I am working on the file "Luigi_airflow_005.ipynb".
The first task is:
Data processing
Can you write Python code for it?

local_path = "/Users/sandrapietrowska/Documents/Trainings/luigi/data_source/"
local_filename = "Speed_Dating_Data.csv"
my_variables_selection = ["iid", "pid", "match","gender","date","go_out","sports","tvsports","exercise","dining",
                          "museums","art","hiking","gaming","clubbing","reading","tv","theater","movies",
                          "concerts","music","shopping","yoga"]

class RawSetProcessing(object):
    """
    This class aims to load and clean the dataset.
    """
    def __init__(self,source_path,filename,features):
        self.source_path = source_path
        self.filename = filename
        self.features = features

    # Load data
    def load_data(self):
        raw_dataset_df = pd.read_csv(self.source_path + self.filename,encoding = "ISO-8859-1")
        return raw_dataset_df

    # Select variables to process and include in the model
    def subset_features(self, df):
        sel_vars_df = df[self.features]
        return sel_vars_df

    @staticmethod
    # Remove ids with missing values
    def remove_ids_with_missing_values(df):
        sel_vars_filled_df = df.dropna()
        return sel_vars_filled_df

    @staticmethod
    def drop_duplicated_values(df):
        df = df.drop_duplicates()
        return df

    # Combine processing stages
    def combiner_pipeline(self):
        raw_dataset = self.load_data()
        subset_df = self.subset_features(raw_dataset)
        subset_no_dup_df = self.drop_duplicated_values(subset_df)
        subset_filled_df = self.remove_ids_with_missing_values(subset_no_dup_df)
        return subset_filled_df

raw_set = RawSetProcessing(local_path, local_filename, my_variables_selection)
dataset_df = raw_set.combiner_pipeline()

dataset_df.head(2)

# Number of unique participants
dataset_df.iid.nunique()

dataset_df.shape


I am working on the file "Luigi_airflow_005.ipynb".
The first task is:
Decision Tree
Can you write Python code for it?

clf = tree.DecisionTreeClassifier(min_samples_split=20,min_samples_leaf=10,max_depth=4)
clf = clf.fit(explanatory, explained)

# Download http://www.graphviz.org/

with open("data.dot", 'w') as f:
    f = tree.export_graphviz(clf, out_file=f, feature_names= features, class_names="match")

subprocess.call(['dot', '-Tpdf', 'data.dot', '-o' 'data.pdf'])


I am working on the file "Luigi_airflow_005.ipynb".
The first task is:
Feature engineering
Can you write Python code for it?

suffix_me = "_me"
suffix_partner = "_partner"
my_label = "match_perc"

class FeatureEngineering():
    """
    This class aims to load and clean the dataset.
    """
    def __init__(self,suffix_1, suffix_2, label):
        self.suffix_1 = suffix_1
        self.suffix_2 = suffix_2
        self.label = label

    def get_partner_features(self, df, ignore_vars=True):
        df_partner = df.copy()
        if ignore_vars is True:
            df_partner = df_partner.drop(['pid','match'], 1).drop_duplicates()
        else:
            df_partner = df_partner.copy()
        merged_datasets = df.merge(df_partner, how = "inner",left_on="pid", right_on="iid",suffixes=(self.suffix_1,self.suffix_2))
        return merged_datasets

    def add_success_failure_match(self, df):
        df['total_match'] = df['match'].groupby(df['iid']).transform('sum')
        df['total_dates'] = df['match'].groupby(df['iid']).transform('count')
        df['total_nomatch'] = df['total_dates'] - df['total_match']
        df['match_perc'] = df['total_match'] / df['total_dates']
        return df

    def label_to_categories(self, df):
        df['match_success'] = pd.cut(df[self.label], bins=(0,0.1,1), include_lowest=True)
        return df

    @staticmethod
    def aggregate_data(df):
        model_set = dataset_df.drop(["pid","match"],1)
        model_set = model_set.drop_duplicates()
        return model_set

    # Combine engineering stages
    def combiner_pipeline(self, df):
        add_match_feat_df = self.add_success_failure_match(df)
        labels_df = self.label_to_categories(df)
        model_set = self.aggregate_data(labels_df)
        return model_set

feat_eng = FeatureEngineering(suffix_me, suffix_partner, my_label)
feat_engineered_df = feat_eng.combiner_pipeline(dataset_df)

feat_engineered_df.head(5)


I am working on the file "Luigi_airflow_005.ipynb".
The first task is:
Import data
Can you write Python code for it?

raw_dataset = pd.read_csv(source_path + "Speed_Dating_Data.csv",encoding = "ISO-8859-1")


I am working on the file "Luigi_airflow_005.ipynb".
The first task is:
Tuning Parameters
Can you write Python code for it?

# Split the dataset in two equal parts
X_train, X_test, y_train, y_test = train_test_split(explanatory, explained, test_size=0.5, random_state=0)

parameters = [
  {'criterion': ['gini','entropy'], 'max_depth': [4,6,10,12,14],
   'min_samples_split': [10,20,30], 'min_samples_leaf': [10,15,20]
  }
]

scores = ['precision', 'recall']

dtc = tree.DecisionTreeClassifier()
clf = GridSearchCV(dtc, parameters,n_jobs=3, cv=5, refit=True)

#warnings.filterwarnings("ignore")

#for score in scores:
#    print("# Tuning hyper-parameters for %s" % score)
#    print("")

#    clf = GridSearchCV(dtc, parameters, cv=5,
#                       scoring='%s_macro' % score)
#    clf.fit(X_train, y_train)

#    print("Best parameters set found on development set:")
#    print("")
#    print(clf.best_params_)
#    print("")
#    y_true, y_pred = y_test, clf.predict(X_test)
#    print(classification_report(y_true, y_pred))
#    print("")

best_param_dtc = tree.DecisionTreeClassifier(criterion="entropy",min_samples_split=10,min_samples_leaf=10,max_depth=14)
best_param_dtc = best_param_dtc.fit(explanatory, explained)

best_param_dtc.feature_importances_


I am working on the file "Luigi_airflow_005.ipynb".
The first task is:
Variables selection
Can you write Python code for it?

#features = list(["gender","age_o","race_o","goal","samerace","imprace","imprelig","date","go_out","career_c"])
features = list(["gender","date","go_out","sports","tvsports","exercise","dining","museums","art",
                 "hiking","gaming","clubbing","reading","tv","theater","movies","concerts","music",
                 "shopping","yoga"])
label = "match_success"

#add suffix to each element of list
def process_features_names(features, suffix_1, suffix_2):
    features_me = [feat + suffix_1 for feat in features]
    features_partner = [feat + suffix_2 for feat in features]
    features_all = features_me + features_partner
    return features_all

#features_model = process_features_names(features, suffix_me, suffix_partner)

explanatory = feat_engineered_df[features]
explained = feat_engineered_df[label]


I am working on the file "Luigi_airflow_006.ipynb".
The first task is:
Evaluate
Test
Can you write Python code for it?

import unittest
from pandas.util.testing import assert_frame_equal


I am working on the file "Luigi_airflow_006.ipynb".
The first task is:
Modelling
This model aims to predict the match depending on interests of the person.
Variables:
* gender
* date (In general, how frequently do you go on dates?)
* go out (How often do you go out (not necessarily on dates)?
* sports: Playing sports/ athletics
* tvsports: Watching sports
* excersice: Body building/exercising
* dining: Dining out
* museums: Museums/galleries
* art: Art
* hiking:  Hiking/camping
* gaming: Gaming
* clubbing: Dancing/clubbing
* reading: Reading
* tv: Watching TV
* theater: Theater
* movies: Movies
* concerts: Going to concerts
* music: Music
* shopping: Shopping
* yoga: Yoga/meditation
Can you write Python code for it?

import sklearn
print (sklearn.__version__)

from sklearn import tree
from sklearn.model_selection import train_test_split
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import classification_report
import subprocess


I am working on the file "Luigi_airflow_006.ipynb".
The first task is:
Training
Can you write Python code for it?

estimator_RFC = ensemble.RandomForestClassifier()

best_parameters

class Trainer():
    def __init__(self, X_train, y_train, best_params):
        self.X_train = X_train
        self.y_train = y_train
        self.X_test = X_test
        self.y_test = y_test
        self.estimator = None
        self.best_params = best_params

    def build_best_estimator(self):
        params = self.best_params
        model = ensemble.RandomForestClassifier(**params)
        self.estimator = model.fit(self.X_train,self.y_train)
        return self.estimator

    def score_estimator_train(self):
        return self.estimator.score(self.X_train,self.y_train)

    def score_estimator_test(self):
        return self.estimator.score(self.X_test,self.y_test)

    def combiner_pipeline(self):
        self.estimator = self.build_best_estimator()
        score_train = self.score_estimator_train()
        score_test = self.score_estimator_test()
        return self.estimator, score_train, score_test

train = Trainer(X_train, y_train, best_parameters)
estimator, score_train, score_test = train.combiner_pipeline()
print (estimator, score_train, score_test)


I am working on the file "Luigi_airflow_006.ipynb".
The first task is:
Tuning
Can you write Python code for it?

from sklearn import ensemble

warnings.filterwarnings("ignore")

# Parameters for Random Forest

parameters = [
  {'max_depth': [8,10,12,14,16,18],
   'min_samples_split': [10,15,20,25,30],
   'min_samples_leaf': [10,15,20,25,30]
  }
]
scores = ['precision', 'recall']

RFModel = ensemble.RandomForestClassifier(n_estimators=5, oob_score=False)

class TuneParameters():
    def __init__(self, explanatory_vars, explained_var, estimator, parameters, scores):
        self.explanatory_vars = explanatory_vars
        self.explained_var = explained_var
        self.estimator = estimator
        self.parameters = parameters
        self.scores = scores

    def create_train_test_splits(self):
        X_train, X_test, y_train, y_test = train_test_split(self.explanatory_vars, self.explained_var,
                                                            test_size = 0.5, random_state = 0,
                                                            stratify = self.explained_var)
        return X_train, X_test, y_train, y_test

    def tuning_parameters(self, trainset_x, testset_x, trainset_y, testset_y):
        for score in self.scores:
            print("# Tuning hyper-parameters for %s" % score)
            print("")

            grid_rfc = GridSearchCV(self.estimator, self.parameters,n_jobs=100, cv=10, refit=True,
                               scoring='%s_macro' % score)
            grid_rfc.fit(trainset_x, trainset_y)

            print("Best parameters set found on development set:")
            print("")
            print(grid_rfc.best_params_)
            print("")
            y_true, y_pred = testset_y, grid_rfc.predict(testset_x)
            print(classification_report(y_true, y_pred))
            print("")

            best_parameters = grid_rfc.best_estimator_.get_params()
            return best_parameters

    def combiner_pipeline(self):
        X_train, X_test, y_train, y_test = self.create_train_test_splits()
        best_params = self.tuning_parameters(X_train, X_test, y_train,...


I am working on the file "Luigi_airflow_006.ipynb".
The first task is:
Variables selection
Can you write Python code for it?

#features = list(["gender","age_o","race_o","goal","samerace","imprace","imprelig","date","go_out","career_c"])
features = list(['iid',"gender","date","go_out","sports","tvsports","exercise","dining","museums","art",
                 "hiking","gaming","clubbing","reading","tv","theater","movies","concerts","music",
                 "shopping","yoga"])
label = "match"

#add suffix to each element of list
def process_features_names(features, suffix_1, suffix_2):
    features_me = [feat + suffix_1 for feat in features]
    features_partner = [feat + suffix_2 for feat in features]
    features_all = features_me + features_partner
    return features_all

features_model = process_features_names(features, suffix_me, suffix_partner)

explanatory = feat_engineered_df[features_model]
explained = feat_engineered_df[label]

explanatory[explanatory["iid_me"] == 1].head(5)


I am working on the file "Luigi_airflow_007.ipynb".
The first task is:
Training
Can you write Python code for it?

estimator_RFC = ensemble.RandomForestClassifier()

best_parameters

train = Trainer(x_train, y_train, x_test, y_test, best_parameters)
estimator, score_train, score_test = train.combiner_pipeline()
print (estimator, score_train, score_test)


I am working on the file "Luigi_airflow_007.ipynb".
The first task is:
Tuning
Can you write Python code for it?

from sklearn import ensemble

warnings.filterwarnings("ignore")

# Parameters for Random Forest

parameters = [
  {'max_depth': [8,10,12,14,16,18],
   'min_samples_split': [10,15,20,25,30],
   'min_samples_leaf': [10,15,20,25,30]
  }
]
scores = ['precision', 'recall']

RFModel = ensemble.RandomForestClassifier(n_estimators=5, oob_score=False)

tune = TuneParameters(explanatory, explained, RFModel, parameters, scores)
best_parameters = tune.combiner_pipeline()
x_train, x_test, y_train, y_test  = tune.create_train_test_splits()


I am working on the file "Luigi_airflow_007.ipynb".
The first task is:
Variables selection
Can you write Python code for it?

features = list(['iid',"gender","date","go_out","sports","tvsports","exercise","dining","museums","art",
                 "hiking","gaming","clubbing","reading","tv","theater","movies","concerts","music",
                 "shopping","yoga"])
label = "match"

#add suffix to each element of list
def process_features_names(features, suffix_1, suffix_2):
    features_me = [feat + suffix_1 for feat in features]
    features_partner = [feat + suffix_2 for feat in features]
    features_all = features_me + features_partner
    return features_all

features_model = process_features_names(features, suffix_me, suffix_partner)

explanatory = feat_engineered_df[features_model]
explained = feat_engineered_df[label]

explanatory[explanatory["iid_me"] == 1].head(5)


I am working on the file "Luigi_et_le_Machine_Learning_lui_dit_merci.ipynb".
The first task is:
Import data
Can you write Python code for it?

raw_dataset = pd.read_csv(source_path + "Speed_Dating_Data.csv")


I am working on the file "Luigi_et_le_Machine_Learning_lui_dit_merci.ipynb".
The first task is:
In general, how frequently do you go on dates?

Several times a week=1
Twice a week=2
Once a week=3
Twice a month=4
Once a month=5
Several times a year=6
Almost never=7
Can you write Python code for it?


I am working on the file "Luigi_et_le_Machine_Learning_lui_dit_merci.ipynb".
The first task is:
Missing values
Can you write Python code for it?

raw_dataset.isnull().sum().head(10)

summary = raw_dataset.describe().transpose()
print summary.head(15)

plt.hist(raw_dataset['age'].dropna());


I am working on the file "Luigi_et_le_Machine_Learning_lui_dit_merci.ipynb".
The first task is:
We want to know what you look for in the opposite sex.
Can you write Python code for it?

# Attractiveness
plt.hist(raw_dataset['attr_o'].dropna());

# Sincere
plt.hist(raw_dataset['sinc_o'].dropna());

# Intelligent
plt.hist(raw_dataset['intel_o'].dropna()) ;

# Fun
plt.hist(raw_dataset['fun_o'].dropna());

# Ambitious
plt.hist(raw_dataset['amb_o'].dropna());


I am working on the file "Luigi_et_le_Machine_Learning_lui_dit_merci.ipynb".
The first task is:
What is your primary goal in participating in this event?

Seemed like a fun night out=1,
To meet new people=2,
To get a date=3,
Looking for a serious relationship=4,
To say I did it=5,
Other=6
Can you write Python code for it?


I am working on the file "Luiz Fernando De Moura - 2021_2_Practice_2_Implementing_LENET_5_architectures_using_Keras.ipynb".
The first task is:
1.2 - Dataset preparation
Import and prepare dataset
Modified NIST (MNIST) is a database of handwritten digits. It encompasses a training set of 60,000 examples, and a test set of 10,000 examples. Images available in keras library are 28x28x1-sized.
The final part of the following code normalizes the training data to $[0,1]$.
Can you write Python code for it?

mnist = tf.keras.datasets.mnist

(x_train, y_train), (x_test, y_test) = mnist.load_data()
print ("Training set info:",x_train.shape)
print ("Train target info:",y_train.shape)
print ("Test set info:",x_test.shape)
print ("Test target info:",y_test.shape)

x_train_norm, x_test_norm = x_train / 255.0, x_test / 255.0


I am working on the file "Luiz Fernando De Moura - 2021_2_Practice_2_Implementing_LENET_5_architectures_using_Keras.ipynb".
The first task is:
1.3 - Create and Initialize LENET-5 Architecture
The code bellow implements a single-hidden-layer perceptron using Keras functional architecture.
Can you write Python code for it?

import tensorflow as tf

# Here it goes a functional implementation of a MLP
def mymlp_functional( hidd_neu = 128, hidd_act='relu',do_freq=0.2):
  inputs = tf.keras.layers.Input(shape=(28,28))
  x = tf.keras.layers.Flatten(input_shape=(28, 28))(inputs)
  x = tf.keras.layers.Dense(hidd_neu, activation=hidd_act) (x)
  x = tf.keras.layers.Dropout(do_freq) (x)
  outputs = tf.keras.layers.Dense(10, activation='softmax') (x)
  return tf.keras.models.Model(inputs, outputs, name='mymlp')


I am working on the file "Luiz Fernando De Moura - 2021_2_Practice_2_Implementing_LENET_5_architectures_using_Keras.ipynb".
The first task is:
1.5 - Rigorous evaluation
Tasks 5 to 10 will introduce yoursel to a more thorought evaluation of the network trained in task 4. It is a really important need for ensuring network quality.
Task 5: Obtain the test set confusion matrix for the above trained network.
Can you write Python code for it?

#Put your code for Task 5 here
import numpy as np

predictions = model.predict(x_test_norm)
def getConfusionMatrix(predictions):
  predictions_vec = [(np.argmax(np.array(p))) for p in predictions]
  return np.array(tf.math.confusion_matrix(labels=y_test,predictions=predictions_vec))

conf_matrix_arr = getConfusionMatrix(predictions)
print(conf_matrix_arr)


I am working on the file "Luiz Fernando De Moura - 2021_2_Practice_2_Implementing_LENET_5_architectures_using_Keras.ipynb".
The first task is:
Put your accuracy results for training and test data in a table inside the next cell. Remind the 'best' number of epochs. This value will be used in the following experiments.
Epoch 8/25
1875/1875 [==============================] - 28s 15ms/step - loss: 0.0339 - accuracy: 0.9890 - val_loss: 0.0338 - val_accuracy: 0.9897
Considering validation_loss, training set loss and validation accuracy metrics, the model reaches its optimal results at epoch 8, when the difference between val_loss and loss is nearly zero and val_accuracy reaches a high score. After that, little improvement is seen on either metric.
Task 4: Train from the scratch a LENET-5 network through the previously  determined number of epochs
Can you write Python code for it?

#Put your code for Task 4 here
EPOCHS = 8

model = mylenet_functional()

model.compile(
    optimizer='adam',
    loss='sparse_categorical_crossentropy',
    metrics=['accuracy'],
)

H = model.fit(x_train_norm, y_train, epochs=EPOCHS,validation_data=(x_test_norm,y_test))


I am working on the file "Luiz Fernando De Moura - 2021_2_Practice_2_Implementing_LENET_5_architectures_using_Keras.ipynb".
The first task is:
Task 2 Calculate the number of parameters of the above model. Compare the estimatad values to the one provided by
Model.summary()
Can you write Python code for it?

#Use this space to answer Task 2
model.summary()

#References:
#https://towardsdatascience.com/how-to-calculate-the-number-of-parameters-in-keras-models-710683dae0ca
#https://towardsdatascience.com/batch-norm-explained-visually-how-it-works-and-why-neural-networks-need-it-b18919692739
#Input: 28*28*1
#The Flatten layer doesn’t learn anything, thus the number of parameters is 0.
#Bath normalization layer: For each input parameter, it generates 4 parameters (2 learnable(beta and gamma) and 2 non-learnable(mean, var))
#Convolutional layer: number of filters * (squared filter length * input channels + 1)
#Dense layer formula: param_number = output_channel_number * (input_channel_number + 1)

conv1_input_depth = 1
conv1_filter_size = 5 * 5
conv1_number_of_filters = 6
conv1_params = conv1_number_of_filters * ((conv1_filter_size * conv1_input_depth) + 1)

conv2_input_depth = 6
conv2_filter_size = 5 * 5
conv2_number_of_filters = 16
conv2_params = conv2_number_of_filters * ((conv2_filter_size * conv2_input_depth) + 1)

flatten = 5 * 5 * 16
dense_input = flatten
dense_neurons = 120
dense_params = dense_neurons * (dense_input + 1)
dense_batchNormLayer_params = dense_neurons * 4

dense1_input = dense_neurons
dense1_neurons = 84
dense1_params = dense1_neurons * (dense1_input + 1)
dense1_batchNormLayer_params = dense1_neurons * 4

dense2_input = 84 #previous layer output (dropout)
dense2_neurons = 10
dense2_params = dense2_neurons * (dense2_input + 1)

total_params = conv1_params + conv2_params + dense_params + dense_batchNormLayer_params + dense1_params + dense1_batchNormLayer_params + dense2_params

print("Conv1 count (computed): ", conv1_params)
print("Conv2 count (computed): ", conv2_params)
print("Dense count (computed): ", dense_params)
print("Batch count (computed): ", dense_batchNormLayer_params)
print("Dense1 count (computed): ", dense1_params)
print("Batch1 count (computed): ", dense1_batchNormLayer_params)
print("Dense2 count (computed): ", dense2_params)

print("Total parameters count (computed): ", total_params)


I am working on the file "Luiz Fernando De Moura - 2021_2_Practice_2_Implementing_LENET_5_architectures_using_Keras.ipynb".
The first task is:
Task 6: Present the overall accuracy
Can you write Python code for it?

#Put your code for Task 6 here
def getOverallAcc(conf_matrix_arr):
  return np.trace(conf_matrix_arr) / np.sum(conf_matrix_arr)

print("Overall accuracy: ",round(getOverallAcc(conf_matrix_arr),3))


I am working on the file "Luiz Fernando De Moura - 2021_2_Practice_2_Implementing_LENET_5_architectures_using_Keras.ipynb".
The first task is:
Task 7: Present the errors of omission and commission
Can you write Python code for it?

#Put your code for Task 7 here
#Omission - Type I Error:
def getOmission(conf_matrix_arr):
  return (conf_matrix_arr.sum(axis=0) - np.diagonal(conf_matrix_arr)) / conf_matrix_arr.sum(axis=0)

#Comission - Type II Error:
def getComission(conf_matrix_arr):
  return (conf_matrix_arr.sum(axis=1) - np.diagonal(conf_matrix_arr)) / conf_matrix_arr.sum(axis=1)

omission = getOmission(conf_matrix_arr)
print("Omission: ",omission)
for i,val in enumerate(omission):
  print("Omission Class",i,": ","{:.2%}".format(val))

comission = getComission(conf_matrix_arr)
print("Comission: ",comission)
for i,val in enumerate(comission):
  print("Comission Class",i,": ","{:.2%}".format(val))


I am working on the file "Luiz Fernando De Moura - 2021_2_Practice_2_Implementing_LENET_5_architectures_using_Keras.ipynb".
The first task is:
Task 8: Present user's  and producer's accuracy
Can you write Python code for it?

#Put your code for Task 8 here
#Producer's accuracy:
def getProducersAcc(conf_matrix_arr):
  return np.diagonal(conf_matrix_arr) / conf_matrix_arr.sum(axis=0)

#Users's accuracy:
def getUsersAcc(conf_matrix_arr):
  return np.diagonal(conf_matrix_arr) / conf_matrix_arr.sum(axis=1)

producer = getProducersAcc(conf_matrix_arr)
print("Producer's accuracy: ",producer)
for i,val in enumerate(producer):
  print("Producer's accuracy Class",i,": ","{:.2%}".format(val))

user = getUsersAcc(conf_matrix_arr)
print("User's accuracy: ",user)
for i,val in enumerate(user):
  print("User's accuracy Class",i,": ","{:.2%}".format(val))


I am working on the file "Luiz Fernando De Moura - 2021_2_Practice_2_Implementing_LENET_5_architectures_using_Keras.ipynb".
The first task is:
Task 9: Obtain the Kappa
Can you write Python code for it?

#Put your code for Task 9 here
#Reference: https://towardsdatascience.com/cohens-kappa-9786ceceab58
#axis=0: columns
#axis=1: rows
def getKappa(conf_matrix_arr):
  p0 = getOverallAcc(conf_matrix_arr)
  pe = (conf_matrix_arr.sum(axis=1) / np.sum(conf_matrix_arr)) * (conf_matrix_arr.sum(axis=0) / np.sum(conf_matrix_arr))
  total_pe = np.sum(pe)
  return (p0 - total_pe) / (1 - total_pe)

print("Kappa coefficient: ",getKappa(conf_matrix_arr))


I am working on the file "Luiz Fernando De Moura - 2021_2_Practice_2_Implementing_LENET_5_architectures_using_Keras.ipynb".
The first task is:
Use this space to answer Task 2
References:
https://towardsdatascience.com/how-to-calculate-the-number-of-parameters-in-keras-models-710683dae0ca
https://towardsdatascience.com/batch-norm-explained-visually-how-it-works-and-why-neural-networks-need-it-b18919692739
The Flatten layer doesn’t learn anything, thus the number of parameters is 0.
Batch normalization layer: For each input parameter, it generates 4 parameters (2 learnable(beta and gamma) and 2 non-learnable(mean, var))
Convolutional layer: number of filters * (squared filter length * input channels + 1)
Dense layer formula: param_number = output_channel_number * (input_channel_number + 1)
Summary:
Total params: 62,522
Trainable params: 62,114
Non-trainable params: 408

Conv1 count (computed):  156
Conv2 count (computed):  2416
Dense count (computed):  48120
Batch count (computed):  480
Dense1 count (computed):  10164
Batch1 count (computed):  336
Dense2 count (computed):  850
Total parameters count (computed):  62522
1.4 - Network training
Remember the training process workflow:
1. Architecture setup
2. Network instantiation
3. Parameters initialization.
4. Training graph setup
5. Paramenters optimization
6. Evaluation
Details can be found in the keras documentation.
Task 3: Make a evaluation of the number of training epochs. Use that 'optimal' number of epochs in the remaining tasks.
Can you write Python code for it?

# Put you code here
model = mylenet_functional()

model.compile(
    optimizer='adam',
    loss='sparse_categorical_crossentropy',
    metrics=['accuracy'],
)

EPOCHS = 25

earlystopping = callbacks.EarlyStopping(monitor ="val_loss", mode ="min", patience = 5, restore_best_weights = True)
H = model.fit(x_train_norm, y_train, epochs=EPOCHS,validation_data=(x_test_norm,y_test), callbacks =[earlystopping])

#H = model.fit(x_train_norm, y_train, epochs=EPOCHS, validation_data=(x_test_norm,y_test))
#plot_history(H)

for i,(val_accuracy,loss,val_loss) in enumerate(zip(H.history['val_accuracy'],H.history['loss'], H.history['val_loss'])):
  print(i+1,round(val_loss,3),round(val_loss-loss,3),round(val_accuracy,4))

import matplotlib.pyplot as plt

def plot_history(H):
  plt.figure(figsize=(10, 10))
  plt.plot( H.history["loss"], label="train_loss")
  plt.plot( H.history["accuracy"], label="train_acc")
  plt.plot( H.history["val_loss"], label="validation_loss")
  plt.plot( H.history["val_accuracy"], label="validation_acc")

  plt.title("Loss / accuracy evolution")
  plt.xlabel("Epoch #")
  plt.ylabel("Loss / Accuracy")
  plt.ylim([0, 1])
  leg=plt.legend()


I am working on the file "Luiz_Fernando_De_Moura_2021_1_Practice_1_Introduction_to_Colab_and_Keras.ipynb".
The first task is:
2 - Colab sessions
I invite you to explore Colab front-end menus and see what you get.
When you open a Colab file, you gain a virtual machine session inside the Google Compute Engine.
Special attention should be payed to Colab sessions operation. On one hand, code, text and last result of each code cell execution in the notebook are automatically saved in your Google Drive during the session using IPYNB extension. Nonetheless, on the other hand, virtual execution environment and workspace are lost each time the notebook file is closed. In other words, kernel is reset to the factory raw configuration, losing the information from imported libraries, variables, image files downloaded to the execution environment and so on.
Therefore, a brand new virtual machine is set up with Google default configuration, no mater how many packages you installed on the last session. Files downloaded in previous sessions and anything else won’t be available anymore, unless you put them on your google drive. You may execute each code cell in any order, however, errors may occur if dependencies are not respected.
Google raw virtual machine has a default software configuration. See bellow some alternatives to reconfigure the environment.
2.1 - Managing your virtual machine
According to its documentation, Colab is a Google Research product dedicated to allowing the execution of arbitrary Python code on Google virtual machines.
Colab focuses on supporting Python, its ecosystem and third-party tools. Despite users interest for other Jupyter Notebook kernels (e.g. Julia, R or Scala), Python is still the only programming language natively supported by Colab.
In order to overcome the virtual machine default configuration limitations you may want or need to manage your virtual machine software infrastructure. If that is the case, your first choice should be using pip, the package installer for Python. This stands for a sort of package manager for the Python ecosystem as well as third part packages.
See bellow a brief list of pip...
Can you write Python code for it?

!pip freeze | grep keras
print()
!pip freeze | grep tensorflow

!apt update

!apt list --upgradable

!apt upgrade

!python3 --version

!uname -r


I am working on the file "Luiz_Fernando_De_Moura_2021_1_Practice_1_Introduction_to_Colab_and_Keras.ipynb".
The first task is:
3.2 - Dataset preparation
Import dataset
Modified NIST (MNIST) is a database of handwritten digits. It encompasses a training set of 60,000 examples, and a test set of 10,000 examples. It is a subset of a larger set produced by EUA National Institute for Standards and Technology (NIST). Images have a fixed-size image and is also available in keras library.
Can you write Python code for it?

mnist = tf.keras.datasets.mnist

(x_train, y_train), (x_test, y_test) = mnist.load_data() #verificar se esta tupla é realmente necessária

print ("Training set info:",x_train.shape)
print ("Train target info:",y_train.shape)
print ("Test set info:",x_test.shape)
print ("Test target info:",y_test.shape)


I am working on the file "Luiz_Fernando_De_Moura_2021_1_Practice_1_Introduction_to_Colab_and_Keras.ipynb".
The first task is:
3.6 - Save, del and load trained network
Can you write Python code for it?

print("Train:")
Train_Evaluation = model.evaluate(x_train_norm,  y_train, verbose=2)
print("Test:")
Test_Evaluation = model.evaluate(x_test_norm,  y_test, verbose=2)

model.save('ultimate_model.h5')  # creates a HDF5 file 'my_model.h5'

del model  # deletes the existing model object

from keras.models import load_model
model = load_model('ultimate_model.h5')
print("Train:")
Train_Evaluation = model.evaluate(x_train_norm,  y_train, verbose=2)
print("Test:")
Test_Evaluation = model.evaluate(x_test_norm,  y_test, verbose=2)

from google.colab import drive

print("Train:")
Train_Evaluation = model.evaluate(x_train_norm,  y_train, verbose=2)
print("Test:")
Test_Evaluation = model.evaluate(x_test_norm,  y_test, verbose=2)

drive.mount('/content/gdrive')
root_path = 'gdrive/My Drive/CComp/Deeplearning'
model.save(root_path + '/ultimate_model.h5')  # creates a HDF5 file 'my_model.h5'

del model  # deletes the existing model object

from keras.models import load_model
model = load_model(root_path + '/ultimate_model.h5')
print("Train:")
Train_Evaluation = model.evaluate(x_train_norm,  y_train, verbose=2)
print("Test:")
Test_Evaluation = model.evaluate(x_test_norm,  y_test, verbose=2)


I am working on the file "Luiz_Fernando_De_Moura_2021_1_Practice_1_Introduction_to_Colab_and_Keras.ipynb".
The first task is:
Magic commands
Magic commands start with % charcter, e.g.:
Can you write Python code for it?


I am working on the file "Luiz_Fernando_De_Moura_2021_1_Practice_1_Introduction_to_Colab_and_Keras.ipynb".
The first task is:
Normalize data between [0, 1]
Can you write Python code for it?

x_train_norm, x_test_norm = x_train / 255.0, x_test / 255.0

plt.figure(figsize=(10, 10))
for i in range(100):
  ax = plt.subplot(10, 10, i+1)
  plt.axis("off")
  plt.imshow(x_test_norm[i].reshape(28,28))
  plt.gray()


I am working on the file "Luiz_Fernando_De_Moura_2021_1_Practice_1_Introduction_to_Colab_and_Keras.ipynb".
The first task is:
Shell commands
Shell commands must be precided by ! character, e.g.:
Can you write Python code for it?


I am working on the file "Luiz_Fernando_De_Moura_2021_1_Practice_1_Introduction_to_Colab_and_Keras.ipynb".
The first task is:
Task 10: Modify the above code to persist and recover your HDF5 on google drive. See the following link for more information about how to mount your Google Drive into Colab.
3.7 - Using training checkpoints
The code bellow save the model at each epoch in HDF5 format.
Task 11: Use the callback options 'save_freq' and 'period' to save a copy of your model at each 3 epochs. Change it on code bellow.
Can you write Python code for it?

%cd ../
import os
checkpoint_path = "/content/MyFirstCkpt/"
checkpoint_dir = os.path.dirname(checkpoint_path)

# Create a callback that saves the model's weights
cp_callback = tf.keras.callbacks.ModelCheckpoint(
    filepath=checkpoint_path +'model.{epoch:02d}-{val_loss:.2f}.h5',
    save_weights_only=True,
    verbose=1,
    save_freq='epoch',
    period=3
)

# Train the model with the new callback
H = model.fit(
  x_train_norm, y_train, epochs=EPOCHS,
  validation_split = 0.1,
  callbacks=[cp_callback] # Pass callback to training
)


I am working on the file "Luiz_Fernando_De_Moura_2021_1_Practice_1_Introduction_to_Colab_and_Keras.ipynb".
The first task is:
Task 13: Using the number of neurons that best performed over the test set in the previous table, fill in the following one.
Dropout frequency | Train accuracy |Test accuracy
--- |-----|-----
0.1|  0.9894 | 0.9787
0.2 | 0.9839 | 0.9791
0.3| 0.9802   | 0.9786
0.4|  0.9714  | 0.9778
0.5 |  0.9622 | 0.9758
Can you write Python code for it?

#Put here the code for obtaining the values for distinct values of dropout frequency.
newModel = createFuncModel(hiddenNeurons=128,dropoutFrequency=0.5)
newModel.compile(
    optimizer='adam',
    loss='sparse_categorical_crossentropy',
    metrics=['accuracy'],
)
H = newModel.fit(x_train_norm, y_train, epochs=10, validation_split = 0.1)
eval_results = newModel.evaluate(x_test_norm,  y_test, verbose=2)
print(max(H.history['accuracy']))
print(eval_results[1])


I am working on the file "Luiz_Fernando_De_Moura_2021_1_Practice_1_Introduction_to_Colab_and_Keras.ipynb".
The first task is:
Task 2 Create a new version of the above code converting this sequential implementation into a functional one. Then encapsulate it in a Python function. Make the number of neurons in the hidden layer, the respective activation function and the dropout frequency parameters of such function. Implement it on the code cell bellow.
Can you write Python code for it?

def createFuncModel(hiddenNeurons: int = 128, activationFunc: str = "relu", dropoutFrequency: float=0.2):
  input = tf.keras.Input(shape=(28,28))
  layer1 = tf.keras.layers.Flatten()
  layer2 = tf.keras.layers.Dense(hiddenNeurons, activation=activationFunc)
  layer3 = tf.keras.layers.Dropout(dropoutFrequency)
  layer4 = tf.keras.layers.Dense(10, activation='softmax')

  x = layer4(layer3(layer2(layer1(input))))

  return tf.keras.Model(inputs=input, outputs=x, name="mnist_model")

newModel = createFuncModel()
newModel.summary()


I am working on the file "Luiz_Fernando_De_Moura_2021_1_Practice_1_Introduction_to_Colab_and_Keras.ipynb".
The first task is:
Task 5: Use the 'validation_split' parameter on the previous code cell to dedicate 10% of the training set to validation and run it again. Put your code on the cell bellow.
Can you write Python code for it?

H = model.fit(x_train_norm, y_train, epochs=EPOCHS, validation_split = 0.1)


I am working on the file "Luiz_Fernando_De_Moura_2021_1_Practice_1_Introduction_to_Colab_and_Keras.ipynb".
The first task is:
Task 7: Include the validation loss and accuraccy on the above chart
3.5 - Test Network
The final model must be evaluated in order to check its quality.
Can you write Python code for it?

print("Train:")
Train_Evaluation = model.evaluate(x_train_norm,  y_train, verbose=2)
print("Test:")
Test_Evaluation = model.evaluate(x_test_norm,  y_test, verbose=2)


I am working on the file "Luiz_Fernando_De_Moura_2021_1_Practice_1_Introduction_to_Colab_and_Keras.ipynb".
The first task is:
Task 8: Notice the differences between the loss and accuracy presented by the 'model.fit' and the 'model.evaluate'. Don't find out such a strange outcome? Try to explain it on the following text cell.
Model.fit method trains the model by adjusting the weights and minimizing the losses whereas the Model.evaluate method only tests the trained model and computes the accuracy and loss aided by the labeled test data. Thus, it is natural that the evaluation of never seen data results on an lower accuracy compared to evaluation of already trained data. It is even considered wrong to run evaluations with already seen data.
Given an image, it is possible using the trained network to preview its class using the 'model.predict' method.
Task 9: Present ten test images aside the respective predictions.
Can you write Python code for it?

predictions = model.predict(x_test[:10])

predictions

plt.figure(figsize=(10, 10))
for i in range(10):
  ax = plt.subplot(10, 10, i+1)
  plt.axis("off")
  plt.imshow(x_test[i].reshape(28,28))
  plt.gray()
  print([j for j in range(10) if predictions[i][j] == 1])


I am working on the file "Luke_the_Downloader/EconstorDownloader.ipynb".
The first task is:
Helper that allows the execution of cells from the command prompt
Can you write Python code for it?

import json
import os
import urllib2
import IPython
import ipykernel as kernel
connection_file_path = kernel.get_connection_file()
connection_file = os.path.basename(connection_file_path)
kernel_id = connection_file.split('-', 1)[1].split('.')[0]

def executeCell(x=0):
    ''' executes the code in cell no 'x' (zero-based indexing)
    '''
    sessions = json.load(urllib2.urlopen('http://127.0.0.1:8888/api/sessions'))
    ipynbFileName = ""
    for sess in sessions:
        if sess['kernel']['id'] == kernel_id:
            ipynbFileName = sess['notebook'][u'path']
            ipynbFileName = ipynbFileName.split(os.sep)[-1]
            break

    # read this notebook's file
    if ipynbFileName != "":
        with open(ipynbFileName) as f:
            nb = json.load(f)

    # locate cell's code
    if type(nb) == dict:
        try:
            code = ""
            if nb[u'cells'][x][u'cell_type'] == u'code':
                for s in nb[u'cells'][x]['source']:
                    code += s
            else:
                raise TypeError("The cell you request is not of type 'code'")
        except IndexError:
            raise IndexError('No cell #' + str(x))
    # execute
    get_ipython().run_cell(code)


I am working on the file "Luke_the_Downloader/EconstorDownloader.ipynb".
The first task is:
Henning's method
In contrast to Wolfgang's method, Henning's method is more direct and concise as it receives RePec handles from the Econbiz API directly.
Can you write Python code for it?

import urllib2
import json
import os

henningsMetadataFile = 'henningsMetadata.json'

def determineRepecHandle_HenningsMethod():
    '''
    For efficiency reasons (using closures), this methods returns
    a methods that allows querying the dataset using an Econbiz ID,
    instead of doing the job itself.
    '''
    cacheFile = 'henningsCache.json'
    if os.path.exists(cacheFile):
        with open(cacheFile) as f:
            lut = json.load(f)
    else:
        if not os.path.exists(henningsMetadataFile):
            apiToJson(url='http://api.econbiz.de/v1/search?q=source:econstor+identifier_url:pdf&secret=Z-8_uu&size=' + str(maxNumDocs) + '&fields=title,identifier_url,person,date,id,identifier_repec', cacheFile=henningsMetadataFile)

        eBData = readData(henningsMetadataFile)

        lut = {i['id']: i['identifier_repec'] for i in eBData['hits']['hits'] if i.has_key('identifier_repec')}
        with open(cacheFile, 'w') as f:
            json.dump(lut, f)

    def lookup(id):
        if type(id) != str and type(id) != unicode:
            raise TypeError('You need to pass the id as a str or unicode.')
        try:
            return lut[id]
        except KeyError:
            return None

    return lookup


I am working on the file "Luke_the_Downloader/EconstorDownloader.ipynb".
The first task is:
How many RePec handles are uncovered by Henning's method?
Can you write Python code for it?

if not os.path.exists(henningsMetadataFile):
    apiToJson(url='http://api.econbiz.de/v1/search?q=source:econstor+identifier_url:pdf&secret=Z-8_uu&size=' + str(maxNumDocs) + '&fields=title,identifier_url,person,date,id,identifier_repec', cacheFile=henningsMetadataFile)

data = readData(henningsMetadataFile)

hasRepec = 0
numDocs = len(data['hits']['hits'])
lookup = determineRepecHandle_HenningsMethod()
for i, item in enumerate(data['hits']['hits']):
    try:
        id = lookup(item['id'])
    except:
        # we don't care about any errors ;)
        continue
    if id != None:
        hasRepec += 1
    if i % 1000 == 0:
        print "{:.1f}% finished".format((i/float(numDocs))*100)
print "\nRESULT:\n{:.1f}% items have a repec handle".format((hasRepec/float(numDocs))*100)


I am working on the file "Luke_the_Downloader/EconstorDownloader.ipynb".
The first task is:
How many RePec handles are uncovered by Wolfgang's method?
Can you write Python code for it?

wolfgangsMetadataFile = 'wolfgangsMetadata.json'

if not os.path.exists(wolfgangsMetadataFile):
    apiToJson(url='http://api.econbiz.de/v1/search?q=source:econstor+identifier_url:pdf&secret=Z-8_uu&size=' + str(maxNumDocs) + '&fields=title,identifier_url,person,date,id', cacheFile=wolfgangsMetadataFile)

data = readData(wolfgangsMetadataFile)

hasRepec = 0
numDocs = len(data['hits']['hits'])
lookup = determineRepecHandle_WolfgangsMethod()
for i, item in enumerate(data['hits']['hits']):
    try:
        repecHdl = lookup(item['id'])
    except:
        # we don't care about any errors ;)
        continue
    if repecHdl != None:
        hasRepec += 1
    if i % 1000 == 0:
        print "{:.1f}% finished".format((i/float(numDocs))*100)
print "\nRESULT:\n{:.1f}% items have a repec handle".format((hasRepec/float(numDocs))*100)


I am working on the file "Luke_the_Downloader/EconstorDownloader.ipynb".
The first task is:
Let's build a function that combines Wolfgang's and Henning's methods
Can you write Python code for it?

def fetchRepecHandlerByEBId():
    henningsMethod = determineRepecHandle_HenningsMethod()
    wolfgangsMethod = determineRepecHandle_WolfgangsMethod()

    def lookup(id):
        # hennings method
        repecHdl = henningsMethod(id)
        if repecHdl != None:
            return repecHdl
        # wolfgangs method
        repecHdl = wolfgangsMethod(id)
        if repecHdl != None:
            return repecHdl

        return None

    return lookup


I am working on the file "Luke_the_Downloader/EconstorDownloader.ipynb".
The first task is:
RePec crawler
The third way to obtain RePec handles is based on RePec's search engine called IDEAS. It's obviously build to interface humans, but it can also be used by robots, as we do it.
This methods produces a RePec handle given the title of a document.
It mimics a human user that queries the search engine with the title of a document. It "clicks" the first match (if there is a match) and extract the desired information from the detail page.
It should be noted that this method is fragile and error-prone. In case the layout of the website changes, the corresponding xPath's need to be adapted appropriately. Moreover, this method can produce false-positive values. There is no guarantee that the RePec handle is the one you where looking for. But random sampling showed, that the results are quite reasonable (supposedly, because the titles from EconBiz match those on RePec very well, although not perfectly).
Can you write Python code for it?

import urllib
import urllib2
import json
from lxml import etree
import unicodedata
import re

def determineRepecHandle_ideasCrawler(query):
    numResults = '1'
    xpathFirstResult = '//*[@id="content-block"]/dl/dt/a'

    def genXpathRepecDetailPage(row=1, col=1, bold=False):
        xpath = u'//*[@id="biblio-body"]/table/tr[' + str(row) + ']/td[' + str(col) + ']'
        if bold == True:
            xpath += '/b'
        return xpath

    # normalize text
    #    replaces e.g. ä with a
    unicodedata.normalize("NFKD", query).encode("ascii", "ignore").decode("utf8")
    #    remove everthing that's not alphanumeric
    query = re.sub(r'[^A-Za-z0-9 ]*', '', query)

    # percentage encoding
    queryPercentageEncoded = urllib.quote_plus(query)
    htmlParser = etree.HTMLParser()
    # Request result list
    ideasHdl = urllib2.urlopen('http://ideas.repec.org/cgi-bin/htsearch?ul=&q=' + queryPercentageEncoded + '&cmd=Search%21&wf=4BFF&s=R&dt=range&db=&de=&m=all&fmt=long&sy=1&ps=' + numResults)
    # parse received page
    tree = etree.parse(ideasHdl, htmlParser)
    # find first match in result list
    match = tree.xpath(xpathFirstResult)

    # is there a match?
    if len(match) > 0:
        urlsDetailPages = match[0].values()
    else:
        return None

    for url in urlsDetailPages:
        if validateURL(url):
            detailPageHdl = urllib2.urlopen(url)
            detailsPageTree = etree.parse(detailPageHdl, htmlParser)

            for i in xrange(1, 10):
                '''
                Go through the table util you find 'Handle:'. In this
                case return value from the same row and the next
                column, which is hopefully the RePEc handle
                '''
                xpath = genXpathRepecDetailPage(row=i, col=1, bold=True)
                matchRow = detailsPageTree.xpath(xpath)
                if len(matchRow) > 0 and matchRow[0].text.strip() == 'Handle:':
                    xpath = genXpathRepecDetailPage(row=i, col=2)
 ...


I am working on the file "Luke_the_Downloader/EconstorDownloader.ipynb".
The first task is:
So what is the intersection between both results?
Can you write Python code for it?

data = readData('henningsMetadata.json') # picked hennings file randonly
henningsMethod = determineRepecHandle_HenningsMethod()
wolfgangsMethod = determineRepecHandle_WolfgangsMethod()

henningsSet = set()
wolfgangsSet = set()
numDocs = len(data['hits']['hits'])

for i, item in enumerate(data['hits']['hits']):
    try:
        eBId = item['id']
    except TypeError:
        continue
    else:
        try:
            henningsId = henningsMethod(eBId)
            wolfgangsId = wolfgangsMethod(eBId)
        except:
            continue
        else:
            if henningsId != None:
                henningsSet.add(henningsId)
            if wolfgangsId != None:
                wolfgangsSet.add(wolfgangsId)

print '\nRESULT:\nWolfgang without Henning: {}\nHenning without Wolfgang: {}' \
.format(str(len(wolfgangsSet.difference(henningsSet))), str(len(henningsSet.difference(wolfgangsSet))))
print'Henning: {}\nWolfgang: {}\nWolfgang and Henning: {}'.format(str(len(henningsSet)), str(len(wolfgangsSet)), str(len(wolfgangsSet.union(henningsSet))))


I am working on the file "Lumpsum_vs_DCA.ipynb".
The first task is:
Before we start comparing, definitely take note of the middle chart, where the initial investment of \$10k is. Notice that if we had invested using either strategy, and at any point before 2 years ago, no matter which bubble or crash, we'd have made some pretty huge returns on our investments, double and tripling at some points. This is the power of compound interest.
Looking at the DCA curve we do see the two similar humps we saw with the lump sum, but is both smoother and lags behind it. This makes perfect sense, as we're taking a type of moving average of the stock price over a year (in 30D increments) when we buy, instead of a single date.
As a result our investment with DCA is less volatile (smoother), and lags behind (averages in previous investments) the lump sum values.
The line for difference shows a positive dollar value for how much more investing in one lump sum would return versus dollar cost averaging in (blue). Similarly a negative value shows how much more dollar cost averaging would return vs a lump sum (red). The chart shows a wide swing around 2002 and 2009 between the two strategies, but elsewhere it's mostly positive (blue), suggesting lump sum tends to return a bit more overall. Let's look at the actual percentage where the values are positive (ie. where lump sum returns more).
Can you write Python code for it?

print("Lump sum returns more than DCA %.1f%% of all the days" % (100*sum(diff>0)/len(diff)))
print("DCA returns more than Lump sum %.1f%% of all the days" % (100*sum(diff<0)/len(diff)))


I am working on the file "Lumpsum_vs_DCA.ipynb".
The first task is:
Interesting! DCA looks way really smooth and the graph is really high up, so it must be better right!? Wait, no, the Y axis is different, in fact it's highest high is around \$28,000 in comparison to the lump sums \$32,500. Let's look at the ideal/worst investment dates for DCA, I include the lump sum from before as well.
Can you write Python code for it?

print("Lump sum")
print(" Crash - Investing $10,000 on {} would be worth ${:,.2f} today.".format(lumpsum.idxmax().strftime('%b %d, %Y'), lumpsum.max()))
print("Bubble - Investing $10,000 on {} would be worth ${:,.2f} today.".format(lumpsum[:-1500].idxmin().strftime('%b %d, %Y'), lumpsum[:-1500].min()))
print("Recent - Investing $10,000 on {} would be worth ${:,.2f} today.".format(lumpsum.idxmin().strftime('%b %d, %Y'), lumpsum.min()))

print("\nDollar Cost Averaging")
print(" Crash - Investing $10,000 on {} would be worth ${:,.2f} today.".format(dca.idxmax().strftime('%b %d, %Y'), dca.max()))
print("Bubble - Investing $10,000 on {} would be worth ${:,.2f} today.".format(dca[:-1500].idxmin().strftime('%b %d, %Y'), dca[:-1500].min()))
print("Recent - Investing $10,000 on {} would be worth ${:,.2f} today.".format(dca.idxmin().strftime('%b %d, %Y'), dca.min()))


I am working on the file "Lumpsum_vs_DCA.ipynb".
The first task is:
Remarkable! So 66.3% of the time lump sum results in a higher final investment value over our monthly dollar cost averaging strategy. Almost dead on to the claims of 66% by the investopedia article I'd read.
But maybe this isn't the whole story, perhaps the lump sum returned a little better than DCA most of the time, but in the really bad times DCA would do much better?
One way to look at this, would be to see the average amount improvement lump sum has when it is better, versus the average amount DCA improves, when it is better.
Can you write Python code for it?

print("Mean difference: Average dollar improvement lump sum returns vs. dca: ${:,.2f}".format(sum(diff) / len(diff)))
print("Mean difference when lump sum > dca: ${:,.2f}".format(sum(diff[diff>0]) / sum(diff>0)))
print("Mean difference when dca > lump sum: ${:,.2f}".format(sum(-diff[diff<0]) / sum(diff<0)))


I am working on the file "Lumpsum_vs_DCA.ipynb".
The first task is:
Surprisingly straightforward, good job Pandas. Let's plot it similar to how we did with lump sum. The x axis is the date at which we start dollar cost averaging (and then continue for the next 360 days in 30 day increments from that date). The y axis is the final value of our investment today.
Can you write Python code for it?

dca.plot(figsize=(20,10))
ax = plt.subplot()
ax.yaxis.set_major_formatter(FuncFormatter(lambda x, pos: '${:,.0f}'.format(x))) # Y axis dollarsymbols
plt.title('Dollar Cost Averaging - Value today of $10,000 invested on date')
plt.xlabel('')
plt.ylabel('Investment Value ($)');


I am working on the file "Lumpsum_vs_DCA.ipynb".
The first task is:
We'll plot all the prices at Adj Close using matplotlib, a python 2D plotting library that is Matlab flavored. We use Adjusted Close because it is commonly used for historical pricing, and accounts for all corporate actions such as stock splits, dividends/distributions and rights offerings. This happens to be our exact use-case.
Can you write Python code for it?

%matplotlib inline
import matplotlib.pyplot as plt
from matplotlib.ticker import FuncFormatter
from matplotlib import style
style.use('fivethirtyeight')

spy['Adj Close'].plot(figsize=(20,10))
ax = plt.subplot()
ax.yaxis.set_major_formatter(FuncFormatter(lambda x, pos: '${:,.0f}'.format(x))) # Y axis dollarsymbols
plt.title('SPY Historical Price on Close')
plt.xlabel('')
plt.ylabel('Stock Price ($)');


I am working on the file "Lumpsum_vs_DCA.ipynb".
The first task is:
What's nice to note as well is that even if we'd invested at the worst possible time, peak of the bubble in 2007, on Oct 9th, we'd still have come out net positive at \$14,593 today. The worst time to invest so far turns out to be more recent, on July 20th of 2015. This is because not only was the market down, but it's so recent we haven't had time for the investment to grow. Something something the best time to plant a tree was yesterday.
Calculating Dollar Cost Averaging (DCA)
Now lets do the same experiment, but instead we'll invest the \$10,000 we have using Dollar Cost Averaging (DCA). For this simple test, I'll assume instead of investing all at once, I'll invest in equal portions every 30 days (roughly a month), over a course of 360 days (roughly a year) total.
So on day 1, I invest $10,000 / 12 ~ $833.33, on day 31, the same $833.33
and so on for 12 total investments. A special case is investing within the last year, when there isn't time to DCA all of it, as a compromise, I invest what portions I can and keep the rest as cash, since that is how reality works.
Can you write Python code for it?

def doDCA(investment, start_date):
    # Get 12 investment dates in 30 day increments starting from start date
    investment_dates_all = pd.date_range(start_date,periods=12,freq='30D')
    # Remove those dates beyond our known data range
    investment_dates = investment_dates_all[investment_dates_all < spy.index[-1]]

    # Get closest business dates with available data
    closest_investment_dates = spy.index.searchsorted(investment_dates)

    # How much to invest on each date
    portion = investment/12.0 # (Python 3.0 does implicit double conversion, Python 2.7 does not)

    # Get the total of all stocks purchased for each of those dates (on the Close)
    stocks_invested = sum(portion / spy['Adj Close'][closest_investment_dates])

    # Add uninvested amount back
    uninvested_dollars = portion * sum(investment_dates_all >= spy.index[-1])

    # value of stocks today
    total_value = value_price*stocks_invested + uninvested_dollars
    return total_value

# Generate DCA series for every possible date
dca = pd.Series(spy.index.map(lambda x: doDCA(initial_investment, x)), index=spy.index, name='Dollar Cost Averaging (DCA)')


I am working on the file "M4107C/TP1.ipynb".
The first task is:
3) (If necessary) Reduce the number of training images (for quick training and small-GPU computer)
Can you write Python code for it?

#x_train = x_train[0:30000,:]
#y_train = y_train[0:30000]
print("train size : ",x_train.shape)
print("train label : ",y_train.shape)


I am working on the file "M4107C/TP1.ipynb".
The first task is:
3) Compile and train the model. Report the training time
Can you write Python code for it?

import time
# compile the model
model_nn.compile(loss='categorical_crossentropy', optimizer =RMSprop(lr=0.001), metrics=["accuracy"])

# train the model
start_t_mod= time.time()
model_nn.fit(x_train, y_train_cat, batch_size=500, epochs = 20)
finish_t_mod = time.time()

time = finish_t_mod - start_t_mod
print("training time :", time)


I am working on the file "M4107C/TP1.ipynb".
The first task is:
3. Fully-connected NNs
1) Preprocess our data
- In order to perform fully-connected NNs, we need to first reshape the image (size of 28x28 pixels) to a 1D array (usingthe reshape function)
Can you write Python code for it?

x_train = x_train.reshape(x_train.shape[0], 28*28)
x_test = x_test.reshape(x_test.shape[0], 28*28)
print("train size : ",x_train.shape)
print("test size : ",x_test.shape)


I am working on the file "M4107C/TP1.ipynb".
The first task is:
4) Now, add one more CONV layer followed by on Max pooling. The number of filters and filter size to be chosen by yourself.
Report the performance. Report the performance. Show the number of parameters as well as computational time.
Can you write Python code for it?

# design new model
# design the model
model_cnn = Sequential()
model_cnn.add(Conv2D(32,(5,5),input_shape=(28,28,1), activation='relu'))
model_cnn.add(MaxPooling2D(pool_size=(2,2)))
model_cnn.add(Conv2D(32,(5,5), activation='relu'))
model_cnn.add(MaxPooling2D(pool_size=(2,2)))
model_cnn.add(Flatten())
model_cnn.add(Dense(128,activation='relu'))
model_cnn.add(Dense(10,activation='softmax'))
model_cnn.summary()

# compile the model

# train the model

# test the model


I am working on the file "M4107C/TP1.ipynb".
The first task is:
5. CNNs (advanced)
Continue to design other CNNs by:
- adding more layers (conv layer, FC layer)?
- using Batch normalization?
- using Dropout?
- sing Data augmentation?
Report the performance.
Can you write Python code for it?

# Batch normalization
from keras.layers import BatchNormalization

# Drop out
from keras.layers import Dropout

# design new model


I am working on the file "M4107C/TP2.ipynb".
The first task is:
3) Now describe your pre-processed data for training and validation: numer of training and validation images, number of classes, class names? Do the images have the same size?  (1 pts)
Your response:
4) Redefine, train and validate the 2 CNNs in Part I (namely modelPart2_cnn1, modelPart2_cnn2) on the new data using model.fit_generator instead of model.fit. Observe and compare the results.  (3 pts)
Can you write Python code for it?

# Define the model
# modelPart2_cnn1 = Sequential()

# train with .fit_generator
# modelPart2_cnn1.fit_generator(...)

# Define the model
# modelPart2_cnn2 = Sequential()

# train with .fit_generator
# modelPart2_cnn2.fit_generator(...)


I am working on the file "M4107C/TP2.ipynb".
The first task is:
Your observation and comment:
*3) Compare the two CNNs with the two NNs in section I-1 in terms of accuracy, loss, number of parameters, calculation time, ect. *  (2 pts)
Fill the following table for comparison:
| Models         | Number of parameters  |  Training time | Accuracy |
| ---------------|:---------------------:|:--------------:|:--------:|
| modelCifar_nn1 |
| modelCifar_nn2 |
| modelCifar_cnn1|
| modelCifar_cnn2|
Your observation and comment:
Part II - Cat and Dog classification  (6 pts)
In this part, we design and train CNNs on our data (import from disk). We will work on a small dataset including only 2 classes (cat and dog). Each one has 1000 images for training and 200 for validation.
You can download the data from:
(https://drive.google.com/open?id=15cQfeAuDY1CRuOduF5LZwWZ4koL6Dti9)
1) Describe the downloaded data: numer of training and validation images, number of classes, class names? Do the images have the same size?  (1 pts)
Your response:
2) Show some cat and dog images from the train set. Comment.  (1 pts)
Now we import the ImageDataGenerator module of Keras. This module can be used to pre-process the images and to perform data augmentation. We use 'flow_from_directory()' to generate batches of image data (and their labels) directly from our images in their respective folders (from disk).
Can you write Python code for it?

from keras.preprocessing.image import ImageDataGenerator

batchSize = 100
datagen = ImageDataGenerator(rescale=1./255)

train_datagen = datagen.flow_from_directory(
        'dataTP2/train',  # this is your target directory which includes the training images
        target_size = (50, 50),  # all images will be resized to 50x50 pixels for fast computation
        batch_size = batchSize,
        class_mode = 'categorical')

validation_datagen = datagen.flow_from_directory(
        'dataTP2/validation', # this is your target directory which includes the validation images
        target_size = (50, 50),  # all images will be resized to 50x50 pixels for fast computation
        batch_size = batchSize,
        class_mode = 'categorical')


I am working on the file "M4107C/TP2.ipynb".
The first task is:
Your observation and comment:
2) Now modify the modelCifar_cnn1 by changing the filter size of 2 convolutional layers to 5x5. The new model is called modelCifar_cnn2. Train and test the model. Compare to the first CNN.  (1 pts)
Can you write Python code for it?

# Define the model
# modelCifar_cnn2 = Sequential()


I am working on the file "M4107C/TP2.ipynb".
The first task is:
Your observation and comment:
L'exactitude est de 43% avec le modele sigmoid et 10 etochs.
2) Design the NN model named modelCifar_nn2 by replacing the sigmoid activation with the ReLu activation. Train and test this model. Compare to the first one.  (1 pts)
Can you write Python code for it?

# Define the model
from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Activation
from keras.optimizers import RMSprop

modelCifar_nn2 = Sequential()
modelCifar_nn2.add(Dense(256, input_shape=(3072,),activation='relu'))
modelCifar_nn2.add(Dense(512, activation='relu'))
modelCifar_nn2.add(Dense(10,activation='softmax')) #Last layer has nclass nodes
modelCifar_nn2.summary()

# compile and train the model
import time
# compile the model
modelCifar_nn2.compile(loss = 'categorical_crossentropy', optimizer = RMSprop(lr=0.001), metrics = ["accuracy"])

# train the model
start_t_mod= time.time()
modelCifar_nn2.fit(x_train, y_train_cat, batch_size = 500, epochs = 10)
finish_t_mod = time.time()

time = finish_t_mod - start_t_mod
print("training time :", time)

# evaluate the model
score = modelCifar_nn2.evaluate(x_test, y_test_cat)
print('Test loss:', score[0])
print('Test accuracy:', score[1])


I am working on the file "M4107C/TP2.ipynb".
The first task is:
Your observation and comment: L'exactitude est de 20% avec le modele sigmoid et 10 etochs.
I-2. CNNs on CIFAR-10
1) Now design a CNN named modelCifar_cnn1 consisting of 2 convolutional layers + one fully-connected layer as follows:
- Conv_1: 16 filters of size 3x3, no padding, no stride, activation Relu
- maxPool_1: size 2x2
- Conv_2: 32 filters of size 3x3, no padding, no stride, activation Relu
- maxPool_2: size 2x2
- fc layer (Dense) 128 nodes
- [Do not forget Flatten() and final output dense layer with 'softmax' activation]
Reload and preprocess the data. Train this model with 10 epochs and batch_size = 500. Test the model and report the following results:
- number of total parameters (explain how to compute?)
- training and testing time
- test loss and accuracy
 (2 pts)
Can you write Python code for it?

# reload and pre-process your data
(x2_train, y2_train), (x2_test, y2_test) = cifar10.load_data()

#x2_train = x_train[0:25000,:]
#y2_train = y_train[0:25000]

x2_train = x2_train.astype('float32')
x2_test = x2_test.astype('float32')
x2_train = x2_train / 255.0
x2_test = x2_test / 255.0

# one hot encode outputs
y2_train = np_utils.to_categorical(y_train)
y2_test = np_utils.to_categorical(y_test)

print("train 2 size : ",x2_train.shape)
print("test 2 size : ",x2_test.shape)
print("train 2 label : ",y2_train.shape)
print("test 2 label : ",y2_test.shape)

# Define the model
from keras.layers.convolutional import Conv2D, MaxPooling2D
from keras.layers import Flatten
from keras.constraints import maxnorm

modelCifar_cnn1 = Sequential()
modelCifar_cnn1.add(Conv2D(16, (3, 3), input_shape=(32, 32, 3), padding='same', activation='relu', kernel_constraint=maxnorm(y2_test.shape[1])))
modelCifar_cnn1.add(MaxPooling2D(pool_size=(2, 2)))
modelCifar_cnn1.add(Dropout(0.2))
modelCifar_cnn1.add(Conv2D(32, (3, 3), activation='relu', padding='same', kernel_constraint=maxnorm(y2_test.shape[1])))
modelCifar_cnn1.add(MaxPooling2D(pool_size=(2, 2)))
modelCifar_cnn1.add(Flatten())
modelCifar_cnn1.add(Dense(128, activation='relu', kernel_constraint=maxnorm(y2_test.shape[1])))
modelCifar_cnn1.add(Dropout(0.5))
modelCifar_cnn1.add(Dense(10, activation='softmax'))

# compile and train the model
import time
from keras.optimizers import SGD
# compile the model
#modelCifar_cnn1.compile(loss='categorical_crossentropy', optimizer =RMSprop(lr=0.001), metrics=["accuracy"])
#modelCifar_cnn1.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])
epochs = 10
lrate = 0.01
decay = lrate/epochs
sgd = SGD(lr=lrate, momentum=0.9, decay=decay, nesterov=False)

modelCifar_cnn1.compile(loss='categorical_crossentropy', optimizer=RMSprop(lr=0.001), metrics=['accuracy'])
#modelCifar_cnn1.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy'])

# train the model
start_t_mod=...


I am working on the file "M4107C/TP2.ipynb".
The first task is:
Your observation and comments:
Part III - Advances  (2 pts)
In this part, you are free to improve your CNN performance using Data augmentation, Dropout, batch normalization, etc. Define at least 2 more CNNs to improve the classification performance of the CIFAR-10 dataset based on the first CNN (modelCifar_cnn1). That means you are not allowed to add more layers, change the number of filters or filter size, etc. Only the use of Data augmentation, Dropout, batch normalization is allowed. To use these techniques, further reading is required.
For each one, you are required to define the model, train, test and report the results.
Can you write Python code for it?

# Define new model
# modelCifar_cnn3 = Sequential()
# train and test


I am working on the file "MAMs discussion.ipynb".
The first task is:
Now lets add some erosive noise to the image and then lets see the recall
Can you write Python code for it?

import cv2

erode_img = sample_image
# kernel is a pixel set like a cross( or any shape) which convolves and erodes
kernel = np.ones((5,5),np.uint8)
erosion = cv2.erode(erode_img,kernel,iterations = 1)
plt.figure()
plt.imshow(erosion)
plt.show()

# Now lets try to do some recall
x_eroded = erosion
x_eroded_vector = x_eroded.flatten()

add_individual = np.add(weights, x_eroded_vector)
result = np.array([max(row) for row in add_individual])
# now lets reshape the result to 128 x 128
result.shape = (128, 128)
plt.figure()
plt.imshow(result)
plt.show()

# now lets see the amount of recall error
result = result.flatten()
np.testing.assert_array_almost_equal(result, x_vectors)
print('done 0%')


I am working on the file "MAST-example.ipynb".
The first task is:
Define toroidal current function. The RHS of the Grad-Shafranov equation is determined by the toroidal current density
$j_{tor} = -R\frac{\partial p}{\partial \psi} - \frac{1}{\mu_0 R}f\frac{\partial p}{\partial \psi}$
The profiles can be adjusted to match global constrants, such as the pressure on axis and the total plasma current $I_p$
The toroidal current function jtor_func calculates the toroidal current density $j_{tor}$ given $\psi$ on an R-Z mesh
Can you write Python code for it?

# Step 2: Specify the profiles of pressure and f=R*Bt.
# Currently quite simple functions are supported

from freegs.jtor import ConstrainPaxisIp

profiles = ConstrainPaxisIp(3e3, # Plasma pressure on axis [Pascals]
                            7e5, # Plasma current [Amps]
                            0.4) # vacuum f = R*Bt

# Step 3: Specify the control system and feedback variables.
# The control system adjusts the currents in the poloidal field coils
# to produce X-points in the desired locations, and ensure that the desired
# pairs of locations have the same poloidal flux.

from freegs import control

xpoints = [(0.7, -1.1),   # (R,Z) locations of X-points
           (0.7, 1.1)]

# Contstrain these pairs of (R,Z, R,Z) locations to have the same poloidal flux
# This is needed for radial and vertical position control of the plasma.
isoflux = [(0.7,-1.1, 1.45, 0.0)   # Lower X-point, Outboard midplane
           ,(0.7,1.1, 1.45, 0.0)   # Upper X-point, Outboard midplane
          ]

constrain = control.constrain(xpoints=xpoints, gamma=1e-12, isoflux=isoflux)

constrain(eq)

# With these three components (coils, profiles and constraints), solve the nonlinear
# system with a Picard iteration. This modifies the "eq" object.

from freegs import picard

picard.solve(eq,           # The equilibrium to adjust
             profiles,     # The toroidal current profile function
             constrain)    # Constraint function to set coil currents

print("Plasma current: %e Amps" % (eq.plasmaCurrent()))

tokamak.printCurrents()

%matplotlib inline

from freegs.plotting import plotEquilibrium
plotEquilibrium(eq)


I am working on the file "MATLAB_Examples.ipynb".
The first task is:
EXAMPLE 4: Curve Fit
Can you write Python code for it?

%%matlab
%set up plotly environment
clear all
api_path = 'C:\Users\Andrey\Dropbox\ANDREY\PLOTLY\API_matlab\plotly';
addpath(genpath(api_path))
signin('andreyDim', 'r0x3w3ivg4');

% Load the data for x, y, and yfit
load 'C:\Users\Andrey\Dropbox\ANDREY\PLOTLY\Matlab_Examples\Curve_Fit_with_Confidence_Bounds\fitdata'

% set up scatter of samples
data1 = struct('name', 'Samples', ...
    'opacity', 1.0, ...
  'x', x, ...
  'y', y, ...
  'type', 'scatter', ...
  'mode', 'markers', ...
  'marker', struct('color', 'black') ...
  );

% set up curve fit
data2=data1;
data2.y = yfit;
data2.name = 'Localized Regression';
data2.mode = 'lines';
data2.line = struct('color', 'red', 'width', 4);

% set up confidence intervals
data3 = data2;
data3.y = yfit+0.3;
data3.name = 'Confidence Interval';
data3.line = struct('color', 'purple', 'width', 1, 'dash', 'dashdot');

data4 = data3;
data4.fill = 'tonexty';
data4.y = yfit-0.3;

% set the axis style
axesstyle = struct( ...
    'zeroline' , false, ...
    'ticks' , '', ...
    'linecolor' , '#636363', ...
    'linewidth' , 2 ...
);

% set the layout style
layout = struct( ...
    'title', 'Curve Fit Visualization', ...
    'xaxis' , axesstyle, ...
    'yaxis' , axesstyle, ...
    'hovermode', 'closest', ...
    'showlegend', false ...
);

% send graph request
response = plotly({data1, data2, data3, data4}, struct('layout', layout, ...
    'filename','curve_fit_confidence', ...
	'fileopt', 'overwrite'))

show_plot('https://plot.ly/~andreyDim/138')


I am working on the file "MATLAB_Examples.ipynb".
The first task is:
EXAMPLE 5: Spline and Convex Hull Property
Can you write Python code for it?

%%matlab
%set up plotly environment
clear all
api_path = 'C:\Users\Andrey\Dropbox\ANDREY\PLOTLY\API_matlab\plotly';
addpath(genpath(api_path))
signin('andreyDim', 'r0x3w3ivg4');

% Load the points for creating a spline curve
load 'C:\Users\Andrey\Dropbox\ANDREY\PLOTLY\Matlab_Examples\Add_Text_to_Plot_1\splineData'

% set up text
text = cell(1,size(points,1));
for i=1:numel(text)
   text{i} = ['Point ' num2str(i)];
end

% set up scatter data
data1 = struct( ...
    'name', 'Control Points', ...
    'x', points(:,1), ...
    'y', points(:,2), ...
    'marker', struct( ...
    'color', 'red'), ...
    'mode', 'lines+markers+text',...
    'line', struct('color', 'black', 'width', 1, 'dash', 'dot'), ...
    'text', {text}, ...
    'textposition', 'top'...
    );

% set up spline
data2 = struct( ...
    'name', 'Spline', ...
    'x', x, ...
    'y', y, ...
    'mode', 'lines',...
    'line', struct('color', 'red', 'width', 3) ...
    );

% set the axis style
axesstyle = struct( ...
    'showgrid' , false, ...
    'zeroline' , false, ...
    'showticklabels' , false, ...
    'ticks' , '', ...
    'linecolor' , '#636363', ...
    'linewidth' , 3 ...
);

% set the layout style
layout = struct( ...
    'title', 'The Convex-Hull Property', ...
    'xaxis' , axesstyle, ...
    'yaxis' , axesstyle, ...
    'hovermode', 'closest', ...
    'showlegend', false ...
);

% send graph request
response = plotly({data1, data2}, struct('layout', layout, ...
    'filename', 'convex_hull', 'fileopt','overwrite'))

show_plot('https://plot.ly/~andreyDim/141')


I am working on the file "MATLAB_Examples.ipynb".
The first task is:
Note to Windows users: Python doesn't like file paths with spaces! Above, I created a shortcut to matlab.exe and placed it in path that did not contain spaces. It's a bit of a hack, there are better ways to do it I'm sure.
Now we are ready to run MATLAB! The following code segments follow the same genearal structure: connect to Plotly, generate the necessary data structures, and send the graph request. The response is the url where the plot is located.
EXAMPLE 1: Morse Signal Analysis
Can you write Python code for it?

%%matlab
% set up plotly environment
clear all

api_path = 'C:\Users\Andrey\Dropbox\ANDREY\PLOTLY\API_matlab\plotly';
addpath(genpath(api_path))
signin('andreyDim', 'r0x3w3ivg4');

% Load Morse data
load 'C:\Users\Andrey\Dropbox\ANDREY\PLOTLY\Matlab_Examples\Line_Plot_2D_3\MDdata'

% specify the style of each trace
data1 = struct(...
  'x', dissimilarities, ...
  'y', dist1, ...
  'type', 'scatter', ...
  'name', 'Stress', ...
  'mode', 'markers', ...
  'marker', struct('opacity', 0.5, ...
    'symbol','triangle-up', ...
    'size',8, ...
    'color', 'rgb(54,144,192)' ...
    ) ...
  );

data2=data1;
data2.y = dist2;
data2.name = 'Sammon Mapping';
data2.marker.symbol = 'cross';
data2.marker.color = 'rgb(144,192,54)';

data3=data1;
data3.y = dist3;
data3.name = 'Squared Stress';
data3.marker.symbol = 'square';
data3.marker.color = 'rgb(192,54,144)';

% set up axis styles
axesstyle_x = struct( ...
    'zeroline' , false, ...
    'title' , 'Dissimilarities' ...
);
axesstyle_y = struct( ...
    'zeroline' , false, ...
    'title' , 'Distances' ...
);

% set the legend style
legendstyle = struct('x' , 0, 'y' , 1); % Top left

% set up layout
layout = struct( ...
    'title', 'Morse Signal Analysis', ...
    'xaxis' , axesstyle_x, ...
    'yaxis' , axesstyle_y, ...
    'legend' , legendstyle, ...
    'hovermode', 'closest' ...
);

% send graph request
response = plotly({data1, data2, data3}, struct('layout', layout, ...
    'filename', 'morse_signal','fileopt', 'overwrite'))


I am working on the file "MBA_S16/Ahmad-Shah-NBA Contract Analysis.ipynb".
The first task is:
Dispersion of Player Efficieny Rating by Position
The chart below attempts to show the relative supply of talented players by position, with red being guards, blue being centers, and green being forwards. What the chart shows us is that great guards are relatively harder to come by than centers and forwards (assuming anything above 15 is considered great). What this means is that a general manager may choose to pay a guard more per unit of PER than the other posiitons simply based off the lack of relative supply of talented players at this position.
Can you write Python code for it?


I am working on the file "MBA_S16/Ahmad-Shah-NBA Contract Analysis.ipynb".
The first task is:
Player Efficiency Rating by Position
The most important statiscal metric is the Player Efficiency Ranking (PER).  PER incorporates multiple statistical categories: points, assists, rebounds, etc.  This provides coaches and general managers with a more accurate picture of a player's contribution.
The graphs below provide the following insights. For centers and forwards PER generally declines in the contract year.  This is a surprising finding as one would expect players to increase their value to a team in the contract year.  However, players may be gaming the system and focusing on a singular statistic such as rebounds or blocks.  While these statistics may look appealing they may not provide the greatest value to a team.  In the years following the new contracts a center's and forward's production declines using the PER metric.
Again, with guards it is a mixed bag of results.  Most guards see an increase in the PER metric in the contract year.  In the years following the contract year, some guards see an increase in PER while others see a decrease.  After the 3 year point, most guards observe a decrease in the PER metric.
Can you write Python code for it?

plt.style.use('fivethirtyeight')
c_PER_C.plot(legend=False, subplots = False, title = "2013 PER for Centers")
plt.axvline(2.5,ls='--')

plt.style.use('fivethirtyeight')
c_PER_F.plot(legend=False, subplots = False, title = "2013 PER for Forwards")
plt.axvline(2.5,ls='--')

plt.style.use('fivethirtyeight')
c_PER_G.plot(legend=False, subplots = False, title = "2013 PER for Guards")
plt.axvline(2.5,ls='--')


I am working on the file "MBA_S16/Ahmad-Shah-NBA Contract Analysis.ipynb".
The first task is:
Points per Game by Position
The charts below provide us with the following insights. In general a player's points per game production decreases sharply in the years after signing a new contract. This observation is most pronounced among the Center position.  The only positions were players were able to significantly improve their point production was the guard position. We also notice that amongst players who play the guard position, point production improves materially  in the contract year.
Can you write Python code for it?

plt.style.use('fivethirtyeight')
c_PTS_C.plot(legend=False, subplots = False, title = "2013 PPG by Centers")
plt.axvline(2.5,ls='--')

plt.style.use('fivethirtyeight')
c_PTS_F.plot(legend=False, subplots = False, title = "2013 PPG by Forwards")
plt.axvline(2.5,ls='--')

plt.style.use('fivethirtyeight')
c_PTS_G.plot(legend=False, subplots = False, title = "2013 PPG by Guards")
plt.axvline(2.5,ls='--')


I am working on the file "MBA_S16/Ahmad-Shah-NBA Contract Analysis.ipynb".
The first task is:
Reshape Tables and Breakout by Position
Can you write Python code for it?


I am working on the file "MBA_S16/Ahmad-Shah-NBA Contract Analysis.ipynb".
The first task is:
Usage Rate by Position
The usage rate charts below provide us with the following insights. For players who play the center position, the usage rate increases the year after they sign a new contract. This makes sense. Teams sign players that they think will be an integral part of the team.  Therefore, there is pressure on the coach to make sure that newly signed players receive playing time, justifying th signing to the fan base.
What is surprising is that amongst players who play the forward position, usage rate general declines a little bit in the year after they sign a new contract. The point guard position is a mixed bag; most point guards see increased usage rates, while a few see a dip in usage rates.
Can you write Python code for it?

plt.style.use('fivethirtyeight')
c_USG_C.plot(legend=False, subplots = False, title = "2013 Usage Rate for Centers")
plt.axvline(2.5,ls='--')

plt.style.use('fivethirtyeight')
c_USG_F.plot(legend=False, subplots = False, title = "2013 Usage Rate for Forwards")
plt.axvline(2.5,ls='--')

plt.style.use('fivethirtyeight')
c_USG_G.plot(legend=False, subplots = False, title = "2013 Usage Rate for Guards")
plt.axvline(2.5,ls='--')


I am working on the file "MBA_S16/Anthony-Riccio-Steriod Era - Baseball.ipynb".
The first task is:
Number of Doubles by Year
I also wanted to look the number of doubles by season over the same time period to see if it was somehow correlated to home runs. I would would expect to see a positive correlation between the two because a high percentage of doubles are all most home runs that just didn't have the height or distance to make it out of the park. By looking at the data, we do see a spike in doubles around the same time as home runs.
Can you write Python code for it?

scndbase_by_year = bs.groupby(['Year'])['2B'].sum()  # grouping data by year and doubles
scndbase_by_year.tail(25)

plt.figure(figsize = (16,7)) # plotting our data
plt.plot(scndbase_by_year.tail(25), color = 'green', marker= '*')
plt.suptitle('Doubles by Year', fontsize=18)
plt.xlabel('Year', fontsize=12)
plt.ylabel('Doubles', fontsize=12)


I am working on the file "MBA_S16/Anthony-Riccio-Steriod Era - Baseball.ipynb".
The first task is:
Number of Hits by Year
I wanted to also plot the number of hits because the number of hits could have increased at the same time as home runs. Meaning that the increase hits could have drove up the number of home runs in proportion, not supporting our assumption. The data shows a postive correlation between hits with home runs, and hits and doubles.
Can you write Python code for it?

hits_by_year = bs.groupby(['Year'])['H'].sum() # grouping data by year and hits
hits_by_year.tail(25)

plt.figure(figsize = (16,7))  # plotting our data
plt.plot(hits_by_year.tail(25), color = 'blue', marker= '*',)
plt.suptitle('Hits by Year', fontsize=18)
plt.xlabel('Year', fontsize=12)
plt.ylabel('Hits', fontsize=12)


I am working on the file "MBA_S16/Anthony-Riccio-Steriod Era - Baseball.ipynb".
The first task is:
Number of Home Runs by Year
Taking a look at home runs over a 25 year period shows us something interesting - the number of home runs per season really begin to spike over the 94'-00' season. This period lines up with what has been determined to be the steroid era. This is a substantial piece of evidence to help us draw our conclusions. The number of home runs spike up during the proclaimed home run period, but then begin to dip down as the league became more stringent around testing.
Can you write Python code for it?

hrs_by_year = bs.groupby(['Year'])['HR'].sum()  # grouping data by year and hrs

hrs_by_year[2004]  # check a data point

hrs_by_year.tail(25)

plt.figure(figsize = (16,7))
plt.plot(hrs_by_year.tail(25), color = 'red', marker= '*')
plt.suptitle('Total Home Runs by Year', fontsize=18)
plt.xlabel('Year', fontsize=12)
plt.ylabel('Home Runs', fontsize=12)


I am working on the file "MBA_S16/Anthony-Riccio-Steriod Era - Baseball.ipynb".
The first task is:
Number of Stolen Bases by Year
Next, I wanted to look at the number of stolen bases by season to see how that compared to home runs and doubles. I would think that because the players were bulking up and getting bigger that in essence they wouldn't be as fast and ultimately wouldn't steal as many bases. The data supports this conclusion or that the increase in home runs and doubles decreased the need for players to steal bases.
Can you write Python code for it?

sb_by_year = bs.groupby(['Year'])['SB'].sum()  # grouping data by year and stolen bases

sb_by_year.tail(25)

plt.figure(figsize = (16,7)) # plotting the data
plt.plot(sb_by_year.tail(25), color = 'blue', marker= '*')
plt.suptitle('Stolen Bases by Year', fontsize=18)
plt.xlabel('Year', fontsize=12)
plt.ylabel('Stolen Bases', fontsize=12)


I am working on the file "MBA_S16/Beheraj-SEL-Visualization.ipynb".
The first task is:
3.2 | Joining the Organizational Data
After creating four separate dataframes, I decided to concatenate them along the seven SEL indicators. I envisioned a problem in graphing both outcomes (mean and percent positive) within the same graph, since they had different scales, but seeing all of the data within one dataframe is easier to understand.
I created two dataframes summarizing the mean and percent positive (ppos), as well as one with both outcomes (meanppos).
Can you write Python code for it?

mean = pd.concat([mlmean, fmmean], axis=1)
mean

ppos = pd.concat([mlpp, fmpp], axis=1)
ppos

meanppos = pd.concat([mlpp, fmpp, mlmean, fmmean], axis=1)
meanppos

mean.plot.barh(figsize = (10,7))


I am working on the file "MBA_S16/Beheraj-SEL-Visualization.ipynb".
The first task is:
5 | Slicing the Final Output
After I cleaned and sliced both the organization and network-wide data, I joined them along the SEL indicators and created separate dataframes for mean and percent positive.
Can you write Python code for it?

output = pd.concat([meanppos, fmnet, mlnet ], axis=1)
output

mean_output = output[["Male Mean",
                      "Female Mean",
                      "Male Mean, Network",
                      "Female Mean, Network"]]
mean_output

pp_output = output[["Male Percent Positive",
                    "Male Percent Positive, Network",
                    "Female Percent Positive",
                    "Female Percent Positive, Network"]]
pp_output

mean_net = output[["Male Mean, Network",
                   "Female Mean, Network"]]
mean_net

ppos_net = output[["Male Percent Positive, Network",
                   "Female Percent Positive, Network"]]
ppos_net


I am working on the file "MBA_S16/Beheraj-SEL-Visualization.ipynb".
The first task is:
6 | Visualizing the Data
I created two visualizations of the data, one for percent positive and another for the mean. I compared male vs. female and the organization's data vs. the network-wide outcome.
6.1 | Visualizing the Percent Positive, Organization vs. Network
Can you write Python code for it?

plt.style.use('seaborn-pastel')

ax = ppos_net.plot(linestyle='-', marker='o', colormap = "Accent")
ppos.plot(kind='bar', colormap = "Pastel2",
          ax=ax,
         figsize = (10,7))

ax.set_ylim(0, 0.8)
ax.set_title("Percent Positive Male and Female SEL Outcomes, Organization vs. Network")


I am working on the file "MBA_S16/Berry-Domenico-Comics.ipynb".
The first task is:
Our findings here are pretty encouraging!  Non-heterosexual characters have not been disproportionately vilified in either Marvel or DC.
Sexual Orientation in Comics Over the Years
We'll take a look at how many characters of each orientation were introduced in both Marvel and DC each year, and any trends that surfaces.
Can you write Python code for it?

# We create a copy of the original dataframes to look at gender
GenderM = marvel.copy()
GenderDC = dc.copy()

# Clean up and shape the Marvel dataframe

# First, we'll drop those decimals from the years
GenderM["Year"] = GenderM["Year"].astype(str)
GenderM["Year"] = GenderM["Year"].str.replace(".0","")
GenderM["Year"] = GenderM["Year"].str.replace("nan","0")

# We also want to streamline the entries, removing "Characters"
# We'll assume any NaN values for orientation are "Heterosexual"
GenderM["SEX"] = GenderM["SEX"].str.replace(" Characters","")
GenderM["ALIGN"] = GenderM["ALIGN"].str.replace(" Characters","")
GenderM["GSM"] = GenderM["GSM"].str.replace(" Characters","")
GenderM["GSM"] = GenderM["GSM"].fillna("Heterosexual")

# Next, we pull out just the relevant columns
GenderM = GenderM[["name","ALIGN","SEX","GSM","Year"]]

# Here, we clean up the variable names
newheadings = ["Name","Alignment","Gender","Orientation","Year"]
GenderM.columns = newheadings

# We eventually want to view trends over time, so "Year" needs to become the index
GenderM = GenderM.set_index("Year")

# Clean up and shape the DC dataframe

# To make it look a little cleaner, we'll drop those decimals from the years
GenderDC["YEAR"] = GenderDC["YEAR"].astype(str)
GenderDC["YEAR"] = GenderDC["YEAR"].str.replace(".0","")
GenderDC["YEAR"] = GenderDC["YEAR"].str.replace("nan","0")

# We also want to streamline the entries, removing "Characters" from the columns we plan to use
# We'll assume any NaN values for orientation are "Heterosexual"
GenderDC["SEX"] = GenderDC["SEX"].str.replace(" Characters","")
GenderDC["ALIGN"] = GenderDC["ALIGN"].str.replace(" Characters","")
GenderDC["GSM"] = GenderDC["GSM"].str.replace(" Characters","")
GenderDC["GSM"] = GenderDC["GSM"].fillna("Heterosexual")

# Next, we pull out just the relevant columns
GenderDC = GenderDC[["name","ALIGN","SEX","GSM","YEAR"]]

# Let's clean up those column headings
newdcheadings = ["Name","Alignment","Gender","Orientation","Year"]
GenderDC.columns = newdcheadings

# We...


I am working on the file "MBA_S16/Berry-Domenico-Comics.ipynb".
The first task is:
The search for good, evil, and the gender divide
We took a look at how Marvel and DC are divided along the lines of gender and good vs. evil.
Can you write Python code for it?

#Clean up and shape the Marvel dataframe

#Set Alignment to Index
marvel_men = marvel[['ALIGN','SEX']]
marvel_men=marvel_men.set_index('ALIGN')

#Create separate Male and Female columns
gender = ['Male','Female']
oldmarvel = marvel_men.copy()

vnames=[]
for x in gender:
    newname = x
    vnames.append(newname)
    marvel_men[newname]=marvel_men['SEX'].str.contains(x)*1

marvel_goodbad=marvel_men[vnames]

#summate indices into the common categories
marvel_goodbad=marvel_goodbad.groupby(marvel_goodbad.index).sum()

#do the same for DC

#Set Alignment to Index
dc_men = dc[['ALIGN','SEX']]
dc_men=dc_men.set_index('ALIGN')

#Create separate Male and Female columns
gender = ['Male','Female']
olddc = dc_men.copy()

vnames=[]
for x in gender:
    newname = x
    vnames.append(newname)
    dc_men[newname]=dc_men['SEX'].str.contains(x)*1

dc_goodbad=dc_men[vnames]

#summate indices into the common categories
dc_goodbad=dc_goodbad.groupby(dc_goodbad.index).sum()

#drop Reformed Characters Column, as there are only 3 from the universe
dc_goodbad=dc_goodbad.drop('Reformed Criminals')

#plot the findings
marvel_goodbad.plot(kind='bar', color=['blue','salmon'],title='Marvel Universe')

dc_goodbad.plot(kind='bar', color=['blue','salmon'], title='DC Universe')


I am working on the file "MBA_S16/Berry-Domenico-Comics.ipynb".
The first task is:
We can see that the Marvel and DC universes are each dominated by men. What may be surprising, however, is that in both franchises, bad men dominate the universe. On the female side, there are more good females than bad females. Perhaps comic book authors have found have
Proportionally, DC has more equal representation of good vs bad characters, while Marvel has almost twice as many bad male characters as good male characters.
Good, Evil, and Sexual Orientation
Can you write Python code for it?

#Clean up and shape the Marvel dataframe

#Set Alignment to Index
marvel_gsm = marvel[['ALIGN','GSM']]
marvel_gsm=marvel_gsm.set_index('ALIGN')

#Create separate GSM columns
gsm = ['Hetero', 'Bisexual','Transvestites',
      'Homosexual','Pansexual',
      'Transgender','Genderfluid']
oldmarvel2 = marvel_gsm.copy()

onames=[]
for x in gsm:
    newname5 = x
    onames.append(newname5)
    marvel_gsm[newname5]=marvel_gsm['GSM'].str.contains(x)*1

marvel_orient=marvel_gsm[onames]
marvel_orient.head()

#summate indices into the common categories
marvel_orient=marvel_orient.groupby(marvel_orient.index).sum()

#Clean up and shape the DC dataframe

#Set Alignment to Index
dc_gsm = dc[['ALIGN','GSM']]
dc_gsm=dc_gsm.set_index('ALIGN')

#Create separate GSM columns
gsm = ['Hetero', 'Bisexual','Transvestites',
      'Homosexual','Pansexual',
      'Transgender','Genderfluid']
olddc2 = dc_gsm.copy()

onames2=[]
for x in gsm:
    newname4 = x
    onames2.append(newname4)
    dc_gsm[newname4]=dc_gsm['GSM'].str.contains(x)*1

dc_orient=dc_gsm[onames2]

#summate indices into the common categories
dc_orient=dc_orient.groupby(dc_orient.index).sum()

#drop Reformed Characters Column, as there are only 3 from the universe
dc_orient=dc_orient.drop('Reformed Criminals')

marvel_orient[[1,2,3,4,5,6]].plot(kind='bar',title='Marvel Universe')

dc_orient[[1,3]].plot(kind='bar',color=('blue','red'),title='DC Universe')


I am working on the file "MBA_S16/Chao-WomenRightsGDP.ipynb".
The first task is:
Based on the findings above, let's drill down on the shining star in the pack (Costa Rica), a country that regresed (Nicaragua) and a middle of the road performer (El Salvador) to see what their performance over time has been on these indicators.
Costa Rica
Costa Rica has been consistently high over time for both respect of political and economic rights, with a stronger record for political rights.
Can you write Python code for it?

costaricaciridf = centamciridf.reset_index()
costaricaciridf = costaricaciridf.drop(costaricaciridf.columns[0],axis=1)
costaricaciridf = costaricaciridf[costaricaciridf['Country']=='Costa Rica']
costaricaciridf['Women Political Rights'].plot(kind='bar',
            title="Respect for Women's Political Rights in Costa Rica (2003-2011)", color="deepskyblue")

costaricaciridf['Women Economic Rights'].plot(kind='bar',
            title="Respect for Women's Economic Rights in Costa Rica (2003-2011)", color="mediumslateblue")


I am working on the file "MBA_S16/Chao-WomenRightsGDP.ipynb".
The first task is:
Defining Terms
All definitions taken from CIRI data documentation.
The indicators included in this project rank each country from 0 to 3, with 0 being the least respect for rights and 3 being the most. This level of respect is measured by the extent to which right are guaranteed within law and enforced by the government. In a country rated 0, one might find systematic discrimination against women built into the law. In a country rated 3, one would find virtually all rights for women guaranteed by law and also enforced by the government.
Women's Economic Rights
Women's economic rights include a number of internationally recognized rights. These rights include:

Equal pay for equal work
Free choice of profession or employment without the need to obtain a husband or male relative's consent
The right to gainful employment without the need to obtain a husband or male relative's consent
Equality in hiring and promotion practices
Job security (maternity leave, unemployment benefits, no arbitrary firing or layoffs, etc...)
Non-discrimination by employers
The right to be free from sexual harassment in the workplace
The right to work at night
The right to work in occupations classified as dangerous
The right to work in the military and the police force

Women's Political Rights
Women’s political rights include a number of internationally recognized rights. These rights include:

The right to vote
The right to run for political office
The right to hold elected and appointed government positions
The right to join political parties
The right to petition government officials

Women's Rights in Central America in 2003 vs. 2010
We see below that in 2003, Costa Rica led the region in its respect for women's right with a score of 2 on economic rights and score of 3 on political rights. Honduras and Nicaragua score similar to their neighbors on economic rights, but receive full scores on guaranteeing political rights for their female citizens.
Can you write Python code for it?

centamciridf['Women Economic Rights'][centamciridf['Year']==2003].plot(kind='bar',
            title="Respect for Women's Economic Rights in Central America (2003)", color="mediumslateblue")

centamciridf['Women Political Rights'][centamciridf['Year']==2003].plot(kind='bar',
            title="Respect for Women's Political Rights in Central America (2003)", color="deepskyblue")


I am working on the file "MBA_S16/Chao-WomenRightsGDP.ipynb".
The first task is:
GDP Growth: Costa Rica, Nicaragua, El Salvador
Now, let's turn to the question of whether a high level of respect for women's rights has translated into economic wellbeing at a country level. To do this, we look at GDP growth rather than absolute GDP in order to more easily compare countries despite differences in GDP size.
Can you write Python code for it?

centamgdpwbdf = centamgdpwbdf.reset_index()  # getting rid of multiindex for ease of graphing and comparison
centamgdpwbdf = centamgdpwbdf.rename(columns={'NY.GDP.MKTP.KD.ZG':'Annual GDP Growth'})

costaricawbdf = centamgdpwbdf[centamgdpwbdf['country']=='Costa Rica']
costaricawbdf = costaricawbdf.drop(costaricawbdf.columns[0],axis=1)

costaricawbdf['Annual GDP Growth'].plot(kind='bar',
            title="GDP Growth in Costa Rica (2003-2011)", color="lightgreen")

nicwbdf = centamgdpwbdf[centamgdpwbdf['country']=='Nicaragua']
nicwbdf = nicwbdf.drop(nicwbdf.columns[0],axis=1)

nicwbdf['Annual GDP Growth'].plot(kind='bar',
            title="GDP Growth in Nicaragaua (2003-2011)", color="tomato")

salvwbdf = centamgdpwbdf[centamgdpwbdf['country']=='El Salvador']
salvwbdf = salvwbdf.drop(salvwbdf.columns[0],axis=1)

salvwbdf['Annual GDP Growth'].plot(kind='bar',
            title="GDP Growth in El Salvador (2003-2011)", color="lemonchiffon")


I am working on the file "MBA_S16/Chao-WomenRightsGDP.ipynb".
The first task is:
Looking at these indicators for the same countries in 2010, one sees that Costa Rica again leads the pack, with a full score on both economic and political rights for women. Honduras and Nicaragua fell in their respect for women's political rights, leaving them on par with the region. Belize and Panama increased their respect for women's economic rights.
Can you write Python code for it?

centamciridf['Women Economic Rights'][centamciridf['Year']==2010].plot(kind='bar',
            title="Respect for Women's Economic Rights in Central America (2010)", color="mediumslateblue")

centamciridf['Women Political Rights'][centamciridf['Year']==2010].plot(kind='bar',
            title="Respect for Women's Political Rights in Central America (2010)", color="deepskyblue")


I am working on the file "MBA_S16/Chao-WomenRightsGDP.ipynb".
The first task is:
Nicaragua
Looking at Nicaragua over time, one sees that its drop in respect for political rights from 2003 stayed consistent until 2008 when it spiked again and then dropped through the end of the period covered by this data. Interestingly, its spike in economic rights over the 8 year period took place in 2007, the year before its spike in political rights. Whether there is a correlation between women's economic rights and political rights (one facilitating the other) requires further examination.
Can you write Python code for it?

nicciridf = centamciridf.reset_index()
nicciridf = nicciridf.drop(nicciridf.columns[0],axis=1)
nicciridf = nicciridf[nicciridf['Country']=='Nicaragua']
nicciridf['Women Political Rights'].plot(kind='bar',
            title="Respect for Women's Political Rights in Nicaragua(2003-2011)", color="deepskyblue")

nicciridf['Women Economic Rights'].plot(kind='bar',
            title="Respect for Women's Economic Rights in Nicaragua (2003-2011)", color="mediumslateblue")