I am working on the file "SVG_Table_Builder.ipynb".
The first task is:
Dictionary to SVG Table
Can you write Python code for it?

def quick_dict_to_SVG(Table_dict, width=500, L_width=200, x_location=0, y_location=0):
    title_color = (173,216,230) #lightblue
    header_color = (213,224,197) #a grey green
    font_family="Arial"
    font_size=12 #font_size=font_size
    header_font_size = 14
    line_width=1 #, line_width=line_width
    #good site for color translations: http://www.yellowpipe.com/yis/tools/hex-to-rgb/color-converter.php
    TL = [x_location, y_location]
    row_horz_shift = 5 #shift the row text over to line up with the header text
    display_text = ''
    for header, values in Table_dict.items():
        entries = list(values.keys())
        #Types = values['_type_']
        heading = True #default since most row groups have a heading
        #There are at least one entry unter each header
        for entry, explained in values.items():
            if entry =='_type_':
                #This entry indicates what type of group of rows this is.

                #Select the header color
                if explained == 'Heading':
                    bg = header_color
                elif explained == 'Title':
                    bg = title_color
                else: #for comment or other headerless row groups
                    heading = False #

                if heading:    #Build a header for the table
                    head=Table_Header(text=header, width=width, TL=list(TL), background=bg, font_family=font_family, size=header_font_size, line_width=line_width)
                    bg=header_color
                    display_text += head.get_SVG_header() #add the header
                    TL[1]=head.bottom #Move the top left cordinates
                    spacer = Table_rows(font_size=3, width=width, top_left=list(TL), line_width=line_width)
                    spacer.set_count(1)
                    display_text += spacer.get_SVG_rows() #add a spacer after the header
                    TL[1]=spacer.bottom #Move the top left cordinates

            elif...


I am working on the file "SVG_Table_Builder.ipynb".
The first task is:
Grab the %quickref Text with a New Magic Command
Creating custom(s) command documentation.
Can you write Python code for it?

from IPython.core.magic import (magics_class, line_magic)
from IPython.core.magics.basic import BasicMagics #Where _magic_docs is defined, also BasicMagics inherits the Magics already
@magics_class
class MyMagics(BasicMagics):
    @line_magic
    def quickref_text(self, line):
        """ Return the quickref text to be assigned to a variable """
        from IPython.core.usage import quick_reference
        qr = quick_reference + self._magic_docs(brief=True)
        return qr

ip = get_ipython()
ip.register_magics(MyMagics)

quickref_text = %quickref_text


I am working on the file "SVG_Table_Builder.ipynb".
The first task is:
Make a table of the Basic commands
Can you write Python code for it?

from IPython.display import SVG #to display the results
from collections import OrderedDict as Or_dict

#load the SVG Table classes
%run SVG_Table_Classes #used instead of import during editing
#Set the table size
width=552
Left_column_width=170
#Build the SVG text
start_stop = 4 #number of headings to have on the left
Table_dict = Or_dict()
for i in range(0,start_stop): #Grab some headings
    Table_dict[heading[i]]=Quick_ref_dic[heading[i]]

display_text, right, bottom = quick_dict_to_SVG(Table_dict, width=width, L_width=Left_column_width)
running_text = display_text
Bottom = bottom
#print("Right = %d"%right)
Table_dict = Or_dict()
for i in range(start_stop,8): #Grab some headings
    Table_dict[heading[i]]=Quick_ref_dic[heading[i]]
width=525
Left_column_width=130
display_text, right, bottom = quick_dict_to_SVG(Table_dict, width=width, L_width=Left_column_width, x_location=right+10)
Bottom =max(Bottom , bottom)
running_text += display_text

Text=Set_SVG_view(right, Bottom, running_text)


I am working on the file "SVG_Table_Builder.ipynb".
The first task is:
Make a table of the Magic commands
Can you write Python code for it?

#load the SVG Table classes
%run SVG_Table_Classes #used instead of import during editing (not needed a second time but left in for clarification)
#Set the table lengths
stops = [32+4+5,65+8+10]

#Build the SVG text

#Make a tempory dictionary of the magic heading entries
magic_dict = Or_dict()
magic_dict[heading[-1]]=Quick_ref_dic[heading[-1]]
entries = list(magic_dict[heading[-1]].keys())

#First table
Table_dict = Or_dict()
magic_dict_list = Or_dict()
for i in range(0,stops[0]): #grab some of the magic entries
    magic_dict_list[entries[i]] = magic_dict[heading[-1]][entries[i]]
#make a table dictionary to display
Table_dict[heading[-1]]=magic_dict_list

width=585
Left_column_width=145
display_text, right, bottom = quick_dict_to_SVG(Table_dict, width=width, L_width=Left_column_width)
Bottom = bottom #keep track of the bottom of the view port
running_text = display_text #start a running total of the display text

#Second table
Table_dict = Or_dict()
magic_dict_list = Or_dict()
for i in range(stops[0],stops[1]): #grab some of the magic entries
    magic_dict_list[entries[i]] = magic_dict[heading[-1]][entries[i]]
#make a table dictionary to display
Table_dict["Continuation"]=magic_dict_list
Table_dict["Continuation"]['_type_']="continuation"

width=600
Left_column_width=110#95
display_text, right, bottom = quick_dict_to_SVG(Table_dict, width=width, L_width=Left_column_width, x_location=right+10)
Bottom =max(Bottom , bottom) #keep track of the bottom of the view port
running_text += display_text #Add the diplay text to the running total of text

#Third table
Table_dict = Or_dict()
magic_dict_list = Or_dict()
last = len(entries)
for i in range(stops[1],last): #grab some of the magic entries
    magic_dict_list[entries[i]] = magic_dict[heading[-1]][entries[i]]
#make a table dictionary to display
Table_dict["Continuation"]=magic_dict_list
Table_dict["Continuation"]['_type_']="continuation"

width=575
Left_column_width=90
display_text, right, bottom = quick_dict_to_SVG(Table_dict, width=width,...


I am working on the file "SVG_Table_Builder.ipynb".
The first task is:
Save and display the results...
Can you write Python code for it?

#Save the result

with open("Basic_Help.svg",'w') as f:
    f.write(Text)

#Display the results
SVG(Text)


I am working on the file "SVM Binary Classification.ipynb".
The first task is:
2.1) Dealing with Missing Values – Imputation
Imputation is the act of replacing missing data values in a data set with meaningful values. Simply removing rows with missing feature values is bad practice if the data is scarce, as a lot of information could be lost. In addition, deletion methods can introduce bias [9].
The incomplete additional training data was combined with the complete original training data because the complete original data was scarce in number. However, the incomplete additional training data was missing values, therefore imputation was appropriate, if not required. Two methods of imputation were employed. The first method of imputation employed was imputation via feature means. However, this method has been heavily criticized. In particular, it has been hypothesized that imputation via mean introduces bias and underestimates variability [10].
The second method of imputation employed was k- Nearest-Neighbours (kNN) [11]. This is a technique, which is part of hot-deck imputation techniques [12], where missing feature values are filled from data points that are similar, or geometrically speaking, points that are closest in distance. This method is more appropriate than using the mean imputation, given the flaws of feature mean imputation. Therefore, kNN was the imputation method used to build the final model. The kNN implementation was found in the ‘fancyimpute’ package [13].
The k of kNN can be considered a parameter that needs to be chosen carefully. Fortunately, the literature provides some direction on this. The work of [14] suggests that kNN with 3 nearest-neighbours is the best for the trade-off between imputation error and preservation of data structure. In summary, kNN was employed for imputation, and k was set to 3.
This section will cover how the incomplete additional training data set was incorporated to develop a larger training data set. In particular, the additional training data was combined with the original training data. The additonal training data was incomplete, with...
Can you write Python code for it?

#Load additional training data
add_training_data = pd.read_csv("/Users/Max/Desktop/Max's Folder/Uni Work/Data Science MSc/Machine Learning/ML Kaggle Competition /Data Sets/Additional Training Data Set .csv", header=0, index_col=0)

#observe additional training data
add_training_data

#quantify class counts of additional training data
add_training_data.prediction.value_counts()

#find number of NAs for each column for additional training data
add_training_data.isnull().sum()

#concatenate original training data with additional training data
full_training_data_inc = pd.concat([training_data, add_training_data])
#observe concatenated training data
full_training_data_inc


I am working on the file "SVM Binary Classification.ipynb".
The first task is:
2.2) Dealing with Confidence Labels
One approach employed to incorporate the confidence labels was to use the confidence label of each instance as the corresponding sample weight for the instance. Theoretically, a confidence label of smaller than 1 would reduce the C parameter, which results in a lower penalty for misclassification of an instance whose label is not known with certainty. However, the implementation of this did not follow the theory; introducing the sample weights reduced the overall accuracy of the model. This matter was complicated more by the fact that samples generated from over-sampling via SMOTE had to be assigned a confidence label, which is difficult to determine objectively. Thus, it was decided that only data instances with a confidence label of 1 should be retained in the training data. This obviously leads to a massive loss of information. However, after removing instances, which do not have a confidence label of 1, 1922 training instances remained, which can be assumed to be a reasonable training data size. After truncating the data set, the procedure described in Section 2.2 was repeated for the truncated training data.
In summary, the training data was truncated to only include instances that have a confidence label of 1. The minority class of the training data was over-sampled using SMOTE to balance the class split. Class weights were then applied during the training of the SVM to ensure that the model was more sensitive to correctly classifying the majority class of the test data.
This section will cover how the confidence labels, one of the additional pieces of information provided in the assignment outline, were incorporated into the final training data set.
Can you write Python code for it?

#Load confidence annotations
confidence_labels = pd.read_csv("/Users/Max/Desktop/Max's Folder/Uni Work/Data Science MSc/Machine Learning/ML Kaggle Competition /Data Sets/Annotation Confidence .csv", header=0, index_col=0)

#quantify confidence labels (how many are 1, how many are 0.66)
print(confidence_labels.confidence.value_counts())

#observe confidence annotations
confidence_labels

#adding confidence of label column to imputed full training data set
full_train_wcl = pd.merge(full_training_data, confidence_labels, left_index=True, right_index=True)
full_train_wcl


I am working on the file "SVM Binary Classification.ipynb".
The first task is:
2.3) Dealing with Class Imbalance
Binary classification tasks suffer from imbalanced class splits. Training a model on a data set with more instances for one class than the other class can result in biases towards the majority class, as sensitivity will be lost in detecting the minority class [17]. This is pertinent because the training data (additional and original data included) has an unbalanced class split, with more instances of Class 1 than Class 0. Thus, training the model on this data would result in a model that is biased towards Class 1 detections. To exacerbate this issue, the test data is also unbalanced, but the majority class for the test data is Class 0.
Some researchers have already attempted to tackle this problem. There are two primary methods of dealing with class imbalances: balancing or further unbalancing the data set as needs fit, or introducing class weights, where the underlying algorithm applies disparate
misclassification penalties to different classes [15]. Both approaches will be combined here, to first balance the data set, and then train the model to be bias towards Class 0 instances, as Class 0 is the majority class in the test data. The ‘imbalanced-learn’ API [16] has implementations of class balancing strategies found in the literature, such as SMOTE [17]. The premise of SMOTE is over-sampling of the minority class until a balance in the data set is reached. The sampling method is based on sampling via kNN. Unlike kNN for imputation, the best k was suggested to be 5 here.
Once the data set was balanced through SMOTE, class weights were introduced. Considering the test data has more instances belonging to Class 0, the class weights were adjusted so that misclassification of Class 0 is penalized more heavily than misclassification of Class 1. The ratio of class weights for training were adjusted to match the class proportions of the test data, i.e. Class 0 weight = 1.33 and Class 1 weight = 1.
The reason over-sampling of the minority class was preferred over under-sampling of...
Can you write Python code for it?

from imblearn.over_sampling import SMOTE
from collections import Counter

#fit over-sampling to training data inputs and putputs
over_sampler = SMOTE(ratio='auto', k_neighbors=5, kind='regular', random_state=0)
over_sampler.fit(conf_ft_inputs, conf_ft_outputs)

#create new inputs and outputs with correct class proportions
resampled_x, resampled_y = over_sampler.fit_sample(conf_ft_inputs, conf_ft_outputs)

#quantify original class proportions prior to over-sampling
Counter(conf_ft_outputs)

#quantify class proportions after over-sampling
Counter(resampled_y)

#assign newly sampled input and outputs to old variable name used for inputs and outputs before
#over-sampling
conf_ft_inputs = resampled_x
conf_ft_outputs = resampled_y
print(Counter(conf_ft_outputs))


I am working on the file "SVM Binary Classification.ipynb".
The first task is:
3. Pre-Processing
The pre-processing of the data consisted of several steps. First, the features were rescaled appropriately. Secondly, Feature Extraction was performed to reduce the unwieldy dimensionality of the training data set, concomitantly increasing the signal-to-noise ratio and decreasing time complexity.
This section will cover the Pre-Processing conducted that produced the model capable of producing the best predictions. Feature Scaling was achieved via several methods. The best method was standardisation. Feature Extraction was achieved via PCA.
3.1) Feature Scaling
Feature scaling is important because it ensures that features have values plotted on the same scale, irrespective of the original units used to describe the original features. Feature scaling can be in the form of standardization, normalization or rescaling. The correct choice of feature scaling method is arbitrary and highly dependent on context. Thus, all three approaches were tried. The optimal results were obtained for standardization.
Can you write Python code for it?

#standardise the full training data with confidence labels 1 only
scaler_2 = preprocessing.StandardScaler().fit(conf_ft_inputs)
std_conf_ft_in = scaler_2.transform(conf_ft_inputs)
std_conf_ft_in


I am working on the file "SVM Binary Classification.ipynb".
The first task is:
4. Model Selection
The optimization was conducted through the use of a Grid search. In addition, the optimization was conducted for two kernels: the polynomial kernel and the RBF kernel. The initial search for optimal parameters was conducted on a logarithmic scale to explore as much of the parameter space as possible. From the results, the parameter ranges were refined and pruned to only include the potential best candidates. The choice of parameters was purely based on accuracy metrics, not on any other practical factors such as memory consumption or time complexity of predictions. The best model was determined on the following merits:
1. Good generalisation - achieving a high testing 356 score during cross-validation.
2. Avoidance of over-fitting - restriction on the  magnitude of training scores during cross-validation. In particular, a training score beyond 360 an arbitrary limit is indicative of over-fitting. 361 Thus, a balance had to be struck to ensure that good 362 generalisation can be assumed.
This section covers how the best model was selected. Two kernels were tried and tested: RBF and polynomial. RBF outperformed polynomial, therefore only the optimisation results of RBF will be presented here. Furthermore, the parameter ranges to try have already been pruned at this point, so only the final ranges will be used to perform a Grid Search.
4.1) Parameter Optimisation
Can you write Python code for it?

#this cell takes around 7 minutes to run
#parameter optimisation with Exhaustive Grid Search, with class weight
original_c_range = np.arange(0.85, 1.01, 0.01)
gamma_range = np.arange(0.00001, 0.00023, 0.00002)

#define parameter ranges to test
param_grid = [{'C': original_c_range, 'gamma': gamma_range, 'kernel': ['rbf'],
             'class_weight':[{0:1.33, 1:1}]}]

#define model to do parameter search on
svr = SVC()
clf = GridSearchCV(svr, param_grid, scoring='accuracy', cv=5,)
clf.fit(std_conf_ft_in_pca, conf_ft_outputs)

#create dictionary of results
results_dict = clf.cv_results_

#convert the results into a dataframe
df_results = pd.DataFrame.from_dict(results_dict)
df_results


I am working on the file "SVM Binary Classification.ipynb".
The first task is:
Making Predictions
The following cells will prepare the test data by getting it into the right format.
Can you write Python code for it?

#Load the complete training data set
test_data = pd.read_csv("/Users/Max/Desktop/Max's Folder/Uni Work/Data Science MSc/Machine Learning/ML Kaggle Competition /Data Sets/Testing Data Set.csv", header=0, index_col=0)

##Observe the test data
test_data

#turn test dataframe into matrix
test_data_matrix = test_data.as_matrix(columns=None)
test_data_matrix.shape


I am working on the file "SVM Binary Classification.ipynb".
The first task is:
The cell below will plot how much more of the variance in the data set is explained as the number of principal components included is increased.
Can you write Python code for it?

#calculate a list of cumulative sums for amount of variance explained
cumulative_variance = np.cumsum(pca_2.explained_variance_ratio_)
len(cumulative_variance)
#add 0 to the beginning of the list, otherwise list starts with variance explained by 1 pc
cumulative_variance = np.insert(cumulative_variance, 0, 0)

#define range of pcs
pcs_4_var_exp = np.arange(0,701,1)
len(pcs_4_var_exp)

fig_1 = plt.figure(figsize=(7,4))
plt.title('Number of PCs and Change In Variance Explained')
plt.xlabel('Number of PCs')
plt.ylabel('Variance Explained (%)')
plt.plot(pcs_4_var_exp, cumulative_variance, 'x-', color="r")
plt.show()


I am working on the file "SVM Binary Classification.ipynb".
The first task is:
The cell below will plot two heat-maps side by side: one for showing how the training accuracy changes during cross-validation for different combinations of parameters, and one for showing how the testing accuracy changes during cross-validation for different combinations of parameters.
Can you write Python code for it?

#Draw heatmap of the validation accuracy as a function of gamma and C
fig = plt.figure(figsize=(10, 10))
ix=fig.add_subplot(1,2,1)
val_scores = clf.cv_results_['mean_test_score'].reshape(len(original_c_range),len(gamma_range))
val_scores

ax = sns.heatmap(val_scores, linewidths=0.5, square=True, cmap='PuBuGn',
                 xticklabels=gamma_range, yticklabels=original_c_range, cbar_kws={'shrink':0.5})
ax.invert_yaxis()
plt.yticks(rotation=0, fontsize=10)
plt.xticks(rotation= 70,fontsize=10)
plt.xlabel('Gamma', fontsize=15)
plt.ylabel('C', fontsize=15)
plt.title('Validation Accuracy', fontsize=15)

#Draw heatmap of the validation accuracy as a function of gamma and C
ix=fig.add_subplot(1,2,2)
train_scores = clf.cv_results_['mean_train_score'].reshape(len(original_c_range),len(gamma_range))
train_scores
#plt.figure(figsize=(6, 6))
ax_1 = sns.heatmap(train_scores, linewidths=0.5, square=True, cmap='PuBuGn',
                 xticklabels=gamma_range, yticklabels=original_c_range, cbar_kws={'shrink':0.5})
ax_1.invert_yaxis()
plt.yticks(rotation=0, fontsize=10)
plt.xticks(rotation= 70,fontsize=10)
plt.xlabel('Gamma', fontsize=15)
plt.ylabel('C', fontsize=15)
plt.title('Training Accuracy', fontsize=15)
plt.show()


I am working on the file "SVM Binary Classification.ipynb".
The first task is:
The cells below will plot a Validation Curves for Gamma.
Can you write Python code for it?

#import module/library
from sklearn.model_selection import validation_curve
import matplotlib.pyplot as plt
%matplotlib inline

#specifying gamma parameter range to plot for validation curve
param_range = gamma_range
param_range

#calculating train and validation scores
train_scores, valid_scores = validation_curve(SVC(C=0.92, kernel='rbf', class_weight={0:1.33, 1:1}), std_conf_ft_in_pca, conf_ft_outputs, param_name='gamma',param_range=param_range,scoring='accuracy')
train_scores_mean = np.mean(train_scores, axis=1)
train_scores_std = np.std(train_scores, axis=1)
valid_scores_mean = np.mean(valid_scores, axis=1)
valid_scores_std = np.std(valid_scores, axis=1)

#plotting validation curve
plt.title('Gamma Validation Curve for SVM With RBF Kernel | C=0.92')
plt.xlabel('Gamma')
plt.ylabel('Score')
plt.xticks(rotation=70)
plt.ylim(0.8,1.0)
plt.xlim(0.0001,0.00021)
plt.xticks(param_range)
lw=2
plt.plot(param_range, train_scores_mean, 'o-',label="Training Score", color='darkorange', lw=lw)
plt.fill_between(param_range, train_scores_mean-train_scores_std, train_scores_mean+train_scores_std, alpha=0.2, color='darkorange', lw=lw)
plt.plot(param_range, valid_scores_mean, 'o-',label="Testing Score", color='navy', lw=lw)
plt.fill_between(param_range, valid_scores_mean-valid_scores_std, valid_scores_mean+valid_scores_std, alpha=0.2, color='navy', lw=lw)
plt.legend(loc='best')
plt.show()


I am working on the file "SVM Binary Classification.ipynb".
The first task is:
The cells below will plot the Learning Curve.
Can you write Python code for it?

#import module/library
from sklearn.model_selection import learning_curve

#define training data size increments
td_size = np.arange(0.1, 1.1, 0.1)
#calculating train and validation scores
train_sizes, train_scores, valid_scores = learning_curve(SVC(C=0.92, kernel='rbf', gamma=0.00011, class_weight={0:1.33, 1:1}), std_conf_ft_in_pca, conf_ft_outputs, train_sizes=td_size ,scoring='accuracy')
train_scores_mean = np.mean(train_scores, axis=1)
train_scores_std = np.std(train_scores, axis=1)
valid_scores_mean = np.mean(valid_scores, axis=1)
valid_scores_std = np.std(valid_scores, axis=1)

#plotting learning curve
fig = plt.figure(figsize=(5,5))
plt.title('Learning Curve with SVM with RBF Kernel| C=0.92 & Gamma = 0.00011', fontsize=9)
plt.xlabel('Train Data Size')
plt.ylabel('Score')
plt.ylim(0.8,1)
lw=2
plt.plot(train_sizes, train_scores_mean, 'o-', color="r", label="Training Score")
plt.fill_between(train_sizes, train_scores_mean-train_scores_std, train_scores_mean+train_scores_std, alpha=0.2, color='red', lw=lw)
plt.plot(train_sizes, valid_scores_mean, 'o-', color="g",label="Testing Score")
plt.fill_between(train_sizes, valid_scores_mean-valid_scores_std, valid_scores_mean+valid_scores_std, alpha=0.2, color='green', lw=lw)
plt.legend(loc='best')
plt.show()


I am working on the file "SVM Binary Classification.ipynb".
The first task is:
The following cell will apply the same pre-processing applied to the training data to the test data.
Can you write Python code for it?

#pre-process test data in same way as train data
scaled_test = scaler_2.transform(test_data_matrix)
transformed_test = pca_2.transform(scaled_test)
transformed_test.shape


I am working on the file "SVM Binary Classification.ipynb".
The first task is:
The graph above suggests that the maximum number of principal components should not exceed 300, as less and less variance is explained as the number of principal components included increases beyond 300. For the optimisation, the optimal number of principal components was initially assumed to be 230.
Can you write Python code for it?

#preprocessing: PCA (feature construction)
pca_2 = PCA(n_components=230, random_state=0)
std_conf_ft_in_pca = pca_2.fit_transform(std_conf_ft_in)
#quantify ratio of variance explain by principal components
print("Total Variance Explained by PCs (%): ", np.sum(pca_2.explained_variance_ratio_))


I am working on the file "SVM Test.ipynb".
The first task is:
Right now we have a dataframe similar to the one with the fit points. We'll use the classifier to add a fourth column with the predicted values. Our goal is to have the same value in both real_value and predicted_value columns.
Can you write Python code for it?

df_test['predicted_value'] = clf.predict(test_points)
print(df_test)


I am working on the file "SVM Test.ipynb".
The first task is:
THe classifier is pretty successfull at predicting values from the x and ycoordinates. We may also apply the classifier to the fit points - it's somewhat pointless, because those are the points used to infer the data classification function.
Can you write Python code for it?

df_fit[''] = clf.predict([x[0:2] for x in fit_points])
print(df_fit)


I am working on the file "SVM Test.ipynb".
The first task is:
The classifier is now trained with the fit points, and is ready to be evaluated with a set of test points, which have a similiar structure as the fit points: x, y coordinates, and a value.
Can you write Python code for it?


I am working on the file "SVM Test.ipynb".
The first task is:
To better understand the data separation between values 1 and 0, we'll plot both the fit points and the test points.
Following the same color code as before, points that belong to class 1 are represented in black, and points that belong to class 0 in red. Fit points are represented in a full cirle, and the test points are represented by circunferences.
Can you write Python code for it?

sns.set(style="darkgrid")
for i in range(0,2):
  pl.scatter(df_fit[df_fit.value==i].x,
             df_fit[df_fit.value==i].y,
             color = 'black' if i == 1 else 'red')
  pl.scatter(df_test[df_test.predicted_value==i].x,
             df_test[df_test.predicted_value==i].y,
             marker='o',
             facecolor='none',
             color='black' if i == 1 else 'red')
pl.grid(True)
pl.show()


I am working on the file "SVM Test.ipynb".
The first task is:
We are now able to build and train the classifier.
Can you write Python code for it?

clf = svm.SVC()
clf.fit(df_fit[train_cols], df_fit.value)


I am working on the file "SVM Test.ipynb".
The first task is:
We begin by defining a set of training points. This is the set which the classifier will use to infer the data classification function. Each row represents a data point, with x,y coordinates and classification value.
Can you write Python code for it?


I am working on the file "SVM Test.ipynb".
The first task is:
We build the test points dataframe with the features.
Can you write Python code for it?

df_test = pd.DataFrame(test_points, columns=['x','y'])
print(df_test)


I am working on the file "SVM Test.ipynb".
The first task is:
We can add the values to the dataframe.
Can you write Python code for it?

df_test['real_value'] = test_points_values
print(df_test)


I am working on the file "SVM Test.ipynb".
The first task is:
We need to select the set of columns with the data features. In our example, those are the x and y coordinates.
Can you write Python code for it?


I am working on the file "SVM Test.ipynb".
The first task is:
We separate the features and values to make clear were the data comes from.
Can you write Python code for it?


I am working on the file "SVM-comparision.ipynb".
The first task is:
3. Not-linearly separable data
Can you write Python code for it?

from sklearn.datasets.samples_generator import make_circles
X, y = make_circles(100, factor=.3, noise=.3)

plt.scatter(X[:, 0], X[:, 1], c=y, s=50, cmap='spring')
compareWRTScore(X, y)


I am working on the file "SVM-comparision.ipynb".
The first task is:
As it can be seen, all the lines separate the classes perfectly. What's more, we can draw more straight lines still separating perfectly. But only one of them will give us the minimum misclassification error. So which line is that?
Can you write Python code for it?


I am working on the file "SVM-comparision.ipynb".
The first task is:
Comparing classifiers
1. Iris data
Can you write Python code for it?

iris = datasets.load_iris()
compareWRTScore(iris.data[:,:2], iris.target)


I am working on the file "SVM-comparision.ipynb".
The first task is:
Let w is a vector perpendicular to the median line. But we do not know its length yet. And u is an unknown example. We project the u vector down on to the w vector so that we have a distance. It goes further out, it ends up on the right side of the street. Or it is a short distance which means it is on the left side. So we can formulate it like this:
$$ \vec{w}  \cdot \vec{u} \geq c $$
or can be rewritten as: $$ c = -b $$
$$ \vec{w}  \cdot \vec{u}{+} + b \geq 1 \ \ \ {{for + samples}}$$
$$ \vec{w}  \cdot \vec{u}{-} + b \leq -1 \ \ \ {{for - samples}}$$
to make it mathematically convenient, reformulate it as:
$$ y \in \left { -1, 1 \right } \ \ \  and \ \ \  y(\vec{w}  \cdot \vec{u} + b) + 1 = 0 $$
It is equal to 0 when the sample is on the street.
$$ width\ of\ the\ street = (\vec{x}{+} - \vec{x}{-}) \cdot \frac{\vec{w}}{\left \| \vec{w} \right \|}$$
$$ width\ of\ the\ street =  \frac{2}{\left \| \vec{w} \right \|}$$
So we want to maximize the margin, or the minimize the following:
$$ \frac{1}{2} \left \| \vec{w} \right \|^{2} $$
So far, we've come up with the two constraints. By using Lagrange multipliers, we can get a formula which gives maximum/minimum without thinking about the constraints anymore.
$$ L = \frac{1}{2} \left \| \vec{w} \right \|^{2}  -
\sum \lambda \left [  y_{i} (\vec{w} \cdot \vec{x_{i}+b} - 1 )  \right ] $$
After takin the derivates of L with respect to w and b:
$$ \frac{\partial L}{\partial \vec{w}} = \vec{w} - \sum \lambda \cdot y_{i} \cdot \vec{x_{i}}  $$
$$ \frac{\partial L}{\partial b} =  - \sum \vec{x_{i}} \cdot y_{i} = 0$$
Now the decision rule will be as:
$$ \sum \lambda {i}  \cdot y{i} \cdot \vec{x_{i}} \cdot \vec{u} + b \geq 0 $$
Kernel trick
SVM can also work with non-linear data with the help of kernel trick. Kernel trick maps data points to a higher dimension in which classification problem becomes linearly separable.
Most popular kernel tricks are:
linear: $$\langle x, x'\rangle$$.
polynomial: $$(\gamma \langle x, x'\rangle + r)^d.d$$ is specified by keyword degree, r...
Can you write Python code for it?

YouTubeVideo("3liCbRZPrZA")

from matplotlib.colors import ListedColormap
# Create color maps for 3-class classification problem, as with iris
cmap_light = ListedColormap(['#FFAAAA', '#AAFFAA', '#AAAAFF'])
cmap_bold = ListedColormap(['#FF0000', '#00FF00', '#0000FF'])

def plot_estimator(estimator, X, y, title=""):

    try:
        X, y = X.values, y.values
    except AttributeError:
        pass

    estimator.fit(X, y)
    x_min, x_max = X[:, 0].min() - .1, X[:, 0].max() + .1
    y_min, y_max = X[:, 1].min() - .1, X[:, 1].max() + .1
    xx, yy = np.meshgrid(np.linspace(x_min, x_max, 100),
                         np.linspace(y_min, y_max, 100))

    Z = estimator.predict(np.c_[xx.ravel(), yy.ravel()])

    # Put the result into a color plot
    Z = Z.reshape(xx.shape)
    plt.figure()
    plt.pcolormesh(xx, yy, Z, cmap=cmap_light)

    # Plot also the training points
    plt.scatter(X[:, 0], X[:, 1], c=y, cmap=cmap_bold)
    plt.axis('tight')
    plt.axis('off')
    plt.title(title)
    plt.tight_layout()

def classify(clfs, X, y):
    for title, clf in clfs:
        X_train, X_test, y_train, y_test = cross_validation.train_test_split(
            X, y, test_size=0.3, random_state=0)

        clf.fit(X_train, y_train)
        score = clf.score(X_test, y_test)
        plot_estimator(clf, X, y, "{} - score: {}".format(title, score))
        plt.show()

def timeit(clfs, X, y):
    results = []
    for title, clf in clfs:
        X_train, X_test, y_train, y_test = cross_validation.train_test_split(
            X, y, test_size=0.3, random_state=0)

        trial = 500

        startTime = time.time()
        for i in range(trial):
            clf.fit(X_train, y_train)
        trainingTime = time.time() - startTime
        print "{} times training {} with the data that has shape {} took {}".format(
            trial, title, X_train.shape, trainingTime)

        startTime = time.time()
        for i in range(trial):
            for i in X_test:
                score =...


I am working on the file "SVM-comparision.ipynb".
The first task is:
Support vector machines (SVM)
SVM is a method that is widely used for classification and regression tasks. The most important characteristic of svm is that it tries to find a decision boundary which separates classes with equal distance. SVM decision boundaries are as wide as possible so that's why sometimes this method is called widest street approach, or large margin classifier.
SVM has other nice properties such as it is convex, theoretically motivated, nonlinear with kernels.
Let's have a look at the space shown below.
Can you write Python code for it?

x1 = [1, 2, 3, 4, 5, 6]
y1 = [12, 18, 14, 8, 10, 6]

x2 = [4, 5, 7, 8]
y2 = [47, 37, 22, 24]
plt.plot(x1, y1, "o", color="yellow")
plt.plot(x2, y2, "x", color="green")

plt.plot(range(3, 8),[ i * (-14.2) + 100 for i in range(3, 8)], color="blue")
#plt.plot(range(12),[ i * - 4.4 + 45 for i in range(12)], color="blue")
plt.plot(range(9),[ i * - 2.0 + 24 for i in range(9)], color="red")
plt.plot(range(9),[ i * - 4.6 + 44 for i in range(9)], color="k")

plt.title("Figure 1: possible hyperplanes")


I am working on the file "SVM-comparision.ipynb".
The first task is:
The inference is that this sex feature can be used in a classification task to determine whether a given person survived or not.
Pclass feature
This stands for Passenger Class. There are three classes as 1 = 1st; 2 = 2nd; 3 = 3rd. We can make a guess saying most probably the first class passengers survived thanks to their nobility. This guess is based on the domain knowledge; in that time classes among the people is more obvious and severe than now. Let's have a look at the data to see the truth.
Can you write Python code for it?

sns.countplot(data=df, hue="Survived", x="Pclass")

sns.countplot(data=df[df['Pclass'] == 1], hue="Survived", x="Sex")


I am working on the file "SVM.ipynb".
The first task is:
GridSearchCV in sklearn
GridSearchCV is a way of systematically working through multiple combinations of parameter tunes, cross-validating as it goes to determine which tune gives the best performance. The beauty is that it can work through many combinations in only a couple extra lines of code.
Here's an example from the sklearn documentation:
python
parameters = {'kernel':('linear', 'rbf'), 'C':[1, 10]}
svr = svm.SVC()
clf = grid_search.GridSearchCV(svr, parameters)
clf.fit(iris.data, iris.target)
Let's break this down line by line.
python
parameters = {'kernel':('linear', 'rbf'), 'C':[1, 10]}
A dictionary of the parameters, and the possible values they may take. In this case, they're playing around with the kernel (possible choices are 'linear' and 'rbf'), and C (possible choices are 1 and 10).
Then all the following combinations of values for (kernel, C) are automatically generated: [('rbf', 1), ('rbf', 10), ('linear', 1), ('linear', 10)]. Each is used to train an SVM, and the performance is then assessed using cross-validation.
python
svr = svm.SVC()
This looks kind of like creating a classifier, just like we've been doing since the first lesson. But note that the "clf" isn't made until the next line--this is just saying what kind of algorithm to use. Another way to think about this is that the "classifier" isn't just the algorithm in this case, it's algorithm plus parameter values. Note that there's no monkeying
around with the kernel or C; all that is handled in the next line.
python
clf = grid_search.GridSearchCV(svr, parameters)
This is where the first bit of magic happens; the classifier is being created. We pass the algorithm (svr) and the dictionary of parameters to try (parameters) and it generates a grid of parameter combinations to try.
python
clf.fit(iris.data, iris.target)
And the second bit of magic. The fit function now tries all the parameter combinations, and returns a fitted classifier that's automatically tuned to the optimal parameter combination. You can now access the parameter...
Can you write Python code for it?

from sklearn.grid_search import GridSearchCV
from sklearn import svm

#features_train = features_train[:len(features_train)/100]
#labels_train = labels_train[:len(labels_train)/100]

features_train, features_test, labels_train, labels_test = preprocess() # full training set

parameters = {'kernel':('linear', 'rbf'), 'C':[10, 100, 1000, 10000]}
svr = svm.SVC()
clf = GridSearchCV(svr, parameters)

t0 = time()
clf.fit(features_train, labels_train)
print("done in %0.3fs" % (time() - t0))
print("Best estimator found by grid search:")
print(clf.best_estimator_)

len(features_train)

from sklearn import datasets
from sklearn.svm import SVC

iris = datasets.load_iris()
features = iris.data
labels = iris.target

###############################################################
### YOUR CODE HERE
###############################################################

### import the relevant code and make your train/test split
### name the output datasets features_train, features_test,
### labels_train, and labels_test

### set the random_state to 0 and the test_size to 0.4 so
### we can exactly check your result

from sklearn import cross_validation

iris.data.shape, iris.target.shape

### We can now quickly sample a training set while holding out 40% of the data for testing
### (evaluating) our classifier:
features_train, features_test, labels_train, labels_test = cross_validation.train_test_split(iris.data,
                                iris.target, test_size=0.4, random_state=0)

features_train.shape, labels_train.shape

features_test.shape, labels_test.shape

###############################################################

clf = SVC(kernel="linear", C=1.)
clf.fit(features_train, labels_train)

print clf.score(features_test, labels_test)

##############################################################
def submitAcc():
    return clf.score(features_test, labels_test)

from sklearn.cross_validation import KFold

t0 = time()


I am working on the file "SVM.ipynb".
The first task is:
Speeding up the algorithm
One way to speed up an algorithm is to train it on a smaller training dataset. The tradeoff is that the accuracy almost always goes down when you do this. Let’s explore this more concretely: add in the following two lines immediately before training your classifier.
python
features_train = features_train[:len(features_train)/100]
labels_train = labels_train[:len(labels_train)/100]
These lines effectively slice the training dataset down to 1% of its original size, tossing out 99% of the training data. You can leave all other code unchanged. What’s the accuracy now?
Can you write Python code for it?

from sklearn import svm
clf = svm.SVC(kernel="linear")

features_train = features_train[:len(features_train)/100]
labels_train = labels_train[:len(labels_train)/100]

t0 = time()
clf.fit(features_train, labels_train)
print "training time:", round(time()-t0, 3), "s"

t0 = time()
pred = clf.predict(features_test)
print "testing time:", round(time()-t0, 3), "s"

from sklearn.metrics import accuracy_score

def submitAccuracy():
    return accuracy_score(pred, labels_test)

submitAccuracy()


I am working on the file "SVM.ipynb".
The first task is:
There are over 1700 test events--how many are predicted to be in the “Chris” (1) class?
(Use the RBF kernel, C=10000., and the full training set.)
Can you write Python code for it?

pred = clf2.predict(features_test)
len(pred)

chrisPred = 0
for item in range(len(pred)):
    if pred[item] == 1:
        chrisPred += 1

print "The number of emails that are predicted to be authored by “Chris” (1) class is: ", chrisPred


I am working on the file "SVM/Example_of_ML_in_Action.ipynb".
The first task is:
Get the separating hyperplane
Can you write Python code for it?

labels = df.loc[:, 'y'].values
data = df.loc[:, ['x1', 'x2']].values

w = np.dot(alphas*lables, data)

w


I am working on the file "SVM/Example_of_ML_in_Action.ipynb".
The first task is:
Get the support vectors
Can you write Python code for it?

for i in range(100):
    if alphas[i] > 0.:
        print df.loc[i, :]


I am working on the file "SVM/Example_of_ML_in_Action.ipynb".
The first task is:
Plot
Can you write Python code for it?


I am working on the file "SVM/Example_of_ML_in_Action.ipynb".
The first task is:
The Full Platt SMO
Some support functions
Can you write Python code for it?

class optStruct:
    def __init__(self, data, labels, C, toler):
        self.X = data
        self.labels = labels
        self.C = C
        self.tol = toler
        self.m = data.shape[0]
        self.alphas = np.zeros((self.m, ))
        self.b = 0
        self.errCache = np.zeros((self.m, 2)) # store error cache


I am working on the file "SVM_Tutorial.ipynb".
The first task is:
Below, we plot the first two features from the dataset (sepal length and width). Normally we would try to use all useful features, but sticking with two allows us to visualize the data more easily.
Then we plot the data to get a look at what we're dealing with. The colormap is used to determine what colors are used for each class when plotting.
Can you write Python code for it?

# Plot the data

# Sepal length and width
X_small = X[:,:2]
# Get the minimum and maximum values with an additional 0.5 border
x_min, x_max = X_small[:, 0].min() - .5, X_small[:, 0].max() + .5
y_min, y_max = X_small[:, 1].min() - .5, X_small[:, 1].max() + .5

plt.figure(figsize=(8, 6))

# Plot the training points
plt.scatter(X_small[:, 0], X_small[:, 1], c=y, cmap=cmap_bold)
plt.xlabel('Sepal length (cm)')
plt.ylabel('Sepal width (cm)')
plt.title('Sepal width vs length')

# Set the plot limits
plt.xlim(x_min, x_max)
plt.ylim(y_min, y_max)

# Plot the legend
plt.legend(labelList, labelNames)

plt.show()


I am working on the file "SVM_Tutorial.ipynb".
The first task is:
Comparing Models with Crossvalidation
To select the kernel to use in our model, we need to use crossvalidation. We can then get our final result using our test data.
First we choose the kernels we want to investigate, then divide our training data into folds. We loop through the sets of training and validation folds. Each time, we train each model on the training data and evaluate on the validation data. We store the accuracy of each classifier on each fold so we can look at them later.
Can you write Python code for it?

# Choose our kernels
kernels = ['linear', 'rbf']

# Create a dictionary of arrays to store accuracies
accuracies = {}
for kernel in kernels:
    accuracies[kernel] = []

# Loop through 5 folds
kf = KFold(n_splits=5)
for trainInd, valInd in kf.split(X_train):
    X_tr = X_train[trainInd,:]
    y_tr = y_train[trainInd]
    X_val = X_train[valInd,:]
    y_val = y_train[valInd]

    # Loop through each kernel
    for kernel in kernels:

        # Create the classifier
        clf = SVC(kernel=kernel, decision_function_shape='ovo')

        # Train the classifier
        clf.fit(X_tr, y_tr)

        # Make our predictions
        pred = clf.predict(X_val)

        # Calculate the accuracy
        accuracies[kernel].append(np.mean(pred == y_val))


I am working on the file "SVM_Tutorial.ipynb".
The first task is:
Final Evaluation
The linear kernel gives us the highest accuracy, so we select it as our best model. Now we can evaluate it on our training set and get our final accuracy rating.
Can you write Python code for it?

clf = SVC(kernel='linear', decision_function_shape='ovo')
clf.fit(X_train, y_train)
predictions = clf.predict(X_test)

accuracy = np.mean(predictions == y_test) * 100

print('The final accuracy is %.2f' % accuracy + '%')


I am working on the file "SVM_Tutorial.ipynb".
The first task is:
Import the dataset
Import the dataset and store it to a variable called iris. Scikit-learn's explanation of the dataset is here. This dataset is similar to a python dictionary, with the keys: ['DESCR', 'target_names', 'target', 'data', 'feature_names']
The data features are stored in iris.data, where each row is an example from a single flower, and each column is a single feature. The feature names are stored in iris.feature_names. Labels are stored as the numbers 0, 1, or 2 in iris.target, and the names of these labels are in iris.target_names.
The dataset consists of measurements made on 50 examples from each of three different species of iris flowers (Setosa, Versicolour, and Virginica). Each example has four features (or measurements): sepal length, sepal width, petal length, and petal width. All measurements are in cm.
Below, we load the labels into y, the corresponding label names into labelNames, the data into X, and the names of the features into featureNames.
Can you write Python code for it?

# Import some data to play with
iris = datasets.load_iris()

# Store the labels (y), label names, features (X), and feature names
y = iris.target       # Labels are stored in y as numbers
labelNames = iris.target_names # Species names corresponding to labels 0, 1, and 2
X = iris.data
featureNames = iris.feature_names


I am working on the file "SVM_Tutorial.ipynb".
The first task is:
Making predictions
Now, let's say we go out and measure the sepals of two new iris plants, and want to know what species they are. We're going to use our classifier to predict the flowers with the following measurements:
Plant | Sepal length | Sepal width
------|--------------|------------
A     |4.3           |2.5
B     |6.3           |2.1
We can use our classifier's predict() function to predict the label for our input features. We pass in the variable examples to the predict() function, which is a list, and each element is another list containing the features (measurements) for a particular example. The output is a list of labels corresponding to the input examples.
Can you write Python code for it?

# Add our new data examples
examples = [[4.3, 2.5], # Plant A
            [6.3, 2.1]] # Plant B

# Create an instance of SVM and fit the data
clf = SVC(kernel='linear', decision_function_shape='ovo')
clf.fit(X_small, y)

# Predict the labels for our new examples
labels = clf.predict(examples)

# Print the predicted species names
print('A: ' + labelNames[labels[0]])
print('B: ' + labelNames[labels[1]])


I am working on the file "SVM_Tutorial.ipynb".
The first task is:
Now the plot looks very different from before! The solid black lines are now all curves, but each decision boundary still falls along one part of those lines. And instead of having dotted lines parallel to the solid, there are smaller ellipsoids on either side of the solid line.
What other kernels exist?
Scikit-learn comes with two other default kernels: polynomial and sigmoid. Advanced users can also creat their own kernels, but we will stick to the defaults for now.
Below, modify our previous examples to try out the other kernels. How do they change the decision boundaries?
Can you write Python code for it?


I am working on the file "SVM_Tutorial.ipynb".
The first task is:
Plotting our predictions
Now let's plot our predictions to see why they were classified that way.
Can you write Python code for it?

# Now plot the results
h = .02  # step size in the mesh

# Plot the decision boundary. For that, we will assign a color to each
# point in the mesh [x_min, m_max]x[y_min, y_max].
x_min, x_max = X_small[:, 0].min() - 1, X_small[:, 0].max() + 1
y_min, y_max = X_small[:, 1].min() - 1, X_small[:, 1].max() + 1
xx, yy = np.meshgrid(np.arange(x_min, x_max, h),
                     np.arange(y_min, y_max, h))

Z = clf.predict(np.c_[xx.ravel(), yy.ravel()]) # Make a prediction oat every point
                                               # in the mesh in order to find the
                                               # classification areas for each label

# Put the result into a color plot
Z = Z.reshape(xx.shape)
plt.figure(figsize=(8, 6))
plt.pcolormesh(xx, yy, Z, cmap=cmap_light)

# Plot the training points
plt.scatter(X_small[:, 0], X_small[:, 1], c=y, cmap=cmap_bold)
plt.xlim(xx.min(), xx.max())
plt.ylim(yy.min(), yy.max())
plt.title("3-Class classification (SVM)")
plt.xlabel('Sepal length (cm)')
plt.ylabel('Sepal width (cm)')

# Display the new examples as labeled text on the graph
plt.text(examples[0][0], examples[0][1],'A', fontsize=14)
plt.text(examples[1][0], examples[1][1],'B', fontsize=14)

# Plot the legend
plt.legend(labelList, labelNames)

plt.show()


I am working on the file "SVM_Tutorial.ipynb".
The first task is:
Select a Model
To select a model, we look at the average accuracy across all folds.
Can you write Python code for it?

for kernel in kernels:
    print('%s: %.2f' % (kernel, np.mean(accuracies[kernel])))


I am working on the file "SVM_Tutorial.ipynb".
The first task is:
Sidenote: Randomness and Results
Every time you run this notebook, you will get slightly different results. Why? Because data is randomly divided among the training/testing/validation data sets. Running the code again will create a different division of the data, and will make the results slightly different. However, the overall outcome should remain consistent and have approximately the same values. If you have drastically different results when running an analysis multiple times, it suggests a problem with your model or that you need more data.
If it's important that you get the exact same results every time you run the code, you can specify a random state in the random_state argument of train_test_split() and KFold.
Can you write Python code for it?

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)


I am working on the file "SVM_Tutorial.ipynb".
The first task is:
Support vector machines: training
Next, we train a SVM classifier on our data.
The first line creates our classifier using the SVC() function. For now we can ignore the parameter kernel='linear', this just means the decision boundaries should be straight lines. The second line uses the fit() method to train the classifier on the features in X_small, using the labels in y.
It is safe to ignore the parameter 'decision_function_shape'. This is not important for this tutorial, but its inclusion prevents warnings from Scikit-learn about future changes to the default.
Can you write Python code for it?

# Create an instance of SVM and fit the data.
clf = SVC(kernel='linear', decision_function_shape='ovo')
clf.fit(X_small, y)


I am working on the file "SVM_Tutorial.ipynb".
The first task is:
This plot is much more visually cluttered than our previous toy example. There are a few points worth noticing if you take a closer look.
First, notice how the three solid lines run right along one of the decision boundaries. These are used to determine the boundaries between the classification areas (where the colors change).
Additionally, while the parallel dotted lines still pas through one or more support vectors, there are now data points located between the decision boundary and the dotted line (and even on the wrong side of the decision boundary!). This happens when our data is not "perfectly separable". A perfectly separable dataset is one where the classes can be separated completely with a single, straight (or at least simple) line. While it makes for nice examples, real world machine learning uses are almost never perfectly separable.
Kernels: Changing The Decision Boundary Lines
In our previous example, all of the decision boundaries are straight lines. But what if our data is grouped into more circular clusters, maybe a curved line would separate the data better.
SVMs use something called kernels to determine the shape of the decision boundary. Remember that when we called the SVC() function we gave it a parameter kernel='linear', which made the boundaries straight. A different kernel, the radial basis function (RBF) groups data into circular clusters instead of dividing by straight lines.
Below we show the same example as before, but with an RBF kernel.
Can you write Python code for it?

# Create an instance of SVM and fit the data.
clf = SVC(kernel='rbf', decision_function_shape='ovo') # Use the RBF kernel this time
clf.fit(X_small, y)

# Now plot the results
h = .02  # step size in the mesh

# Plot the decision boundary. For that, we will assign a color to each
# point in the mesh [x_min, m_max]x[y_min, y_max].
x_min, x_max = X_small[:, 0].min() - 1, X_small[:, 0].max() + 1
y_min, y_max = X_small[:, 1].min() - 1, X_small[:, 1].max() + 1
xx, yy = np.meshgrid(np.arange(x_min, x_max, h),
                     np.arange(y_min, y_max, h))

Z = clf.predict(np.c_[xx.ravel(), yy.ravel()]) # Make a prediction oat every point
                                               # in the mesh in order to find the
                                               # classification areas for each label

# Put the result into a color plot
Z = Z.reshape(xx.shape)
plt.figure(figsize=(8, 6))
plt.pcolormesh(xx, yy, Z, cmap=cmap_light)

# Plot the training points
plt.scatter(X_small[:, 0], X_small[:, 1], c=y, cmap=cmap_bold)
plt.xlim(xx.min(), xx.max())
plt.ylim(yy.min(), yy.max())
plt.title("3-Class classification (SVM)")
plt.xlabel('Sepal length (cm)')
plt.ylabel('Sepal width (cm)')

# Display the new examples as labeled text on the graph
plt.text(examples[0][0], examples[0][1],'A', fontsize=14)
plt.text(examples[1][0], examples[1][1],'B', fontsize=14)

# Plot the legend
plt.legend(labelList, labelNames)

plt.show()


I am working on the file "SVM_Tutorial.ipynb".
The first task is:
What about my other features?
We've been looking at two features: the length and width of the plant's sepal. But what about the other two featurs, petal length and width? What does the graph look like when train on the petal length and width? How does it change when you change the SVM kernel?
How would you plot our two new plants, A and B, on these new plots? Assume we have all four measurements for each plant, as shown below.
Plant | Sepal length | Sepal width| Petal length | Petal width
------|--------------|------------|--------------|------------
A     |4.3           |2.5         | 1.5          | 0.5
B     |6.3           |2.1         | 4.8          | 1.5
Can you write Python code for it?


I am working on the file "SVM_Tutorial.ipynb".
The first task is:
What are the support vectors in this example?
Below, we define a function to plot the solid decision boundary and corresponding dashed lines, as shown in the introductory picture. Because there are three classes to separate, there will now be three sets of lines.
Can you write Python code for it?

def plot_svc_decision_function(clf):
    """Plot the decision function for a 2D SVC"""
    x = np.linspace(plt.xlim()[0], plt.xlim()[1], 30)
    y = np.linspace(plt.ylim()[0], plt.ylim()[1], 30)
    Y, X = np.meshgrid(y, x)
    P = np.zeros((3,X.shape[0],X.shape[1]))
    for i, xi in enumerate(x):
        for j, yj in enumerate(y):
            P[:, i,j] = clf.decision_function([[xi, yj]])[0]
    for ind in range(3):
        plt.contour(X, Y, P[ind,:,:], colors='k',
                levels=[-1, 0, 1],
                linestyles=['--', '-', '--'])


I am working on the file "SVM_theano.ipynb".
The first task is:
.build_update might take a while for FAST_COMPILE (that flag command that's typed in before the notebook starts for theano)
Can you write Python code for it?

SVM_iris.build_update();

SVM_iris.train_model_full();

SVM_iris.build_b();

SVM_iris.make_predict(X_2d_valid[0])

SVM_iris.make_predictions(X_2d_valid)

X_2d_test

y_2d_test

y_test_pred= SVM_iris.make_predictions(X_2d_test)

np.array( [np.array(yhat) for yhat in y_test_pred] )

y_valid_pred = [ SVM_iris.make_predict(X_2d_valid_ele) for X_2d_valid_ele in X_2d_valid ]

y_valid_pred = [y_valid_pred_ele[0] for y_valid_pred_ele in y_valid_pred]

y_valid_pred = np.array( y_valid_pred).flatten()

#y_valid_pred[ y_valid_pred>0 ] = 1
#y_valid_pred[ y_valid_pred<0 ] = -1
y_valid_pred = np.sign( y_valid_pred)

(y_2d_valid == y_valid_pred).astype(theano.config.floatX).sum()/len(y_valid_pred)

y_valid_pred

y_2d_valid

SVM_iris_X

SVM_iris = SVM(X_2d_train,y_2d_train,len(y_2d_train),0.1,1.0,0.001)

SVM_iris.build_W();
SVM_iris.build_update();
SVM_iris.train_model_full();
SVM_iris.build_b();

y_valid_pred = np.array( [ SVM_iris.make_predict(X_2d_valid_ele)[0] for X_2d_valid_ele in X_2d_valid ] ).flatten()

y_valid_pred[ y_valid_pred>0 ] = 1
y_valid_pred[ y_valid_pred<0 ] = -1

(y_2d_valid == y_valid_pred).astype(theano.config.floatX).sum()/len(y_valid_pred)

SVM_iris = SVM(X_2d_train,y_2d_train,len(y_2d_train),0.1,0.1,0.001)

SVM_iris.build_W();
SVM_iris.build_update();
SVM_iris.train_model_full();
SVM_iris.build_b();

y_valid_pred = np.array( [ SVM_iris.make_predict(X_2d_valid_ele)[0] for X_2d_valid_ele in X_2d_valid ] ).flatten()

y_valid_pred[ y_valid_pred>0 ] = 1
y_valid_pred[ y_valid_pred<0 ] = -1

(y_2d_valid == y_valid_pred).astype(theano.config.floatX).sum()/len(y_valid_pred)

SVM_iris = SVM(X_2d_train,y_2d_train,len(y_2d_train),0.01,0.1,0.001)

SVM_iris.build_W();
SVM_iris.build_update();
SVM_iris.train_model_full();
SVM_iris.build_b();

y_valid_pred = np.array( [ SVM_iris.make_predict(X_2d_valid_ele)[0] for X_2d_valid_ele in X_2d_valid ] ).flatten()

y_valid_pred[ y_valid_pred>0 ] = 1
y_valid_pred[ y_valid_pred<0 ] = -1

(y_2d_valid == y_valid_pred).astype(theano.config.floatX).sum()/len(y_valid_pred)

m_val =...


I am working on the file "SVM_theano.ipynb".
The first task is:
Clean the data where I choose to fill in missing values, NaN values, with the mean, due to the distribution of the data
Can you write Python code for it?

train_3_X_pd = pd.DataFrame(train_3_X)
train_3_X_pd_cleaned = train_3_X_pd.where( pd.notnull( train_3_X_pd ), train_3_X_pd.mean(), axis='columns')

train_3_X_pd.describe()

train_3_X_pd_cleaned.describe()

train_3_X_scaled = scaler.fit_transform( train_3_X_pd_cleaned.as_matrix() )

train_3_y

SVM_3 = SVM_parallel(train_3_X_scaled,train_3_y,len(train_3_y),128.0,2.0,0.001)  # sigma=2.0

SVM_3.build_W();
SVM_3.build_update();

%time SVM_3.train_model_full(20)

SVM_3.build_b()

print(test_3_y.shape)
test_3_y

print(test_3_X.shape)
test_3_X_scaled = scaler.transform( test_3_X)

pd.DataFrame( train_3_X_scaled).describe()

pd.DataFrame( test_3_X_scaled).describe()

%time yhat_test3 = SVM_3.make_predictions_parallel( test_3_X_scaled)

yhat_test3 = np.sign( yhat_test3[0]);

(yhat_test3 == test_3_y).sum()/float(len(test_3_y))


I am working on the file "SVM_theano.ipynb".
The first task is:
Dataset examples
from sci-kit learn, sklearn
Can you write Python code for it?


I am working on the file "SVM_theano.ipynb".
The first task is:
Load and prepare data set
dataset for grid search
Can you write Python code for it?

iris = load_iris()
X = iris.data
y = iris.target

# Dataset for decision function visualization: we only keep the first two
# features in X and sub-sample the dataset to keep only 2 classes and
# make it a binary classification problem

X_2d = X[:,:2]
X_2d=X_2d[y>0]
y_2d=y[y>0]
y_2d -= 1

# It is usually a good idea to scale the data for SVM training.
# We are cheating a bit in this example in scaling all of the data,
# instead of fitting the transformation on the training set and
# just applying it on the test set.

scaler = StandardScaler()
X= scaler.fit_transform(X)
X_2d=scaler.fit_transform(X_2d)

print(type(X)); print(X.shape); print(type(X_2d));print(X_2d.shape);print(type(y));print(y.shape);
print(type(y_2d));print(y_2d.shape)

ratio_of_train_to_total = 0.6
numberofexamples = len(y_2d)
numberoftrainingexamples = int(numberofexamples*ratio_of_train_to_total)
numbertovalidate = (numberofexamples - numberoftrainingexamples)/2
numbertotest= numberofexamples - numberoftrainingexamples - numbertovalidate
print(numberofexamples);print(numbertotest);print(numberoftrainingexamples);print(numbertovalidate)

shuffledindices = np.random.permutation( numberofexamples)

X_2d_train = X_2d[:numberoftrainingexamples]
y_2d_train = y_2d[:numberoftrainingexamples]
X_2d_valid = X_2d[numberoftrainingexamples:numberoftrainingexamples + numbertovalidate]
y_2d_valid = y_2d[numberoftrainingexamples:numberoftrainingexamples + numbertovalidate]
X_2d_test = X_2d[numberoftrainingexamples + numbertovalidate:]
y_2d_test = y_2d[numberoftrainingexamples + numbertovalidate:]


I am working on the file "SVM_theano.ipynb".
The first task is:
Scale the train.3 Vehicle data
Can you write Python code for it?

scaler = StandardScaler()
train_3_X_scaled = scaler.fit_transform(train_3_X)

train_3_X


I am working on the file "SVM_theano.ipynb".
The first task is:
Test values
Can you write Python code for it?

m=4
d=2
X_val=np.arange(2,m*d+2).reshape(m,d).astype(theano.config.floatX)
X=theano.shared( X_val)
y_val=np.random.randint(2,size=m).astype(theano.config.floatX)
y=theano.shared( y_val )
lambda_mult_val = np.random.rand(m).astype(theano.config.floatX)
lambda_mult = theano.shared( lambda_mult_val ) # lambda Lagrange multipliers
sigma_val = 2.0
sigma = theano.shared( np.float32(sigma_val))

np.random.randint(2,size=4)

X[1]

np.random.rand(4)

#lambda_mult = theano.shared( np.zeros(m_val).astype(theano.config.floatX) ) # lambda Lagrange multipliers

prodi = lambda_mult[1]*y[1]

sigma=0.5
def step(Xj,Xi):
    rbf = T.exp(-(Xj-Xi)**2/(np.float32(2.*sigma**2)))
    return sandbox.cuda.basic_ops.gpu_from_host(rbf)

output,update=theano.scan(fn=step, sequences=[X,],non_sequences=[X[1],])

test_rbf = theano.function(inputs=[],outputs=output,updates=update )

print(test_rbf().shape)
test_rbf()

#Check
prodi_val = lambda_mult_val[1]*y_val[1]

for j in range(4):
    print( np.exp(-((X_val[j]-X_val[1])**2).sum(0)/(np.float32(2.*sigma**2))) )

X_val

X_val[3]

prodi = lambda_mult[0]*y[0]

sigma=0.5
def step(Xj,yj,lambda_multj,Xi):
    rbf = lambda_multj*yj*T.exp(-((Xj-Xi)**2).sum()/(np.float32(2.*sigma**2)))
    return sandbox.cuda.basic_ops.gpu_from_host(rbf)

output,update=theano.scan(fn=step, sequences=[X,y,lambda_mult],non_sequences=[X[0],])

test_rbf = theano.function(inputs=[],outputs=output,updates=update )

print(test_rbf().shape)
test_rbf()

sigma=0.5

def rbf(Xj,Xi,sigma):
    rbf = T.exp(-((Xj-Xi)**2).sum()/(np.float32(2.*sigma**2)))
    return rbf

def step(Xj,yj,lambda_multj,Xi,yi,lambda_multi):
#    W_i = lambda_multi*yi*lambda_multj*yj*T.exp(-((Xj-Xi)**2).sum()/(np.float32(2.*sigma**2)))
    W_i = lambda_multi*yi*lambda_multj*yj*rbf(Xj,Xi,sigma)
    return W_i

output,update=theano.scan(fn=step, sequences=[X,y,lambda_mult],non_sequences=[X[0],y[0],lambda_mult[0]])

test_rbf = theano.function(inputs=[],outputs=output,updates=update )

test_rbf()

output1,update1=theano.scan(fn=step,...


I am working on the file "SVM_theano.ipynb".
The first task is:
Using SVM
Can you write Python code for it?

SVM_iris = SVM(X_2d_train,y_2d_train,len(y_2d_train),1.0,1,0.001)

SVM_iris.build_W();


I am working on the file "SVM_theano.ipynb".
The first task is:
cf. A Practical Guide to Support Vector Classication, Chih-Wei Hsu, Chih-Chung Chang, and Chih-Jen Lin
Can you write Python code for it?

with open("./Data/train.1",'rb') as f:
    train_1_lst = f.readlines()
f.close()
# strip of '\n'
train_1_lst = [x.strip() for x in train_1_lst]
print(len(train_1_lst))

train_1_lst=[line.replace('1:','').replace('2:','').replace('3:','').replace('4:','') for line in train_1_lst]

train_1_lst=[line.split() for line in train_1_lst]
train_1_arr=np.array( [[float(ele) for ele in line] for line in train_1_lst] )

train_1_y=train_1_arr[:,0]
train_1_X=train_1_arr[:,1:]

print(train_1_y.shape)
print(train_1_X.shape)

with open("./Data/test.1",'rb') as f:
    test_1_lst = f.readlines()
f.close()
# strip of '\n'
test_1_lst = [x.strip() for x in test_1_lst]
print(len(test_1_lst))

test_1_lst=[line.replace('1:','').replace('2:','').replace('3:','').replace('4:','') for line in test_1_lst]

test_1_lst=[line.split() for line in test_1_lst]
test_1_arr=np.array( [[float(ele) for ele in line] for line in test_1_lst] )

test_1_y=test_1_arr[:,0]
test_1_X=test_1_arr[:,1:]

with open("./Data/train.3",'rb') as f:
    train_3_lst = f.readlines()
f.close()
# strip of '\n'
train_3_lst = [x.strip() for x in train_3_lst]
print(len(train_3_lst))

train_3_lst=[line.replace('1:','').replace('2:','').replace('3:','').replace('4:','').replace('5:','').replace('6:','').replace('7:','').replace('8:','').replace('9:','').replace('10:','').replace('11:','').replace('12:','').replace('13:','').replace('14:','').replace('15:','').replace('16:','').replace('17:','').replace('18:','').replace('19:','').replace('20:','').replace('21:','').replace('22:','') for line in train_3_lst]
train_3_lst=[line.split() for line in train_3_lst]
train_3_DF=pd.DataFrame( train_3_lst)

train_3_y = train_3_DF[0].as_matrix().astype(theano.config.floatX)
train_3_X = train_3_DF.ix[:,1:].as_matrix().astype(theano.config.floatX)
print(train_3_X.shape)

ratiotraintotot = 0.2
numberofexamples1 = len(train_1_y)
numberoftrain1 = int( numberofexamples1 * ratiotraintotot )
numberofvalid1 = numberofexamples1 - numberoftrain1

shuffled_idx =...


I am working on the file "SVM_theano.ipynb".
The first task is:
from Coursera's Machine Learning Introduction by Andrew Ng, Ex. 6, i.e. Programming Exercise 6
Can you write Python code for it?

where_ex6_is_str = './coursera_Ng/machine-learning-ex6/ex6/'
ex6data1_mat_data = scipy.io.loadmat( where_ex6_is_str + "ex6data1.mat")


I am working on the file "SWAN_pytimber_for_injectors.ipynb".
The first task is:
<hr style="border-top-width: 4px; border-top-color: #34609b;">

Example  1
Let us assume that you start to download some raw data from CALS. Here I decided to download some  variables recorded by acqStamp and some others by cycleStamp from the S. The fundamental filtering makes sense only on the variable recorded by cycleStamp.
Can you write Python code for it?

# Heavily using pyTimber we get the variable
varSet1=log.search('%TPS15%') #recorded by acqStamp since is not PPM
varSet2=log.search('CPS.TGM%')#recorded by cyclestamp
print(varSet1) # just to show what is inside
print(varSet2) # just to show what is inside

extractFromTime=myToolbox.time_1_hour_ago(hours=2)
extractToTime=myToolbox.time_1_hour_ago(hours=1)
# Heavily using PANDAS
# we cannot use the fundamental (recorded by acqStamp)!
myDataFrame1=myToolbox.cals2pnd(varSet1,extractFromTime,extractToTime)
# we can use the fundamental
myDataFrame2=myToolbox.cals2pnd(varSet2,extractFromTime,extractToTime,fundamental='%TOF')

# Now I can merge and create a postprocessing DF

rawDF=myToolbox.mergeDF(myDataFrame1,myDataFrame2)

#eventually I can define a postprocessing
def postprocess(df):
    aux=pnd.DataFrame()
    aux['PE.TPS15.359.CONTROLLER:ANGLE filled']= df['PE.TPS15.359.CONTROLLER:ANGLE'].ffill()
    aux['PE.TPS15.359.CONTROLLER:ANGLE filled, doubled']= aux['PE.TPS15.359.CONTROLLER:ANGLE filled']*2+1
    return aux;

postDF=postprocess(rawDF)

#I suggest not to merge the rawDF with the postDF, this will allow to extend the raw.

# starting from the original DF is trivial now to extend them.
# It is important to note that the we have somehow to remember that the second DF needs a fundamental filter
myDataFrame1=myToolbox.cals2pnd(myDataFrame1.columns,rawDF.index[-1],
                                rawDF.index[-1]+datetime.timedelta(hours=1))
myDataFrame2=myToolbox.cals2pnd(myDataFrame2.columns,rawDF.index[-1],
                                rawDF.index[-1]+datetime.timedelta(hours=1),
                                fundamental='%TOF')

# and now we can iterate with the merging
aux=myToolbox.mergeDF(myDataFrame1,myDataFrame2)

# and with the concatenation
rawDF=myToolbox.concatDF(rawDF,aux)
postDF=myToolbox.concatDF(postDF,postprocess(aux))

# we suggest to maintain well separated the raw data, postprocessing functions and postprocessed data.

# print the dataFrame head
rawDF.head()

# describe the...


I am working on the file "SWAN_pytimber_for_injectors.ipynb".
The first task is:
Summary
The indexesFromCALS method gives a great flexibility to reshuffle CALS data. For teh moment this method is not very fast: it is cycling on the rows of the DF, in future perhaps could be improved by using a primitive from CALS.
<hr style="border-top-width: 4px; border-top-color: #34609b;">

Example 3
Let us consider now another user case. We would like to extract the list of the cyclestamp and some basic variable from MTE cycles along the injector chains. This is an example of cycleStamp arithmetic. We have to remember that the offeset between teh C0 time of the PSB, PS and SPS is 635 ms.
Can you write Python code for it?

CPSDF=myToolbox.cals2pnd(['CPS.TGM:USER','CPS.TGM:BATCH','PR.DCAFTINJ_1:INTENSITY'],myToolbox.time_1_hour_ago(hours=1./6.),myToolbox.time_now(),fundamental='%MTE%')

firstBatch=CPSDF['CPS.TGM:BATCH']==1
SPSDF=myToolbox.indexesFromCALS(CPSDF[firstBatch].index+datetime.timedelta(seconds=0.635), ['SPS.TGM:USER'])
PSBDF=myToolbox.indexesFromCALS(CPSDF.index-datetime.timedelta(seconds=0.635), ['PSB.TGM:USER']+log.search('%_BCT_ACC_INTENSITY'))

print('==================================')
print('PSB')
print(PSBDF['PSB.TGM:USER'].head(4))
print('==================================')
print('CPS')
print(CPSDF['CPS.TGM:USER'].head(4))
print('==================================')
print('SPS')
print(SPSDF['SPS.TGM:USER'].head(2))

postPSBDF=pnd.DataFrame()
# note the use of the filtering with regular expression and the sum done on the rows (axis=1)
postPSBDF['Total Intensity']=PSBDF.filter(regex='BR*').sum(axis=1)
postPSBDF.head()

postCPSDF=pnd.DataFrame()
# we are now building a series using data from different DF and adopting the indexing (cycleStamp) from PS.
postCPSDF['transmission']=pnd.Series(CPSDF['PR.DCAFTINJ_1:INTENSITY'].values/postPSBDF['Total Intensity'].values,index=CPSDF.index)
plt.plot(postCPSDF['transmission'])
myToolbox.setXlabel(ax=plt.gca(), hours=1/30.)


I am working on the file "SWAN_pytimber_for_injectors.ipynb".
The first task is:
Summary
The pandas dateframe (DF) are a very flexible data type for postprocessing CALS data. They comes with a rich asset of methods and a very active community.
Think the DF as LEGO: play with them, merge and contatenate them. Keep well separated the raw DF (extracted from CALS), the postprocessing methods and the postrprocessed DF.
NB: numpy is of course more performant of pandas DF, but I tend to buy flexibility with performace.
<hr style="border-top-width: 4px; border-top-color: #34609b;">

Example 2
Let us consider now another user case. We would like to extract the list of the cycles following TOF in the PS in the last 10 min. We will start extracting the TOF cycle and from that DF create a new dataframe. This is an example of cycleStamp arithmetic.
Can you write Python code for it?

CPSDF=myToolbox.cals2pnd(['CPS.TGM:USER'],
                         myToolbox.time_1_hour_ago(hours=1./6.),
                         myToolbox.time_now(),fundamental='%TOF')
CPSDF.head()

# the important method to use is indexesFromCALS
cycleAfterDF=myToolbox.indexesFromCALS(CPSDF.index+datetime.timedelta(seconds=1.2),CPSDF.columns)
cycleAfterDF.head()


I am working on the file "SWE_square_well/swe_square_well.ipynb".
The first task is:
Next we generate the lattice for the simulation. For now we consider 2000 lattice points.
Can you write Python code for it?

delta = 1 # The spacing between neighboring lattice points
L = 1000 # The ends of the lattice
length = 10
momentum = 1
lattice = arange(-L,L,delta)
v = vectorize(potential)
plot(v(lattice))

hamiltonian = np.zeros((lattice.shape[0],lattice.shape[0]))
for row in range(lattice.shape[0]):
    for col in range(lattice.shape[0]):
        hamiltonian[row,col] = (kronecker_delta(col+1, row)\
                                + kronecker_delta(col-1, row)\
                                - 2*kronecker_delta(col,row))/-(delta)**2\
                                + potential(row)*kronecker_delta(col,row)
hamiltonian


I am working on the file "SWE_square_well/swe_square_well.ipynb".
The first task is:
Now that we know the wavefunction as a linear combination of basis vectors, the time evolution of the complete wavefuction is just the linear combination of the time evolutions of the basis states. Lets do this for 10 time steps.
Can you write Python code for it?

func = zeros_like(eigenvalues,dtype=complex128)
for index in range(eigenvalues.shape[0]):
        func += coeff[index]*eigenvectors[index]
plot(lattice,absolute(func))

time_steps = arange(0,200,2)
evolution = []
for time in time_steps:
    wavefunc = zeros_like(eigenvalues,dtype=complex128)
    for index in range(eigenvalues.shape[0]):
        wavefunc += coeff[index]*eigenvectors[index]*exp(-1j*eigenvalues[index]*time)
    evolution.append(wavefunc)

fig = plt.figure()
ax = plt.axes(xlim=(-L, L), ylim=(0, 0.2))
line, = ax.plot([], [], lw=2)
def init():
    line.set_data([], [])
    return line,

def animate(i):
    line.set_data(lattice,absolute(evolution[i]))
    return line,

from matplotlib import animation
anim = animation.FuncAnimation(fig,animate,init_func=init,frames=100,interval=50,blit=True)
HTML(anim.to_html5_video())


I am working on the file "SWE_square_well/swe_square_well.ipynb".
The first task is:
Now that we've computed the hamiltonian matrix, let's diagonalize it
Can you write Python code for it?

eigenvalues, eigenvectors = linalg.eigh(hamiltonian)
plot(lattice,absolute(eigenvectors[0]))

# eigenvectors[0].shape
wavefunction.shape


I am working on the file "SWE_wavepacket/swe_wavepacket.ipynb".
The first task is:
Next we generate the lattice for the simulation. For now we consider 200 lattice points.
Can you write Python code for it?

# defining our own kronecker delta and hamiltonian as vectorized functions

psi = vectorize(lambda x,p: (1/pow(np.pi*(length**2),0.25))*np.exp(-((x-peak)**2)/(2.0*length**2) - 1j*p*x))
kronecker = vectorize(lambda i,j: 1 if i == j else 0)
h= vectorize(lambda i,j: (-kronecker(i+1,j) + 2*kronecker(i,j) - kronecker(i-1,j))/delta**2)

delta = .1 # The spacing between neighboring lattice points
L = 20. # The ends of the lattice
length = 1. # spread of the Gaussian wavefunction
peak = 0. # Centre of gaussian
N = int(L/delta) # half the points of the lattice
momentum = 2. # initial momentum of the wavepacket
rows,cols = reshape(arange(-N,N+1),(2*N+1,1)),reshape(arange(-N,N+1),(1,2*N+1))
lattice = linspace(-L,L,2*N+1) #The lattice points

hamiltonian = h(rows,cols)


I am working on the file "SYDE 556 Lecture 1 Introduction.ipynb".
The first task is:

Multielectrode recordings

Put 'tetrodes' or multi-electrode arrays into the brain
Post-processing:
"Spike sorting"
Local field potentials (LFPs)

High temporal resolution, max ~100 cells

Can you write Python code for it?

from IPython.display import YouTubeVideo
YouTubeVideo('lfNVv0A8QvI', width=640, height=390)


I am working on the file "SYDE 556 Lecture 1 Introduction.ipynb".
The first task is:

Multielectrode recordings

- Put 'tetrodes' or multi-electrode arrays into the brain
- Post-processing:
    - "Spike sorting"
    - Local field potentials (LFPs)
- High temporal resolution, max ~100 cells

Can you write Python code for it?


I am working on the file "SYDE 556 Lecture 1 Introduction.ipynb".
The first task is:
Billions of neurons

Simplify the neuron model and you can run more of them
Can you write Python code for it?

from IPython.display import YouTubeVideo
YouTubeVideo('WmChhExovzY', width=600, height=400)


I am working on the file "SYDE 556 Lecture 1 Introduction.ipynb".
The first task is:
Brain structures

Lots of visually obvious structure
Lots of greek and latin names to remember
locus coeruleus, thalamus, amygdala, hypothalamus, substantia nigra, etc etc

Can you write Python code for it?

from IPython.display import HTML
HTML('
A Neuron

')


I am working on the file "SYDE 556 Lecture 1 Introduction.ipynb".
The first task is:
Calcium Imaging

Use calcium to glow when Ca2+ ions bond
Happens a lot during neural activity and spike generation

Good spatial and good temporal resolution

E.g. In a fish embryo
Can you write Python code for it?

from IPython.display import YouTubeVideo
YouTubeVideo('DGBy-BGiZIM', width=640, height=360)


I am working on the file "SYDE 556 Lecture 1 Introduction.ipynb".
The first task is:
In a stalking fish
Can you write Python code for it?

from IPython.display import YouTubeVideo
YouTubeVideo('CpejbZ-XEyM', width=640, height=360)


I am working on the file "SYDE 556 Lecture 1 Introduction.ipynb".
The first task is:
Neurons in the brain

100 billion
100's or 1000's of distinct types (distinguished via anatomy and/or physiology)
Axon length: from $10^{-4}$ to $5$ m
Each neuron: 500-200,000 inputs and outputs
72km of axons

Communication: 100's of different neurotransmitters

Neuron communication: Synapses

What it really looks like

What it really really looks like
Can you write Python code for it?

from IPython.display import YouTubeVideo
YouTubeVideo('F37kuXObIBU', width=720, height=500, start=8*60+35)


I am working on the file "SYDE 556 Lecture 1 Introduction.ipynb".
The first task is:
Optogenetics

Allows stimulation and recording from select parts of the brain
Just those parts expressing a light sensitive proteins that are stimulated

High spatial and temporal resolution (but local)
Can you write Python code for it?

from IPython.display import YouTubeVideo
YouTubeVideo('v7uRFVR9BPU', width=640, height=390)


I am working on the file "SYDE 556 Lecture 1 Introduction.ipynb".
The first task is:
Overall Goal

- Building brains!
- Why?
    - To figure out how brains work (health applications)
    - To apply this knowledge to building systems (AI applications)
Administration

Course website: http://compneuro.uwaterloo.ca/courses/syde-750.html
Contact information
Chris Eliasmith: celiasmith@uwaterloo.ca
Course times: Mon & Wed. 9:00a-10:20a (10:30a-11:20p Wed for 750)
Location: Mon: E5-6004, Wed: E7-5343
Office hours: by appointment

Coursework

Four assignments (60%)

20%, 20%, 10%, 10%
About two weeks for each assignment
Everyone writes their own code, generates their own graphs, writes their own answers

Final project (40%)

Make a model of some neural system
For 556 students, this can be an extension of something seen in class
For 750 students, this must be more of a research project with more novelty
ideas
Get your idea approved via email before Reading Week (Feb 18)

Schedule

Week	Reading	Topic	Assignments

Jan 7	Chpt 1	Introduction

Jan 9, 14	Chpt 2,4	Neurons, Population Representation	#1 posted

Jan 16, 21	Chpt 4	Temporal Representation

Jan 23, 28, 30	Chpt 5,6	Feedforward Transformations	#1 due (23rd at midnight); #2 posted

Feb 4, 6, 11	Chpt...
Can you write Python code for it?

from IPython.display import YouTubeVideo
YouTubeVideo('jHxyP-nUhUY', width=500, height=400, autoplay=0, start=60)


I am working on the file "SYDE 556 Lecture 1 Introduction.ipynb".
The first task is:
Spaun: addition by counting
Can you write Python code for it?

from IPython.display import YouTubeVideo
YouTubeVideo('mP7DX6x9PX8', width=640, height=390)


I am working on the file "SYDE 556 Lecture 1 Introduction.ipynb".
The first task is:
Spaun: digit recognition
Can you write Python code for it?

from IPython.display import YouTubeVideo
YouTubeVideo('f6Ul5TYK5-o', width=640, height=360)


I am working on the file "SYDE 556 Lecture 1 Introduction.ipynb".
The first task is:
Spaun: pattern completion
Can you write Python code for it?

from IPython.display import YouTubeVideo
YouTubeVideo('Q_LRvnwnYp8', width=640, height=390)


I am working on the file "SYDE 556 Lecture 1 Introduction.ipynb".
The first task is:
Vision: >1000 categories
Can you write Python code for it?

from IPython.display import YouTubeVideo
YouTubeVideo('VWUhCzUDZ70', width=640, height=390)


I am working on the file "SYDE 556 Lecture 1 Introduction.ipynb".
The first task is:
What do we know so far?

Lots of details
Data: "The proportion of type A neurons in area X is Y"
Conclusion: "Therefore, the proportion of type A neurons in area X is Y".

Hard to get a big picture

No good methods for generalizing from data

"Data-rich and theory-poor" (Churchland & Sejnowski, 1994; still true)

Need some way to connect these details
Need unifying theory

Recall: Neural Modeling

What I cannot create, I do not understand
Build a computer simulation
Do to neuroscience what Newton did to physics
Too complex to be analytically tractable, so use computer simulation

Can we use this to connect the levels?

Single neuron simulation

Hodgkin & Huxley, 1952

Single neuron simulation

Hodgkin & Huxley, 1952

Single neuron simulation

Millions of neurons
Can you write Python code for it?

from IPython.display import YouTubeVideo
YouTubeVideo('_UFOSHZ22q4', width=600, height=400, start=60)


I am working on the file "SYDE 556 Lecture 2 Representation.ipynb".
The first task is:
But that's not how people normally plot it
It might not make sense to sample every possible x
Instead they might do some subset
For example, what if we just plot the points around the unit circle?
Can you write Python code for it?

import nengo
import numpy

n = nengo.neurons.LIFRate()

theta = numpy.linspace(0, 2*numpy.pi, 100)
x = numpy.array([numpy.cos(theta), numpy.sin(theta)])
plot(x[0],x[1])
axis('equal')

e = numpy.array([1.0, 1.0])
e = e/numpy.linalg.norm(e)

plot([0,e[0]], [0,e[1]],'r')

gain = 1
bias = 2.5

figure()
plot(theta, n.rates(numpy.dot(x.T, e), gain=gain, bias=bias))
plot([numpy.arctan2(e[1],e[0])],0,'rv')
xlabel('angle')
ylabel('firing rate')
xlim(0, 2*numpy.pi);


I am working on the file "SYDE 556 Lecture 2 Representation.ipynb".
The first task is:
For mapping #1, the NEF uses a linear map:
$ J = \alpha x + J^{bias} $

But what about type (c) in this graph?

Easy enough:

$ J = - \alpha x + J^{bias} $

But what about type(b)?  Or these ones?

There's usually some $x$ which gives a maximum firing rate
...and thus a maximum $J$

Firing rate (and $J$) decrease as you get farther from the preferred $x$ value
So something like $J = \alpha [sim(x, x_{pref})] + J^{bias}$

What sort of similarity measure?
Let's think about $x$ for a moment
$x$ can be anything... scalar, vector, etc.
Does thinking of it as a vector help?

The Encoding Equation (i.e. Tuning Curves)

Here is the general form we use for everything (it has both 'mappings' in it)
$a_i = G_i[\alpha_i x \cdot e_i + J_i^{bias}] $
$\alpha$ is a gain term (constrained to always be positive)
$J^{bias}$ is a constant bias term
$e$ is the encoder, or the preferred direction vector
$G$ is the neuron model
$i$ indexes the neuron

To simplify life, we always assume $e$ is of unit length
Otherwise we could combine $\alpha$ and $e$

In the 1D case, $e$ is either +1 or -1
In higher dimensions, what happens?
Can you write Python code for it?

#assume this has been run
#%pylab inline

import numpy
import nengo

n = nengo.neurons.LIFRate()

e = numpy.array([1.0, 1.0])
e = e/numpy.linalg.norm(e)

a = numpy.linspace(-1,1,50)
b = numpy.linspace(-1,1,50)

X,Y = numpy.meshgrid(a, b)

from mpl_toolkits.mplot3d.axes3d import Axes3D
fig = figure()
ax = fig.add_subplot(1, 1, 1, projection='3d')
p = ax.plot_surface(X, Y, n.rates((X*e[0]+Y*e[1]), gain=1, bias=1.5),
                    linewidth=0, cstride=1, rstride=1, cmap=pylab.cm.jet)


I am working on the file "SYDE 556 Lecture 2 Representation.ipynb".
The first task is:
How good is the representation?
Can you write Python code for it?

#Have to run previous code cell first
noise = 0.2

with model:
    connection = nengo.Connection(neurons, neurons, #This is just to generate the decoders
                                  solver=nengo.solvers.LstsqNoise(noise=0.2)) #Add noise ###NEW

sim = nengo.Simulator(model)

d = sim.data[connection].weights.T
x, A = tuning_curves(neurons, sim)
A_noisy = A + numpy.random.normal(scale=noise*numpy.max(A), size=A.shape)
xhat = numpy.dot(A_noisy, d)

print('RMSE with %d neurons is %g'%(N, np.sqrt(np.average((x-xhat)**2))))

figure()
plot(x, x)
plot(x, xhat)
xlabel('$x$')
ylabel('$\hat{x}$')
ylim(-1, 1)
xlim(-1, 1);


I am working on the file "SYDE 556 Lecture 2 Representation.ipynb".
The first task is:
Leaky integrate-and-fire neuron
$ a = {1 \over {\tau_{ref}-\tau_{RC}ln(1-{1 \over J})}}$
Can you write Python code for it?

#assume this has been run
#%pylab inline

# Leaky integrate and fire
import numpy
import nengo

n = nengo.neurons.LIFRate(tau_rc=0.02, tau_ref=0.002) #n is a Nengo LIF neuron, these are defaults

J = numpy.linspace(-1,10,100)

plot(J, n.rates(J, gain=1, bias=-3))
xlabel('J (current)')
ylabel('$a$ (Hz)');


I am working on the file "SYDE 556 Lecture 2 Representation.ipynb".
The first task is:
Number of neurons

What happens to the error with more neurons?
Note that the error has two parts:

$
\begin{align}
E = {1 \over 2} \int_{-1}^1 \left(x-\sum_i a_i d_i \right)^2 \;dx + \sigma^2 \sum_i d_i^2
\end{align}
$

Error due to static distortion (i.e. the error introduced by the decoders themselves)
This is present regardless of noise

$
\begin{align}
E_{distortion} = {1 \over 2} \int_{-1}^1 \left(x-\sum_i a_i d_i \right)^2 dx
\end{align}
$

Error due to noise

$
\begin{align}
E_{noise} = \sigma^2 \sum_i d_i^2
\end{align}
$

What do these look like as number of neurons $N$ increases?

- Noise error is proportional to $1/N$
- Distortion error is proportional to $1/N^2$
- Remember this error $E$ is defined as
$ E = {1 \over 2} \int_{-1}^1 (x-\hat{x})^2 dx $

So that's actually a squared error term

Also, as number of neurons is greater than 100 or so, the error is dominated by the noise term ($1/N$).

Examples

Methodology for building models with the Neural Engineering Framework (outlined in Chapter 1)
System Description: Describe the system of interest in terms of the neural data, architecture, computations, representations, etc. (e.g. response functions, tuning curves, etc.)
Design Specification: Add additional performance constraints (e.g. bandwidth, noise, SNR, dynamic range, stability, etc.)
Implement the model: Employ the NEF principles given the System Description and Design Specification

Example 1: Horizontal Eye Control (1D)
From http://www.nature.com/nrn/journal/v3/n12/full/nrn986.html

There are also neurons whose response goes the other way.  All of the neurons are directly connected to the muscle controlling the horizontal direction of the eye, and that's the only thing that muscle does, so we're pretty sure this is what's being repreesnted.

System Description

We've only done the first NEF principle, so that's all we'll worry about
What is being...
Can you write Python code for it?

#%pylab inline
import numpy
import nengo
from nengo.utils.ensemble import tuning_curves
from nengo.dists import Uniform

N = 10
tau_rc = 20
tau_ref = .001
lif_model = nengo.LIFRate(tau_rc=tau_rc, tau_ref=tau_ref)

model = nengo.Network(label='Neurons')
with model:
    neurons = nengo.Ensemble(N, dimensions=1,
                             max_rates = Uniform(250,300),
                             neuron_type = lif_model)
sim = nengo.Simulator(model)

x, A = tuning_curves(neurons, sim)

plot(x, A)
xlabel('x')
ylabel('firing rate (Hz)');


I am working on the file "SYDE 556 Lecture 2 Representation.ipynb".
The first task is:
Possible questions
How many neurons do we need for a particular level of accuracy?
What happens with different firing rates?
What happens with different distributions of x-intercepts?

Example 2: Arm Movements (2D)

Georgopoulos et al., 1982.  "On the relations between the direction of two-dimensional arm movements and cell discharge in primate motor cortex."

System Description

What is being represented?
$x$ is the hand position
Note that this is different from what Georgopoulos talks about in this initial paper
Initial paper only looks at those 8 positions, so it only talks about direction of movement (angle but not magnitude)
More recent work in the same area shows the cells do respond to both (Fu et al, 1993; Messier and Kalaska, 2000)

Bell-shaped tuning curves
Encoders: randomly distributed around the unit circle
Firing rates of up to 60Hz

Design Specification

Range of values for $x$:  Anywhere within a unit circle (or perhaps some other radius)
Normal levels of noise: $\sigma$ is 20% of maximum firing rate
the book goes a bit higher, with $\sigma^2=0.1$, meaning that $\sigma = \sqrt{0.1} \approx 0.32$ times the maximum

Implementation
Examine the tuning curves
Can you write Python code for it?

import numpy
import nengo

n = nengo.neurons.LIFRate()

theta = numpy.linspace(-numpy.pi, numpy.pi, 100)
x = numpy.array([numpy.sin(theta), numpy.cos(theta)])

e = numpy.array([1.0, 0])

plot(theta*180/numpy.pi, n.rates(numpy.dot(x.T, e), bias=1, gain=0.2)) #bias 1->1.5
xlabel('angle')
ylabel('firing rate')
xlim(-180, 180)
show()


import numpy
import nengo

n = nengo.neurons.LIFRate()

theta = numpy.linspace(-numpy.pi, numpy.pi, 100)
x = numpy.array([numpy.sin(theta), numpy.cos(theta)])

e = numpy.array([-1.0, 0])

plot(theta*180/numpy.pi, n.rates(numpy.dot(x.T, e), bias=1., gain=0.2)) #bias 1->1.5
xlabel('angle')
ylabel('firing rate')
xlim(-180, 180)
show()


I am working on the file "SYDE 556 Lecture 2 Representation.ipynb".
The first task is:
Rectified Linear Neuron
Can you write Python code for it?

# Rectified linear neuron
%pylab inline
import numpy
import nengo

n = nengo.neurons.RectifiedLinear()

J = numpy.linspace(-1,1,100)

plot(J, n.rates(J, gain=10, bias=-5))
xlabel('J (current)')
ylabel('$a$ (Hz)');


I am working on the file "SYDE 556 Lecture 2 Representation.ipynb".
The first task is:
Response functions

These are called "response functions"
How much neural firing changes with change in current
Similar for many classes of cells (e.g. pyramidal cells - most of cortex)
This is the $G_i$ function in the NEF: it can be pretty much anything

Tuning Curves

Neurons seem to be sensitive to particular values of $x$

How are neurons 'tuned' to a representation?  or...

What's the mapping between $x$ and $a$?

Recall 'place cells', and 'edge detectors'

Sometimes they are fairly straight forward:

But not often:

Is there a general form?

Tuning curves (cont.)

The NEF suggests that there is...
Something generic and simple
That covers all the above cases (and more)

Let's start with the simpler case:

Note that the experimenters are graphing $a$, as a function of $x$

$x$ is much easier to measure than $J$
So, there are two mappings of interest:
$x$->$J$
$J$->$a$ (response function)

Together these give the tuning curve

$x$ is the volume of the sound in this case

Any ideas?
Can you write Python code for it?

#assume this has been run
#%pylab inline
import numpy
import nengo

n = nengo.neurons.LIFRate() #n is a Nengo LIF neuron

x = numpy.linspace(-100,0,100)

plot(x, n.rates(x, gain=1, bias=50), 'b') # x*1+50
plot(x, n.rates(x, gain=0.1, bias=10), 'r') # x*0.1+10
plot(x, n.rates(x, gain=0.5, bias=5), 'g') # x*0.05+5
plot(x, n.rates(x, gain=0.1, bias=4), 'c') #x*0.1+4))

xlabel('x')
ylabel('a');


I am working on the file "SYDE 556 Lecture 2 Representation.ipynb".
The first task is:
Some sort of mapping between neural activity and a state in the world

my location
head tilt
image
remembered location

Intuitively, we call this "representation"

In neuroscience, people talk about the 'neural code'
To formalize this notion, the NEF uses information theory (or coding theory)

Representation formalism

Value being represented:  $x$
Neural activity:  $a$
Neuron index: $i$

Encoding and decoding

Have to define both to define a code
Lossless code (e.g. Morse Code):
encoding: $a = f(x)$
decodng: $x = f^{-1}(a)$

Lossy code:
encoding: $a = f(x)$
decoding: $\hat{x} = g(a) \approx x$

Distributed representation

Not just one neuron per $x$ value (or per $x$)
Many different $a$ values for a single $x$

Encoding: $a_i = f_i(x)$
Decoding: $\hat{x} = g(a_0, a_1, a_2, a_3, ...)$

Example: binary representation
Encoding (nonlinear):
$$
a_i = \begin{cases}
    1 &\mbox{if } x \mod {2^{i}} > 2^{i-1} \
    0 &\mbox{otherwise}
    \end{cases}
$$
Decoding (linear):
$$
\hat{x} = \sum_i a_i 2^{i-1}
$$

Suppose: $x = 13$
Encoding:
$a_1 = 1$, $a_2 = 0$, $a_3 = 1$, $a_4 = 1$
Decoding:
$\hat{x} = 11+02+14+18 = 13$
Linear decoding

Write decoder as $\hat{x} = \sum_ia_i d_i$

Linear decoding is nice and simple

Works fine with non-linear encoding (!)

The NEF uses linear decoding, but what about the encoding?

Neuron encoding
$a_i = f_i(x)$

What do we know about neurons?

<img src=files/lecture1/NeuronStructure.jpg>

Firing rate goes up as total input current goes up

$a_i = G_i(J)$

What is $G_i$?

depends on how detailed a neuron model we want.
Can you write Python code for it?

from IPython.display import YouTubeVideo
YouTubeVideo('hxdPdKbqm_I', width=720, height=400, loop=1, autoplay=0)


I am working on the file "SYDE 556 Lecture 2 Representation.ipynb".
The first task is:
What happens to the error with more neurons?

Noise

Neurons aren't perfect
Axonal jitter
Neurotransmitter vesicle release failure (~80%)
Amount of neurotransmitter per vesicle
Thermal noise
Ion channel noise (# of channels open and closed)
Network effects
More information: http://icwww.epfl.ch/~gerstner/SPNM/node33.html

How do we include this noise as well?
Make the neuron model more complicated
Simple approach: add gaussian random noise to $a_i$
Set noise standard deviation $\sigma$ to 20% of maximum firing rate
Each $a_i$ value for each $x$ value gets a different noise value added to it

What effect does this have on decoding?
Can you write Python code for it?

#Have to run previous python cell first
A_noisy = A + numpy.random.normal(scale=0.2*numpy.max(A), size=A.shape)

xhat = numpy.dot(A_noisy, d)

pyplot.plot(x, A_noisy)
xlabel('x')
ylabel('firing rate (Hz)')

figure()
plot(x, x)
plot(x, xhat)
xlabel('$x$')
ylabel('$\hat{x}$')
ylim(-1, 1)
xlim(-1, 1)

print 'RMSE', np.sqrt(np.average((x-xhat)**2))


I am working on the file "SYDE 556 Lecture 2 Representation.ipynb".
The first task is:
What happens to the error with more or fewer neurons?

Noise

Neurons aren't perfect
Axonal jitter
Neurotransmitter vesicle release failure (~80%)
Amount of neurotransmitter per vesicle
Thermal noise
Ion channel noise (# of channels open and closed)
Network effects
More information: http://icwww.epfl.ch/~gerstner/SPNM/node33.html

How do we include this noise as well?
Make the neuron model more complicated
Simple approach: add gaussian random noise to $a_i$
Set noise standard deviation $\sigma$ to 20% of maximum firing rate
Each $a_i$ value for each $x$ value gets a different noise value added to it

What effect does this have on decoding?
Can you write Python code for it?

#Have to run previous python cell first
A_noisy = A + numpy.random.normal(scale=0.2*numpy.max(A), size=A.shape)

xhat = numpy.dot(A_noisy, d)

pyplot.plot(x, A_noisy)
xlabel('x')
ylabel('firing rate (Hz)')

figure()
plot(x, x)
plot(x, xhat)
xlabel('$x$')
ylabel('$\hat{x}$')
ylim(-1, 1)
xlim(-1, 1)

print('RMSE', np.sqrt(np.average((x-xhat)**2)))


I am working on the file "SYDE 556 Lecture 2 Representation.ipynb".
The first task is:
What if we just increase the number of neurons?  Will it help?

Taking noise into account

Include noise while solving for decoders

Introduce noise term $\eta$

$
\begin{align}
\hat{x} &= \sum_i(a_i+\eta)d_i \
E &= {1 \over 2} \int_{-1}^1 (x-\hat{x})^2 \;dx d\eta\
 &= {1 \over 2} \int_{-1}^1 \left(x-\sum_i(a_i+\eta)d_i\right)^2 \;dx d\eta\
 &= {1 \over 2} \int_{-1}^1 \left(x-\sum_i a_i d_i - \sum \eta d_i \right)^2 \;dx d\eta
\end{align}
$
- Assume noise is gaussian, independent, mean zero, and has the same variance for each neuron
    - $\eta = \mathcal{N}(0, \sigma)$
    - All the noise cross-terms disappear (independent)
$
\begin{align}
E &= {1 \over 2} \int_{-1}^1 \left(x-\sum_i a_i d_i \right)^2 \;dx + \sum_{i,j} d_i d_j <\eta_i \eta_j>\eta  \
 &= {1 \over 2} \int{-1}^1 \left(x-\sum_i a_i d_i \right)^2 \;dx + \sum_{i} d_i d_i <\eta_i \eta_i>_\eta
\end{align}
$

Since the average of $\eta_i \eta_i$ noise is its variance (since the mean is zero), $\sigma^2$, we get

$
\begin{align}
E = {1 \over 2} \int_{-1}^1 \left(x-\sum_i a_i d_i \right)^2 \;dx + \sigma^2 \sum_i d_i^2
\end{align}
$

The practical result is that, when computing the decoder, we get

$
\begin{align}
\Gamma_{ij} = \sum_x a_i a_j / S + \sigma^2 \delta_{ij}
\end{align}
$

Where $\delta_{ij}$ is the Kronecker delta: http://en.wikipedia.org/wiki/Kronecker_delta

To simplfy computing this using matrices, this can be written as $\Gamma=A^T A /S + \sigma^2 I$
Can you write Python code for it?

import numpy
import nengo
from nengo.utils.ensemble import tuning_curves
from nengo.dists import Uniform

N = 100

model = nengo.Network(label='Neurons')
with model:
    neurons = nengo.Ensemble(N, dimensions=1,
                             max_rates=Uniform(100,200)) #Defaults to LIF neurons,
                                        #with random gains and biases for
                                        #neurons between 100-200hz over -1,1

    connection = nengo.Connection(neurons, neurons, #This is just to generate the decoders
                                  solver=nengo.solvers.LstsqNoise(noise=0.2)) #Add noise ###NEW

sim = nengo.Simulator(model)

d = sim.data[connection].weights.T

x, A = tuning_curves(neurons, sim)

A_noisy = A + numpy.random.normal(scale=0.2*numpy.max(A), size=A.shape)

xhat = numpy.dot(A_noisy, d)

pyplot.plot(x, A_noisy)
xlabel('x')
ylabel('firing rate (Hz)')

figure()
plot(x, x)
plot(x, xhat)
xlabel('$x$')
ylabel('$\hat{x}$')
ylim(-1, 1)
xlim(-1, 1)

print 'RMSE', np.sqrt(np.average((x-xhat)**2))