I am working on the file "Miscellaneous/Gesture-Phase-Detection.ipynb".
The first task is:
Random Forest accuracy and time elapsed caculation
Can you write Python code for it?

t2=time()
print ("RandomForest")
rf = RandomForestClassifier(n_estimators=100,n_jobs=-1)
clf_rf = rf.fit(X_train,y_train)
print ("Acurracy: ", clf_rf.score(X_test,y_test))
t3=time()
print ("time elapsed: ", t3-t2)

tt2=time()
print ("cross result========")
scores = cross_validation.cross_val_score(rf, X,y, cv=5)
print (scores)
print (scores.mean())
tt3=time()
print ("time elapsed: ", tt3-tt2)


I am working on the file "Miscellaneous/Gesture-Phase-Detection.ipynb".
The first task is:
Unsupervised Learning
PCA
Can you write Python code for it?

len(features)

# Apply PCA with the same number of dimensions as variables in the dataset
from sklearn.decomposition import PCA
pca = PCA(n_components=50)
pca.fit(X)

# Print the components and the amount of variance in the data contained in each dimension
print(pca.components_)
print(pca.explained_variance_ratio_)

%matplotlib inline
import matplotlib.pyplot as plt
plt.plot(list(pca.explained_variance_ratio_),'-o')
plt.title('Explained variance ratio as function of PCA components')
plt.ylabel('Explained variance ratio')
plt.xlabel('Component')
plt.show()

# First we reduce the data to two dimensions using PCA to capture variation
pca = PCA(n_components=2)
reduced_data = pca.fit_transform(X)
print(reduced_data[:10])  # print upto 10 elements

# Import clustering modules
from sklearn.cluster import KMeans
from sklearn.mixture import GMM

kmeans = KMeans(n_clusters=5)
clusters = kmeans.fit(reduced_data)
print(clusters)

# Plot the decision boundary by building a mesh grid to populate a graph.
x_min, x_max = reduced_data[:, 0].min() - 1, reduced_data[:, 0].max() + 1
y_min, y_max = reduced_data[:, 1].min() - 1, reduced_data[:, 1].max() + 1
hx = (x_max-x_min)/1000.
hy = (y_max-y_min)/1000.
xx, yy = np.meshgrid(np.arange(x_min, x_max, hx), np.arange(y_min, y_max, hy))

# Obtain labels for each point in mesh. Use last trained model.
Z = clusters.predict(np.c_[xx.ravel(), yy.ravel()])

# Find the centroids for KMeans or the cluster means for GMM

centroids = kmeans.cluster_centers_
print('*** K MEANS CENTROIDS ***')
print(centroids)

# TRANSFORM DATA BACK TO ORIGINAL SPACE FOR ANSWERING 7
print('*** CENTROIDS TRANSFERED TO ORIGINAL SPACE ***')
print(pca.inverse_transform(centroids))

# Put the result into a color plot
Z = Z.reshape(xx.shape)
plt.figure(1)
plt.clf()
plt.imshow(Z, interpolation='nearest',
           extent=(xx.min(), xx.max(), yy.min(), yy.max()),
           cmap=plt.cm.Paired,
           aspect='auto', origin='lower')

plt.plot(reduced_data[:, 0], reduced_data[:, 1], 'k.',...


I am working on the file "Miscellaneous/Indian-Liver-Patient-Classification.ipynb".
The first task is:
Fine tuning the model using GridSearch
Can you write Python code for it?

from sklearn.cross_validation import cross_val_score
from sklearn.pipeline import Pipeline
from sklearn import grid_search

knn = KNeighborsClassifier()

parameters = {'n_neighbors':[1,10]}

grid = grid_search.GridSearchCV(knn, parameters, n_jobs=-1, verbose=1, scoring='accuracy')

grid.fit(X_train, y_train)

print ('Best score: %0.3f' % grid.best_score_)

print ('Best parameters set:')
best_parameters = grid.best_estimator_.get_params()

for param_name in sorted(parameters.keys()):
    print ('\t%s: %r' % (param_name, best_parameters[param_name]))

predictions = grid.predict(X_test)
print( classification_report(y_test, predictions))


I am working on the file "Miscellaneous/Indian-Liver-Patient-Classification.ipynb".
The first task is:
KNN accuracy and time elapsed caculation
Can you write Python code for it?

t6=time()
print ("KNN")
# knn = KNeighborsClassifier(n_neighbors=3)
knn = KNeighborsClassifier(n_neighbors=3)
clf_knn=knn.fit(X_train, y_train)
print ("Acurracy: ", clf_knn.score(X_test,y_test) )
t7=time()
print ("time elapsed: ", t7-t6)


I am working on the file "Miscellaneous/Indian-Liver-Patient-Classification.ipynb".
The first task is:
Naive Bayes accuracy and time elapsed caculation
Can you write Python code for it?

t4=time()
print ("NaiveBayes")
nb = BernoulliNB()
clf_nb=nb.fit(X_train,y_train)
print ("Acurracy: ", clf_nb.score(X_test,y_test))
t5=time()
print ("time elapsed: ", t5-t4)


I am working on the file "Miscellaneous/Indian-Liver-Patient-Classification.ipynb".
The first task is:
Random Forest accuracy and time elapsed caculation
Can you write Python code for it?

t2=time()
print ("RandomForest")
rf = RandomForestClassifier(n_estimators=100,n_jobs=-1)
clf_rf = rf.fit(X_train,y_train)
print ("Acurracy: ", clf_rf.score(X_test,y_test))
t3=time()
print ("time elapsed: ", t3-t2)


I am working on the file "Miscellaneous/Indian-Liver-Patient-Classification.ipynb".
The first task is:
SVM with RBF kernel accuracy and time elapsed caculation
Can you write Python code for it?

t7=time()
print ("SVM")

svc_r = SVC(kernel='rbf')
clf_svc=svc.fit(X_train, y_train)
print ("Acurracy: ", clf_svc.score(X_test,y_test) )
t8=time()
print ("time elapsed: ", t8-t7)


I am working on the file "Miscellaneous/Indian-Liver-Patient-Classification.ipynb".
The first task is:
Tuning our hyperparameters using GridSearch
Can you write Python code for it?

from sklearn.metrics import classification_report

pipeline = Pipeline([
    ('clf', DecisionTreeClassifier(criterion='entropy'))
])

parameters = {
    'clf__max_depth': (5, 25 , 50),
    'clf__min_samples_split': (2, 5, 10),
    'clf__min_samples_leaf': (1, 2, 3)
}

grid_search = GridSearchCV(pipeline, parameters, n_jobs=-1, verbose=1, scoring='f1')
grid_search.fit(X_train, y_train)

print ('Best score: %0.3f' % grid_search.best_score_)
print ('Best parameters set:')

best_parameters = grid_search.best_estimator_.get_params()
for param_name in sorted(parameters.keys()):
    print ('\t%s: %r' % (param_name, best_parameters[param_name]))

predictions = grid_search.predict(X_test)

print (classification_report(y_test, predictions))


I am working on the file "Miscellaneous/Indian-Liver-Patient-Classification.ipynb".
The first task is:
cross validation for KNN
Can you write Python code for it?

tt6=time()
print ("cross result========")
scores = cross_validation.cross_val_score(knn, X,y, cv=5)
print (scores)
print (scores.mean())
tt7=time()
print ("time elapsed: ", tt7-tt6)


I am working on the file "Miscellaneous/Indian-Liver-Patient-Classification.ipynb".
The first task is:
cross validation for SVM
Can you write Python code for it?

tt7=time()
print ("cross result========")
scores = cross_validation.cross_val_score(svc,X,y, cv=5)
print (scores)
print (scores.mean())
tt8=time()
print ("time elapsed: ", tt7-tt6)


tt7=time()
print ("cross result========")
scores = cross_validation.cross_val_score(svc_r,X,y, cv=5)
print (scores)
print (scores.mean())
tt8=time()
print ("time elapsed: ", tt7-tt6)


I am working on the file "Miscellaneous/Leaf Classification.ipynb".
The first task is:
Decision Tree accuracy and time elapsed caculation
Can you write Python code for it?

t0=time()
print "DecisionTree"

dt = DecisionTreeClassifier(min_samples_split=20,random_state=99)
# dt = DecisionTreeClassifier(min_samples_split=20,max_depth=5,random_state=99)

clf_dt=dt.fit(X_train,y_train)

print "Acurracy: ", clf_dt.score(X_test,y_test)
t1=time()
print "time elapsed: ", t1-t0


I am working on the file "Miscellaneous/Leaf Classification.ipynb".
The first task is:
Feature importances with forests of trees
This examples shows the use of forests of trees to evaluate the importance of features on an artificial classification task. The red bars are the feature importances of the forest, along with their inter-trees variability.
Can you write Python code for it?

%matplotlib inline
import numpy as np
import matplotlib.pyplot as plt

from sklearn.ensemble import ExtraTreesClassifier

# Build a classification task using 3 informative features

# Build a forest and compute the feature importances
forest = ExtraTreesClassifier(n_estimators=250,
                              random_state=0)

forest.fit(X, y)
importances = forest.feature_importances_
std = np.std([tree.feature_importances_ for tree in forest.estimators_],
             axis=0)
indices = np.argsort(importances)[::-1]

# Print the feature ranking
print("Feature ranking:")

for f in range(X.shape[1]):
    print("%d. feature %d - %s (%f) " % (f + 1, indices[f], features[indices[f]], importances[indices[f]]))

# Plot the feature importances of the forest
plt.figure(num=None, figsize=(14, 10), dpi=80, facecolor='w', edgecolor='k')
plt.title("Feature importances")
plt.bar(range(X.shape[1]), importances[indices],
       color="r", yerr=std[indices], align="center")
plt.xticks(range(X.shape[1]), indices)
plt.xlim([-1, X.shape[1]])
plt.show()

importances[indices[:5]]

for f in range(5):
    print("%d. feature %d - %s (%f)" % (f + 1, indices[f], features[indices[f]] ,importances[indices[f]]))

best_features = []
for i in indices[:5]:
    best_features.append(features[i])

# Plot the top 5 feature importances of the forest
plt.figure(num=None, figsize=(8, 6), dpi=80, facecolor='w', edgecolor='k')
plt.title("Feature importances")
plt.bar(range(5), importances[indices][:5],
       color="r",  yerr=std[indices][:5], align="center")
plt.xticks(range(5), best_features)
plt.xlim([-1, 5])
plt.show()


I am working on the file "Miscellaneous/Leaf Classification.ipynb".
The first task is:
Fine tuning the model using GridSearch
Can you write Python code for it?

from sklearn.svm import SVC
from sklearn.cross_validation import cross_val_score
from sklearn.pipeline import Pipeline
from sklearn import grid_search

knn = KNeighborsClassifier()

parameters = {'n_neighbors':[1,10]}

grid = grid_search.GridSearchCV(knn, parameters, n_jobs=-1, verbose=1, scoring='accuracy')

grid.fit(X_train, y_train)

print 'Best score: %0.3f' % grid.best_score_

print 'Best parameters set:'
best_parameters = grid.best_estimator_.get_params()

for param_name in sorted(parameters.keys()):
    print '\t%s: %r' % (param_name, best_parameters[param_name])

predictions = grid.predict(X_test)
print classification_report(y_test, predictions)


I am working on the file "Miscellaneous/Leaf Classification.ipynb".
The first task is:
KNN accuracy and time elapsed caculation
Can you write Python code for it?

t6=time()
print "KNN"
# knn = KNeighborsClassifier(n_neighbors=3)
knn = KNeighborsClassifier(n_neighbors=3)
clf_knn=knn.fit(X_train, y_train)
print "Acurracy: ", clf_knn.score(X_test,y_test)
t7=time()
print "time elapsed: ", t7-t6


I am working on the file "Miscellaneous/Leaf Classification.ipynb".
The first task is:
SVM accuracy and time elapsed caculation
Can you write Python code for it?

t7=time()
print "SVM"

svc = SVC()
clf_svc=svc.fit(X_train, y_train)
print "Acurracy: ", clf_svc.score(X_test,y_test)
t8=time()
print "time elapsed: ", t8-t7


I am working on the file "Miscellaneous/Leaf Classification.ipynb".
The first task is:
cross validation for KNN
Can you write Python code for it?

tt6=time()
print "cross result========"
scores = cross_validation.cross_val_score(knn, X,y, cv=5)
print scores
print scores.mean()
tt7=time()
print "time elapsed: ", tt7-tt6
print "\n"


I am working on the file "Miscellaneous/Leaf Classification.ipynb".
The first task is:
cross validation for RF
Can you write Python code for it?

tt2=time()
print "cross result========"
scores = cross_validation.cross_val_score(rf, X, y, cv=3)
print scores
print scores.mean()
tt3=time()
print "time elapsed: ", tt3-tt2
print "\n"


I am working on the file "Miscellaneous/Lenses Data Classification.ipynb".
The first task is:
Affinity Propogation
Can you write Python code for it?

# Affinity propagation
aff = cluster.AffinityPropagation()
aff.fit(X_train)
print aff.cluster_centers_indices_.shape

y_pred = aff.predict(X_test)

from sklearn import metrics
print "Addjusted rand score:{:.2}".format(metrics.adjusted_rand_score(y_test, y_pred))
print "Homogeneity score:{:.2} ".format(metrics.homogeneity_score(y_test, y_pred))
print "Completeness score: {:.2} ".format(metrics.completeness_score(y_test, y_pred))
print "Confusion matrix"
print metrics.confusion_matrix(y_test, y_pred)

#MeanShift
ms = cluster.MeanShift()
ms.fit(X_train)
print ms.cluster_centers_

y_pred = ms.predict(X_test)

from sklearn import metrics
print "Addjusted rand score:{:.2}".format(metrics.adjusted_rand_score(y_test, y_pred))
print "Homogeneity score:{:.2} ".format(metrics.homogeneity_score(y_test, y_pred))
print "Completeness score: {:.2} ".format(metrics.completeness_score(y_test, y_pred))
print "Confusion matrix"
print metrics.confusion_matrix(y_test, y_pred)


I am working on the file "Miscellaneous/Lenses Data Classification.ipynb".
The first task is:
Applying agglomerative clustering via scikit-learn
Can you write Python code for it?

from sklearn.cluster import AgglomerativeClustering

ac = AgglomerativeClustering(n_clusters=3, affinity='euclidean', linkage='complete')
labels = ac.fit_predict(X)
print('Cluster labels: %s' % labels)


I am working on the file "Miscellaneous/Lenses Data Classification.ipynb".
The first task is:
As shown in the following table, the linkage matrix consists of several rows where each row represents one merge. The first and second columns denote the most dissimilar members in each cluster, and the third row reports the distance between those members. The last column returns the count of the members in each cluster.
Now that we have computed the linkage matrix, we can visualize the results in the form of a dendrogram:
Can you write Python code for it?

from scipy.cluster.hierarchy import dendrogram

# make dendrogram black (part 1/2)
# from scipy.cluster.hierarchy import set_link_color_palette
# set_link_color_palette(['black'])

row_dendr = dendrogram(row_clusters,
                       labels=labels,
                       # make dendrogram black (part 2/2)
                       # color_threshold=np.inf
                       )
plt.tight_layout()
plt.ylabel('Euclidean distance')
#plt.savefig('./figures/dendrogram.png', dpi=300,
#            bbox_inches='tight')
plt.show()

# plot row dendrogram
fig = plt.figure(figsize=(8,8))
axd = fig.add_axes([0.09,0.1,0.2,0.6])
row_dendr = dendrogram(row_clusters, orientation='right')

# reorder data with respect to clustering
df_rowclust = df.ix[row_dendr['leaves'][::-1]]

axd.set_xticks([])
axd.set_yticks([])

# remove axes spines from dendrogram
for i in axd.spines.values():
        i.set_visible(False)

# plot heatmap
axm = fig.add_axes([0.23,0.1,0.6,0.6]) # x-pos, y-pos, width, height
cax = axm.matshow(df_rowclust, interpolation='nearest', cmap='hot_r')
fig.colorbar(cax)
axm.set_xticklabels([''] + list(df_rowclust.columns))
axm.set_yticklabels([''] + list(df_rowclust.index))

# plt.savefig('./figures/heatmap.png', dpi=300)
plt.show()


I am working on the file "Miscellaneous/Lenses Data Classification.ipynb".
The first task is:
Elbow Method
Can you write Python code for it?

distortions = []
for i in range(1, 11):
    km = KMeans(n_clusters=i,
                init='k-means++',
                n_init=10,
                max_iter=300,
                random_state=0)
    km.fit(X)
    distortions .append(km.inertia_)
plt.plot(range(1,11), distortions , marker='o')
plt.xlabel('Number of clusters')
plt.ylabel('Distortion')
plt.tight_layout()
#plt.savefig('./figures/elbow.png', dpi=300)
plt.show()


I am working on the file "Miscellaneous/Lenses Data Classification.ipynb".
The first task is:
Feature importances with forests of trees
This examples shows the use of forests of trees to evaluate the importance of features on an artificial classification task. The red bars are the feature importances of the forest, along with their inter-trees variability.
Can you write Python code for it?


I am working on the file "Miscellaneous/Lenses Data Classification.ipynb".
The first task is:
Fine tuning the model using GridSearch
Can you write Python code for it?

from sklearn.svm import SVC
from sklearn.cross_validation import cross_val_score
from sklearn.pipeline import Pipeline
from sklearn import grid_search

knn = KNeighborsClassifier()

parameters = {'n_neighbors':[1,]}

grid = grid_search.GridSearchCV(knn, parameters, n_jobs=-1, verbose=1, scoring='accuracy')

grid.fit(X_train, y_train)

print 'Best score: %0.3f' % grid.best_score_

print 'Best parameters set:'
best_parameters = grid.best_estimator_.get_params()

for param_name in sorted(parameters.keys()):
    print '\t%s: %r' % (param_name, best_parameters[param_name])

predictions = grid.predict(X_test)
print classification_report(y_test, predictions)


I am working on the file "Miscellaneous/Lenses Data Classification.ipynb".
The first task is:
Mixture of Guassian Models
Can you write Python code for it?

from sklearn import mixture

# Define a heldout dataset to estimate covariance type
X_train_heldout, X_test_heldout, y_train_heldout, y_test_heldout = train_test_split(
        X_train, y_train,test_size=0.25, random_state=42)
for covariance_type in ['spherical','tied','diag','full']:
    gm=mixture.GMM(n_components=3, covariance_type=covariance_type, random_state=42, n_init=5)
    gm.fit(X_train_heldout)
    y_pred=gm.predict(X_test_heldout)
    print "Adjusted rand score for covariance={}:{:.2}".format(covariance_type, metrics.adjusted_rand_score(y_test_heldout, y_pred))

gm = mixture.GMM(n_components=3, covariance_type='tied', random_state=42)
gm.fit(X_train)

# Print train clustering and confusion matrix
y_pred = gm.predict(X_test)
print "Addjusted rand score:{:.2}".format(metrics.adjusted_rand_score(y_test, y_pred))
print "Homogeneity score:{:.2} ".format(metrics.homogeneity_score(y_test, y_pred))
print "Completeness score: {:.2} ".format(metrics.completeness_score(y_test, y_pred))

print "Confusion matrix"
print metrics.confusion_matrix(y_test, y_pred)


I am working on the file "Miscellaneous/Lenses Data Classification.ipynb".
The first task is:
Naive Bayes accuracy and time elapsed caculation
Can you write Python code for it?

t4=time()
print "NaiveBayes"
nb = BernoulliNB()
clf_nb=nb.fit(X_train,y_train)
print "Acurracy: ", clf_nb.score(X_test,y_test)
t5=time()
print "time elapsed: ", t5-t4


I am working on the file "Miscellaneous/Lenses Data Classification.ipynb".
The first task is:
Organizing clusters as a hierarchical tree
Performing hierarchical clustering on a distance matrix
To calculate the distance matrix as input for the hierarchical clustering algorithm, we will use the pdist function from SciPy's spatial.distance submodule:
Can you write Python code for it?

labels = []
for i in range(df1.shape[0]):
    str = 'ID_{}'.format(i)
    labels.append(str)

from scipy.spatial.distance import pdist,squareform

row_dist = pd.DataFrame(squareform(pdist(df1, metric='euclidean')), columns=labels, index=labels)
row_dist[:5]

# 1. incorrect approach: Squareform distance matrix

from scipy.cluster.hierarchy import linkage

row_clusters = linkage(row_dist, method='complete', metric='euclidean')
pd.DataFrame(row_clusters,
             columns=['row label 1', 'row label 2', 'distance', 'no. of items in clust.'],
             index=['cluster %d' %(i+1) for i in range(row_clusters.shape[0])])

# 2. correct approach: Condensed distance matrix

row_clusters = linkage(pdist(df, metric='euclidean'), method='complete')
pd.DataFrame(row_clusters,
             columns=['row label 1', 'row label 2', 'distance', 'no. of items in clust.'],
             index=['cluster %d' %(i+1) for i in range(row_clusters.shape[0])])

# 3. correct approach: Input sample matrix

row_clusters = linkage(df.values, method='complete', metric='euclidean')
pd.DataFrame(row_clusters,
             columns=['row label 1', 'row label 2', 'distance', 'no. of items in clust.'],
             index=['cluster %d' %(i+1) for i in range(row_clusters.shape[0])])


I am working on the file "Miscellaneous/Lenses Data Classification.ipynb".
The first task is:
PCA
Can you write Python code for it?

# Apply PCA with the same number of dimensions as variables in the dataset
from sklearn.decomposition import PCA
pca = PCA(n_components=4) # 6 components for 6 variables
pca.fit(df1)

# Print the components and the amount of variance in the data contained in each dimension
print(pca.components_)
print(pca.explained_variance_ratio_)

%matplotlib inline
import matplotlib.pyplot as plt
plt.plot(list(pca.explained_variance_ratio_),'-o')
plt.title('Explained variance ratio as function of PCA components')
plt.ylabel('Explained variance ratio')
plt.xlabel('Component')
plt.show()

%pylab inline
import IPython
import sklearn as sk
import numpy as np
import matplotlib
import matplotlib.pyplot as plt

y = df['Target-Lenses']

target = array(y.unique())

target

def plot_pca_scatter():
    colors = ['blue', 'red', 'green']
    for i in xrange(len(colors)):
        px = X_pca[:, 0][y == i+1]
        py = X_pca[:, 1][y == i+1]
        plt.scatter(px, py, c=colors[i])
    plt.legend(target)
    plt.xlabel('First Principal Component')
    plt.ylabel('Second Principal Component')

from sklearn.decomposition import PCA

estimator = PCA(n_components=3)
X_pca = estimator.fit_transform(X)
plot_pca_scatter() # Note that we only plot the first and second principal component


I am working on the file "Miscellaneous/Lenses Data Classification.ipynb".
The first task is:
Quantifying the quality of clustering via silhouette plots
Can you write Python code for it?

import numpy as np
from matplotlib import cm
from sklearn.metrics import silhouette_samples

km = KMeans(n_clusters=3,
            init='k-means++',
            n_init=10,
            max_iter=300,
            tol=1e-04,
            random_state=0)
y_km = km.fit_predict(X)

cluster_labels = np.unique(y_km)
n_clusters = cluster_labels.shape[0]
silhouette_vals = silhouette_samples(X, y_km, metric='euclidean')
y_ax_lower, y_ax_upper = 0, 0
yticks = []
for i, c in enumerate(cluster_labels):
    c_silhouette_vals = silhouette_vals[y_km == c]
    c_silhouette_vals.sort()
    y_ax_upper += len(c_silhouette_vals)
    color = cm.jet(i / n_clusters)
    plt.barh(range(y_ax_lower, y_ax_upper), c_silhouette_vals, height=1.0,
            edgecolor='none', color=color)

    yticks.append((y_ax_lower + y_ax_upper) / 2)
    y_ax_lower += len(c_silhouette_vals)

silhouette_avg = np.mean(silhouette_vals)
plt.axvline(silhouette_avg, color="red", linestyle="--")

plt.yticks(yticks, cluster_labels + 1)
plt.ylabel('Cluster')
plt.xlabel('Silhouette coefficient')

plt.tight_layout()
# plt.savefig('./figures/silhouette.png', dpi=300)
plt.show()


I am working on the file "Miscellaneous/Lenses Data Classification.ipynb".
The first task is:
Random Forest accuracy and time elapsed caculation
Can you write Python code for it?

t2=time()
print "RandomForest"
rf = RandomForestClassifier(n_estimators=100,n_jobs=-1)
clf_rf = rf.fit(X_train,y_train)
print "Acurracy: ", clf_rf.score(X_test,y_test)
t3=time()
print "time elapsed: ", t3-t2


I am working on the file "Miscellaneous/Lenses Data Classification.ipynb".
The first task is:
SVM accuracy and time elapsed caculation
Can you write Python code for it?


I am working on the file "Miscellaneous/Lenses Data Classification.ipynb".
The first task is:
Tuning our hyperparameters using GridSearch
Can you write Python code for it?

from sklearn.metrics import classification_report

pipeline = Pipeline([
    ('clf', DecisionTreeClassifier(criterion='entropy'))
])

parameters = {
    'clf__max_depth': (5, 25 , 50),
    'clf__min_samples_split': (1, 5, 10),
    'clf__min_samples_leaf': (1, 2, 3)
}

grid_search = GridSearchCV(pipeline, parameters, n_jobs=-1, verbose=1, scoring='f1')
grid_search.fit(X_train, y_train)

print 'Best score: %0.3f' % grid_search.best_score_
print 'Best parameters set:'

best_parameters = grid_search.best_estimator_.get_params()
for param_name in sorted(parameters.keys()):
    print '\t%s: %r' % (param_name, best_parameters[param_name])

predictions = grid_search.predict(X_test)

print classification_report(y_test, predictions)


I am working on the file "Miscellaneous/Lenses Data Classification.ipynb".
The first task is:
cross validation for KNN
Can you write Python code for it?


I am working on the file "Miscellaneous/Lenses Data Classification.ipynb".
The first task is:
cross validation for SVM
Can you write Python code for it?

tt7=time()
print "cross result========"
scores = cross_validation.cross_val_score(svc,X,y, cv=5)
print scores
print scores.mean()
tt8=time()
print "time elapsed: ", tt7-tt6
print "\n"

from sklearn.svm import SVC
from sklearn.cross_validation import cross_val_score
from sklearn.pipeline import Pipeline
from sklearn import grid_search

svc = SVC()

parameters = {'kernel':('linear', 'rbf'), 'C':[1, 10]}

grid = grid_search.GridSearchCV(svc, parameters, n_jobs=-1, verbose=1, scoring='accuracy')

grid.fit(X_train, y_train)

print 'Best score: %0.3f' % grid.best_score_

print 'Best parameters set:'
best_parameters = grid.best_estimator_.get_params()

for param_name in sorted(parameters.keys()):
    print '\t%s: %r' % (param_name, best_parameters[param_name])

predictions = grid.predict(X_test)
print classification_report(y_test, predictions)

pipeline = Pipeline([
    ('clf', SVC(kernel='linear', gamma=0.01, C=1))
])

parameters = {
    'clf__gamma': (0.01, 0.03, 0.1, 0.3, 1),
    'clf__C': (0.1, 0.3, 1, 3, 10, 30),
}

grid_search = GridSearchCV(pipeline, parameters, n_jobs=-1, verbose=1, scoring='accuracy')

grid_search.fit(X_train, y_train)

print 'Best score: %0.3f' % grid_search.best_score_

print 'Best parameters set:'
best_parameters = grid_search.best_estimator_.get_params()

for param_name in sorted(parameters.keys()):
    print '\t%s: %r' % (param_name, best_parameters[param_name])

predictions = grid_search.predict(X_test)
print classification_report(y_test, predictions)


I am working on the file "Miscellaneous/Lenses Data Classification.ipynb".
The first task is:
cross-validation for NB
Can you write Python code for it?

tt4=time()
print "cross result========"
scores = cross_validation.cross_val_score(nb, X,y, cv=3)
print scores
print scores.mean()
tt5=time()
print "time elapsed: ", tt5-tt4
print "\n"


I am working on the file "Miscellaneous/Plants Clustering.ipynb".
The first task is:

Can you write Python code for it?

X = df[features].values
y= df['Class'].values
pca = PCA(n_components=2)
X = pca.fit_transform(X)

c = []
from matplotlib.pyplot import cm
n=100
color=iter(cm.rainbow(np.linspace(0,1,n)))
for i in range(n):
    c.append(next(color))

c[99]

n = 100
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(20,10))

km = KMeans(n_clusters= n , random_state=0)
y_km = km.fit_predict(X)

for i in range(n):
    ax1.scatter(X[y_km==i,0], X[y_km==i,1], c=c[i], marker='o', s=40, label='cluster{}'.format(i))
ax1.set_title('K-means clustering')

ac = AgglomerativeClustering(n_clusters=100, affinity='euclidean', linkage='complete')
y_ac = ac.fit_predict(X)
for i in range(n):
    ax2.scatter(X[y_ac==i,0], X[y_ac==i,1], c=c[i], marker='o', s=40, label='cluster{}'.format(i))
ax2.set_title('Agglomerative clustering')

# Put a legend below current axis
plt.legend(loc='upper center', bbox_to_anchor=(0, -0.05),
          fancybox=True, shadow=True, ncol=10)

plt.tight_layout()
#plt.savefig('./figures/kmeans_and_ac.png', dpi=300)
plt.show()


I am working on the file "Miscellaneous/Plants Clustering.ipynb".
The first task is:
Affinity Propogation
Can you write Python code for it?

# Affinity propagation
aff = cluster.AffinityPropagation()
aff.fit(X_train)
print aff.cluster_centers_indices_.shape

y_pred = aff.predict(X_test)

from sklearn import metrics
print "Addjusted rand score:{:.2}".format(metrics.adjusted_rand_score(y_test, y_pred))
print "Homogeneity score:{:.2} ".format(metrics.homogeneity_score(y_test, y_pred))
print "Completeness score: {:.2} ".format(metrics.completeness_score(y_test, y_pred))
print "Confusion matrix"
print metrics.confusion_matrix(y_test, y_pred)


I am working on the file "Miscellaneous/Plants Clustering.ipynb".
The first task is:
Applying agglomerative clustering via scikit-learn
Can you write Python code for it?

from sklearn.cluster import AgglomerativeClustering

ac = AgglomerativeClustering(n_clusters=100, affinity='euclidean', linkage='complete')
labels = ac.fit_predict(X)
print('Cluster labels: %s' % labels)


I am working on the file "Miscellaneous/Plants Clustering.ipynb".
The first task is:
Classification
Can you write Python code for it?

import os
from sklearn.tree import DecisionTreeClassifier, export_graphviz
import pandas as pd
import numpy as np
from sklearn.cross_validation import train_test_split
from sklearn import cross_validation, metrics
from sklearn.ensemble import RandomForestClassifier
from sklearn.naive_bayes import BernoulliNB
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
from time import time
from sklearn.pipeline import Pipeline
from sklearn.metrics import roc_auc_score , classification_report
from sklearn.grid_search import GridSearchCV
from sklearn.pipeline import Pipeline
from sklearn.metrics import precision_score, recall_score, accuracy_score, classification_report

X = df[features]
y = df['Class']

# split dataset to 60% training and 40% testing
X_train, X_test, y_train, y_test = cross_validation.train_test_split(X, y, test_size=0.4, random_state=0)

print (X_train.shape, y_train.shape)


I am working on the file "Miscellaneous/Plants Clustering.ipynb".
The first task is:
Decision Tree accuracy and time elapsed caculation
Can you write Python code for it?

t0=time()
print ("DecisionTree")

dt = DecisionTreeClassifier(min_samples_split=20,random_state=99)
# dt = DecisionTreeClassifier(min_samples_split=20,max_depth=5,random_state=99)

clf_dt=dt.fit(X_train,y_train)

print ("Acurracy: ", clf_dt.score(X_test,y_test))
t1=time()
print ("time elapsed: ", t1-t0)

tt0=time()
print ("cross result========")
scores = cross_validation.cross_val_score(dt, X,y, cv=5)
print (scores)
print (scores.mean())
tt1=time()
print ("time elapsed: ", tt1-tt0)

from sklearn.metrics import classification_report

pipeline = Pipeline([
    ('clf', DecisionTreeClassifier(criterion='entropy'))
])

parameters = {
    'clf__max_depth': (5, 25 , 50),
    'clf__min_samples_split': (1, 5, 10),
    'clf__min_samples_leaf': (1, 2, 3)
}

grid_search = GridSearchCV(pipeline, parameters, n_jobs=-1, verbose=1, scoring='f1')
grid_search.fit(X_train, y_train)

print 'Best score: %0.3f' % grid_search.best_score_
print 'Best parameters set:'

best_parameters = grid_search.best_estimator_.get_params()
for param_name in sorted(parameters.keys()):
    print '\t%s: %r' % (param_name, best_parameters[param_name])

predictions = grid_search.predict(X_test)

print classification_report(y_test, predictions)


I am working on the file "Miscellaneous/Plants Clustering.ipynb".
The first task is:
KNN accuracy and time elapsed caculation
Can you write Python code for it?

t6=time()
print ("KNN")
# knn = KNeighborsClassifier(n_neighbors=3)
knn = KNeighborsClassifier()
clf_knn=knn.fit(X_train, y_train)
print ("Acurracy: ", clf_knn.score(X_test,y_test) )
t7=time()
print ("time elapsed: ", t7-t6)

tt0=time()
print ("cross result========")
scores = cross_validation.cross_val_score(knn, X,y, cv=5)
print (scores)
print (scores.mean())
tt1=time()
print ("time elapsed: ", tt1-tt0)


I am working on the file "Miscellaneous/Plants Clustering.ipynb".
The first task is:
Random Forest accuracy and time elapsed caculation
Can you write Python code for it?

t2=time()
print ("RandomForest")
rf = RandomForestClassifier(n_estimators=100,n_jobs=-1)
clf_rf = rf.fit(X_train,y_train)
print ("Acurracy: ", clf_rf.score(X_test,y_test))
t3=time()
print ("time elapsed: ", t3-t2)

tt0=time()
print ("cross result========")
scores = cross_validation.cross_val_score(rf, X,y, cv=5)
print (scores)
print (scores.mean())
tt1=time()
print ("time elapsed: ", tt1-tt0)

pipeline2 = Pipeline([
('clf', RandomForestClassifier(criterion='entropy'))
])

parameters = {
    'clf__n_estimators': (5, 25, 50, 100),
    'clf__max_depth': (5, 25 , 50),
    'clf__min_samples_split': (1, 5, 10),
    'clf__min_samples_leaf': (1, 2, 3)
}

grid_search = GridSearchCV(pipeline2, parameters, n_jobs=-1, verbose=1, scoring='accuracy', cv=3)

grid_search.fit(X_train, y_train)

print 'Best score: %0.3f' % grid_search.best_score_

print 'Best parameters set:'
best_parameters = grid_search.best_estimator_.get_params()

for param_name in sorted(parameters.keys()):
    print '\t%s: %r' % (param_name, best_parameters[param_name])

predictions = grid_search.predict(X_test)
print 'Accuracy:', accuracy_score(y_test, predictions)
print classification_report(y_test, predictions)


I am working on the file "Miscellaneous/Plants Clustering.ipynb".
The first task is:
Unsupervised Learning
PCA
Can you write Python code for it?

y.unique()

len(features)

# Apply PCA with the same number of dimensions as variables in the dataset
from sklearn.decomposition import PCA
pca = PCA(n_components=64)
pca.fit(X)

# Print the components and the amount of variance in the data contained in each dimension
print(pca.components_)
print(pca.explained_variance_ratio_)

%matplotlib inline
import matplotlib.pyplot as plt
plt.plot(list(pca.explained_variance_ratio_),'-o')
plt.title('Explained variance ratio as function of PCA components')
plt.ylabel('Explained variance ratio')
plt.xlabel('Component')
plt.show()

# First we reduce the data to two dimensions using PCA to capture variation
pca = PCA(n_components=2)
reduced_data = pca.fit_transform(X)
print(reduced_data[:10])  # print upto 10 elements

kmeans = KMeans(n_clusters=100)
clusters = kmeans.fit(reduced_data)
print(clusters)

# Plot the decision boundary by building a mesh grid to populate a graph.
x_min, x_max = reduced_data[:, 0].min() - 1, reduced_data[:, 0].max() + 1
y_min, y_max = reduced_data[:, 1].min() - 1, reduced_data[:, 1].max() + 1
hx = (x_max-x_min)/1000.
hy = (y_max-y_min)/1000.
xx, yy = np.meshgrid(np.arange(x_min, x_max, hx), np.arange(y_min, y_max, hy))

# Obtain labels for each point in mesh. Use last trained model.
Z = clusters.predict(np.c_[xx.ravel(), yy.ravel()])

# Find the centroids for KMeans or the cluster means for GMM

centroids = kmeans.cluster_centers_
print('*** K MEANS CENTROIDS ***')
print(centroids)

# TRANSFORM DATA BACK TO ORIGINAL SPACE FOR ANSWERING 7
print('*** CENTROIDS TRANSFERED TO ORIGINAL SPACE ***')
print(pca.inverse_transform(centroids))

# Put the result into a color plot
Z = Z.reshape(xx.shape)
plt.figure(1)
plt.clf()
plt.imshow(Z, interpolation='nearest',
           extent=(xx.min(), xx.max(), yy.min(), yy.max()),
           cmap=plt.cm.Paired,
           aspect='auto', origin='lower')

plt.plot(reduced_data[:, 0], reduced_data[:, 1], 'k.', markersize=2)
plt.scatter(centroids[:, 0], centroids[:, 1],
            marker='x', s=169, linewidths=3,
    ...


I am working on the file "Miscellaneous/Student-Performance-Evaluation-Classification-Regression.ipynb".
The first task is:

Can you write Python code for it?

# read .csv from provided dataset
csv_filename="student/student-mat.csv"

# df=pd.read_csv(csv_filename,index_col=0)
df=pd.read_csv(csv_filename, sep=";")

df.head()

df.describe()


I am working on the file "Miscellaneous/Student-Performance-Evaluation-Classification-Regression.ipynb".
The first task is:
An interesting side effect of random forest classification, is that you can measure how 'important' each feature is when predicting the final result
Can you write Python code for it?


I am working on the file "Miscellaneous/Student-Performance-Evaluation-Classification-Regression.ipynb".
The first task is:
CASE 1: Binary Classification : G3>10: 1 else 0
Can you write Python code for it?

df.G3.describe()

# handle G3 attrubte to binary
high = df.G3 >= 10
low = df.G3 < 10
df.loc[high,'G3'] = 1
df.loc[low,'G3'] = 0

df.head()

df.G3.describe()

cols = list(df.columns)

categorical_features = []
for f in cols:
    if df[f].dtype != 'int64':
        categorical_features.append(f)
categorical_features

for f in categorical_features:

    #Get binarized columns
    df[f] = pd.get_dummies(df[f])

df.head()

features=list(df.columns[:-1])

X = df[features]
y = df['G3']

# split dataset to 60% training and 40% testing
X_train, X_test, y_train, y_test = cross_validation.train_test_split(X,y, test_size=0.4, random_state=0)

print (X_train.shape, y_train.shape)


I am working on the file "Miscellaneous/Student-Performance-Evaluation-Classification-Regression.ipynb".
The first task is:
CASE 2: Multi Class Classification :

Class	G3	Label

I   (excellent/very good)	16-20	A

II  (good)	14-15	B

III (satisfactory)	12-13	C

IV  (sufficient)	10-11	D

V   (fail)	0-9	E

Can you write Python code for it?

df.G3.describe()

for i in range(len(df.G3)):
    if df.G3.loc[i] < 10:
        df.G3.loc[i] = 5
    elif df.G3.loc[i] < 12:
        df.G3.loc[i] = 4
    elif df.G3.loc[i] < 14:
        df.G3.loc[i] = 3
    elif df.G3.loc[i] < 16:
        df.G3.loc[i] = 2
    elif df.G3.loc[i] < 21:
        df.G3.loc[i] = 1

df.G3.unique()

df.head()

df.G3.describe()

cols = list(df.columns)

categorical_features = []
for f in cols:
    if df[f].dtype != 'int64':
        categorical_features.append(f)
categorical_features

for f in categorical_features:

    #Get binarized columns
    df[f] = pd.get_dummies(df[f])

df.head()

features=list(df.columns[:-1])

X = df[features]
y = df['G3']

# split dataset to 60% training and 40% testing
X_train, X_test, y_train, y_test = cross_validation.train_test_split(X,y, test_size=0.4, random_state=0)

print (X_train.shape, y_train.shape)


I am working on the file "Miscellaneous/Student-Performance-Evaluation-Classification-Regression.ipynb".
The first task is:
Case 3 : Regression
Can you write Python code for it?

import numpy as np
from sklearn.linear_model import LinearRegression
from sklearn.linear_model import SGDRegressor
from sklearn.preprocessing import StandardScaler

from sklearn.cross_validation import train_test_split
from sklearn. cross_validation import cross_val_score

from sklearn.feature_selection import *
from sklearn import metrics

# read .csv from provided dataset
csv_filename="student/student-mat.csv"

# df=pd.read_csv(csv_filename,index_col=0)
df=pd.read_csv(csv_filename,sep=";")

df.head()

cols = list(df.columns)

categorical_features = []
for f in cols:
    if df[f].dtype != 'int64':
        categorical_features.append(f)
categorical_features

for f in categorical_features:

    #Get binarized columns
    df[f] = pd.get_dummies(df[f])

df.head()

features=list(df.columns[:-1])

X = df[features]
y = df['G3']

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=0)

from sklearn.feature_selection import *
fs=SelectKBest(score_func=f_regression,k=5)
X_new=fs.fit_transform(X_train,y_train)
z =  zip(fs.get_support(),features)
print z

x_min, x_max = X_new[:,0].min() - .5, X_new[:, 0].max() + .5
y_min, y_max = y_train.min() - .5, y_train.max() + .5
#fig=plt.figure()
#fig.subplots_adjust(left=0, right=1, bottom=0, top=1, hspace=0.05, wspace=0.05)

# Two subplots, unpack the axes array immediately
fig, axes = plt.subplots(1,5)
fig.set_size_inches(12,12)

for i in range(5):
    axes[i].set_aspect('equal')
    axes[i].set_title('Feature {}'.format(i))
    axes[i].set_xlabel('Feature')
    axes[i].set_ylabel('Grades')
    axes[i].set_xlim(x_min, x_max)
    axes[i].set_ylim(y_min, y_max)
    plt.sca(axes[i])
    plt.scatter(X_new[:,i],y_train)

best_features = []
for bool,feature in z:
    if bool:
        best_features.append(feature)

correlated = best_features + ['G3']

correlated

%matplotlib inline
import matplotlib.pyplot as plt
import seaborn as sns
sns.set(style='whitegrid', context='notebook')

sns.pairplot(df[correlated], size=2.0);
plt.tight_layout()
#...


I am working on the file "Miscellaneous/Student-Performance-Evaluation-Classification-Regression.ipynb".
The first task is:
Cross Validation
Can you write Python code for it?

scores = cross_val_score(regressor, X, y, cv=5)
print "Average of scores: ", scores.mean()
print "Cross validation scores: ", scores

plt.scatter(y_test,y_predictions)
plt.xlabel('True Quality')
plt.ylabel('Predicted Quality')
plt.title('Predicted Quality Against True Quality')
plt.show()


I am working on the file "Miscellaneous/Student-Performance-Evaluation-Classification-Regression.ipynb".
The first task is:
Decision Tree accuracy and time elapsed caculation
Can you write Python code for it?

t0=time()
print ("DecisionTree")

dt = DecisionTreeClassifier(min_samples_split=20,random_state=99)
# dt = DecisionTreeClassifier(min_samples_split=20,max_depth=5,random_state=99)

clf_dt=dt.fit(X_train,y_train)

print ("Acurracy: ", clf_dt.score(X_test,y_test))
t1=time()
print ("time elapsed: ", t1-t0)


I am working on the file "Miscellaneous/Student-Performance-Evaluation-Classification-Regression.ipynb".
The first task is:
Feature importances with forests of trees
This examples shows the use of forests of trees to evaluate the importance of features on an artificial classification task. The red bars are the feature importances of the forest, along with their inter-trees variability.
Can you write Python code for it?

import numpy as np
import matplotlib.pyplot as plt

from sklearn.ensemble import ExtraTreesClassifier

# Build a classification task using 3 informative features

# Build a forest and compute the feature importances
forest = ExtraTreesClassifier(n_estimators=250,
                              random_state=0)

forest.fit(X, y)
importances = forest.feature_importances_
std = np.std([tree.feature_importances_ for tree in forest.estimators_],
             axis=0)
indices = np.argsort(importances)[::-1]

# Print the feature ranking
print("Feature ranking:")

for f in range(X.shape[1]):
    print("%d. feature %d - %s (%f) " % (f + 1, indices[f], features[indices[f]], importances[indices[f]]))

# Plot the feature importances of the forest
plt.figure(num=None, figsize=(14, 10), dpi=80, facecolor='w', edgecolor='k')
plt.title("Feature importances")
plt.bar(range(X.shape[1]), importances[indices],
       color="r", yerr=std[indices], align="center")
plt.xticks(range(X.shape[1]), indices)
plt.xlim([-1, X.shape[1]])
plt.show()

importances[indices[:5]]

for f in range(5):
    print("%d. feature %d - %s (%f)" % (f + 1, indices[f], features[indices[f]] ,importances[indices[f]]))

best_features = []
for i in indices[:5]:
    best_features.append(features[i])

# Plot the top 5 feature importances of the forest
plt.figure(num=None, figsize=(8, 6), dpi=80, facecolor='w', edgecolor='k')
plt.title("Feature importances")
plt.bar(range(5), importances[indices][:5],
       color="r",  yerr=std[indices][:5], align="center")
plt.xticks(range(5), best_features)
plt.xlim([-1, 5])
plt.show()


I am working on the file "Miscellaneous/Student-Performance-Evaluation-Classification-Regression.ipynb".
The first task is:
Fitting models with gradient descent
SGDRegressor
Can you write Python code for it?

# Scaling the features using StandardScaler:
X_scaler = StandardScaler()
y_scaler = StandardScaler()
X_train = X_scaler.fit_transform(X_train)
y_train = y_scaler.fit_transform(y_train)
X_test = X_scaler.transform(X_test)
y_test = y_scaler.transform(y_test)

regressor = SGDRegressor(loss='squared_loss')
scores = cross_val_score(regressor, X_train, y_train, cv=5)
print 'Cross validation r-squared scores:', scores
print 'Average cross validation r-squared score:', np.mean(scores)
regressor.fit_transform(X_train, y_train)
print 'Test set r-squared score', regressor.score(X_test, y_test)

from sklearn.cross_validation import *
def train_and_evaluate(clf, X_train, y_train):

    clf.fit(X_train, y_train)

    print "Coefficient of determination on training set:",clf.score(X_train, y_train)

    # create a k-fold croos validation iterator of k=5 folds
    cv = KFold(X_train.shape[0], 5, shuffle=True, random_state=33)
    scores = cross_val_score(clf, X_train, y_train, cv=cv)
    print "Average coefficient of determination using 5-fold crossvalidation:",np.mean(scores)


I am working on the file "Miscellaneous/Student-Performance-Evaluation-Classification-Regression.ipynb".
The first task is:
For example, if we set l1_ratio to 1.0, the ElasticNet regressor would be
equal to LASSO regression.
Decision tree regression
Can you write Python code for it?

X = df[features].values
y = df['G3'].values

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=0)

from sklearn.tree import DecisionTreeRegressor

tree = DecisionTreeRegressor(max_depth=3)
tree.fit(X_train, y_train)
y_train_pred = tree.predict(X_train)
y_test_pred = tree.predict(X_test)

print('MSE train: %.3f, test: %.3f' % (
        mean_squared_error(y_train, y_train_pred),
        mean_squared_error(y_test, y_test_pred)))
print('R^2 train: %.3f, test: %.3f' % (
        r2_score(y_train, y_train_pred),
        r2_score(y_test, y_test_pred)))


I am working on the file "Miscellaneous/Student-Performance-Evaluation-Classification-Regression.ipynb".
The first task is:
Lastly, the ElasticNet implementation allows us to vary the L1 to L2 ratio:
Can you write Python code for it?

from sklearn.linear_model import ElasticNet
en = ElasticNet(alpha=1.0, l1_ratio=0.5)
en.fit(X_train, y_train)
y_train_pred = en.predict(X_train)
y_test_pred = en.predict(X_test)
print(en.coef_)


I am working on the file "Miscellaneous/Student-Performance-Evaluation-Classification-Regression.ipynb".
The first task is:
Naive Bayes accuracy and time elapsed caculation
Can you write Python code for it?

t4=time()
print ("NaiveBayes")
nb = BernoulliNB()
clf_nb=nb.fit(X_train,y_train)
print ("Acurracy: ", clf_nb.score(X_test,y_test))
t5=time()
print ("time elapsed: ", t5-t4)


I am working on the file "Miscellaneous/Student-Performance-Evaluation-Classification-Regression.ipynb".
The first task is:
Random Forests for Regression
Finally, let's try again Random Forests, in their Extra Trees, and Regression version
Can you write Python code for it?

from sklearn import ensemble
clf_et=ensemble.ExtraTreesRegressor(n_estimators=10,random_state=42)
train_and_evaluate(clf_et,X_train,y_train)


I am working on the file "Miscellaneous/Student-Performance-Evaluation-Classification-Regression.ipynb".
The first task is:
Random forest regression
Can you write Python code for it?

X = df[features].values
y = df['G3'].values

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=0)

from sklearn.ensemble import RandomForestRegressor

forest = RandomForestRegressor(n_estimators=1000,
                               criterion='mse',
                               random_state=1,
                               n_jobs=-1)
forest.fit(X_train, y_train)
y_train_pred = forest.predict(X_train)
y_test_pred = forest.predict(X_test)

print('MSE train: %.3f, test: %.3f' % (
        mean_squared_error(y_train, y_train_pred),
        mean_squared_error(y_test, y_test_pred)))
print('R^2 train: %.3f, test: %.3f' % (
        r2_score(y_train, y_train_pred),
        r2_score(y_test, y_test_pred)))

plt.scatter(y_train_pred,
            y_train_pred - y_train,
            c='black',
            marker='o',
            s=35,
            alpha=0.5,
            label='Training data')
plt.scatter(y_test_pred,
            y_test_pred - y_test,
            c='lightgreen',
            marker='s',
            s=35,
            alpha=0.7,
            label='Test data')

plt.xlabel('Predicted values')
plt.ylabel('Residuals')
plt.legend(loc='upper left')
plt.hlines(y=0, xmin=0, xmax=22, lw=2, color='red')
plt.xlim([0, 22])
plt.tight_layout()

# plt.savefig('./figures/slr_residuals.png', dpi=300)
plt.show()


I am working on the file "Miscellaneous/Student-Performance-Evaluation-Classification-Regression.ipynb".
The first task is:
Receiver Operating Characteristic (ROC) curve
Can you write Python code for it?

roc_auc_score(y_test,rf.predict(X_test))

%matplotlib inline
import matplotlib.pyplot as plt
from sklearn.metrics import roc_curve, auc

predictions = rf.predict_proba(X_test)

false_positive_rate, recall, thresholds = roc_curve(y_test, predictions[:, 1])
roc_auc = auc(false_positive_rate, recall)
plt.title('Receiver Operating Characteristic')
plt.plot(false_positive_rate, recall, 'b', label='AUC = %0.2f' % roc_auc)
plt.legend(loc='lower right')

plt.plot([0, 1], [0, 1], 'r--')
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.0])
plt.ylabel('Recall')
plt.xlabel('Fall-out')
plt.show()


I am working on the file "Miscellaneous/Student-Performance-Evaluation-Classification-Regression.ipynb".
The first task is:
SVM accuracy and time elapsed caculation
Can you write Python code for it?

t7=time()
print ("SVM")

svc = SVC()
clf_svc=svc.fit(X_train, y_train)
print ("Acurracy: ", clf_svc.score(X_test,y_test) )
t8=time()
print ("time elapsed: ", t8-t7)


I am working on the file "Miscellaneous/Student-Performance-Evaluation-Classification-Regression.ipynb".
The first task is:
Similiarly Ridge regression can be used:
Can you write Python code for it?

from sklearn.linear_model import Ridge
ridge = Ridge(alpha=1.0)
ridge.fit(X_train, y_train)
y_train_pred = ridge.predict(X_train)
y_test_pred = ridge.predict(X_test)
print(ridge.coef_)


I am working on the file "Miscellaneous/Student-Performance-Evaluation-Classification-Regression.ipynb".
The first task is:
Support Vector Machines for regression
The regression version of SVM can be used instead to find the hyperplane (note how easy is to change the classification method in scikit-learn!). We will try a linear kernel, a polynomial kernel, and finally, a rbf kernel. For more information on kernels, see http://scikit-learn.org/stable/modules/svm.html#svm-kernels
Can you write Python code for it?

from sklearn import svm
clf_svr= svm.SVR(kernel='linear')
train_and_evaluate(clf_svr,X_train,y_train)

clf_svr_poly= svm.SVR(kernel='poly')
train_and_evaluate(clf_svr_poly,X_train,y_train)

clf_svr_rbf= svm.SVR(kernel='rbf')
train_and_evaluate(clf_svr_rbf,X_train,y_train)

clf_svr_poly2= svm.SVR(kernel='poly',degree=2)
train_and_evaluate(clf_svr_poly2,X_train,y_train)


I am working on the file "Miscellaneous/Student-Performance-Evaluation-Classification-Regression.ipynb".
The first task is:
Tuning our hyperparameters using GridSearch
Can you write Python code for it?

from sklearn.metrics import classification_report

pipeline = Pipeline([
    ('clf', DecisionTreeClassifier(criterion='entropy'))
])

parameters = {
    'clf__max_depth': (5, 25 , 50),
    'clf__min_samples_split': (1, 5, 10),
    'clf__min_samples_leaf': (1, 2, 3)
}

grid_search = GridSearchCV(pipeline, parameters, n_jobs=-1, verbose=1, scoring='f1')
grid_search.fit(X_train, y_train)

print 'Best score: %0.3f' % grid_search.best_score_
print 'Best parameters set:'

best_parameters = grid_search.best_estimator_.get_params()
for param_name in sorted(parameters.keys()):
    print '\t%s: %r' % (param_name, best_parameters[param_name])

predictions = grid_search.predict(X_test)

print classification_report(y_test, predictions)


I am working on the file "Miscellaneous/Student-Performance-Evaluation-Classification-Regression.ipynb".
The first task is:
Using regularized methods for regression
A Lasso Regression model can be initialized as follows:
Can you write Python code for it?

from sklearn.linear_model import Lasso
lasso = Lasso(alpha=0.1)
lasso.fit(X_train, y_train)
y_train_pred = lasso.predict(X_train)
y_test_pred = lasso.predict(X_test)
print(lasso.coef_)

print('MSE train: %.3f, test: %.3f' % (
        mean_squared_error(y_train, y_train_pred),
        mean_squared_error(y_test, y_test_pred)))
print('R^2 train: %.3f, test: %.3f' % (
        r2_score(y_train, y_train_pred),
        r2_score(y_test, y_test_pred)))


I am working on the file "Miscellaneous/Student-Performance-Evaluation-Classification-Regression.ipynb".
The first task is:
cross validation for DT
Can you write Python code for it?

tt0=time()
print ("cross result========")
scores = cross_validation.cross_val_score(dt, X,y, cv=5)
print (scores)
print (scores.mean())
tt1=time()
print ("time elapsed: ", tt1-tt0)


I am working on the file "Miscellaneous/Student-Performance-Evaluation-Classification-Regression.ipynb".
The first task is:
cross validation for KNN
Can you write Python code for it?

tt0=time()
print ("cross result========")
scores = cross_validation.cross_val_score(knn, X,y, cv=5)
print (scores)
print (scores.mean())
tt1=time()
print ("time elapsed: ", tt1-tt0)


I am working on the file "Miscellaneous/Student-Performance-Evaluation-Classification-Regression.ipynb".
The first task is:
cross validation for RF
Can you write Python code for it?

tt0=time()
print ("cross result========")
scores = cross_validation.cross_val_score(rf, X,y, cv=5)
print (scores)
print (scores.mean())
tt1=time()
print ("time elapsed: ", tt1-tt0)


I am working on the file "Miscellaneous/Student-Performance-Evaluation-Classification-Regression.ipynb".
The first task is:
cross validation for SVM
Can you write Python code for it?

tt0=time()
print ("cross result========")
scores = cross_validation.cross_val_score(svc, X,y, cv=5)
print (scores)
print (scores.mean())
tt1=time()
print ("time elapsed: ", tt1-tt0)

from sklearn.svm import SVC
from sklearn.cross_validation import cross_val_score
from sklearn.pipeline import Pipeline
from sklearn import grid_search

svc = SVC()

parameters = {'kernel':('linear', 'rbf'), 'C':[1, 10]}

grid = grid_search.GridSearchCV(svc, parameters, n_jobs=-1, verbose=1, scoring='accuracy')

grid.fit(X_train, y_train)

print 'Best score: %0.3f' % grid.best_score_

print 'Best parameters set:'
best_parameters = grid.best_estimator_.get_params()

for param_name in sorted(parameters.keys()):
    print '\t%s: %r' % (param_name, best_parameters[param_name])

predictions = grid.predict(X_test)
print classification_report(y_test, predictions)

pipeline = Pipeline([
    ('clf', SVC(kernel='rbf', gamma=0.01, C=100))
])

parameters = {
    'clf__gamma': (0.01, 0.03, 0.1, 0.3, 1),
    'clf__C': (0.1, 0.3, 1, 3, 10, 30),
}

grid_search = GridSearchCV(pipeline, parameters, n_jobs=-1, verbose=1, scoring='accuracy')

grid_search.fit(X_train, y_train)

print 'Best score: %0.3f' % grid_search.best_score_

print 'Best parameters set:'
best_parameters = grid_search.best_estimator_.get_params()

for param_name in sorted(parameters.keys()):
    print '\t%s: %r' % (param_name, best_parameters[param_name])

predictions = grid_search.predict(X_test)
print classification_report(y_test, predictions)


I am working on the file "Miscellaneous/Student-Performance-Evaluation-Classification-Regression.ipynb".
The first task is:
cross-validation for NB
Can you write Python code for it?

tt0=time()
print ("cross result========")
scores = cross_validation.cross_val_score(nb, X,y, cv=5)
print (scores)
print (scores.mean())
tt1=time()
print ("time elapsed: ", tt1-tt0)


I am working on the file "Miscellaneous/Tic-Tac-Toe Endgame Classification.ipynb".
The first task is:
After we have installed GraphViz on our computer, we can convert the tree.dot file into a PNG file by executing the following command from the command line in the location where we saved the tree.dot file:

dot -Tpng tree.dot -o tree.png

Random Forest accuracy and time elapsed caculation
Can you write Python code for it?

t2=time()
print "RandomForest"
rf = RandomForestClassifier(n_estimators=100,n_jobs=-1)
clf_rf = rf.fit(X_train,y_train)
print "Acurracy: ", clf_rf.score(X_test,y_test)
t3=time()
print "time elapsed: ", t3-t2


I am working on the file "Miscellaneous/Tic-Tac-Toe Endgame Classification.ipynb".
The first task is:
Decision Tree accuracy and time elapsed caculation
Can you write Python code for it?

t0=time()
print "DecisionTree"

dt = DecisionTreeClassifier(min_samples_split=20,random_state=99)
# dt = DecisionTreeClassifier(min_samples_split=20,max_depth=5,random_state=99)

clf_dt=dt.fit(X_train,y_train)

print "Acurracy: ", clf_dt.score(X_test,y_test)
t1=time()
print "time elapsed: ", t1-t0


I am working on the file "Miscellaneous/Tic-Tac-Toe Endgame Classification.ipynb".
The first task is:
Feature importances with forests of trees
This examples shows the use of forests of trees to evaluate the importance of features on an artificial classification task. The red bars are the feature importances of the forest, along with their inter-trees variability.
Can you write Python code for it?

%matplotlib inline
import numpy as np
import matplotlib.pyplot as plt

from sklearn.ensemble import ExtraTreesClassifier

# Build a classification task using 3 informative features

# Build a forest and compute the feature importances
forest = ExtraTreesClassifier(n_estimators=250,
                              random_state=0)

forest.fit(X, y)
importances = forest.feature_importances_
std = np.std([tree.feature_importances_ for tree in forest.estimators_],
             axis=0)
indices = np.argsort(importances)[::-1]

# Print the feature ranking
print("Feature ranking:")

for f in range(X.shape[1]):
    print("%d. feature %d - %s (%f) " % (f + 1, indices[f], features[indices[f]], importances[indices[f]]))

# Plot the feature importances of the forest
plt.figure(num=None, figsize=(14, 10), dpi=80, facecolor='w', edgecolor='k')
plt.title("Feature importances")
plt.bar(range(X.shape[1]), importances[indices],
       color="r", yerr=std[indices], align="center")
plt.xticks(range(X.shape[1]), indices)
plt.xlim([-1, X.shape[1]])
plt.show()

importances[indices[:5]]

for f in range(5):
    print("%d. feature %d - %s (%f)" % (f + 1, indices[f], features[indices[f]] ,importances[indices[f]]))

best_features = []
for i in indices[:5]:
    best_features.append(features[i])

# Plot the top 5 feature importances of the forest
plt.figure(num=None, figsize=(8, 6), dpi=80, facecolor='w', edgecolor='k')
plt.title("Feature importances")
plt.bar(range(5), importances[indices][:5],
       color="r",  yerr=std[indices][:5], align="center")
plt.xticks(range(5), best_features)
plt.xlim([-1, 5])
plt.show()


I am working on the file "Miscellaneous/Tic-Tac-Toe Endgame Classification.ipynb".
The first task is:
Fine tuning the model using GridSearch
Can you write Python code for it?

from sklearn.svm import SVC
from sklearn.cross_validation import cross_val_score
from sklearn.pipeline import Pipeline
from sklearn import grid_search

knn = KNeighborsClassifier()

parameters = {'n_neighbors':[1,10]}

grid = grid_search.GridSearchCV(knn, parameters, n_jobs=-1, verbose=1, scoring='accuracy')

grid.fit(X_train, y_train)

print 'Best score: %0.3f' % grid.best_score_

print 'Best parameters set:'
best_parameters = grid.best_estimator_.get_params()

for param_name in sorted(parameters.keys()):
    print '\t%s: %r' % (param_name, best_parameters[param_name])

predictions = grid.predict(X_test)
print classification_report(y_test, predictions)


I am working on the file "Miscellaneous/Tic-Tac-Toe Endgame Classification.ipynb".
The first task is:
KNN accuracy and time elapsed caculation
Can you write Python code for it?

t6=time()
print "KNN"
# knn = KNeighborsClassifier(n_neighbors=3)
knn = KNeighborsClassifier(n_neighbors=3)
clf_knn=knn.fit(X_train, y_train)
print "Acurracy: ", clf_knn.score(X_test,y_test)
t7=time()
print "time elapsed: ", t7-t6


I am working on the file "Miscellaneous/Tic-Tac-Toe Endgame Classification.ipynb".
The first task is:
SVM accuracy and time elapsed caculation
Can you write Python code for it?

t7=time()
print "SVM"

svc = SVC()
clf_svc=svc.fit(X_train, y_train)
print "Acurracy: ", clf_svc.score(X_test,y_test)
t8=time()
print "time elapsed: ", t8-t7


I am working on the file "Miscellaneous/Tic-Tac-Toe Endgame Classification.ipynb".
The first task is:
Tuning Models using GridSearch
Can you write Python code for it?

pipeline2 = Pipeline([
('clf', RandomForestClassifier(criterion='entropy'))
])

parameters = {
    'clf__n_estimators': (5, 25, 50, 100),
    'clf__max_depth': (5, 25 , 50),
    'clf__min_samples_split': (1, 5, 10),
    'clf__min_samples_leaf': (1, 2, 3)
}

grid_search = GridSearchCV(pipeline2, parameters, n_jobs=-1, verbose=1, scoring='accuracy', cv=3)

grid_search.fit(X_train, y_train)

print 'Best score: %0.3f' % grid_search.best_score_

print 'Best parameters set:'
best_parameters = grid_search.best_estimator_.get_params()

for param_name in sorted(parameters.keys()):
    print '\t%s: %r' % (param_name, best_parameters[param_name])

predictions = grid_search.predict(X_test)
print 'Accuracy:', accuracy_score(y_test, predictions)
print classification_report(y_test, predictions)


I am working on the file "Miscellaneous/Tic-Tac-Toe Endgame Classification.ipynb".
The first task is:
Tuning our hyperparameters using GridSearch
Can you write Python code for it?

from sklearn.metrics import classification_report

pipeline = Pipeline([
    ('clf', DecisionTreeClassifier(criterion='entropy'))
])

parameters = {
    'clf__max_depth': (5, 25 , 50),
    'clf__min_samples_split': (1, 5, 10),
    'clf__min_samples_leaf': (1, 2, 3)
}

grid_search = GridSearchCV(pipeline, parameters, n_jobs=-1, verbose=1, scoring='f1')
grid_search.fit(X_train, y_train)

print 'Best score: %0.3f' % grid_search.best_score_
print 'Best parameters set:'

best_parameters = grid_search.best_estimator_.get_params()
for param_name in sorted(parameters.keys()):
    print '\t%s: %r' % (param_name, best_parameters[param_name])

predictions = grid_search.predict(X_test)

print classification_report(y_test, predictions)

t0=time()
print "DecisionTree"

dt1 = DecisionTreeClassifier(min_samples_split=1,max_depth=25,min_samples_leaf=1, random_state=99)

clf_dt1=dt1.fit(X_train,y_train)

print "Acurracy: ", clf_dt1.score(X_test,y_test)
t1=time()
print "time elapsed: ", t1-t0

export_graphviz(clf_dt1,
                out_file='tree.dot',
                feature_names=features)


I am working on the file "Miscellaneous/Tic-Tac-Toe Endgame Classification.ipynb".
The first task is:
cross validation for DT
Can you write Python code for it?

tt0=time()
print "cross result========"
scores = cross_validation.cross_val_score(dt, X, y, cv=3)
print scores
print scores.mean()
tt1=time()
print "time elapsed: ", tt1-tt0
print "\n"


I am working on the file "Miscellaneous/Tic-Tac-Toe Endgame Classification.ipynb".
The first task is:
cross validation for KNN
Can you write Python code for it?

tt6=time()
print "cross result========"
scores = cross_validation.cross_val_score(knn, X,y, cv=5)
print scores
print scores.mean()
tt7=time()
print "time elapsed: ", tt7-tt6
print "\n"


I am working on the file "Miscellaneous/Tic-Tac-Toe Endgame Classification.ipynb".
The first task is:
cross validation for RF
Can you write Python code for it?

tt2=time()
print "cross result========"
scores = cross_validation.cross_val_score(rf, X, y, cv=3)
print scores
print scores.mean()
tt3=time()
print "time elapsed: ", tt3-tt2
print "\n"


I am working on the file "Miscellaneous/Tic-Tac-Toe Endgame Classification.ipynb".
The first task is:
cross validation for SVM
Can you write Python code for it?

tt7=time()
print "cross result========"
scores = cross_validation.cross_val_score(svc,X,y, cv=5)
print scores
print scores.mean()
tt8=time()
print "time elapsed: ", tt7-tt6
print "\n"

from sklearn.svm import SVC
from sklearn.cross_validation import cross_val_score
from sklearn.pipeline import Pipeline
from sklearn import grid_search

svc = SVC()

parameters = {'kernel':('linear', 'rbf'), 'C':[1, 10]}

grid = grid_search.GridSearchCV(svc, parameters, n_jobs=-1, verbose=1, scoring='accuracy')

grid.fit(X_train, y_train)

print 'Best score: %0.3f' % grid.best_score_

print 'Best parameters set:'
best_parameters = grid.best_estimator_.get_params()

for param_name in sorted(parameters.keys()):
    print '\t%s: %r' % (param_name, best_parameters[param_name])

predictions = grid.predict(X_test)
print classification_report(y_test, predictions)

pipeline = Pipeline([
    ('clf', SVC(kernel='linear', gamma=0.01, C=10))
])

parameters = {
    'clf__gamma': (0.01, 0.03, 0.1, 0.3, 1),
    'clf__C': (0.1, 0.3, 1, 3, 10, 30),
}

grid_search = GridSearchCV(pipeline, parameters, n_jobs=-1, verbose=1, scoring='accuracy')

grid_search.fit(X_train, y_train)

print 'Best score: %0.3f' % grid_search.best_score_

print 'Best parameters set:'
best_parameters = grid_search.best_estimator_.get_params()

for param_name in sorted(parameters.keys()):
    print '\t%s: %r' % (param_name, best_parameters[param_name])

predictions = grid_search.predict(X_test)
print classification_report(y_test, predictions)


I am working on the file "Miscellaneous/Tic-Tac-Toe Endgame Classification.ipynb".
The first task is:
cross-validation for NB
Can you write Python code for it?

tt4=time()
print "cross result========"
scores = cross_validation.cross_val_score(nb, X,y, cv=3)
print scores
print scores.mean()
tt5=time()
print "time elapsed: ", tt5-tt4
print "\n"


I am working on the file "Miscellaneous/Topic Modelling using LDA.ipynb".
The first task is:
Displaying and Evaluating Topics
The structure of the resulting matrices returned by both NMF and LDA is the same and the Scikit Learn interface to access the returned matrices is also the same. This is great and allows for a common Python method that is able to display the top words in a topic. Topics are not labeled by the algorithm — a numeric index is assigned.
Can you write Python code for it?

def display_topics(model, feature_names, no_top_words):
    for topic_idx, topic in enumerate(model.components_):
        print ("Topic %d:" % (topic_idx))
        print (" ".join([feature_names[i]
                        for i in topic.argsort()[:-no_top_words - 1:-1]]))

no_top_words = 10
display_topics(nmf, tfidf_feature_names, no_top_words)


I am working on the file "Miscellaneous/Topic Modelling using LDA.ipynb".
The first task is:
The creation of the bag of words matrix is very easy in Scikit Learn — all the heavy lifting is done by the feature extraction functionality provided for text datasets. A tf-idf transformer is applied to the bag of words matrix that NMF must process with the TfidfVectorizer. LDA on the other hand, being a probabilistic graphical model (i.e. dealing with probabilities) only requires raw counts, so a CountVectorizer is used. Stop words are removed and the number of terms included in the bag of words matrix is restricted to the top 1000.
Can you write Python code for it?

from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer

no_features = 1000

# NMF is able to use tf-idf
tfidf_vectorizer = TfidfVectorizer(max_df=0.95, min_df=2, max_features=no_features, stop_words='english')
tfidf = tfidf_vectorizer.fit_transform(documents)
tfidf_feature_names = tfidf_vectorizer.get_feature_names()

# LDA can only use raw term counts for LDA because it is a probabilistic graphical model
tf_vectorizer = CountVectorizer(max_df=0.95, min_df=2, max_features=no_features, stop_words='english')
tf = tf_vectorizer.fit_transform(documents)
tf_feature_names = tf_vectorizer.get_feature_names()


I am working on the file "Miscellaneous/Topic Modelling using LDA.ipynb".
The first task is:
This was using NMF
And now using LDA:
Can you write Python code for it?


I am working on the file "MissingValue_Outlier_exploratory.ipynb".
The first task is:
After setting up the standard range of outliers, we lost at most 600 points for each variable in all_encounter_data. (And set the times of IQR from 1.5 to 2 does not really remain as many points as I was expecting. So I choose the standard times of 1.5)
And after removing the outliers, the variables seem a lot more normal distributed.
Can you write Python code for it?

import matplotlib.pyplot as plt
for column in quantitive_columns:
    f, (ax1, ax2) = plt.subplots(1, 2, sharey=True)

    temp0 = pd.to_numeric(d_enc[column], errors='coerce')
    ax1.hist(temp0[temp0.notnull()])
    ax1.set_title('before')

    temp = temp0[temp0.notnull()]

    Q2 = temp.quantile(0.75)
    Q1 = temp.quantile(0.25)
    IQR = Q2-Q1

    d_enc[column] = temp0.map(lambda x: x if Q1 - 1.5 * IQR < x < Q2 + 1.5 * IQR else NaN)
    ax2.hist(d_enc[column][d_enc[column].notnull()])
    ax2.set_title('after')

    f.suptitle(column)

plt.show()


I am working on the file "MissingValue_Outlier_exploratory.ipynb".
The first task is:
Decision Tree modeling exploration
Can you write Python code for it?

from sklearn import tree
from sklearn.model_selection import train_test_split
from sklearn import metrics
from sklearn.metrics import confusion_matrix

for method, temp in person_data_fillup.items():
    print(method)

    X = temp[quantitive_columns + dummy_columns]
    y = temp[target_columns[method]]
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)

    clf = tree.DecisionTreeClassifier()
    clf = clf.fit(X_train, y_train)

    preds = clf.predict(X = X_test)

    #preds = label_encoder.inverse_transform(preds.tolist())
    #y_test = label_encoder.inverse_transform(y_test)

    print(pd.crosstab(y_test, preds))
    print(metrics.classification_report(y_true = y_test, y_pred=preds))

tree.export_graphviz(clf, feature_names = quantitive_columns + dummy_columns,
                     class_names = ['MNPDR','PDR','SNPDR','mNPDR','no_DR'], out_file='DT.dot')


I am working on the file "MissingValue_Outlier_exploratory.ipynb".
The first task is:
Fill up the missing values
with the mean of the non-null variable grouped by DR diagnosis
Can you write Python code for it?

DR_diagnoses = ['PDR', 'SNPDR', 'MNPDR', 'mNPDR', 'no_DR']

temp = person_data_new.copy()
for column in quantitive_columns:
    mean_value = temp.groupby('recent_DR').apply(lambda x: x[column][x[column].notnull()].mean())
    missing_index = temp.groupby('recent_DR').apply(lambda x: x[column][x[column].isnull()])
    for diagnosis in DR_diagnoses:
        temp.set_value(missing_index[diagnosis].index, column, mean_value[diagnosis])
person_data_fillup['recent_groupbyDR_mean'] = temp

temp = person_data_new.copy()
for column in quantitive_columns:
    mean_value = temp.groupby('worst_DR').apply(lambda x: x[column][x[column].notnull()].mean())
    missing_index = temp.groupby('worst_DR').apply(lambda x: x[column][x[column].isnull()])
    for diagnosis in DR_diagnoses:
        temp.set_value(missing_index[diagnosis].index, column, mean_value[diagnosis])
person_data_fillup['worst_groupbyDR_mean'] = temp


I am working on the file "MissingValue_Outlier_exploratory.ipynb".
The first task is:
Fill up the missing values
with the mean of the non-null variable grouped by age group
Can you write Python code for it?

person_data_fillup = {}

temp = person_data_new.copy()
for column in quantitive_columns:
    mean_value = temp.groupby('Age_group').apply(
        lambda x: x[column][x[column].notnull()].mean())
    missing_index = temp.groupby('Age_group').apply(
        lambda x: x[column][x[column].isnull()])
    for i in mean_value.index.to_series().tolist():
        if i in missing_index.index:
            temp.set_value(missing_index[i].index, column, mean_value[i])
person_data_fillup['groupbyAgegroup_mean'] = temp


I am working on the file "MissingValue_Outlier_exploratory.ipynb".
The first task is:
First read in the original data
Can you write Python code for it?

import re
data = pd.read_pickle(os.getcwd() + '/data/all_encounter_data.pickle')


I am working on the file "MissingValue_Outlier_exploratory.ipynb".
The first task is:
From above we can tell, after I identify the outliers, no more than 4% points of each variable are removed, which is acceptable for me.
Can you write Python code for it?

plt.bar(range(0,5),
        person_data_new[quantitive_columns].isnull().sum(axis=0)/person_data_new.shape[0])
plt.gca().set_ylim([0,1])
plt.xticks(range(0,5), quantitive_columns)
plt.ylabel('Missing value percentage')
plt.xlabel('Quantative variables')
plt.show()


I am working on the file "MissingValue_Outlier_exploratory.ipynb".
The first task is:
Get the dummy value for the categorical features
Can you write Python code for it?

dummy_columns = ['DM', 'ME', 'Glaucoma_Suspect', 'Open_angle_Glaucoma', 'Cataract']
categorical_columns = ['Gender', 'Race', 'recent_smoking_status', 'family_DM', 'family_G']
for column in categorical_columns:
    temp = pd.get_dummies(person_data_new[column], prefix=column)
    person_data_new[temp.columns.values]=temp
    dummy_columns.extend(temp.columns.values.tolist())


I am working on the file "MissingValue_Outlier_exploratory.ipynb".
The first task is:
Group the quantitive features by Age and Gender
But there are still null values within the grouped mean values.
Can you write Python code for it?

temp = person_data_new.copy()
mean_value = temp.groupby(['Gender', pd.cut(temp['Age'], 6)]).apply(
    lambda x: x['A1C'][x['A1C'].notnull()].mean())
missing_index = temp.groupby(['Gender', pd.cut(temp['Age'], 6)]).apply(
    lambda x: x['A1C'][x['A1C'].isnull()])
for i in mean_value.index.to_series().tolist():
    if i in missing_index.index:
        temp.set_value(missing_index[i].index, 'A1C', mean_value[i])

mean_value

temp[temp['A1C'].isnull()].shape[0]


I am working on the file "MissingValue_Outlier_exploratory.ipynb".
The first task is:
Group the quantitive features by DR diagnosis
The missing value percentage in different diagnosis as following:
Can you write Python code for it?

person_data_new.groupby('recent_DR').apply(lambda x: x.shape[0])

person_data_new.groupby('recent_DR').apply(lambda x: x[quantitive_columns].isnull().sum(axis=0)/x.shape[0])

person_data_new.groupby('worst_DR').apply(lambda x: x.shape[0])

person_data_new.groupby('worst_DR').apply(lambda x: x[quantitive_columns].isnull().sum(axis=0)/x.shape[0])


I am working on the file "MissingValue_Outlier_exploratory.ipynb".
The first task is:
Implement ANOVA test
To test if the mean values of the each feature are equal after being grouped
Can you write Python code for it?

from scipy.stats import f_oneway

for column in quantitive_columns:
    temp = {k:list(v[column]) for k,v in person_data_new[person_data_new[column].notnull()].groupby('Age_group_numeric')}
    print column
    print f_oneway(temp[1], temp[2], temp[3], temp[4], temp[5], temp[6])

for column in quantitive_columns:
    temp = {k:list(v[column]) for k,v in person_data_new[person_data_new[column].notnull()].groupby('Gender')}
    print column
    print f_oneway(temp['F'], temp['M'])


I am working on the file "MissingValue_Outlier_exploratory.ipynb".
The first task is:
Now use 04/15 processed person data (added new features)
Can you write Python code for it?

person_data_new = pd.read_pickle(os.getcwd() + '/data/all_person_data_Dan_20170415.pickle')

person_data_new.columns.values

quantitive_columns = ["A1C", "BMI", "Glucose", "BP_Systolic", "BP_Diastolic",
                        'MR_OD_SPH_Numeric', 'MR_OD_CYL_Numeric',
                        'MR_OS_SPH_Numeric', 'MR_OS_CYL_Numeric',
                        'MR_OS_DVA_ability', 'MR_OD_DVA_ability',
                        'MR_OS_NVA_ability', 'MR_OD_NVA_ability']


I am working on the file "MissingValue_Outlier_exploratory.ipynb".
The first task is:
Output the filled up data
Can you write Python code for it?

temp = person_data_fillup['groupbyAgegroup_mean'][quantitive_columns + dummy_columns + ['worst_DR', 'recent_DR']]
temp.describe(include='all')

#temp.to_pickle('baseline_missingHandled_Dan_20170406.pickle')
temp.to_pickle('Morefeatures_missingHandled_Dan_20170415.pickle')

temp = person_data_new[quantitive_columns + dummy_columns + ['worst_DR', 'recent_DR']]
temp.describe(include='all')

#temp.to_pickle('baseline_raw_Dan_20170406.pickle')
temp.to_pickle('Morefeatures_raw_Dan_20170415.pickle')


I am working on the file "MissingValue_Outlier_exploratory.ipynb".
The first task is:
What about we crush all_encounter_data to all_person_data by group Person_Nbr?
Can you write Python code for it?

person_data_old = pd.read_pickle(os.getcwd() + '/data/all_person_data_Richard_20170307.pickle')
person_data_new = pd.read_pickle(os.getcwd() + '/data/all_person_data_Dan_20170406.pickle')

person_data_old[quantitive_columns].isnull().sum(axis=0)/person_data_old.shape[0]

person_data_new[quantitive_columns].isnull().sum(axis=0)/person_data_new.shape[0]


I am working on the file "MissingValue_Outlier_exploratory.ipynb".
The first task is:
repeat the processing with all_encounter_data in ICO.py
Can you write Python code for it?

d_enc = data.drop(["Enc_ID","Person_ID"], axis=1)

pattern0= re.compile("\d+\s*\/\s*\d+")
index1 = d_enc['Glucose'].str.contains(pattern0, na=False)
temp = d_enc.loc[index1, 'Glucose']
d_enc.loc[index1, 'Glucose'] = d_enc.loc[index1, 'BP']
d_enc.loc[index1, 'BP'] = temp

index2 = d_enc.BP[d_enc.BP.notnull()][~d_enc.BP[d_enc.BP.notnull()].str.contains('/')].index
temp = d_enc.loc[index2, 'Glucose']
d_enc.loc[index2, 'Glucose'] = d_enc.loc[index2, 'BP']
d_enc.loc[index2, 'BP'] = temp

# Split up the BP field into Systolic and Diastolic readings
pattern1 = re.compile("(?P<BP_Systolic>\d+)\s*\/\s*(?P<BP_Diastolic>\d+)")
d_enc = pd.merge(d_enc, d_enc["BP"].str.extract(pattern1, expand=True),
                 left_index=True, right_index=True).drop("BP", axis=1)

# Define ranges for reasonable values. Identify the data outside of 1.5 times of IQR as outliers
NaN = float("NaN")
quantitive_columns=['A1C', 'BMI', 'Glucose', 'BP_Diastolic', 'BP_Systolic']

for column in quantitive_columns:
    d_enc[column] = pd.to_numeric(d_enc[column], errors='coerce')
    temp = d_enc[column][d_enc[column].notnull()]

    Q2 = temp.quantile(0.75)
    Q1 = temp.quantile(0.25)
    IQR = Q2-Q1

    print(temp[Q1 - 2 * IQR < temp][temp[Q1 - 2 * IQR < temp] < Q2 + 2 * IQR].shape[0], temp.shape[0], d_enc.shape[0])
    print(column, Q1 - 2 * IQR, Q2 + 2 * IQR)

for column in quantitive_columns:
    d_enc[column] = pd.to_numeric(d_enc[column], errors='coerce')
    temp = d_enc[column][d_enc[column].notnull()]

    Q2 = temp.quantile(0.75)
    Q1 = temp.quantile(0.25)
    IQR = Q2-Q1

    print(temp[Q1 - 1.5 * IQR < temp][temp[Q1 - 1.5 * IQR < temp] < Q2 + 1.5 * IQR].shape[0], temp.shape[0], d_enc.shape[0])
    print(column, Q1 - 1.5 * IQR, Q2 + 1.5 * IQR)


I am working on the file "Mixmatch/streetview_dataset/OCR_lib_test.ipynb".
The first task is:
Test for function word_to_vec
Given a valid word string and make sure the output array of embeddings is the same as the ground truth.
Note: The word_to_vec will ignore words that are not in the dictionary. So the TEST_STRING should be a sentence with words that are in the English vocabulary.
Can you write Python code for it?

TEST_STRING = "Test string"

word_embedding = word_to_vec("Test")
nlp = spacy.load("en_core_web_lg")
ground_truth_embedding = nlp("Test")

assert ground_truth_embedding.vector.all() == word_embedding.all()


I am working on the file "Mixmatch/streetview_dataset/OCR_lib_test.ipynb".
The first task is:
Test for reshape_embeddings()
Test the reshape function with random generated arrays
Can you write Python code for it?

DEFAULT_SHAPE = (64, 64, 1)
TEST_ARRAYS = [np.random.rand(0), np.random.rand(10), np.random.rand(1000)]

for array in TEST_ARRAYS:
    new_embedding = reshape_embeddings(array)
    assert new_embedding.numpy().shape == DEFAULT_SHAPE


I am working on the file "Mixmatch/streetview_dataset/parse_data_to_tfrecord_lib_test.ipynb".
The first task is:
Read back the generated tfrecords and check if the data stored inside the file meets the expectation.
Can you write Python code for it?

# Read the files back from the generated tfrecords
def parse_tf_records(file_dir):
    raw_image_dataset = tf.data.TFRecordDataset(file_dir)

    # Create a dictionary describing the features.
    image_feature_description = {
        'label': tf.io.FixedLenFeature([], tf.int64),
        'image': tf.io.FixedLenFeature([], tf.string),
    }

    def _parse_image_function(example_proto):
      # Parse the input tf.Example proto using the dictionary above.
      return tf.io.parse_single_example(example_proto, image_feature_description)

    parsed_image_dataset = raw_image_dataset.map(_parse_image_function)

    return parsed_image_dataset

parsed_image_dataset = parse_tf_records(TF_FILE_DIR)

for image_features in parsed_image_dataset:
    print(int(image_features['label']))
    image_raw = image_features['image'].numpy()
    display.display(display.Image(data=image_raw))


I am working on the file "Mixmatch/streetview_dataset/parse_data_to_tfrecord_lib_test.ipynb".
The first task is:
Test function img_to_example():
The function reads in image files and outputs tf.examples. The output should store the information feeded into the function.
Can you write Python code for it?

IMG_PATH = './TC11/svt1/img/19_00.jpg'

features={'image': tf.FixedLenFeature([], tf.string),
          'label': tf.FixedLenFeature([], tf.int64)}

try:
    img = Image.open(IMG_PATH, "r")
except Exception as e:
    print(e)
    print(IMG_PATH + " is not valid")

example = img_to_example(img, label=0)
features = tf.io.parse_single_example(example.SerializeToString(), features)

# Testing
# The label feature should be value of 0
assert features['label'].numpy() == 0

# The pixel values of the original image and the stored image should be the same
decode_image = tf.image.decode_image(features['image']).numpy()
original_image = np.array(img.getdata())
assert decode_image.flatten().all() == original_image.flatten().all()


I am working on the file "Mixmatch/streetview_dataset/parse_data_to_tfrecord_main.ipynb".
The first task is:
streetview_v1
Run this part of code to build streetview_v1 dataset
Can you write Python code for it?

# name format of the output tfrecord files
OUTPUT_TRAIN_RECORD_FILENAME = "streetview-train.tfrecord"
OUTPUT_TEST_RECORD_FILENAME = "streetview-test.tfrecord"

# tfrecord file writer
train_writer = tf.io.TFRecordWriter(OUTPUT_RECORD_DIR + OUTPUT_TRAIN_RECORD_FILENAME)
test_writer = tf.io.TFRecordWriter(OUTPUT_RECORD_DIR + OUTPUT_TEST_RECORD_FILENAME)

# Write train and test tfrecord from TC11 dataset
train_image_path = os.path.join(INPUT_TC11_IMG_DIR, 'train')
test_image_path = os.path.join(INPUT_TC11_IMG_DIR, 'test')

write_tfrecord_from_images(train_image_path, 1, train_writer)
write_tfrecord_from_images(test_image_path, 1, test_writer)

# Write train and test tfrecords from UCF dataset
detection_property = {'include_top_camera':True, 'only_keep_top_confidence':True, 'balance':False}
train_batch_range, test_batch_range = [0, int(0.9 * INPUT_RECORD_CNT)], [int(0.9 * INPUT_RECORD_CNT), INPUT_RECORD_CNT]

batch_read_write_tfrecords(train_batch_range, INPUT_RECORD_DIR, INPUT_UCF_IMG_DIR, train_writer, detection_property)
batch_read_write_tfrecords(test_batch_range, INPUT_RECORD_DIR, INPUT_UCF_IMG_DIR, test_writer, detection_property)

train_writer.close()
test_writer.close()


I am working on the file "Mixmatch/streetview_dataset/parse_data_to_tfrecord_main.ipynb".
The first task is:
streetview_v2
Run this part of code to build streetview_v2 dataset
Can you write Python code for it?

# name format of the output tfrecord files
OUTPUT_TRAIN_RECORD_FILENAME = "streetview_v2_64-train.tfrecord"
OUTPUT_TEST_RECORD_FILENAME = "streetview_v2_64-test.tfrecord"

# tfrecord file writer
train_writer = tf.io.TFRecordWriter(OUTPUT_RECORD_DIR + OUTPUT_TRAIN_RECORD_FILENAME)
test_writer = tf.io.TFRecordWriter(OUTPUT_RECORD_DIR + OUTPUT_TEST_RECORD_FILENAME)

# Write train and test tfrecord for TC11 dataset
train_image_path = os.path.join(INPUT_TC11_img_dir, 'train')
test_image_path = os.path.join(INPUT_TC11_img_dir, 'test')

write_tfrecord_from_images(train_image_path, 1, train_writer)
write_tfrecord_from_images(test_image_path, 1, test_writer)

# Write train and test tfrecords from UCF dataset
detection_property = {'include_top_camera':False, 'only_keep_top_confidence':False, 'balance':False}
train_batch_range, test_batch_range = [0, int(0.9 * INPUT_RECORD_CNT)], [int(0.9 * INPUT_RECORD_CNT), INPUT_RECORD_CNT]

batch_read_write_tfrecords(train_batch_range, INPUT_RECORD_DIR, INPUT_UCF_IMG_DIR, train_writer, detection_property)
batch_read_write_tfrecords(test_batch_range, INPUT_RECORD_DIR, INPUT_UCF_IMG_DIR, test_writer, detection_property)

train_writer.close()
test_writer.close()


I am working on the file "Mixmatch/streetview_dataset/parse_data_to_tfrecord_main.ipynb".
The first task is:
streetview_v3
Run this part of code to build streetview_v3 dataset
Can you write Python code for it?

# Constant input and output path
# NOTE: The images within INPUT_UCF_FILTER_IMG_DIR will be handpicked from outputs of OUTPUT_UCF_IMG_DIR
INPUT_UCF_FILTER_IMG_DIR = './UCF_Streetview_Dataset/test/'
OUTPUT_UCF_IMG_DIR = './UCF_Streetview_Dataset/test_neg/'
OUTPUT_TEST_RECORD_FILENAME = 'streetview_v3_256-test.tfrecord'
OUTPUT_TRAIN_RECORD_FILENAME = 'streetview_v3_256-train.tfrecord'


I am working on the file "Mixmatch/streetview_dataset/parse_data_to_tfrecord_main.ipynb".
The first task is:
streetview_v3-test
This test set uses all TC11 images as positive cases, and handpick UCF images with confidence < 0.2 as negative cases.
Can you write Python code for it?

# Filter UCF images and copy files for furthur handpicking test set.
for i in range(int(0.9 * INPUT_RECORD_CNT), INPUT_RECORD_CNT):
    file_name = "./streetlearn_detections_tfexample-" + str(i).zfill(5) + "-of-01000.tfrecord"
    parsed_image_dataset = read_tfrecord(os.path.join(INPUT_RECORD_DIR, file_name))
    filter_image_with_confidence_threshold(parsed_image_dataset, INPUT_UCF_IMG_DIR, OUTPUT_UCF_IMG_DIR, 0.2)

# Build streetview_v3_64-test. Write test tfrecord from TC11 dataset and filtered UCF
test_writer = tf.io.TFRecordWriter(OUTPUT_RECORD_DIR + OUTPUT_TEST_RECORD_FILENAME)
test_image_path = os.path.join(INPUT_TC11_IMG_DIR, 'img')
write_tfrecord_from_images(test_image_path, 1, test_writer)
write_tfrecord_from_images(INPUT_UCF_FILTER_IMG_DIR, 0, test_writer)
test_writer.close()


I am working on the file "Mixmatch/streetview_dataset/parse_data_to_tfrecord_main.ipynb".
The first task is:
streetview_v3-train (Balanced UCF training set)
Can you write Python code for it?

train_writer = tf.io.TFRecordWriter(OUTPUT_RECORD_DIR + OUTPUT_TRAIN_RECORD_FILENAME)

# Write train and test tfrecords from UCF dataset
detection_property = {'include_top_camera':False, 'only_keep_top_confidence':False, 'balance':True}
train_batch_range = [0, int(0.9 * INPUT_RECORD_CNT)]

batch_read_write_tfrecords(train_batch_range, INPUT_RECORD_DIR, INPUT_UCF_IMG_DIR, train_writer, detection_property)
train_writer.close()


I am working on the file "Mixture of Simple Linear Regressions.ipynb".
The first task is:
Division by zero should be avoided
Let $ x = \frac{\pi_1  e^{-p}}{\pi_1  e^{-p} + \pi_2  e^{-q}} $
Now the problem is p and q can be both very large, and hence a 0 by 0 division.
$\log(x) = -\log(1+\frac{\pi_2 e^{p-q}}{\pi_1})$
Can you write Python code for it?

# Press ^Enter to run each iteration

# E Step: compute gammas (dividing directly causes 0 by 0 division. Go log)
p_ = -0.5/(sig*sig) * (np.dot(X, w1) - Y)**2
q_ = -0.5/(sig*sig) * (np.dot(X, w2) - Y)**2
g1 = np.exp(-np.log(1 + np.exp(p_ - q_)*p2/p1))
g2 = 1-g1 # np.exp(-np.log(1 + np.exp(q_ - p_)*p1/p2))

# M Step: recompute pi and w
p1 = np.sum(g1)/N
p2 = np.sum(g2)/N
assert (np.abs(p1+p2-1) < 0.000001), "Not normalized - sad face puppy face"

Z1 = np.diag(g1)
Z2 = np.diag(g2)
T1 = np.dot(np.transpose(X), Z1) # X'*Z1
T2 = np.dot(np.transpose(X), Z2) # X'*Z2
w1 = np.dot(np.linalg.pinv(np.dot(T1, X)), np.dot(T1, Y))
w2 = np.dot(np.linalg.pinv(np.dot(T2, X)), np.dot(T2, Y))

print w1, w2, p1, p2

plt.plot(seq, line1, 'ro')
plt.plot(seq, line2, 'b+')
plt.plot(seq, np.dot(x, w1))
plt.plot(seq, np.dot(x, w2))
plt.show()


I am working on the file "Mixture of Simple Linear Regressions.ipynb".
The first task is:
EM Algorithm
Run EM algorithm on the above dataset
E Step
$\gamma(z_{nk}) = \frac{\pi_k \mathcal{N}(y_n/w_k^{T}x_n, \sigma^2)} {\sum_{j=1}^K \pi_j \mathcal{N}(y_n/w_j^{T}x_n, \sigma^2)}$ where $z_{nk}$ indicates whether $n^{th}$ data point is generated by the $k^{th}$ line.
M Step
Update $\pi_k$, $w_k$ as follows;
$\pi_k = \frac{\sum_{i=1}^{N}\gamma(z_{nk})}{\sum_{i=1}^{N} \sum_{k=1}^{K}\gamma(z_{nk})} = \frac{\sum_{i=1}^{N}\gamma(z_{nk})}{N} = \frac{N_k}{N}$
$w_k = (X^T Z_k X)^{-1} X^T Z_k Y$, where $X$ is the design matrix, Y is the column vector of $y_n$ values and $Z$ is a diagonal matrix with $\gamma(z_{nk})$ on the diagonals.
Note that we are keeping $\sigma$ as a known fixed value for simplicity. But it can be estimated as well.
Can you write Python code for it?

# Constants
N = 2*n # total number of data points
X = np.vstack((x, x)) # Design Matrix;
Y = np.concatenate((line1, line2)) # Target vector;

# Initialize the parameters
p1 = np.random.rand()
p2 = 1-p1
w1 = np.array([np.random.normal(0, 5), np.random.normal(0,2)])
w2 = np.array([np.random.normal(0, 5), np.random.normal(0,2)])


I am working on the file "Mixture of Simple Linear Regressions.ipynb".
The first task is:
Generate data
line1 is $f_1(x) = w_1^{T}x$ and line2 is $f_2(x) = w_2^{T}x$
Generate data with gaussian noise around f(x) with std $\sigma^2$
Can you write Python code for it?

n = 20 # number of datapoints in each line
v1 = np.array([-13, 0.9]) # first line
v2 = np.array([7, -1]) # second line
sig = 1.0
seq = np.array(range(n))+1
x = np.transpose(np.array([np.ones(n), seq])) # Half of Design Matrix as there are two y's for each x

line1 = np.zeros(n)
line2 = np.zeros(n)
for i in range(n):
    line1[i] = np.random.normal(np.dot(v1, x[i]), sig)
    line2[i] = np.random.normal(np.dot(v2, x[i]), sig)

plt.plot(seq, line1, 'ro')
plt.plot(seq, line2, 'b+')
plt.show()


I am working on the file "Mixture of Simple Linear Regressions.ipynb".
The first task is:
Local Minima problem
Note that EM algorithm maynot converge to the required optimal minima always. Many times the convergence is to a local minimum.
K Means like algorithm
Next we will try an algorithm inspired by the K-Means. The algorithm is as follows: Randomly initialize two lines. Assign each point to one of the lines (hard assignment as opposed to EM above) based on the nearest distance to the lines. Then refit the lines to the assigned points.
Can you write Python code for it?

def closer_to_first(d, w1, w2):
    # d[0] is x and d[1] is y
    return 1 if (np.dot(w1, d[0])-d[1])**2 <= (np.dot(w2, d[0])-d[1])**2 else 0

# Initialization
w1 = np.array([np.random.normal(0, 5), np.random.normal(0,2)])
w2 = np.array([np.random.normal(0, 5), np.random.normal(0,2)])

# Assign each point to the nearest line
z1 = np.array([closer_to_first(d, w1, w2) for d in zip(X, Y)])
Z1 = np.diag(z1)
Z2 = np.diag(1-z1)
# recompute the equations of lines - same as M step in EM algorithm
T1 = np.dot(np.transpose(X), Z1) # X'*Z1
T2 = np.dot(np.transpose(X), Z2) # X'*Z2
w1 = np.dot(np.linalg.pinv(np.dot(T1, X)), np.dot(T1, Y))
w2 = np.dot(np.linalg.pinv(np.dot(T2, X)), np.dot(T2, Y))

print w1, w2
plt.plot(seq, line1, 'ro')
plt.plot(seq, line2, 'b+')
plt.plot(seq, np.dot(x, w1))
plt.plot(seq, np.dot(x, w2))
plt.show()


I am working on the file "Mixture of Simple Linear Regressions.ipynb".
The first task is:
What if we minimize the perpendicular distance to the line
The alternative K Means like algorithm by minimizing the sum of perpendicular distances<br/>
The perpendicular distance to a line $y=mx+c$ from a point $(a,b)$ is given by $\frac{ma+c-b}{\sqrt{m^2+1}}$
Can you write Python code for it?

def perpend_to_first(d, w1, w2):
    # d[0] is x and d[1] is y
    return 1 if np.abs(np.dot(w1, d[0])-d[1])/np.linalg.norm([1,w1[1]]) <= np.abs(np.dot(w2, d[0])-d[1])/np.linalg.norm([1,w2[1]]) else 0

# Initialization
w1 = np.array([np.random.normal(0, 5), np.random.normal(0,2)])
w2 = np.array([np.random.normal(0, 5), np.random.normal(0,2)])

# Assign each point to the nearest line
z1 = np.array([perpend_to_first(d, w1, w2) for d in zip(X, Y)])
Z1 = np.diag(z1)
Z2 = np.diag(1-z1)
# recompute the equations of lines - same as M step in EM algorithm
T1 = np.dot(np.transpose(X), Z1) # X'*Z1
T2 = np.dot(np.transpose(X), Z2) # X'*Z2
w1 = np.dot(np.linalg.pinv(np.dot(T1, X)), np.dot(T1, Y))
w2 = np.dot(np.linalg.pinv(np.dot(T2, X)), np.dot(T2, Y))

print w1, w2
plt.plot(seq, line1, 'ro')
plt.plot(seq, line2, 'b+')
plt.plot(seq, np.dot(x, w1))
plt.plot(seq, np.dot(x, w2))
plt.show()


I am working on the file "MixtureModel.ipynb".
The first task is:
A typical call to the Gaussian Mixture Model algorithm looks like this:
Can you write Python code for it?

# Execute this cell
import numpy as np
from sklearn.mixture import GMM

X = np.random.normal(size=(1000,2)) #1000  points in 2D
gmm = GMM(3) #three components
gmm.fit(X)
log_dens = gmm.score(X)
BIC = gmm.bic(X)


I am working on the file "MoL_June15/lab-cfg.ipynb".
The first task is:
Finally we can have access to the set of terminals and nonterminals of the grammar
Can you write Python code for it?

'[S]' in G.nonterminals

'a' in G.terminals


I am working on the file "MoL_June15/lab-cfg.ipynb".
The first task is:
Grammar
A PCFG is organised pretty much as a dictionary mapping from LHS symbols to their rewrite rules.
Can you write Python code for it?


I am working on the file "MoL_June15/lab-cfg.ipynb".
The first task is:
We also have an ambiguous grammar which discourage solving sums before solving products, but still allows it.
Can you write Python code for it?

G2 = WCFG(read_grammar_rules(open('examples/ambiguous')))
print G2


I am working on the file "MoL_June15/lab-cfg.ipynb".
The first task is:
We can add rules
Can you write Python code for it?


I am working on the file "MoL_June15/lab-cfg.ipynb".
The first task is:
We can also iterate through rules in the grammar.
Note that the followin is basically counting how many rules we have in the grammar.
Can you write Python code for it?


I am working on the file "MoL_June15/lab-cfg.ipynb".
The first task is:
We can print the grammar
Can you write Python code for it?


I am working on the file "MoL_June15/lab-cfg.ipynb".
The first task is:
You can print a rule, you can access its attributes, and you can hash rules with containers such as dict and set.
Can you write Python code for it?

print r

print r.prob

r in set([r])

D = {r: 1}
D


I am working on the file "MoL_June15/lab-cfg.ipynb".
The first task is:
we can get the set of rewrite rules for a certain LHS symbol
Can you write Python code for it?


I am working on the file "MoL_June15/lab-cfg.ipynb".
The first task is:
we can test whether there are rewrite rules for a certain LHS symbol
Can you write Python code for it?


I am working on the file "MoL_June15/lab-cky.ipynb".
The first task is:
Agenda
We need an agenda of items. Some items are active, others are passive.
The active agenda is nothing but a stack or queue, whereas the passive agenda is simply a set (all items that have already been processed). However, we can further organise the passive items for easy/quick access within inference rules.
Can you write Python code for it?

class Agenda(object):  # this class is also available in `agenda.py`

    def __init__(self):
        # we are organising active items in a stack (last in first out)
        self._active = []
        # an item should never queue twice, thus we will manage a set of items which we have already seen
        self._seen = set()
        # we organise incomplete items by the symbols they wait for at a certain position
        # that is, if the key is a pair (Y, i)
        # the value is a set of items of the form
        # [X -> alpha * Y beta, [...i]]
        self._incomplete = defaultdict(set)
        # we organise complete items by their LHS symbol spanning from a certain position
        # if the key is a pair (X, i)
        # then the value is a set of items of the form
        # [X -> gamma *, [i ... j]]
        self._complete = defaultdict(set)

    def __len__(self):
        """return the number of active items"""
        return len(self._active)

    def push(self, item):
        """push an item into the queue of active items"""
        if item not in self._seen:  # if an item has been seen before, we simply ignore it
            self._active.append(item)
            self._seen.add(item)
            return True
        return False

    def pop(self):
        """pop an active item"""
        assert len(self._active) > 0, 'I have no items left.'
        return self._active.pop()

    def make_passive(self, item):
        if item.is_complete():  # complete items offer a way to rewrite a certain LHS from a certain position
            self._complete[(item.lhs, item.start)].add(item)
        else:  # incomplete items are waiting for the completion of the symbol to the right of the dot
            self._incomplete[(item.next, item.dot)].add(item)

    def waiting(self, symbol, dot):
        return self._incomplete.get((symbol, dot), set())

    def complete(self, lhs, start):
        return self._complete.get((lhs, start), set())

    def itercomplete(self):
    ...


I am working on the file "MoL_June15/lab-cky.ipynb".
The first task is:
Axioms (CKY)
Basic version
For every rule X -> alpha, and every input position (i) between 0 and n-1, we have an item of the kind:

[X -> * alpha, i, i]

which gets represented as

X -> alpha, [i]
Can you write Python code for it?

def axioms(cfg, sentence):
    """
    :params cfg: a context-free grammar (an instance of WCFG)
    :params sentence: the input sentence (as a list or tuple)
    :returns: a list of items
    """
    items = []
    for rule in cfg:
        for i in range(len(sentence)):
            items.append(Item(rule, [i]))
    return items

sentence = 'a * a'.split()
axioms(G1, sentence)