I am working on the file "ML-LR-part4.ipynb".
The first task is:
Now define functions for the formulas of the model, cost function, gradient and parameter updates.
Can you write Python code for it?

def h(x,a0,a1) :  # The model function h
    h = a0 + a1*x
    return h

def J(a0, a1, x, y, m):  # The Cost function J
    J = 0
    for i in range(m):
        J += ( h(x[i],a0,a1) - y[i] )**2
    return J/(2.0*m)

def gradJ(a0,a1,x,y,m) :  # The Gradient vector of J, gradJ
    DJa0 = 0; DJa1 = 0;
    for i in range(m):
        DJa0 += (h(x[i],a0,a1) - y[i])
        DJa1 += (h(x[i],a0,a1) - y[i])*x[i]
    gradJ = [DJa0/m, DJa1/m]
    return gradJ

def ap1(a0, a1, gJ, alpha) :  # The update to the parameter vector a, ap1 (a plus 1)
                              # gJ is the gradient vector and alpha is the step length
    a0p1 = a0 - alpha * gJ[0]
    a1p1 = a1 - alpha * gJ[1]
    ap1 = [a0p1, a1p1]
    return ap1


I am working on the file "ML-LR-part4.ipynb".
The first task is:
That looks more reasonable. We'll do a gradient descent with this scaling and make a contour plot of the progress. Note, without doing this scaling I couldn't even get the plot to display or the optimization to start!
Can you write Python code for it?

fig, ax = plt.subplots()
# setup the contour axis
p0 = np.linspace(-8,8, 500)
p1 = np.linspace(-3,3, 500)
pa0, pa1 = np.meshgrid(p0, p1)

# plot the Cost function J
ax.contour(pa0,pa1,J(pa0,pa1,x/1000,y/1000000,m=len(x)) , colors='C0', levels=[i for i in np.arange(0,80,5)])

# starting point
a0 = 6; a1 = 2.2
ax.plot(a0,a1, 'o')

# do the gradient descent loop and plot the progress
for i in range(300):
    a0old, a1old = a0, a1
    a0,a1 = ap1(a0,a1, gJ=gradJ(a0,a1,x/1000,y/1000000,m), alpha=0.1 )
    #print(a0,a1) # the updated parameters
    ax.plot(a0,a1,'o')
    ax.plot([a0old,a0],[a1old,a1], '-') # connect the dots

# the values of the parameters after the optimization run
print(a0,a1)


I am working on the file "ML-LR-part6.ipynb".
The first task is:
Notice now that the training-set fit is looking very good but the test-set is not a good fit.
Add $x^5$,
$h(x) = a_0 + a_1x +a_2x^2 +a_3x^3 +a_4x^4 +a_5x^5$
Can you write Python code for it?

X = np.column_stack( (np.ones((x.size,1)) , x, x**2, x**3, x**4, x**5 )) # construct the augmented matrix X
Xtest = np.column_stack( (np.ones((x.size,1)) , xtest, xtest**2, xtest**3, xtest**4, xtest**5 )) # needed for error analysis
a = a_opt(X,y)   # solve the regression

fig, ax = plt.subplots()
ax.plot( x,y, "o", color='g', label='training data')  # plot the test data
ax.plot(xtest,ytest, 'o', color='r', label='testset data')

xt = np.linspace(-1,10,100)
ax.plot(xt, f(xt), '.', color='c', label='source function')
ax.plot(xt, a[0] + a[1]*xt + a[2]*xt**2 + a[3]*xt**3 + a[4]*xt**4 + a[5]*xt**5,
        label='h(x) = %.2f + %.2f*x + %.2f*x**2 + %.2f*x**3 + %.2f*x**4 + %.2f*x**5'
        %(a[0],a[1],a[2],a[3],a[4],a[5]))  # plot the model function
ax.legend()

print('a =', a)  # print the values of a
print('training-set: ','J =', J(a,X,y), "   R2 =", R2(a,X,y))

print('test-set:     ','J =', J(a,Xtest,ytest), "       R2 =", R2(a,Xtest,ytest))


I am working on the file "ML-LR-part6.ipynb".
The first task is:
Now the training-set fit is exact! The cost function $J$ is 0 and $R^2$ is 1. The test-set still has a poor fit. Even though the model function fits the training-set perfectly it is not a very good model overall. It's not as bad as it could be!
Lets add 3 more terms just for fun,
$h(x) = a_0 + a_1x +a_2x^2 +a_3x^3 +a_4x^4 +a_5x^5 +a_6x^6 +a_7x^7 +a_8x^8$
Can you write Python code for it?

X = np.column_stack( (np.ones((x.size,1)) , x, x**2, x**3, x**4, x**5, x**6, x**7, x**8 )) # construct the augmented matrix X
Xtest = np.column_stack( (np.ones((x.size,1)) , xtest, xtest**2, xtest**3, xtest**4, xtest**5, xtest**6, xtest**7, xtest**8 )) # needed for error analysis
a = a_opt(X,y)   # solve the regression

fig, ax = plt.subplots()
ax.plot( x,y, "o", color='g', label='training data')  # plot the test data
ax.plot(xtest,ytest, 'o', color='r', label='testset data')

xt = np.linspace(-1,10,100)
ax.plot(xt, f(xt), '.', color='c', label='source function')
ax.plot(xt, a[0] + a[1]*xt + a[2]*xt**2 + a[3]*xt**3 + a[4]*xt**4 + a[5]*xt**5 + a[6]*xt**6 + a[7]*xt**7 + a[8]*xt**8,
        label='h(x) = %.2f + %.2f*x + %.2f*x**2 + %.2f*x**3 + %.2f*x**4 + %.2f*x**5 + %.2f*x**6 + %.2f*x**7 + %.2f*x**8'
        %(a[0],a[1],a[2],a[3],a[4],a[5],a[6],a[7],a[8]))  # plot the model function
ax.legend()

print('a =', a)  # print the values of a
print('training-set: ','J =', J(a,X,y), "   R2 =", R2(a,X,y))

print('test-set:     ','J =', J(a,Xtest,ytest), "       R2 =", R2(a,Xtest,ytest))


I am working on the file "ML-LR-part6.ipynb".
The first task is:
Now we will start generating models with increasing powers of $x$ and looking at the goodness of fit.
$h(x) = a_0 + a_1x$
Can you write Python code for it?

X = np.column_stack( (np.ones((x.size,1)) , x )) # construct the augmented matrix X
Xtest = np.column_stack( (np.ones((x.size,1)) , xtest )) # needed for error analysis
a = a_opt(X,y)   # solve the regression

fig, ax = plt.subplots()
ax.plot( x,y, "o", color='g', label='training data')  # plot the test data
ax.plot(xtest,ytest, 'o', color='r', label='testset data')

xt = np.linspace(-1,10,100)
ax.plot(xt, f(xt), '.', color='c', label='source function')
ax.plot(xt, a[0] + a[1]*xt ,
        label='h(x) = %.2f + %.2f*x'
        %(a[0],a[1]))  # plot the model function
ax.legend()

print('a =', a)  # print the values of a
print('training-set: ','J =', J(a,X,y), "   R2 =", R2(a,X,y))

print('test-set:     ','J =', J(a,Xtest,ytest), "   R2 =", R2(a,Xtest,ytest))


I am working on the file "ML-LR-part6.ipynb".
The first task is:
That is a better fit. The cost, $J$ has gone down and $R^2$ is closer to 1, However, he model is not behaving the way we want (it's rising at the end of the data).
Lets try a different model. $1/x$ would descend rapidly and then tapper off toward 0.
$h(x) = a_0 + a_1 1/x$
Can you write Python code for it?

X = np.column_stack( (np.ones((x.size,1)) , 1/x )) # construct the augmented matrix X
a = a_opt(X,y)   # solve the regression

fig, ax = plt.subplots()
ax.plot( x,y, "o", color='g', label='training data')  # plot the test data

xt = np.linspace(1,11,100)
ax.plot(xt, a[0] + a[1]/xt  ,  label='h(x) = %.2f + %.2f * 1/x' %(a[0], a[1]))  # plot the model function
ax.legend()

print('a =', a)  # print the values of a
print('J =', J(a,X,y)) # value of Cost function
print("R2 =", R2(a,X,y)) # R squared


I am working on the file "ML-LR-part6.ipynb".
The first task is:
That is a significant improvement and now $R^2$ is getting closer to 1. It's still not converging to a constant value so lets try one more model function.
$h(x) = a_0 + a_1\exp(-x)$
Can you write Python code for it?

X = np.column_stack( (np.ones((x.size,1)) , np.exp(-x) )) # construct the augmented matrix X
a = a_opt(X,y)   # solve the regression

fig, ax = plt.subplots()
ax.plot( x,y, "o", color='g', label='training data')  # plot the test data

xt = np.linspace(1,11,100)
ax.plot(xt, a[0] + a[1]*np.exp(-xt)  ,  label='h(x) = %.2f + %.2f * exp(-x)' %(a[0], a[1]))  # plot the model function
ax.legend()

print('a =', a)  # print the values of a
print('J =', J(a,X,y)) # value of Cost function
print("R2 =", R2(a,X,y)) # R squared


I am working on the file "ML-LR-part6.ipynb".
The first task is:
That is a very good fit and the model has the behavior that we want. The $R^2$ is also very close to 1 indication that the fit to the training data is very good and the cost, $J$ is near 0.
Key points to note;

Linear regression can be easily extended to fit non-linear data by adding non-linear feature variables as long as the model optimization parameters, $a$ are kept linear.
Python with numpy is simple to use for experimenting with data and different model functions.
Your intuition about a candidate for a model function may not be correct!
Having a measure like $R^2$ for the "goodness of fit" is important. It will become very important when we look at over-fitting in the next section.

Another point in addition to the above is that if your data has very complex behavior it may be difficult to come up with a suitable model. One solution to this problem is to try a very general model with lots of terms. The regression may be able to pick out the terms that are useful to reduce the error by making the parameters of important terms large and unimportant terms small. There is a very serious problem that can occur if you use a complex model with many terms -- over-fitting!
Over and Under Fitting
Under-fitting is usually fairly obvious.If your model is inadequate to represent the data you are trying to fit you can usually see that visually in a plot or just observe that the value of the cost function, $J$, is high or $R^2$ is far from 1. Looking at how well or poorly your model predicts a result from a few data points in your training set will tell you right away if your model is inadequate. [It's better to have a separate test set but we will get to that later.]
Over-fitting can be more subtle. You may have a model that fits your training data very well, maybe even perfectly, and still have a model that does a terrible job of predicting unknown values.

The goal of fitting model parameters to a training set is to produce a model that "generalizes" well for making predictions from data that is NOT in your...
Can you write Python code for it?

def f(x) :  # sourse of the data
    f = x*np.sin(x/2)
    return f

np.random.seed(22)
noise = np.random.randn(12)  # some random "normal" noise

x = np.linspace(0,10,6) # x for training-set
y = f(x) + noise[0:6]   # y for training-set

xtest = np.linspace(-1,9,6)     # x for test-set
ytest = f(xtest) + noise[6:12]  # y for test-set


I am working on the file "ML-LR-part6.ipynb".
The first task is:
That's not so good! No surprise! Next lets try changing $x$ to $x^2$ thinking that a simple parabola may be a better fit.
Note, since we are using numpy arrays it is easy to change our model by changing how the augmented matrix $X$ is constructed.
$h(x) = a_0 + a_1x^2$
Can you write Python code for it?

X = np.column_stack( (np.ones((x.size,1)) , x**2 )) # construct the augmented matrix X
a = a_opt(X,y)   # solve the regression

fig, ax = plt.subplots()
ax.plot( x,y, "o", color='g', label='training data')  # plot the test data

xt = np.linspace(1,10,100)
ax.plot(xt, a[0] + a[1]*xt**2  ,  label='h(x) = %.2f + %.2f x**2' %(a[0], a[1]))  # plot the model function
ax.legend()

print('a =', a)  # print the values of a
print('J =', J(a,X,y)) # value of Cost function
print("R2 =", R2(a,X,y)) # R squared


I am working on the file "ML-LR-part6.ipynb".
The first task is:
That's worse! It looks like the regression found a minimum for $a$ that has the curve opposite of what I expected.
So, lets try something different. This time I'll keep $x$ and add an additional feature $x^2$.
$h(x) = a_0 + a_1x + a_2x^2$
Can you write Python code for it?

X = np.column_stack( (np.ones((x.size,1)) , x, x**2 )) # construct the augmented matrix X
a = a_opt(X,y)   # solve the regression

fig, ax = plt.subplots()
ax.plot( x,y, "o", color='g', label='training data')  # plot the test data

xt = np.linspace(1,11,100)
ax.plot(xt, a[0] + a[1]*xt + a[2]*xt**2  ,  label='h(x) = %.2f + %.2f x + %.2f x**2' %(a[0], a[1], a[2]))  # plot the model function
ax.legend()

print('a =', a)  # print the values of a
print('J =', J(a,X,y)) # value of Cost function
print("R2 =", R2(a,X,y)) # R squared


I am working on the file "ML-LR-part6.ipynb".
The first task is:
The fit is not very good as expected, but, note that $J$ and $R^2$ for training and test sets are similar.
Now add an $x^2$ term, do the regression and look at the results,
$h(x) = a_0 + a_1x +a_2x^2$
Can you write Python code for it?

X = np.column_stack( (np.ones((x.size,1)) , x, x**2 )) # construct the augmented matrix X
Xtest = np.column_stack( (np.ones((x.size,1)) , xtest, xtest**2 )) # needed for error analysis
a = a_opt(X,y)   # solve the regression

fig, ax = plt.subplots()
ax.plot( x,y, "o", color='g', label='training data')  # plot the test data
ax.plot(xtest,ytest, 'o', color='r', label='testset data')

xt = np.linspace(-1,10,100)
ax.plot(xt, f(xt), '.', color='c', label='source function')
ax.plot(xt, a[0] + a[1]*xt + a[2]*xt**2 ,
        label='h(x) = %.2f + %.2f*x + %.2f*x**2'
        %(a[0],a[1],a[2]))  # plot the model function
ax.legend()

print('a =', a)  # print the values of a
print('training-set: ','J =', J(a,X,y), "   R2 =", R2(a,X,y))

print('test-set:     ','J =', J(a,Xtest,ytest), "   R2 =", R2(a,Xtest,ytest))


I am working on the file "ML-LR-part6.ipynb".
The first task is:
The value of $J$ has come down and $R^2$ is better for the training and test set.
Now add an $x^3$ term,
$h(x) = a_0 + a_1x +a_2x^2 +a_3x^3$
Can you write Python code for it?

X = np.column_stack( (np.ones((x.size,1)) , x, x**2, x**3 )) # construct the augmented matrix X
Xtest = np.column_stack( (np.ones((x.size,1)) , xtest, xtest**2, xtest**3 )) # needed for error analysis
a = a_opt(X,y)   # solve the regression

fig, ax = plt.subplots()
ax.plot( x,y, "o", color='g', label='training data')  # plot the test data
ax.plot(xtest,ytest, 'o', color='r', label='testset data')

xt = np.linspace(-1,10,100)
ax.plot(xt, f(xt), '.', color='c', label='source function')
ax.plot(xt, a[0] + a[1]*xt + a[2]*xt**2 + a[3]*xt**3,
        label='h(x) = %.2f + %.2f*x + %.2f*x**2 + %.2f*x**3'
        %(a[0],a[1],a[2],a[3]))  # plot the model function
ax.legend()

print('a =', a)  # print the values of a
print('training-set: ','J =', J(a,X,y), "   R2 =", R2(a,X,y))

print('test-set:     ','J =', J(a,Xtest,ytest), "   R2 =", R2(a,Xtest,ytest))


I am working on the file "ML-LR-part6.ipynb".
The first task is:
To start lets just try fitting a straight line $h(x) = a_0 + a_1x$. We'll construct the augmented matrix $X$, solve the regression equation to find the best values of $a$, then plot the result and look at the value of $J$ and $R^2$
$h(x) = a_0 + a_1x$
Can you write Python code for it?

X = np.column_stack( (np.ones((x.size,1)) , x )) # construct the augmented matrix X
a = a_opt(X,y)   # solve the regression

fig, ax = plt.subplots()
ax.plot( x,y, "o", color='g', label='training data')  # plot the test data

xt = np.linspace(1,10,100)
ax.plot(xt, a[0] + a[1]*xt  ,  label='h(x) = %.2f + %.2f x' %(a[0], a[1]))  # plot the model function
ax.legend()

print('a =', a)  # print the values of a
print('J =', J(a,X,y)) # value of Cost function
print("R2 =", R2(a,X,y)) # R squared


I am working on the file "ML-Logistic-Regression-Multinomial.ipynb".
The first task is:
<hr />

Mean normalize the data sets
Each of the data sets are normalized using the mean and standard deviation from the whole 42000 element data set. The make_multinomial_features functions is used here simply to add the column of 1's to the data for the bias term.
Can you write Python code for it?

X_mean = data_09.mean(axis=0)
X_std  = data_09.std(axis=0)
X_std[X_std==0]=1.0 # if there are any 0 values in X_std set them to 1

order = [1]

X_train = make_multinomial_features(data_train_09, order=order)
X_train[:,1:] = apply_normalizer(X_train[:,1:],X_mean,X_std)
Y_train = Y_train_09

X_val = make_multinomial_features(data_val_09, order=order)
X_val[:,1:] = apply_normalizer(X_val[:,1:],X_mean,X_std)
Y_val = Y_val_09

X_test = make_multinomial_features(data_test_09, order=order)
X_test[:,1:] = apply_normalizer(X_test[:,1:],X_mean,X_std)
Y_test = Y_test_09


I am working on the file "ML-Logistic-Regression-Multinomial.ipynb".
The first task is:
<hr />
Checking how well the model did for each of the datasets
The next function will give a printout of the percentage of correct prediction in a dataset. We first look at the training and validation sets.
Can you write Python code for it?

def print_num_correct(datasets):
    for dataset in datasets :
        set_name, yl, Xd = dataset
        yls = yl.size
        probs = np.zeros((10,2))
        count = 0
        for samp in range(yls):
            for num in range(10):
                a_opt = A_opt[:,num]
                probs[num,0] = num
                probs[num,1] = h_logistic(Xd[samp,:],a_opt)

            probs = probs[probs[:,1].argsort()[::-1]]
            if probs[0,0] == yl[samp] :
                count +=1
        print('\n{}'.format(set_name))
        print("{} correct out of {} : {}% correct".format(count, yls, count/yls * 100))

datasets = [('Training Set:', y_labels_train, X_train),('Validation Set:',y_labels_val, X_val)]
print_num_correct(datasets)


I am working on the file "ML-Logistic-Regression-Multinomial.ipynb".
The first task is:
<hr />
Find optimal parameters for the 10 models
This is the main training loop. All 10 models are optimized using the columns of $Y$ and the training data set. Each model is fit to it's number (0-9) by evaluation it's cost function against all of the other numbers "the rest".
You can see that some of the models required many more iterations before convergence. There was also some numerical overflow present. I'm not too concerned about this since it is an artifact of the optimization run. The models converged OK and gave reasonably good set of parameters for each of the 10 models. It is possible to work on each model separately to try to get better fits and the regularization term could be adjusted per model. I did play with the optimization somewhat but wont worry about it too much since in teh next post I'll be doing an implementation of "Stochastic Gradient Descent" and will likely use this data again as an example.
Can you write Python code for it?

reg =300.0 # Regularization term
np.random.seed(42)
aguess = np.random.randn(X_train.shape[1]) # A random guess for the parameters
A_opt = np.zeros((X_train.shape[1],10)) # The matrix of optimized parameters
Res=[] # List to hold the full optimizitaion output of each model

for i in range(10):
    print('\nFitting {} against the rest\n'.format(i))
    def opt_J_reg(a) :
        return J(X_train,a,Y_train[:,i])
    def opt_gradJ_reg(a) :
        return gradJ_reg(X_train,a,Y_train[:,i],reg)

    res = minimize(opt_J_reg, aguess, method='CG', jac=opt_gradJ_reg, tol=1e-6, options={'disp': True})
    Res.append(res)
    A_opt[:,i] = res.x


I am working on the file "ML-Logistic-Regression-Multinomial.ipynb".
The first task is:
<hr />
Following are a few random images picked from the test set.
The first image is of an "8". You can see that the model did not give a very high probability for "8" but it was higher than any of the other probabilities so it did give the correct answer!
Can you write Python code for it?

samp = 23
samp_label = y_labels_test[samp]
sample = X_test[samp,:]
predict(sample, samp_label)

samp = 147
samp_label = y_labels_test[samp]
sample = X_test[samp,:]
predict(sample,samp_label)

samp = 6200
samp_label = y_labels_test[samp]
sample = X_test[samp,:]
predict(sample,samp_label)


I am working on the file "ML-Logistic-Regression-Multinomial.ipynb".
The first task is:
<hr />
Separate out the label column and remove columns that are all 0's
We got rid of 76 feature columns that were all 0's
Can you write Python code for it?

y_labels, data_09 = data_full_matrix[:,0], data_full_matrix[:,1:]
print(data_09.shape)
data_09 = data_09[:,data_09.sum(axis=0)!=0]
print(data_09.shape)


I am working on the file "ML-Logistic-Regression-Multinomial.ipynb".
The first task is:
<hr />
The fit for the "0" model has a low cost function and the quality of fit looks much better than that for "8".
Can you write Python code for it?

num=0
a_opt = A_opt[:,num]
h_prob = h_logistic(X_val,a_opt)
h_predict = to_0_1(h_prob)
print_model_quality('Validation-data fit', h_predict, Y_val[:,num])


I am working on the file "ML-Logistic-Regression-Multinomial.ipynb".
The first task is:
You can show any of the images in that matrix with the following snipit of code. This would show the image in the 4th row (index 3) which is a hand written 4.
Can you write Python code for it?


I am working on the file "ML-Logistic-Regression-examples1.ipynb".
The first task is:
<hr />
After a few minutes of trying different values for the regularization 15.0 seemed to be about as good as was going to get for both the training and validation set. This value resulted in just 1 missed prediction out of 1323.
Now lets see how the test set looks,
Results on Test set
Can you write Python code for it?

a_opt = res.x
h_prob = h_logistic(X_test,a_opt)
h_predict = to_0_1(h_prob)
print_model_quality('Test-data fit', h_predict, y_test)


I am working on the file "ML-Logistic-Regression-examples1.ipynb".
The first task is:
<hr />
All we had to do here is use the optimized vales in $a$ with our validation set to see how well the model predicts values on that. It looks pretty good with only 7 total miss-classifications out or 1323 total.
Maybe our "perfect" fit on the training set was too good. I can try adjusting the regularization term to see if we can get a model that is still good on the training set but does a better job on the validation set. (this is what the validation set is for!)
Can you write Python code for it?

reg = 15.0

np.random.seed(42) # just make sure I'm using the same random guess every time I run

aguess = np.random.randn(X_train.shape[1])
%time res = minimize(opt_J_reg, aguess, method='CG', jac=opt_gradJ_reg, tol=1e-4, options={'disp': True})

a_opt = res.x
h_prob = h_logistic(X_train,a_opt)
h_predict = to_0_1(h_prob)
print_model_quality('Traing-data fit', h_predict, y_train)

a_opt = res.x
h_prob = h_logistic(X_val,a_opt)
h_predict = to_0_1(h_prob)
print_model_quality('Validation-data fit', h_predict, y_val)


I am working on the file "ML-Logistic-Regression-examples1.ipynb".
The first task is:
<hr />
Before the optimization run we need to setup wrappers around the cost functions and the gradient so they are in the correct for the optimization code.
Can you write Python code for it?

def opt_J_reg(a) :
    return J_reg(X,a,y,reg)

def opt_gradJ_reg(a) :
    return gradJ_reg(X,a,y,reg)


I am working on the file "ML-Logistic-Regression-examples1.ipynb".
The first task is:
<hr />
For the model checking function I have added all of the data for a "confusion matrix" i.e. true positive, true negative, false positive and false negative. Using that data I ahve added "Precision" and "Recall" measures.
Can you write Python code for it?

#
# Some model checking functions
#
def to_0_1(h_prob) : # convert probabilites to true (1) or false (0) at cut-off 0.5
    return np.where(h_prob >= 0.5, 1, 0)

def model_accuracy(h,y) : # Overall accuracy of model
    return np.sum(h==y)/y.size * 100

def model_accuracy_pos(h,y) : # Accuracy on positive cases
    return np.sum(y[h==1] == 1)/y[y==1].size * 100

def model_accuracy_neg(h,y) : # Accuracy on negative cases
    return np.sum(y[h==0] == 0)/y[y==0].size * 100

def false_pos(h,y) : # Number of false positives
    return np.sum((y==0) & (h==1))

def false_neg(h,y) : # Number of false negatives
    return np.sum((y==1) & (h==0))

def true_pos(h,y) : # Number of true positives
    return np.sum((y==1) & (h==1))

def true_neg(h,y) : # Number of true negatives
    return np.sum((y==0) & (h==0))

def model_precision(h,y) : # Precision = TP/(TP+FP)
    return true_pos(h,y)/(true_pos(h,y) + false_pos(h,y))

def model_recall(h,y) : # Recall = TP/(TP+FN)
    return true_pos(h,y)/(true_pos(h,y) + false_neg(h,y))

def print_model_quality(title, h, y) : # Print the results of the functions above
    print( '\n# \n# {} \n#'.format(title) )
    print( 'Total number of data points   = {}'.format(y.size))
    print( 'Number of Positive values(1s) = {}'.format(y[y==1].size))
    print( 'Number of Negative values(0s) = {}'.format(y[y==0].size))
    print( '\nNumber of True Positives = {}'.format(true_pos(h,y)) )
    print( 'Number of False Positives = {}'.format(false_pos(h,y)) )
    print( '\nNumber of True Negatives = {}'.format(true_neg(h,y)) )
    print( 'Number of False Negatives = {}'.format(false_neg(h,y)) )
    print( '\nModel Accuracy = {:.2f}%'.format( model_accuracy(h,y) ) )
    print( 'Model Accuracy Positive Cases = {:.2f}%'.format( model_accuracy_pos(h,y) ) )
    print( 'Model Accuracy Negative Cases = {:.2f}%'.format( model_accuracy_neg(h,y) ) )
    print( '\nModel Precision = {}'.format(model_precision(h,y)) )
    print( '\nModel Recall = {}'.format(model_recall(h,y)) )


I am working on the file "ML-Logistic-Regression-examples1.ipynb".
The first task is:
<hr />
The next thing to do is break up the data set in training, validation and test sets. I'll print out the shapes of the datasets and label vectors. We will have 6170 training samples and 1323 samples for validation and testing.
Can you write Python code for it?


I am working on the file "ML-Logistic-Regression-examples1.ipynb".
The first task is:
<hr />
The plot boundary is smoother and probably a more reasonable model. I hope this example has given you some idea of how multinomial features and regularization can effect model fitting for Logistic Regression.
This is really kind of fun to play around with! If you want to try the code and experiment yourself feel free to grad the Jupyter notebook from the GitHub link I gave near the top of the post.
In the next section we'll look at a simple image classification example.
Two class Image classification example using data from the MNIST dataset
For this example I pulled the MNIST training set from Kaggle. For information on the dataset itself see Yann Lecun's site http://yann.lecun.com/exdb/mnist/index.html. I used the data from Kaggel since it is in an easy to use format. (Kaggle is a great site for Machine Learning and there are many useful datasets and example problems there.)
The reason I only pulled the training set from Kaggel is because I want to explicitly show splitting up a dataset into separate sets for training. validation, and testing. The training set should be the largest set since that is where the model fitting takes place. The validation set is used for evaluating the effect of making adjustments to "hyper-parameters" of the method used to construct the model. The test set is the finial judge of the "generalization" or usefulness of the model for making predictions.
The first thing to do is load up the data and have a look at it,
Can you write Python code for it?

data_full = pd.read_csv("./data/kg-mnist/train.csv")
data_full.head(10)


I am working on the file "ML-Logistic-Regression-examples1.ipynb".
The first task is:
<hr />
The plot looks OK but the model boundary is getting a bit wobbly. Let's try it again with a small regularization value added.
Can you write Python code for it?

reg = 1.0 # no regularization on this run

aguess = np.ones(X.shape[1]) # make a random starting guess

res = minimize(opt_J_reg, aguess, method='BFGS', jac=opt_gradJ_reg, tol=1e-4, options={'disp': True})

a_opt = res.x
print(a_opt)

h_prob = h_logistic(X,a_opt)  # get the probabilities of the from the model using a_opt
h_predict = to_0_1(h_prob)    # convert the probabilities to 0,1 predictions at cutoff 0.5
print_model_quality('Traing-data fit', h_predict, y)


I am working on the file "ML-Logistic-Regression-examples1.ipynb".
The first task is:
<hr />
We will obviously need a non-linear feature model to fit this data. We'll start by setting the data up for a logistic regression optimization run using a hybrid multinomial feature set that looks like,
$ [1, f_1, f_2, f_1^2, f_1 f_2, f_2^2 ]$
Can you write Python code for it?

f1 = np.concatenate((nx1,px1))
f2 = np.concatenate((nx2,px2))
fs = np.column_stack((f1,f2))
y = np.concatenate( (np.zeros(100), (np.ones(100))) )

order = np.array([1,2])
X = make_multinomial_features(fs,order=order)
X[:,1:] = mean_normalize(X[:,1:])  # normalize but leave the bias term alone
print('Number of multinolial features = {}'.format(X.shape[1]))


I am working on the file "ML-Logistic-Regression-examples1.ipynb".
The first task is:
Example of 2-D data using a multinomial feature model
This first example is a variation of data that I used when describing "decision boundaries". I have made it a little more complicated and will use multinomial "feature expansion" on the two feature variables. This will also include some over-fitting and Regularization cases.
Can you write Python code for it?

# Generate some "interesting" random 2-D data
np.random.seed(42)
px1, px2 = np.random.multivariate_normal([0,0], [[1,0.2],[0.8,1]] , 100).T
t = np.linspace(0,2*np.pi,100)
nx1, nx2 = (3+px1)*np.sin(t), (3+px2)*np.cos(t)

# Plot the data
fig, ax = plt.subplots()
ax.plot(nx1,nx2, "o", label='neg data' )
ax.plot(px1,px2, "P", label='pos data')
ax.axis('equal')
plt.title("2-D dataset", fontsize=24)
ax.legend();


I am working on the file "ML-Logistic-Regression-examples1.ipynb".
The first task is:
The image is shown fairly large on the screen. You can clearly see the pixels and their relative gray-scale values.
<hr />
Now I'll pull out just the 0 1nd 1 images (data_01) and the corresponding labels (y_01),
Can you write Python code for it?

idx = np.logical_or(data_full_matrix[:,0]==0,  data_full_matrix[:,0]==1)

y_01, data_01 = data_full_matrix[idx,0], data_full_matrix[idx,1:]
print(data_01.shape)


I am working on the file "ML-Logistic-Regression-examples1.ipynb".
The first task is:
There are now 8816 images of 0's and 1's in (data_01). Some of the feature columns in data_01 have all 0 gray-scale values i.e. they are empty boarders. Let's remove those "features". We will have 588 columns in data_01 after their removal.
Can you write Python code for it?

data_01 = data_01[:,data_01.sum(axis=0)!=0]
print(data_01.shape)


I am working on the file "ML-Logistic-Regression-theory.ipynb".
The first task is:
In this plot the positive outcomes are in a circular region in the center of the plot. The decision boundary the red circle.
## Cost Function for Logistic Regression
A cost function's main purpose is to penalize bad choices for the parameters to be optimized and reward good ones. It should be easy to minimize by having a single global minimum and not be overly sensitive to changes in its arguments. It is also nice if it is differentiable, (without difficulty) so you can find the gradient for the minimization problem. That is, it's best if it is "convex", "well behaved" and "smooth".
The cost function for logistic regression is written with logarithmic functions. An argument for using the log form of the cost function comes from the statistical derivation of the likelihood estimation for the probabilities. With the exponential form that's is a product of probabilities and the log-likelihood is a sum. [The statistical derivations are always interesting but usually complex. We don't really need to look at that to justify the cost function we will use.] The log function is also a monotonically increasing function so the negative of the log is decreasing. The minimization of a function and minimizing the negative log of that function will give the same values for the parameters. The log form will also be convex which means it will have a single global minimum whereas a simple "least-squares" cost function using the sigmoid function can have multiple minimum and abrupt changes. The log form is just better behaved!
To see some of this lets looks at a plot of the sigmoid function and the negative log of the sigmoid function.
Can you write Python code for it?

z = np.linspace(-10,10,100)
fig, ax = plt.subplots()
ax.plot(z, g(z))
ax.set_title('Sigmoid Function 1/(1 + exp(-z))', fontsize=24)
ax.annotate('Convex', (-7.5,0.2), fontsize=18 )
ax.annotate('Concave', (3,0.8), fontsize=18 )

z = np.linspace(-10,10,100)
plt.plot(z, -np.log(g(z)))
plt.title("Log Sigmoid Function -log(1/(1 + exp(-z)))", fontsize=24)
plt.annotate('Convex', (-2.5,3), fontsize=18 )


I am working on the file "ML-Logistic-Regression-theory.ipynb".
The first task is:
Recall that in the training-set $y$ are labels with a values or 0 or 1. The cost function will be broken down into two cases for each data point $(i)$, one for $y=1$ and one for $y=0$. These two cases can then be combined into a single cost function $J$
$$ \bbox[25px,border:2px solid green]{
\begin{align}  J^{(i)}{y=1}(a) & = -log(h_a(x^{(i)}))   \ \
 J^{(i)}{y=0}(a) & = -log(1 - h_a(x^{(i)}))   \ \
 J(a) & = -\frac{1}{m}\sum^{m}_{i=1} y^{(i)} log(h_a(x^{(i)})) + (1-y^{(i)})log(1 - h_a(x^{(i)}))
 \end{align}  }$$
You can see that the factors $y$ and $(1-y)$ effectively pick out the terms for the cases $y=1$ and $y=0$.
Vectorized form of $J(a)$
$J(a)$ can be written in vector form eliminating the summation sign as,
 $$ \bbox[25px,border:2px solid green]{
   \begin{align} h_a(X) &= g(Xa) \
  J(a)  &= -\frac{1}{m} \left( y' log(h_a(X) + (1-y)'log(1 - h_a(X) \right)
   \end{align} }$$
To visualize how the cost functions works look at the following plots,
Can you write Python code for it?

x = np.linspace(-10,10,50)
plt.plot(g(x), -np.log(g(x)))
plt.title("h(x) vs J(a)=-log(h(x)) for y = 1", fontsize=24)
plt.xlabel('h(x)')
plt.ylabel('J(a)')


I am working on the file "ML-Logistic-Regression-theory.ipynb".
The first task is:
The plot above shows 2 sets of training-data. The positive case is represented by green '+' and the negative case by blue 'o'. The red line is the decision boundary $b(x) = 0.55 -1.3x$. Any test cases that are above the line are negative and any below are positive. The parameters for that red line would be what we could have determined from doing a Logistic Regression run on those 2 sets of training data.
The next plot shows a case where the decision boundry is more complicated. It's represented by $b(x_1,x_2) = x_1^2 +x_2^2 - 2.5$
Can you write Python code for it?

fig, ax = plt.subplots()
x3, y3 = np.random.multivariate_normal([0,0], [[.5,0],[0,.5]] , 400).T
t = np.linspace(0,2*np.pi,400)
ax.plot((3+x3)*np.sin(t), (3+y3)*np.cos(t), "o")
ax.plot(x3, y3, "P")

xb1 = np.linspace(-5.0, 5.0, 100)
xb2 = np.linspace(-5.0, 5.0, 100)
Xb1, Xb2 = np.meshgrid(xb1,xb2)
b = Xb1**2 + Xb2**2 - 2.5
ax.contour(Xb1,Xb2,b,[0], colors='r')
plt.title("Decision Boundary", fontsize=24)
ax.axis('equal')


I am working on the file "ML-Logistic-Regression-theory.ipynb".
The first task is:
There are several features of $g$ to note,
- For larger values of $z$ $g(z)$ approaches 1
- For more negative values of $z$ $g(z)$ approaches 0
- The value of $g(0) = 0.5$
- For $z \ge 0$, $g(z)\ge 0.5$
- For $z \lt 0$, $g(z)\lt 0.5$
0.5 will be the cutoff for decisions. That is, if $g(z) \ge 0.5$ then the "answer" is "the positive case", 1,  if $g(z) \lt 0.5$ then the answer is "the negative case", 0.
Decision Boundary
The value 0.5 mentioned above creates a boundary for classification by our model (hypothesis)  $h_a(x)$
$$
\begin{align} \text{if } h_a(x) \ge 0.5 & \text{ then we say } &y=1 \ \
 \text{if } h_a(x) \lt 0.5 & \text{ then  } &y=0
 \end{align}  $$
Looking at $g(z)$ more closely gives,
 $$
 \begin{align}  h_a(x) = g(a'x) \ge 0.5 & \text{ when} & a'x \ge 0 \ \
  h_a(x) = g(a'x) \lt 0.5 & \text{ when} & a'x \le 0
  \end{align}  $$
Therefore,
$$ \bbox[25px,border:2px solid green]{
\begin{align}  a'x \ge 0.5 & \text{ implies } & y = 1 \ \
 a'x \lt 0.5 & \text{ implies} & y = 0
 \end{align}  }$$

The Decision Boundary is the "line" defined by $a'x$  that separates the area where $y=0$ and $y=1$. The "line" defined by $a'x$ can be non-linear since the feature variables $x_i$ can be non-linear. The decision boundary can be any shape (curve) that fits the data. We use a Cost Function derived from the logistic regression sigmoid function to helps us find the parameters $a$ that define the optimal decision boundary $a'x$. After we have found the optimal values of $a$ the model function $h_a(x$), which uses the sigmoid function, will tell us which side of the decision boundary our "question" lies on based on the values of the features $x$ that we give it.

If you understand the paragraph above then you have a good idea of what logistic regression about!
Here's some examples of what that Decision Boundary might look like;
Can you write Python code for it?

# Generate 2 clusters of data
S = np.eye(2)
x1, y1 = np.random.multivariate_normal([1,1], S, 40).T
x2, y2 = np.random.multivariate_normal([-1,-1], S, 40).T

fig, ax = plt.subplots()
ax.plot(x1,y1, "o", label='neg data' )
ax.plot(x2,y2, "P", label='pos data')
xb = np.linspace(-3,3,100)
a = [0.55,-1.3]
ax.plot(xb, a[0] + a[1]*xb  ,  label='b(x) = %.2f + %.2f x' %(a[0], a[1]))
plt.title("Decision Boundary", fontsize=24)
plt.legend();


I am working on the file "ML-Logistic-Regression-theory.ipynb".
The first task is:
You can see from this plot that when $y=1$ the cost $J(a)$ is large if $h(x)$ goes toward 0. That is, it favors $h(x)$ going to 1 which is what we want.
Can you write Python code for it?

x = np.linspace(-10,10,50)
plt.plot(g(x), -np.log(1-g(x)))
plt.title("h(x) vs J(a)=-log(1-h(x)) for y = 0", fontsize=24)
plt.xlabel('h(x)')
plt.ylabel('J(a)')


I am working on the file "ML-Python/Week3-4/submissions/A4/Shashank/A4_regular_logistic.ipynb".
The first task is:
Feature Mapping
One way to fit the data better is to create more features from each data point. The function mapFeature, maps the features into all polynomial terms of x1 and x2 up to the sixth power. As a result of this mapping, our vector of two features (the scores on two QA tests) has been transformed into a 28-dimensional vector.
Can you write Python code for it?

def mapFeature(X1, X2):
    if isinstance(X1, np.float64) :
        l = 1
    else:
        l = len(X1)
    X1.reshape((l, 1))
    X2.reshape((l, 1))
    degree = 6
    out = np.ones((l, 28))
    k = 1
    for i in range(1, degree + 1):
        for j in range(0, i + 1):
            out[:, k] = (np.power(X1, (i-j))) * np.power(X2, j)
            k = k + 1
    return out

X = mapFeature(X[:, 1], X[:, 2])


I am working on the file "ML-Python/Week3-4/submissions/A4/Shashank/A4_regular_logistic.ipynb".
The first task is:
Gradient function
Can you write Python code for it?

def dif_cost():
    n = len(theta)
    grad = np.zeros((n, 1))
    new_grad = np.zeros((n, 1))
    z = np.dot(X, theta)
    h = sigmoid(z)
    # if sigmoid function gives error uncomment the 2 statements below and comment the statement above
    #from scipy.special import expit
    #h = expit(z)
    for j in range(0, n):
        x = np.zeros((m,1))
        x[:,0] = X[:,j]
        t = (h - y) * x
        sum_ = np.sum(t)
        grad[j, 0] = sum_ / m
    new_grad[0,0] = grad[0,0]
    new_grad[1:, 0] = grad[1:, 0] + lambda_ * theta[1:,0] / m
    return new_grad


I am working on the file "ML-Python/Week3-4/submissions/A4/Shashank/A4_regular_logistic.ipynb".
The first task is:
Initialize theta values and lambda
Can you write Python code for it?


I am working on the file "ML-Python/Week3-4/submissions/A4/Shashank/A4_regular_logistic.ipynb".
The first task is:
Training
Can you write Python code for it?

no_iter = 1000
alpha = 1
for i in range(1, no_iter):
    grad = dif_cost()
    theta = theta - alpha * grad


I am working on the file "ML-Python/Week3-4/submissions/A4/Shashank/A4_regular_logistic.ipynb".
The first task is:
Visualizing the data
Can you write Python code for it?

pos = []
neg = []
for i in range(0, m):
    if y[i] == 1:
        pos.append(i)
    else:
        neg.append(i)
import matplotlib.pyplot as plt
plt.plot(X[pos, 1], X[pos, 2], '+', label = "y=1")
plt.plot(X[neg, 1], X[neg, 2], 'yo', label = "y=0")
plt.axis([-1, 1.5, -1, 1.25])

plt.xlabel('Microchip Test 1')
plt.ylabel('Microchip Test 2')
plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
plt.show()


I am working on the file "ML-SQL/ML-SQL-initialDemo.ipynb".
The first task is:
Classify
Define an algorithm to perform a classification task on
Supported classifier: SVM, Logistic Regression
Can you write Python code for it?

#Algorithm Definitions
algoPhrase = (Literal ("algorithm") + Literal("=")).suppress()
svmPhrase = oneOf(["svm", "SVM"])
logPhrase = oneOf(["logistic", "Logistic", "LOGISTIC"])

#Options for classifiers

#Compositions
svm = svmPhrase + Optional(openParen + closeParen)
log = logPhrase + Optional(openParen + closeParen)
algo = algoPhrase + MatchFirst([svm, log]).setResultsName("algorithm")

#define so that there can be multiple verisions of Classify
classifyKeyword = oneOf(["Classify", "CLASSIFY"]).suppress()

#Phrases to organize predictor and label column numbers
predPhrase = (Literal("predictors") + Literal("=")).suppress()
labelPhrase = (Literal("label") + Literal("=")).suppress()

#define predictor and label column numbers
predictorsDef = OneOrMore(Word(numbers) + ocomma).setResultsName("predictors")
labelDef = Word(numbers).setResultsName("label")

#combine phrases with found column numbers
preds = predPhrase + openParen + predictorsDef + closeParen
labels = labelPhrase + labelDef

classify = Optional(classifyKeyword + openParen + preds + ocomma + labels + ocomma + algo + closeParen)

classifyTest = "CLASSIFY (predictors = (1,2,3,4), label = 5, algorithm = SVM)"

print(classify.parseString(classifyTest))


I am working on the file "ML-SQL/ML-SQL-initialDemo.ipynb".
The first task is:
Examples
I include the 3 words: READ, SPLIT, CLASSIFY and show a basic example of a classification task on the Iris-data set
Can you write Python code for it?

read_split_classify = read + split + classify
read_split_classify_regression = read + split + classify + regression

query1 = readTest + " " + splitTest + " " + classifyTest

print(query1)

#define a pipeline to accomplish all of the data tasks we envision
result1 = read_split_classify.parseString(query1)

#Extract relevant features from the query
filename1 = result1.filename
header1 = result1.header
sep1 = result1.sep
train1 = result1.train_split
test1 = result1.test_split
predictors1 = result1.predictors
label1 = result1.label
algo1 = str(result1.algorithm)

#Preform classification dataflow

#read file
file1 = pd.read_csv(filename1, header = str_to_bool(header1), sep = sep1)

#predictors and labels
pred_cols = map(int, predictors1)
pred_cols = map(lambda x: x - 1, pred_cols)
label_col = int(label1) - 1

X = file1.ix[:,pred_cols]
y = file1.ix[:,label_col]

#Split data into training and testing sets
X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=float(train1), test_size=float(test1))

#choose classification algorithm
if algo1.lower() == "svm":
    clf = svm.SVC()
elif algo.lower() == "logistic":
    clf = LogisticRegression()

#Train model
clf.fit(X_train, y_train)

#Performance on test data
clf.score(X_test, y_test)


I am working on the file "ML-SQL/ML-SQL-initialDemo.ipynb".
The first task is:
Grammer Definition
Literals and Valid Symbols that are possible in the ML-SQL language
Can you write Python code for it?

letters = string.ascii_letters
punctuation = string.punctuation
numbers = string.digits
whitespace = string.whitespace

#combinations
everything = letters + punctuation + numbers
everythingWOQuotes = everything.replace("\"", "").replace("'", "")

#Booleans
bools = Literal("True") + Literal("False")

#Parenthesis and Quotes
openParen = Literal("(").suppress()
closeParen = Literal(")").suppress()
Quote = Literal('"').suppress()

#includes every combination except whitespace
everything


I am working on the file "ML-SQL/ML-SQL-initialDemo.ipynb".
The first task is:
READ
Read files into memory by speicifying the file name, header presence, and the separator.
Issues:
- Read cannot handle spaces in file names
- Handling names of columns
Can you write Python code for it?

filename = Word(everything).setResultsName("filename")

#define so that there can be multiple verisions of READ
readKeyword = oneOf(["Read", "READ"]).suppress()

#Define Read Optionals
#header
headerLiteral = (Literal("header") + Literal("=")).suppress()
header = Optional(headerLiteral + Or(bools).setResultsName("header"), default = "False" )

#separator
separatorLiteral = (Or([Literal("sep"), Literal("separator")]) + Literal("=")).suppress()
definesep = Quote + Word(everythingWOQuotes + whitespace).setResultsName("sep") + Quote
separator = Optional(separatorLiteral + definesep, default = ",")

#Compose Read Optionals
readOptions = Optional(openParen + separator + header + closeParen)

read = readKeyword + filename + readOptions

readTest = 'READ /home/ubuntu/notebooks/ML-SQL/Classification/iris.data (sep="," header=False)'

readTestResult = read.parseString(readTest)

filename = readTestResult.filename
header = readTestResult.header
sep = readTestResult.sep

#Function to lower a string value of "True" or "False" to an actual python boolean value
def str_to_bool(s):
    if s == 'True':
         return True
    elif s == 'False':
         return None
    else:
         raise ValueError ("Cannot lower value " + s + " to a boolean value")

#read parameters from parsed statement and read the file
f = pd.read_csv(filename, sep = sep, header = str_to_bool(header))
f.head()


I am working on the file "ML-SQL/ML-SQL-initialDemo.ipynb".
The first task is:
SPLIT
Splits dataset into training, testing, and validation sets. Give 3 non-negative decimals that sum to 1 to specify these quantities.
Can you write Python code for it?

#define so that there can be multiple verisions of Split
splitKeyword = oneOf(["Split", "SPLIT"]).suppress()

#Phrases used to organize splits
trainPhrase = (Literal("train") + Literal("=")).suppress()
testPhrase = (Literal("test") + Literal("=")).suppress()
valPhrase = (Literal("validation") + Literal("=")).suppress()

#train, test, validation split values
trainS = Combine(Literal(".") + Word(numbers)).setResultsName("train_split")
testS = Combine(Literal(".") + Word(numbers)).setResultsName("test_split")
valS = Combine(Literal(".") + Word(numbers)).setResultsName("validation_split")

#Compose phrases and values together
training = trainPhrase + trainS
testing = testPhrase + testS
val = valPhrase + valS

#Creating Optional Split phrase
ocomma = Optional(",").suppress()
split = Optional(splitKeyword + openParen + training + ocomma + testing + ocomma + val + closeParen)

#Combining READ and SPLIT keywords into one clause for combined use
read_split = read + split

#Split test
splitTest = "SPLIT (train = .8, test = .2, validation = .0)"

print(split.parseString(splitTest))

#Read with Split test
read_split_test = readTest + " "+ splitTest

print(read_split.parseString(read_split_test))


I am working on the file "ML-SQL/Pyparsing tutorial.ipynb".
The first task is:
Chemical Formula parser
Mentioned in the tutorial
Grammer
- integer       :: '0'..'9'+
- cap           :: 'A'..'Z'
- lower         :: 'a'..'z'
- elementSymbol :: cap lower*
- elementRef    :: elementSymbol [ integer ]
- formula       :: elementRef+
Can you write Python code for it?

#Define Grammer
caps       = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
lowers     = caps.lower()
digits     = "0123456789"

element    = Word( caps, lowers )

#Groups elements so that element and numbers appear together
elementRef = Group( element + Optional( Word( digits ), default="1" ) )
formula    = OneOrMore( elementRef )

testString = "CO2"
elements   = formula.parseString( testString )
print(elements)

tests = [ "H2O", "C6H5OH", "NaCl" ]
for t in tests:
    try:
        results = formula.parseString( t )
        print (t,"->", results)
    except ParseException as pe:
        print (pe)
    else:
        wt = sum( [atomicWeight[elem]*int(qty) for elem,qty in results] )
        print ("(%.3f)" % wt)


I am working on the file "ML-SQL/Pyparsing tutorial.ipynb".
The first task is:
Phone number parser
Mentioned in the tutorial
Grammer:
- number      :: '0'.. '9'*
- phoneNumber :: [ '(' number ')' ] number '-' number
Can you write Python code for it?

#Definitions of literals
dash   = Literal( "-" )
lparen = Literal( "(" )
rparen = Literal( ")" )

#Variable lengths and patterns of number => Word token
digits = "0123456789"
number = Word( digits )

#Define phone number with And (+'s)
#Literals can also be defined with direct strings
phoneNumber = lparen + number + rparen + number + dash + number

#Create a results name for easy access
areacode = number.setResultsName("areacode")

#Make the area code optional
phoneNumber = Optional( "(" + areacode + ")" ) + number + "-" + number

#List of phone numbers
phoneNumberList = OneOrMore( phoneNumber )

#Using the grammer
inputString = "(978) 844-0961"
data = phoneNumber.parseString( inputString )

data.areacode

#Bad input
inputStringBad = "978) 844-0961"

data2 = phoneNumber.parseString( inputStringBad )


I am working on the file "ML-W1/Linear Regression With One Variable.ipynb".
The first task is:
Cost Function
For $x$ as input variable (feature) and $y$ as output/target variable, we have cost function $J$ of parameter $\theta$ as follow.
$$J(\theta) = \frac{1}{2m} \sum_{i=1}^{m}\left(h_{\theta}(x^{(i)}) - y^{(i)}\right)^2$$
Where hypothesis $h_{\theta}(x)$ is the linear model as follow.
$$h_{\theta}(x) = \theta^Tx = \theta_0 + \theta_1x$$
And $m$ is total training samples and $i$ is the index.
Can you write Python code for it?

def compute_cost(X, y, theta):
    # Total training samples
    m = len(y)

    h = theta[0] + (theta[1] * X[:,1])
    J = (1 / (2*m)) * sum((h - y) ** 2)
    return J

J = compute_cost(X, y, theta)
print(J)


I am working on the file "ML-W1/Linear Regression With One Variable.ipynb".
The first task is:
Display Settings
Can you write Python code for it?

# Display graph inline
%matplotlib inline

# Display graph in 'retina' format for Mac with retina display. Others, use PNG or SVG format.
%config InlineBackend.figure_format = 'retina'
#%config InlineBackend.figure_format = 'PNG'
#%config InlineBackend.figure_format = 'SVG'

# For displaying 3D graph
from mpl_toolkits.mplot3d import Axes3D
from matplotlib import cm


I am working on the file "ML-W1/Linear Regression With One Variable.ipynb".
The first task is:
Gradient Descent Algorithm
Our objective is to have the parameters ($\theta_0$ and $\theta_1$) to produce a hypothesis $h_{\theta}(x)$ as close as possible to target variable $y$. As we have defined our cost function $J(\theta)$ as the error of the two therefore all we need to do is to minimize the cost function $\left(\underset{\theta_0, \theta_1}{\text{minimize}} \hspace{2mm} J(\theta_0, \theta_1)\right)$.
Following is the algorithm where $\alpha$ is the learning rate.

repeat until convergence {
$\hspace{10mm} \theta_j := \theta_j - \alpha \frac{\partial}{\partial\theta_j} J(\theta_0, \theta_1) \hspace{10mm} (\text{for } j=0 \text{ and } j=1)$
}

Derive cost function $J(\theta_j)$ for both $j=0$ and $j=1$.

$j = 0$ :
$$\frac{\partial}{\partial\theta_0} J(\theta_0, \theta_1) =$$
$$\frac{\partial}{\partial\theta_0} \frac{1}{2m} \sum_{i=1}^{m}\left(h_{\theta}(x^{(i)}) - y^{(i)}\right)^2 =$$
$$\frac{\partial}{\partial\theta_0} \frac{1}{2m} \sum_{i=1}^{m}\left(\theta_0 + \theta_1x^{(i)}-y^{(i)}\right)^2 =$$
$$\frac{1}{m} \sum_{i=1}^{m}\left(\theta_0 + \theta_1x^{(i)} - y^{(i)}\right) =$$
$$\frac{1}{m} \sum_{i=1}^{m}\left(h_{\theta}(x^{(i)}) - y^{(i)}\right)$$
$j = 1$ :
$$\frac{\partial}{\partial\theta_1} J(\theta_0, \theta_1) =$$
$$\frac{\partial}{\partial\theta_1} \frac{1}{2m} \sum_{i=1}^{m}\left(h_{\theta}(x^{(i)}) - y^{(i)}\right)^2 =$$
$$\frac{\partial}{\partial\theta_1} \frac{1}{2m} \sum_{i=1}^{m}\left(\theta_0 + \theta_1x^{(i)}-y^{(i)}\right)^2 =$$
$$\frac{1}{m} \sum_{i=1}^{m}\left(\theta_0 + \theta_1x^{(i)} - y^{(i)}\right) . x^{(i)} =$$
$$\frac{1}{m} \sum_{i=1}^{m}\left(h_{\theta}(x^{(i)}) - y^{(i)}\right) . x^{(i)}$$

For this linear case, the algorithm would be as follow.

repeat until convergence {
$\hspace{10mm} \theta_0 := \theta_0 - \alpha \frac{1}{m} \sum_{i=1}^{m}\left(h_{\theta}(x^{(i)}) - y^{(i)}\right)$
$\hspace{10mm} \theta_1 := \theta_1 - \alpha \frac{1}{m} \sum_{i=1}^{m}\left(h_{\theta}(x^{(i)}) - y^{(i)}\right) . x^{(i)}$
}
Can you write Python code for it?

def perform_gradient_descent(X, y, theta, alpha, iterations):
    # Total training samples
    m = len(y)
    # History of costs
    J_history = np.zeros([iterations, 1])

    for i in range(iterations):
        h = theta[0] + (theta[1] * X[:,1])
        error = h - y
        theta_tmp1 = theta[0] - ((alpha / m) * sum(error))
        theta_tmp2 = theta[1] - ((alpha / m) * sum(error * X[:,1]))
        theta[0] = theta_tmp1
        theta[1] = theta_tmp2

        J_history[i] = compute_cost(X, y, theta)

    return theta, J_history

iterations = 1500
alpha = 0.01

theta, J_history = perform_gradient_descent(X, y, theta, alpha, iterations)

theta

J_history


I am working on the file "ML-W1/Linear Regression With One Variable.ipynb".
The first task is:
Make Prediction
Predict profit for cities with population of 35,000 and 70,000 people.
Can you write Python code for it?

ttl_population = 35000
print('For population = 35,000, we predict a profit of %s' % \
      int(np.dot([1, ttl_population/10000], theta) * 10000))

ttl_population = 70000
print('For population = 70,000, we predict a profit of %s' % \
      int(np.dot([1, ttl_population/10000], theta) * 10000))


I am working on the file "ML-W1/Linear Regression With One Variable.ipynb".
The first task is:
Plot Linear Fit
Can you write Python code for it?

plt.scatter(data['population'], data['profit'], color='red', s=2)
plt.plot(data['population'], np.dot(X, theta), color='blue')
plt.title('Restaurant Sells')
plt.xlabel('Population of City in 10,000s')
plt.ylabel('Profit in $10,000s')
plt.show()


I am working on the file "ML-W1/Linear Regression With One Variable.ipynb".
The first task is:
Visualize Cost for Different Range of Parameters
Can you write Python code for it?

fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
ax.view_init(azim =-120)

theta0_vals = np.linspace(-10, 10, 100)
theta1_vals = np.linspace(-1, 4, 100)
theta0_mesh, theta1_mesh = np.meshgrid(theta0_vals, theta1_vals)
zs = np.array([compute_cost(X, y, [theta0_val,theta1_val]) for theta0_val,theta1_val in zip(np.ravel(theta0_mesh), np.ravel(theta1_mesh))])
Z = zs.reshape(theta0_mesh.shape)

ax.plot_surface(theta0_mesh, theta1_mesh, Z, cmap=cm.coolwarm)
ax.set_xlabel('Theta 0')
ax.set_ylabel('Theta 1')
ax.set_zlabel(r'Cost $J(\theta)$')

plt.show()


I am working on the file "ML-W2/Linear Regression With Multiple Variables.ipynb".
The first task is:
Another way to look at the data.
Can you write Python code for it?

fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
ax.view_init(azim =-120)
ax.dist = 11

scatter = ax.scatter3D(data['size'] / 100, data['ttl_bedrooms'], data['price'] / 100000,
                       c=data['price'], cmap=cm.coolwarm, s=50, edgecolors='none')

ax.set_xlabel('\nSize (in 100 square feet)', linespacing=2)
ax.set_ylabel('\n# Bedroom', linespacing=2)
ax.set_zlabel('\nHouse Price (in 100,000 USD)', linespacing=2)
plt.colorbar(scatter)

plt.show()

# Total training samples
m = len(data)

# Collect features
X = np.append(data['size'].reshape(m,1), data['ttl_bedrooms'].reshape(m,1), 1)
# Feature Normalization
mu = np.mean(X, axis=0)
sigma = np.std(X, axis=0)
X_n = (X - mu) / sigma

# Total features
n = X_n.shape[1]

# Add column of ones to variable x
X = np.append(np.ones([m,1]),X,1)
X_n = np.append(np.ones([m,1]),X_n,1)

# Output
y = data['price']

# Initialize fitting parameters
theta = np.zeros([n+1, 1])
theta_n = np.zeros([n+1, 1])


I am working on the file "ML-W2/Linear Regression With Multiple Variables.ipynb".
The first task is:
Cost Function
For $x$ as input variable (feature) and $y$ as output/target variable, we have cost function $J$ of parameter $\theta$ as follow.
$$J(\theta) = \frac{1}{2m} \sum_{i=1}^{m}\left(h_{\theta}(x^{(i)}) - y^{(i)}\right)^2$$
Where hypothesis $h_{\theta}(x)$ is the linear model as follow.
$$h_{\theta}(x) = \theta^Tx = \theta_0 + \theta_1x_1 + \theta_2x_2$$
And $m$ is total training samples and $i$ is the index.
Can you write Python code for it?

def compute_cost(X, y, theta):
    # Total training samples
    m = len(y)
    # Calculate hypothesis
    h = np.sum(theta.transpose() * X, axis=1)
    # Calculate cost
    J = (1 / (2*m)) * sum((h - y) ** 2)
    return J

J = compute_cost(X, y, theta)
print(J)


I am working on the file "ML-W2/Linear Regression With Multiple Variables.ipynb".
The first task is:
Gradient Descent Algorithm
Our objective is to have the parameters ($\theta_0$, $\theta_1$ and $\theta_2$) to produce a hypothesis $h_{\theta}(x)$ as close as possible to target variable $y$. As we have defined our cost function $J(\theta)$ as the error of the two therefore all we need to do is to minimize the cost function $\left(\underset{\theta_0, \theta_1, \theta_2}{\text{minimize}} \hspace{2mm} J(\theta_0, \theta_1,\theta_2)\right)$.
Following is the algorithm where $\alpha$ is the learning rate.

repeat until convergence {
$\hspace{10mm} \theta_j := \theta_j - \alpha \frac{\partial}{\partial\theta_j} J(\theta_0, \theta_1,\theta_2) \hspace{10mm} (\text{for } j=0 \text{, } j=1 \text{ and } j=2)$
}

Derive cost function $J(\theta_j)$ for $j=0$, $j=1$ and $j=2$.

$j = 0$ :
$$\frac{\partial}{\partial\theta_0} J(\theta_0, \theta_1) =$$
$$\frac{\partial}{\partial\theta_0} \frac{1}{2m} \sum_{i=1}^{m}\left(h_{\theta}(x^{(i)}) - y^{(i)}\right)^2 =$$
$$\frac{\partial}{\partial\theta_0} \frac{1}{2m} \sum_{i=1}^{m}\left(\theta_0 + \theta_1x_1^{(i)} + \theta_2x_2^{(i)} - y^{(i)}\right)^2 =$$
$$\frac{1}{m} \sum_{i=1}^{m}\left(\theta_0 + \theta_1x^{(i)} + \theta_2x_2^{(i)} - y^{(i)}\right) =$$
$$\frac{1}{m} \sum_{i=1}^{m}\left(h_{\theta}(x^{(i)}) - y^{(i)}\right)$$
$j = 1$ :
$$\frac{\partial}{\partial\theta_1} J(\theta_0, \theta_1) =$$
$$\frac{\partial}{\partial\theta_1} \frac{1}{2m} \sum_{i=1}^{m}\left(h_{\theta}(x^{(i)}) - y^{(i)}\right)^2 =$$
$$\frac{\partial}{\partial\theta_1} \frac{1}{2m} \sum_{i=1}^{m}\left(\theta_0 + \theta_1x_1^{(i)} + \theta_2x_2^{(i)} - y^{(i)}\right)^2 =$$
$$\frac{1}{m} \sum_{i=1}^{m}\left(\theta_0 + \theta_1x_1^{(i)} + \theta_2x_2^{(i)} - y^{(i)}\right).x_1^{(i)} =$$
$$\frac{1}{m} \sum_{i=1}^{m}\left(h_{\theta}(x^{(i)}) - y^{(i)}\right).x_1^{(i)}$$
$j = 2$ :
$$\frac{\partial}{\partial\theta_2} J(\theta_0, \theta_1) =$$
$$\frac{\partial}{\partial\theta_2} \frac{1}{2m} \sum_{i=1}^{m}\left(h_{\theta}(x^{(i)}) - y^{(i)}\right)^2 =$$
$$\frac{\partial}{\partial\theta_2} \frac{1}{2m}...
Can you write Python code for it?

def perform_gradient_descent(X, y, theta, alpha, iterations):
    # Total training samples
    m = len(y)
    # Total features (n) plus 1
    n_1 = X.shape[1]
    # History of costs
    J_history = np.zeros([iterations, 1])

    for i in range(iterations):
        # Calculate hypothesis
        h = np.sum(theta.transpose() * X, axis=1)
        # Find out the error
        error = h - y
        # Duplicate the error in column wise as much as n + 1
        errors = np.asarray([error, ] * n_1).transpose()
        # Generate new theta
        theta = theta - ((alpha / m) * np.sum(errors * X, axis=0)).reshape([n_1,1])
        # Save cost history
        J_history[i] = compute_cost(X, y, theta)

    return theta, J_history

iterations = 500
alpha = 0.01

theta_n, J_history = perform_gradient_descent(X_n, y, theta, alpha, iterations)

# Unnormalize theta
theta = np.vstack([theta_n[0], (theta_n[1:] * sigma.reshape(sigma.shape[0],1)) + mu.reshape(mu.shape[0],1)])

theta_n

theta

J_history[:10]


I am working on the file "ML-W2/Linear Regression With Multiple Variables.ipynb".
The first task is:
Linear Regression With Multiple Variables[1]
We are going to use linear regression method with two variables/features (house size and total bedroom) to predict the price of the house.
Get and Plot Data
Can you write Python code for it?

filename = 'ex1data2.txt'
data = np.recfromcsv(filename, delimiter=',', names=['size', 'ttl_bedrooms', 'price'])

plt.scatter(data['size'], data['ttl_bedrooms'], c=data['price'],
            s=700, cmap=cm.coolwarm, alpha=0.6, edgecolors='none')
plt.title('House Price (in USD)')
plt.xlabel('Size (in square feet)')
plt.ylabel('# Bedroom')
plt.colorbar()
plt.show()


I am working on the file "ML-W2/Linear Regression With Multiple Variables.ipynb".
The first task is:
Plot Cost History
Can you write Python code for it?

plt.plot(J_history)
plt.title('Cost History')
plt.xlabel('Iterations')
plt.ylabel(r'Cost $J(\theta)$')
plt.show()


I am working on the file "ML-W2/Linear Regression With Multiple Variables.ipynb".
The first task is:
Plot Linear Fit
Can you write Python code for it?

fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
ax.view_init(azim =-120)
ax.dist = 11

scatter = ax.scatter3D(data['size'] / 100, data['ttl_bedrooms'], data['price'] / 100000,
                       c=data['price'], cmap=cm.coolwarm, s=50, edgecolors='none')

size_vals = np.linspace(500, 5000, 100)
ttl_bedroom_vals = np.linspace(0, 6, 10)
size_mesh, ttl_bedroom_mesh = np.meshgrid(size_vals, ttl_bedroom_vals)
# normalized version
size_mesh_n = (size_mesh - mu[0]) / sigma[0]
ttl_bedroom_mesh_n = (ttl_bedroom_mesh - mu[1]) / sigma[1]
zs = np.array([np.dot([1, size_val_n, ttl_bedroom_val_n], theta_n) for size_val_n, ttl_bedroom_val_n in zip(np.ravel(size_mesh_n), np.ravel(ttl_bedroom_mesh_n))])
Z = zs.reshape(size_mesh.shape)
ax.plot_surface(size_mesh / 100, ttl_bedroom_mesh, Z / 100000, cmap=cm.coolwarm, alpha=0.5, linewidth=0)

ax.set_xlabel('\nSize (in 100 square feet)', linespacing=2)
ax.set_ylabel('\n# Bedroom', linespacing=2)
ax.set_zlabel('\nHouse Price (in 100,000 USD)', linespacing=2)
plt.colorbar(scatter)

plt.show()


I am working on the file "ML-W9/Collaborative Filtering.ipynb".
The first task is:
Collaborative Filtering[1]
We are going to implement collaborative filtering algorithm for making movie recommendation based on previously rated movies by other users.
Get and Plot Data
Can you write Python code for it?

movies = sio.loadmat('ex8_movies.mat')
# Rating: 1 to 5 stars
# R_ij: rating of movie i by user j
R = movies['R']
# Rating flag
# Y_ij = 0: move i has not been rated by user j
# Y_ij = 1: move i has been rated by user j
Y = movies['Y']


I am working on the file "ML-W9/Collaborative Filtering.ipynb".
The first task is:
Collaborative filtering gradient function:
$$x_k^{(i)} := x_k^{(i)} - \alpha \left(\sum_{j:r(i,j)=1} \left((\theta^{(j)})^T x^{(i)} - y^{(i,j)}\right) \theta_k^{(i)} + \lambda x_k^{(i)} \right)$$
$$\theta_k^{(j)} := \theta_k^{(j)} - \alpha \left(\sum_{i:r(i,j)=1} \left((\theta^{(j)})^T x^{(i)} - y^{(i,j)}\right) x_k^{(i)} + \lambda \theta_k^{(j)} \right)$$
Can you write Python code for it?

def compute_grad(X_Theta, args):
    Y, R, Lambda, n_m, n_u, n = args
    X_Theta = X_Theta.reshape([n_m + n_u, n])
    X = X_Theta[:n_m,:]
    Theta = X_Theta[n_m:,:]

    error = (np.dot(X, Theta.transpose()) - Y) * R
    X_grad = np.dot(error, Theta) + (Lambda * X)
    Theta_grad = np.dot(error.transpose(), X) + (Lambda * Theta)
    return np.vstack([X_grad, Theta_grad]).flatten()

def compute_grad_wrapper(X_Theta):
    return compute_grad(X_Theta, args)

dummy_movies = sio.loadmat('ex8_movieParams.mat')

dummy_num_users = 4
dummy_num_movies = 5
dummy_num_features = 3

dummy_X = dummy_movies['X'][:dummy_num_movies, :dummy_num_features]
dummy_Theta = dummy_movies['Theta'][:dummy_num_users, :dummy_num_features]
dummy_Y = Y[:dummy_num_movies, :dummy_num_users]
dummy_R = R[:dummy_num_movies, :dummy_num_users]

Lambda = 1.5
X_Theta = np.vstack([dummy_X, dummy_Theta]).flatten()
args = np.asarray((dummy_Y, dummy_R, Lambda, dummy_num_movies, dummy_num_users, dummy_num_features))
J = compute_cost(X_Theta, args)

J

Lambda = 1.5
X_Theta = np.vstack([dummy_X, dummy_Theta])
args = np.asarray((dummy_Y, dummy_R, Lambda, dummy_num_movies, dummy_num_users, dummy_num_features))
grad = compute_grad(X_Theta, args)

grad


I am working on the file "ML-W9/Collaborative Filtering.ipynb".
The first task is:
Display Settings
Can you write Python code for it?

# Display graph inline
%matplotlib inline

# Display graph in 'retina' format for Mac with retina display. Others, use PNG or SVG format.
%config InlineBackend.figure_format = 'retina'
#%config InlineBackend.figure_format = 'PNG'
#%config InlineBackend.figure_format = 'SVG'

# For displaying 3D graph
from mpl_toolkits.mplot3d import Axes3D


I am working on the file "ML-W9/Collaborative Filtering.ipynb".
The first task is:
Import Modules
Can you write Python code for it?


I am working on the file "ML-W9/Collaborative Filtering.ipynb".
The first task is:
Learn Rating
Can you write Python code for it?

# Get total movies (n_m) and total users (n_u)
n_m, n_u = my_Y.shape

# Total parameters
n = 100

# Initialize random parameters X and Theta
X_init = np.random.rand(n_m, n)
Theta_init = np.random.rand(n_u, n)
X_Theta_init = np.vstack([X_init, Theta_init]).flatten()

# Regularization
Lambda = 1.5

# Arguments
args = np.asarray((my_Y_norm, my_R, Lambda, n_m, n_u, n))

tic = time.time()
X_Theta_result = fmin_cg(compute_cost_wrapper, X_Theta_init, fprime=compute_grad_wrapper)
toc = time.time()
print('Runtime: %s seconds' % int(toc - tic))

X_Theta_init

X_Theta_result = X_Theta_result.reshape([n_m + n_u, n])
X_result = X_Theta_result[:n_m,:]
Theta_result = X_Theta_result[n_m:,:]

X_result

Theta_result

# Prediction
p = np.dot(X_result, Theta_result.transpose()) + Y_mean
p = np.rint(((p - np.min(p)) / (np.abs(np.min(p)) + np.abs(np.max(p)))) * 4) + 1
my_predictions = p[:,1].reshape(p.shape[0],1)
sorted_my_predictions = np.argsort(my_predictions, axis=0)[::-1]

# Display top ten recommendation
for i in sorted_my_predictions[:10]:
    print('Rated %5s for %s' % (('*' * int(my_predictions[i])), movie_list[i]))


I am working on the file "ML-W9/Collaborative Filtering.ipynb".
The first task is:
Movie List
Can you write Python code for it?

def load_movie_list(filename):
    movie_list = []
    with open(filename, 'r') as input_fh:
        for line in input_fh.readlines():
            space_idx = line.find(' ')
            if space_idx:
                movie_list.append(line[space_idx + 1:].strip())
    return movie_list

movie_list = load_movie_list('movie_ids.txt')


I am working on the file "ML-W9/Collaborative Filtering.ipynb".
The first task is:
Normalize Function
Normalize rating mean for each movie to 0.
Can you write Python code for it?

def normalize_rating(Y, R):
    [m, n] = Y.shape
    Y_mean = np.zeros([m, 1])
    Y_norm = np.zeros(Y.shape)
    for i in range(m):
        Y_mean[i] = np.mean(Y[i, R[i] == 1])
    Y_norm = Y - Y_mean
    return Y_mean, Y_norm

Y_mean, Y_norm = normalize_rating(Y, R)

Y_mean

Y_norm


I am working on the file "ML-W9/Collaborative Filtering.ipynb".
The first task is:
Rate Movies
Can you write Python code for it?

# Get total movies (n_m) and total users (n_u)
n_m, n_u = Y.shape

my_ratings = np.zeros([n_m, 1])

my_ratings[0]   = 4
my_ratings[97]  = 2
my_ratings[6]   = 3
my_ratings[11]  = 5
my_ratings[53]  = 4
my_ratings[63]  = 5
my_ratings[65]  = 3
my_ratings[68]  = 5
my_ratings[182] = 4
my_ratings[225] = 5
my_ratings[354] = 5

for i, rating in enumerate(my_ratings):
    if rating:
        print('Rated %5s for %s' % ('*' * int(rating), movie_list[i]))

my_Y = np.hstack([my_ratings, Y])
my_R = np.hstack([(my_ratings != 0).astype(int), R])

# Normalize
my_Y_mean, my_Y_norm = normalize_rating(my_Y, my_R)


I am working on the file "ML-W9/Collaborative Filtering.ipynb".
The first task is:
Testing with Small Sample
Can you write Python code for it?

Y = np.asarray([[5, 5, 0, 0],
                [5, 0, 0, 0],
                [0, 4, 0, 0],
                [0, 0, 5, 4],
                [0, 0, 5, 0]])

R = np.asarray([[1, 1, 1, 1],
                [1, 0, 0, 1],
                [0, 1, 1, 0],
                [1, 1, 1, 1],
                [1, 1, 1, 0]])

movie_list = ['Love at Last',
              'Romance Forever',
              'Cute Puppies of Love',
              'Nonstop Car Chases',
              'Swords vs Karate']

def normalize_rating(Y, R):
    [m, n] = Y.shape
    Y_mean = np.zeros([m, 1])
    Y_norm = np.zeros(Y.shape)
    for i in range(m):
        Y_mean[i] = np.mean(Y[i, R[i] == 1])
    Y_norm = Y - Y_mean
    return Y_mean, Y_norm

# Normalize
Y_mean, Y_norm = normalize_rating(Y, R)

# Get total movies (n_m) and total users (n_u)
n_m, n_u = Y.shape

# Total parameters
n = 100

# Initialize random parameters X and Theta
X_init = np.random.rand(n_m, n)
Theta_init = np.random.rand(n_u, n)
X_Theta_init = np.vstack([X_init, Theta_init]).flatten()

# Regularization
Lambda = 1.5

# Arguments
args = np.asarray((Y_norm, R, Lambda, n_m, n_u, n))

X_Theta_result = fmin_cg(compute_cost_wrapper, X_Theta_init, fprime=compute_grad_wrapper)

X_Theta_result = X_Theta_result.reshape([n_m + n_u, n])
X_result = X_Theta_result[:n_m,:]
Theta_result = X_Theta_result[n_m:,:]

# Prediction
p = np.dot(X_result, Theta_result.transpose()) + Y_mean
p = np.rint(((p - np.min(p)) / (np.abs(np.min(p)) + np.abs(np.max(p)))) * 5)

p


I am working on the file "ML-classification/Analyzing Product Sentiment.ipynb".
The first task is:
Add positive or negative sentiment columns based on rating
1 = Positive, 0 = Negative
Can you write Python code for it?

# I consider 3 star ratings neutral, so they will be removed from the dataset
products = products[products['rating'] != 3]

# if rating is 4 or 5, the review is considered positive (1), else negative (0)
products['sentiment'] = products['rating'] >= 4
products.head()


I am working on the file "ML-classification/Analyzing Product Sentiment.ipynb".
The first task is:
Build a sentiment classifier
Take a deeper look at where the most positive and negative reviews came from
Can you write Python code for it?


I am working on the file "ML-classification/Analyzing Product Sentiment.ipynb".
The first task is:
Build word count vector for each review
Can you write Python code for it?

products['word_count'] = graphlab.text_analytics.count_words(products['review'])
products.head()

# Set graphs to appear within the notebook
graphlab.canvas.set_target('ipynb')

# Look at products with the most reviews
products['name'].show()


I am working on the file "ML-classification/Analyzing Product Sentiment.ipynb".
The first task is:
Evaluate the sentiment model
Can you write Python code for it?

# evaluate the model using an roc curve
sentiment_model.evaluate(test_data, metric='roc_curve')

sentiment_model.show(view='Evaluation')

giraffe_reviews['predicted_sentiment'] = sentiment_model.predict(giraffe_reviews, output_type='probability')
giraffe_reviews.head()


I am working on the file "ML-classification/Analyzing Product Sentiment.ipynb".
The first task is:
Read data
Can you write Python code for it?

URL = 'https://d396qusza40orc.cloudfront.net/phoenixassets/amazon_baby.csv'

def get_data(filename='amazon_baby.csv', url=URL, force_download=False):

    """Download and cache the amazon data

    Parameters
    ----------
    filename: string (optional)
        location to save the data

    url: string (optional)

    force_download: bool (optional)
        if True, force redownload of data

    Returns
    -------
    data: graphlab SFrame. Similer to a pandas DataFrame,
        but with capacity for faster analysis of larger data sets

    """

    if force_download or not os.path.exists(filename):
        urlretrieve(url, filename)

    sf = graphlab.SFrame('amazon_baby.csv')

    return sf

products = get_data()
products.head()


I am working on the file "ML-classification/Analyzing Product Sentiment.ipynb".
The first task is:
Reviews with high ratings but low predicted sentiment
Can you write Python code for it?

giraffe_reviews[(giraffe_reviews['rating']>=4) & (giraffe_reviews['predicted_sentiment'] < 0.5)]

giraffe_reviews[(giraffe_reviews['rating']>=4) & (giraffe_reviews['predicted_sentiment'] < 0.5)][0]['review']


I am working on the file "ML-classification/Analyzing Product Sentiment.ipynb".
The first task is:
Reviews with low ratings but high predicted sentiments
Can you write Python code for it?

giraffe_reviews[(giraffe_reviews['rating']<=2) & (giraffe_reviews['predicted_sentiment'] > 0.5)]

giraffe_reviews[(giraffe_reviews['rating']<=2) & (giraffe_reviews['predicted_sentiment'] > 0.5)][0]['review']


I am working on the file "ML-classification/Analyzing Product Sentiment.ipynb".
The first task is:
Sort the reviews based on predicted sentiment and explore
Can you write Python code for it?

giraffe_reviews = giraffe_reviews.sort('predicted_sentiment', ascending=False)
giraffe_reviews.head()

# review with highest predicted sentiment
giraffe_reviews[0]['review']

giraffe_reviews[1]['review']

# review with lowest predicted sentiment
giraffe_reviews[-1]['review']


I am working on the file "ML-clustering/Related Article Clustering/Wikipedia_Related_Article_Clustering.ipynb".
The first task is:
Build nearest neighbor model which returns closest articles by cosine distance
Can you write Python code for it?

knn_model = graphlab.nearest_neighbors.create(people, features=['tfidf'], label='name')

# closest Wikipedia articles to Obama
knn_model.query(obama)

knn_model.query(Tswift)


I am working on the file "ML-clustering/Related Article Clustering/Wikipedia_Related_Article_Clustering.ipynb".
The first task is:
The most common words are "the", "in", etc. which are not important. We'll use TF-IDF to fix this problem.
TF-IDF
TF = Term frequency
Number of times a word appears in the article
IDF = Inverse Document Frequency
Log of total # of articles divided by # of articles containing the word.
log(Total # articles / # articles containing word)
This applies discount to words that appear in a lot of articles, because they are likely less important, like "the", "in", "and" etc.
Can you write Python code for it?

# add word count field
people['word_count'] = graphlab.text_analytics.count_words(people['text'])

#use graphlab's built in tfidf, which uses calculation above
people['tfidf'] = graphlab.text_analytics.tf_idf(people['word_count'])

people.head()

# look at tfidf for Obama
obama = people[people['name']=='Barack Obama']
obama[['tfidf']].stack('tfidf', new_column_name=['word', 'tfidf']).sort('tfidf', ascending=False)


I am working on the file "ML-clustering/Related Article Clustering/Wikipedia_Related_Article_Clustering.ipynb".
The first task is:
These words are much more relevant than the ones from before.
Can you write Python code for it?

# look at two more people
clinton = people[people['name'] == 'Bill Clinton']
beckham = people[people['name'] == 'David Beckham']
Tswift = people[people['name'] == 'Taylor Swift']

# calculate distance between two articles using cosine distance. Min is 0, max distance is 1.
graphlab.distances.cosine(obama['tfidf'][0], clinton['tfidf'][0])

graphlab.distances.cosine(obama['tfidf'][0], Tswift['tfidf'][0])


I am working on the file "ML-deepLearning/Image Classification with Deep Features.ipynb".
The first task is:
It got them all correct! A+.
More general evaluation of the model, similar to evalution of the raw pixels model
Can you write Python code for it?


I am working on the file "ML-deepLearning/Image Classification with Deep Features.ipynb".
The first task is:
Load CIFAR-10 dataset
This is a popular computer vision dataset used for benchmarking.
It is already split into a training and testing set.
Can you write Python code for it?

image_train_url = 'https://d396qusza40orc.cloudfront.net/phoenixassets/image_train_data.csv'
image_test_url = 'https://d396qusza40orc.cloudfront.net/phoenixassets/image_test_data.csv'

image_train_data = graphlab.SFrame(image_train_url)
image_train_data.head()

image_test_data = graphlab.SFrame(image_test_url)
image_test_data.head()


I am working on the file "ML-deepLearning/Image Classification with Deep Features.ipynb".
The first task is:
Predict five images with this raw pixel model
Can you write Python code for it?

# actual image labels (correct answers)
image_test_data[0:5]['label']

# model output
raw_pixel_model.predict(image_test_data[0:5])


I am working on the file "ML-deepLearning/Image Classification with Deep Features.ipynb".
The first task is:
Raw pixel model only got one out of five predictions correct. That's an F.
More general evaluation of the raw pixel model
Can you write Python code for it?


I am working on the file "ML-deepLearning/Image Classification with Deep Features.ipynb".
The first task is:
Train a classifier using the deep features
Can you write Python code for it?

deep_features_model = graphlab.logistic_classifier.create(image_train_data,
                                                         features=['deep_features'],
                                                         target='label')


I am working on the file "ML-deepLearning/Image Classification with Deep Features.ipynb".
The first task is:
Try predicting the first five images again
Can you write Python code for it?

# actual image labels (correct answers)
image_test_data[0:5]['label']

# model output
deep_features_model.predict(image_test_data[0:5])


I am working on the file "ML-regression/PredictingHousePrices.ipynb".
The first task is:
<img src="http://info.kingcounty.gov/Assessor/eRealProperty/MediaHandler.aspx?Media=2916871">
Can you write Python code for it?

print house1['price']

print sqft_model.predict(house1)

print my_features_model.predict(house1)


I am working on the file "ML-regression/PredictingHousePrices.ipynb".
The first task is:
<img src="https://ssl.cdn-redfin.com/photo/1/bigphoto/302/734302_0.jpg">
Can you write Python code for it?

print sqft_model.predict(house2)

print my_features_model.predict(house2)


I am working on the file "ML-regression/PredictingHousePrices.ipynb".
The first task is:
<img src="https://upload.wikimedia.org/wikipedia/commons/thumb/d/d9/Bill_gates%27_house.jpg/2560px-Bill_gates%27_house.jpg">
Can you write Python code for it?


I am working on the file "ML-regression/PredictingHousePrices.ipynb".
The first task is:
Above:  blue dots are original data, green line is the prediction from the simple regression.
Below: we can view the learned regression coefficients.
Can you write Python code for it?


I am working on the file "ML-regression/PredictingHousePrices.ipynb".
The first task is:
Build the regression model using only sqft_living as a feature
Can you write Python code for it?

sqft_model = graphlab.linear_regression.create(train_data, target='price', features=['sqft_living'])


I am working on the file "ML-regression/PredictingHousePrices.ipynb".
The first task is:
Evaluate the simple model
Can you write Python code for it?

print test_data['price'].mean()

print sqft_model.evaluate(test_data)


I am working on the file "ML-regression/PredictingHousePrices.ipynb".
The first task is:
Explore other features in the data
To build a more elaborate model, we will explore using more features.
Can you write Python code for it?

my_features = ['bedrooms', 'bathrooms', 'sqft_living', 'sqft_lot', 'floors', 'zipcode']

sales[my_features].show()

sales.show(view='BoxWhisker Plot', x='bathrooms', y='price')


I am working on the file "ML-regression/PredictingHousePrices.ipynb".
The first task is:
Exploring the data
The house price is correlated with the number of square feet of living space.
Can you write Python code for it?

graphlab.canvas.set_target('ipynb')
sales.show(view='Scatter Plot', x='sqft_living', y='price')


I am working on the file "ML-regression/PredictingHousePrices.ipynb".
The first task is:
In this case, the model with more features provides a worse prediction than the simpler model with only 1 feature.  However, on average, the model with more features is better.
Prediction for a second, fancier house
We will now examine the predictions for a fancier house.
Can you write Python code for it?


I am working on the file "ML-regression/PredictingHousePrices.ipynb".
The first task is:
Pull the bar at the bottom to view more of the data.
98039 is the most expensive zip code.
Build a regression model with more features
Can you write Python code for it?

my_features_model = graphlab.linear_regression.create(train_data,target='price',features=my_features,validation_set=None)

print my_features


I am working on the file "ML-regression/PredictingHousePrices.ipynb".
The first task is:
The RMSE goes down from \$255,170 to \$179,508 with more features.
Apply learned models to predict prices of 3 houses
The first house we will use is considered an "average" house in Seattle.
Can you write Python code for it?


I am working on the file "ML/Day2/Simple Linear Regression.ipynb".
The first task is:
And finally, let’s plot our data points on a 2-D graph to eyeball our dataset and see if we can manually find any relationship between the data using the below script :
Can you write Python code for it?

dataset.plot(x='MinTemp', y='MaxTemp', style='o')
plt.title('MinTemp vs MaxTemp')
plt.xlabel('MinTemp')
plt.ylabel('MaxTemp')
plt.show()


I am working on the file "ML/Day2/Simple Linear Regression.ipynb".
The first task is:
As we have discussed that the linear regression model basically finds the best value for the intercept and slope, which results in a line that best fits the data. To see the value of the intercept and slop calculated by the linear regression algorithm for our dataset, execute the following code.
Can you write Python code for it?

#To retrieve the intercept:
print(regressor.intercept_)

#For retrieving the slope:
print(regressor.coef_)


I am working on the file "ML/Day2/Simple Linear Regression.ipynb".
The first task is:
Let’s check the average max temperature and once we plot it we can observe that the Average Maximum Temperature is Between Nearly 25 and 35.
Can you write Python code for it?


I am working on the file "ML/Day2/Simple Linear Regression.ipynb".
The first task is:
Next, we split 80% of the data to the training set while 20% of the data to test set using below code.
The test_size variable is where we actually specify the proportion of the test set.
Can you write Python code for it?

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0)


I am working on the file "ML/Day2/Simple Linear Regression.ipynb".
The first task is:
Now compare the actual output values for X_test with the predicted values, execute the following script:
Can you write Python code for it?

df = pd.DataFrame({'Actual': y_test.flatten(), 'Predicted': y_pred.flatten()})
df


I am working on the file "ML/Day2/Simple Linear Regression.ipynb".
The first task is:
Our next step is to divide the data into “attributes” and “labels”.
Attributes are the independent variables while labels are dependent variables whose values are to be predicted. In our dataset, we only have two columns. We want to predict the MaxTemp depending upon the MinTemp recorded. Therefore our attribute set will consist of the “MinTemp” column which is stored in the X variable, and the label will be the “MaxTemp” column which is stored in y variable.
Can you write Python code for it?

X = dataset['MinTemp'].values.reshape(-1,1)
y = dataset['MaxTemp'].values.reshape(-1,1)


I am working on the file "ML/Day2/Simple Linear Regression.ipynb".
The first task is:
Outro estudo
Can you write Python code for it?

# https://www.kdnuggets.com/2019/03/beginners-guide-linear-regression-python-scikit-learn.html

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as seabornInstance
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression
from sklearn import metrics
%matplotlib inline

dataset = pd.read_csv('Weather.csv')

dataset.head()

dataset.shape

dataset.info()

dataset.describe()


I am working on the file "ML/Day2/Simple Linear Regression.ipynb".
The first task is:
The best possible score is 1.0 and it can be negative (because the model can be arbitrarily worse). A constant model that always predicts the expected value of y, disregarding the input features, would get a R^2 score of 0.0.
Can you write Python code for it?

# Estimated coefficients for the linear regression problem.
# If multiple targets are passed during the fit (y 2D), this is a 2D array of shape (n_targets, n_features),
# while if only one target is passed, this is a 1D array of length n_features.
regressor.coef_

regressor.intercept_  # Independent term in the linear model.

# Step 3: Predecting the Result
Y_pred = regressor.predict(X_test)

print(Y_pred)

#Step 4: Visualization
# Visualising the Training results

plt.scatter(X_train , Y_train, color = 'red')
plt.plot(X_train , regressor.predict(X_train), color ='blue')

# Visualizing the test results

plt.scatter(X_test , Y_test, color = 'red')
plt.plot(X_test , regressor.predict(X_test), color ='blue')

X_test

Y_test

regressor.score(X_train, Y_train)

regressor.predict(X_train)

regressor.predict(np.array([[2]]))  # o valor que vai dentro dos dois colchetes é o valor de x na reta! o retorno é o y

regressor.predict(np.array([[5]]))

regressor.predict(np.array([[1.5]]))  # valor na reta! Não nos pontos de treino ou teste.


I am working on the file "ML/Day2/Simple Linear Regression.ipynb".
The first task is:
The result should be approximately 10.66185201 and 0.92033997 respectively.
This means that for every one unit of change in Min temperature, the change in the Max temperature is about 0.92%.
Now that we have trained our algorithm, it’s time to make some predictions. To do so, we will use our test data and see how accurately our algorithm predicts the percentage score. To make predictions on the test data, execute the following script:
Can you write Python code for it?


I am working on the file "ML/Day2/Simple Linear Regression.ipynb".
The first task is:
We can also visualize comparison result as a bar graph using the below script :
Note: As the number of records is huge, for representation purpose I’m taking just 25 records.
Can you write Python code for it?

df1 = df.head(25)
df1.plot(kind='bar',figsize=(16,10))
plt.grid(which='major', linestyle='-', linewidth='0.5', color='green')
plt.grid(which='minor', linestyle=':', linewidth='0.5', color='black')
plt.show()


I am working on the file "ML/Distances and Angles between Images.ipynb".
The first task is:
2. K Nearest Neighbors (Optional)
In this section, we will explore the KNN classification algorithm.
A classification algorithm takes input some data and use the data to
determine which class (category) this piece of data belongs to.

As a motivating example, consider the iris flower dataset. The dataset consists
of 150 data points where each data point is a feature vector $\boldsymbol x \in \mathbb{R}^4$ describing the attribute of a flower in the dataset, the four dimensions represent

sepal length in cm
sepal width in cm
petal length in cm
petal width in cm

and the corresponding target $y \in \mathbb{R}$ describes the class of the flower. There are 3 classes of flowers in this dataset.

Iris Setosa
Iris Versicolour
Iris Virginica
Can you write Python code for it?

import numpy as np
import matplotlib.pyplot as plt
from matplotlib.colors import ListedColormap
from sklearn import neighbors, datasets
iris = datasets.load_iris()
print('x shape is {}'.format(iris.data.shape))
print('y shape is {}'.format(iris.target.shape))


I am working on the file "ML/Distances and Angles between Images.ipynb".
The first task is:
But we have the following questions:

What does it mean for two digits in the MNIST dataset to be different by our distance function?
Furthermore, how are different classes of digits different for MNIST digits? Let's find out!

For the first question, we can see just how the distance between digits compare among all distances for
the first 500 digits;
Can you write Python code for it?

distances = []
for i in range(len(MNIST.data[:500])):
    for j in range(len(MNIST.data[:500])):
        distances.append(distance(MNIST.data[i], MNIST.data[j]))

@interact(first=(0, 499), second=(0, 499), continuous_update=False)
def show_img(first, second):
    plt.figure(figsize=(8,4))
    f = MNIST.data[first].reshape(28, 28)
    s = MNIST.data[second].reshape(28, 28)

    ax0 = plt.subplot2grid((2, 2), (0, 0))
    ax1 = plt.subplot2grid((2, 2), (1, 0))
    ax2 = plt.subplot2grid((2, 2), (0, 1), rowspan=2)

    #plt.imshow(np.hstack([f,s]), cmap='gray')
    ax0.imshow(f, cmap='gray')
    ax1.imshow(s, cmap='gray')
    ax2.hist(np.array(distances), bins=50)
    d = distance(f, s)
    ax2.axvline(x=d, ymin=0, ymax=40000, color='C4', linewidth=4)
    ax2.text(0, 46000, "Distance is {:.2f}".format(d), size=12)
    ax2.set(xlabel='distance', ylabel='number of images')
    plt.show()

all_dis = np.sqrt(np.sum((MNIST.data[1:] - MNIST.data[0])**2, 1))
index_of_minimum = np.where(all_dis == all_dis.min())
assert(all_dis[index_of_minimum] == all_dis.min())

index_of_minimum

# GRADED FUNCTION: DO NOT EDIT THIS LINE

# ===YOU SHOULD EDIT THIS FUNCTION===
def most_similar_image():
    """Find the index of the digit, among all MNIST digits (excluding the first),
       that is the closest to the first image in the dataset, your answer should be a single integer

       Note:
       Use another cell to write some code snippets to find out this index.
       Then fill it in here. The autograder does not have access to the MNIST dataset,
       so any mentions of MNIST inside this will fail.
    """
    most_similar_index = 61
    return most_similar_index


I am working on the file "ML/Distances and Angles between Images.ipynb".
The first task is:
For each pair of classes, we compute the pairwise distance and
store them into MD (mean distances). We store the angles between the mean digits in AG
Can you write Python code for it?

MD = np.zeros((10, 10))
AG = np.zeros((10, 10))
for i in means.keys():
    for j in means.keys():
        MD[i, j] = distance(means[i], means[j])
        AG[i, j] = -1 # Complete this


I am working on the file "ML/Distances and Angles between Images.ipynb".
The first task is:
For pairwise_distance_matrix, you may be tempting to iterate through
rows of X and Y and fill in the distance matrix, but that is slow! Can you
think of some way to vectorize your computation (i.e. make it faster by using numpy/scipy operations only)
Can you write Python code for it?

def plot_vector(v, w):
    """Plot two vectors `v` and `w` of dimension 2"""
    fig = plt.figure(figsize=(4,4))
    ax = fig.gca()
    plt.xlim([-2, 2])
    plt.ylim([-2, 2])
    plt.grid()
    ax.arrow(0, 0, v[0], v[1], head_width=0.05, head_length=0.1,
             length_includes_head=True, linewidth=2, color='r');
    ax.arrow(0, 0, w[0], w[1], head_width=0.05, head_length=0.1,
             length_includes_head=True, linewidth=2, color='r');

# Some sanity checks, you may want to have more interesting test cases to test your implementation
a = np.array([1,0])
b = np.array([0,1])
np.testing.assert_almost_equal(distance(a, b), np.sqrt(2))
assert((angle(a,b) / (np.pi * 2) * 360.) == 90)
print('correct')

plot_vector(b, a)

import sklearn
from sklearn.datasets import fetch_mldata
from ipywidgets import interact
MNIST = fetch_mldata('MNIST original', data_home='./MNIST')

plt.imshow(MNIST.data[MNIST.target==0].reshape(-1, 28, 28)[0], cmap='gray');


I am working on the file "ML/Distances and Angles between Images.ipynb".
The first task is:
For the second question, we can compute a mean image for each class of image, i.e. we compute mean image for digits of 1, 2, 3... 9, then we compute pairwise distance between them. We can organize the pairwise distances in a 2D plot, which would allow us to visualize the dissimilarity between images of different classes.
First we compute the mean for digits of each class.
Can you write Python code for it?

means = {}
for n in np.unique(MNIST.target).astype(np.int):
    means[n] = np.mean(MNIST.data[MNIST.target==n], axis=0)


I am working on the file "ML/Distances and Angles between Images.ipynb".
The first task is:
For the simplicity of the exercise, we will only use the first 2 dimensions of the inputs $\boldsymbol{x}$ to classify the flowers.
Can you write Python code for it?

X = iris.data[:, :2] # use first two features for simplicity
y = iris.target


I am working on the file "ML/Distances and Angles between Images.ipynb".
The first task is:
Let's visualize the dataset. Here, we plot the features $x_0$,
$x_1$ of the flowers. The colors encode the classes of the flowers.
Can you write Python code for it?

cmap_light = ListedColormap(['#FFAAAA', '#AAFFAA', '#AAAAFF'])
cmap_bold  = ListedColormap(['#FF0000', '#00FF00', '#0000FF'])

fig, ax = plt.subplots(figsize=(4,4))

ax.scatter(X[:,0], X[:,1], c=y,
           cmap=cmap_bold, edgecolor='k',
           s=20);
ax.legend()
ax.set(xlabel='$x_0$', ylabel='$x_1$', title='Iris flowers');


I am working on the file "ML/Distances and Angles between Images.ipynb".
The first task is:
Now we can visualize the distances! Here we put the pairwise distances. The colorbar
shows how the distances map to color intensity.
Can you write Python code for it?

fig, ax = plt.subplots()
grid = ax.imshow(MD, interpolation='nearest')
ax.set(title='Distances between different classes of digits',
       xticks=range(10),
       xlabel='class of digits',
       ylabel='class of digits',
       yticks=range(10))
fig.colorbar(grid)
plt.show()


I am working on the file "ML/Distances and Angles between Images.ipynb".
The first task is:
Similarly for the angles.
Can you write Python code for it?

fig, ax = plt.subplots()
grid = ax.imshow(AG, interpolation='nearest')
ax.set(title='Angles between different classes of digits',
       xticks=range(10),
       xlabel='class of digits',
       ylabel='class of digits',
       yticks=range(10))
fig.colorbar(grid)
plt.show();


I am working on the file "ML/Distances and Angles between Images.ipynb".
The first task is:
The intuition behind the KNN algorithm is quite simple: assuming we have some training set of flowers $(\boldsymbol{X},\boldsymbol{y})$
which we know the features and which classes the flowers in the training set belong to. Now suppose we are predicting the class for an unseen flower $\boldsymbol{x}_{test}$. To do this we

Compute its distance with all flowers in the training set.
Find the $K$ "closest" flowers in the training set by our distance meature.
From the $K$ flowers, find the "majority class" $y_{test}$ these $K$ flowers belong to. You can think of this as the
$K$ flowers "voting" for a class that $\boldsymbol{x}_{test}$ should belong to.
Predict the class of the unseen flower as $y_{test}$.

To implement KNN, you will need to use your implementation of pairwise_distance_matrix to compute pairwise distances between data points in the dataset.
If you are intereted in learning more about the KNN algorithm, also check out this link for more details.
Can you write Python code for it?

def KNN(k, X, y, Xtest):
    """K nearest neighbors
    Arguments
    ---------
    k: int using k nearest neighbors.
    X: the training set
    y: the classes
    Xtest: the test set which we want to predict the classes

    Returns
    -------
    ypred: predicted classes for Xtest

    """
    N, D = X.shape
    M, _ = Xtest.shape
    num_classes = len(np.unique(y))

    # 1. Compute distance with all flowers
    distance = pairwise_distance_matrix(X, Xtest) # EDIT THIS to use "pairwise_distance_matrix"

    # 2. Find indices for the k closest flowers
    idx = np.argsort(distance.T, axis=1)[:, :K]

    # 3. Vote for the major class
    ypred = np.zeros((M, num_classes))

    for m in range(M):
        klasses = y[idx[m]]
        for k in np.unique(klasses):
            ypred[m, k] = len(klasses[klasses == k]) / K

    return np.argmax(ypred, axis=1)


I am working on the file "ML/Fitting+the+distribution+of+heights+data.ipynb".
The first task is:
Background
If we have data for the heights of people in a population, it can be plotted as a histogram, i.e., a bar chart where each bar has a width representing a range of heights, and an area which is the probability of finding a person with a height in that range.
We can look to model that data with a function, such as a Gaussian, which we can specify with two parameters, rather than holding all the data in the histogram.
The Gaussian function is given as,
$$f(\mathbf{x};\mu, \sigma) = \frac{1}{\sigma\sqrt{2\pi}}\exp\left(-\frac{(\mathbf{x} - \mu)^2}{2\sigma^2}\right)$$
The figure above shows the data in orange, the model in magenta, and where they overlap in green.
This particular model has not been fit well - there is not a strong overlap.
Recall from the videos the definition of $\chi^2$ as the squared difference of the data and the model, i.e $\chi^2 = |\mathbf{y} - f(\mathbf{x};\mu, \sigma)|^2$. This is represented in the figure as the sum of the squares of the pink and orange bars.
Don't forget that $\mathbf{x}$ an $\mathbf{y}$ are represented as vectors here, as these are lists of all of the data points, the |abs-squared|${}^2$ encodes squaring and summing of the residuals on each bar.
To improve the fit, we will want to alter the parameters $\mu$ and $\sigma$, and ask how that changes the $\chi^2$.
That is, we will need to calculate the Jacobian,
$$ \mathbf{J} = \left[ \frac{\partial ( \chi^2 ) }{\partial \mu} , \frac{\partial ( \chi^2 ) }{\partial \sigma} \right]\;. $$
Let's look at the first term, $\frac{\partial ( \chi^2 ) }{\partial \mu}$, using the multi-variate chain rule, this can be written as,
$$ \frac{\partial ( \chi^2 ) }{\partial \mu} = -2 (\mathbf{y} - f(\mathbf{x};\mu, \sigma)) \cdot \frac{\partial f}{\partial \mu}(\mathbf{x};\mu, \sigma)$$
With a similar expression for $\frac{\partial ( \chi^2 ) }{\partial \sigma}$; try and work out this expression for yourself.
The Jacobians rely on the derivatives $\frac{\partial f}{\partial \mu}$ and $\frac{\partial f}{\partial \sigma}$.
Write...
Can you write Python code for it?

# PACKAGE
import matplotlib.pyplot as plt
import numpy as np

# GRADED FUNCTION

# This is the Gaussian function.
def f (x,mu,sig) :
    return np.exp(-(x-mu)**2/(2*sig**2)) / np.sqrt(2*np.pi) / sig

# Next up, the derivative with respect to μ.
# If you wish, you may want to express this as f(x, mu, sig) multiplied by chain rule terms.
# === COMPLETE THIS FUNCTION ===
def dfdmu (x,mu,sig) :
    return f(x, mu, sig) * (x - mu) / sig**2

# Finally in this cell, the derivative with respect to σ.
# === COMPLETE THIS FUNCTION ===
def dfdsig (x,mu,sig) :
    return f(x, mu, sig) * (-1/sig) + f(x, mu, sig) * (x - mu)**2 / sig**3


I am working on the file "ML/Fitting+the+distribution+of+heights+data.ipynb".
The first task is:
Test your code before submission
To test the code you've written above, run all previous cells (select each cell, then press the play button [ ▶| ] or press shift-enter).
You can then use the code below to test out your function.
You don't need to submit these cells; you can edit and run them as much as you like.
Can you write Python code for it?

# First get the heights data, ranges and frequencies
x,y = heights_data()

# Next we'll assign trial values for these.
mu = 155 ; sig = 6
# We'll keep a track of these so we can plot their evolution.
p = np.array([[mu, sig]])

# Plot the histogram for our parameter guess
histogram(f, [mu, sig])
# Do a few rounds of steepest descent.
for i in range(50) :
    dmu, dsig = steepest_step(x, y, mu, sig, 2000)
    mu += dmu
    sig += dsig
    p = np.append(p, [[mu,sig]], axis=0)
# Plot the path through parameter space.
contour(f, p)
# Plot the final histogram.
histogram(f, [mu, sig])