I am working on the file "MillikanOilDrop/Analysis.ipynb".
The first task is:
Correction Factor
Can you write Python code for it?


I am working on the file "MillikanOilDrop/Analysis.ipynb".
The first task is:
Down Charge
Can you write Python code for it?


I am working on the file "MillikanOilDrop/Analysis.ipynb".
The first task is:
Down Uncer
Can you write Python code for it?


I am working on the file "MillikanOilDrop/Analysis.ipynb".
The first task is:
Drop Charge
Can you write Python code for it?

def down_charge(vd,vf,gamma,mass):
    return gamma*mass*g*((vd/vf) - 1)/E

def delta_qd(qu,dm, m, dvu, vu, dvf, vf):
    return np.sqrt((dm/m)**2 + (dvu/(vu-vf))**2 + ((vu/vf)*dvf/(vf-vu))**2)*qu

qd0 = down_charge(td0_speed, speeds[0], gammas[0], masses[0])
qd1 = down_charge(td1_speed, speeds[1], gammas[1], masses[1])
qd2 = down_charge(td2_speed, speeds[2], gammas[2], masses[2])
qd3 = down_charge(td3_speed, speeds[3], gammas[3], masses[3])
qd = np.array([qd0,qd1,qd2,qd3])
qd*1e19

qd0_u = delta_qd(qd0,masses_u[0],masses[0],td0_speed_u, td0_speed, speeds_u[0], speeds[0])
qd1_u = delta_qd(qd1,masses_u[1],masses[1],td1_speed_u, td1_speed, speeds_u[1], speeds[1])
qd2_u = delta_qd(qd2,masses_u[2],masses[2],td2_speed_u, td2_speed, speeds_u[2], speeds[2])
qd3_u = delta_qd(qd3,masses_u[3],masses[3],td3_speed_u, td3_speed, speeds_u[3], speeds[3])
qd_u = np.array([qd0_u,qd1_u,qd2_u,qd3_u])
qd_u*1e19


I am working on the file "MillikanOilDrop/Analysis.ipynb".
The first task is:
Drop Charge
Downward Movement
Can you write Python code for it?

df['Run 1']

# Fall time in seconds when V = -
td = np.array([df[key]['t_d (s)'] for key in keys])
td

#### tu0: Time Up 0'th drop.
#### Average Time
#### Std of Times
#### Resulting Speed
#### Speed Uncertainty

td0 = td[0] #All values here are about the same
avg_td0 = np.mean(td0)
std_td0 = np.std(td0)
td0_speed = .5*1e-3/avg_td0
td0_speed_u = .5*1e-3/(avg_td0**2)*std_td0/np.sqrt(len(td0))

td1 = np.array([td[1]]) #All values here are about the same
avg_td1 = np.mean(td1)
std_td1 = np.std(td1)
td1_speed = .5*1e-3/avg_td1
td1_speed_u = .5*1e-3/(avg_td1**2)*std_td1/np.sqrt(len(td1))

td2 = np.array([td[2]]) #All values here are about the same
avg_td2 = np.mean(td2)
std_td2 = np.std(td2)
td2_speed = .5*1e-3/avg_td2
td2_speed_u = .5*1e-3/(avg_td2**2)*std_td2/np.sqrt(len(td2))

td3 = td[3] #All values here are about the same
avg_td3 = np.mean(td3)
std_td3 = np.std(td3)
td3_speed = .5*1e-3/avg_td3
td3_speed_u = .5*1e-3/(avg_td3**2)*std_td3/np.sqrt(len(td3))

avg_td0

std_td0/np.sqrt(len(td0))


I am working on the file "MillikanOilDrop/Analysis.ipynb".
The first task is:
E Field
Can you write Python code for it?

d = 7.63*1e-3 #Electrode spacer thickness in meters
V = 301.8 #Volts. Constant through all measurements

E = V/d #V/m
E


I am working on the file "MillikanOilDrop/Analysis.ipynb".
The first task is:
Fall Speed
All measurements were of the time to travel .5mm
Can you write Python code for it?

# Units of m/s
speeds = np.array([(.5*1e-3)/entry for entry in avg_tf ])
speeds


I am working on the file "MillikanOilDrop/Analysis.ipynb".
The first task is:
Mask points that lie outside 2.5$\sigma$
Can you write Python code for it?

tf = np.array([ma.masked_outside(entry, np.mean(entry) - 2.5*np.std(entry), np.mean(entry) + 2.5*np.std(entry))
                     for entry in tf])


I am working on the file "MillikanOilDrop/Analysis.ipynb".
The first task is:
Read-In
Can you write Python code for it?

df = pd.read_excel('Data.xlsx', sheetname=None)

df['Run 1']

keys = ['Run 1', 'Run 2', 'Run 3', 'Run 4']

# Fall time in seconds when V = 0
tf = np.array([df[key]['t_f (s)'] for key in keys])
tf

# Average Fall times for each run (seconds)
avg_tf = np.array([np.mean(tf[i]) for i in np.arange(4)])
avg_tf

# Standard Deviation Fall times for each run (seconds)
std_tf = np.array([np.std(tf[i]) for i in np.arange(4)])
std_tf


I am working on the file "MillikanOilDrop/Analysis.ipynb".
The first task is:
Search for Charge Unit
Upwards Charge
Can you write Python code for it?


I am working on the file "MillikanOilDrop/Analysis.ipynb".
The first task is:
Upward Movement
Can you write Python code for it?

# Rise time in seconds when V = +
tu = np.array([df[key]['t_u (s)'] for key in keys])
tu

#### tu0: Time Up 0'th drop.
#### Average Time
#### Std of Times
#### Resulting Speed
#### Speed Uncertainty

tu0 = ma.masked_array(tu[0],mask=[0,0,0,0,0,0,0,0,0,1])
avg_tu0 = np.mean(tu0)
std_tu0 = np.std(tu0)
tu0_speed = .5*1e-3/avg_tu0
tu0_speed_u = .5*1e-3/(avg_tu0**2)*std_tu0/np.sqrt(len(tu0))

tu1 = ma.masked_array(tu[1],mask=[0,0,1,1,0,0,0,0,0,0])
avg_tu1 = np.mean(tu1)
std_tu1 = np.std(tu1)
tu1_speed = .5*1e-3/avg_tu1
tu1_speed_u = .5*1e-3/(avg_tu1**2)*std_tu1/np.sqrt(len(tu1))

tu2_1 = ma.masked_array(tu[2],mask=[0,1,1,0,1,0,0,0,0,0])
avg_tu2_1 = np.mean(tu2_1)
std_tu2_1 = np.std(tu2_1)
tu2_1_speed = .5*1e-3/avg_tu2_1
tu2_1_speed_u = .5*1e-3/(avg_tu2_1**2)*std_tu2_1/np.sqrt(len(tu2_1))

tu2_2 = ma.masked_array(tu[2],mask=[1,0,0,1,0,1,1,1,1,1])
avg_tu2_2 = np.mean(tu2_2)
std_tu2_2 = np.std(tu2_2)
tu2_2_speed = .5*1e-3/avg_tu2_2
tu2_2_speed_u = .5*1e-3/(avg_tu2_2**2)*std_tu2_2/np.sqrt(len(tu2_2))

tu3 = tu[3]
avg_tu3 = np.mean(tu3)
std_tu3 = np.std(tu3)
tu3_speed = .5*1e-3/avg_tu3
tu3_speed_u = .5*1e-3/(avg_tu3**2)*std_tu3/np.sqrt(len(tu3))

#This data can't be used unless the free fall time data is analyzed
tu4 = np.array([4.20, 3.51, 3.44, 3.21, 3.56, 3.27, 3.25]) #This data from an extra column not entered into excel
avg_tu4 = np.mean(tu4)
std_tu4 = np.std(tu4)
tu4_speed = .5*1e-3/avg_tu4
tu4_speed_u = .5*1e-3/(avg_tu4**2)*std_tu4/np.sqrt(len(tu4))

avg_tu0

std_tu0/np.sqrt(len(tu0))

def up_charge(vu,vf,gamma,mass):
    return gamma*mass*g*((vu/vf) + 1)/E

qu0 = up_charge(tu0_speed, speeds[0], gammas[0], masses[0])
qu1 = up_charge(tu1_speed, speeds[1], gammas[1], masses[1])
qu2_1 = up_charge(tu2_1_speed, speeds[2], gammas[2], masses[2])
qu2_2 = up_charge(tu2_2_speed, speeds[2], gammas[2], masses[2])
qu3 = up_charge(tu3_speed, speeds[3], gammas[3], masses[3])
qu = np.array([qu0,qu1,qu2_1,qu2_2,qu3])
qu*1e19


I am working on the file "MillikanOilDrop/Analysis.ipynb".
The first task is:
Upwards Uncer
Can you write Python code for it?


I am working on the file "MillikanOilDrop/Analysis.ipynb".
The first task is:
\begin{equation}
    \frac{\delta q_u}{q_u} =  \sqrt{(\frac{\delta m}{m})^2 + (\frac{\delta v_u}{v_u + v_f})^2 + (\frac{v_u}{v_f}\frac{\delta v_f}{v_f+v_u})^2 }
\end{equation}
Can you write Python code for it?

def delta_qu(qu,dm, m, dvu, vu, dvf, vf):
    return np.sqrt((dm/m)**2 + (dvu/(vu+vf))**2 + ((vu/vf)*dvf/(vf+vu))**2)*qu

qu0_u = delta_qu(qu0,masses_u[0],masses[0],tu0_speed_u, tu0_speed, speeds_u[0], speeds[0])
qu1_u = delta_qu(qu1,masses_u[1],masses[1],tu1_speed_u, tu1_speed, speeds_u[1], speeds[1])
qu2_1_u = delta_qu(qu2_1,masses_u[2],masses[2],tu2_1_speed_u, tu2_1_speed, speeds_u[2], speeds[2])
qu2_2_u = delta_qu(qu2_2,masses_u[2],masses[2],tu2_2_speed_u, tu2_2_speed, speeds_u[2], speeds[2])
qu3_u = delta_qu(qu3,masses_u[3],masses[3],tu3_speed_u, tu3_speed, speeds_u[3], speeds[3])
qu_u = np.array([qu0_u,qu1_u,qu2_1_u,qu2_2_u,qu3_u])
qu_u*1e19


I am working on the file "MillikanOilDrop/Analysis.ipynb".
The first task is:
\begin{equation}
    m = \frac{4}{3} \pi r^3 \rho
\end{equation}
\begin{equation}
    \delta m = 4 \pi r^2 \rho \delta r
\end{equation}
\begin{equation}
    r = \sqrt{\frac{9 \eta v_f}{2 g \rho}}
\end{equation}
\begin{equation}
    \delta r = \frac{1}{2}\sqrt{\frac{9 \eta}{2 g \rho}} \frac{\delta v_f}{\sqrt{v_f}}
\end{equation}
\begin{equation}
    \delta m =  \pi r^2 \sqrt{\frac{18 \eta \rho}{ g}} \frac{\delta v_f}{\sqrt{v_f}}
\end{equation}
Can you write Python code for it?

masses_u = np.array([np.pi*(radii[i]**2)*np.sqrt(18*viscosities[i]*rho/g)*np.sqrt(1/speeds[i])*speeds_u[i] for i in np.arange(4)])
masses_u


I am working on the file "MillionSong_Dataset_Exploration.ipynb".
The first task is:
As expected (and fortunately) results are very similar.
Let's visualize the result :
Can you write Python code for it?

plt.figure(figsize=(8,7))
plt.scatter(songs.artist_familiarity, songs.artist_hotttnesss,
            marker='.', edgecolors='none', c='C2', alpha=0.7)
plt.plot([0.0,1.0], [0.0,songs_linreg[0]] + songs_linreg[1], 'k', lw=1.0)
plt.xlabel('Artist Familiarity')
plt.ylabel('Artist Hotness');


I am working on the file "MillionSong_Dataset_Exploration.ipynb".
The first task is:
Despite a couple of pretty long songs (nearly 27minutes for the maximum) the distribution appears roughly symmetrical. This is partially confirmed by the small difference between the mean and the median.
Confirmation of the degree of symmetry can be found in the skewness of the distribution :
Can you write Python code for it?

# computing  distribution skewness
print('Skewness : {:.3f}'.format(skew(songs.duration.values)))


I am working on the file "MillionSong_Dataset_Exploration.ipynb".
The first task is:
Familiarity vs Hotness Clustering
When looking at the previous figures one can easily spot a "main sequence" (term borrowed from astronomy) where hotness linearly increase with familiarity and a number of cases where hotness is nearly zero if not zero. In addition there seems to be a different relationship between hotness and familiarity at high familiarity (>0.8) and numerous outliers surrounding the "main sequence" and at low familiarity (<0.4).
To better understand the pattern(s) in this 2-D space we choose to use an unsupervised clustering approach with the hope that it will reveal interesting structure.
We decided to illustrate this approach with two methods :
* The Gaussian Mixture Model (GMM) which makes the assumptions that the distribution can be viewed as samples from k Gaussian distributions.
* Agglomerative Clustering, a hierarchical clustering approach, grouping samples by increasing distance.
Both methods require the user to define a priori the number of clusters which can then be validated using measures such as the $\log[Likelihood]$, Bayesian Information Criteria (BIC) or the Akaike Information Criteria (AIC).
Hereafter we exclude the collection of points with hotness at zero as it may very well prevents clustering from giving sensible results.
Can you write Python code for it?

# checking the number of points at hotness = 0
songs[songs.artist_hotttnesss == 0.0].song_id.nunique()

# importing the methods
from sklearn.mixture import GaussianMixture
from sklearn.cluster import AgglomerativeClustering

# Selecting only the useful part of the dataset
songs_cl = songs[songs.artist_hotttnesss != 0.0][['artist_familiarity','artist_hotttnesss']]


I am working on the file "MillionSong_Dataset_Exploration.ipynb".
The first task is:
Having information about the minimum, maximum, mean, median, quartiles, etc. is interesting but it is probably more informative to look at the distribution of songs per year in details.
Can you write Python code for it?

# first we count the number of songs per year in chronological ordre.
songs_per_yr = songs.year.value_counts().sort_index()
songs_per_yr.head(5)

# add missing years in the original dataset with 0
songs_per_yr = songs_per_yr.reindex(index=list(range(songs.year.min(),songs.year.max())),
                                    fill_value=0)

# Visualizing the number of songs per year
l_col = songs_per_yr / songs_per_yr.max()
songs_per_yr.plot.bar(color=cm.plasma(l_col), figsize=(12,7))

plt.xlabel('Year')
plt.ylabel('Number of Songs')
plt.title('Number of Songs vs Year');


I am working on the file "MillionSong_Dataset_Exploration.ipynb".
The first task is:
In order to estimate the optimal number of clusters we iteratively compute the different metrics with GMM :
Can you write Python code for it?

# Here we compute metrics for GMM with different number of clusters
scores_n = []

for i in range(2,12):
    print('number of clusters : {}'.format(i))
    gmm = GaussianMixture(n_components=i, covariance_type='full')
    bic, aic, log = 0.0, 0.0, 0.0
    for j in range(7):
        gmm.fit(songs_cl)
        bic += gmm.bic(songs_cl)
        aic += gmm.aic(songs_cl)
        log += gmm.score(songs_cl)
    scores_n.append([np.mean(bic), np.mean(aic), np.mean(log)])

scores_n = np.asarray(scores_n)

# Visualizing the results
fig, ax = plt.subplots(nrows=1, ncols=2, sharex=True, figsize=(10,4))

ax[0].plot(range(2,12), scores_n[:,0] - np.min(scores_n[:,0]), 'C0')
ax[0].plot(range(2,12), scores_n[:,1] - np.min(scores_n[:,1]), 'C1')
ax[0].legend(['BIC','AIC'])
ax[0].set_xlabel('Number of Clusters $k$')

ax[1].plot(range(2,12), scores_n[:,2] - np.min(scores_n[:,2]), 'C2')
ax[1].legend(['Log Likelihood'])
ax[1].set_xlabel('Number of Clusters $k$');


I am working on the file "MillionSong_Dataset_Exploration.ipynb".
The first task is:
It is possible to look separately at the resulting clusters from both methods.
Can you write Python code for it?

songs_cl['gmm'] = np.argmax(gmm_3_pred, axis=1)
songs_cl['agg_clus'] = agg_3_pred

sb.lmplot(data=songs_cl, x='artist_familiarity', y='artist_hotttnesss',
          hue='gmm', col='gmm',
          fit_reg=False, markers='.', scatter_kws={'alpha':0.7});

sb.lmplot(data=songs_cl, x='artist_familiarity', y='artist_hotttnesss',
          hue='agg_clus', col='agg_clus',
          fit_reg=False, markers='.', scatter_kws={'alpha':0.7});


I am working on the file "MillionSong_Dataset_Exploration.ipynb".
The first task is:
Looking at the very unbalanced distribution of songs per year it appears that the initial statistical indicators are not carrying very meaningful information.
We can clearly see that the dataset is biased toward more recent songs.
Songs and Albums Authors
Now we know there is a year bias, let's look at the songs author. Is there another bias ?
Can you write Python code for it?

# Counting the number of entry per artist
songs_per_artist = songs.artist_name.value_counts()
songs_per_artist[songs_per_artist >= 8].sort_values().plot.barh(color='midnightblue', figsize=(7,7))
plt.title('Songs per Artist (8 and more)')
plt.xlabel('Number of Songs')
plt.ylabel('Artist Name');

# Number of different albums release per artist
songs_per_art_and_rel = songs.groupby(['artist_name']).release.nunique()
songs_per_art_and_rel[songs_per_art_and_rel >= 6].sort_values().plot.barh(color='midnightblue', figsize=(6,10))
plt.title('Albums per Artist (6 and more)')
plt.xlabel('Number of Albums')
plt.ylabel('Artist Name');


I am working on the file "MillionSong_Dataset_Exploration.ipynb".
The first task is:
Reading the Metadata SQL table
For this demo the database .db file is in a separate folder named MillionSongSubset/AdditionalFiles/.
An alternative to the lines below can be found at the following address : https://labrosa.ee.columbia.edu/millionsong/sites/default/files/tutorial1.py.txt
The latter demo was specifically tailored for reading these files.
Can you write Python code for it?

# creating the connector for the SQL database
con_meta = sqlite3.connect(path+'AdditionalFiles/subset_track_metadata.db')
cur_meta = con_meta.cursor()

# creating and executing an SQL request
res = con_meta.execute("SELECT name FROM sqlite_master WHERE type='table';")

# printing available table names
for name in res:
    print(name[0])


I am working on the file "MillionSong_Dataset_Exploration.ipynb".
The first task is:
Songs Hotness and Familiarity
In this section we focus on two engineered featured, namely the artist hotness and familiarity
Can you write Python code for it?

songs[['artist_hotttnesss','artist_familiarity']].describe()

fig, ax = plt.subplots(nrows=1, ncols=2, figsize=(12,6), sharex=True, sharey=True)

songs.artist_familiarity.hist(bins=np.arange(0.0,1.02,0.02), ax=ax[0], color='C2');
ax[0].set_title('Artist Familiarity')
ax[0].axvline(0.54075, linestyle=':', lw=1.0, c='k')
ax[0].axvline(0.622875, linestyle='-.', lw=1.0, c='k')
ax[0].axvline(0.72723, linestyle=':', lw=1.0, c='k')

songs.artist_hotttnesss.hist(bins=np.arange(0.0,1.02,0.02), ax=ax[1], color='C1');
ax[1].set_title('Artist Hotness')
ax[1].axvline(0.36682, linestyle=':', lw=1.0, c='k')
ax[1].axvline(0.420938, linestyle='-.', lw=1.0, c='k')
ax[1].axvline(0.511054, linestyle=':', lw=1.0, c='k')

ax[-1].set_xlim(0.0,1.0)
plt.subplots_adjust(wspace=0.05);


I am working on the file "MillionSong_Dataset_Exploration.ipynb".
The first task is:
That is much closer to the intuition we had.
Let's have a look at what Agglomerative clustering suggests (with 4 and 3 clusters) :
Can you write Python code for it?

# Agglomerative Clustering with k=4 clusters
agg_4 = AgglomerativeClustering(n_clusters=4, linkage='average')
agg_4_pred = agg_4.fit_predict(songs_cl)

# Agglomerative Clustering with k=3 clusters
agg_3 = AgglomerativeClustering(n_clusters=3, linkage='average')
agg_3_pred = agg_3.fit_predict(songs_cl)

fig, ax = plt.subplots(nrows=1, ncols=2, sharex=True, sharey=True, figsize=(12,6))

ax[0].scatter(songs_cl.artist_familiarity, songs_cl.artist_hotttnesss, c=agg_3_pred,
              marker='.', alpha=1.0, cmap='Dark2_r')
ax[0].set_title('Agg. Clus. with k=3')
ax[0].set_xlabel('Artist Familiarity')
ax[0].set_ylabel('Artist Hotness')

ax[1].scatter(songs_cl.artist_familiarity, songs_cl.artist_hotttnesss, c=agg_4_pred,
              marker='.', alpha=1.0, cmap='Dark2_r')
ax[1].set_title('Agg. Clus. with k=4')
ax[1].set_xlabel('Artist Familiarity')

plt.subplots_adjust(wspace=0.025);


I am working on the file "MillionSong_Dataset_Exploration.ipynb".
The first task is:
The nature of the GMM method result in our case in two clusters presenting a gap while both methods recover a pretty similar "main sequence". Therefore we will keep Agglomerative Clustering results.
Clustering quality assessment can be very tedious and there exists a lot of metrics for this purpose. However, we choose to limit ourselves to the previous qualitative analysis.
Linear Regression of the Clustering
Now we have defined three different groups it is interesting to introduce some quantification.
First we perform a linear regression of the "main sequence" as it exhibits a clear correlation.
Can you write Python code for it?

songs_linreg = np.polyfit(songs_cl.artist_familiarity[songs_cl.agg_clus == 2].values,
                          songs_cl.artist_hotttnesss[songs_cl.agg_clus == 2].values, 1)
print("Linear Regression of the 'main sequence' :\n\
Hotness = {:.4f} * Familiarity + {:.4f}".format(songs_linreg[0],songs_linreg[1]))


I am working on the file "MillionSong_Dataset_Exploration.ipynb".
The first task is:
There are no missing values but that does not mean there are not any weird values.
In the following we will explore the songs distributions in the multi-dimensional feature space. Such descriptive analysis step is essential to later have a critical understanding of any trends or interesting facts we might discover about the Million Song dataset.
One Dimensional Analysis
At first we decided to look at features individually (aka along one dimension of the feature space only).
Songs per Year
Can you write Python code for it?

# simple statistical description
songs.year.describe()


I am working on the file "MillionSong_Dataset_Exploration.ipynb".
The first task is:
This is not much information to extract from the one-dimensional anlaysis of these two features, later we will come back to the relationship between hotness and familiarity that shows much more interesting patterns.
Looking at Patterns and Correlations
One dimensional analysis is by definition limited and in this section we will investigate further correlations between features and possible existing patterns.
An easy way to visualize possible patterns is to use a pair grid where every feature is plotted against the others :
Can you write Python code for it?

g = sb.PairGrid(songs, vars=['year','duration','artist_familiarity','artist_hotttnesss'])
g.map_upper(sb.kdeplot)
g.map_lower(plt.scatter, marker='.', edgecolor='w')
g.map_diag(plt.hist, bins=51);


I am working on the file "MillionSong_Dataset_Exploration.ipynb".
The first task is:
Well, artist familiarity and hotness are highly correlated with a coefficient of ~0.81 while other features are much more independent from each other (coefficient < 0.1).
In the following we will take a closer look at the pair familiarity-hotness and try to better understand the correlation between these two features.
Artist Hotness vs Artist Familiarity
Let's first have a look again at the dataset in this 2-D space defined by the artist hotness and familiarity :
Can you write Python code for it?

fig, ax = plt.subplots(nrows=1, ncols=2, sharex=True, sharey=True,
                       figsize=(12,6))

songs.plot.scatter(x='artist_familiarity', y='artist_hotttnesss', marker='+', ax=ax[0])
songs.plot.hexbin(x='artist_familiarity', y='artist_hotttnesss', ax=ax[1],
                  gridsize=41, mincnt=1.0, cmap='viridis', colorbar=False)

ax[0].set_title('Scatter Plot')
ax[1].set_title('Density Plot')
plt.subplots_adjust(wspace=0.03);


I am working on the file "MillionSong_Dataset_Exploration.ipynb".
The first task is:
Well... even if metrics recommend four clusters, resulting groups differ significantly from the intuition we have. Let's try with maybe $k=3$ :
Can you write Python code for it?

# Defining GMM with k=3 clusters
gmm_3 = GaussianMixture(n_components=3, covariance_type='full')
gmm_3.fit(songs_cl)
gmm_3_pred = gmm_3.predict_proba(songs_cl)

# Visualizing the result
plt.scatter(songs_cl.artist_familiarity, songs_cl.artist_hotttnesss, c=np.argmax(gmm_3_pred, axis=1),
            marker='.', alpha=0.5, cmap='Dark2_r')
plt.title('GMM with k=3')
plt.xlabel('Artist Familiarity')
plt.ylabel('Artist Hotness');


I am working on the file "Mimic-iii_Model - Capstone Supervised.ipynb".
The first task is:
Ab_name indicates which antibiotic is sensitive the microorganism, this field together with the interpretation indicates if the microorganism the degree of resistance of this one to the antibiotic., the main antibiotics evaluated are gentamicin, trimethoprim/sulfa, levofloxacin, ceftazidime, tobramycin, cefepime, ciprofloxacin, meropenem, erythromycin, oxacillin, vancomycin, ceftriaxone, tetracycline, clindamycin, piperacillin/tazo, which represent 80% of the sample.
Can you write Python code for it?

patients_pn['ab_name'].unique()

patients_pn['ab_name'].value_counts()

patients_ab_name = patients_pn['ab_name'].value_counts().reset_index()
patients_ab_name.columns=['ab_name','Count']
patients_ab_name['Count'].apply('{:,.2f}'.format)
patients_ab_name['cum_perc'] = 100*patients_ab_name.Count/patients_ab_name.Count.sum()
patients_ab_name['cum_perc'] = patients_ab_name ['cum_perc'].map('{:,.4f}%'.format)

print (patients_ab_name)

patients_pn['ab_name'].value_counts().plot(kind='bar')


I am working on the file "Mimic-iii_Model - Capstone Supervised.ipynb".
The first task is:
Decision Tree Classifier is a no parametric method that learns through binary decisions that when deployed are forming a decision tree.
Can you write Python code for it?

print ("Fitting the Decision Tree Classifier to the training set")

param_grid = {
         'max_depth': [2, 3, 4, 5, 6,7],
          #'gamma': [0.0001, 0.0005, 0.001, 0.005, 0.01, 0.1, 1],
          }
# for sklearn version 0.16 or prior, the class_weight parameter value is 'auto'
clf_DTC = GridSearchCV(DecisionTreeClassifier(criterion='entropy', random_state=123,
                       class_weight='balanced'), param_grid, n_jobs=-1)

clf_DTC = clf_DTC.fit(X_train, y_train)

scores_DTC = cross_val_score(clf_DTC, X_train, y_train, cv=4, n_jobs=-1)
print(scores_DTC)
print("Accuracy: %0.4f (+/- %0.4f)" % (scores_DTC.mean(), scores_DTC.std() * 2))

print ("Best estimator found by grid search:")
print (clf_DTC.best_estimator_)
y_predicted_DTC = clf_DTC.predict(X_test)

print (metrics.classification_report(y_test, y_predicted_DTC))
process = time.process_time()
print (process)

sns.heatmap(confusion_matrix(y_test, y_predicted_DTC), annot = True, fmt = '', cmap = "GnBu")


I am working on the file "Mimic-iii_Model - Capstone Supervised.ipynb".
The first task is:
Ensemble methods like Random Forest, Extremely Tree and Ada Boost Classifiers.  These methods “combine the predictions of several base estimators built with a given learning algorithm in order to improve generalizability / robustness over a single estimator ”.  The first two are in averaging methods, where independent estimators are used over random samples and resulting predictions are averaged, getting as result a lower variance than a single estimator.
Can you write Python code for it?

print ("Fitting the Random Forest Classifier to the training set")

param_grid = {
         'n_estimators' :[3,5,7,10],
         'max_depth': [2, 3, 4, 5, 6,7],
          #'gamma': [0.0001, 0.0005, 0.001, 0.005, 0.01, 0.1, 1],
          }
# for sklearn version 0.16 or prior, the class_weight parameter value is 'auto'
clf_RFC = GridSearchCV(RandomForestClassifier(min_samples_split=2, random_state=123, class_weight='balanced'),
                       param_grid, n_jobs=-1)

clf_RFC = clf_RFC.fit(X_train, y_train)

scores_RFC = cross_val_score(clf_RFC, X_train, y_train, cv=4, n_jobs=-1)
print(scores_RFC)
print("Accuracy: %0.4f (+/- %0.4f)" % (scores_RFC.mean(), scores_RFC.std() * 2))

print ("Best estimator found by grid search:")
print (clf_RFC.best_estimator_)
y_predicted_RFC = clf_RFC.predict(X_test)

print (metrics.classification_report(y_test, y_predicted_RFC))
process = time.process_time()
print (process)

sns.heatmap(confusion_matrix(y_test, y_predicted_RFC), annot = True, fmt = '', cmap = "GnBu")

print ("Fitting the Extremely Tree Classifier to the training set")

param_grid = {
         'n_estimators' :[3,5,10],
         'max_depth': [2, 3, 4, 5, 6,7],
          #'gamma': [0.0001, 0.0005, 0.001, 0.005, 0.01, 0.1, 1],
          }
# for sklearn version 0.16 or prior, the class_weight parameter value is 'auto'
clf_EFC = GridSearchCV(ExtraTreesClassifier(min_samples_split=2, random_state=123, class_weight='balanced'),
                       param_grid, n_jobs=-1)

clf_EFC = clf_EFC.fit(X_train, y_train)

scores_EFC = cross_val_score(clf_EFC, X_train, y_train, cv=4, n_jobs=-1)
print(scores_EFC)
print("Accuracy: %0.4f (+/- %0.4f)" % (scores_EFC.mean(), scores_EFC.std() * 2))

print ("Best estimator found by grid search:")
print (clf_EFC.best_estimator_)
y_predicted_EFC = clf_EFC.predict(X_test)

print (metrics.classification_report(y_test, y_predicted_EFC))
process = time.process_time()
print (process)

sns.heatmap(confusion_matrix(y_test, y_predicted_EFC), annot = True, fmt = '', cmap = "GnBu")

print...


I am working on the file "Mimic-iii_Model - Capstone Supervised.ipynb".
The first task is:
For each subject, I categorized the population in five types of groups according to the age recorded at the time of admission to the ICU, which are neonates [0,1], middle (1, 14), adults (14, 65), Older adults [65, 85] and older elderly people (85, 91.4].
Can you write Python code for it?


I am working on the file "Mimic-iii_Model - Capstone Supervised.ipynb".
The first task is:
Hospital_expire_flag has the binary dependent variable to the model, 0 when the patient goes out from ICU alive, and 1 when the patient has deceased while stay in ICU.

Subject_id is the key value which relates the respective record with an acute patient in ICU.  gender give the patient sex of the subject.  Last_admit_age (is a computed field) has the age when the patient is admitted in ICU.

Age_group (is a computed field) serves to categorize the sample by age.

Valuenum_avg is the average number for valuenum of this respective label measure.  Org_name contains the names of the microorganisms (bacteria) related to pneumonia, where the main ones are staph aureus coag +, klebsiella pneumoniae, escherichia coli, pseudomonas aeruginosa, staphylococcus, coagulase negative, klebsiella oxytoca, enterococcus sp, which represent 80% of the sample.

Category is employed to categorize the charted events, the main categories of this data column are Labs, Respiratory, Alarms, Routine Vital Signs, Chemistry which gathering the 82% of the records which are present in this query.
Can you write Python code for it?

patients_pn['category'].unique()

patients_pn['category'].value_counts()

patients_category = patients_pn['category'].value_counts().reset_index()
patients_category.columns=['category','Count']
patients_category['Count'].apply('{:,.2f}'.format)
patients_category['cum_perc'] = 100*patients_category.Count/patients_category.Count.sum()
patients_category['cum_perc'] = patients_category['cum_perc'].map('{:,.4f}%'.format)

print (patients_category)

patients_pn['category'].value_counts().plot(kind='bar')


I am working on the file "Mimic-iii_Model - Capstone Supervised.ipynb".
The first task is:
I was struggling a lot with this model, for that reason I will not use this for capstone project

print ("Fitting the classifier to the training set")
from sklearn.tree import DecisionTreeClassifier
param_grid = {
         'C': [1e-3, 1e-2, 1, 1e3, 5e3, 1e4, 5e4, 1e5],
         'gamma': [0.0001, 0.0005, 0.001, 0.005, 0.01, 0.1, 1],
         'degree': [3,4,5]
          }
- for sklearn version 0.16 or prior, the class_weight parameter value is 'auto'
clf_poly = GridSearchCV(SVC(kernel='poly', class_weight='balanced'), param_grid, n_jobs=-1)
clf_poly = clf_poly.fit(X_train, y_train)
scores_poly = cross_val_score(clf_poly, X_train, y_train, cv=4,n_jobs=-1)
print(scores_poly)
print("Accuracy: %0.4f (+/- %0.4f)" % (scores_poly.mean(), scores_poly.std() * 2))
print ("Best estimator found by grid search:")
print (clf_poly.best_estimator_)
y_predicted_poly = clf_poly.predict(X_test)
print (metrics.classification_report(y_test, y_predicted_poly))
process = time.process_time()
print (process)
Can you write Python code for it?

print ("Fitting the Support Vector Classification - kernel Sigmoid classifier to the training set")

param_grid = {
         'C': [1e3, 1e4, 1e5],
         'gamma': [0.0001, 0.0005, 0.001, 0.005, 0.01, 0.1, 1],
         'coef0':[-1,0,1]
          }
# for sklearn version 0.16 or prior, the class_weight parameter value is 'auto'
clf_SIGMOID = GridSearchCV(SVC(kernel='sigmoid', class_weight='balanced'), param_grid, n_jobs=-1)
clf_SIGMOID = clf_SIGMOID.fit(X_train, y_train)

scores_SIGMOID = cross_val_score(clf_SIGMOID, X_train, y_train, cv=4, n_jobs=-1)
print(scores_SIGMOID)
print("Accuracy: %0.4f (+/- %0.4f)" % (scores_SIGMOID.mean(), scores_SIGMOID.std() * 2))

print ("Best estimator found by grid search:")
print (clf_SIGMOID.best_estimator_)

y_predicted_SIGMOID = clf_SIGMOID.predict(X_test)

print (metrics.classification_report(y_test, y_predicted_SIGMOID))

process = time.process_time()
print (process)

sns.heatmap(confusion_matrix(y_test, y_predicted_SIGMOID), annot = True, fmt = '', cmap = "GnBu")


I am working on the file "Mimic-iii_Model - Capstone Supervised.ipynb".
The first task is:
Logistic Regression Classifier is the most traditional method applied to classification problems.  Here a logistic probability function is applied to data, and the result obtained is a probability of occurrence of the binary categorical variable
Can you write Python code for it?

print ("Fitting the Logistic Regression Classification - Hingue loss classifier to the training set")

param_grid = {
         'C': [1e-3, 1e-2, 1, 1e3, 5e3, 1e4, 5e4, 1e5],
          #'gamma': [0.0001, 0.0005, 0.001, 0.005, 0.01, 0.1, 1],
          }
# for sklearn version 0.16 or prior, the class_weight parameter value is 'auto'
clf_LOGREG = GridSearchCV(LogisticRegression(random_state=123), param_grid, n_jobs=-1)
clf_LOGREG= clf_LOGREG .fit(X_train, y_train)

scores_LOGREG = cross_val_score(clf_LOGREG, X_train, y_train, cv=4, n_jobs=-1)
print(scores_LOGREG)
print("Accuracy: %0.4f (+/- %0.4f)" % (scores_LOGREG.mean(), scores_LOGREG.std() * 2))

print ("Best estimator found by grid search:")
print (clf_LOGREG.best_estimator_)

y_predicted_LOGREG = clf_LOGREG.predict(X_test)

print (metrics.classification_report(y_test, y_predicted_LOGREG))

process = time.process_time()
print (process)

sns.heatmap(confusion_matrix(y_test, y_predicted_LOGREG), annot = True, fmt = '', cmap = "GnBu")

# Best Models

clf_RBF_b = SVC(C=1, cache_size=200, class_weight='balanced', coef0=0.0,
                decision_function_shape=None, degree=3, gamma=1, kernel='rbf',
                max_iter=-1, probability=False, random_state=None, shrinking=True,
                tol=0.001, verbose=False)

y_predicted_RBF_b = clf_RBF_b.fit(X_train,y_train).predict(X_test)

clf_LSCV_b = LinearSVC(C=0.001, class_weight=None, dual=True, fit_intercept=True,
                       intercept_scaling=1, loss='hinge', max_iter=1000, multi_class='ovr',
                       penalty='l2', random_state=None, tol=0.0001, verbose=0)

y_predicted_LSCV_b = clf_LSCV_b.fit(X_train,y_train).predict(X_test)

clf_SIGMOID_b = SVC(C=1000.0, cache_size=200, class_weight='balanced', coef0=1,
                    decision_function_shape=None, degree=3, gamma=0.001, kernel='sigmoid',
                    max_iter=-1, probability=False, random_state=None, shrinking=True,
                    tol=0.001, verbose=False)

y_predicted_SIGMOID_b =...


I am working on the file "Mimic-iii_Model - Capstone Supervised.ipynb".
The first task is:
Support Vector Machine:  Machine Support Vector (SVM) is a classification method that separates a sample points in different hyperplanes in multidimensional spaces, which are separated by different labels. The algorithm seeks to group the data (classify) by optimal search (minimum distance) between the hyperplanes, the resulting vectors are called support vectors .   The optimization is made over kernels (mathematical functions), in this analysis I used different methods: linear, radial basis function (rbf) and sigmoid.  I purposely avoided using the polynomial kernels, more because of parameterization problems that did not allow me to run the data with this algorithm.
Can you write Python code for it?

# the same model as above, only has changed the jobs

clf_SVC = SVC(kernel='linear', C=1)
scores_SVC = cross_val_score(clf_SVC, X_train, y_train, cv=4, n_jobs=-1)
print(scores_SVC)
print("Accuracy: %0.4f (+/- %0.4f)" % (scores_SVC.mean(), scores_SVC.std() * 2))
clf_SVC = clf_SVC.fit(X_train, y_train)
y_predicted_SVC = clf_SVC.predict(X_test)

print (metrics.classification_report(y_test, y_predicted_SVC))

process = time.process_time()
print (process)

sns.heatmap(confusion_matrix(y_test, y_predicted_SVC), annot = True, fmt = '', cmap = "GnBu")

print ("Fitting the Support Vector Classification - kernel Radial Basis Function classifier to the training set")

param_grid = {
          'C': [1e-3, 1e-2, 1, 1e3, 5e3, 1e4, 5e4, 1e5],
          'gamma': [0.0001, 0.0005, 0.001, 0.005, 0.01, 0.1, 1],
          }
# for sklearn version 0.16 or prior, the class_weight parameter value is 'auto'
clf_RBF = GridSearchCV(SVC(kernel='rbf', class_weight='balanced', cache_size=1000), param_grid)
clf_RBF = clf_RBF.fit(X_train, y_train)

scores_RBF = cross_val_score(clf_RBF, X_train, y_train, cv=4, n_jobs=-1)
print(scores_RBF)
print("Accuracy: %0.2f (+/- %0.2f)" % (scores_RBF.mean(), scores_RBF.std() * 2))
print ("Best estimator found by grid search:")
print (clf_RBF.best_estimator_)

y_predicted_RBF = clf_RBF.predict(X_test)

print (metrics.classification_report(y_test, y_predicted_RBF))

process = time.process_time()
print (process)

sns.heatmap(confusion_matrix(y_test, y_predicted_RBF), annot = True, fmt = '', cmap = "GnBu")

# Mimic-iii_Model-Pulmonary.ipynb
print ("Fitting the Linear Support Vector Classification - Hingue loss classifier to the training set")

param_grid = {
         'C': [1e-3, 1e-2, 1, 1e3, 5e3, 1e4, 5e4, 1e5],
          #'gamma': [0.0001, 0.0005, 0.001, 0.005, 0.01, 0.1, 1],
          }
# for sklearn version 0.16 or prior, the class_weight parameter value is 'auto'
clf_LSCV = GridSearchCV(LinearSVC(C=1, loss= 'hinge'), param_grid, n_jobs=-1)
clf_LSCV = clf_LSCV.fit(X_train, y_train)

scores_LSCV =...


I am working on the file "Mimic-iii_Model - Capstone Supervised.ipynb".
The first task is:
The best model found here is the ensemble model with the decision tree and the extremely tree, even though the ensemble model with five aggregate methods shows a slightly better score, applying the principle of Occam's razor make the simplest better.  At this time, the resulting model can accurately predict the deaths given a series of medical examinations, as proposed in the hypothesis. While accuracy is not the best (76%), I think it may be a good start for future investigations of interdisciplinary teams in ICU forecasting diseases.
From the forest model is possible to find how features weights in the results, such weights are called importance.  As you can see only 129 features are important to the model, the rest has no weights.  As you can expect, the antibiotic sensitivity are the most important feature (together weights 57% of importance) and AMIKACIN antibiotic the most important feature of the sample.  Every feature from age group weight 0.97% in average, followed by category feature which everyone weights less than 1%.
Can you write Python code for it?

# Code source:  http://scikit-learn.org/stable/auto_examples/ensemble/plot_forest_importances.html
from sklearn.datasets import make_classification
forest = clf_EFC_b

forest.fit(X_train, y_train)
feature_names = X_train.columns
importances = forest.feature_importances_
std = np.std([tree.feature_importances_ for tree in forest.estimators_],
             axis=0)
indices = np.argsort(importances)[::-1]

# Print the feature ranking
print("Feature ranking:")

for f in range(X_train.shape[1]):
    if importances[indices[f]]>0:
        print("%d. feature %d (%f)" % (f + 1, indices[f], importances[indices[f]]))

#plt.xticks(range(heart_train.shape[1]), )

# Plot the feature importances of the forest
plt.figure()
plt.title("Feature importances")
plt.bar(range(X_train.shape[1]), importances[indices],
       color="r", yerr=std[indices], align="center")
plt.xticks(range(X_train.shape[1]), feature_names)
plt.xlim([-1, X_train.shape[1]])
plt.show()

from sklearn.datasets import make_classification
forest = clf_EFC_b

forest.fit(X_train, y_train)
feature_names = X_train.columns
importances = forest.feature_importances_
std = np.std([tree.feature_importances_ for tree in forest.estimators_],
             axis=0)
indices = np.argsort(importances)[::-1]

impor = []
for f in range(X_train.shape[1]):
    if importances[indices[f]]>0:
        impor.append({'Feature': feature_names[f] , 'Importance': importances[indices[f]]})

feature_importance = pd.DataFrame(impor).sort_values('Importance',ascending = False)
print(feature_importance.to_string())

feature_importance.info()

feature_importance['Importance'].sum()

#surveys_df[surveys_df.year == 2002]
more_important= feature_importance[feature_importance.Importance >= 0.009]
more_important

more_important.plot(kind='bar')


I am working on the file "Mimic-iii_Model - Capstone Supervised.ipynb".
The first task is:
The following ensemble model only aggregate five models:
Can you write Python code for it?

eclf3 = VotingClassifier(estimators=[
    ('rbf',clf_RBF), ('sigmoid',clf_SIGMOID), ('DTC',clf_DTC),
    ('RFC',clf_RFC),('EFC',clf_EFC)],
                         voting='hard')
eclf3 = eclf3.fit(X_train, y_train)
y_predict_eclf3 =  eclf3.predict(X_test)

print (eclf3.get_params(deep=True))
print (eclf3.score(X_train, y_train, sample_weight=None))

print(y_predict_eclf3)

scores_eclf3 = cross_val_score(eclf3 , X_train, y_train, cv=4, n_jobs=-1)
print(scores_eclf3 )
print("Accuracy: %0.4f (+/- %0.4f)" % (scores_eclf3.mean(), scores_eclf3.std() * 2))

print (metrics.classification_report(y_test, y_predict_eclf3))

cnf_matrix_eclf3 = confusion_matrix(y_test, y_predict_eclf3)
np.set_printoptions(precision=2)

# Plot non-normalized confusion matrix
plt.figure()
plot_confusion_matrix(cnf_matrix_eclf3, classes='1',
                      title='ECLF Confusion matrix, without normalization')

# Plot normalized confusion matrix
plt.figure()
plot_confusion_matrix(cnf_matrix_eclf3, classes='1', normalize=True,
                      title='ECLF Normalized confusion matrix')

plt.show()


I am working on the file "Mimic-iii_Model - Capstone Supervised.ipynb".
The first task is:
The next step, is to transform the matrix into a PivotTable, the purpose of this transformation is to be able to have the medical data in individual lines per subject and numerical form.
To do that, I employed pandas pivot_table command, and using as indexes subject_id and hospital_expire_flag.  With this transformation, the resulting panda data frame has 829 rows by 724 columns.  The data transformed in this form allow apply the classifier models to this data.
Can you write Python code for it?

patients_data = pd.pivot_table(patients_dummy,index=["subject_id", "hospital_expire_flag" ])
process = time.process_time()
print (process)

patients_data.head()

patients_data.info()

patients = patients_data.reset_index()

patients.head()

p_data= patients.ix[:,2:]

p_data.head()

p_target= patients['hospital_expire_flag']

p_target.head()


I am working on the file "Mimic-iii_Model - Capstone Supervised.ipynb".
The first task is:
To transform the matrix in a pivot table, the first step is transform some categorical variables as dummy variables.  The chosen variables are gender, age_group, category, label, org_name, ab_name, and interpretation.  This operation was done with pandas get_dummies command.  The result of this transformation is a panda data frame with shape 2,430,640 rows by 716 columns, these new columns are binaries variables and only take the number 1 once the categorical effect happened.
Can you write Python code for it?

patients_dummy = pd.get_dummies(patients_pn,prefix=['gender', 'age_group', 'category','label',
                                                    'org_name','ab_name', 'interpretation'])
patients_dummy.head()


I am working on the file "Mimic-iii_Model - Capstone Supervised.ipynb".
The first task is:
interpretation indicates the results of the test,  “S” when the antibiotic is sensitive, “R” when is resistant, “I” when the antibiotic is intermediate, and “P” when is pending.
Can you write Python code for it?

patients_pn['interpretation'].unique()

patients_pn['interpretation'].value_counts()

patients_interpretation = patients_pn['interpretation'].value_counts().reset_index()
patients_interpretation.columns=['interpretation','Count']
patients_interpretation['Count'].apply('{:,.2f}'.format)
patients_interpretation['cum_perc'] = 100*patients_interpretation.Count/patients_interpretation.Count.sum()
patients_interpretation['cum_perc'] = patients_interpretation ['cum_perc'].map('{:,.4f}%'.format)

print (patients_interpretation)

patients_pn['interpretation'].value_counts().plot(kind='bar')

patients_pn.head()


I am working on the file "Minesweeper_results.ipynb".
The first task is:
Evolutionary Strategies
Can you write Python code for it?

# Plot rewards and winrate
ES_adjust_axis = 1000000
ES_smoothing = 10
ES_color = [0.2,0.5,0.1]
ES_color_test = [0.5,0.8,0.4]

with open(os.path.join('Results/Evolutionary/results.pkl'), 'rb') as f:
    ES_results = pickle.load(f)
ES_steps = pd.Series(ES_results['steps'])/ES_adjust_axis
ES_rewards_mean = ES_results['mean_pop_rewards']
ES_rewards_mean_smooth = smooth(ES_rewards_mean,ES_smoothing)
ES_rewards_test = ES_results['test_rewards']
ES_rewards_test_smooth = smooth(ES_rewards_test,ES_smoothing)
ES_winrate = ES_results['win_rate']

fig = plt.figure(figsize=(6, 4))
ax1 = plt.subplot(211)
plt.plot(ES_steps, ES_rewards_mean_smooth, color=ES_color+[1.0])
plt.plot(ES_steps, ES_rewards_mean, color=ES_color+[0.1], label='_nolegend_')
plt.plot(ES_steps, ES_rewards_test_smooth, color=ES_color_test+[1.0])
plt.plot(ES_steps, ES_rewards_test, color=ES_color_test+[0.1], label='_nolegend_')
plt.setp(ax1.get_xticklabels(), visible=False)
plt.ylabel('Reward')
plt.legend(['Mean population reward', 'Test reward'])
plt.grid()
plt.subplot(212, sharex=ax1)
plt.plot(ES_steps, ES_winrate, color=ES_color+[1.0])
plt.xlabel('Steps (millions)')
plt.ylabel('Win rate')
plt.tight_layout()
plt.grid()
plt.savefig(os.path.join('Results/Evolutionary/evo-res.pdf'))
plt.show()


I am working on the file "Minesweeper_results.ipynb".
The first task is:
Policy Gradient
Can you write Python code for it?

# Plot rewards and winrate
PG_adjust_axis = 1000000
PG_smoothing = 100
PG_color = [0.1, 0.2, 0.5]

PG_steps = []
PG_reward = []
PG_loss = []
PG_winrate = []

stats = pickle.load(open("Results/Policy-Gradients/stats.p", "rb"))
for stat in stats:
    PG_steps.append(stat[1])
    PG_reward.append(stat[2]/1.5)
    PG_loss.append(stat[3])
    PG_winrate.append(stat[4])
PG_steps = pd.Series(PG_steps)/PG_adjust_axis
PG_reward_smooth = smooth(PG_reward,PG_smoothing)
PG_loss_smooth = smooth(PG_loss,PG_smoothing)
PG_winrate_smooth = smooth(PG_winrate,PG_smoothing)

fig = plt.figure(figsize=(6, 6))
ax1 = plt.subplot(311)
plt.plot(PG_steps, PG_reward_smooth, color=PG_color+[1.0])
plt.plot(PG_steps, PG_reward, color=PG_color+[0.1], label='_nolegend_')
plt.setp(ax1.get_xticklabels(), visible=False)
plt.ylabel('Reward')
plt.grid()

ax2 = plt.subplot(312, sharex=ax1)
plt.plot(PG_steps, PG_loss_smooth, color=PG_color+[1.0])
plt.plot(PG_steps, PG_loss, color=PG_color+[0.1], label='_nolegend_')
plt.setp(ax1.get_xticklabels(), visible=False)
plt.ylabel('Loss')
plt.grid()

ax3 = plt.subplot(313, sharex=ax2)
plt.plot(PG_steps, PG_winrate_smooth, color=PG_color+[1.0])
plt.plot(PG_steps, PG_winrate, color=PG_color+[0.1], label='_nolegend_')
plt.xlabel('Steps (millions)')
plt.ylabel('Win Rate')
plt.grid()
plt.tight_layout()
plt.savefig(os.path.join('Results/Policy-Gradients/policy-res.pdf'))
plt.show()


I am working on the file "Minesweeper_results.ipynb".
The first task is:
Q-Learning
Can you write Python code for it?

# Load the function
import os
import sys
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
os.chdir(filepath)
sys.path.append('q_learning')
from train import *
tf.reset_default_graph()

# Get the 90.20 win-rate
setup_model("test")

# Get the win-rates for different number of mines
tf.reset_default_graph()
# setup_model("test_random_mines") # runs forever

tf.reset_default_graph()


I am working on the file "Minesweeper_results.ipynb".
The first task is:
Q-Learning
Q-Learning no Discount
Can you write Python code for it?

# Plot rewards and winrate
QL_batch_size = 32
QL_adjust_axis = 1000000
QL_smoothing = 10
QL_color = [0.5,0.1,0.2]

QL_reward_pd = pd.read_csv("Results/Q-Learning/evaluation_total_reward_net2_d_0.csv")
QL_reward_num_pd = pd.read_csv("Results/Q-Learning/evaluation_num_rewards_net2_d_0.csv")
QL_reward_pd['Value'] = QL_reward_pd['Value']/QL_reward_num_pd['Value'] # Scale with the number of rewards

QL_reward_smooth_q = smooth(QL_reward_pd['Value'],QL_smoothing)
QL_steps_reward_q = QL_reward_pd['Step']*QL_batch_size/QL_adjust_axis
QL_reward_q = QL_reward_pd['Value']

QL_loss_pd = pd.read_csv("Results/Q-Learning/train_loss_net2_d_0.csv")
QL_steps_loss = QL_loss_pd['Step']*QL_batch_size/QL_adjust_axis
QL_loss_smooth_q = smooth(QL_loss_pd['Value'],QL_smoothing)
QL_loss_q = QL_loss_pd['Value']

QL_loss_smooth_q = QL_loss_smooth_q[1:840]
QL_loss_q = QL_loss_q[1:840]
QL_steps_loss_q = QL_steps_loss[1:840]

QL_winrate_pd = pd.read_csv("Results/Q-Learning/evaluation_win_rate_net2_d_0.csv")
QL_steps_q = QL_winrate_pd['Step']*QL_batch_size/QL_adjust_axis
QL_winrate_q = QL_winrate_pd['Value']/100

fig = plt.figure(figsize=(6, 6))
ax1 = plt.subplot(311)
plt.plot(QL_steps_reward_q, QL_reward_q, color=QL_color+[1.0])
plt.setp(ax1.get_xticklabels(), visible=False)
plt.ylabel('Reward')
plt.grid()
ax2 = plt.subplot(312, sharex=ax1)
plt.plot(QL_steps_loss_q, QL_loss_smooth_q, color=QL_color+[1.0])
plt.plot(QL_steps_loss_q, QL_loss_q, color=QL_color+[0.1], label='_nolegend_')
plt.setp(ax1.get_xticklabels(), visible=False)
plt.ylabel('Loss')
plt.grid()
ax3 = plt.subplot(313, sharex=ax1)
plt.plot(QL_steps_q, QL_winrate_q, color=QL_color+[1.0])
plt.ylabel('Win rate')
plt.xlabel('Steps (millions)')
plt.grid()
plt.tight_layout()
plt.savefig(os.path.join('Results/Q-Learning/Q-Learning-res_net2_d_0.pdf'))
plt.show()


I am working on the file "Minesweeper_results.ipynb".
The first task is:
Testing the Agents
Policy Gradients
Execute the block below to test the Policy Gradient Agent on 10000 games.
If you want to see the agent play, you will have to launch the following bash-command from within the /policy_graident-folder:
python3 test_condensed_6x6_CNN.py --display 1
Can you write Python code for it?

import os
filepath = os.getcwd()
os.chdir(filepath)
os.chdir('policy_gradients')
os.environ["KERAS_BACKEND"] = "tensorflow"
%run test_condensed_6x6_CNN.py


I am working on the file "Minesweeper_results.ipynb".
The first task is:
Training
Our training script will resume the training from the last checkpoint. You are free to interup training whenver you want, it saves progress every 100. epoch (100 epochs is equivalent to 20000 moves with current batch size).
To prevent overwriting saved the networks we present as our final solution, below is shown traning for an older network and non-converged network, but the process is the same. All the training and test files a found in the /policy_gradients folder
Win Rate is evaluated on 1000 games and printed every 400. epoch
Can you write Python code for it?

### De-comment the code below to initiate training of the Policy Gradient agent
# import os
# os.chdir(filepath)
# os.chdir('policy_gradients/')
# %run train_condensed_6x6_v4.py


I am working on the file "Mini Project Student Admissions in Keras/imdb/Student_Admissions.ipynb".
The first task is:
3. Split the data into training and testing sets
Can you write Python code for it?

# break training set into training and validation sets
(X_train, X_test) = X[50:], X[:50]
(y_train, y_test) = y[50:], y[:50]

# print shape of training set
print('x_train shape:', X_train.shape)

# print number of training, validation, and test images
print(X_train.shape[0], 'train samples')
print(X_test.shape[0], 'test samples')


I am working on the file "Mini Project Student Admissions in Keras/imdb/Student_Admissions.ipynb".
The first task is:
4. Define the model architecture
Can you write Python code for it?

# Imports
import numpy as np
from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Activation
from keras.optimizers import SGD
from keras.utils import np_utils

# Building the model
# Note that filling out the empty rank as "0", gave us an extra column, for "Rank 0" students.
# Thus, our input dimension is 7 instead of 6.
model = Sequential()
model.add(Dense(128, activation='relu', input_shape=(7,)))
model.add(Dropout(.2))
model.add(Dense(64, activation='relu'))
model.add(Dropout(.1))
model.add(Dense(2, activation='softmax'))

# Compiling the model
model.compile(loss = 'categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
model.summary()


I am working on the file "Mini Project Student Admissions in Keras/imdb/Student_Admissions.ipynb".
The first task is:
5. Train the model
Can you write Python code for it?

# Training the model
model.fit(X_train, y_train, epochs=200, batch_size=100, verbose=0)


I am working on the file "Mini Project Student Admissions in Keras/imdb/Student_Admissions.ipynb".
The first task is:
6. Score the model
Can you write Python code for it?

# Evaluating the model on the training and testing set
score = model.evaluate(X_train, y_train)
print("\n Training Accuracy:", score[1])
score = model.evaluate(X_test, y_test)
print("\n Testing Accuracy:", score[1])


I am working on the file "Mini Project Student Admissions in Keras/imdb/Student_Admissions.ipynb".
The first task is:
Let's plot the data and see how it looks.
Can you write Python code for it?

import matplotlib.pyplot as plt
import numpy as np
def plot_points(data):
    X = np.array(data[["gre","gpa"]])
    y = np.array(data["admit"])
    admitted = X[np.argwhere(y==1)]
    rejected = X[np.argwhere(y==0)]
    plt.scatter([s[0][0] for s in rejected], [s[0][1] for s in rejected], s = 25, color = 'red', edgecolor = 'k')
    plt.scatter([s[0][0] for s in admitted], [s[0][1] for s in admitted], s = 25, color = 'cyan', edgecolor = 'k')
    plt.xlabel('Test (GRE)')
    plt.ylabel('Grades (GPA)')
plot_points(data)
plt.show()


I am working on the file "Mini Project/Analysis on the Movie Lens dataset-Copy1.ipynb".
The first task is:
Based on the above exploratory commands, I believe that the following questions can be answered using the dataset:

Is there a correlation or a trend between the year of release of a movie and the genre?
Which genres were more dominant in each decade of the range available in the dataset?
Do science fiction movies tend to be rated more highly than other movie genres?

For the mini-project, I have chosen question 3 for further analysis.
Using pandas methods to explore the dataset
Includes matplotlib visualization
Can you write Python code for it?

# List of genres as a Python list

genres = ['Action','Adventure','Animation','Children','Comedy','Crime','Documentary','Drama','Fantasy','Film-Noir','Horror','Musical','Mystery','Romance','Sci-Fi','Thriller','War','Western']

genres_rating_list = []

# The loop reads each element of the above list
    # For each iteration, one genre is selected from the movies data frame
    # This selection of the data frame is then merged with the rating data frame to get the rating for that genre
    # Once the new merged data frame is created, we use the mean function to get the mean rating for the genre
    # The genre and the corresponding mean rating are then appended to the genres_rating Data Frame
    # The entire looping takes long - can certainly be optimized for performance

for i in range(len(genres)):
    fil = genres[i]+'_filter'
    mov = genres[i]+'_movies'
    rat = genres[i]+'_ratings'
    rat_mean = rat+'_mean'
    fil = movies['genres'].str.contains(genres[i])
    mov = movies[fil]
    rat = mov.merge(ratings, on='movieId', how='inner')
    rat_mean = round(rat['rating'].mean(), 2)
    #print(genres[i], round(rat_mean,2))
    genres_rating_list.append(rat_mean)

df = {'Genre':genres, 'Genres Mean Rating':genres_rating_list}

genres_rating = pd.DataFrame(df)

genres_rating

genres_rating['Genres Standard Deviation'] = genres_rating['Genres Mean Rating'].std()

genres_rating['Mean'] = genres_rating['Genres Mean Rating'].mean()
genres_rating['Zero'] = 0

genres_rating

overall_mean = round(genres_rating['Genres Mean Rating'].mean(), 2)
overall_std = round(genres_rating['Genres Mean Rating'].std(),2)
scifi_rating = genres_rating[genres_rating['Genre'] == 'Sci-Fi']['Genres Mean Rating']

print(overall_mean)

print(overall_std)

print(scifi_rating)

genres_rating['Diff from Mean'] = genres_rating['Genres Mean Rating'] - overall_mean

genres_rating


I am working on the file "Mini Project/Analysis on the Movie Lens dataset-Copy1.ipynb".
The first task is:
Now that we have a move year column, let us list the data types of the columns in the movies data frame.
Can you write Python code for it?


I am working on the file "Mini Project/Analysis on the Movie Lens dataset-Copy1.ipynb".
The first task is:
movie_year is of float64 datat type. We must convert the data type of the movie_year column to int64. Before we go ahead and do that, we must replace all NULL and inifinite entries in the column with zero. If we do not perform this step, we will get the following errror message.
Can you write Python code for it?

#movies['movie_year'].astype(np.int64)

#movies['movie_year'] = movies['movie_year'].astype(np.int64, inplace=True)

movies.dtypes

movies.describe()

movies.head()

movies_and_years = pd.DataFrame(movies['movie_year', 'title_only'])

movies_and_years.tail()

#a = pd.DataFrame(movies['title_only'].groupby(movies['movie_year']).count(), movies['movie_year'], movies['title_only'])

#plt.plot(a['title_only'], a['movie_year'])
#plt.show()

#movies_and_years

#movies_and_years['title_only'].groupby[movies_and_years['movie_year']]

#create a unique array of years from the newly created data frame column
#years_array_raw = movies['movie_year'].unique()
#years_array_raw  = years_array_raw.astype(int)

#years_array_raw


I am working on the file "Mini Project/Analysis on the Movie Lens dataset.ipynb".
The first task is:
Based on the above exploratory commands, I believe that the following questions can be answered using the dataset:

Is there a correlation or a trend between the year of release of a movie and the genre?
Which genres were more dominant in each decade of the range available in the dataset?
Do science fiction movies tend to be rated more highly than other movie genres?

For the mini-project, I have chosen question 3 for further analysis.
Using pandas methods to explore the dataset
Includes matplotlib visualization
Can you write Python code for it?


I am working on the file "Mini Project/Analysis on the Movie Lens dataset.ipynb".
The first task is:
Now that we have a data frame of information about each genre and the corresponding mean rating, we will visualize the data using matplotlib
Can you write Python code for it?

genre_list = list(genres_rating['Genre'])

genres_rating_list = list(genres_rating['Genres Mean Rating'])
genres_diff_list = list(genres_rating['Diff from Mean'])

%matplotlib inline
import matplotlib.pyplot as plt

plt.figure(figsize=(20, 10))

ax1 = plt.subplot(2,1,1)
x = [x for x in range(0, 18)]
xticks_genre_list = genre_list
y = genres_rating_list
plt.xticks(range(len(x)), xticks_genre_list)
plt.scatter(x,y, color='g')
plt.plot(x, genres_rating['Mean'], color="red")
plt.autoscale(tight=True)
#plt.rcParams["figure.figsize"] = (10,2)
plt.title('Movie ratings by genre')
plt.xlabel('Genre')
plt.ylabel('Rating')
plt.ylim(ymax = 4, ymin = 3)
plt.grid(True)
plt.savefig(r'movie-ratings-by-genre.png')

plt.annotate("Sci-Fi Rating",
            xy=(14.25,3.5), xycoords='data',
            xytext=(14.20, 3.7), textcoords='data',
            arrowprops=dict(arrowstyle="->",
                            connectionstyle="arc3"),
            )

for i,j in enumerate( y ):
    ax1.annotate( j, ( x[i] + 0.03, y[i] + 0.02))

ax2 = plt.subplot(2,1,2)
x = [x for x in range(0, 18)]
xticks_genre_list = genre_list
y = genres_rating['Diff from Mean']
plt.xticks(range(len(x)), xticks_genre_list)
plt.plot(x,y)
plt.plot(x, genres_rating['Zero'])
plt.autoscale(tight=True)
#plt.rcParams["figure.figsize"] = (10,2)
plt.title('Deviation of each genre\'s rating from the overall mean rating')
plt.xlabel('Genre')
plt.ylabel('Deviation from mean rating')
plt.grid(True)
plt.savefig(r'deviation-from-mean-rating.png')

plt.annotate("Sci-Fi Rating",
            xy=(14,-0.13), xycoords='data',
            xytext=(14.00, 0.0), textcoords='data',
            arrowprops=dict(arrowstyle="->",
                            connectionstyle="arc3"),
            )

plt.show()


I am working on the file "Mini Project/Analysis on the Movie Lens dataset.ipynb".
The first task is:
Now that we have a move year column, let us list the data types of the columns in the movies data frame.
movie_year is of float64 datat type. We must convert the data type of the movie_year column to int64. Before we go ahead and do that, we must replace all NULL and inifinite entries in the column with zero. If we do not perform this step, we will get the following errror message.
Can you write Python code for it?

movie_year = pd.DataFrame(movies['title_only'].groupby(movies['movie_year']).count())

movie_year.reset_index(inplace=True)

X=movie_year['movie_year']
Y=movie_year['title_only']

plt.plot_date(X,Y,'bo-')
plt.grid(True)
plt.rcParams["figure.figsize"] = (15,5)
plt.title('Number of movies per year')
plt.xlabel('Years')
plt.ylabel('Number of movies')
plt.xlim('1885-01-01','2020-01-01')
plt.show()


I am working on the file "Mini Project/Analysis on the Movie Lens dataset.ipynb".
The first task is:
Reporting findings/analyses
Now that we have a couple plots, let us revisit the question we want to answer using the dataset.
Again, the question is  - Do science fiction movies tend to be rated more highly than other movie genres?

The scatter plot shows the mean rating value for each genre. Each genre has a value on the scatter plot for the mean rating value for that genre. Let us now see if the plot is able to help us answer the question above.

The mean rating for Sci-Fi genre is about 3.45. When looking at the plot, we see that there are only three other genres out of 18 genres in total, that have lesser mean ratings than Sci-Fi - Horror, Children and Comedy. The remaining 10 genres have mean ratings higher than Science Fiction.

This gives us enough information to answer the question. Sci-Fi movies do not tend to be rated higher than other genres.

The second plot, a bar plot, shows how much each genre's ratings deviate from the overall mean of ratings. Science Fiction is around -0.13 lower than the mean rating of 3.58, showing lesser deviation than Horror at the lower end and Film-Noir at the higher end.

To conclude - no, science fiction movies are not rated higher than other movie genres. The ratings for science fiction movies hover around the mean ratings for all movies.
I have submitted my work to the mini project section of the course. Now, we will explore the dataset further and try to answer the remaining questions I have listed at the beginning of the notebook.

- Is there a correlation or a trend between the year of release of a movie and the genre?
- Which genres were more dominant in each decade of the range available in the dataset?
Can you write Python code for it?

# extract year of release of each movie from the title column
# convert the data type of the movie_year column to numeric (from str)

import numpy as np
import re

movies['movie_year'] = movies['title']

movies['movie_year'] = movies['movie_year'].str.extract(r"\(([0-9]+)\)", expand=False)

# creating a new column with just the movie titles
movies['title_only'] = movies['title']
movies['title_only'] = movies['title_only'].str.extract('(.*?)\s*\(', expand=False)

 movies['movie_year'].fillna(0, inplace=True)

#Drop all rows containing incorrect year values - such as 0, 6, 69, 500 and -2147483648
movies.drop(movies[movies.movie_year == '0'].index, inplace=True)
movies.drop(movies[movies.movie_year == '6'].index, inplace=True)
movies.drop(movies[movies.movie_year == '06'].index, inplace=True)
movies.drop(movies[movies.movie_year == '69'].index, inplace=True)
movies.drop(movies[movies.movie_year == '500'].index, inplace=True)
movies.drop(movies[movies.movie_year == '-2147483648'].index, inplace=True)

movies.drop(movies[movies.movie_year == 0].index, inplace=True)
movies.drop(movies[movies.movie_year == 6].index, inplace=True)
movies.drop(movies[movies.movie_year == 69].index, inplace=True)
movies.drop(movies[movies.movie_year == 500].index, inplace=True)
movies.drop(movies[movies.movie_year == -2147483648].index, inplace=True)

#convert the string values to numeric
movies['movie_year'] = pd.to_datetime(movies['movie_year'], format='%Y')


I am working on the file "Mini Project/Analysis on the Movie Lens dataset.ipynb".
The first task is:
The above plot provides some interesting insight:
* There was a steady increase in the number of movies after 1930 and till 2008.
* In this dataset, 2009 was the year when the highest number of movies were produced - 1112 in all.
* The decades between 1970 and 2000 saw the highest year-on-year increase in the number of movies produced.
* 2014 saw a sharp drop in the nimber of movies produced, from 1011 in 2013 to only 740 movies.
* The movie count of 2015 is only 120. This could possibly be due to the lack of information available for the entire year of 2015.
Can you write Python code for it?

movies.head()

list(movies)

a = pd.Series(movies.iloc[0])

a

def flat(str1):
    c = pd.DataFrame(columns=list(movies))
    for i in range(len(str1)):
        #print(str1[i])
        if i == 2:
            a = str1[i].split('|')
    for j in range(len(a)):
        c.loc[j] = [str1[0], str1[1], a[j], str1[3], str1[4]]
    return c

c = flat(a)

c


I am working on the file "Mini curso Python - SEC 2015.ipynb".
The first task is:
Banco de dados SQLite
Can you write Python code for it?

import sqlite3
conn = sqlite3.connect('exemplo.db')
c = conn.cursor()

c.execute("CREATE TABLE IF NOT EXISTS pessoas (id integer primary key, nome text, telefone text, data_nascimento date)")
c.execute("""INSERT INTO pessoas (nome, telefone, data_nascimento)
VALUES ('Herberth Amaral','32222222', '1988-11-05')""")
conn.commit()
c.execute('SELECT * from pessoas')
c.fetchone()


I am working on the file "Mini curso Python - SEC 2015.ipynb".
The first task is:
Demonstração do NumPy, pedra fundamental para álgebra linear no Python
Can you write Python code for it?

import numpy as np

matriz = np.matrix([[1,2,3,4], [5,6,7,8]])
matriz

matriz.T #transposta

matriz*matriz.T


I am working on the file "Mini curso Python - SEC 2015.ipynb".
The first task is:
Exemplo do Matplotlib, a principal ferramenta de plotagem de gráficos em Python
Can you write Python code for it?

import numpy as np
x = np.linspace(0, 5)
plot(x, cos(x))


I am working on the file "Mini curso Python - SEC 2015.ipynb".
The first task is:
Pequena demonstração de ferramentas úteis para engenheiros
Can you write Python code for it?


I am working on the file "MiniProjeto2/Mini-Projeto-2.ipynb".
The first task is:
Resultado
Podemos perceber que a medida que aumentamos o n, a amostra acaba tendendo a normal e para isso bastou apenas o n = 50. A correlaçao que foi mostrada entre o primeiro e o nono numero foi quase igual a 0, o que significa que essas duas variaveis não tem relação nenhuma entre elas, mostrando que o nosso experimento foi um sucesso.
Can you write Python code for it?

media50=[]
primeiro= []
nono = []

for i in range (0,10001):
    valores = np.random.choice(x,50)
    primeiro.append(valores[0])
    nono.append(valores[9])
    media50.append(np.mean(valores))

media = np.mean(media50)
variancia = np.var(media50)
print("Media:{}".format(media))
print("Variancia:{}".format(variancia))
plt.hist(media50)
plt.title("n valendo 50")
plt.show()

dp1= (np.var(primeiro))**1/2
dp9=(np.var(nono))**1/2
cov = np.cov(primeiro,nono)
plt.hist(primeiro)
plt.title("primeiro")
plt.show()

plt.hist(nono)
plt.title("nono")
plt.show()

print(cov/(dp1*dp9))


I am working on the file "Mini_Project_Clustering.ipynb".
The first task is:
Choosing K: The Silhouette Method
There exists another method that measures how well each datapoint $x_i$ "fits" its assigned cluster and also how poorly it fits into other clusters. This is a different way of looking at the same objective. Denote $a_{x_i}$ as the average distance from $x_i$ to all other points within its own cluster $k$. The lower the value, the better. On the other hand $b_{x_i}$ is the minimum average distance from $x_i$ to points in a different cluster, minimized over clusters. That is, compute separately for each cluster the average distance from $x_i$ to the points within that cluster, and then take the minimum. The silhouette $s(x_i)$ is defined as
$$s(x_i) = \frac{b_{x_i} - a_{x_i}}{\max{\left( a_{x_i}, b_{x_i}\right)}}$$
The silhouette score is computed on every datapoint in every cluster. The silhouette score ranges from -1 (a poor clustering) to +1 (a very dense clustering) with 0 denoting the situation where clusters overlap. Some criteria for the silhouette coefficient is provided in the table below.

| Range       | Interpretation                                |
|-------------|-----------------------------------------------|
| 0.71 - 1.0  | A strong structure has been found.            |
| 0.51 - 0.7  | A reasonable structure has been found.        |
| 0.26 - 0.5  | The structure is weak and could be artificial.|
| < 0.25      | No substantial structure has been found.      |

Source: http://www.stat.berkeley.edu/~spector/s133/Clus.html
Fortunately, scikit-learn provides a function to compute this for us (phew!) called sklearn.metrics.silhouette_score. Take a look at this article on picking $K$ in scikit-learn, as it will help you in the next exercise set.

Checkup Exercise Set III

Exercise: Using the documentation for the `silhouette_score` function above, construct a series of silhouette plots like the ones in the article linked above.

Exercise: Compute the average silhouette score for each...
Can you write Python code for it?

from __future__ import print_function

from sklearn.datasets import make_blobs
from sklearn.cluster import KMeans
from sklearn.metrics import silhouette_samples, silhouette_score

import matplotlib.pyplot as plt
import matplotlib.cm as cm
import numpy as np

print(__doc__)

# Generating the sample data from make_blobs
# This particular setting has one distinct cluster and 3 clusters placed close
# together.
X, y = make_blobs(n_samples=500,
                  n_features=2,
                  centers=4,
                  cluster_std=1,
                  center_box=(-10.0, 10.0),
                  shuffle=True,
                  random_state=1)  # For reproducibility

range_n_clusters = [2, 3, 4, 5, 6]

for n_clusters in range_n_clusters:
    # Create a subplot with 1 row and 2 columns
    fig, (ax1, ax2) = plt.subplots(1, 2)
    fig.set_size_inches(18, 7)

    # The 1st subplot is the silhouette plot
    # The silhouette coefficient can range from -1, 1 but in this example all
    # lie within [-0.1, 1]
    ax1.set_xlim([-0.1, 1])
    # The (n_clusters+1)*10 is for inserting blank space between silhouette
    # plots of individual clusters, to demarcate them clearly.
    ax1.set_ylim([0, len(X) + (n_clusters + 1) * 10])

    # Initialize the clusterer with n_clusters value and a random generator
    # seed of 10 for reproducibility.
    clusterer = KMeans(n_clusters=n_clusters, random_state=10)
    cluster_labels = clusterer.fit_predict(X)

    # The silhouette_score gives the average value for all the samples.
    # This gives a perspective into the density and separation of the formed
    # clusters
    silhouette_avg = silhouette_score(X, cluster_labels)
    print("For n_clusters =", n_clusters,
          "The average silhouette_score is :", silhouette_avg)

    # Compute the silhouette scores for each sample
    sample_silhouette_values = silhouette_samples(X, cluster_labels)

    y_lower = 10
    for i in range(n_clusters):
        # Aggregate the silhouette scores for samples belonging to
        # cluster...


I am working on the file "Mini_Project_Clustering.ipynb".
The first task is:
Data
The dataset contains information on marketing newsletters/e-mail campaigns (e-mail offers sent to customers) and transaction level data from customers. The transactional data shows which offer customers responded to, and what the customer ended up buying. The data is presented as an Excel workbook containing two worksheets. Each worksheet contains a different dataset.
Can you write Python code for it?

df_offers = pd.read_excel("./WineKMC.xlsx", sheetname=0)
df_offers.columns = ["offer_id", "campaign", "varietal", "min_qty", "discount", "origin", "past_peak"]
df_offers.head()


I am working on the file "Mini_Project_Clustering.ipynb".
The first task is:
Data wrangling
We're trying to learn more about how our customers behave, so we can use their behavior (whether or not they purchased something based on an offer) as a way to group similar minded customers together. We can then study those groups to look for patterns and trends which can help us formulate future offers.
The first thing we need is a way to compare customers. To do this, we're going to create a matrix that contains each customer and a 0/1 indicator for whether or not they responded to a given offer.

Checkup Exercise Set I

Exercise: Create a data frame where each row has the following columns (Use the pandas [`merge`](http://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.merge.html) and [`pivot_table`](http://pandas.pydata.org/pandas-docs/stable/generated/pandas.pivot_table.html) functions for this purpose):

 customer_name

 One column for each offer, with a 1 if the customer responded to the offer

Make sure you also deal with any weird values such as `NaN`. Read the documentation to develop your solution.

Can you write Python code for it?

#your turn

# merge the dataframes based on offer id
df_merged = pd.merge(df_transactions, df_offers, on='offer_id')

# create a matrix of customer name and offer id.  Replace NaN values with zero and reset index to offer id rather than customer
x_cols = pd.pivot_table(df_merged, values='n', index=['customer_name'], columns=['offer_id']).fillna(0).reset_index()

# create dataframe without customer name
X = x_cols[x_cols.columns[1:]]


I am working on the file "Mini_Project_Clustering.ipynb".
The first task is:
K-Means Clustering
Recall that in K-Means Clustering we want to maximize the distance between centroids and minimize the distance between data points and the respective centroid for the cluster they are in. True evaluation for unsupervised learning would require labeled data; however, we can use a variety of intuitive metrics to try to pick the number of clusters K. We will introduce two methods: the Elbow method, the Silhouette method and the gap statistic.
Choosing K: The Elbow Sum-of-Squares Method
The first method looks at the sum-of-squares error in each cluster against $K$. We compute the distance from each data point to the center of the cluster (centroid) to which the data point was assigned.
$$SS = \sum_k \sum_{x_i \in C_k} \sum_{x_j \in C_k} \left( x_i - x_j \right)^2 = \sum_k \sum_{x_i \in C_k} \left( x_i - \mu_k \right)^2$$
where $x_i$ is a point, $C_k$ represents cluster $k$ and $\mu_k$ is the centroid for cluster $k$. We can plot SS vs. $K$ and choose the elbow point in the plot as the best value for $K$. The elbow point is the point at which the plot starts descending much more slowly.

Checkup Exercise Set II

Exercise:

 What values of $SS$ do you believe represent better clusterings? Why?
 The value of $SS$ that is mininum and the number of clusters is also a minimum

 Create a numpy matrix `x_cols` with only the columns representing the offers (i.e. the 0/1 colums)
 Done

 Write code that applies the [`KMeans`](http://scikit-learn.org/stable/modules/generated/sklearn.cluster.KMeans.html) clustering method from scikit-learn to this matrix.
 Done

 Construct a plot showing $SS$ for each $K$ and pick $K$ using this plot. For simplicity, test $2 \le K \le 10$.
 Done

 Make a bar chart showing the number of points in each cluster for k-means under the best $K$.
 Done

 What challenges did you experience using the Elbow method to pick $K$?
 1) Selecting the number of...
Can you write Python code for it?

#your turn
from scipy.spatial.distance import cdist, pdist
from sklearn.cluster import KMeans
import numpy as np

# get Kmean and centroids
K = range(2, 11)
KM = [KMeans(n_clusters=k).fit(X) for k in K]
centroids = [k.cluster_centers_ for k in KM]

# compute euclidean distance
D_k = [cdist(X, mid, 'euclidean') for mid in centroids]
cIdx = [np.argmin(D,axis=1) for D in D_k]
dist = [np.min(D,axis=1) for D in D_k]

# Total with-in sum of square
tss = [sum(d**2) for d in dist]

# Construct a plot showing  SSSS  for each  KK
fig = plt.figure()
ax = fig.add_subplot(111)
ax.set_xlim([1, 11])
ax.plot(K, tss, 'b*-')
ax.plot(K[6], tss[6], marker='o', markersize=12,
markeredgewidth=2, markeredgecolor='r', markerfacecolor='None')
plt.grid(True)
plt.xlabel('Number of clusters')
plt.ylabel('Within-cluster sum of squares')
plt.title('Elbow for KMeans clustering')

# setup KMEans for cluster = 8
cluster = KMeans(n_clusters=8)

# predict and assign to a cluster
x_cols['cluster'] = cluster.fit_predict(X)
y = x_cols.cluster.value_counts()

# index number is the cluster number
cluster = y.index.values
x_lim = np.arange(len(y))

# plot bar chart
plt.bar(x_lim, y, align='center', alpha=0.5)
plt.xticks(x_lim, cluster)
plt.ylabel('Counts')
plt.title('Number of points per cluster')

plt.show()


I am working on the file "Mini_Project_Clustering.ipynb".
The first task is:
Other Clustering Algorithms
k-means is only one of a ton of clustering algorithms. Below is a brief description of several clustering algorithms, and the table provides references to the other clustering algorithms in scikit-learn.

Affinity Propagation does not require the number of clusters $K$ to be known in advance! AP uses a "message passing" paradigm to cluster points based on their similarity.

Spectral Clustering uses the eigenvalues of a similarity matrix to reduce the dimensionality of the data before clustering in a lower dimensional space. This is tangentially similar to what we did to visualize k-means clusters using PCA. The number of clusters must be known a priori.

Ward's Method applies to hierarchical clustering. Hierarchical clustering algorithms take a set of data and successively divide the observations into more and more clusters at each layer of the hierarchy. Ward's method is used to determine when two clusters in the hierarchy should be combined into one. It is basically an extension of hierarchical clustering. Hierarchical clustering is divisive, that is, all observations are part of the same cluster at first, and at each successive iteration, the clusters are made smaller and smaller. With hierarchical clustering, a hierarchy is constructed, and there is not really the concept of "number of clusters." The number of clusters simply determines how low or how high in the hierarchy we reference and can be determined empirically or by looking at the dendogram.

Agglomerative Clustering is similar to hierarchical clustering but but is not divisive, it is agglomerative. That is, every observation is placed into its own cluster and at each iteration or level or the hierarchy, observations are merged into fewer and fewer clusters until convergence. Similar to hierarchical clustering, the constructed hierarchy contains all possible numbers of clusters and it is up to the analyst to pick the number by reviewing statistics or the dendogram.

DBSCAN is based on point density rather than...
Can you write Python code for it?

# your turn
# Affinity propagation
from sklearn.cluster import AffinityPropagation
from sklearn import metrics

af = AffinityPropagation().fit(X)
cluster_centers_indices = af.cluster_centers_indices_
labels = af.labels_

n_clusters_ = len(cluster_centers_indices)

print('Estimated number of clusters: %d' % n_clusters_)
print("Silhouette Coefficient: %0.3f"
      % metrics.silhouette_score(X, labels, metric='sqeuclidean'))

import matplotlib.pyplot as plt
from itertools import cycle

plt.close('all')
plt.figure(1)
plt.clf()

colors = cycle('bgrcmykbgrcmykbgrcmykbgrcmyk')
for k, col in zip(range(n_clusters_), colors):
    class_members = labels == k
    cluster_center = X[cluster_centers_indices[k]]
    plt.plot(X[class_members, 0], X[class_members, 1], col + '.')
    plt.plot(cluster_center[0], cluster_center[1], 'o', markerfacecolor=col,
             markeredgecolor='k', markersize=14)
    for x in X[class_members]:
        plt.plot([cluster_center[0], x[0]], [cluster_center[1], x[1]], col)

plt.title('Estimated number of clusters: %d' % n_clusters_)
plt.show()

# your turn
# Spectral Clustering
from sklearn import cluster

for n_clusters in range(2,3):
#n_clusters = 4
    spectral = cluster.SpectralClustering(n_clusters=n_clusters,
                                          eigen_solver='arpack',
                                          affinity="nearest_neighbors")
    spectral.fit(X)
    labels = spectral.labels_

    print('Assigned number of clusters: %d' % n_clusters)
    print("Silhouette Coefficient: %0.3f"
      % metrics.silhouette_score(X, labels))

plt.scatter(X[:, 0], X[:, 1], c=spectral.labels_, cmap=plt.cm.spectral)
plt.title('Assigned number of clusters: %d' % n_clusters)

# AgglomerativeClustering
from sklearn.cluster import AgglomerativeClustering

for n_clusters in range(2,3):
#n_clusters = 4
    linkage= 'ward'
    model = AgglomerativeClustering(n_clusters=n_clusters)
    model.fit(X)
    labels = model.labels_
    print("Silhouette Coefficient: %0.3f"
          %...


I am working on the file "Mini_Project_Clustering.ipynb".
The first task is:
What we've done is we've taken those columns of 0/1 indicator variables, and we've transformed them into a 2-D dataset. We took one column and arbitrarily called it x and then called the other y. Now we can throw each point into a scatterplot. We color coded each point based on it's cluster so it's easier to see them.

Exercise Set V

As we saw earlier, PCA has a lot of other uses. Since we wanted to visualize our data in 2 dimensions, restricted the number of dimensions to 2 in PCA. But what is the true optimal number of dimensions?

Exercise: Using a new PCA object shown in the next cell, plot the `explained_variance_` field and look for the elbow point, the point where the curve's rate of descent seems to slow sharply. This value is one possible value for the optimal number of dimensions. What is it?

Can you write Python code for it?

#your turn
# Initialize a new PCA model with a default number of components.
from sklearn.decomposition import PCA

# Do the rest on your own :)

from sklearn import decomposition
pca = PCA()
pca.fit(X)
pca_ratio = (np.round(pca.explained_variance_, decimals=4)*100)
K = [1, 2]

fig = plt.figure()
ax = fig.add_subplot(111)
ax.set_xlim([1, 3])
ax.plot(K, pca_ratio, 'b*-')

plt.grid(True)
plt.xlabel('Number of dimensions')
plt.ylabel('PCA Explained variance')
plt.title('Elbow for PCA explained variance')


I am working on the file "Mini_Project_Linear_Regression.ipynb".
The first task is:
Another way, is to split the data into random train and test subsets using the function train_test_split in sklearn.cross_validation. Here's the documentation.
Can you write Python code for it?

#X_train, X_test, Y_train, Y_test = sklearn.cross_validation.train_test_split(
#    X, bos.PRICE, test_size=0.33, random_state = 5)
print X_train.shape
print X_test.shape
print Y_train.shape
print Y_test.shape
X_train


X_train, X_test, Y_train, Y_test = sklearn.cross_validation.train_test_split(
    X, bos.PRICE, test_size=0.33, random_state = 5)
print (X_train.shape)
print (X_test.shape)
print (Y_train.shape)
print (Y_test.shape)


I am working on the file "Mini_Project_Linear_Regression.ipynb".
The first task is:
EDA and Summary Statistics

Let's explore this data set.  First we use describe() to get basic summary statistics for each of the columns.
Can you write Python code for it?


I am working on the file "Mini_Project_Linear_Regression.ipynb".
The first task is:
Histograms

Histograms are a useful way to visually summarize the statistical properties of numeric variables. They can give you an idea of the mean and the spread of the variables as well as outliers.
Can you write Python code for it?

plt.hist(bos.CRIM)
plt.title("CRIM")
plt.xlabel("Crime rate per capita")
plt.ylabel("Frequency")
plt.show()


I am working on the file "Mini_Project_Linear_Regression.ipynb".
The first task is:
Interpreting coefficients
There is a ton of information in this output. But we'll concentrate on the coefficient table (middle table). We can interpret the RM coefficient (9.1021) by first noticing that the p-value (under P>|t|) is so small, basically zero. We can interpret the coefficient as, if we compare two groups of towns, one where the average number of rooms is say $5$ and the other group is the same except that they all have $6$ rooms. For these two groups the average difference in house prices is about $9.1$ (in thousands) so about $\$9,100$ difference. The confidence interval fives us a range of plausible values for this difference, about ($\$8,279, \$9,925$), deffinitely not chump change.
statsmodels formulas

This formula notation will seem familiar to R users, but will take some getting used to for people coming from other languages or are new to statistics.
The formula gives instruction for a general structure for a regression call. For statsmodels (ols or logit) calls you need to have a Pandas dataframe with column names that you will add to your formula. In the below example you need a pandas data frame that includes the columns named (Outcome, X1,X2, ...), bbut you don't need to build a new dataframe for every regression. Use the same dataframe with all these things in it. The structure is very simple:
Outcome ~ X1
But of course we want to to be able to handle more complex models, for example multiple regression is doone like this:
Outcome ~ X1 + X2 + X3
This is the very basic structure but it should be enough to get you through the homework. Things can get much more complex, for a quick run-down of further uses see the statsmodels help page.
Let's see how our model actually fit our data. We can see below that there is a ceiling effect, we should probably look into that. Also, for large values of $Y$ we get underpredictions, most predictions are below the 45-degree gridlines.
Your turn: Create a scatterpot between the predicted prices, available in m.fittedvalues and the original prices....
Can you write Python code for it?

# your turn
plt.scatter(m.fittedvalues, bos.PRICE)
plt.xlabel("Predicted Price")
plt.ylabel("Original Housing Price")
plt.title("Predicted vs. Original Prices");


I am working on the file "Mini_Project_Linear_Regression.ipynb".
The first task is:
Linear regression with  Boston housing data example

Here,
$Y$ = boston housing prices (also called "target" data in python)
and
$X$ = all the other features (or independent variables)
which we will use to fit a linear regression model and predict Boston housing prices. We will use the least squares method as the way to estimate the coefficients.
We'll use two ways of fitting a linear regression. We recommend the first but the second is also powerful in its features.
Fitting Linear Regression using statsmodels

Statsmodels is a great Python library for a lot of basic and inferential statistics. It also provides basic regression functions using an R-like syntax, so it's commonly used by statisticians. While we don't cover statsmodels officially in the Data Science Intensive, it's a good library to have in your toolbox. Here's a quick example of what you could do with it.
Can you write Python code for it?

# Import regression modules
# ols - stands for Ordinary least squares, we'll use this
import statsmodels.api as sm
from statsmodels.formula.api import ols

# statsmodels works nicely with pandas dataframes
# The thing inside the "quotes" is called a formula, a bit on that below
m = ols('PRICE ~ RM',bos).fit()
print m.summary()


I am working on the file "Mini_Project_Linear_Regression.ipynb".
The first task is:
Mean squared error

This is simple the mean of the residual sum of squares.
Your turn: Calculate the mean squared error and print it.
Can you write Python code for it?

#your turn
print ('Mean squared error: ', np.mean((bos.PRICE - lm.predict(X)) ** 2))


#your turn
print np.mean((bos.PRICE - lm.predict(X)) ** 2)


I am working on the file "Mini_Project_Linear_Regression.ipynb".
The first task is:
Now let's explore the data set itself.
Can you write Python code for it?

bos = pd.DataFrame(boston.data)
bos.head()


I am working on the file "Mini_Project_Linear_Regression.ipynb".
The first task is:
Now we have a pandas DataFrame called bos containing all the data we want to use to predict Boston Housing prices.  Let's create a variable called PRICE which will contain the prices. This information is contained in the target data.
Can you write Python code for it?

print (boston.target.shape)

bos['PRICE'] = boston.target
bos.head()


I am working on the file "Mini_Project_Linear_Regression.ipynb".
The first task is:
Other important things to think about when fitting a linear regression model

**Linearity**. The dependent variable $Y$ is a linear combination of the regression coefficients and the independent variables $X$.

**Constant standard deviation**. The SD of the dependent variable $Y$ should be constant for different values of X.

e.g. PTRATIO

 **Normal distribution for errors**.  The $\epsilon$ term we discussed at the beginning are assumed to be normally distributed.
  $$ \epsilon_i \sim N(0, \sigma^2)$$
Sometimes the distributions of responses $Y$ may not be normally distributed at any given value of $X$.  e.g. skewed positively or negatively.

 **Independent errors**.  The observations are assumed to be obtained independently.

e.g. Observations across time may be correlated

Part 3: Training and Test Data sets
Purpose of splitting data into Training/testing sets

 Let's stick to the linear regression example:

 We built our model with the requirement that the model fit the data well.

 As a side-effect, the model will fit THIS dataset well. What about new data?

 We wanted the model for predictions, right?

 One simple solution, leave out some data (for testing) and train the model on the rest

 This also leads directly to the idea of cross-validation, next section.

One way of doing this is you can create training and testing data sets manually.
Can you write Python code for it?

X_train = X[:-50]
X_test = X[-50:]
Y_train = bos.PRICE[:-50]
Y_test = bos.PRICE[-50:]
print X_train.shape
print X_test.shape
print Y_train.shape
print Y_test.shape
X_train.head()


I am working on the file "Mini_Project_Linear_Regression.ipynb".
The first task is:
Other important things to think about when fitting a linear regression model

**Linearity**. The dependent variable $Y$ is a linear combination of the regression coefficients and the independent variables $X$.

**Constant standard deviation**. The SD of the dependent variable $Y$ should be constant for different values of X.

e.g. PTRATIO

 **Normal distribution for errors**.  The $\epsilon$ term we discussed at the beginning are assumed to be normally distributed.
  $$ \epsilon_i \sim N(0, \sigma^2)$$
Sometimes the distributions of responses $Y$ may not be normally distributed at any given value of $X$.  e.g. skewed positively or negatively.

 **Independent errors**.  The observations are assumed to be obtained independently.

e.g. Observations across time may be correlated

Can you write Python code for it?

sns.set(font_scale=.8)
sns.heatmap(X.corr(), vmax=.8, square=True, annot=True)


I am working on the file "Mini_Project_Linear_Regression.ipynb".
The first task is:
Output | Description
--- | ---
lm.coef_ | Estimated coefficients
lm.intercept_ | Estimated intercept
Fit a linear model

The lm.fit() function estimates the coefficients the linear regression using least squares.
Can you write Python code for it?

# Use all 13 predictors to fit linear regression model
lm.fit(X, bos.PRICE)


I am working on the file "Mini_Project_Linear_Regression.ipynb".
The first task is:
Part 1: Linear Regression
Purpose of linear regression

 Given a dataset $X$ and $Y$, linear regression can be used to:

 Build a predictive model to predict future values of $X_i$ without a $Y$ value.

 Model the strength of the relationship between each dependent variable $X_i$ and $Y$

 Sometimes not all $X_i$ will have a relationship with $Y$

 Need to figure out which $X_i$ contributes most information to determine $Y$

Linear regression is used in so many applications that I won't warrant this with examples. It is in many cases, the first pass prediction algorithm for continuous outcomes.

A brief recap (feel free to skip if you don't care about the math)

Linear Regression is a method to model the relationship between a set of independent variables $X$ (also knowns as explanatory variables, features, predictors) and a dependent variable $Y$.  This method assumes the relationship between each predictor $X$ is linearly related to the dependent variable $Y$.
$$ Y = \beta_0 + \beta_1 X + \epsilon$$
where $\epsilon$ is considered as an unobservable random variable that adds noise to the linear relationship. This is the simplest form of linear regression (one variable), we'll call this the simple model.

$\beta_0$ is the intercept of the linear model

Multiple linear regression is when you have more than one independent variable

$X_1$, $X_2$, $X_3$, $\ldots$

$$ Y = \beta_0 + \beta_1 X_1 + \ldots + \beta_p X_p + \epsilon$$

Back to the simple model. The model in linear regression is the conditional mean of $Y$ given the values in $X$ is expressed a linear function.

$$ y = f(x) = E(Y | X = x)$$

http://www.learner.org/courses/againstallodds/about/glossary.html

The goal is to estimate the coefficients (e.g. $\beta_0$ and $\beta_1$). We represent the estimates of the coefficients with a "hat" on top of the letter.

$$ \hat{\beta}_0,...
Can you write Python code for it?

from sklearn.datasets import load_boston
boston = load_boston()

boston.keys()

boston.data.shape

# Print column names
print (boston.feature_names)

# Print description of Boston housing data set
print (boston.DESCR)


I am working on the file "Mini_Project_Linear_Regression.ipynb".
The first task is:
Part 3: Training and Test Data sets
Purpose of splitting data into Training/testing sets

 Let's stick to the linear regression example:

 We built our model with the requirement that the model fit the data well.

 As a side-effect, the model will fit THIS dataset well. What about new data?

 We wanted the model for predictions, right?

 One simple solution, leave out some data (for testing) and train the model on the rest

 This also leads directly to the idea of cross-validation, next section.

One way of doing this is you can create training and testing data sets manually.
Can you write Python code for it?

X_train = X[:-50]
X_test = X[-50:]
Y_train = bos.PRICE[:-50]
Y_test = bos.PRICE[-50:]
print (X_train.shape)
print (X_test.shape)
print (Y_train.shape)
print (Y_test.shape)


I am working on the file "Mini_Project_Linear_Regression.ipynb".
The first task is:
Predict Prices
We can calculate the predicted prices ($\hat{Y}_i$) using lm.predict.
$$ \hat{Y}i = \hat{\beta}_0 + \hat{\beta}_1 X_1 + \ldots \hat{\beta}{13} X_{13} $$
Can you write Python code for it?

# first five predicted prices
lm.predict(X)[0:5]


I am working on the file "Mini_Project_Linear_Regression.ipynb".
The first task is:
Relationship between PTRATIO and housing price

Try fitting a linear regression model using only the 'PTRATIO' (pupil-teacher ratio by town)
Calculate the mean squared error.
Can you write Python code for it?

lm = LinearRegression()
lm.fit(X[['PTRATIO']], bos.PRICE)
#pd.DataFrame(zip(X.columns, lm.coef_), columns = ['features', 'estimatedCoefficients'])

print np.mean((bos.PRICE - lm.predict(X[['PTRATIO']])) ** 2)

msePTRATIO = np.mean((bos.PRICE - lm.predict(X[['PTRATIO']])) ** 2)
print msePTRATIO


lm = LinearRegression()
lm.fit(X[['PTRATIO']], bos.PRICE)

msePTRATIO = np.mean((bos.PRICE - lm.predict(X[['PTRATIO']])) ** 2)
print (msePTRATIO)


I am working on the file "Mini_Project_Linear_Regression.ipynb".
The first task is:
Residual sum of squares
Let's calculate the residual sum of squares
$$ S = \sum_{i=1}^N r_i = \sum_{i=1}^N (y_i - (\beta_0 + \beta_1 x_i))^2 $$
Can you write Python code for it?

print np.sum((bos.PRICE - lm.predict(X)) ** 2)


I am working on the file "Mini_Project_Linear_Regression.ipynb".
The first task is:
Scatter Plots using Seaborn

Seaborn is a cool Python plotting library built on top of matplotlib. It provides convenient syntax and shortcuts for many common types of plots, along with better-looking defaults.
We can also use seaborn regplot for the scatterplot above. This provides automatic linear regression fits (useful for data exploration later on). Here's one example below.
Can you write Python code for it?

sns.regplot(y="PRICE", x="RM", data=bos, fit_reg = True)


I am working on the file "Mini_Project_Linear_Regression.ipynb".
The first task is:
Scatter plots

Let's look at some scatter plots for three variables: 'CRIM', 'RM' and 'PTRATIO'.
What kind of relationship do you see? e.g. positive, negative?  linear? non-linear?
Can you write Python code for it?

plt.scatter(bos.CRIM, bos.PRICE)
plt.xlabel("Per capita crime rate by town (CRIM)")
plt.ylabel("Housing Price")
plt.title("Relationship between CRIM and Price")


I am working on the file "Mini_Project_Linear_Regression.ipynb".
The first task is:
There are no column names in the DataFrame. Let's add those.
Can you write Python code for it?

bos.columns = boston.feature_names
bos.head()


I am working on the file "Mini_Project_Linear_Regression.ipynb".
The first task is:
We can also plot the fitted linear regression line.
Can you write Python code for it?

plt.scatter(bos.PTRATIO, bos.PRICE)
plt.xlabel("Pupil-to-Teacher Ratio (PTRATIO)")
plt.ylabel("Housing Price")
plt.title("Relationship between PTRATIO and Price")

plt.plot(bos.PTRATIO, lm.predict(X[['PTRATIO']]), color='blue', linewidth=3)
plt.show()


I am working on the file "Mini_Project_Linear_Regression.ipynb".
The first task is:
What can you do with a LinearRegression object?

Check out the scikit-learn docs here. We have listed the main functions here.
Main functions | Description
--- | ---
lm.fit() | Fit a linear model
lm.predit() | Predict Y using the linear model with estimated coefficients
lm.score() | Returns the coefficient of determination (R^2). A measure of how well observed outcomes are replicated by the model, as the proportion of total variation of outcomes explained by the model
What output can you get?
Can you write Python code for it?


I am working on the file "Mini_Project_Linear_Regression.ipynb".
The first task is:
Your turn

Try fitting a linear regression model using three independent variables

'CRIM' (per capita crime rate by town)
'RM' (average number of rooms per dwelling)
'PTRATIO' (pupil-teacher ratio by town)

Calculate the mean squared error.
Can you write Python code for it?

# your turn
lm = LinearRegression()
lm.fit(X[['CRIM']], bos.PRICE)

print np.mean((bos.PRICE - lm.predict(X[['CRIM']])) ** 2)

plt.scatter(bos.CRIM, bos.PRICE)
plt.xlabel("Crime per Capita")
plt.ylabel("Housing Price")
plt.title("Relationship between CRIM and Price")

plt.plot(bos.CRIM, lm.predict(X[['CRIM']]), color='blue', linewidth=3)
plt.show()


# your turn
lm.fit(X[['CRIM']], bos.PRICE)
print ('(MSE) Per capita crime rate by town: ', np.mean((bos.PRICE - lm.predict(X[['CRIM']])) ** 2))
lm.fit(X[['RM']], bos.PRICE)
print ('(MSE) Average number of rooms per dwelling: ', np.mean((bos.PRICE - lm.predict(X[['RM']])) ** 2))
lm.fit(X[['PTRATIO']], bos.PRICE)
print ('(MSE) Pupil-teacher ratio by town: ', np.mean((bos.PRICE - lm.predict(X[['PTRATIO']])) ** 2))


I am working on the file "Mini_Project_Linear_Regression.ipynb".
The first task is:
Your turn:
Calculate the mean squared error

using just the test data
using just the training data

Are they pretty similar or very different? What does that mean?
Can you write Python code for it?

# your turn
lm = LinearRegression()
lm.fit(X_test[['RM']], Y_test)

print np.mean((Y_test - lm.predict(X_test[['RM']])) ** 2)

plt.scatter(X_test.RM, Y_test)
plt.xlabel("No. of Rooms")
plt.ylabel("Housing Price")
plt.title("Relationship between No. of Rooms and Price")

plt.plot(X_test.RM, lm.predict(X_test[['RM']]), color='blue', linewidth=3)
plt.show()