I am working on the file ".ipynb_checkpoints/pggNumAdd-checkpoint.ipynb".
The first task is:
Makes dict with keys pointing to the 10s numbers.
The value needs the list of random numbers updated.
Currently it just adds the number one.
How to add the random number list?
Can you write Python code for it?

for ronum in ranumlis:
    #print ronum
    if ronum in othgues:
        print (str(ronum) + ' You Win!')
    else:
        print (str(ronum) + ' You Lose!')

#dieci = dict()

#for ranz in range(10):
    #print str(ranz) + str(1)#
#    dieci.update({str(ranz) + str(1): str(ranz)})
#    for numz in range(10):
        #print str(ranz) + str(numz)
#        print numz
#print zetoo

#for diez in dieci:
#    print diez

#for sinum  in ranumlis:
#    print str(sinum) + (str('\n'))
    #if str(sinum) in othhigh:
    #    print 'Win'

#import os

#os.system('sudo adduser joemanz --disabled-login --quiet -D')

#uslis = os.listdir('/home/wcmckee/signinlca/usernames/')
#print ('User List: ')
#for usl in uslis:
#    print usl
#    os.system('sudo adduser ' + usl + ' ' + '--disabled-login --quiet')

#    os.system('sudo mv /home/wcmckee/signinlca/usernames/' + usl  + ' ' + '/home/' + usl + ' ')

#print dieci


I am working on the file ".ipynb_checkpoints/pggNumAdd-checkpoint.ipynb".
The first task is:
Testing of playing pyguessgame.
Generates random numbers and plays a game.
Create two random lists of numbers 0/9,10/19,20/29 etc to 100.
Compare the two lists. If win mark, if lose mark.
Debian
Can you write Python code for it?

#for ronum in ranumlis:
#    print ronum

randict = dict()

othgues = []
othlow = 0
othhigh = 9

for ranez in range(10):
      randxz = random.randint(othlow, othhigh)
      othgues.append(randxz)

      othlow = (othlow + 10)
      othhigh = (othhigh + 10)

#print othgues

tenlis = ['zero', 'ten', 'twenty', 'thirty', 'fourty',
          'fifty', 'sixty', 'seventy', 'eighty',
          'ninety']

#for telis in tenlis:
#    for diez in dieci:
#        print telis

#randict


I am working on the file ".ipynb_checkpoints/play_notebook-checkpoint.ipynb".
The first task is:
If you already trained the dataset there will be a pickle file with the trained network available. Now underneath we test in on the test-set
Can you write Python code for it?

import cpickle as pickle

#has definition accuracy, accuracy(y_target, y_predict)
%run modelSelection


I am working on the file ".ipynb_checkpoints/play_notebook-checkpoint.ipynb".
The first task is:
SVM
Visualize with (k)PCA
Here I import some code to plot the (k)PCA. I use the PCA libary from Sklearn
Can you write Python code for it?

#important to have this magic line inplace, otherwise the notebook will not plot
%matplotlib inline
#this imports the file from the folder by running all definitions from file will be in the memory of the kernel
%run PCA_visualization

kPCA_visualization2d(X, y)


I am working on the file ".ipynb_checkpoints/play_notebook-checkpoint.ipynb".
The first task is:
test using python definitons from other files
Instead of import we use %run, which works because it will store all the functions in the memory of the kernel.
As you can see, the example works.
Can you write Python code for it?

%run notebook_import_test

print_import()


I am working on the file ".ipynb_checkpoints/plot_forest_importances-checkpoint.ipynb".
The first task is:
Feature importances with forests of trees
This examples shows the use of forests of trees to evaluate the importance of
features on an artificial classification task. The red bars are the feature
importances of the forest, along with their inter-trees variability.
Inspired from sciki-tutorials.
Can you write Python code for it?

print(__doc__)

import numpy as np
import matplotlib.pyplot as plt
import pandas as pd

from sklearn.datasets import make_classification
from sklearn.ensemble import ExtraTreesClassifier


I am working on the file ".ipynb_checkpoints/plot_forest_importances-checkpoint.ipynb".
The first task is:
Read dataset with Pandas.
As previous examples, we use pandas to read datasets, and standarize the data.
Can you write Python code for it?

#I use this dataset because this has clearly separated cathegories,
#Read the database using pandas,
#Note that bad lines are omitted with error_bad_lines=False
df = pd.read_csv('https://archive.ics.uci.edu/ml/'
        'machine-learning-databases/00236/seeds_dataset.txt', header=None, sep="\t", error_bad_lines=False)

#The headers are not given in the dataset, so we give them afterwords:
#1. area A,
#2. perimeter P,
#3. compactness C = 4*pi*A/P^2,
#4. length of kernel,
#5. width of kernel,
#6. asymmetry coefficient
#7. length of kernel groove.
#8. Class: 1=Kama, 2=Rosa, 3=Canadian
column_header= ["area","perimeter","compactness","kernel-length","kernel-width",
              "asymmetry","kernel-groove-length","class"]
df.columns = column_header

#This shows the header of the database:
df.head()

#This sets class=2 to 0 and 3 to 1:
y = df.loc[:,'class']

#Extract some cathegories:
X=df.iloc[:,0:7]
#This is to convert the csv dictionary into a numpy matrix to later standarize:
X=X.as_matrix()

nfeature=X.shape[1]

# standardize features
X_std = np.copy(X)
for ifeat in range(0,nfeature):
    X_std[:,ifeat] = (X[:,ifeat] - X[:,ifeat].mean()) / X[:,ifeat].std()


I am working on the file ".ipynb_checkpoints/profile_analysis-checkpoint.ipynb".
The first task is:
Exploratory Data Analysis
Can you write Python code for it?

# examines when users first joined
entry_year = df_profile.loc[df_profile.join_year != 'NA', 'join_year']
entry = entry_year.value_counts().sort_index()
entry.plot()

plt.xlim([int(np.min(entry_year)),16])
plt.show()


I am working on the file ".ipynb_checkpoints/profile_analysis-checkpoint.ipynb".
The first task is:
Note the dip in 2002 and 2012 that coincides with the Great Fanfiction Purge of 2002 and 2012!
Can you write Python code for it?

# examines status of users
status = df_profile['status'].value_counts()
status.plot.bar()
plt.xticks(rotation=0)

plt.show()

# examines distribution of top 10 countries
country = df_profile['country'].value_counts()
country[1:10].plot.bar()
plt.xticks(rotation=45)

plt.show()

# examines distribution of stories written
st = df_profile.loc[df_profile.status != 'inactive', 'st']
st.plot.hist(range = [1, 60])

plt.show()

# examines distribution of favorited
fav = df_profile.loc[df_profile.status != 'inactive', ['fs', 'fa']]
fav.plot.hist(alpha = 0.5, range = [1, 100])

plt.show()


I am working on the file ".ipynb_checkpoints/profile_analysis-checkpoint.ipynb".
The first task is:
Regression Model
Can you write Python code for it?

# runs OLS regression
formula = 'st ~ fa + fs + cc + age'
reg = smf.ols(data=df_active, formula=formula).fit()
print(reg.summary())


I am working on the file ".ipynb_checkpoints/profile_analysis-checkpoint.ipynb".
The first task is:
Results indicate there is some correlation between two of the independent variables: 'fa' and 'fs', implying one of them may not be necessary in the model.
Nonlinearity
We know from earlier distributions that some of the variables are heavily right-skewed. We created some scatter plots to confirm that the assumption of linearity holds.
Can you write Python code for it?

sns.pairplot(data=df_active, y_vars=['st'], x_vars=['fa', 'fs', 'age'])

plt.show()


I am working on the file ".ipynb_checkpoints/profile_analysis-checkpoint.ipynb".
The first task is:
The log transformations helped increase the fit from and R-squared of ~0.05 to ~0.20.
From these results, we can see that:

A 1% change in number of authors favorited is associated with a ~15% change in the number of stories written.
A 1% change in number of stories favorited is associated with a ~4% change in the number of stories written.
Being in a community is associated with a ~0.7 increase in the number of stories written.
One more year on the site is associated with a ~3% change in the number of stories written.

We noted earlier that 'fa' and 'fs' had a correlation of ~0.7. As such, we reran the regression without 'fa' first, then again without 'fs'. The model without 'fs' yielded a better fit (R-squared), as well as AIC and BIC.
Can you write Python code for it?

# runs OLS regression
formula = 'st ~ fa + cc + age'
reg = smf.ols(data=df_active, formula=formula).fit()
print(reg.summary())


I am working on the file ".ipynb_checkpoints/profile_analysis-checkpoint.ipynb".
The first task is:
Without 'fs', we lost some information but not much:

A 1% change in number of authors favorited is associated with a ~20% change in the number of stories written.
Being in a community is associated with a ~0.7 increase in the number of stories written.
One more year on the site is associated with a ~3% change in the number of stories written.

All these results seem to confirm a basic intuition that the more active an user reads (as measured by favoriting authors and stories), the likely it is that user will write more stories. Being longer on the site and being part of a community is also correlated to publications.
To get a sense of the actual magnitude of these effects, let's attempt some plots:
Can you write Python code for it?

def graph(formula, x_range):
    y = np.array(x_range)
    x = formula(y)
    plt.plot(y,x)

graph(lambda x : (np.exp(reg.params[0]+reg.params[1]*(np.log(x-1)))-1),
      range(2,100,1))
graph(lambda x : (np.exp(reg.params[0]+reg.params[1]*(np.log(x-1))+reg.params[2])-1),
      range(2,100,1))

plt.show()

ages = [0, 1, 5, 10, 15]
for age in ages:
    graph(lambda x : (np.exp(reg.params[0]+reg.params[1]*(np.log(x-1))+reg.params[3]*age)-1),
          range(2,100,1))

plt.show()


I am working on the file ".ipynb_checkpoints/project-checkpoint.ipynb".
The first task is:
After several tests we found in almost all cases that training a function on every possibility is much more effective than randomly generating data.
Can you write Python code for it?

data_inorder = pd.read_csv('Data\\adder_inorder_data.csv')
data_inorder = data_inorder[['Steps', 'MSE']]
data_inorder = data_inorder.sort_values(['Steps'])
data_inorder.head(9)

data_rnd_0 = pd.read_csv('Data\\adder_random_0_data.csv')
data_rnd_0 = data_rnd_0[['Steps', 'MSE']]
data_rnd_0 = data_rnd_0.sort_values(['Steps'])

data_rnd_1 = pd.read_csv('Data\\adder_random_1_data.csv')
data_rnd_1 = data_rnd_1[['Steps', 'MSE']]
data_rnd_1 = data_rnd_1.sort_values(['Steps'])

data_rnd_2 = pd.read_csv('Data\\adder_random_2_data.csv')
data_rnd_2 = data_rnd_2[['Steps', 'MSE']]
data_rnd_2 = data_rnd_2.sort_values(['Steps'])

data_rnd_3 = pd.read_csv('Data\\adder_random_3_data.csv')
data_rnd_3 = data_rnd_3[['Steps', 'MSE']]
data_rnd_3 = data_rnd_3.sort_values(['Steps'])

data_rnd_4 = pd.read_csv('Data\\adder_random_4_data.csv')
data_rnd_4 = data_rnd_4[['Steps', 'MSE']]
data_rnd_4 = data_rnd_4.sort_values(['Steps'])

plt.plot(data_inorder['Steps'].ix[:20], data_inorder['MSE'].ix[:20], 'bo',
         data_rnd_0['Steps'].ix[:20], data_rnd_0['MSE'].ix[:20],
        data_rnd_1['Steps'].ix[:20], data_rnd_1['MSE'].ix[:20],
        data_rnd_2['Steps'].ix[:20], data_rnd_2['MSE'].ix[:20],
        data_rnd_3['Steps'].ix[:20], data_rnd_3['MSE'].ix[:20],
        data_rnd_4['Steps'].ix[:20], data_rnd_4['MSE'].ix[:20])
plt.show()

plt.plot(data_inorder['Steps'].ix[30:], data_inorder['MSE'].ix[30:], 'bo',
         data_rnd_0['Steps'].ix[30:], data_rnd_0['MSE'].ix[30:],
        data_rnd_1['Steps'].ix[30:], data_rnd_1['MSE'].ix[30:],
        data_rnd_2['Steps'].ix[30:], data_rnd_2['MSE'].ix[30:],
        data_rnd_3['Steps'].ix[30:], data_rnd_3['MSE'].ix[30:],
        data_rnd_4['Steps'].ix[30:], data_rnd_4['MSE'].ix[30:])
plt.show()

plt.plot(data_rnd_1['Steps'].ix[30:], data_rnd_1['MSE'].ix[30:],
        data_rnd_2['Steps'].ix[30:], data_rnd_2['MSE'].ix[30:],
        data_rnd_4['Steps'].ix[30:], data_rnd_4['MSE'].ix[30:])
plt.show()


I am working on the file ".ipynb_checkpoints/project-checkpoint.ipynb".
The first task is:
adder(n): Adds 42 to n
Process:
* Tried different methods of generating data to find which worked the best
    * All possible values 0-100
    * 1000 random values 0-100
    * All possible values 0-100 10 times for a total of 1000 datum
    * All possible values 0-100 100 times for a total of 10000 datum
* Experimented with single layer hidden units 1-20
* Experimented with two layer hidden units [1-20, 1-20]
* Found how well each different neural net extrapolated to other values
* Tried to scale data up to learn 1-1000
Results:

        Data
        MSE

        100 data: 1-100
        .065861

        1000 datum: random(100)
        .028475

        1000 datum: 1-100 10 times
        .007759

        10000 datum: 1-100 100 times
        409.116

The results seemed to show that iterating through every possibility multiple times and then training on that is the best method of data gathering. As with many things in this, you have to find the fine line of having the right amount of data without having too much.
I was interested in how far a neural net could extrapolate to numbers that it had never seen before. Although this should have been an easy problem because it is linear, it wasn't because SkFlow doesn't allow you to choose the activation function for your regressor. Most all of the single and double layer neural nets I trained failed around 200-300, however there was one that stood out and was able to correctly predict 1-1145 with just training on 1-100
Can you write Python code for it?

data_inorder = pd.read_csv('Data\\adder_inorder_data.csv')
data_inorder = data_inorder[['Steps', 'MSE']]
data_inorder = data_inorder.sort_values(['Steps'])

arr = np.zeros(5)
arr[0] = 5

arr = ['100', '200', '300', '400', '500', '600', '700',
       '1000','1100','1200','1300', '1400','1500', '1600','1700','1800', '1900',
      '2000', '2100', '2300', '2400', '2500']
df_arr = []
for i in range(len(arr)):
    temp = pd.read_csv('Data\\determinant_' + arr[i] +'_layer_by_100.csv', header=None)
    temp = temp.T
    temp.columns=['Second', 'MSE']
    temp['First'] = arr[i]
    temp = temp.sort_values(['First', 'Second'])
    df_arr.append(temp)
len(df_arr)

temp = pd.read_csv('Data\\determinant_layer_by_100.csv', header=None)
    temp = temp.T
    temp.columns=['Second', 'MSE']

frames = [df_arr[0], df_arr[1], df_arr[2], df_arr[3], df_arr[4], df_arr[5],
          df_arr[6], df_arr[7], df_arr[8], df_arr[9], df_arr[10], df_arr[11],
          df_arr[12], df_arr[13], df_arr[14], df_arr[15], df_arr[16], df_arr[17],
          df_arr[18], df_arr[19], df_arr[20], df_arr[21]]
result = pd.concat(frames)
result = result.reset_index(drop=True)
result.sort_values(['MSE'])

res1 = result.as_matrix(columns=['First'])
res2 = result.as_matrix(columns=['Second'])
res3 = result.as_matrix(columns=['MSE'])

fig = plt.figure()
ax = fig.gca(projection='3d')
ax.plot_trisurf(res1[:,0], res2[:,0], res3[:,0], cmap=cm.jet, linewidth=0.2)
plt.show()

min = 30
for i in range(len(arr)):
    plt.plot(df_arr[i]['Second'], df_arr[i]['MSE'])
plt.show()

num = 5
df_arr[5].as_matrix(columns=['MSE']))
df_arr[5].head(21)

for i in range(len(arr)):
    plt.plot(df_arr[i]['Second'], df_arr[i]['MSE'])
plt.ylim(.3, .5)
plt.show()

plt.plot(df_arr[0]['Second'], df_arr[0]['MSE'])

from mpl_toolkits.mplot3d import Axes3D
from matplotlib import cm
import matplotlib.pyplot as plt
import numpy as np

n_angles = 36
n_radii = 8

# An array of radii
# Does not include radius r=0, this is to eliminate duplicate points
radii = np.linspace(0.125, 1.0, n_radii)

# An...


I am working on the file ".ipynb_checkpoints/prune-X-checkpoint.ipynb".
The first task is:
After pickling the sensor to power model, use it to predict power from satellite data!
Can you write Python code for it?

from sklearn.externals import joblib
etr2 = joblib.load('data/sensor-to-power-model/sensor-to-power-model.pkl')

etr2.predict(Y).shape #power predictions

with np.load('data/y.npz') as data:
    y = data['y']

y_pred2 = etr2.predict(Y_pred) #the predicted

y_pred2.shape

y.shape


I am working on the file ".ipynb_checkpoints/prune-X-checkpoint.ipynb".
The first task is:
Not too good. Or at least not an improvement over itself even.
I may have to do some feature engineering for the images.  Averaging, filtering or both. I wonder if I do some ratios of channels?  Sort of like the "greenness" parameter that farmers use.  Also, maybe +/- 3 hours is too long for searching for a satellite image. (Reduce to +/- 1 hour?)
Can you write Python code for it?


I am working on the file ".ipynb_checkpoints/prune-X-checkpoint.ipynb".
The first task is:
Seems like dimensions are working out.  Probably will need to pickle the sat to sensor model too and just feed in data start to finish and see how it does.
Can you write Python code for it?

with np.load('data/good_times.npz') as data:
    good_times = data['good_times']

print good_times.shape
print etr2.predict(Y).shape


I am working on the file ".ipynb_checkpoints/prune-X-checkpoint.ipynb".
The first task is:
So, from the plot above, there are values above 0, even though my Y_train and Y_test are all below zero.
Can you write Python code for it?

val = randint(0,2301)
_ = plt.hist(X_reduced[:,0:1972][val], alpha = 0.5, normed=True, bins=25, label='visible',range=(0,25000)) #visible light histogram
_ = plt.hist(X_reduced[:,1973:2476][val], alpha = 0.5, normed=True, bins=25, label='IR',range=(0,25000))
plt.legend(loc='upper right');

X_hist = []
for i in xrange(X_reduced.shape[0]):
    hist1, _ = np.histogram(X_reduced[:,0:1972][i], density=True, bins=25, range=(0,25000)) #vis
    hist2, _ = np.histogram(X_reduced[:,1973:2476][i], density=True, bins=25, range=(0,25000)) #IR
    X_hist.append(np.hstack((hist1,hist2)))
X_hist = np.array(X_hist)


I am working on the file ".ipynb_checkpoints/prune-X-checkpoint.ipynb".
The first task is:
The score has improved (0.2) but the most remarkable is that all the predicted values are now below 0, as they should be.
Let's try something Extra Trees!
Can you write Python code for it?

X_hist = []
for i in xrange(X_reduced.shape[0]):
    hist1, _ = np.histogram(X_reduced[:,0:1972][i], density=True, bins=30, range=(0,25000)) #vis
    hist2, _ = np.histogram(X_reduced[:,1973:2476][i], density=True, bins=30, range=(0,25000)) #IR
    X_hist.append(np.hstack((hist1,hist2)))
X_hist = np.array(X_hist)

from sklearn.ensemble import ExtraTreesRegressor
etr = ExtraTreesRegressor(oob_score=True, bootstrap=True, n_jobs=-1, n_estimators=100) #njobs uses all cores!
X_train, X_test, Y_train, Y_test = train_test_split(X_hist, Y_reduced, test_size = 0.3, random_state = 14)

etr.fit(X_train,Y_train)

print etr.score(X_test,Y_test)
print etr.oob_score_

from random import randint
val = randint(0,Y_test.shape[0])
print Y_pred[val]
print Y_test[val]

plt.plot(Y_pred,'go');


I am working on the file ".ipynb_checkpoints/prune-X-checkpoint.ipynb".
The first task is:
Tres beau, call it a day!  That's a pretty graph!
Can you write Python code for it?


I am working on the file ".ipynb_checkpoints/prune-X-checkpoint.ipynb".
The first task is:
Try it all again with just X, not X_reduced?
Can you write Python code for it?

from scipy.ndimage import zoom
from __future__ import division
from random import randint

X_ratio_1_2 = []
for i in xrange(X.shape[0]):
    CH1 = zoom(X[:,0:1972][i].reshape((29,68)),zoom=(0.48, 0.53), order=5)
    CH2 = X[:,1972:2476][i].reshape((14,36))
    X_ratio_1_2.append(25000* (CH2)  / (CH1 + CH2+1.0) )
X_ratio_1_2 = np.array(X_ratio_1_2)

X_ratio_1_6 = []
for i in xrange(X.shape[0]):
    CH1 = zoom(X[:,0:1972][i].reshape((29,68)),zoom=(0.48, 0.53), order=5)
    CH6 = X[:,3484:3988][i].reshape((14,36))
    X_ratio_1_6.append(25000* CH6 / (CH1 + CH6 + 0.1) )
X_ratio_1_6 = np.array(X_ratio_1_6)

X_ratio_2_6 = []
for i in xrange(X.shape[0]):
    CH2 = X[:,1972:2476][i].reshape((14,36))
    CH6 = X[:,3484:3988][i].reshape((14,36))
    X_ratio_2_6.append(25000* CH6 / (CH2 + CH6 + 0.1) )
X_ratio_2_6 = np.array(X_ratio_2_6)

val = randint(0,2301)
_ = plt.hist(X[:,0:1972][val], alpha = 0.2, normed=True, bins=30, label='CH1',range=(0,25000)) #visible light histogram
_ = plt.hist(X[:,1973:2476][val], alpha = 0.2, normed=True, bins=30, label='CH2',range=(0,25000))
_ = plt.hist(X[:,2477:2980][val], alpha = 0.2, normed=True, bins=30, label='CH3',range=(0,25000))
_ = plt.hist(X[:,2981:3484][val], alpha = 0.2, normed=True, bins=30, label='CH4',range=(0,25000))
_ = plt.hist(X[:,3484:3988][val], alpha = 0.2, normed=True, bins=30, label='CH6',range=(0,25000))
_ = plt.hist(np.ravel(X_ratio_1_2[val]), alpha = 0.2, normed=True, bins=30, label='CH2/CH1',range=(0,25000))
_ = plt.hist(np.ravel(X_ratio_1_6[val]), alpha = 0.2, normed=True, bins=30, label='CH6/CH1',range=(0,25000))
_ = plt.hist(np.ravel(X_ratio_2_6[val]), alpha = 0.2, normed=True, bins=30, label='CH6/CH2',range=(0,25000))
plt.legend(loc='upper right');

import pandas as pd

X_hist = []
bins = 25
for i in xrange(X.shape[0]):
    myval1 = pd.DataFrame(np.ravel(X_ratio_1_2[i])).fillna(np.mean).values.flatten();
    myval2 = pd.DataFrame(np.ravel(X_ratio_1_6[i])).fillna(np.mean).values.flatten();
    myval3 =...


I am working on the file ".ipynb_checkpoints/prune-X-checkpoint.ipynb".
The first task is:
Wow, even better, but is it good enough?  If I include ratios between channels, spatial and other information will be implicitly collected.
Downsample CH1 and divide by CH6:
Can you write Python code for it?

from scipy.ndimage import zoom
from __future__ import division

X_reduced_ratio_1_2 = []
for i in xrange(X_reduced.shape[0]):
    CH1 = zoom(X_reduced[:,0:1972][i].reshape((29,68)),zoom=(0.48, 0.53), order=5)
    CH2 = X_reduced[:,1972:2476][i].reshape((14,36))
    X_reduced_ratio_1_2.append(25000* CH2  / (CH1 + CH2) )
X_reduced_ratio_1_2 = np.array(X_reduced_ratio_1_2)

X_reduced_ratio_1_6 = []
for i in xrange(X_reduced.shape[0]):
    CH1 = zoom(X_reduced[:,0:1972][i].reshape((29,68)),zoom=(0.48, 0.53), order=5)
    CH6 = X_reduced[:,3484:3988][i].reshape((14,36))
    X_reduced_ratio_1_6.append(25000* CH6 / (CH1 + CH6) )
X_reduced_ratio_1_6 = np.array(X_reduced_ratio_1_6)

X_reduced_ratio_2_6 = []
for i in xrange(X_reduced.shape[0]):
    CH2 = X_reduced[:,1972:2476][i].reshape((14,36))
    CH6 = X_reduced[:,3484:3988][i].reshape((14,36))
    X_reduced_ratio_2_6.append(25000* CH6 / (CH2 + CH6) )
X_reduced_ratio_2_6 = np.array(X_reduced_ratio_2_6)

val = randint(0,2301)
_ = plt.hist(X_reduced[:,0:1972][val], alpha = 0.2, normed=True, bins=30, label='CH1',range=(0,25000)) #visible light histogram
_ = plt.hist(X_reduced[:,1973:2476][val], alpha = 0.2, normed=True, bins=30, label='CH2',range=(0,25000))
_ = plt.hist(X_reduced[:,2477:2980][val], alpha = 0.2, normed=True, bins=30, label='CH3',range=(0,25000))
_ = plt.hist(X_reduced[:,2981:3484][val], alpha = 0.2, normed=True, bins=30, label='CH4',range=(0,25000))
_ = plt.hist(X_reduced[:,3484:3988][val], alpha = 0.2, normed=True, bins=30, label='CH6',range=(0,25000))
_ = plt.hist(np.ravel(X_reduced_ratio_1_2[val]), alpha = 0.2, normed=True, bins=30, label='CH2/CH1',range=(0,25000))
_ = plt.hist(np.ravel(X_reduced_ratio_1_6[val]), alpha = 0.2, normed=True, bins=30, label='CH6/CH1',range=(0,25000))
_ = plt.hist(np.ravel(X_reduced_ratio_2_6[val]), alpha = 0.2, normed=True, bins=30, label='CH6/CH2',range=(0,25000))
plt.legend(loc='upper right');

X_hist = []
bins = 30
for i in xrange(X_reduced.shape[0]):
    hist1, _ = np.histogram(X_reduced[:,0:1972][i], density=True,...


I am working on the file ".ipynb_checkpoints/pycharts-checkpoint.ipynb".
The first task is:
Example 4
Can you write Python code for it?

data = dict(
    labels=['City ABCD','City EFGHM','City OPQRSTUV'],
    values=[200,100,500]
)
out = StdCharts.VBar(data=data,values_sorted=True,color="#446699",fill="#999999")
HTML(out)


I am working on the file ".ipynb_checkpoints/pycharts-checkpoint.ipynb".
The first task is:
Vertical Bar Charts

Ideal for a small number of labels or a portion of time dependent values
Fixed width

Example 3: defaults
Can you write Python code for it?


I am working on the file ".ipynb_checkpoints/pycharts-checkpoint.ipynb".
The first task is:
You can also save your chart with the save method:
Can you write Python code for it?


I am working on the file ".ipynb_checkpoints/pyplearnr_test_code-checkpoint.ipynb".
The first task is:
Binary scaling fed into a KNN classifier appears to have the best training score.
KNN with custom min-max and binary scaling settings
MinMaxScaler scales each feature value to between 0 and 1 by default. Different scaling ranges can be gridded over by setting the 'scaler__feature_range' keyword argument in param_dist.
Binarizer sets each value to 0 or 1 depending on a threshold. The default for pyplearnr is 0.5. This can be changed by setting 'scaler__threshold' using param_dist.
Here is an example of setting both:
Can you write Python code for it?

%%time

reload(ppl)

estimator = 'knn'

scaling_options = ['min_max','binary']

param_dists = {
    'min_max': {
        'scaler__feature_range': [(1,2),(3,4)]
    },
    'binary': {
        'scaler__threshold': np.arange(0,1,0.1)
    }
}

for scaling_option in scaling_options:
    model_name = 'custom_%s_%s'%(scaling_option,estimator_name)

    optimized_pipeline_kwargs = {
        'scale_type': scaling_option
    }

    # Initialize pipeline
    optimized_pipeline = ppl.OptimizedPipeline(estimator,**optimized_pipeline_kwargs)

    # Set pipeline fitting parameters
    fit_kwargs = {
        'random_state': 6,
        'use_default_param_dist': True,
        'suppress_output': True,
        'param_dist': param_dists[scaling_option]

    }

    # Fit data
    optimized_pipeline.fit(X,y,**fit_kwargs)

    # Save
    optimized_pipelines[model_name] = optimized_pipeline

# Visualize results
pipeline_keys = optimized_pipelines.keys()

test_scores = [optimized_pipelines[key].test_score_ for key in pipeline_keys]

ax = pd.Series(test_scores,index=pipeline_keys).sort_values().plot(kind='barh',color='black')


I am working on the file ".ipynb_checkpoints/pyplearnr_test_code-checkpoint.ipynb".
The first task is:
Feature selection and KNN did had a mid-level training score:
Can you write Python code for it?


I am working on the file ".ipynb_checkpoints/pyplearnr_test_code-checkpoint.ipynb".
The first task is:
KNN using feature interactions
Feature products of different degrees can be used as additional features by setting the 'feature_interaction' OptimizedPipeline keyword argument to True:
Can you write Python code for it?

%%time

reload(ppl)

estimator = 'knn'

model_name = 'interaction_%s'%(estimator_name)

optimized_pipeline_kwargs = {
    'feature_interactions': True
}

# Initialize pipeline
optimized_pipeline = ppl.OptimizedPipeline(estimator,**optimized_pipeline_kwargs)

# Set pipeline fitting parameters
fit_kwargs = {
    'random_state': 6,
    'use_default_param_dist': True,
    'suppress_output': True
}

# Fit data
optimized_pipeline.fit(X,y,**fit_kwargs)

# Save
optimized_pipelines[model_name] = optimized_pipeline

# Visualize results
pipeline_keys = optimized_pipelines.keys()

test_scores = [optimized_pipelines[key].test_score_ for key in pipeline_keys]

ax = pd.Series(test_scores,index=pipeline_keys).sort_values().plot(kind='barh',color='black')

print optimized_pipelines['interaction_knn']


I am working on the file ".ipynb_checkpoints/pyplearnr_test_code-checkpoint.ipynb".
The first task is:
Logistic regression, random forest, multilayer perceptron, and adaboost outperform KNN, even with all of the attempted pre-processing so far.
Putting it all together
Different combinations of these options can be strung together simultaneously to iterate over multiple models:
Can you write Python code for it?

%%time

import itertools

estimators = ['knn','logistic_regression','svm',
              'multilayer_perceptron','random_forest','adaboost']

feature_interaction_options = [True,False]
feature_selection_options = [None,'select_k_best']
scaling_options = [None,'standard','normal','min_max','binary']
transformations = [None,'pca']

pipeline_steps = [feature_interaction_options,feature_selection_options,scaling_options,
                  transformations,estimators]

pipeline_options = list(itertools.product(*pipeline_steps))

optimized_pipelines = {}

for pipeline_step_combo in pipeline_options:
    model_name = []

    feature_interactions = pipeline_step_combo[0]

    if feature_interactions:
        model_name.append('interactions')

    feature_selection_type = pipeline_step_combo[1]

    if feature_selection_type:
        model_name.append('select')

    scale_type = pipeline_step_combo[2]

    if scale_type:
        model_name.append(scale_type)

    transform_type = pipeline_step_combo[3]

    if transform_type:
        model_name.append(transform_type)

    estimator = pipeline_step_combo[4]

    model_name.append(estimator)

    model_name = '_'.join(model_name)

    print model_name

    # Set pipeline keyword arguments
    optimized_pipeline_kwargs = {
        'feature_selection_type': feature_selection_type,
        'scale_type': scale_type,
        'feature_interactions': feature_interactions,
        'transform_type': transform_type
        }

    # Initialize pipeline
    optimized_pipeline = ppl.OptimizedPipeline(estimator,**optimized_pipeline_kwargs)

    # Set pipeline fitting parameters
    fit_kwargs = {
        'cv': 10,
        'num_parameter_combos': None,
        'n_jobs': -1,
        'random_state': None,
        'suppress_output': True,
        'use_default_param_dist': True,
        'param_dist': None,
        'test_size': 0.2 # 20% saved as test set
    }

    # Fit data
    optimized_pipeline.fit(X,y,**fit_kwargs)

    #...


I am working on the file ".ipynb_checkpoints/pyplearnr_test_code-checkpoint.ipynb".
The first task is:
Note how the estimator__weights parameter isn't set for the KNN estimator.
KNN with scaling
The currently supported scaling options are standard, normal, min-max, and binary using scikit-learn's StandardScaler, Normalizer, MinMaxScaler, and Binarizer, respectively. These are set by the pipeline initialization kwarg 'scale_type' like this:
Can you write Python code for it?

%%time

estimator = 'knn'

scaling_options = ['standard','normal','min-max','binary']

for scaling_option in scaling_options:
    model_name = '%s_%s'%(scaling_option,estimator_name)

    optimized_pipeline_kwargs = {
        'scale_type': scaling_option
    }

    # Initialize pipeline
    optimized_pipeline = ppl.OptimizedPipeline(estimator,**optimized_pipeline_kwargs)

    # Set pipeline fitting parameters
    fit_kwargs = {
        'random_state': 6,
        'use_default_param_dist': True,
        'suppress_output': True

    }

    # Fit data
    optimized_pipeline.fit(X,y,**fit_kwargs)

    # Save
    optimized_pipelines[model_name] = optimized_pipeline


I am working on the file ".ipynb_checkpoints/pyplearnr_test_code-checkpoint.ipynb".
The first task is:
Note the default OptimizedPipeline parameters and those for its fit() method.
The OptimizedPipeline class contains all of the data associated with the nested stratified k-folds cross-validation.
After use of the fit() method, this includes the data, its test/train splits (based on the test_size percentage keyword argument), the GridSearchCV or RandomizedGridSearchCV object, the Pipeline object that has been retrained using all of the data with the best parameters, test/train scores, and validation metrics/reports.
A report can be printed immediately after the fit by setting the suppress_output keyword argument to True.
Printing the OptimizedPipeline instance also shows the report:
Can you write Python code for it?


I am working on the file ".ipynb_checkpoints/pyplearnr_test_code-checkpoint.ipynb".
The first task is:
Null model
Can you write Python code for it?


I am working on the file ".ipynb_checkpoints/pyplearnr_test_code-checkpoint.ipynb".
The first task is:
One-hot encode categorical features
Can you write Python code for it?

simulation_df = pd.get_dummies(simulation_df,drop_first=True)

simulation_df.info()


I am working on the file ".ipynb_checkpoints/pyplearnr_test_code-checkpoint.ipynb".
The first task is:
Out of 240 different possible pipelines, best pipeline, with a test score of 0.899, appears to be min-max scaling between 0 and 1 funneled into a PCA and then into a multilayer perceptron with one hidden layer of size 5.
It took roughly 3 hours to find it.
Predicting survival with the optimal model
All one has to do to make a prediction is use the .predict method of the pipeline in the .pipeline field.
Here's an example of predicting whether I would survive on the Titanic. I'm 32, would probably have one family member with me, might be Pclass1 (I'd hope), male, have a Ph.D (if that's what they mean by Dr.). I'm using the median Fare for Pclass 1 and randomly chose a city to have embarked from:
Can you write Python code for it?

personal_stats = [32,1,0,df[df['Pclass']==1]['Fare'].median(),0,0,1,1,0,1,0,0,0,0,0,0]

zip(personal_stats,X.columns)

optimized_pipelines['min_max_pca_multilayer_perceptron'].pipeline.predict(personal_stats)


I am working on the file ".ipynb_checkpoints/pyplearnr_test_code-checkpoint.ipynb".
The first task is:
Regression
Can you write Python code for it?

%%time
%autoreload

import numpy as np
import pyplearnr as ppl

from sklearn.preprocessing import StandardScaler
from sklearn.feature_selection import SelectKBest

reload(ppl)

data = pd.read_csv('Advertising.csv',index_col=0)

# Start with all features
feature_cols = ['TV','Radio','Newspaper']

# Split data
X = data[feature_cols]
y = data.Sales

kfcv = ppl.NestedKFoldCrossValidation(outer_loop_fold_count=5,
                                      inner_loop_fold_count=3)

pipeline_bundle_schematic = [
    {'scaler': {
        'none': {},
        'standard': {}
    }},
    {'pre_estimator': {
            'polynomial_features': {
                'degree': range(1,5)
            }
        }},
    {'estimator': {
            'linear_regression': {},
    }}
]

pipelines = ppl.PipelineBuilder().build_pipeline_bundle(pipeline_bundle_schematic)

print 'Number of pipelines: %d'%(len(pipelines)), '\n'

kfcv.fit(X.values, y.values, pipelines, scoring_metric='rmse')

kfcv.fit(X.values, y.values, pipelines, scoring_metric='rmse', best_outer_fold_pipeline=1)

%autoreload
kfcv.print_report()

%autoreload
kfcv.print_report()

%%time
%autoreload
import itertools

estimators = ['knn','logistic_regression','svm',
              'multilayer_perceptron','random_forest','adaboost']

feature_interaction_options = [True,False]
feature_selection_options = [None,'select_k_best']
scaling_options = [None,'standard','normal','min_max','binary']
transformations = [None,'pca']

pipeline_steps = [feature_interaction_options,feature_selection_options,scaling_options,
                  transformations,estimators]

pipeline_options = list(itertools.product(*pipeline_steps))

optimized_pipelines = {}

for pipeline_step_combo in pipeline_options:
    model_name = []

    feature_interactions = pipeline_step_combo[0]

    if feature_interactions:
        model_name.append('interactions')

    feature_selection_type = pipeline_step_combo[1]

    if feature_selection_type:
        model_name.append('select')

   ...


I am working on the file ".ipynb_checkpoints/pyplearnr_test_code-checkpoint.ipynb".
The first task is:
Switching the range for min_max scaling boosted it to rank 1 for pipeline training scores:
Can you write Python code for it?


I am working on the file ".ipynb_checkpoints/pyplearnr_test_code-checkpoint.ipynb".
The first task is:
The best parameter combination, of those attempted by RandomizedSearchCV, was 12 nearest neighbors with the 'uniform' weight.
Other models
This code currently supports K-nearest neighbors, logistic regression, support vector machines, multilayer perceptrons, random forest, and adaboost:
Can you write Python code for it?

%%time

classifiers = ['knn','logistic_regression','svm',
               'multilayer_perceptron','random_forest','adaboost']

for estimator in classifiers:
    # Set pipeline keyword arguments
    optimized_pipeline_kwargs = {}

    # Initialize pipeline
    optimized_pipeline = ppl.OptimizedPipeline(estimator,**optimized_pipeline_kwargs)

    # Set pipeline fitting parameters
    fit_kwargs = {
        'random_state': 6,
        'suppress_output': True,
        'use_default_param_dist': True
    }

    # Fit data
    optimized_pipeline.fit(X,y,**fit_kwargs)

    # Save
    optimized_pipelines[estimator] = optimized_pipeline

# Visualize results
pipeline_keys = optimized_pipelines.keys()

test_scores = [optimized_pipelines[key].test_score_ for key in pipeline_keys]

ax = pd.Series(test_scores,index=pipeline_keys).sort_values().plot(kind='barh',color='black')


I am working on the file ".ipynb_checkpoints/pyplearnr_test_code-checkpoint.ipynb".
The first task is:
The optimal number of interactions (number of features multiplied by each other at once) was found to be 1.
KNN using custom number of feature interactions
The 'feature_interactions__degree' dictates the number of interactions. The default setting is to try no interactions (degree 1) and 2 interactions. Setting this in param_dist allows custom numbers:
Can you write Python code for it?

%%time

reload(ppl)

estimator = 'knn'

model_name = 'custom_interaction_%s'%(estimator_name)

optimized_pipeline_kwargs = {
    'feature_interactions': True
}

# Initialize pipeline
optimized_pipeline = ppl.OptimizedPipeline(estimator,**optimized_pipeline_kwargs)

param_dist = {
    'feature_interactions__degree': [2,3,4]
}

# Set pipeline fitting parameters
fit_kwargs = {
    'random_state': 6,
    'use_default_param_dist': True,
    'suppress_output': True,
    'param_dist': param_dist
}

# Fit data
optimized_pipeline.fit(X,y,**fit_kwargs)

# Save
optimized_pipelines[model_name] = optimized_pipeline

# Visualize results
pipeline_keys = optimized_pipelines.keys()

test_scores = [optimized_pipelines[key].test_score_ for key in pipeline_keys]

ax = pd.Series(test_scores,index=pipeline_keys).sort_values().plot(kind='barh',color='black')

print optimized_pipelines['custom_interaction_knn']


I am working on the file ".ipynb_checkpoints/pyplearnr_test_code-checkpoint.ipynb".
The first task is:
The range of 1 to 2 for the MinMaxScaler appeared to be the best.
KNN with feature selection using SelectKBest with f_classif
Currently only one form of feature selection, SelectKBest with f_classif, is supported. This is set using the 'feature_selection_type' keyword argument.
Can you write Python code for it?

%%time

reload(ppl)

estimator = 'knn'

model_name = 'select_%s'%(estimator_name)

optimized_pipeline_kwargs = {
    'feature_selection_type': 'select_k_best'
}

# Initialize pipeline
optimized_pipeline = ppl.OptimizedPipeline(estimator,**optimized_pipeline_kwargs)

# Set pipeline fitting parameters
fit_kwargs = {
    'random_state': 6,
    'use_default_param_dist': True,
    'suppress_output': True
}

# Fit data
optimized_pipeline.fit(X,y,**fit_kwargs)

# Save
optimized_pipelines[model_name] = optimized_pipeline

# Visualize results
pipeline_keys = optimized_pipelines.keys()

test_scores = [optimized_pipelines[key].test_score_ for key in pipeline_keys]

ax = pd.Series(test_scores,index=pipeline_keys).sort_values().plot(kind='barh',color='black')


I am working on the file ".ipynb_checkpoints/pyplearnr_test_code-checkpoint.ipynb".
The first task is:
Thus, null accuracy of ~62% if always predict death.
Import pyplearnr and initialize optimized pipeline collection
Can you write Python code for it?

%matplotlib inline

%load_ext autoreload

import sys

import os

sys.path.append("./pyplearnr")

optimized_pipelines = {}

%%time
%autoreload

import numpy as np
import pyplearnr as ppl

reload(ppl)

kfcv = ppl.NestedKFoldCrossValidation(outer_loop_fold_count=3,
                                      inner_loop_fold_count=3)

pipeline_schematic = [
    {'scaler': {
            'none': {},
            'standard': {},
            'min_max': {},
            'normal': {}
        }},
    {'estimator': {
            'knn': {
                'n_neighbors': range(1,31),
                'weights': ['uniform','distance']
    }}}
]

pipelines = ppl.PipelineBuilder().build_pipeline_bundle(pipeline_schematic)

print 'Number of pipelines: %d'%(len(pipelines)), '\n'

kfcv.fit(X.values, y.values, pipelines, scoring_metric='auc')

kfcv.fit(X.values, y.values, pipelines,
			 best_inner_fold_pipeline_inds = {0:59})

kfcv.fit(X.values, y.values, pipelines, best_outer_fold_pipeline=59)

%autoreload
kfcv.plot_best_pipeline_scores(number_size=10,markersize=8, figsize=(9,3), box_line_thickness=1)

%autoreload
kfcv.plot_contest(color_by='scaler', markersize=3)

%autoreload
kfcv.fit(X.values, y.values, pipelines,
			 best_inner_fold_pipeline_inds = {1:6})

kfcv.fit(X.values, y.values, pipelines, best_outer_fold_pipeline=8)

%autoreload

%matplotlib inline

kfcv.plot_best_pipeline_scores(number_size=18, markersize=14)

%autoreload

%matplotlib inline

kfcv.plot_contest(number_size=8, markersize=7, all_folds=True, figsize=(10,40),
                  color_by='scaler', box_line_thickness=2)

kfcv.pipelines[29]

# cmap = pylab.cm.viridis

# print cmap.__doc__

worst_pipelines = [85, 67, 65, 84, 69, 83]

for pipeline_ind in worst_pipelines:
    print pipeline_ind, kfcv.pipelines[pipeline_ind]

print '\n'

worst_pipelines = [86, 75, 84, 79, 85, 83]

for pipeline_ind in worst_pipelines:
    print pipeline_ind, kfcv.pipelines[pipeline_ind]

print '\n'

worst_pipelines = [77, 61, 81, 83, 74, 82, 84]

for pipeline_ind in...


I am working on the file ".ipynb_checkpoints/pyqt5_qsetting_note-checkpoint.ipynb".
The first task is:
最简单的用法
Can you write Python code for it?

myOrganization = 'MyOrg'  # 组织名称
myApplication = 'MyApp'  # 应用名称
settings = QSettings(myOrganization, myApplication)
settings.setValue("editor/wrapMargin", 68)
print (settings.value("editor/wrapMargin"))
print (settings.value("editor/someth"))

# 如果在程序中多处用到 QSettings ，那么可以一次性定义，然后统一使用
from PyQt5.QtCore import QCoreApplication
QCoreApplication.setOrganizationName("MyOrg2nd")
QCoreApplication.setOrganizationDomain("MyOrg2nd.com")
QCoreApplication.setApplicationName("MyApp2nd")
settings_2nd = QSettings()
print (settings.applicationName())
print (settings_2nd.applicationName())
print (settings_2nd.organizationName())

# 测试键是否存在
settings.contains("editor")
settings.contains("editor/wrapMargin")

settings.remove("editor") #删除一个键，键不存在也不会报错
settings.contains("editor/wrapMargin")

settings.clear()  #清空所有键（只是清空自己添加的，系统固有的还在）
settings.allKeys()  #所有键

settings.fileName() #储存位置

settings_ini = QSettings("conf.ini", QSettings.IniFormat) #使用 ini 文件来储存
settings_ini.fileName()


I am working on the file ".ipynb_checkpoints/python-data-files-checkpoint.ipynb".
The first task is:
CSV to Dictionary
Can you write Python code for it?

# Read CSV into dictionary data structure
with open('data/fileops/game-scores.csv', 'rb') as df:
    dict_rdr = csv.DictReader(df)
    for row in dict_rdr:
        print('\t'.join([row['Game'], row['Team A'], row['Team B']]))
    print('\t'.join(row.keys()))


I am working on the file ".ipynb_checkpoints/python-data-files-checkpoint.ipynb".
The first task is:
CSV to List
Can you write Python code for it?

# The rb flag opens file for reading
with open('data/fileops/vehicles.csv', 'rb') as csv_file:
    rdr = csv.reader(csv_file, delimiter=',', quotechar='"')
    for row in rdr:
        print '\t'.join(row)


I am working on the file ".ipynb_checkpoints/python-data-files-checkpoint.ipynb".
The first task is:
DataFrame to Excel
Can you write Python code for it?

# DataFrame to XLSX Excel file
df.to_excel('data/fileops/sales.xlsx', index=False)

print(check_output(["ls", "data/fileops"]).decode("utf8"))


I am working on the file ".ipynb_checkpoints/python-data-files-checkpoint.ipynb".
The first task is:
Dictionary to CSV
Can you write Python code for it?

# Dictionary data structures can be used to represent rows
game1_scores = {'Game':'Quarter', 'Team A': 45, 'Team B': 90}
game2_scores = {'Game':'Semi', 'Team A': 80, 'Team B': 32}
game3_scores = {'Game':'Final', 'Team A': 70, 'Team B': 68}

headers = ['Game', 'Team A', 'Team B']

# Create CSV from dictionaries
with open('data/fileops/game-scores.csv', 'wb') as df:
    dict_wtr = csv.DictWriter(df, fieldnames=headers)
    dict_wtr.writeheader()
    dict_wtr.writerow(game1_scores)
    dict_wtr.writerow(game2_scores)
    dict_wtr.writerow(game3_scores)

print(check_output(["ls", "data/fileops"]).decode("utf8"))


I am working on the file ".ipynb_checkpoints/python-data-files-checkpoint.ipynb".
The first task is:
Excel to DataFrame
Can you write Python code for it?

# Excel to DataFrame
df3 = pd.read_excel('data/fileops/sales.xlsx')

df3


I am working on the file ".ipynb_checkpoints/python-data-files-checkpoint.ipynb".
The first task is:
Pandas for CSV file operations
Pandas goal is to become the most powerful and flexible open source data analysis / manipulation tool available in any language. Pandas includes file operations capabilities for CSV, among other formats.
CSV operations in Pandas are much faster than in native Python.
DataFrame to CSV
Can you write Python code for it?

import pandas as pd

# Create a DataFrame
df = pd.DataFrame({
        'Name' : ['Josh', 'Eli', 'Ram', 'Bil'],
        'Sales' : [34.32, 12.1, 4.77, 31.63],
        'Region' : ['North', 'South', 'West', 'East'],
        'Product' : ['PC', 'Phone', 'SW', 'Cloud']})
df

# DataFrame to CSV
df.to_csv('data/fileops/sales.csv', index=False)

print(check_output(["ls", "data/fileops"]).decode("utf8"))


I am working on the file ".ipynb_checkpoints/python-tutorial-checkpoint.ipynb".
The first task is:
Array indexing
Numpy offers several ways to index into arrays.
Slicing: Similar to Python lists, numpy arrays can be sliced. Since arrays may be multidimensional, you must specify a slice for each dimension of the array:
Can you write Python code for it?

import numpy as np

# Create the following rank 2 array with shape (3, 4)
# [[ 1  2  3  4]
#  [ 5  6  7  8]
#  [ 9 10 11 12]]
a = np.array([[1,2,3,4], [5,6,7,8], [9,10,11,12]])

# Use slicing to pull out the subarray consisting of the first 2 rows
# and columns 1 and 2; b is the following array of shape (2, 2):
# [[2 3]
#  [6 7]]
b = a[:2, 1:3]
print (b)


I am working on the file ".ipynb_checkpoints/python-tutorial-checkpoint.ipynb".
The first task is:
Arrays
A numpy array is a grid of values, all of the same type, and is indexed by a tuple of nonnegative integers. The number of dimensions is the rank of the array; the shape of an array is a tuple of integers giving the size of the array along each dimension.
We can initialize numpy arrays from nested Python lists, and access elements using square brackets:
Can you write Python code for it?

a = np.array([1, 2, 3])  # Create a rank 1 array
print (type(a), a.shape, a[0], a[1], a[2])
a[0] = 5                 # Change an element of the array
print (a)

b = np.array([[1,2,3],[4,5,6]])   # Create a rank 2 array
print (b)

print (b.shape)
print (b[0, 0], b[0, 1], b[1, 0])


I am working on the file ".ipynb_checkpoints/python-tutorial-checkpoint.ipynb".
The first task is:
As usual, you can find all the gory details about lists in the documentation.
Slicing
In addition to accessing list elements one at a time, Python provides concise syntax to access sublists; this is known as slicing:
Can you write Python code for it?

# nums = range(5)    # range is a built-in function that creates a list of integers
nums = [2,3,5,1,2,8]
print (nums)         # Prints "[0, 1, 2, 3, 4]"
print (nums[2:4])    # Get a slice from index 2 to 4 (exclusive); prints "[2, 3]"
print (nums[2:])     # Get a slice from index 2 to the end; prints "[2, 3, 4]"
print (nums[:2])     # Get a slice from the start to index 2 (exclusive); prints "[0, 1]"
print (nums[:])      # Get a slice of the whole list; prints ["0, 1, 2, 3, 4]"
print (nums[:-2])    # Slice indices can be negative; prints ["0, 1, 2, 3]"


I am working on the file ".ipynb_checkpoints/python-tutorial-checkpoint.ipynb".
The first task is:
By running this special iPython command, we will be displaying plots inline:
Can you write Python code for it?


I am working on the file ".ipynb_checkpoints/python-tutorial-checkpoint.ipynb".
The first task is:
Classes
The syntax for defining classes in Python is straightforward:
Can you write Python code for it?

class Greeter:

    # Constructor
    def __init__(self, name):
        self.name = name  # Create an instance variable

    # Instance method
    def greet(self, loud=False):
        if loud:
            print ('HELLO, %s!' % self.name.upper())
        else:
            print ('Hello, %s' % self.name)

g = Greeter('Fred')  # Construct an instance of the Greeter class
g.greet()            # Call an instance method; prints "Hello, Fred"
g.greet(loud=True)   # Call an instance method; prints "HELLO, FRED!"


I am working on the file ".ipynb_checkpoints/python-tutorial-checkpoint.ipynb".
The first task is:
Dictionaries
A dictionary stores (key, value) pairs, similar to a Map in Java or an object in Javascript. You can use it like this:
Can you write Python code for it?

d = {'cat': 'cute', 'dog': 'furry'}  # Create a new dictionary with some data
print (d['cat'])       # Get an entry from a dictionary; prints "cute"
print ('cat' in d)     # Check if a dictionary has a given key; prints "True"

d['fish'] = 'wet'    # Set an entry in a dictionary
print (d['fish'])      # Prints "wet"

print (d['monkey'])  # KeyError: 'monkey' not a key of d

print (d.get('monkey', 'N/A'))  # Get an element with a default; prints "N/A"
print (d.get('fish', 'N/A'))    # Get an element with a default; prints "wet"

del d['fish']        # Remove an element from a dictionary
print (d.get('fish', 'N/A')) # "fish" is no longer a key; prints "N/A"


I am working on the file ".ipynb_checkpoints/python-tutorial-checkpoint.ipynb".
The first task is:
Dictionary comprehensions: These are similar to list comprehensions, but allow you to easily construct dictionaries. For example:
Can you write Python code for it?

nums = [0, 1, 2, 3, 4]
even_num_to_square = {x: x ** 2 for x in nums if x % 2 == 0}
print (even_num_to_square)


I am working on the file ".ipynb_checkpoints/python-tutorial-checkpoint.ipynb".
The first task is:
For brevity we have left out a lot of details about numpy array indexing; if you want to know more you should read the documentation.
Datatypes
Every numpy array is a grid of elements of the same type. Numpy provides a large set of numeric datatypes that you can use to construct arrays. Numpy tries to guess a datatype when you create an array, but functions that construct arrays usually also include an optional argument to explicitly specify the datatype. Here is an example:
Can you write Python code for it?

x = np.array([1, 2])  # Let numpy choose the datatype
y = np.array([1.0, 2.0])  # Let numpy choose the datatype
z = np.array([1, 2], dtype=np.int64)  # Force a particular datatype

print (x.dtype, y.dtype, z.dtype)


I am working on the file ".ipynb_checkpoints/python-tutorial-checkpoint.ipynb".
The first task is:
Functions
Python functions are defined using the def keyword. For example:
Can you write Python code for it?

def sign(x):
    if x > 0:
        return 'positive'
    elif x < 0:
        return 'negative'
    else:
        return 'zero'

for x in [-1, 0, 1]:
    print (sign(x))


I am working on the file ".ipynb_checkpoints/python-tutorial-checkpoint.ipynb".
The first task is:
If you want access to the index of each element within the body of a loop, use the built-in enumerate function:
Can you write Python code for it?

animals = ['cat', 'dog', 'monkey']
for idx, animal in enumerate(animals):
    print ('#%d: %s' % (idx + 1, animal))


I am working on the file ".ipynb_checkpoints/python-tutorial-checkpoint.ipynb".
The first task is:
Images
Can you write Python code for it?

from IPython.display import Image
Image(filename='lectures/images/01_02.png', width=500)

Image(filename='lectures/images/01_01.png', width=500)


I am working on the file ".ipynb_checkpoints/python-tutorial-checkpoint.ipynb".
The first task is:
KNN Classifier
Can you write Python code for it?

# read X and y
# cols = ['pclass','sex','age','fare']
cols = ['pclass','sex','age']
X = dframe[cols]
y = dframe[["survived"]]

dframe.head()

# Use scikit-learn KNN classifier to predit survival probability
from sklearn.neighbors import KNeighborsClassifier
neigh = KNeighborsClassifier(n_neighbors=3)
neigh.fit(X, y)

# check accuracy
neigh.score(X,y)

# define a passenger
passenger = [1,1,29]

# predict survial label
print(neigh.predict([passenger]))

# predict survial probability
print(neigh.predict_proba([passenger]))

# find k-nearest neighbors
neigh.kneighbors(passenger,3)

# Let's create some data for DiCaprio and Winslet and you
import numpy as np
colsidx = [0,2,3];
dicaprio = np.array([3, 'Jack Dawson', 0, 19, 0, 0, 'N/A', 5.0000])
winslet = np.array([1, 'Rose DeWitt Bukater', 1, 17, 1, 2, 'N/A', 100.0000])
you = np.array([1, 'user', 1, 24, 0, 2, 'N/A', 50.0000])
# Preprocess data
dicaprio = dicaprio[colsidx]
winslet = winslet[colsidx]
you = you[colsidx]
# # Predict surviving chances (class 1 results)
pred = neigh.predict([dicaprio, winslet, you])
prob = neigh.predict_proba([dicaprio, winslet, you])
print("DiCaprio Surviving:", pred[0], " with probability", prob[0])
print("Winslet Surviving Rate:", pred[1], " with probability", prob[2])
print("user Surviving Rate:", pred[2], " with probability", prob[2])


I am working on the file ".ipynb_checkpoints/python-tutorial-checkpoint.ipynb".
The first task is:
List comprehensions can also contain conditions:
Can you write Python code for it?

nums = [0, 1, 2, 3, 4]
even_squares = [x ** 2 for x in nums if x % 2 == 0]
print (even_squares)


I am working on the file ".ipynb_checkpoints/python-tutorial-checkpoint.ipynb".
The first task is:
Loops
You can loop over the elements of a list like this:
Can you write Python code for it?

animals = ['cat', 'dog', 'monkey']
for animal in animals:
    print (animal)
    print(1)

x =1
print(x)


I am working on the file ".ipynb_checkpoints/python-tutorial-checkpoint.ipynb".
The first task is:
Loops: Iterating over a set has the same syntax as iterating over a list; however since sets are unordered, you cannot make assumptions about the order in which you visit the elements of the set:
Can you write Python code for it?

animals = {'cat', 'dog', 'fish'}
for idx, animal in enumerate(animals):
    print ('#%d: %s' % (idx + 1, animal))
# Prints "#1: fish", "#2: dog", "#3: cat"


I am working on the file ".ipynb_checkpoints/python-tutorial-checkpoint.ipynb".
The first task is:
Numpy
Numpy is the core library for scientific computing in Python. It provides a high-performance multidimensional array object, and tools for working with these arrays. If you are already familiar with MATLAB, you might find this tutorial useful to get started with Numpy.
To use Numpy, we first need to import the numpy package:
Can you write Python code for it?


I am working on the file ".ipynb_checkpoints/python-tutorial-checkpoint.ipynb".
The first task is:
Numpy also provides many functions to create arrays:
Can you write Python code for it?

a = np.zeros((2,2))  # Create an array of all zeros
print (a)

b = np.ones((1,2))   # Create an array of all ones
print (b)

c = np.full((2,2), 7) # Create a constant array
print (c)

d = np.eye(2)        # Create a 2x2 identity matrix
print (d)

e = np.random.random((2,2)) # Create an array filled with random values
print (e)


I am working on the file ".ipynb_checkpoints/python-tutorial-checkpoint.ipynb".
The first task is:
Numpy broadcasting allows us to perform this computation without actually creating multiple copies of v. Consider this version, using broadcasting:
Can you write Python code for it?

import numpy as np

# We will add the vector v to each row of the matrix x,
# storing the result in the matrix y
x = np.array([[1,2,3], [4,5,6], [7,8,9], [10, 11, 12]])
v = np.array([1, 0, 1])
y = x + v  # Add v to each row of x using broadcasting
print (y)


I am working on the file ".ipynb_checkpoints/python-tutorial-checkpoint.ipynb".
The first task is:
Numpy provides many useful functions for performing computations on arrays; one of the most useful is sum:
Can you write Python code for it?

x = np.array([[1,2],[3,4]])

print np.sum(x)  # Compute sum of all elements; prints "10"
print np.sum(x, axis=0)  # Compute sum of each column; prints "[4 6]"
print np.sum(x, axis=1)  # Compute sum of each row; prints "[3 7]"


I am working on the file ".ipynb_checkpoints/python-tutorial-checkpoint.ipynb".
The first task is:
One useful trick with integer array indexing is selecting or mutating one element from each row of a matrix:
Can you write Python code for it?

# Create a new array from which we will select elements
a = np.array([[1,2,3], [4,5,6], [7,8,9], [10, 11, 12]])
print (a)

# Create an array of indices
b = np.array([0, 2, 0, 1])

# Select one element from each row of a using the indices in b
print (a[np.arange(4), b])  # Prints "[ 1  6  7 11]"

# Mutate one element from each row of a using the indices in b
a[np.arange(4), b] += 10
print (a)


I am working on the file ".ipynb_checkpoints/python-tutorial-checkpoint.ipynb".
The first task is:
Python versions
There are currently two different supported versions of Python, 2.7 and 3.6. Somewhat confusingly, Python 3.X introduced many backwards-incompatible changes to the language, so code written for 2.7 may not work under 3.6 and vice versa. For this class all code will use Python 2.7.
You can check your Python version at the command line by running python --version.
Basic data types
Numbers
Integers and floats work as you would expect from other languages:
Can you write Python code for it?

x,y = 3,4
print (x,y)

# type of variable
print(type(x))

print (x + 1)   # Addition;
print (x - 1)   # Subtraction;
print (x * 2)   # Multiplication;
print (x ** 2)  # Exponentiation;

x += 1
print (x)  # Prints "4"
x *= 2
print (x)  # Prints "8"

y = 2.5
print (type(y)) # Prints "<type 'float'>"
print (y, y + 1, y * 2, y ** 2) # Prints "2.5 3.5 5.0 6.25"


I am working on the file ".ipynb_checkpoints/python-tutorial-checkpoint.ipynb".
The first task is:
Sets
A set is an unordered collection of distinct elements. As a simple example, consider the following:
Can you write Python code for it?

animals = {'cat', 'dog'}
print ('cat' in animals)   # Check if an element is in a set; prints "True"
print ('fish' in animals)  # prints "False"

animals.add('fish')      # Add an element to a set
print ('fish' in animals)
print (animals)       # Number of elements in a set;

animals.add('cat')       # Adding an element that is already in the set does nothing
print (animals)
animals.remove('cat')    # Remove an element from a set
print (animals)


I am working on the file ".ipynb_checkpoints/python-tutorial-checkpoint.ipynb".
The first task is:
String objects have a bunch of useful methods; for example:
Can you write Python code for it?

s = "hello"
print (s.capitalize())  # Capitalize a string; prints "Hello"
print (s.upper())       # Convert a string to uppercase; prints "HELLO"
print (s.rjust(7))      # Right-justify a string, padding with spaces; prints "  hello"
print (s.center(7))     # Center a string, padding with spaces; prints " hello "
print (s.replace('l', '(ell)'))  # Replace all instances of one substring with another;
                               # prints "he(ell)(ell)o"
print ('  world '.strip())  # Strip leading and trailing whitespace; prints "world"


I am working on the file ".ipynb_checkpoints/python-tutorial-checkpoint.ipynb".
The first task is:
Subplots
You can plot different things in the same figure using the subplot function. Here is an example:
Can you write Python code for it?

# Compute the x and y coordinates for points on sine and cosine curves
x = np.arange(0, 3 * np.pi, 0.1)
y_sin = np.sin(x)
y_cos = np.cos(x)

# Set up a subplot grid that has height 2 and width 1,
# and set the first such subplot as active.
plt.subplot(2, 1, 1)

# Make the first plot
plt.plot(x, y_sin)
plt.title('Sine')

# Set the second subplot as active, and make the second plot.
plt.subplot(2, 1, 2)
plt.plot(x, y_cos)
plt.title('Cosine')

# Show the figure.
plt.show()


I am working on the file ".ipynb_checkpoints/python-tutorial-checkpoint.ipynb".
The first task is:
The line y = x + v works even though x has shape (4, 3) and v has shape (3,) due to broadcasting; this line works as if v actually had shape (4, 3), where each row was a copy of v, and the sum was performed elementwise.
Broadcasting two arrays together follows these rules:

If the arrays do not have the same rank, prepend the shape of the lower rank array with 1s until both shapes have the same length.
The two arrays are said to be compatible in a dimension if they have the same size in the dimension, or if one of the arrays has size 1 in that dimension.
The arrays can be broadcast together if they are compatible in all dimensions.
After broadcasting, each array behaves as if it had shape equal to the elementwise maximum of shapes of the two input arrays.
In any dimension where one array had size 1 and the other array had size greater than 1, the first array behaves as if it were copied along that dimension

If this explanation does not make sense, try reading the explanation from the documentation or this explanation.
Functions that support broadcasting are known as universal functions. You can find the list of all universal functions in the documentation.
Here are some applications of broadcasting:
Can you write Python code for it?

# Compute outer product of vectors
v = np.array([1,2,3])  # v has shape (3,)
w = np.array([4,5])    # w has shape (2,)
# To compute an outer product, we first reshape v to be a column
# vector of shape (3, 1); we can then broadcast it against w to yield
# an output of shape (3, 2), which is the outer product of v and w:

print (np.reshape(v, (3, 1)) * w)

# Add a vector to each row of a matrix
x = np.array([[1,2,3], [4,5,6]])
# x has shape (2, 3) and v has shape (3,) so they broadcast to (2, 3),
# giving the following matrix:

print (x + v)

# Add a vector to each column of a matrix
# x has shape (2, 3) and w has shape (2,).
# If we transpose x then it has shape (3, 2) and can be broadcast
# against w to yield a result of shape (3, 2); transposing this result
# yields the final result of shape (2, 3) which is the matrix x with
# the vector w added to each column. Gives the following matrix:

print ((x.T + w).T)

# Another solution is to reshape w to be a row vector of shape (2, 1);
# we can then broadcast it directly against x to produce the same
# output.
print x + np.reshape(w, (2, 1))

# Multiply a matrix by a constant:
# x has shape (2, 3). Numpy treats scalars as arrays of shape ();
# these can be broadcast together to shape (2, 3), producing the
# following array:
print x * 2


I am working on the file ".ipynb_checkpoints/python-tutorial-checkpoint.ipynb".
The first task is:
This works; however when the matrix x is very large, computing an explicit loop in Python could be slow. Note that adding the vector v to each row of the matrix x is equivalent to forming a matrix vv by stacking multiple copies of v vertically, then performing elementwise summation of x and vv. We could implement this approach like this:
Can you write Python code for it?

vv = np.tile(v, (4, 1))  # Stack 4 copies of v on top of each other
print (vv)                 # Prints "[[1 0 1]
                         #          [1 0 1]
                         #          [1 0 1]
                         #          [1 0 1]]"

y = x + vv  # Add x and vv elementwise
print (y)


I am working on the file ".ipynb_checkpoints/python-tutorial-checkpoint.ipynb".
The first task is:
Two ways of accessing the data in the middle row of the array.
Mixing integer indexing with slices yields an array of lower rank,
while using only slices yields an array of the same rank as the
original array:
Can you write Python code for it?

row_r1 = a[1, :]    # Rank 1 view of the second row of a
row_r2 = a[1:2, :]  # Rank 2 view of the second row of a
row_r3 = a[[1], :]  # Rank 2 view of the second row of a
print (row_r1, row_r1.shape)
print (row_r2, row_r2.shape)
print (row_r3, row_r3.shape)

# We can make the same distinction when accessing columns of an array:
col_r1 = a[:, 1]
col_r2 = a[:, 1:2]
print (col_r1, col_r1.shape)
print (col_r2, col_r2.shape)


I am working on the file ".ipynb_checkpoints/python-tutorial-checkpoint.ipynb".
The first task is:
With just a little bit of extra work we can easily plot multiple lines at once, and add a title, legend, and axis labels:
Can you write Python code for it?

y_cos = np.cos(x)

# Plot the points using matplotlib
plt.plot(x, y)
plt.plot(x, y_cos)
plt.xlabel('x axis label')
plt.ylabel('y axis label')
plt.title('Sine and Cosine')
plt.legend(['Sine', 'Cosine'])


I am working on the file ".ipynb_checkpoints/python-tutorial-checkpoint.ipynb".
The first task is:
You can also mix integer indexing with slice indexing. However, doing so will yield an array of lower rank than the original array. Note that this is quite different from the way that MATLAB handles array slicing:
Can you write Python code for it?

# Create the following rank 2 array with shape (3, 4)
a = np.array([[1,2,3,4], [5,6,7,8], [9,10,11,12]])
print (a)
print(a.shape)


I am working on the file ".ipynb_checkpoints/python-tutorial-checkpoint.ipynb".
The first task is:
You can find a list of all string methods in the documentation.
Containers
Python includes several built-in container types: lists, dictionaries, sets, and tuples.
Lists
A list is the Python equivalent of an array, but is resizeable and can contain elements of different types:
Can you write Python code for it?

xs = [3, 1, 2]   # Create a list
print (xs, xs[2])
print (xs[-1])     # Negative indices count from the end of the list; prints "2"

ys = [[1,2,3],[2,3,4]]
print(ys)
print(ys[1][2])

xs[2] = 'foo'    # Lists can contain elements of different types
print (xs)

xs.append('bar') # Add a new element to the end of the list
print (xs)

x = xs.pop()     # Remove and return the last element of the list
print (x, xs)


I am working on the file ".ipynb_checkpoints/python-tutorial-checkpoint.ipynb".
The first task is:
You can find all you need to know about dictionaries in the documentation.
It is easy to iterate over the keys in a dictionary:
Can you write Python code for it?

d = {'person': 2, 'cat': 4, 'spider': 8}
for animal in d:
    legs = d[animal]
    print ('A %s has %d legs' % (animal, legs))


I am working on the file ".ipynb_checkpoints/python-tutorial-checkpoint.ipynb".
The first task is:
You can find the full list of mathematical functions provided by numpy in the documentation.
Apart from computing mathematical functions using arrays, we frequently need to reshape or otherwise manipulate data in arrays. The simplest example of this type of operation is transposing a matrix; to transpose a matrix, simply use the T attribute of an array object:
Can you write Python code for it?

print x
print x.T

v = np.array([[1,2,3]])
print v
print v.T


I am working on the file ".ipynb_checkpoints/python-tutorial-checkpoint.ipynb".
The first task is:
You can make this code simpler using a list comprehension:
Can you write Python code for it?

nums = [0, 1, 2, 3, 4]
squares = [x ** 2 for x in nums]
print (squares)


I am working on the file ".ipynb_checkpoints/python-tutorial-checkpoint.ipynb".
The first task is:
You can read all about numpy datatypes in the documentation.
Array math
Basic mathematical functions operate elementwise on arrays, and are available both as operator overloads and as functions in the numpy module:
Can you write Python code for it?

x = np.array([[1,2],[3,4]], dtype=np.float64)
y = np.array([[5,6],[7,8]], dtype=np.float64)

# Elementwise sum; both produce the array
print (x + y)
print (np.add(x, y))

# Elementwise difference; both produce the array
print x - y
print np.subtract(x, y)

# Elementwise product; both produce the array
print x * y
print np.multiply(x, y)

# Elementwise division; both produce the array
# [[ 0.2         0.33333333]
#  [ 0.42857143  0.5       ]]
print x / y
print np.divide(x, y)

# Elementwise square root; produces the array
# [[ 1.          1.41421356]
#  [ 1.73205081  2.        ]]
print np.sqrt(x)


I am working on the file ".ipynb_checkpoints/python_data_analysis-checkpoint.ipynb".
The first task is:
Add another level to the index
Can you write Python code for it?

ufo.set_index('City', inplace=True, append=True) # Adds to existing index
ufo.sort_index(inplace=True)
ufo.head(25)

ufo.loc[['ND', 'WY'],:] # Select all records from ND AND WY

ufo.loc['ND':'WY',:]    # Select all records from ND THROUGH WY

ufo.loc[('ND', 'Bismarck'),:]  # Select all records from Bismark, ND

ufo.loc[('ND', 'Bismarck'):('ND','Casselton'),:] # Select all records from Bismark, ND through Casselton, ND

ufo.reset_index(level='City', inplace=True) # Remove the City from the index
ufo.head()

ufo.reset_index(inplace=True)               # Remove all columns from the index

ufo.head()


I am working on the file ".ipynb_checkpoints/python_data_analysis-checkpoint.ipynb".
The first task is:
Analyzing Across Time
Can you write Python code for it?

# Reset the index
ufo.dtypes

# Convert Time column to date-time format (defined in Pandas)
# Reference: https://docs.python.org/2/library/time.html#time.strftime
ufo['Time'] = pd.to_datetime(ufo['Time'], format="%m/%d/%Y %H:%M")
ufo.dtypes

# Compute date range
ufo.Time.min()

ufo.Time.max()

# Slice using time
ufo[ufo.Time > pd.datetime(1995, 1, 1)]  # Slice using the time

ufo[(ufo.Time > pd.datetime(1995, 1, 1)) & (ufo.State =='TX')]  # Works with other logical conditions, as expected

# Set the index to time
ufo.set_index('Time', inplace=True)
ufo.sort_index(inplace=True)
ufo.head()

# Access particular times/ranges
ufo.loc['1995',:]
ufo.loc['1995-01',:]
ufo.loc['1995-01-01',:]

# Access range of times/ranges
ufo.loc['1995':,:]
ufo.loc['1995':'1996',:]
ufo.loc['1995-12-01':'1996-01',:]

# Access elements of the timestamp
# Reference: http://pandas.pydata.org/pandas-docs/stable/timeseries.html#time-date-components
ufo.index.year
ufo.index.month
ufo.index.weekday
ufo.index.day
ufo.index.time
ufo.index.hour

# Create a new variable with time element
ufo['Year'] = ufo.index.year
ufo['Month'] = ufo.index.month
ufo['Day'] = ufo.index.day
ufo['Weekday'] = ufo.index.weekday
ufo['Hour'] = ufo.index.hour


I am working on the file ".ipynb_checkpoints/python_data_analysis-checkpoint.ipynb".
The first task is:
Filtering and Sorting Data
Can you write Python code for it?

# Select a single column
ufo['State']

ufo.State                       # This is equivalent

# Select multiple columns
ufo[['State', 'City','Shape Reported']]

my_cols = ['State', 'City', 'Shape Reported']
ufo[my_cols]                    # This is equivalent

# Logical filtering
ufo[ufo.State == 'TX']          # Select only rows where State == 'TX'

ufo[~(ufo.State == 'TX')]       # Select everything where the test fails

ufo[ufo.State != 'TX']          # Same thing as before

ufo.City[ufo.State == 'TX']     # Select only city columm where State == 'TX'

ufo[ufo.State == 'TX'].City     # Same thing as before

ufo[(ufo.State == 'CA') | (ufo.State =='TX')] # Select only records where State is 'CA' or State is 'TX'

ufo_dallas = ufo[(ufo.City == 'Dallas') & (ufo.State =='TX')]  # Select only Dallas, TX records

ufo[ufo.City.isin(['Austin','Dallas', 'Houston'])] # Select only Austin, Dallas, or Houston records


I am working on the file ".ipynb_checkpoints/python_data_analysis-checkpoint.ipynb".
The first task is:
Indexing and Slicing Data
Create a new index
Can you write Python code for it?

ufo.set_index('State', inplace=True)
ufo.index

ufo.index.is_unique

ufo.sort_index(inplace=True)
ufo.head(25)


I am working on the file ".ipynb_checkpoints/python_data_analysis-checkpoint.ipynb".
The first task is:
Other Useful Features
Detect duplicate rows
Can you write Python code for it?

ufo.duplicated()                                # Series of logicals

ufo.duplicated().sum()                          # count of duplicates

ufo[ufo.duplicated(['State','Time'])]           # only show duplicates

ufo[ufo.duplicated()==False]                    # only show unique rows

ufo_unique = ufo[~ufo.duplicated()]             # only show unique rows

ufo.duplicated(['State','Time']).sum()          # columns for identifying duplicates


I am working on the file ".ipynb_checkpoints/python_data_analysis-checkpoint.ipynb".
The first task is:
Pivot rows to columns
Can you write Python code for it?

ufo.groupby(['Weekday','Hour']).City.count()

ufo.groupby(['Weekday','Hour']).City.count().unstack(0) # Make first row level a column

ufo.groupby(['Weekday','Hour']).City.count().unstack(1) # Make second row level a column
# Note: .stack() transforms columns to rows


I am working on the file ".ipynb_checkpoints/python_data_analysis-checkpoint.ipynb".
The first task is:
Randomly sample a DataFrame
Can you write Python code for it?

idxs = np.random.rand(len(ufo)) < 0.66   # create a Series of booleans
train = ufo[idxs]                        # will contain about 66% of the rows
test = ufo[~idxs]                        # will contain the remaining rows


I am working on the file ".ipynb_checkpoints/python_data_analysis-checkpoint.ipynb".
The first task is:
Replace all instances of a value
Can you write Python code for it?

ufo.Shape.replace('DELTA', 'TRIANGLE')   # replace values in a Series

ufo.replace('PYRAMID', 'TRIANGLE')       # replace values throughout a DataFrame


I am working on the file ".ipynb_checkpoints/python_data_analysis-checkpoint.ipynb".
The first task is:
Solutions
Can you write Python code for it?

# Read drinks.csv (in the drinks_data folder) into a DataFrame called 'drinks'
drinks = pd.read_csv('drinks_data/drinks.csv')

# Print the first 10 rows
drinks.head(10)

# Examine the data types of all columns
drinks.dtypes
drinks.info()

# Print the 'beer_servings' Series
drinks.beer_servings
drinks['beer_servings']

# Calculate the average 'beer_servings' for the entire dataset
drinks.describe()                   # Mean is provided in the summary from describe()
drinks.beer_servings.mean()         # Alternatively, calculate the mean directly

# Print all columns, but only show rows where the country is in Europe
drinks[drinks.continent=='EU']

# Calculate the average 'beer_servings' for all of Europe (hint: use the .mean() function)
drinks[drinks.continent=='EU'].beer_servings.mean()

# Only show European countries with 'wine_servings' greater than 300
drinks[(drinks.continent=='EU') & (drinks.wine_servings > 300)]

# Determine which 10 countries have the highest 'total_litres_of_pure_alcohol'
drinks.sort_index(by='total_litres_of_pure_alcohol').tail(10)

# Determine which country has the highest value for 'beer_servings' (hint: use the .max() function)
drinks[drinks.beer_servings==drinks.beer_servings.max()].country

drinks[['country', 'beer_servings']].sort_index(by='beer_servings', ascending=False).head(1) # This is equivalent

# Count the number of occurrences of each 'continent' value and see if it looks correct
drinks.continent.value_counts()

# Determine which countries do not have continent designations
drinks[drinks.continent.isnull()].country

# Due to "na_filter = True" default within pd.read_csv()
help(pd.read_csv)


I am working on the file ".ipynb_checkpoints/python_data_analysis-checkpoint.ipynb".
The first task is:
Sorting
Can you write Python code for it?

ufo.State.order()                               # only works for a Series

ufo.sort_index(inplace=True)                    # sort rows by label

ufo.sort_index(ascending=False, inplace=False)

ufo.sort_index(by='State')                      # sort rows by specific column

ufo.sort_index(by=['State', 'Shape Reported'])  # sort by multiple columns

ufo.sort_index(by=['State', 'Shape Reported'], ascending=[False, True], inplace=True)  # specify sort order


I am working on the file ".ipynb_checkpoints/python_data_analysis-checkpoint.ipynb".
The first task is:
Summarize the data that was just read in
Can you write Python code for it?

ufo.head(10)          # Look at the top 10 observations

ufo.tail()            # Bottom x observations (defaults to 5)

ufo.describe()        # get summary statistics for columns

ufo.index             # "the index" (aka "the labels")

ufo.columns           # column names (which is "an index")

ufo.dtypes            # data types of each column

ufo.values            # underlying numpy array

ufo.info()            # concise summary


I am working on the file ".ipynb_checkpoints/python_data_analysis-checkpoint.ipynb".
The first task is:
Writing Data
Can you write Python code for it?

ufo.to_csv('ufo_new.csv')

ufo.to_csv('ufo_new.csv', index=False)  # Index is not included in the csv


I am working on the file ".ipynb_checkpoints/python_data_analysis-checkpoint.ipynb".
The first task is:
iloc: filter rows by POSITION, and select columns by POSITION
Can you write Python code for it?

ufo.iloc[0,:]                       # row with 0th position (first row)

ufo.iloc[0:3,:]                     # rows with positions 0 through 2 (not 3)

ufo.iloc[0:3, 0:3]                  # rows and columns with positions 0 through 2

ufo.iloc[:, 0:3]                    # all rows, columns with positions 0 through 2

ufo.iloc[[0,2], [0,1]]              # 1st and 3rd row, 1st and 2nd column


I am working on the file ".ipynb_checkpoints/python_data_analysis-checkpoint.ipynb".
The first task is:
loc: filter rows by LABEL, and select columns by LABEL
Can you write Python code for it?

ufo.loc['FL',:]                                     # row with label FL`

ufo.loc[:'FL',:]                                    # rows with labels  through'FL'

ufo.loc['FL':'HI', 'City':'Shape']         # rows FL, columns 'City' through 'Shape Reported'

ufo.loc[:, 'City':'Shape']                 # all rows, columns 'City' through 'Shape Reported'

ufo.loc[['FL', 'TX'], ['City','Shape']]    # rows FL and TX, columns 'City' and 'Shape Reported'


I am working on the file ".ipynb_checkpoints/scraping_form_submissions-checkpoint.ipynb".
The first task is:
It's magic, I swear!
But then...
Sometimes requests.get just isn't enough. Why? It mostly has to do with JavaScript or complicated forms - when a site reacts and changes without loading a new page, you can't use requests for that (think "Load more" buttons on Instagram).
For those sites you need Selenium! Selenium = you put your browser on autopilot. As in, literally, it takes control over your browser. There are "headless" versions that use invisible browsers but if you don't like to install a bunch of stuff, the normal version is usually fine.
Installing Selenium
Selenium isn't just a Python package, but you'll need to install python bindings in order to have Python talk to Selenium.
pip install selenium
You'll also need the Firefox browser, since that's the browser we're going to be controlling.
Selenium is built on WebDrivers, which are libraries that let you... drive a browser. I believe it comes with a Firefox WebDriver, whereas Safari/Chrome/etc take a little more effort to set up.
Using Selenium
Can you write Python code for it?

# Imports, of course

# Initialize a Firefox webdriver

# Grab the web page

# You'll use selenium.webdriver.support.ui.Select
# that we imported above to grab the Seelct element called
# t_web_lookup__license_type_name, then select Acupuncturists

# We use .find_element_by_name here because we know the name

# We use .find_element_by_id here because we know the id

# Then we'll fake typing into it

# Now we can grab the search button and click it

# Instead of using requests.get, we just look at .page_source of the driver

# We can feed that into Beautiful Soup

# It's a tricky table, but this grabs the linked names inside of the A
#rows = doc.select("#datagrid_results tr")


I am working on the file ".ipynb_checkpoints/scraping_form_submissions-checkpoint.ipynb".
The first task is:
Normal scraping
By now we all know how to scrape normal sites (kind of, mostly, somewhat).
Can you write Python code for it?

# Grab the NYT's homepage
response = requests.get("http://nytimes.com")
doc = BeautifulSoup(response.text)

# Snag all of the headlines (h3 tags with 'story-heading' class)
headlines = doc.find_all("h3", {'class': 'story-heading'})
# Getting the headline text out using list comprehensions
# is a lot more fun but I guess you just learned those
# like a day ago, so we'll go ahead and use a for loop.
# But for the curious:
#   [headline.text.strip() for headline in headlines]

# Print the text of the headlines
for headline in headlines:
    print(headline.)


I am working on the file ".ipynb_checkpoints/scratchbook-checkpoint.ipynb".
The first task is:
Can the site see the sun???
Can you write Python code for it?

df_viz = df_sun.join(df_allbearings, on='az')
df_viz['visible'] = False
df_viz.loc[df_viz['alt'] > df_viz['inclination_from_site'], 'visible'] = True

series_dailysum = df_viz.groupby(pd.TimeGrouper(freq='D'))['visible'].sum()/samples_per_hour
series_dailysum.plot(figsize=(18,6), style='.')

series_monthlysum = df_viz.groupby(pd.TimeGrouper(freq='M'))['visible'].sum()/samples_per_hour
series_monthlysum.plot(kind='bar', figsize=(18,6))

series_dailysum.to_csv(path='dailysum.csv', index=True, date_format='%Y-%m-%d', float_format='%.2f')
series_monthlysum.to_csv(path='monthlysum.csv', index=True, date_format='%B', float_format='%.2f')

series_monthlysum


I am working on the file ".ipynb_checkpoints/scratchbook-checkpoint.ipynb".
The first task is:
Interpolate the DEM derived skyview
Can you write Python code for it?

skyview_bearings = df_skyview.index.tolist()
azimuths = df_sun.az.tolist()
all_bearings = list(skyview_bearings)
all_bearings.extend(x for x in azimuths if x not in all_bearings)
df_allbearings = df_skyview.reindex(all_bearings)
df_allbearings.sort_index(inplace=True)
df_allbearings = df_allbearings.inclination_from_site.interpolate().ffill().bfill()

# df_allbearings.plot(kind='area', figsize=(18,6))


I am working on the file ".ipynb_checkpoints/scratchbook-checkpoint.ipynb".
The first task is:
Load digital elevation model
Can you write Python code for it?


I am working on the file ".ipynb_checkpoints/scratchbook-checkpoint.ipynb".
The first task is:
Not integrated in this notebook yet...
Did the viewshed analysis in QGIS using https://github.com/zoran-cuckovic/QGIS-visibility-analysis
Options:
- horizon search radius 20 km
- target (site) height 2 meters above surface
- adapting out 3 pixels (15 meters/~50 feet) from site to find high spot
Kept working in qgis for speed, will convert to this notebook eventually
In QGIS the horizon was:
- converted to ascii
- imported as points
Then:
- the dem was sampled with the horizon pts
- distance to site from each point calculated (m)
- inclination angle to each point calculation: degrees(atan((horizon_elev-site_elev)/distance)
Bearing calculation:
degrees(atan(($x_at(0)-site_lon)/($y_at(0)-site_lat))) + (180 *((($y_at(0)-site_lat) < 0) + ((($x_at(0)-site_lon) < 0 AND ($y_at(0)-site_lat) >0)*2)))
Can you write Python code for it?

df = pd.read_csv('inputs/viewshed/horizon_dist.csv')
df_skyview = df.set_index('bearing_from_site')
df_skyview.plot(y='inclination_from_site', style='.', figsize=(18,6), xlim=(0, 360))

pd.set_option('display.float_format', lambda x: '%.3f' % x)
df_skyview.loc[df_skyview['inclination_from_site'].idxmax()]


I am working on the file ".ipynb_checkpoints/scratchbook-checkpoint.ipynb".
The first task is:
Reproject the site to coords of dem for sampling elevation
Can you write Python code for it?

site = ogr.Geometry(ogr.wkbPoint) # create an ogr geom instead of shapely
site.AddPoint(s.x, s.y)

sr = dem.GetProjection()
destSR = osr.SpatialReference()
inSRS_converter = osr.SpatialReference()
inSRS_converter.ImportFromWkt(sr)
inSRS_proj4 = inSRS_converter.ExportToProj4()
destSR.ImportFromProj4(inSRS_proj4)

srcSR = osr.SpatialReference()
srcSR.ImportFromProj4("+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs")
srTrans = osr.CoordinateTransformation(srcSR,destSR)

site_reproj = site
site.Transform(srTrans)
print(site_reproj.ExportToWkt())


I am working on the file ".ipynb_checkpoints/scratchbook-checkpoint.ipynb".
The first task is:
Solar analysis
Get the azimuth and altitude for the sun for every hour during the year
Can you write Python code for it?

panel = ephem.Observer()
panel.lon = str(s.x) # this needs to be a string. derp
panel.lat = str(s.y)
panel.elevation = site_elev+2 # +2meters stays consistent with viewshed analysis
# panel.date = '2017/05/17 20:30:00' # utc time
panel

from pytz import all_timezones, common_timezones
local = pytz.timezone('America/Anchorage')

def get_sunangles(time_utc, observer):
    observer.date = time_utc
    solar = ephem.Sun(observer)
    a = {}
    a['az'] = math.degrees(solar.az)
    a['alt'] = math.degrees(solar.alt)
    return a

get_sunangles(datetime.datetime.utcnow(), panel)

df_sun = pd.DataFrame(index=pd.date_range('2018-01-01 00:00', '2018-12-31 23:59', freq='5min', tz=local))
samples_per_hour=12 # adjust based on freq

df_sun['ts_utc'] = (df_sun.index).tz_convert(pytz.timezone('UTC'))
df_sun['az'] = df_sun.apply(lambda row: get_sunangles(row['ts_utc'], panel)['az'], axis=1)
df_sun['alt'] = df_sun.apply(lambda row: get_sunangles(row['ts_utc'], panel)['alt'], axis=1)

# df_sun.alt.plot(figsize=(18,6))


I am working on the file ".ipynb_checkpoints/seeds-sciki-mfeatures-checkpoint.ipynb".
The first task is:
Plot learning curves of different classifiers
This example is a small modification of the sciki-learn tutorial test.
Comparison of different linear SVM classifiers on a 2D projection of the iris
dataset. Here I consider only two features of the dataset:

Seed area
Seed asymmetry

Test linear and other models, and use multiple features of the seeds dataset.
Can you write Python code for it?

print(__doc__)

import numpy as np
import matplotlib.pyplot as plt
from sklearn.naive_bayes import GaussianNB
from sklearn.svm import SVC
from sklearn.datasets import load_digits
from sklearn.model_selection import learning_curve
from sklearn.model_selection import ShuffleSplit

def plot_learning_curve(estimator, title, X, y, ylim=None, cv=None,
                        n_jobs=1, train_sizes=np.linspace(.1, 1.0, 5)):
    """
    Generate a simple plot of the test and training learning curve.

    Parameters
    ----------
    estimator : object type that implements the "fit" and "predict" methods
        An object of that type which is cloned for each validation.

    title : string
        Title for the chart.

    X : array-like, shape (n_samples, n_features)
        Training vector, where n_samples is the number of samples and
        n_features is the number of features.

    y : array-like, shape (n_samples) or (n_samples, n_features), optional
        Target relative to X for classification or regression;
        None for unsupervised learning.

    ylim : tuple, shape (ymin, ymax), optional
        Defines minimum and maximum yvalues plotted.

    cv : int, cross-validation generator or an iterable, optional
        Determines the cross-validation splitting strategy.
        Possible inputs for cv are:
          - None, to use the default 3-fold cross-validation,
          - integer, to specify the number of folds.
          - An object to be used as a cross-validation generator.
          - An iterable yielding train/test splits.

        For integer/None inputs, if ``y`` is binary or multiclass,
        :class:`StratifiedKFold` used. If the estimator is not a classifier
        or if ``y`` is neither binary nor multiclass, :class:`KFold` is used.

        Refer :ref:`User Guide <cross_validation>` for the various
        cross-validators that can be used here.

    n_jobs : integer, optional
        Number of jobs to run in parallel (default 1).
    """
    plt.figure()
    plt.title(title)
    if...


I am working on the file ".ipynb_checkpoints/seeds-sciki-mfeatures-checkpoint.ipynb".
The first task is:
Testing sklearn classifiers.
Here I test different classifiers provided by sklearn into my specific test set.
Can you write Python code for it?

#In the database there are 3 classes of seeds:
#And skilearn can handle multiple classes

import numpy as np

#This sets class=2 to 0 and 3 to 1:
y = df.loc[:,'class']

#Extract all cathegories:
X=df.iloc[:,0:7]
#This is to convert the csv dictionary into a numpy matrix to later standarize:
X=X.as_matrix()

nfeature=X.shape[1]

# standardize features
X_std = np.copy(X)
for ifeat in range(0,nfeature):
    X_std[:,ifeat] = (X[:,ifeat] - X[:,ifeat].mean()) / X[:,ifeat].std()

#Here since we have many features, we just plot the learning curves for the training and cross-validation sets.

title = "Learning Curves (Naive Bayes)"
# Cross validation with 100 iterations to get smoother mean test and train
# score curves, each time with 20% data randomly selected as a validation set.
cv = ShuffleSplit(n_splits=100, test_size=0.2, random_state=0)

estimator = GaussianNB()
plot_learning_curve(estimator, title, X, y, ylim=(0.7, 1.01), cv=cv, n_jobs=4)

title = "Learning Curves (SVC, Poly kernel, $\gamma=0.001$)"
cv = ShuffleSplit(n_splits=100, test_size=0.2, random_state=0)
estimator = SVC(kernel='poly',gamma=0.001)
plot_learning_curve(estimator, title, X, y, (0.7, 1.01), cv=cv, n_jobs=4)

title = "Learning Curves (SVC, RBF kernel, $\gamma=0.001$)"
cv = ShuffleSplit(n_splits=100, test_size=0.2, random_state=0)
estimator = SVC(kernel='rbf',gamma=0.001)
plot_learning_curve(estimator, title, X, y, (0.7, 1.01), cv=cv, n_jobs=4)

title = "Learning Curves (Linear SVC)"
cv = ShuffleSplit(n_splits=100, test_size=0.2, random_state=0)
estimator = svm.LinearSVC(C=1.0)
plot_learning_curve(estimator, title, X, y, (0.7, 1.01), cv=cv, n_jobs=4)

plt.show()


I am working on the file ".ipynb_checkpoints/semantic_similarity_with_tf_hub_universal_encoder-checkpoint.ipynb".
The first task is:
More detailed information about installing Tensorflow can be found at https://www.tensorflow.org/install/.
Can you write Python code for it?

#@title Load the Universal Sentence Encoder's TF Hub module
from absl import logging

import tensorflow as tf

import tensorflow_hub as hub
import matplotlib.pyplot as plt
import numpy as np
import os
import pandas as pd
import re
import seaborn as sns

module_url = "https://tfhub.dev/google/universal-sentence-encoder/4" #@param ["https://tfhub.dev/google/universal-sentence-encoder/4", "https://tfhub.dev/google/universal-sentence-encoder-large/5"]
model = hub.load(module_url)
print ("module %s loaded" % module_url)
def embed(input):
  return model(input)

model(["I love things"])[0]

hub.load?

#@title Compute a representation for each message, showing various lengths supported.
word = "Elephant"
sentence = "I am a sentence for which I would like to get its embedding."
paragraph = (
    "Universal Sentence Encoder embeddings also support short paragraphs. "
    "There is no hard limit on how long the paragraph is. Roughly, the longer "
    "the more 'diluted' the embedding will be.")
messages = [word, sentence, paragraph]

# Reduce logging output.
logging.set_verbosity(logging.ERROR)

message_embeddings = embed(messages)

for i, message_embedding in enumerate(np.array(message_embeddings).tolist()):
  print("Message: {}".format(messages[i]))
  print("Embedding size: {}".format(len(message_embedding)))
  message_embedding_snippet = ", ".join(
      (str(x) for x in message_embedding[:3]))
  print("Embedding: [{}, ...]\n".format(message_embedding_snippet))


I am working on the file ".ipynb_checkpoints/semantic_similarity_with_tf_hub_universal_encoder-checkpoint.ipynb".
The first task is:
Semantic Textual Similarity Task Example
The embeddings produced by the Universal Sentence Encoder are approximately normalized. The semantic similarity of two sentences can be trivially computed as the inner product of the encodings.
Can you write Python code for it?

def plot_similarity(labels, features, rotation):
  corr = np.inner(features, features)
  sns.set(font_scale=1.2)
  g = sns.heatmap(
      corr,
      xticklabels=labels,
      yticklabels=labels,
      vmin=0,
      vmax=1,
      cmap="YlOrRd")
  g.set_xticklabels(labels, rotation=rotation)
  g.set_title("Semantic Textual Similarity")

def run_and_plot(messages_):
  message_embeddings_ = embed(messages_)
  plot_similarity(messages_, message_embeddings_, 90)


I am working on the file ".ipynb_checkpoints/semantic_similarity_with_tf_hub_universal_encoder-checkpoint.ipynb".
The first task is:
Similarity Visualized
Here we show the similarity in a heat map. The final graph is a 9x9 matrix where each entry [i, j] is colored based on the inner product of the encodings for sentence i and j.
Can you write Python code for it?

messages = [
    # Smartphones
    "I like my phone",
    "My phone is not good.",
    "Your cellphone looks great.",

    # Weather
    "Will it snow tomorrow?",
    "Recently a lot of hurricanes have hit the US",
    "Global warming is real",

    # Food and health
    "An apple a day, keeps the doctors away",
    "Eating strawberries is healthy",
    "Is paleo better than keto?",

    # Asking about age
    "How old are you?",
    "How many years have you been alive?",
]

run_and_plot(messages)


I am working on the file ".ipynb_checkpoints/semantic_similarity_with_tf_hub_universal_encoder-checkpoint.ipynb".
The first task is:
Universal Sentence Encoder

    View on TensorFlow.org

    Run in Google Colab

    View on GitHub

    Download notebook

    See TF Hub models

This notebook illustrates how to access the Universal Sentence Encoder and use it for sentence similarity and sentence classification tasks.
The Universal Sentence Encoder makes getting sentence level embeddings as easy as it has historically been to lookup the embeddings for individual words. The sentence embeddings can then be trivially used to compute sentence level meaning similarity as well as to enable better performance on downstream classification tasks using less supervised training data.
Setup
This section sets up the environment for access to the Universal Sentence Encoder on TF Hub and provides examples of applying the encoder to words, sentences, and paragraphs.
Can you write Python code for it?


I am working on the file ".ipynb_checkpoints/sf_crime-checkpoint.ipynb".
The first task is:
Loading the data, version 2 with some weather features
Can you write Python code for it?

data_path = "./data/train_transformed.csv"

df = pd.read_csv(data_path, header=0)
x_data = df.drop('category', 1)
y = df.category.as_matrix()

########## adding the date back in
import csv
import time
import calendar
data_path = "./data/train.csv"
dataCSV = open(data_path, 'rt')
csvData = list(csv.reader(dataCSV))
csvFields = csvData[0] #['Dates', 'Category', 'Descript', 'DayOfWeek', 'PdDistrict', 'Resolution', 'Address', 'X', 'Y']
allData = csvData[1:]
dataCSV.close()

df2 = pd.DataFrame(allData)
df2.columns = csvFields
dates = df2['Dates']
dates = dates.apply(time.strptime, args=("%Y-%m-%d %H:%M:%S",))
dates = dates.apply(calendar.timegm)
print(dates.head())
#dates = pd.to_datetime(dates)

x_data['secondsFromEpoch'] = dates
colnames = x_data.columns.tolist()
colnames = colnames[-1:] + colnames[:-1]
x_data = x_data[colnames]
##########

########## adding in weather data
weatherData1 = "./data/1027175.csv"
weatherData2 = "./data/1027176.csv"
dataCSV = open(weatherData1, 'rt')
csvData = list(csv.reader(dataCSV))
csvFields = csvData[0] #['Dates', 'Category', 'Descript', 'DayOfWeek', 'PdDistrict', 'Resolution', 'Address', 'X', 'Y']
allWeatherData1 = csvData[1:]
dataCSV.close()

dataCSV = open(weatherData2, 'rt')
csvData = list(csv.reader(dataCSV))
csvFields = csvData[0] #['Dates', 'Category', 'Descript', 'DayOfWeek', 'PdDistrict', 'Resolution', 'Address', 'X', 'Y']
allWeatherData2 = csvData[1:]
dataCSV.close()

weatherDF1 = pd.DataFrame(allWeatherData1)
weatherDF1.columns = csvFields
dates1 = weatherDF1['DATE']

weatherDF2 = pd.DataFrame(allWeatherData2)
weatherDF2.columns = csvFields
dates2 = weatherDF2['DATE']

dates1 = dates1.apply(time.strptime, args=("%Y-%m-%d %H:%M",))
dates1 = dates1.apply(calendar.timegm)
dates2 = dates2.apply(time.strptime, args=("%Y-%m-%d %H:%M",))
dates2 = dates2.apply(calendar.timegm)
weatherDF1['DATE'] = dates1
weatherDF2['DATE'] = dates2
weatherDF = pd.concat([weatherDF1,weatherDF2[32:]],ignore_index=True)

#starting off with some of the easier features to work with-- more to...


I am working on the file ".ipynb_checkpoints/sf_crime-checkpoint.ipynb".
The first task is:
Model Prototyping
Rapidly assessing the viability of different model forms:
Can you write Python code for it?

def model_prototype(train_data, train_labels, eval_data, eval_labels):
    knn = KNeighborsClassifier(n_neighbors=5).fit(train_data, train_labels)
    bnb = BernoulliNB(alpha=1, binarize = 0.5).fit(train_data, train_labels)
    mnb = MultinomialNB().fit(train_data, train_labels)
    log_reg = LogisticRegression().fit(train_data, train_labels)
    support_vm = svm.SVC().fit(train_data, train_labels)
    neural_net = MLPClassifier().fit(train_data, train_labels)
    random_forest = RandomForestClassifier().fit(train_data, train_labels)

    models = [knn, bnb, mnb, log_reg, support_vm, neural_net, random_forest]
    for model in models:
        eval_preds = model.predict(eval_data)
        print(model, "Accuracy:", np.mean(eval_preds==eval_labels), "\n\n")

model_prototype(mini_train_data, mini_train_labels, mini_dev_data, mini_dev_labels)