I am working on the file "NWB pvc-7 use-case.ipynb".
The first task is:
Explore video and imaging data
Can you write Python code for it?

# get movie arrays from file
movies = filter(lambda x: x.type == 'movie', recording.references)
print_stats(movies)

# get mouse image at the beginning of the selected stimulus
mouse = movies[1]
image_index = int(np.where(np.array(mouse.dimensions[0].ticks) > start)[0][0])

plt.imshow(mouse.data[image_index])

# get eye image at the end of the selected stimulus
eye = movies[0]
image_index = int(np.where(np.array(eye.dimensions[0].ticks) > end)[0][0])

plt.imshow(eye.data[image_index])

# get 2-photon image at the beginning of the selected stimulus
imaging = filter(lambda x: x.type == 'imaging', recording.references)[0]

image_index = int(np.where(np.array(imaging.dimensions[0].ticks) > start)[0][0])

plt.imshow(imaging.data[image_index])

# plot mouse speed in the whole window (TODO: add stimulus events)
speeds = filter(lambda x: x.type == 'runspeed', recording.references)[0]

p = Plotter()
p.add(speeds)
p.plot()

f.close()


I am working on the file "NWB pvc-7 use-case.ipynb".
The first task is:
Open a file and inspect its content
Can you write Python code for it?

f = File.open("data/pvc-7.nix.h5", FileMode.ReadOnly)

print_stats(f.blocks)

block = f.blocks[0]

print_stats(block.data_arrays)
print_stats(block.tags)


I am working on the file "NYC Green Taxi.ipynb".
The first task is:
- Another interesting question is how the speed changes over the course of the day. In this case I use  one-way anova test on multiple samples. We find that the speed is different in different hours with a zero pvalue of the anova test. The boxplot reveals that traffic is faster early morning and gets really slow in the evening.
Can you write Python code for it?

# calculate anova
hours = range(24)
cmd = "f_oneway("
for h in hours:
    cmd+="data[data.Hour=="+str(h)+"].Speed_mph,"
cmd=cmd[:-1]+")"
print "one way anova test:", eval(cmd) #evaluate the command and print

# boxplot
data.boxplot('Speed_mph','Hour')
plt.ylim([5,24]) # cut off outliers
plt.ylabel('Speed (mph)')
plt.show()


I am working on the file "NYC Green Taxi.ipynb".
The first task is:
-> We observe long range trips in the morning and evenings. Are these people commuting to work? If so how do they get back home. The evening peak are shorter than the morning peak. I would hypothesize that people are okay to take cabs in the morning to avoid being late to their early appointments while they would take public transportation in the evening. However, this might not apply to NYC
Let's also compare trips that originate (or terminate) from (at) one of the NYC airports. We can look at how many they are, the average fair, ...
Reading through the dictionary of variables, I found that the variable RateCodeID contains values indicating the final rate that was applied. Among those values, I realized that there is Newark and JFK which are the major airports in New York. In this part, I will use this knowledge and group data with RateCodeID 2 (JFK) and 3 (Newark). - An alternative (which I didn't due to time constraint) is to (1) get coordinates of airports from google map or http://transtats.bts.gov (2) get at least 4 points defining a rectangular buffer zone near the airport (3) build a polygon shape using shapely [https://pypi.python.org/pypi/Shapely] and (3) check if any pickup/dropoff location coordinates is within the polygon using shapely again. This method was first tried but was found to be time consuming -
Can you write Python code for it?

# select airport trips
airports_trips = data[(data.RateCodeID==2) | (data.RateCodeID==3)]
print "Number of trips to/from NYC airports: ", airports_trips.shape[0]
print "Average fare (calculated by the meter) of trips to/from NYC airports: $", airports_trips.Fare_amount.mean(),"per trip"
print "Average total charged amount (before tip) of trips to/from NYC airports: $", airports_trips.Total_amount.mean(),"per trip"


I am working on the file "NYC Green Taxi.ipynb".
The first task is:
1 Data cleaning
This part concerns work done to treat invalid data.

Ehail_fee was removed since 99% of the data are missing
Missing values in Trip_type were replace with the most common value that was 1
Invalid data were found in:
RateCodeID: about 0.01% of the values were 99. These were replaced by the most common value 2
Extra: 0.08% of transactions had negative Extra. These were replaced by 0 as the most frequent
Total_amount, Fare_amount, improvement_surcharge, Tip_amount: 0.16% of values were negative. The cases were considered as being machine errors during the data entry. They were replaced by their absolute values. Furthermore, as the minimum Total_amount that is chargeable for any service is $2.5, every transaction falling below that amount was replaced by the median value of the Total_amount 11.76.

The code is provided below
Can you write Python code for it?

# define a function to clean a loaded dataset

def clean_data(adata):
    """
    This function cleans the input dataframe adata:
    . drop Ehail_fee [99% transactions are NaNs]
    . impute missing values in Trip_type
    . replace invalid data by most frequent value for RateCodeID and Extra
    . encode categorical to numeric
    . rename pickup and dropff time variables (for later use)

    input:
        adata: pandas.dataframe
    output:
        pandas.dataframe

    """
    ## make a copy of the input
    data = adata.copy()
    ## drop Ehail_fee: 99% of its values are NaNs
    if 'Ehail_fee' in data.columns:
        data.drop('Ehail_fee',axis=1,inplace=True)

    ##  replace missing values in Trip_type with the most frequent value 1
    data['Trip_type '] = data['Trip_type '].replace(np.NaN,1)

    ## replace all values that are not allowed as per the variable dictionary with the most frequent allowable value
    # remove negative values from Total amound and Fare_amount
    print "Negative values found and replaced by their abs"
    print "Total_amount", 100*data[data.Total_amount<0].shape[0]/float(data.shape[0]),"%"
    print "Fare_amount", 100*data[data.Fare_amount<0].shape[0]/float(data.shape[0]),"%"
    print "Improvement_surcharge", 100*data[data.improvement_surcharge<0].shape[0]/float(data.shape[0]),"%"
    print "Tip_amount", 100*data[data.Tip_amount<0].shape[0]/float(data.shape[0]),"%"
    print "Tolls_amount", 100*data[data.Tolls_amount<0].shape[0]/float(data.shape[0]),"%"
    print "MTA_tax", 100*data[data.MTA_tax<0].shape[0]/float(data.shape[0]),"%"
    data.Total_amount = data.Total_amount.abs()
    data.Fare_amount = data.Fare_amount.abs()
    data.improvement_surcharge = data.improvement_surcharge.abs()
    data.Tip_amount = data.Tip_amount.abs()
    data.Tolls_amount = data.Tolls_amount.abs()
    data.MTA_tax = data.MTA_tax.abs()

    # RateCodeID
    indices_oi = data[~((data.RateCodeID>=1) & (data.RateCodeID<=6))].index
    data.loc[indices_oi, 'RateCodeID'] = 2 # 2 =...


I am working on the file "NYC Green Taxi.ipynb".
The first task is:
2. Similarly to the comparison between trips to/from airports with the rest of the trips, it is worthy to spend more time and check wether trips originating from upper manhattan have different percentage tip.
To identify trips originating from upper manhattan:
- From googgle map, collect latitude and longitude data of at least 12 points that approximately define the bounding box of upper Manhattan
- Create a polygon using shapely.geometry.Polygon [https://pypi.python.org/pypi/Shapely]
- Check if the polygon contains  a location defined by (latitude,longitude)
Can you write Python code for it?

# import library
from shapely.geometry import Point,Polygon,MultiPoint
# data points that define the bounding box of the Upper Manhattan
umanhattan = [(40.796937, -73.949503),(40.787945, -73.955822),(40.782772, -73.943575),
              (40.794715, -73.929801),(40.811261, -73.934153),(40.835371, -73.934515),
              (40.868910, -73.911145),(40.872719, -73.910765),(40.878252, -73.926350),
              (40.850557, -73.947262),(40.836225, -73.949899),(40.806050, -73.971255)]

poi = Polygon(umanhattan)
# create a function to check if a location is located inside Upper Manhattan
def is_within_bbox(loc,poi=poi):
    """
    This function returns 1 if a location loc(lat,lon) is located inside a polygon of interest poi
    loc: tuple, (latitude, longitude)
    poi: shapely.geometry.Polygon, polygon of interest
    """
    return 1*(Point(loc).within(poi))
tic = dt.datetime.now()
# Create a new variable to check if a trip originated in Upper Manhattan
data['U_manhattan'] = data[['Pickup_latitude','Pickup_longitude']].apply(lambda r:is_within_bbox((r[0],r[1])),axis=1)
print "Processing time ", dt.datetime.now()-tic


I am working on the file "NYC Green Taxi.ipynb".
The first task is:
5.2 Regression Model
Following a similar pipeline of optimization as in the classification model, a model was built on top of the random forest algorithm.
-- Key points --:
- Sample size for training and optimization was chosen as 100000 with 5 folds cross-validation
- Only the number of trees were optimized as they are the controlling key of boosting model accuracy. Other parameters were not optimized since the improvement yield was too small compared to the computation time and cost
- The mean square error was used as a valuation metric
-- Results --:

optimized number of trees: 150
optimized variables: Total_amount, Trip_duration, Speed_mph
mean square error on a different test sample: 14.3648

The following code shows the optimization process
Can you write Python code for it?

train = data1.copy()
train = train.loc[np.random.choice(train.index,size=100000,replace=False)]
indices = data1.index[~data1.index.isin(train.index)]
test = data1.loc[np.random.choice(indices,size=100000,replace=False)]

train['ID'] = train.index
IDCol = 'ID'
target = 'Tip_percentage'

predictors = ['VendorID', 'Passenger_count', 'Trip_distance', 'Total_amount',
              'Extra', 'MTA_tax', 'Tolls_amount', 'Payment_type',
              'Hour', 'U_manhattan', 'Week', 'Week_day', 'Month_day', 'Shift_type',
              'Direction_NS', 'Direction_EW', 'Trip_duration', 'Speed_mph']
predictors = ['Trip_distance','Tolls_amount', 'Direction_NS', 'Direction_EW', 'Trip_duration', 'Speed_mph']
predictors = ['Total_amount', 'Trip_duration', 'Speed_mph']

# Random Forest
tic = dt.datetime.now()
from sklearn.ensemble import RandomForestRegressor
# optimize n_estimator through grid search
param_test = {'n_estimators':range(50,200,25)} # define range over which number of trees is to be optimized
# initiate classification model
#rfr = RandomForestRegressor(min_samples_split=2,max_depth=5,max_features='auto',random_state = 10)
rfr = RandomForestRegressor()#n_estimators=100)
# get results of the search grid
gs_rfr = optimize_num_trees(rfr,param_test,'mean_squared_error',train,predictors,target)

# print optimization results
print gs_rfr.grid_scores_, gs_rfr.best_params_, gs_rfr.best_score_

# plot optimization results
#plot_opt_results(gs_rfr)

# cross validate the best model with optimized number of estimators
modelfit(gs_rfr.best_estimator_,train,predictors,target,'mean_squared_error')

# save the best estimator on disk as pickle for a later use
with open('my_rfr_reg2.pkl','wb') as fid:
    pickle.dump(gs_rfr.best_estimator_,fid)
    fid.close()

ypred = gs_rfr.best_estimator_.predict(test[predictors])

print 'RFR test mse:',metrics.mean_squared_error(ypred,test.Tip_percentage)
print 'RFR r2:', metrics.r2_score(ypred,test.Tip_percentage)
print dt.datetime.now()-tic
plot_opt_results(gs_rfr)


I am working on the file "NYC Green Taxi.ipynb".
The first task is:
A further analysis of all continuous variables revealed similar lognormal and non-linearlity behaviors. Since there is no linear relationship between the the Tip percentage and variables, random forest algorithm will be considered to build the regression part of the model
As far as categorical variables concerned, the function visualize_categories was used to explore each variable as it was done for continuous numerical variables (See demostration below)
Can you write Python code for it?

# exploration of the U_manhattan (trip originating from Upper Manhattan) variable
visualize_categories(data1,'U_manhattan','boxplot',[13,20])
test_classification(data,'U_manhattan')


I am working on the file "NYC Green Taxi.ipynb".
The first task is:
A negative t-test value and null p-value imply that the means of Total_amount are significantly different in the group of transactions with tips compared to the group with no tip. Therefore, this variable would used to train the classification model.
Using the same function, a plot of the tip percentage as function of trip duration showed a cluster of points at duration time greater than 1350 min (22 hours).
<img src="Q4_trip_duration.png">
These points look like outliers since it doesn't make sense to have a trip of 22 hours within NYC. Probably, tourists can!
The following code was used to analyze the cluser with trip duration greater than 1350 min
Can you write Python code for it?

# Code to generate the heat map to uncover hidden information in the cluster
# We will first source NYC boroughs shape files,
# then create polygons and check to which polygon does each of the point of the cluster begongs

## download geojson of NYC boroughs
nyc_boros = json.loads(requests.get("https://raw.githubusercontent.com/dwillis/nyc-maps/master/boroughs.geojson").content)

# parse boros into Multipolygons
boros = {}
for f in nyc_boros['features']:
    name = f['properties']['BoroName']
    code = f['properties']['BoroCode']
    polygons = []
    for p in f['geometry']['coordinates']:
        polygons.append(Polygon(p[0]))
    boros[code] = {'name':name,'polygon':MultiPolygon(polygons=polygons)}

# creae function to assign each coordinates point to its borough
def find_borough(lat,lon):
    """
    return the borough of a location given its latitude and longitude
    lat: float, latitude
    lon: float, longitude
    """
    boro = 0 # initialize borough as 0
    for k,v in boros.iteritems(): # update boro to the right key corresponding to the parent polygon
        if v['polygon'].contains(Point(lon,lat)):
            boro = k
            break # break the loop once the borough is found
    return [boro]

## Analyse the cluster now
# create data frame of boroughs
df = data1[data1.Trip_duration>=1350]
orig_dest = []
for v in df[['Pickup_latitude','Pickup_longitude','Dropoff_latitude','Dropoff_longitude']].values:
    orig_dest.append((find_borough(v[0],v[1])[0],find_borough(v[2],v[3])[0]))
df2 = pd.DataFrame(orig_dest)

## creae pivot table for the heat map plot
df2['val']=1 # dummy variable
mat_cluster1 = df2.pivot_table(index=0,columns=1,values='val',aggfunc='count')

## generate the map
fig,ax = plt.subplots(1,2,figsize=(15,4))
im =...


I am working on the file "NYC Green Taxi.ipynb".
The first task is:
Compare distributions of the two groups
Can you write Python code for it?

# create a vector to contain Tip percentage for
v1 = data[(data.U_manhattan==0) & (data.Tip_percentage>0)].Tip_percentage
v2 = data[(data.U_manhattan==1) & (data.Tip_percentage>0)].Tip_percentage

# generate bins and histogram values
bins = np.histogram(v1,bins=10)[1]
h1 = np.histogram(v1,bins=bins)
h2 = np.histogram(v2,bins=bins)

# generate the plot
# First suplot: visualize all data with outliers
fig,ax = plt.subplots(1,1,figsize=(10,5))
w = .4*(bins[1]-bins[0])
ax.bar(bins[:-1],h1[0],width=w,color='b')
ax.bar(bins[:-1]+w,h2[0],width=w,color='g')
ax.set_yscale('log')
ax.set_xlabel('Tip (%)')
ax.set_ylabel('Count')
ax.set_title('Tip')
ax.legend(['Non-Manhattan','Manhattah'],title='origin')
plt.show()
print 't-test results:', ttest_ind(v1,v2,equal_var=False)


I am working on the file "NYC Green Taxi.ipynb".
The first task is:
In addition to the number and mean fare of airport trips, let's have aso look at how trips are distributed by trip distances and hour of the day
Can you write Python code for it?

# create a vector to contain Trip Distance for
v2 = airports_trips.Trip_distance # airport trips
v3 = data.loc[~data.index.isin(v2.index),'Trip_distance'] # non-airport trips

# remove outliers:
# exclude any data point located further than 3 standard deviations of the median point and
# plot the histogram with 30 bins
v2 = v2[~((v2-v2.median()).abs()>3*v2.std())]
v3 = v3[~((v3-v3.median()).abs()>3*v3.std())]

# define bins boundaries
bins = np.histogram(v2,normed=True)[1]
h2 = np.histogram(v2,bins=bins,normed=True)
h3 = np.histogram(v3,bins=bins,normed=True)

# plot distributions of trip distance normalized among groups
fig,ax = plt.subplots(1,2,figsize = (15,4))
w = .4*(bins[1]-bins[0])
ax[0].bar(bins[:-1],h2[0],alpha=1,width=w,color='b')
ax[0].bar(bins[:-1]+w,h3[0],alpha=1,width=w,color='g')
ax[0].legend(['Airport trips','Non-airport trips'],loc='best',title='group')
ax[0].set_xlabel('Trip distance (miles)')
ax[0].set_ylabel('Group normalized trips count')
ax[0].set_title('A. Trip distance distribution')
#ax[0].set_yscale('log')

# plot hourly distribution
airports_trips.Pickup_hour.value_counts(normalize=True).sort_index().plot(ax=ax[1])
data.loc[~data.index.isin(v2.index),'Pickup_hour'].value_counts(normalize=True).sort_index().plot(ax=ax[1])
ax[1].set_xlabel('Hours after midnight')
ax[1].set_ylabel('Group normalized trips count')
ax[1].set_title('B. Hourly distribution of trips')
ax[1].legend(['Airport trips','Non-airport trips'],loc='best',title='group')
plt.savefig('Question3_2.jpeg',format='jpeg')
plt.show()


I am working on the file "NYC Green Taxi.ipynb".
The first task is:
Let's have a look at the distribution of trip distance
Can you write Python code for it?

# define the figure with 2 subplots
fig,ax = plt.subplots(1,2,figsize = (15,4))

# histogram of the number of trip distance
data.Trip_distance.hist(bins=30,ax=ax[0])
ax[0].set_xlabel('Trip Distance (miles)')
ax[0].set_ylabel('Count')
ax[0].set_yscale('log')
ax[0].set_title('Histogram of Trip Distance with outliers included')

# create a vector to contain Trip Distance
v = data.Trip_distance
# exclude any data point located further than 3 standard deviations of the median point and
# plot the histogram with 30 bins
v[~((v-v.median()).abs()>3*v.std())].hist(bins=30,ax=ax[1]) #
ax[1].set_xlabel('Trip Distance (miles)')
ax[1].set_ylabel('Count')
ax[1].set_title('A. Histogram of Trip Distance (without outliers)')

# apply a lognormal fit. Use the mean of trip distance as the scale parameter
scatter,loc,mean = lognorm.fit(data.Trip_distance.values,
                               scale=data.Trip_distance.mean(),
                               loc=0)
pdf_fitted = lognorm.pdf(np.arange(0,12,.1),scatter,loc,mean)
ax[1].plot(np.arange(0,12,.1),600000*pdf_fitted,'r')
ax[1].legend(['data','lognormal fit'])

# export the figure
plt.savefig('Question2.jpeg',format='jpeg')
plt.show()


I am working on the file "NYC Green Taxi.ipynb".
The first task is:
Make predictions on a sample of 100000 transactions
Can you write Python code for it?

test = data.loc[np.random.choice(data.index,size = 100000,replace=False)]
ypred = predict_tip(test)
print "final mean_squared_error:", metrics.mean_squared_error(ypred,test.Tip_percentage)
print "final r2_score:", metrics.r2_score(ypred,test.Tip_percentage)


I am working on the file "NYC Green Taxi.ipynb".
The first task is:
NYC Green Taxi
In this notebook, I will explore data on New York City Green Taxi of september 2015. I will start with some warm up questions about the dataset. Later, I will build a model to predict the percentage tip a driver would exepect on each trip. The code is fully written in python with few additional open-source libraries easy to install.
- shapely
- scikit learn
- tabulate
In this analysis, some notion of statistics and hypothesis test are used but are very easy to follow. This handbook of statistics can be used as a reference to explain basics.
Warm up
Let's first download the dataset and print out the its size
Can you write Python code for it?

# Download the September 2015 dataset
if os.path.exists('data_september_2015.csv'): # Check if the dataset is present on local disk and load it
    data = pd.read_csv('data_september_2015.csv')
else: # Download dataset if not available on disk
    url = "https://s3.amazonaws.com/nyc-tlc/trip+data/green_tripdata_2015-09.csv"
    data = pd.read_csv(url)
    data.to_csv(url.split('/')[-1])

# Print the size of the dataset
print "Number of rows:", data.shape[0]
print "Number of columns: ", data.shape[1]


I am working on the file "NYC Green Taxi.ipynb".
The first task is:
Next, each variable distribution and its relationship with the Tip percentage were explored. Few functions were implemented to quickly explore those variables:
Can you write Python code for it?

# Functions for exploratory data analysis
def visualize_continuous(df,label,method={'type':'histogram','bins':20},outlier='on'):
    """
    function to quickly visualize continous variables
    df: pandas.dataFrame
    label: str, name of the variable to be plotted. It should be present in df.columns
    method: dict, contains info of the type of plot to generate. It can be histogram or boxplot [-Not yet developped]
    outlier: {'on','off'}, Set it to off if you need to cut off outliers. Outliers are all those points
    located at 3 standard deviations further from the mean
    """
    # create vector of the variable of interest
    v = df[label]
    # define mean and standard deviation
    m = v.mean()
    s = v.std()
    # prep the figure
    fig,ax = plt.subplots(1,2,figsize=(14,4))
    ax[0].set_title('Distribution of '+label)
    ax[1].set_title('Tip % by '+label)
    if outlier=='off': # remove outliers accordingly and update titles
        v = v[(v-m)<=3*s]
        ax[0].set_title('Distribution of '+label+'(no outliers)')
        ax[1].set_title('Tip % by '+label+'(no outliers)')
    if method['type'] == 'histogram': # plot the histogram
        v.hist(bins = method['bins'],ax=ax[0])
    if method['type'] == 'boxplot': # plot the box plot
        df.loc[v.index].boxplot(label,ax=ax[0])
    ax[1].plot(v,df.loc[v.index].Tip_percentage,'.',alpha=0.4)
    ax[0].set_xlabel(label)
    ax[1].set_xlabel(label)
    ax[0].set_ylabel('Count')
    ax[1].set_ylabel('Tip (%)')

def visualize_categories(df,catName,chart_type='histogram',ylimit=[None,None]):
    """
    This functions helps to quickly visualize categorical variables.
    This functions calls other functions generate_boxplot and generate_histogram
    df: pandas.Dataframe
    catName: str, variable name, it must be present in df
    chart_type: {histogram,boxplot}, choose which type of chart to plot
    ylim: tuple, list. Valid if chart_type is histogram
    """
    print catName
    cats = sorted(pd.unique(df[catName]))
    if chart_type ==...


I am working on the file "NYC Green Taxi.ipynb".
The first task is:
The Trip Distance is asymmetrically distributed. It is skewed to the right and it has a median smaller than its mean and both smaller than the standard deviation. The skewness is due to the fact that the variable has a lower boundary of 0. The distance can't be negative. This distribution has a structure of a lognormal distribution. To the left is plotted the distribution of the entire raw set of Trip distance. To the right, outliers have been removed before plotting. Outliers are defined as any point located further than 3 standard deviations from the mean
The hypothesis: The trips are not random. If there were random, we would have a (symmetric) Gaussian distribution. The non-zero autocorrelation may be related the fact that people taking ride are pushed by a common cause, for instance, people rushing to work.
Let's see if the time of the day has any impact on the trip distance
Can you write Python code for it?

# First, convert pickup and drop off datetime variable in their specific righ format
data['Pickup_dt'] = data.lpep_pickup_datetime.apply(lambda x:dt.datetime.strptime(x,"%Y-%m-%d %H:%M:%S"))
data['Dropoff_dt'] = data.Lpep_dropoff_datetime.apply(lambda x:dt.datetime.strptime(x,"%Y-%m-%d %H:%M:%S"))

# Second, create a variable for pickup hours
data['Pickup_hour'] = data.Pickup_dt.apply(lambda x:x.hour)

# Mean and Median of trip distance by pickup hour
# I will generate the table but also generate a plot for a better visualization

fig,ax = plt.subplots(1,1,figsize=(9,5)) # prepare fig to plot mean and median values
# use a pivot table to aggregate Trip_distance by hour
table1 = data.pivot_table(index='Pickup_hour', values='Trip_distance',aggfunc=('mean','median')).reset_index()
# rename columns
table1.columns = ['Hour','Mean_distance','Median_distance']
table1[['Mean_distance','Median_distance']].plot(ax=ax)
plt.ylabel('Metric (miles)')
plt.xlabel('Hours after midnight')
plt.title('Distribution of trip distance by pickup hour')
#plt.xticks(np.arange(0,30,6)+0.35,range(0,30,6))
plt.xlim([0,23])
plt.savefig('Question3_1.jpeg',format='jpeg')
plt.show()
print '-----Trip distance by hour of the day-----\n'
print tabulate(table1.values.tolist(),["Hour","Mean distance","Median distance"])


I am working on the file "NYC Green Taxi.ipynb".
The first task is:
The above plot compares the means and range of the Tip_percentage between trips originating from Manhattan and the rest of the trips. The t-test reported says that these groups have different means. Furthermore, a chi-square test shows that this variable can be used to significantly distinguish transactions with tips from those without tips.
Another interesting figure is that of the Payment_type.
Can you write Python code for it?

# visualization of the Payment_type
visualize_categories(data1,'Payment_type','histogram',[13,20])


I am working on the file "NYC Green Taxi.ipynb".
The first task is:
The output shows that the optimum number of trees in 130 and that the important variables for this specific number of tree are as shown on the barchart.
Let's test it on a different sample with the following code
Can you write Python code for it?

# testing on a different set
indices = data.index[~data.index.isin(train.index)]
test = data.loc[np.random.choice(indices,size=100000,replace=False)]

ypred = gs_cls.best_estimator_.predict(test[predictors])

print "ROC AUC:", metrics.roc_auc_score(ypred,test.With_tip)


I am working on the file "NYC Green Taxi.ipynb".
The first task is:
The residual is pretty much symmetrically distributed. This indicate that the model is equally biased. The best model would be the one with mean at 0 and 0 variance
Recommendations

As a future work, I would find a transformation function that can linearlize indipendent values. I would also optimize different algorithms such as extreme gradient boosting and make a bag of multiple models as a final model. This was actually tried but failed because of the computational power.

The following section is the instruction on how to use the predictor script
Can you write Python code for it?

def read_me():
    """
    This is a function to print a read me instruction
    """
    print ("=========Introduction=========\n\nUse this code to predict the percentage tip expected after a trip in NYC green taxi. \nThe code is a predictive model that was built and trained on top of the Gradient Boosting Classifer and the Random Forest Gradient both provided in scikit-learn\n\nThe input: \npandas.dataframe with columns:This should be in the same format as downloaded from the website\n\nThe data frame go through the following pipeline:\n\t1. Cleaning\n\t2. Creation of derived variables\n\t3. Making predictions\n\nThe output:\n\tpandas.Series, two files are saved on disk,  submission.csv and cleaned_data.csv respectively.\n\nTo make predictions, run 'tip_predictor.make_predictions(data)', where data is any 2015 raw dataframe fresh from http://www.nyc.gov/html/tlc/html/about/trip_record_data.shtml\nRun tip_predictor.read_me() for further instructions\n")
read_me()


I am working on the file "NYC Green Taxi.ipynb".
The first task is:
The two distributions look the same however the t-test results in a zero p-value to imply that the two groups are different at 95% level of condidence
The Model
Summary
The initial dataset contained 1494926 transactions with 21 time-series, categorical and numerical variables. In order to build the final model, four phases were followed (1) data cleaning, (2) feature engineering (3) exploratory data analysis and (4) model creation
The cleaning consisted in drop zero variance variables (Ehail_fee), replacing invalid with the most frequent values in each categorical variable whereas the median was used for continuous numerical variables. Invalid values could be missing values or values not allowed for specific variables as per the dictionary of variables. In this part, variables were also converted in their appropriate format such datetime.
The feature engineering part created 10 new variables derived from pickup and dropoff locations and timestamps, trip distance.
During the exploration, each variable was carefully analyzed and compared to other variables and eventually the target variable, Percentage tip. All numerical variables were found to follow lognormal or power law distributions althouth there was found no linear relationship between numerical and the target variable. An interesting insight was uncovered in the distribution of the percentage tip. It was found that only 40% of the transactions paid tip. And 99.99% of these payments were done by credit cards. This inspired me to build the predictive model in two stages (1) classification model to find out weither a transaction will pay tip and (2) regression model to find the percentage of the tip only if the transaction was classified as a tipper. Another insight was that the most frequent percentage is 18% which corresponds to the usual restaurant gratuity rate.
With lack of linear relationship between independent and depend variables, the predictive model was built on top of the random forest regression and gradient boosting classifier algorithms...
Can you write Python code for it?

# Download the September 2015 dataset
if os.path.exists('data_september_2015.csv'): # Check if the dataset is present on local disk and load it
    data = pd.read_csv('data_september_2015.csv')
else: # Download dataset if not available on disk
    url = "https://s3.amazonaws.com/nyc-tlc/trip+data/green_tripdata_2015-09.csv"
    data = pd.read_csv(url)
    data.to_csv(url.split('/')[-1])

# Print the size of the dataset
print "Number of rows:", data.shape[0]
print "Number of columns: ", data.shape[1]

# create backup dataset
backup_data = data.copy()


I am working on the file "NYC Traffic Collision Data.ipynb".
The first task is:

        15 Collision Hotspots of Manhattan

You can observe that a small area between the 25th and 60th Streets, the Midtown Manhattan area, has as many collision hotspots as the northern part of Manhattan, including uptown Manhattan area, and twice as many collision hotspots as Lower Manhattan. This is an interesting result that suggests just how many more collisions happen Midtown compared to the rest of Manhattan.
We can focus on parts of New York City to get a more granular understanding of problematic areas. Let's take Midtown Manhattan, for example. We can run kmeans only on the collisions that occur in Midtown Manhattan and plot them. To (roughly) get the collisions in Midtown Manhattan, we can define and use a function to return the collisions that are in a circle:
Can you write Python code for it?

def get_collisions_in_circle(collisions, latitude, longitude, radius_miles):
    location_data = collisions.loc[:, ['LATITUDE', 'LONGITUDE']]
    center = np.array([latitude, longitude])
    # Calculate distances of each collision to the center
    distances = location_data.apply(lambda x: distance((x['LATITUDE'], x['LONGITUDE']), center).miles, axis = 1)
    # Return the collisions inside the circle
    return collisions[distances <= radius].reset_index(drop=True)


I am working on the file "NYC Traffic Collision Data.ipynb".
The first task is:

        Taxis, Weekdays, Daytime (with bike lanes)
        Taxis, Weekends, Daytime (with bike lanes)

Similar to the results above, the severity of the collisions are greater on the weekdays and more concentrated towards the area of South Manhattan. This appears to be the opposite in that part of the borough during the weekend. It appears that the South-West corner of Central Park remains a problem area.
The results shown here demonstrate that collisions are more likely to happen during the night and close to Central Park or the South. We are not able to determine only based on this data as to what causes these hotspots. It may be that they are favorite places to ride a bike (like Central Park) or that it is an important commercial or business center. However, it is clear that collisions in this parts of town are frequent during certain parts of the day or of the week.

Let's see if buses follow a similar pattern since they are in many places popular means of transportation.
Can you write Python code for it?

filter_features = ['DAY_TIME', 'BUS', 'HAS_BIKE_LANE']

print plot_collisions(collisions_in_manhattan,
                      filter_features=filter_features + ['WEEKDAY'],
                      show_severity=True,
                      output_file_name='BUS_WEEKDAY_BIKE_DAY',
                      open_map_after_plot=True)

print plot_collisions(collisions_in_manhattan,
                      filter_features=filter_features + ['WEEKEND'],
                      show_severity=True,
                      output_file_name='BUS_WEEKEND_BIKE_DAY',
                      open_map_after_plot=True)


I am working on the file "NYC Traffic Collision Data.ipynb".
The first task is:
As you can observe, the borough wasn't much better at predicting the severity of collisions than the baseline either. It's as bad as just using the time of day.
Now, let's use all of our features and see how good they are at predicting the severity of collisions:
Can you write Python code for it?

# Train the classifier on all the features
classifier = linear_model.LogisticRegressionCV()
classifier.fit_transform(X_tr, Y_tr)
score = classifier.score(X_te, Y_te)

# Assess the score of the classifier through the validation set
accuracy = '{:.4f}%'.format(score * 100)
print 'All the features:', accuracy


I am working on the file "NYC Traffic Collision Data.ipynb".
The first task is:
As you can observe, the time of day wasn't much better at predicting the severity of collisions than the baseline.
Let's try another feature: the borough. We will see if the general area is a good indicator of the severity of collisions:
Can you write Python code for it?

# Throw out all the features but the borough
borough_features = ['BRONX', 'BROOKLYN', 'MANHATTAN', 'QUEENS', 'STATEN ISLAND']
X_tr_borough = X_tr.loc[:, borough_features]
X_te_borough = X_te.loc[:, borough_features]

# Train the classifier on the 'borough' features
classifier = linear_model.LogisticRegressionCV()
classifier.fit_transform(X_tr_borough, Y_tr)
score = classifier.score(X_te_borough, Y_te)

# Assess the score of the classifier through the validation set
accuracy = '{:.4f}%'.format(score * 100)
print 'Just borough:', accuracy


I am working on the file "NYC Traffic Collision Data.ipynb".
The first task is:
Let's try to see how well some individual features can predict the severity in order to compare it to the combined features' effectiveness. Let's try the time of day.
It might seem reasonable to expect the most severe collisions to happen during rush hours. Therefore, the time of day might also seem like a good predictor of the severity of the collision. We can test this hypothesis by randomly choosing a subset of our collisions, running logistic regression on it by using just the 'time of day' feature and testing it on a verification set, also randomly chosen.
Can you write Python code for it?

def get_training_sets(collision):
    """Gets a list of collisions and returns the set of training features and
    labels from them. The severity score of each collision are their respective
    label.

    Parameters:
        X (Pandas DataFrame): a dataframe that contains collisions

    Returns:
        X, Y (Pandas DataFrames): tuple of DataFrames that have features and labels of
                                  collisions, respectively
    """
    X = collision
    Y = collision['SEVERITY SCORE']

    # Convert BOROUGH to one-hot encoding
    if 'BOROUGH' in X.columns:
        borough_onehot = pd.get_dummies(X['BOROUGH']).astype(int)
        X = pd.concat([X, borough_onehot], axis=1)

    # Drop unneeded columns
    columns_to_drop = ['BOROUGH', 'ZIP CODE', 'LATITUDE', 'LONGITUDE',
                       'ON STREET NAME', 'CROSS STREET NAME', 'SEVERITY SCORE',
                       'UNIQUE KEY', 'NUMBER OF PERSONS INJURED',
                       'NUMBER OF PERSONS KILLED', 'NUMBER OF PEDESTRIANS INJURED',
                       'NUMBER OF PEDESTRIANS KILLED', 'NUMBER OF CYCLIST INJURED',
                       'NUMBER OF CYCLIST KILLED', 'NUMBER OF MOTORIST INJURED',
                       'NUMBER OF MOTORIST KILLED']
    for column in columns_to_drop:
        if column in X.columns:
            X = X.drop(column, axis=1)

    return X, Y

# The number of collisions we have
num_collisions_total = collision.shape[0]
# Number of collisions we want in our training set
num_collisions_training = 40000
# Number of collisions we want in our test/validation set
num_collisions_test = 4000

# Get a random choice of collisions
collisions_shuffled = collision.loc[np.random.choice(num_collisions_total,
                                                     (num_collisions_training + num_collisions_test),
                                                     replace=False), :]
training_collisions = collisions_shuffled[:num_collisions_training]
test_collisions =...


I am working on the file "NYC Traffic Collision Data.ipynb".
The first task is:
The bicycle lanes are defined by four features:

Which street they're on (ON STREET NAME)
Which cross street they begin on (BEGINS)
Which cross street they end on (ENDS)
The borough they're on (BOROUGH)

By using these four values, we'll match them to the location of the collisions and add a 'HAS_BIKE_LANE' column to the collisions data frame, indicating whether the collision occurred on a street that has a bike lane.
Can you write Python code for it?

def merge_bicycle_lanes(df1, df2, columns):
    # Populate collisions with bicycle lane data by matching on 'columns'
    result = pd.merge(df1, df2, how='left', on=columns)

    # Create a 'HAS_BIKE_LANE' column to indicate the presenc of a bike lane
    result = result.assign(HAS_BIKE_LANE = [0 if x is np.nan else 1 for x in result.loc[:, 'BEGINS']])

    # Drop unneeded columns
    result = result.drop('BEGINS', axis=1)
    result = result.drop('ENDS', axis=1)

    # Multiple bike lanes can be on the same street, disregard that fact
    result = result.drop_duplicates()

    # Reset index to account for duplicates
    result = result.reset_index(drop=True)

    return result

collision = merge_bicycle_lanes(collision, bicycle_lanes, ['ON STREET NAME', 'BOROUGH'])

print collision.head()
print collision.dtypes
print len(collision)


I am working on the file "NYC Traffic Collision Data.ipynb".
The first task is:
We further proceeded to create a one-hot encoding of the vehicle types that were available in the dataset. NYPD has put almost half of the vehicles in a general group called 'PASSENGER VEHICLE', which seems to represent the common sedan type of vehicle. Other types, like SUVs, motorcycles, small and large commercial vehicles, have their own types.
We removed some collision types (like the collisions in which the vehicle was an 'AMBULANCE') with very few collisions. This further cut about 5% of the dataset.
Can you write Python code for it?

def create_vehicle_features(collision):
    # Create one-hot vehicle type representation
    vehicle_types_onehot = pd.get_dummies(collision.loc[:, 'VEHICLE TYPE CODE 1']).astype(int)

    # Merge Motorcycle & Scooter columns
    motorcycle = vehicle_types_onehot.loc[:, 'MOTORCYCLE'] + vehicle_types_onehot.loc[:, 'SCOOTER']
    vehicle_types_onehot = vehicle_types_onehot.drop('MOTORCYCLE', axis=1)
    vehicle_types_onehot = vehicle_types_onehot.drop('SCOOTER', axis=1)
    vehicle_types_onehot = vehicle_types_onehot.assign(MOTORCYCLE = motorcycle.values)

    # Concatanate one-hot with collisions
    collision = pd.concat([collision, vehicle_types_onehot], axis=1)

    # Drop unneeded collisions
    vehicles_to_drop = ['OTHER', 'AMBULANCE', 'PEDICAB', 'FIRE TRUCK', 'LIVERY VEHICLE']
    collisions_to_drop = vehicle_types_onehot.loc[:, vehicles_to_drop[0]]
    for i in xrange(1, len(vehicles_to_drop)):  # Start from 1 since we already have OTHER
        collisions_to_drop += vehicle_types_onehot.loc[:, vehicles_to_drop[i]]
    collisions_to_keep = (collisions_to_drop == 0)
    collision = collision[collisions_to_keep]
    collision = collision.reset_index(drop=True)  # Reset index due to dropped rows

    # Drop unneeded vehicle columns
    for column in vehicles_to_drop:
        collision = collision.drop(column, axis=1)

    # Drop vehicle type column
    collision = collision.drop('VEHICLE TYPE CODE 1', axis=1)

    return collision


I am working on the file "NYC Traffic Collision Data.ipynb".
The first task is:
We proceeded by assigning a 'severity score' to each collision. Each collision, by default, has a severity score of 1. The score increases by 2 for each injury and by 10 for each death. These are arbitrary values we came up with and have no bearing on the cost of life in these collisions.
Can you write Python code for it?

# Default score values
INJURY_SCORE = 2
DEATH_SCORE = 10

def create_severity_score_feature(collision):
    # Give all collisions a score of 1 by default
    collision['SEVERITY SCORE'] = 1

    # For each injury, add to the score of the collision
    collision['SEVERITY SCORE'] += (collision.loc[:, 'NUMBER OF PERSONS INJURED'] * INJURY_SCORE)

    # For each death, add to the score of the collision
    collision['SEVERITY SCORE'] += (collision.loc[:, 'NUMBER OF PERSONS KILLED'] * DEATH_SCORE)

    return collision


I am working on the file "NYC Traffic Collision Data.ipynb".
The first task is:
We proceeded to create two temporal features:

Whether it occurred on a weekday or weekend
In what 'time period' of the day it occurred

The 'time period' requires some explanation: We all know that morning and evening rush hours can be especially more collision-prone than daytime (around noon) and night time (after the evening rush and before the morning rush). We decided to create these four bins with the following time ranges:

Night Time: 00:00-06:59 & 20:00-00:00
Morning Rush: 07:00-10:59
Night Time: 00:00-06:59 & 20:00-00:00
Evening Rush: 16:00-19:59

Together with the weekday/weekend feature, this allowed us to represent the date in a way that is meaningful to a machine learning algorithm. We also assumed that, as long as it's a weekday, the exact day (whether it's a Monday or a Thursday) is not significant.
Can you write Python code for it?

def create_temporal_features(collision):
    collision = _create_time_features(collision)
    collision = _create_date_features(collision)

    # We're done with date, we can drop it
    collision = collision.drop('DATE', axis=1)

    return collision

def _create_time_features(collision):
    # Create one-hot time of day representation
    ## Date part is unimportant
    morning_rush_begin = pd.datetime(2000, 01, 01, 07, 00, 00).time()
    morning_rush_end = pd.datetime(2000, 01, 01, 11, 00, 00).time()
    evening_rush_begin = pd.datetime(2000, 01, 01, 16, 00, 00).time()
    evening_rush_end = pd.datetime(2000, 01, 01, 20, 00, 00).time()
    collision_time = collision['DATE'].dt.time

    ## Night Time 00:00-06:59 & 20:00-00:00
    night_time = (collision_time >= evening_rush_end) | (collision_time < morning_rush_begin)
    night_time_onehot = pd.get_dummies(night_time).loc[:, True].astype(int)
    collision = collision.assign(NIGHT_TIME = night_time_onehot.values)

    ## Morning Rush 07:00-10:59
    morning_rush = (collision_time >= morning_rush_begin) & (collision_time < morning_rush_end)
    morning_rush_onehot = pd.get_dummies(morning_rush).loc[:, True].astype(int)
    collision = collision.assign(MORNING_RUSH = morning_rush_onehot.values)

    ## Night time 00:00-06:59 & 20:00-00:00
    day_time = (collision_time >= morning_rush_end) & (collision_time < evening_rush_begin)
    day_time_onehot = pd.get_dummies(day_time).loc[:, True].astype(int)
    collision = collision.assign(DAY_TIME = day_time_onehot.values)

    ## Evening Rush 16:00-19:59
    evening_rush = (collision_time >= evening_rush_begin) & (collision_time < evening_rush_end)
    evening_rush_onehot = pd.get_dummies(evening_rush).loc[:, True].astype(int)
    collision = collision.assign(EVENING_RUSH = evening_rush_onehot.values)

    return collision

def _create_date_features(collision):
    # Create one-hot weekday/weekend representation
    collision_day = collision['DATE'].dt.dayofweek

    ## Weekday...


I am working on the file "NYC Traffic Collision Data.ipynb".
The first task is:
We will also use a function that takes collisions and plots them, along with some of its features (such as the centers, the membership of collisions, etc.). Effectively visualizing this kind of data is of paramount importance, since the evaluation depends on observation of collision hotspots.
The function takes a list of collisions and plots them on an interactive map of New York City. We provide the function with the collisions we would like to plot, whether we want to show the severity of the collisions, whether we want to demonstrate their cluster membership (which are obtained through kmeans), etc.
Can you write Python code for it?

def plot_collisions(collision, filter_features=[], show_severity=False, centroids=None, labels_of_collisions=None,
                    color_collisions_based_on_labels=False, only_show_num_collisions=None,
                    open_map_after_plot=False, output_file_name='map', lat=40.77, lon=-74.009725, zoom=12):
    # Create the output files
    create_map_html_file(output_file_name)
    create_map_script_file(output_file_name, lat, lon, zoom)

    collisions_to_plot = collision.copy().reset_index(drop=True)

    if labels_of_collisions is not None:
        # Append the labels
        collisions_to_plot = collisions_to_plot.assign(LABELS = labels_of_collisions)

    # Filter the collisions based on the list of features that were given
    for feature in filter_features:
        collisions_to_plot = collisions_to_plot.loc[collisions_to_plot[feature] == 1, :].reset_index(drop=True)

    # After filtering, throw away the unneeded features
    if labels_of_collisions is not None:
        collisions_to_plot = collisions_to_plot.loc[:, ['LATITUDE', 'LONGITUDE', 'SEVERITY SCORE', 'LABELS']]
    else:
        collisions_to_plot = collisions_to_plot.loc[:, ['LATITUDE', 'LONGITUDE', 'SEVERITY SCORE']]

    # Randomly pick some number of collisions if specified
    num_collisions_total = collisions_to_plot.shape[0]
    if only_show_num_collisions and only_show_num_collisions < num_collisions_total:
        collisions_shuffled = collisions_to_plot.loc[np.random.choice(num_collisions_total,
                                                                      only_show_num_collisions,
                                                                      replace=False), :]
        collisions_to_plot = collisions_shuffled[:only_show_num_collisions].reset_index(drop=True)

    if show_severity:  # Plot with heatmap
        # Get the color range
        unique_severity_scores = sorted(collisions_to_plot.loc[:, 'SEVERITY SCORE'].unique())
        num_unique_severity_scores =...


I am working on the file "NYCSubway.ipynb".
The first task is:
Features
I will try to compute the prediction for several possible features.
Please check the code below for the features.
I aim to have R^2 score of 0.40 or better.
The features of interest are listed below along with the reason:

rain : I think people will decide to ride the subway if it is raining
precipi : same with the above, it includes snow, drizzle, hail, etc.
meantempi : I think people will ride the subway more if it is cold
fog : I think people will ride the subway more if it is fogging, it is probably not fun to drive
meanwindspi: If it is windy, people who use bike or walk might opt to take subway
Hour : People tend to ride subway on certain hour, for example to get to or back from work.

I will create combinations out of these feature, compute the predictions and get the R^2 value.
In addition to this, I have added dummy variables 'UNIT' for the features.
Can you write Python code for it?

from itertools import combinations

# Build our features list combination
# feature_of_interest = ['rain', 'meantempi', 'fog', 'meanwindspdi', 'precipi', 'Hour']
# features_list = []
# for count in range(1, len(feature_of_interest) + 1):
#     for item in combinations(feature_of_interest, count):
#         features_list.append(list(item))

features_list = [['rain', 'meantempi', 'fog', 'meanwindspdi', 'precipi', 'Hour']]

best_score = {'score': 0}
for features in features_list:
    predictions, intercept, params = calculate_predictions(df, df[features])
    r2_score = compute_r_squared(df['ENTRIESn_hourly'], predictions)

    # Save the best score
    if r2_score > best_score['score']:
        best_score['score'] = r2_score
        best_score['feature_variables'] =  features
        best_score['intercept'] = intercept
        best_score['param'] = params
        best_score['predictions_values'] =  predictions

print "\n\n"
print "The highest score :", best_score['score']
print "Feature variables for the highest score:", ', '.join(best_score['feature_variables'])
print "Coefficient for the highest score"
print "             Intercept:", best_score['intercept']
print "  Non-dummy Parameters:", best_score['param'][:len(best_score['feature_variables'])]


I am working on the file "NYCSubway.ipynb".
The first task is:
I split up the data into two category, with rain and without rain.
I then summarize and plot the distribution for this two samples.
Can you write Python code for it?

gb = df.groupby('rain')

gb.get_group(1).ENTRIESn_hourly

with_rain_df = df['ENTRIESn_hourly'][df['rain'] == 1]
without_rain_df = df['ENTRIESn_hourly'][df['rain'] == 0]

# Let's take a look at the summary of these two samples
summary = pandas.concat([with_rain_df.describe(), without_rain_df.describe()], axis=1)
summary.columns = ['With rain summary', 'Without rain summary']

print summary

plot_description = ('This is the histogram of the ENTRIESn_hourly when it is raining vs when it is not raining.\n'
                    'In both instances, they are not normally distributed and are positively skewed.')

figure, plot_list = plt.subplots(1, 2, sharey=True, figsize=(16, 8))
figure.subplots_adjust(wspace=0.08)
figure.text(0.08, 0.5, 'Frequency', va='center', rotation='vertical')
figure.text(0.5, 0.95, 'Histogram of ENTRIESn_hourly', ha='center')
figure.text(0.5, 0.01, plot_description, ha='center')

df_list = [(with_rain_df, 'Rain'), (without_rain_df, 'No rain')]

for idx, plot in enumerate(plot_list):
    hist, bins = np.histogram(df_list[idx][0], bins=60)
    width = 0.8 * (bins[1] - bins[0])
    center = (bins[:-1] + bins[1:]) / 2
    plot.bar(center, hist, align='center', width=width)
    plot.set_title(df_list[idx][1])
    plot.set_xlabel('ENTRIESn_hourly')


I am working on the file "NYCSubway.ipynb".
The first task is:
It shows that there are some non-linearity in the data.
And this should be addressed by designing a non linear model.
In conclusion, we achieve a R^2 value that we set (> 0.40), but on further inspection we find out that the linear model is not appropriate to predict ridership.
Visualization
Let's use some visualization to ask get some more answer from the data.
First, let's see ridership by day of week.
Can you write Python code for it?

import datetime
from ggplot import *

df_with_weekday = df.copy()
df_with_weekday['weekday'] = df_with_weekday['DATEn'].apply(lambda x: datetime.datetime.strptime(x, '%Y-%m-%d').weekday())

days = ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"]

df_with_weekday = df_with_weekday[['weekday', 'ENTRIESn_hourly', 'EXITSn_hourly']].groupby('weekday').sum().reset_index()
plot = ggplot(aes(x='weekday'), data=df_with_weekday) + \
         geom_bar(aes(x='weekday', y='ENTRIESn_hourly'), stat='bar') + \
         xlab('Day of the week') + \
         ylab('Total entries') + \
         scale_x_continuous(breaks=range(7), labels=days) + \
         ggtitle('Ridership by day of the week')

print plot


I am working on the file "NYCSubway.ipynb".
The first task is:
Now, let's try to see the top 20 station with the highest total entry.
Can you write Python code for it?

unit_entries_df = df[['UNIT', 'ENTRIESn_hourly']].groupby(['UNIT']).sum().sort(['ENTRIESn_hourly'], ascending =[0]).reset_index()[:20]
plot= ggplot(aes(x='UNIT', y='ENTRIESn_hourly'), data=unit_entries_df) + \
         geom_bar(aes(x='UNIT', weight='ENTRIESn_hourly'), stat='bar') + \
         xlab('Station') + \
         ylab('Total entries') + \
         ggtitle('Top 20 ridership per station')

print plot


I am working on the file "NYCSubway.ipynb".
The first task is:
Results
The features variables we use :
rain, meantempi, fog, meanwindspdi, precipi, Hour
The parameters of the non-dummy features are:
-32.26725174, -5.91215877, 120.27580878, 26.27992382, -22.61805652, 67.39739472
The R2 value is 0.458621326664
Intrepretation
In order to see the goodness of fit for the regression model, let's try to plot the residuals
(that is, the difference between the original hourly entry data and the predicted values).
Can you write Python code for it?

figure = plt.figure()
plt.title('Histogram of residual value')
plt.xlabel('Residual value')
plt.ylabel('Relative frequency')

(df['ENTRIESn_hourly'] - best_score['predictions_values']).hist(bins=100)

plot_description = 'Residual value histogram of the model. The plot shows long tails.'
figure.text(0.5, 0, plot_description, ha='center')


I am working on the file "NYCSubway.ipynb".
The first task is:
Statistical test, tail and null hypothesis
I perform Mann-Whitney U test on the two samples. We will be performing a two-tail P value with .05 p-critical value, with the following hypothesis:
The null hypothesis is:
The distributions of both groups are identical, so that there is a 50% probability that an observation from a value randomly selected from one population exceeds an observation randomly selected from the other population.
P(x > y) = 0.5
Alternative hypothesis:
The distributions of both groups are not identical, the probability that an observation from a value randomly selected from one population exceeds an observation randomly selected from the other population is not 50%.
P(x > y) != 0.5
Statistical test reasoning
I am using this test because the samples is non-normal.
Generally, we use Mann-Whitney U test, if it follow several assumptions:

The dependent variable is ordinal or continous
The independent variable (with rain, without rain) is 2 categorical data
There is no relationship between the observations in each group or between the groups themselves
The two variables are not normaly distributed

Result
Below the computed result:
Can you write Python code for it?

import scipy.stats

U, p = scipy.stats.mannwhitneyu(with_rain_df, without_rain_df)
print "U :", U
print "two-tail p :", p * 2  # the module returns one-tail value, we need to multiply by 2 to get two-tail value


I am working on the file "NYCSubway.ipynb".
The first task is:
The histogram has long tails, which suggests that there are some very large residuals. To make sure I plot normal probablity plot.
Can you write Python code for it?

import scipy.stats

figure = plt.figure()
prob_plot = scipy.stats.probplot(df['ENTRIESn_hourly'] - best_score['predictions_values'], plot=plt)

plot_description = ('Normal probability plot of the residual values.\n'
                    'The curve which starts below the normal line,\n'
                    'bends to follow it, and ends above it indicates long tails.')
figure.text(0.5, -0.02, plot_description, ha='center')


I am working on the file "NYCSubway.ipynb".
The first task is:
The plot confirms that we have long tails problem which means we are seeing more variance than we would expect in a normal distribution.
In fact if we simply plot the residual values we got:
Can you write Python code for it?

figure = plt.figure()
plt.title('Residual value plot')
plt.xlabel('Number of data')
plt.ylabel('Residual value (data - prediction)')

plt.plot(df['ENTRIESn_hourly'] - best_score['predictions_values'])

plot_description = 'The plot shows that the residuals follow a cyclical pattern.'
figure.text(0.5, 0, plot_description, ha='center')


I am working on the file "NYCSubway.ipynb".
The first task is:
We can see that the p < .05.
Interpretation
Since our p-value of 0.049999825587 is < 0.05 our critical value.
We conclude that there is statistically significant difference between the distributions of the two sample. This results are significant at the .05 level.

Linear Regression
Now let's build a model so we can make predictions.
Approach
To compute the coefficients theta and produce prediction for ENTRIESn_hourly in the regression model I choose to
use OLS using Statsmodels.
I added 3 helper function here:

A function that does linear regression,
A function that calculate R^2. We need this to check on our result and,
A function that generate list of prediction value based on data and list of features
Can you write Python code for it?

import statsmodels.api as sm

def linear_regression(features, values):
    """
    Perform linear regression given a data set with an arbitrary number of features.
    """
    features = sm.add_constant(features)
    model = sm.OLS(values, features)
    results = model.fit()
    intercept, params = results.params[0], results.params[1:]
    return intercept, params

def compute_r_squared(data, predictions):
    '''
    Compute r_squared given a data set and the predictions.
    '''

    data_mean = np.mean(data)
    ss_res = ((data - predictions)**2).sum()
    ss_tot = ((data - data_mean)**2).sum()
    r_squared = 1 - (ss_res/ss_tot)

    return r_squared

def calculate_predictions(dataframe, features):
    '''
    Generate prediction given dataframe and features.
    '''

    # Add UNIT to features using dummy variables
    dummy_units = pandas.get_dummies(dataframe['UNIT'], prefix='unit')
    features = features.join(dummy_units)

    # Values
    values = dataframe['ENTRIESn_hourly']

    # Get the numpy arrays
    #features_array = features.values
    #values_array = values.values

    # Perform linear regression
    intercept, params = linear_regression(features, values)

    predictions = intercept + np.dot(features, params)

    return predictions, intercept, params


I am working on the file "NYTimes_API_Final.ipynb".
The first task is:

Obtaining the Data

Consumer Confidence Index

New York Times Articles

Article Search API
Peculiarities of the API
Downloading the Data
Working with the Files

Consumer Confidence Index

The consumer confidence index (CCI) is based on survey results of real consumers. They are asked their opinions of current and future economic conditions as well as about their personal economic situation. These survey questions are encoded and normalized to a baseline of 100 coming from the 1985 results. These results are obtained on a monthly basis by the Organisation for Economic Co-operation and Development and can be downloaded directly as a CSV from https://data.oecd.org/leadind/consumer-confidence-index-cci.htm.
New York Times Articles

Article Search API
The New York Times Article Search API allows for searching and obtaining headlines and lead paragraphs of articles dating back to 1851. Along with each article, there is metadata like the date it was published and the section in which it appeared. There is definitely some possibility that not all articles make it into the database, but an inspection of modern articles finds an order of 10s of articles per day which seems reasonable. The API call returns the data as JSON which can be used as such or transformed into CSV.
To access the API, one needs to obtain an API key from https://developer.nytimes.com/signup. And install the API using:
python
! pip install nytimesarticle
Can you write Python code for it?

from nytimesarticle import articleAPI
api = articleAPI('ca372b5c9318406780fe9ebef28e96a1')


I am working on the file "NYTimes_API_Final.ipynb".
The first task is:
Working with the Files
Let's just check what we have in the files now. We can iterate over the yearly CSV files to make a dataframe with all of the data.
Can you write Python code for it?

all_data_list = []
for year in range(1990,1992):
    data = pd.read_csv('{}_Output.csv'.format(year), header=None)
    all_data_list.append(data) # list of dataframes
data = pd.concat(all_data_list, axis=0)
data.columns = ['id','date','headline', 'lead']
data.head()


I am working on the file "Naive Bayes/Naive Bayes with scikit Learn.ipynb".
The first task is:
And don't forget to standardize your featuresets
Can you write Python code for it?

from sklearn.preprocessing import StandardScaler

stdsc = StandardScaler()

X_train_std = stdsc.fit_transform(X_train)

X_test_std = stdsc.transform(X_test)


I am working on the file "Naive Bayes/Naive Bayes with scikit Learn.ipynb".
The first task is:
So from the df.head() function you can see that column 1 contains the label which denotes if it is benign or malignant cancer. From column 2 to column 31 contains the features.
So we are going to prepare our training set in the following lines. X will contain featuresets and y will contain labels of each row
Can you write Python code for it?

X = df.loc[:, 2:].values

y = df.loc[:, 1].values


I am working on the file "Naive Bayes/Naive Bayes with scikit Learn.ipynb".
The first task is:
So here we are. Time for fitting our estimator with the training data.
Can you write Python code for it?

from sklearn.naive_bayes import GaussianNB

clf = GaussianNB()

clf.fit(X_train_std, y_train)

y_pred = clf.predict(X_test_std)


I am working on the file "Naive Bayes/Naive Bayes with scikit Learn.ipynb".
The first task is:
The segment of df is given below. Which shows Upper 5 rows of our data
Can you write Python code for it?


I am working on the file "Naive+Bayes+for+Classification+of+Positive-Negative+reviews.ipynb".
The first task is:
Keywords are chosen for input into the Naive Bayes model. The keywords were chosen by browsing through the raw Amazon dataset and selecting "emotionally-charged" words that might be good predictors. The model was then run, and if the chosen keyword provided no benefit, it was discarded from the list.
Can you write Python code for it?

keywords = ['must have', 'excellent', 'awesome', 'recommend', 'good',
            'great', 'happy', 'love', 'satisfied', 'best', 'works',
            'liked', 'easy', 'quick', 'incredible', 'perfectly',
           'right', 'cool', 'joy', 'easier', 'fast', 'nice', 'family',
           'incredible', 'sweetest', 'poor', 'broke', 'doesn\'t work',
            'not work', 'died', 'don\'t buy', 'problem', 'useless',
            'awful', 'failed', 'terrible', 'horrible', 'the', '10',
           'cool']

for key in keywords:
    # Note that we add spaces around the key so that we're getting the word,
    # not just pattern matching.
    amazon_raw[str(key)] = amazon_raw.message.str.contains(
        '' + str(key) + '',
        case = False
    )


I am working on the file "Naive+Bayes+for+Classification+of+Positive-Negative+reviews.ipynb".
The first task is:
One disadvantage inherent in the naive Bayes model is that it does not work well with keywords which are correlated in the reviews
Can you write Python code for it?

plt.figure(figsize=(15,10))
sns.heatmap(amazon_raw.corr())
plt.title("Correlation between keywords in the model")
plt.show()

data = amazon_raw[keywords]
target = amazon_raw['satisfaction']

# Our data is binary / boolean, so we're importing the Bernoulli classifier.
from sklearn.naive_bayes import BernoulliNB

# Instantiate our model and store it in a new variable.
bnb = BernoulliNB()

# Fit our model to the data.
bnb.fit(data, target)

# Classify, storing the result in a new variable.
y_pred = bnb.predict(data)

# Display our results.
print("Number of mislabeled points out of a total {} points : {}".format(
    data.shape[0],
    (target != y_pred).sum()
))


I am working on the file "Naive+Bayes+for+Classification+of+Positive-Negative+reviews.ipynb".
The first task is:
This set of keywords in the model correctly predicted the outcome 464/748 = 62.0% of the time.
Yelp
This set of keywords in the model correctly guessed the outcome 70.1% of the time.
Can you write Python code for it?

data_path = ("/Users/jacquelynzuker/Desktop/sentiment labelled sentences/yelp_labelled.txt"
            )
yelp_raw = pd.read_csv(data_path, delimiter= '\t', header=None)
yelp_raw.columns = ['message', 'satisfaction']

amazon_raw = yelp_raw


I am working on the file "Naive_Bayes/Supervised+Learning+Naive+Bayes.ipynb".
The first task is:
Now that we have our model, we will continue by seperating into training and testing sets:
Can you write Python code for it?

from sklearn.cross_validation import train_test_split
# Split the data into Trainging and Testing sets
X_train, X_test, Y_train, Y_test = train_test_split(X, Y)


I am working on the file "Naive_Bayes/Supervised+Learning+Naive+Bayes.ipynb".
The first task is:
Now that we have our module imports, let's go ahead and import the Iris Data Set. We have previously worked with this dataset, so go ahead and look at Lectures on MultiClass Classification for a complete breakdown on this data set!
Can you write Python code for it?

# load the iris datasets
iris = datasets.load_iris()

# Grab features (X) and the Target (Y)
X = iris.data

Y = iris.target

# Show the Built-in Data Description
print iris.DESCR


I am working on the file "Naive_Bayes/Supervised+Learning+Naive+Bayes.ipynb".
The first task is:
Now we predict the outcomes from the Testing Set:
Can you write Python code for it?

# Predicted outcomes
predicted = model.predict(X_test)

# Actual Expected Outvomes
expected = Y_test


I am working on the file "Name_Data.ipynb".
The first task is:
2. Basic interactions with the Census dataset API
Combining data from two API endpoints into a human-readable table
The dataset in our example is not excessively large, so we can explore different approaches to interacting with it:

Download some or all data to the computer we're using ('local'). Keep the data in memory and do stuff.
Only download what we need when we need it. Doing stuff may require additional calls to the API.

Both have pros and cons. Both are used in the following examples.
Some points of interest:

The data are not provided in tabular form
Human-readable variable names and definitions are stored separately from the data

In order to make a human readable table we need to:

Download variable definitions
Download data
Replace shorthand variable codes with human readable names
Reformat the data into a table

Get API and variable information:
Can you write Python code for it?

# First, get the basic info about the dataset.
# References: Dataset API (https://api.census.gov/data/2010/surname.html)
#             Requests API (http://docs.python-requests.org/en/master/)
#             Python 3 JSON API (https://docs.python.org/3/library/json.html)

api_base_url = "http://api.census.gov/data/2010/surname"
api_info = requests.get(api_base_url)
api_json = api_info.json()

# Uncomment the next line(s) to see the response content.
# NOTE: JSON and TEXT don't look much different to us. They can look very different to a machine!
#print(api_info.text)
print(json.dumps(api_json, indent=4))

# The output is a dictionary - data are stored as key:value pairs and can be nested.

# Request and store a local copy of the dataset variables.
# Note that the URL could be hard coded just from referencing the API, but
#   we are navigating the JSON data.
var_link = api_json['dataset'][0]['c_variablesLink']
print(var_link)

# Use the variable info link to make a new request

variables = requests.get(var_link)
jsonData = variables.json()
variable_data = jsonData['variables']

# Note that this is a dictionary of dictionaries.
print(json.dumps(variable_data, indent=4))

print(variable_data.keys())


I am working on the file "Name_Data.ipynb".
The first task is:
5. Fix the data type error
Can you write Python code for it?

# First try - just replace string with a zero.
# Here, the for loop iterates through items in a list.
pcts2 = []
for p in pcts:
    if p != '(S)':
        pcts2.append(p)
    else:
        pcts2.append(0)

# Create a pie chart (https://matplotlib.org/2.0.2/examples/pie_and_polar_charts/pie_demo_features.html)
plt.pie(
    # using data percentages
    pcts2,
    # Use labels defined above
    labels=labels,
    # with no shadows
    shadow=False,
    # with the start angle at 90%
    startangle=90,
    # with the percent listed as a fraction
    autopct='%1.1f%%',
    )

# View the plot drop above
plt.axis('equal')

# View the plot
plt.tight_layout()
plt.show()

# Second try - exclude and corresponding label if source data for a given demographic == (S)
# This requires the list index of the data and the label.
# The for loop in this case iterates across a range of integers equal to the length of the list.
pcts3 = []
edit_labels = []
for i in range(len(pcts)):
    print(pcts[i])
    if pcts[i] != '(S)':
        pcts3.append(pcts[i])
        edit_labels.append(labels[i])
    else:
        pass

# Create a pie chart (https://matplotlib.org/2.0.2/examples/pie_and_polar_charts/pie_demo_features.html)
plt.pie(
    # using data percentages
    pcts3,
    # Use labels defined above
    labels=edit_labels,
    # with no shadows
    shadow=False,
    # with the start angle at 90%
    startangle=90,
    # with the percent listed as a fraction
    autopct='%1.1f%%',
    )

# View the plot drop above
plt.axis('equal')

# View the plot
plt.tight_layout()
plt.show()


I am working on the file "Name_Data.ipynb".
The first task is:
Create a dataframe (table) with variable labels as column names and append data
Can you write Python code for it?

df = pd.DataFrame([surname_data[1]], columns=column_list)  # Create a dataframe using the column names created above. Data
                                                           # for the dataframe comes from rows 2-10 (positions 1-9)
                                                           # of surname_data.

# The table we just created is empty. Here we add the surname data:
for surname in d[2:]:
    tdf = pd.DataFrame([surname], columns=column_list)
    df = df.append(tdf)

print('\n\nPandas dataframe:')
df.sort_values(by=["National Rank"])


I am working on the file "Name_Data.ipynb".
The first task is:
Exercises

Change the table to include the 50 most common surnames. Alternatively, create a table for the 11th - 20th most common surnames.
Sort the output table by surname or a demographic.
Change the request/table to include only surname, rank, and count.
Correct the sort order in the example table.

3. Download and tabulate statistics for a given surname
Now let's find out the rank and demographic breakdown of a particular name.
To make it easy to change the name we're looking up, assign it to a variable.
Can you write Python code for it?

# Try 'STEUBEN' in order to break the first pie chart example.
# Update 2020-02-26: Surnames should be all caps!
name = 'WHEELER'
name_query = '&NAME=' + name


I am working on the file "Name_Data.ipynb".
The first task is:
Laying out the API response like a table helps illustrate what we're doing here. For easier reading the "surname_data" variable has been replace with "d" in the image below.
The variable codes in d[0] will be replaced with human readable descriptions from the variable list (v).

Replace variable codes with human readable labels
Can you write Python code for it?

# Pass the data to a Pandas dataframe.
# In addition to being easier to read, dataframes simplify further analysis.

# The simplest dataframe would use the variable names returned with the data. Example: PCTWHITE
# It's easier to read the descriptive labels provide via the variables API.
# The code block below replaces variable names with labels as it builds the dataframe.
column_list = []

for each in surname_data[0]:                        # For each variable in the response data (stored as surname_data[0])
    label = variable_data[each]['label']            #     look up that variable's label in the variable dictionary
    column_list.append(label)                       #     add the variable's label to the list of column headers
    print(each, ":", label)

print('\n', column_list)


I am working on the file "Name_Data.ipynb".
The first task is:
Referring to the variables API, decide which variables are of interest and edit accordingly.
Can you write Python code for it?

# Default vars: 'RANK,COUNT,PCTWHITE,PCTAPI,PCT2PRACE,PCTAIAN,PCTBLACK,PCTHISPANIC'
desired_vars = 'RANK,COUNT,PCTWHITE,PCTAPI,PCT2PRACE,PCTAIAN,PCTBLACK,PCTHISPANIC'


I am working on the file "Natasha/intro-to-data-analysis.ipynb".
The first task is:
.describe() returns useful summary statistics
Can you write Python code for it?


I am working on the file "Natasha/intro-to-data-analysis.ipynb".
The first task is:
A list of built-in aggregatation functions can be found here
Plotting
Pandas provides a built-in plotting command to create matplotlib plots from your dataframes
To display our plots, we need to import first matplotlib
Can you write Python code for it?

import matplotlib.pyplot as plt

grouped['Google Scholar'].mean().plot(kind='bar')
plt.show()

data.plot(x='Google Scholar', y='Both', kind='scatter', alpha=0.6)
plt.ylabel('Total replications and extensions')
plt.xlabel('Number of citations')
plt.grid()
plt.show()


I am working on the file "Natasha/intro-to-data-analysis.ipynb".
The first task is:
Alternatively, we can filter our dataframe (select rows) using boolean conditions
Can you write Python code for it?

data.reset_index(inplace=True)

data['Field'] == 'Development'


I am working on the file "Natasha/intro-to-data-analysis.ipynb".
The first task is:
Bokeh example
Bokeh is a Python library that makes creating interactive plots super easy - an example gallery is here
Can you write Python code for it?

from bokeh.plotting import figure, output_notebook, show
from bokeh.models import ColumnDataSource, HoverTool
output_notebook()

# Add data source
source = ColumnDataSource(data[['Title', 'Web of Science', 'Replications', 'Extensions']])
colors = ['red', 'blue']

# Add title tooltips
hover = HoverTool(tooltips=[
    ("Title", "@Title"),
    ("(x,y)", "($x, $y)")
])

# Create figure
p = figure(tools=[hover])

# Plot scatter
for to_plot, color in zip(['Replications', 'Extensions'], colors):
    p.circle(x='Web of Science', y=to_plot, source=source, size=7, alpha=0.5, legend=[to_plot.title()], color=color)

p.legend.click_policy = 'hide'
p.xaxis.axis_label = 'Number of citations'
p.yaxis.axis_label = 'Number of replications or extensions'

show(p)


I am working on the file "Natasha/intro-to-data-analysis.ipynb".
The first task is:
Exercise
Plot the average number of replications and extensions for each field, sorted from highest to lowest
<button data-toggle="collapse" data-target="#sol9" class='btn btn-primary'>Solution</button>

```python
grouped['Both'].mean().sort_values(ascending=False).plot(kind='bar')
plt.title('Average number of replications/extensions by field')
plt.show()
```

## Another plotting library: `seaborn`

* `seaborn` is a plotting library built on top of `matplotlib`

* It is geared towards producing pretty plots for statistical applications

* You can find an example gallery of `seaborn` plots [here](https://seaborn.pydata.org/examples/index.html)
Can you write Python code for it?

import seaborn as sns  # Import the package

sns.jointplot(x='Google Scholar', y='Both', data=data)
plt.show()

sns.kdeplot(data=data[data['Replications'] > 0].loc[:, 'Web of Science'], cumulative=True, bw=2)
sns.kdeplot(data=data[data['Replications'] == 0].loc[:, 'Web of Science'], cumulative=True, linestyle='--', bw=2)
plt.xlim(0, 200)
plt.xlabel('Web of Science Citations')
plt.ylabel('CDF')
plt.legend(['One or more replications', 'No replications'])
plt.show()


I am working on the file "Natasha/intro-to-data-analysis.ipynb".
The first task is:
Exercises
Find the means of Top 200 and Google Scholar citations
<button data-toggle="collapse" data-target="#sol2" class='btn btn-primary'>Solution</button>

```python
data[['Top 200', 'Google Scholar']].mean()
```
```
Top 200            22.257143
Google Scholar    227.571429
dtype: float64
```

#### Create a table of summary statistics of Top 200 citations and Google Scholar citations

<button data-toggle="collapse" data-target="#sol4" class='btn btn-primary'>Solution</button>

```python
data[['Top 200', 'Google Scholar']].describe()
```

      Top 200
      Google Scholar

      count
      70.000000
      70.000000

      mean
      22.257143
      227.571429

      std
      23.182131
      257.258375

      min
      0.000000
      7.000000

      25%
      9.000000
      73.250000

      50%
      15.000000
      139.000000

      75%
      23.750000
      267.250000

      max
      108.000000
      1246.000000

#### Create a table of summary statistics of the paper's fields (according to the variable `Field`)

<button data-toggle="collapse" data-target="#sol5" class='btn btn-primary'>Solution</button>

```python
data['Field'].describe()
```
```
count                  70
unique                  5
top       Macro/Int/Trade
freq                   25
Name: Field, dtype: object
```

## Selecting and filtering

We can use integer slicing to select rows as follows
Can you write Python code for it?


I am working on the file "Natasha/intro-to-data-analysis.ipynb".
The first task is:
If we want to know the average number of Web of Science citations...
Can you write Python code for it?


I am working on the file "Natasha/intro-to-data-analysis.ipynb".
The first task is:
Note: we haven't actually changed data
Can you write Python code for it?


I am working on the file "Natasha/intro-to-data-analysis.ipynb".
The first task is:
Selecting rows with this boolean condition will return only rows of the dataframe where Field == 'Development' is True
Can you write Python code for it?

data[data['Field'] == 'Development']

data[(data['Field'] == 'Development') & (data['Web of Science'] > 100)]


I am working on the file "Natasha/intro-to-data-analysis.ipynb".
The first task is:
To return an aggregated dataframe, we need to specify the function we would like pandas to use to aggregate our groups
Can you write Python code for it?

grouped.mean()

grouped['Both'].mean()

grouped['Both'].agg(['mean', 'median', 'count'])


I am working on the file "Natasha/intro-to-data-analysis.ipynb".
The first task is:
We can also select multiple columns, returning a new dataframe
Can you write Python code for it?


I am working on the file "Natasha/intro-to-data-analysis.ipynb".
The first task is:
We might want to find the top cited papers
First we will sort our values by a column in the dataframe
Can you write Python code for it?

pd.set_option('max_colwidth', 100)  # adjust column width

data[['Title', 'Google Scholar']].sort_values(by='Google Scholar', ascending=False)[:5]


I am working on the file "Natasha/intro-to-data-analysis.ipynb".
The first task is:
We need to reassign the variable name data to the new copy
Can you write Python code for it?

# data.reset_index(inplace=True)   # Uncomment this if you receive an error

data = data.set_index('Title')   # Can also use data.set_index('Title', inplace=True)
data.head()

data.loc['Matching and Sorting in Online Dating']

data.loc[['Matching and Sorting in Online Dating',
          'Learning about a New Technology: Pineapple in Ghana']]


I am working on the file "Natural Language Processing - SMS Spam Detection.ipynb".
The first task is:
Model evaluation
Can you write Python code for it?

from sklearn.metrics import classification_report, confusion_matrix
from mlxtend.plotting import plot_confusion_matrix

print(classification_report(test_labels, spam_detection_model.predict(test_tfidfmatrix)))


I am working on the file "Natural Language Processing - SMS Spam Detection.ipynb".
The first task is:
Note that the x and y axes for the two plots above are not the same.
Ham messages tend to be shorter than spam messages.
Clean text
Remove punctuation and stopwords from messages then tokenize them.
Can you write Python code for it?

import string
from nltk.corpus import stopwords

def strippunc(message):
    nopunc = [c for c in message if c not in string.punctuation]
    nopunc = ''.join(nopunc)
    return [word for word in nopunc.split()]

def stripstopwords(message):
    nostripwords = [word for word in message.split() if word.lower() not in stopwords.words('english')]
    return ' '.join(nostripwords).split(" ")

def stripuseless(message):
    noextras = [character for character in message if character not in string.punctuation]
    noextras = ''.join(noextras)
    return [word for word in noextras.split() if word.lower() not in stopwords.words('english')]

messages['Message'].head().apply(strippunc)

messages['Message'].head().apply(stripstopwords)

messages['Message'].head().apply(stripuseless)


I am working on the file "Natural Language Processing - SMS Spam Detection.ipynb".
The first task is:
Splitting training and test data
Can you write Python code for it?

from sklearn.model_selection import train_test_split

training_messages, test_messages, train_labels, test_labels = train_test_split(messages['Message'], messages['Label'], test_size = 0.33, random_state=47)


I am working on the file "Natural Language Processing - SMS Spam Detection.ipynb".
The first task is:
Vectorize messages and create term-document matrix
Create bag-of-words model and term-document matrix.
Can you write Python code for it?

from sklearn.feature_extraction.text import CountVectorizer

bagofwords_transformer = CountVectorizer(analyzer=stripuseless)
bagofwords_transformer.fit(training_messages)

print(len(bagofwords_transformer.vocabulary_))

training_termdocmatrix = bagofwords_transformer.transform(training_messages)

training_termdocmatrix.shape

from sklearn.feature_extraction.text import TfidfTransformer

tfidf_transformer = TfidfTransformer()
tfidf_transformer.fit(training_termdocmatrix)

training_tfidfmatrix = tfidf_transformer.transform(training_termdocmatrix)

test_termdocmatrix = bagofwords_transformer.transform(test_messages)
test_tfidfmatrix = tfidf_transformer.transform(test_termdocmatrix)


I am working on the file "Natural Language Processing.ipynb".
The first task is:
"the set of all w such that w is an element of V (the vocabulary) and w has property P".
{w | w ∈ V & P(w)}
[w for w in V if p(w)]
Can you write Python code for it?

V = set(text1)
long_words = [w for w in V if len(w) > 16]
sorted(long_words)


I am working on the file "Natural Language Processing.ipynb".
The first task is:
A collocation is a sequence of words that occur together unusually often. Thus red wine is a collocation, whereas the wine is not. A characteristic of collocations is that they are resistant to substitution with words that have similar senses; for example, maroon wine sounds definitely odd.
To get a handle on collocations, we start off by extracting from a text a list of word pairs, also known as bigrams. This is easily accomplished with the function bigrams():
Can you write Python code for it?

#list(bigrams(['more', 'is', 'said', 'than', 'done']))
text6.collocations()

fdist = FreqDist(len(w) for w in text6)
fdist.most_common()

fdist.plot(cumulative=True)

[len(w) for w in text6]


I am working on the file "Natural Language Processing.ipynb".
The first task is:
Measure the lexical richness of the text ~ 12.8% of the tokens are unique in this text
Can you write Python code for it?

len(set(text6)) / len(text6)

text6.count("Allo")

sentence = ['Melby', 'is', 'the', 'best']
text5[16715:16735]

FreqDist(text6)


I am working on the file "NaturalLanguageProcessing/TextSummarization.ipynb".
The first task is:
Add edges to the graph (weighted by Levenshtein distance)
Can you write Python code for it?

def levenshtein_distance(first,second):
    """ Return the levenshtein distance between two strings.

    http://rosettacode.org/wiki/Levenshtein_distance#Python
    """
    if len(first) > len(second):
        first, second = second, first
    distances = range(len(first)+1)
    for index2, char2 in enumerate(second):
        new_distances = [index2 + 1]
        for index1, char1 in enumerate(first):
            if char1 == char2:
                new_distances.append(distances[index1])
            else:
                new_distances.append(1 + min((distances[index1],
                                                distances[index1+1],
                                                 new_distances[-1])))
        distances = new_distances
    return distances[-1]


I am working on the file "NaturalLanguageProcessing/TextSummarization.ipynb".
The first task is:
Assign POS tags to the words in the text
Can you write Python code for it?

tagged = nltk.pos_tag(word_tokens)

textlist = [x[0] for x in tagged]

# filter_for_tags
defaulttags = ['NN','JJ','NNP']
tagged_filtered = [item for item in tagged if item[1] in defaulttags]

tagged_filtered


I am working on the file "NaturalLanguageProcessing/TextSummarization.ipynb".
The first task is:
Most important words in ascending order of importance
Can you write Python code for it?

keyphrases = sorted(calculated_page_rank, key=calculated_page_rank.get, reverse=True)


I am working on the file "NaturalLanguageProcessing/TextSummarization.ipynb".
The first task is:
Normalize - return a list of tuples with the first item's periods removed.
Can you write Python code for it?

tagged_filtered_normalized = [(item[0].replace('.',''), item[1]) for item in tagged_filtered]

def unique_everseen(iterable, key=None):
    """ List unique elements in order of appearance.

    Examples:
    unique_everseen('AAAABBBCCDAABBB') --> A B C D
    unique_everseen('ABBCcAD', str.lower) --> A B C D
    """
    seen = set()
    seen_add = seen.add
    if key is None:
        for element in [x for x in iterable if x not in seen]:
            seen_add(element)
            yield element
    else:
        for element in iterable:
            k = key(element)
            if k not in seen:
                seen_add(k)
                yield element

unique_word_set=unique_everseen([x[0] for x in tagged_filtered_normalized])

word_set_list = list(unique_word_set)

word_set_list


I am working on the file "NaturalLanguageProcessing/TextSummarization.ipynb".
The first task is:
This will be used to determine adjacent words in order to construct keyphrases with two words
Build Graph
Return a networkx graph instance.
Initialize an undirected graph.
Can you write Python code for it?

gr = nx.Graph()

gr.add_nodes_from(word_set_list)

nodePairs = list(itertools.combinations(word_set_list,2))


I am working on the file "NaturalLanguageProcessing/TextSummarization.ipynb".
The first task is:
Tokenize the text using nltk
Can you write Python code for it?


I am working on the file "NaturalLanguageProcessing/TextSummarization.ipynb".
The first task is:
pageRank -
initial value of 1.0, error tolerance of 0.0001,
Can you write Python code for it?

calculated_page_rank = nx.pagerank(gr, weight='weight')


I am working on the file "Naturales/Naturales.ipynb".
The first task is:
Algunas sumatorias
Calculemos por ejemplo $\sum_{i=1}^n i$
Can you write Python code for it?


I am working on the file "Naturales/Naturales.ipynb".
The first task is:
Con esto podemos definir una suma
Can you write Python code for it?

def suma(m,n):
    if n == 0:
        return m
    return sucesor(suma(m,prec(n)))

suma(2,3)


I am working on the file "Naturales/Naturales.ipynb".
The first task is:
Cuando vamos a utilizar un símbolo tenemos que declararlo primero. Para el ejemplo que sigue vamos a utilizar n como entero e i como contador
Can you write Python code for it?


I am working on the file "Naturales/Naturales.ipynb".
The first task is:
Hagámoslo ahora a mano. La solución tendrá un polinomio característico asociado $(x-2)(x-1)(x-1)^2(x-2)=(x-2)^2(x-1)^3$. Por tanto la solución será de la forma $x_n=r2^n+sn2^n+t 1^n+u n 1^n + vn^2 1^n$.
Can you write Python code for it?

def f(n):
    if n==0:
        return 0
    if n==1:
        return 1
    return 3*f(n-1)-2*f(n-2)+2**n+2*n

[f(i) for i in range(5)]

from diofant import *

r,s,t,u,v = symbols("r,s,t,u,v")
n=Symbol("n", integer=True)
eq = lambda n:r*2**n+s*n*2**n+t+u*n+v*n**2
solve([eq(0)-f(0),eq(1)-f(1),eq(2)-f(2),eq(3)-f(3), eq(4)-f(4)],[r,s,t,u,v])

eq(n).subs(_)

f = lambdify(n,_)

simplify(f(n)-3*f(n-1)+2*f(n-2)-2**n-2*n)


I am working on the file "Naturales/Naturales.ipynb".
The first task is:
Inducción
Veamos cómo podemos utilizar sympy para hacer algunos ejemplos de inducción
Por ejemplo, veamos que para todo $6\mid 7^n-1$ para todo $n\in \mathbb N$. Empezamos definiendo una función que nos devuelva $7^n-1$
Can you write Python code for it?


I am working on the file "Naturales/Naturales.ipynb".
The first task is:
La funciones sucesor y predecesor quedarían como sigue
Can you write Python code for it?

sucesor = lambda x: x+1

sucesor(2)

def prec(n):
    if not isnatural(n):
        raise TypeError("El argumento debe ser un número natural")
    if n==0:
        raise ValueError("El 0 no tiene predecesor")
    return n-1

prec(1)


I am working on the file "Naturales/Naturales.ipynb".
The first task is:
Otro ejemplo más
Can you write Python code for it?


I am working on the file "Naturales/Naturales.ipynb".
The first task is:
Podemos con esta función definir otra que detecte si un objeto es un número es natural
Can you write Python code for it?

def isnatural(n):
    if not isinstance(n,int):
        return false
    return n>=0

isnatural(3)

isnatural(-3)


I am working on the file "Naturales/Naturales.ipynb".
The first task is:
Por lo que $a_n=-2^n+3^n$, como habíamos visto arriba
Veamos un ejemplo de una que no sea homogénea
Can you write Python code for it?


I am working on the file "Naturales/Naturales.ipynb".
The first task is:
Por tanto, si $3\times 7^{2n}+1$ es un múltiplo de $4$, habremos acabado
Can you write Python code for it?


I am working on the file "Naturales/Naturales.ipynb".
The first task is:
Por último, veamos algunos ejemplos de la relación de problemas
Can you write Python code for it?

from diofant import *
n=Symbol("n", integer = True)
a=Function('a')
rsolve(a(n)-2*a(n-1)-n,a(n),[1])


I am working on the file "Naturales/Naturales.ipynb".
The first task is:
Primero veamos qué valor tiene en el 0
Can you write Python code for it?


I am working on the file "Naturales/Naturales.ipynb".
The first task is:
Recursividad e iteración
Veamos cómo las funciones recursivas se pueden acelerar usando memorización de los pasos anteriores
Can you write Python code for it?

fib = lambda n: n if n<2 else fib(n-2)+fib(n-1)

fib(10)

fibonacci(10)

import time

start=time.time()
fib(30)
time.time()-start

start=time.time()
fibonacci(30)
time.time()-start


I am working on the file "Naturales/Naturales.ipynb".
The first task is:
Resolvamos ahora $x_0=0$, $x_1=1$, $x_n=3x_{n-1}-2x_{n-2}+2^n+2n$.
Can you write Python code for it?


I am working on the file "Naturales/Naturales.ipynb".
The first task is:
Si por hipótesis de inducción $f(n)$ es un múltiplo de 6, entonces para probar que $f(n+1)$ también lo es, bastará demostrar que la diferencia $f(n+1)-f(n)$ es un múltiplo de 6
Can you write Python code for it?


I am working on the file "Naturales/Naturales.ipynb".
The first task is:
Si queremos calcular el "valor" de esta sumatoria, podemos utilizar el método doit
Can you write Python code for it?

s.doit()

pprint(_)

summation(i,(i,1,n))


I am working on the file "Naturales/Naturales.ipynb".
The first task is:
Sucesiones, inducción
Con la librería sympy podemos hacer cálculo simbólico. En particular, podemos calcular el valor de algunas sumas con parámetros. Una ramificación de sympy, diofant, permite resolver algunos de los problemas que tenía sympy con rsolve.
Can you write Python code for it?


I am working on the file "Naturales/Naturales.ipynb".
The first task is:
Veamos ahora un ejemplo con término no lineal
Can you write Python code for it?

from diofant import *
n=Symbol("n", integer = True)
x=Function('x')
rsolve(x(n)-2*x(n-1)-3**n,x(n),[1])

f = lambdify(n,_)

f(1)

f(n)-2*f(n-1)

expand(_)


I am working on the file "Naturales/Naturales.ipynb".
The first task is:
Y ahora otro ejemplo no lineal
Can you write Python code for it?

f=a(n)-(n-1)*(a(n-1)+a(n-2))

rsolve(f,a(n),{a(0):1})

simplify(factorial(n)-(n-1)*(factorial(n-1)+factorial(n-2)))


I am working on the file "Naturales/Naturales.ipynb".
The first task is:
Y por tanto la solución general será de la forma $a_n=u 2^n + v 3^n$, para ciertas constantes $u$ y $v$ que tenemos que encontrar a partir de las condiciones iniciales
Imponemos por tanto que $a_0=0=u+v$ y que $a_1=1=2u+3v$
Can you write Python code for it?

u,v = symbols("u,v")
solve([u+v,2*u+3*v-1])


I am working on the file "Nature-Cooperation.ipynb".
The first task is:
Direct Respirocity vs. Always Defect.
Tomorrow never dies!
Payoff matrix:
\begin{align}
\begin{pmatrix}
   & GRIM & ALLD\
  GRIM & m3 &  0+(m-1)1 \
  ALLD & 5+(m-1)1 &  m1 \
  \end{pmatrix}
\end{align}
Where $m$ is expected days which the game will be repeated.
if $3m > 5+(m-1)$ then GRIM is a strict Nash equilibrium when competing with ALLD.
In terms of evolutionary dynamics, if the whole population uses GRIM, then ALLD cannot invade: selection opposes ALLD at low frequency. GRIM is stable against invasion by ALLD if the number of rounds, $m$, exceeds a critical value:
\begin{align}
m> \frac{T-P}{R-P} = \frac{4}{2} = 2
\end{align}
In following widget you can play with the value of $m$ to see how the two strategies perform.
Can you write Python code for it?

def _EvolutionaryGamesProb(v):
  R = 3
  S = 0
  T = 5
  P = 1
  m_ = prob_tomorrow.value
  payoff = [[R*m_,S+(m_-1)*P],[T+(m_-1)*P,m_*P]]
  return evolutionaryGame([0.99,0.01],payoff,["GRIM","ALLD"])

prob_tomorrow = widgets.FloatSlider(
                value=1,
                min=0,
                max=10.0,
                description="m_",layout=widgets.Layout(width='100%', height='80px'))

prob_tomorrow.observe(_EvolutionaryGamesProb,names="value")
display(prob_tomorrow)


I am working on the file "Nature-Cooperation.ipynb".
The first task is:
Evolutionary Games
Two player games
\begin{align}
\dot{x_A} = x_A ~ [f_A(\vec{x}) - \phi ]\
\dot{x_B} = x_B ~ [f_B(\vec{x}) - \phi ]
\end{align}
\begin{align}
f_A(\vec{x}) = a~x_A+b~x_B\
f_B(\vec{x}) = c~x_A+d~x_B
\end{align}
Payoff matrix:
\begin{align}
\begin{pmatrix}
    a &  b \
    c &  d \
  \end{pmatrix}
\end{align}
In following demo you can determine values for $a, b, c$ and $d and see how their values change determine the outcome of the game. You can also run the demo with different number of players.
Can you write Python code for it?

fig = plt.figure()
plt.close(fig)

def evolutionaryGame(x_,f,labels = None):
  np.random.seed(0)
  clear_output(wait=True)
  t_f = 10
  dt = 0.1
  x_ = np.asarray(x_)
  x_ = np.atleast_2d(x_).T
  f = np.asarray(f)

  def int_(t,x):
    mean = x.T.dot(f.dot(x))
    dev = x*(f.dot(x)-mean)
    return dev

  integ = integrate.ode(int_)
  integ.set_integrator("dopri5").set_initial_value(np.asarray(x_))
  y = np.zeros((int(t_f/dt)+1,x_.shape[0]))
  x = np.zeros((int(t_f/dt)+1,x_.shape[0]))
  xdot = np.zeros((int(t_f/dt)+1,x_.shape[0]))

  i = 0
  while integ.successful() and integ.t<t_f:
    y[i,:] = integ.y[:,0]
    x[i,:] = integ.t
    xdot[i,:] = int_(integ.t,y[i,:])
    integ.integrate(integ.t+dt)
    i=i+1

  fig.clf()
  ax = fig.gca()
  with sns.color_palette("PuBuGn_d",n_colors=2):
    for ind_ in range(x.shape[1]):

      ax.plot(x[:,ind_], y[:,ind_], label="Type: %d" % (ind_+1) if labels is None else labels[ind_])

  ax.set_ylim(-0.1,1.1)
  ax.set_xlabel("time")
  ax.set_ylabel("Quasi specie")
  ax.legend(ncol=min(4,x.shape[1]))

  display(fig)

items_strat = [
  widgets.IntText(
    value=2,
    min=2,
    max=5.0,
    description="r",layout=widgets.Layout(width='50%', height='80px'))
  ,widgets.Button(
    description="submit")]

def _EvolutionaryGames(v):
  init = []
  payoff = []
  for ind_,obj in enumerate(objects_strat):
    if ind_ < len(objects_strat)-1:
      init.append(obj[0].value)
    else:
      if sum(init)>1:
        status_labelstrat.value = "Initial rates should sum to <1"
        return
      else:
        status_labelstrat.value = ""
      init.append(1-sum(init))
    rate_ = []
    for j in range(0,len(objects_strat)):
      rate_.append(obj[j+1].value)
    payoff.append(rate_)
  init = np.asarray(init)
  payoff = np.asarray(payoff)
  if len(objects_strat)==3:
    status_labelstrat.value = "Determinant: %.2f" % linalg.det(payoff)
  return evolutionaryGame(init,payoff)

objects_strat = []
status_labelstrat = None
box_vstrat = None

def call_back_mute(v):
  global...