df1 = pd.read_csv('tobag/11884.csv')
df2 = pd.read_csv('tobag/12003.csv')
df3 = pd.read_csv('tobag/12017.csv')
df4 = pd.read_csv('tobag/12300.csv')
df5 = pd.read_csv('tobag/12488.csv')
df6 = pd.read_csv('tobag/12553.csv')
df7 = pd.read_csv('tobag/12628.csv')
df8 = pd.read_csv('tobag/12769.csv')
df9 = pd.read_csv('tobag/12792.csv')
df10 = pd.read_csv('tobag/13235.csv')
df11 = pd.read_csv('tobag/13462.csv')
df12 = pd.read_csv('tobag/13484.csv')
df13 = pd.read_csv('tobag/13491.csv')
df14 = pd.read_csv('tobag/13550.csv')
df15 = pd.read_csv('tobag/13580.csv')
df16 = pd.read_csv('tobag/13913.csv')
df17 = pd.read_csv('tobag/13945.csv')
df18 = pd.read_csv('tobag/13962.csv')

results = (df1.Sales * 0.985 + df2.Sales * 0.985 + df3.Sales * 0.985 + \
            df4.Sales * 0.985 + df5.Sales * 0.985 + df6.Sales * 0.985 + \
            df7.Sales * 0.985 + df8.Sales * 0.985 + df9.Sales * 0.985 + \
            df10.Sales * 0.985 + df11.Sales * 0.985 + df12.Sales * 0.985 + df13.Sales * 0.985 + \
            df14.Sales * 0.985 + df15.Sales * 0.985 + df16.Sales * 0.985 + \
            df17.Sales * 0.985 + df18.Sales) / 18.0

results = pd.DataFrame(results)
results.T
results.columns = ['Sales']
results.index += 1
results.to_csv('bagged.csv', index_label='Id')


I am working on the file "MeetupPrez.ipynb".
The first task is:
Create dummy columns

Categorical data is handled as dummy columns
At this point we create a validation set
Can you write Python code for it?

#create dummy columns function creates dummy columns where values are value+_+columns name
train = rosshellfun.create_dummy_columns(train, ['StoreCluster','DayOfWeek','DaysLeft','DaysAfter'])
test = rosshellfun.create_dummy_columns(test, ['StoreCluster','DayOfWeek','DaysLeft','DaysAfter'])

train.head(5)

# note Id is not sorted now
test = test.sort('Id')
test.head(5)

# set up training features

# from this we have to remove the original categorical and the string columns
# AND the customers column that is not available in the test
# Gert predicted the customers also and used that to further help his prediction on the sales
numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']

features = train.select_dtypes(include=numerics).columns.values.tolist()
features.remove('Customers')
features.remove('Store') # categorical but is numeric
features.remove('Sales') # Of course let's remove the outcome
print(features)

from sklearn.cross_validation import train_test_split

# create test set
X = train.loc[:,features]
y = train.loc[:,'Sales']
train_base_x, train_validation_x, train_base_y, train_validation_y = train_test_split( X, y, test_size = 0.2 )

# Note Gert here did not randomly chose the hold out set but
# he chose consecutive days


I am working on the file "MeetupPrez.ipynb".
The first task is:
Creating store categories based on sales similarity

We have 1000+ stores
It is reasonable to assume that the covariation is large
It is reasonable to assume that there are similar stores (store type group)
This is a posteriori grouping (We had storetype already in the data)
We used hierarchical clustering
Can you write Python code for it?

table = pd.pivot_table(train.loc[:,['Date','Sales','Store']],index = ['Date'], columns = ['Store'], values = ['Sales'] )

store_type_labels = rosshellfun.cluster_all_stores(table, cut_thresh = 0.05, method = 'ward')
# ward's method focuses on variance and not distance itself


I am working on the file "MeetupPrez.ipynb".
The first task is:
Data exploration – Closing periods
- Short closing periods (i.e. weekend) are visible on the day of week plot
- But what about longer periods? (renovation, etc.)
Can you write Python code for it?

for fest in ['a','b','c']:
    print('Max. consecutive day(s) for holiday type "%s" is %i'
          % (fest, rosshellfun.longest_streak(train['StateHoliday'].values, fest)))

print('Max. consecutive day(s) where a store was marked as closed is %i!'
      % rosshellfun.longest_streak(train['Open'].values, 0))
print('Max. consecutive day(s) where a store was marked with zero sales is %i!'
      % rosshellfun.longest_streak(train['Sales'].values, 0))


I am working on the file "MeetupPrez.ipynb".
The first task is:
Data exploration – days of month
„Payday waves”, otherwise stable
Can you write Python code for it?

fig, (axis1) = plt.subplots(1, 1, figsize=(15,7))
sns.boxplot(x='Day',y='Sales', data=train_plot.sort('Day'), palette="husl", ax=axis1)


I am working on the file "MeetupPrez.ipynb".
The first task is:
Example store
Can you write Python code for it?

plt.plot(train.loc[train['Store'] == 708, 'Sales'], 'b.')
plt.ylim(ymax=20000, ymin=200)


I am working on the file "MeetupPrez.ipynb".
The first task is:
Exploration and preparation
- Explore the data
- Drop bad values and impute missing ones
- Feature widening
Data exploration – days of week
Sundays (where open) and Mondays dominate
Can you write Python code for it?

fig, (axis1) = plt.subplots(1, 1, figsize=(15,4))
train_plot = train[train["Open"] == 1]
sns.boxplot(x='DayOfWeek',y='Sales', data=train_plot.sort('DayOfWeek'), palette="husl", ax=axis1)
plt

fig, (axis1) = plt.subplots(1, 1, figsize=(15,4))
train_plot = train[train["Open"] == 1]
sns.boxplot(x='DayOfWeek',y='Sales', data=train_plot.sort('DayOfWeek'), palette="husl", ax=axis1)
plt.ylim(ymax=12500, ymin=3000)


I am working on the file "MeetupPrez.ipynb".
The first task is:
Let's check if everything is added correctly

This is again a crucial step
'Pandas eat only indexes'
Can you write Python code for it?

# add store_type_labels to training and test
train = pd.merge(train, pd.DataFrame([store_type_labels,table.columns.get_level_values(1)],
                             columns = table.columns.get_level_values(1),
                             index = ['StoreCluster','Store'] ).T, on='Store')
test = pd.merge(test, pd.DataFrame([store_type_labels,table.columns.get_level_values(1)],
                             columns = table.columns.get_level_values(1),
                             index = ['StoreCluster','Store'] ).T, on='Store')

train.loc[:,['Store','StoreCluster']].drop_duplicates(inplace = False).sort('Store').head(10)

test.loc[:,['Store','StoreCluster']].drop_duplicates(inplace = False).sort('Store').head(10)


I am working on the file "MeetupPrez.ipynb".
The first task is:
Models
What models have we tried?

Linear regression, Ridge, Elasticnet, SVM etc. (we are not going to talk about these)
Trees, forests (we are not going to talk about these either)
Gradient Boosting (scikit)
Neural networks (keras + theano)

Gradient boosting
Gradient boosting is the Holy Grail of several machine learning problems today
- Almost every winning solution on Kaggle used it
Gradient boosting is practically a huge ensemble of decision trees
- The term gradient refers to gradient descent
- Each tree is a step on the gradient
- Adds weight to training examples
- Learning rate ~ number of trees
Strong regularization:
- Control model complexity:
   - tree size and structure
- Add randomness to the training
   - subset of features, subset of samples etc.
The difference between scikit GBM and XGBoost is... well actually as far as I see XGBoost gives you a little more option in logistic regression problems (e.g. max delta step)
Why gradient boosting

Handles well features of different scales
Support different loss functions
Can handle complex feature interactions and patterns

However:
1. Slow to train
2. Often comes as a black box
How it works
The objective function consists of the sum of the Loss and Regularization parameter ($\Omega$). So:
$$Objective(\Theta) = Loss(\Theta) + \Omega(\Theta)$$
Where $\Theta$ is the parameter set to be defined.
The loss function that we used was
$$\begin{align}
Loss(\Theta) = \left{ \begin{array}{cl}
\frac{1}{2} \left[y-\hat{y}\right]^2 & \text{for }|y-\hat{y}| \le \alpha, \
\alpha \left(|y-\hat{y}|-\alpha/2\right) & \text{for }|y-\hat{y}| \gt \alpha
\end{array}\right.
\end{align} $$
Where $\alpha$ was equal to $Q(.56)$ (that is ~median). This loss function applies squared loss for small residuals, but absolute loss for larger. Could be better then squared loss, when outliers would pull the squared loss from optimum.
Huber, P. J. (1964). Robust estimation of a location parameter. The Annals of Mathematical Statistics, 35(1), 73-101.
The hyperparameters
There are...
Can you write Python code for it?

import random
from sklearn.grid_search import RandomizedSearchCV
from sklearn import ensemble
from scipy.stats import uniform, randint

# it is not necessary to search the parameters using the complete training set
training_subset = random.sample(range(len(train_base_x)),800) # this is intentionnaly so low it was 80000 in the competition
train_sample_x = train_base_x.iloc[training_subset,]
train_sample_y = train_base_y.iloc[training_subset,]

# note, this is not the whole parameter set of scikit (neither of XGBoost)
parameters = { 'learning_rate': uniform(), #0.426348401220647
               'max_depth' : [4, None], # None
               'min_samples_leaf' : randint(10, 100), #45
               'loss' : ['ls', 'lad', 'huber', 'quantile'], #'huber'
               'subsample' : uniform(), #0.7675104474139473
               'alpha': uniform()} #0.560116434370429

model = ensemble.GradientBoostingRegressor(n_estimators = 100, verbose = 0)

params_found = RandomizedSearchCV(model, param_distributions = parameters,
                               n_iter = 100, n_jobs = -1).fit(train_sample_x,train_sample_y)
params_found.best_params_


I am working on the file "MeetupPrez.ipynb".
The first task is:
Neural Networks and Deep Learning

Can we use neural networks/deep learning for time series prediciton?
“Yes, you can kill a mouse with a rocket propelled grenade. And yes, you can use neural networks on time series.”
(answer from Quora)
Building a Model is not necessarily easy

Weight initialization

Activation functions

Size of network

Number of layers
Number of neurons per layer

Normalization

Dropout
Batch Normalization

Optimizers

Batch Size

We used Keras
- Keras is a deep learning framework that runs on top of Theano or TensorFlow
    - Theano is one of the most three most important deep learning frameworks
    - (others important frameworks being Torch and Caffee)
    - TensorFlow is Google’s deep learning framework, that was made publicly available last year
Weight initialization
Weight initialization defines the probability distribution used to set the initial random weights of layers.
Available methods in Keras:

uniform
lecun_uniform: Uniform initialization scaled by the square root of the number of inputs (LeCun 98)
normal
identity: Use with square 2D layers (shape[0] == shape[1])
orthogonal: Use with square 2D layers (shape[0] == shape[1])
zero
glorot_normal: Gaussian initialization scaled by fan_in + fan_out (Glorot 2010)
glorot_uniform
he_normal: Gaussian initialization scaled by fan_in (He et al., 2014)
he_uniform

Activation functions
Activations that are more complex than a simple Theano/TensorFlow function (eg. learnable activations, configurable activations, etc.) are also available in Keras:

Relu
LeakyReLU
PReLU
Parametric Rectified Linear Unit (PReLU). This activation function adaptively learns the parameters of the rectifiers, and improves accuracy at negligible extra computational cost
http://arxiv.org/pdf/1502.01852v1.pdf

Optimizers

SGD
Stochastic gradient descent, with support for momentum, decay, and Nesterov momentum.

RMSProp
Adagrad
Adadelta
Adam
the name Adam is derived from adaptive moment estimation
http://arxiv.org/pdf/1412.6980v8.pdf
Can you write Python code for it?

from keras.models import Sequential
from keras.layers.core import Dense, Activation, Dropout
from keras.optimizers import Adam
from keras.layers.advanced_activations import PReLU
from keras.layers.normalization import BatchNormalization

model = Sequential()
model.add(Dense(50, input_dim=1242, init='he_normal'))
model.add(PReLU())
#model.add(BatchNormalization())
#model.add(Dropout(0.5))

model.add(Dense(50, init='he_normal'))
model.add(PReLU())
#model.add(BatchNormalization())
#model.add(Dropout(0.5))

model.add(Dense(1, init='he_normal'))
model.add(Activation('sigmoid'))

adam = Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-8)
model.compile(loss='mse', optimizer=adam)


I am working on the file "MeetupPrez.ipynb".
The first task is:
Scaling relevant features for use in neural networks
Can you write Python code for it?

def featureScaling(arr):
    min_arr = min(arr); max_arr = max(arr); new_arr = []
    for i in arr:
        new_arr.append((i - min_arr) / (max_arr - min_arr))
    return new_arr

class Vectorizer():
    def __init__(self):
        self.vectCache = dict()

    def vect(self, size, idx):
        """Retrieves a vector such as [0,0,0,...,idx=1,0,0,...,0] where the size of the vector is 'size'
        and all items are zero except the 'idx' numbered element, which is one.
        Note: 'idx' numbering starts from 1..size, not from zero, it is 1-indexed.
        """
        if size not in self.vectCache:
            print("Vectorizer: Generating vectors for size", size)
            arr = []
            for i in range(size):
                e = [0] * size
                e[i] = 1.0
                arr.append(e)
            self.vectCache[size] = arr
        return self.vectCache[size][idx-1]

v = Vectorizer()


I am working on the file "MeetupPrez.ipynb".
The first task is:
Scaling train file
Can you write Python code for it?

df = pd.read_csv('train_extended.csv')
df.Sales = df.Sales.apply(np.log1p)
df.Customers = df.Customers.apply(np.log1p)
df['CompetitionDistanceLog'] = df.CompetitionDistance.apply(np.log1p)
df.CompetitionOpen = df.CompetitionOpen + 31
df['CompetitionOpenLog'] = df.CompetitionOpen.apply(np.log1p)
df.EurUsdRate = df.EurUsdRate - 0.7
df.CompetitionOpenNull = df.CompetitionOpenNull + 31
df['CompetitionOpenNullLog'] = df.CompetitionOpenNull.apply(np.log1p)

new = df

a = new['Sales']
b = new['Customers']
c = new['CompetitionDistance']
d = new['CompetitionOpen']
e = new['PromoOpen']
f = new['CompetitionOpenNull']
g = new['CompetitionDistanceLog']
h = new['CompetitionOpenLog']
i = new['CompetitionOpenNullLog']

a_scaled = featureScaling(a)
b_scaled = featureScaling(b)
c_scaled = featureScaling(c)
d_scaled = featureScaling(d)
e_scaled = featureScaling(e)
f_scaled = featureScaling(f)
g_scaled = featureScaling(g)
h_scaled = featureScaling(h)
i_scaled = featureScaling(i)

new['Sales'] = a_scaled
new['Customers'] = b_scaled
new['CompetitionDistance'] = c_scaled
new['CompetitionOpen'] = d_scaled
new['PromoOpen'] = e_scaled
new['CompetitionOpenNull'] = f_scaled
new['CompetitionDistanceLog'] = g_scaled
new['CompetitionOpenLog'] = h_scaled
new['CompetitionOpenNullLog'] = i_scaled

new.to_csv('scaled_train.csv', na_rep='0', header=True, index = False)


I am working on the file "MeetupPrez.ipynb".
The first task is:
Summary of the gradient boosting

It is a good, transparent (at least we think), realiable solution for regression
This model was not able to compete with the highest ranked models
Reason is that normally they used additional steps (such as Customers number prediction)
We made a mistake that took a long time to find: the dummy columns for the daysofweek were converted at some point to float, and so this feature was not included in the prediction
Can you write Python code for it?

test['Sales'] = model.predict(test.loc[:,features])
test.loc[test['Open'] == 0,'Sales'] = 0
test.loc[:,['Id','Sales']].set_index('Id',drop = True).to_csv('submission_gbrt.csv')


I am working on the file "MeetupPrez.ipynb".
The first task is:
Train Test Split
Can you write Python code for it?

train = pd.read_csv('scaled_train.csv')
shape =  train.shape

#shuffle data set, and split to train and test set.
df = pd.DataFrame(train)
shuffled_train = df.reindex(np.random.permutation(df.index))

indice_5_percent = int((shape[0]/100.0)* 5)
indice_10_percent = int((shape[0]/100.0)* 10)
indice_15_percent = int((shape[0]/100.0)* 15)

#for experimenting
shuffled_train[:indice_5_percent].to_csv('test_5_split.csv', index=True, columns=train.columns)
shuffled_train[indice_5_percent:indice_10_percent].to_csv('train_5_split.csv', index=True, columns=train.columns)

#for full scale training
shuffled_train[:indice_15_percent].to_csv('test_15_split.csv', index=True, columns=train.columns)
shuffled_train[indice_15_percent:].to_csv('train_85_split.csv', index=True, columns=train.columns)


I am working on the file "MeetupPrez.ipynb".
The first task is:
Training the neural network
Can you write Python code for it?

import csv
import time

training_subset_size = 10000
loss = 1
losslog = []
Time = (time.strftime("Day:%d Time:%H:%M"))

df = pd.DataFrame.from_csv(train_file)
print ("Assembling training set and training neural network...")

for epoch in range(1, num_epochs + 1):
    n = 1
    i = 1
    list_a = []
    list_b = []

    df = df.reindex(np.random.permutation(df.index))
    for index, row in df.iterrows():
        if i <= training_subset_size:
            if row['Open'] == 1:
                features = feature_list()
                list_a.append(features)
                list_b.append(row['Sales'])
                i = i+1

        else:
            x = np.array(list_a, dtype=float)
            y = np.array(list_b, dtype=float)
            model.fit(x, y, batch_size=batch_size, nb_epoch=1, verbose=0)

            i = 1
            list_a = []
            list_b = []

        if (n % 10000 == 0):
            print (n)
        n = n + 1

    '''
    Evaluating model
    '''
    n = 1
    list_a = []
    list_b = []
    with open(test_file) as csvfile:
        reader = csv.DictReader(csvfile)
        for row in reader:
            features = feature_list()
            list_a.append(features)
            list_b.append(row['Sales'])

            if (n % 10000 == 0):
                print (n)
            n = n + 1

        test_x = np.array(list_a, dtype=float)
        test_y = np.array(list_b, dtype=float)

        print ("Performance of the model over validation data (loss score): \
        " + str(model.evaluate(test_x, test_y, verbose=2)) + " at epoch:" + str(epoch))

        if logging == True:
            losslog.append(("Performance of the model over validation data (loss score): \
            " + str(model.evaluate(test_x, test_y, verbose=2)) + " at epoch:" + str(epoch) \
            + "_"+time.strftime("Day:%d Time:%H:%M")))
            losslog_df = pd.DataFrame(losslog)
           ...


I am working on the file "Mention/Mention_example.ipynb".
The first task is:
1. Find articles
The fist goal to achieve is to find all the Italian and Portuguese Wikipedia articles that mention Matteo Renzi. In order to do so, we use the WikiHandler class which goes through the raw data and then keeps and stores the title and the text of the elements of the corpora that mention the Italian (almost) ex-Prime Minister.
In this example we focus on the sets of articles written in Italian and in Portugal, they are collected respectively until 20th November 2016 and 1st December 2016. In general, the code allows you to take into account more than two languages. In the README file, you can find the information related to the collection of data.
Can you write Python code for it?

# Define the path of the corpora
path = '/Users/cristinamenghini/Downloads/'
# Xml file
xml_files = ['itwiki-20161120-pages-articles-multistream.xml',
             'ptwiki-20161201-pages-articles-multistream.xml']


I am working on the file "Mention/Mention_example.ipynb".
The first task is:
A scatterplot is used to get how an article is positioned according to these two variables. The plot shows:
* IT: when the mentions are equal to 1 the number of page views is spread between 0 and ~20k. Where the number of mentions increases the number of page visualizations belongs to a smaller range.
* PT: also for Portuguese article the same is observed.
Can you write Python code for it?

# def scatter_plot(df_it_mension_pageview, df_pt_mension_pageview, 'Number of mentions', 'Pageviews', 'Italian', 'Portuguese')
tls.embed('https://plot.ly/~crimenghini/36')


I am working on the file "Mention/Mention_example.ipynb".
The first task is:
About these two features, we can think that another way to explore should be the following:

Consider how the number of pageviews of an article changes when the number of Matteo Renzi citations increases from a revision to another. In particular, the importance(I) is re-defined as:
$$I = \sum_{t = 1}^{T} \frac{(p_t-p_{t-1}) \times m_t}{|D_t|}$$
Where t is the time of sequential revision of the article, p is the number of page views at time and m is the number of mentions.

Thus we proceed to look for the presence of same articles (in different languages) that mention Matteo Renzi. To do so we make a request for each Portugues Wikipedia page (that cites Renzi) than we parse the HTML source to extract - where available- the title of the IT article related to that the request has been sent. Precisely, the requests are sent for each title of the language that has less article that match Matteo Renzi. The function get_matches is stored in this library.
Can you write Python code for it?

# Built the common articles matches
dict_italian = get_matches(df_pt_titles, 'it')
# Create the inverted one
inverted_dict = {v : k for k, v in dict_italian.items()}

print ('The Portuguese articles that mention Matteo Renzi and correspond to an Italian article are: ', len(dict_italian),
       '. The number of PT articles that have not been matched is: ', len(df_pt_titles)-len(dict_italian), '.')


I am working on the file "Mention/Mention_example.ipynb".
The first task is:
Add a column containing the matches to join the two dfs.
Can you write Python code for it?

# Create new column
new_column_it = ['/'.join([k]+[v]) for i in df_match_it_mentions.Title for k,v in dict_italian_mentions.items()  if i == v]
new_column_pt = ['/'.join([k]+[v]) for i in df_match_pt_mentions.Title for k,v in dict_italian_mentions.items()  if i == k]

# Add the new column to the two dataframes
df_match_it_mentions['Matches'] = new_column_it
df_match_pt_mentions['Matches'] = new_column_pt


I am working on the file "Mention/Mention_example.ipynb".
The first task is:
Add new variable to allow the join
Can you write Python code for it?

# Create new column
new_column_it = ['/'.join([k]+[v]) for i in df_match_it_pageviews.Title for k,v in dict_italian_pageviews.items()  if i == v]
new_column_pt = ['/'.join([v]+[k]) for i in df_match_pt_pageviews.Title for k,v in dict_pt_pageviews.items()  if i == v]

# Add the new column to the two dataframes
df_match_it_pageviews['Matches'] = new_column_it
df_match_pt_pageviews['Matches'] = new_column_pt

df_match_it_pageviews.head()

df_match_pt_pageviews.head()


I am working on the file "Mention/Mention_example.ipynb".
The first task is:
After having a quick peek at a snippet of the XML. The elements we are interested in are on the child page, which identifies an article. Then we want to get the contents of title and text.
Due to the big size of the XML we opted for a parser which registers callbacks for events of interest and then let the parser proceed through the document. The text of the article has not been preprocessed since for the purpose of our analysis we are not going to analyze the text in itself.
Hence, we proceed to parse the Italian corpus using the parse_articles function stored in the wiki_parser library - it basically activates the parser.
Can you write Python code for it?

# Parse italian corpus
parse_articles('ita', path + xml_files[0], 'Matteo Renzi')


I am working on the file "Mention/Mention_example.ipynb".
The first task is:
Comparing the two DataFrames we immediately notice that even if the maximum number of mentions that Matteo Renzi received for Italian and Portuguese articles are very different. In the Portuguese corpus there are only two articles that have more than 5 mentions. Thus, can be interesting to visualize the distribution of the mentions both for the IT and PT corpora.
The distributions are represented using the boxplots. They show that for both the languages the 75% of the articles contain no more than 3 mentions of the Italian premier. For the Portuguese corpus stand out two outliers that correspond to Matteo Renzi 11 mentions and Partido Democrático (Itália) 6 mentions, rather for the Italians the number of outliers is bigger and the maximum number of mentions are contained in Matteo Renzi 62 mentions. Moreover, zooming in the boxes, we observe that the two distributions are skewed toward left (number of mentions equal to 1).
Can you write Python code for it?

#boxplot_mentions(df_pt_mentions, df_it_mentions, 'PT', 'IT', 'Number of mentions')
tls.embed("https://plot.ly/~crimenghini/20")


I am working on the file "Mention/Mention_example.ipynb".
The first task is:
Define the matching dictionaries according to what said above.
Can you write Python code for it?

# Re-define the two dictionaries according to this evidence
dict_italian_pageviews = {k:v for k,v in dict_italian.items() if v in it_titles_with_pageviews}

# PT
dict_pt_pageviews = {v : k for k, v in dict_italian.items() if k in pt_titles_with_pageviews}

# Create df for IT mentions
df_match_it_pageviews = ranked_df_ita.query('Title in @it_titles_with_pageviews').sort_values('Pageviews', ascending = False)

# Create df for PT mentions
df_match_pt_pageviews = ranked_df_port.query('Title in @pt_titles_with_pageviews').sort_values('Pageviews', ascending = False)


I am working on the file "Mention/Mention_example.ipynb".
The first task is:
From the plot:
* Among this group of articles, the two that mention Matteo Renzi more result to be the same.
From this kind of analysis a question one can think about is the following:

Given articles in different languages that correspond one to each other, if we are interested in measuring the proximity of these articles, an element that may be considered is the number of common mentions. It is likely that the necessity of quoting s.o./s.t. derives from the fact that the two articles are talking about the same topics that need to refer to the same thing.

The same procedure is repeated for the page views.

Check whether some articles have not been visited.
Can you write Python code for it?

# Run the query
match_with_pageviews_it = ranked_df_ita.query('Title in @italian_titles')
match_with_pageviews_pt = ranked_df_port.query('Title in @portugues_titles')
# Get the number
print ('There are ', len(portugues_titles)-len(match_with_pageviews_it), 'IT articles that have not been visited.')
print ('There are ', len(portugues_titles)-len(match_with_pageviews_pt), 'PT articles that have not been visited.')

# Define list of articles that have been visualized
it_titles_with_pageviews = list(match_with_pageviews_it.Title)
pt_titles_with_pageviews = list(match_with_pageviews_pt.Title)


I am working on the file "Mention/Mention_example.ipynb".
The first task is:
Having a quick glance at the two top 10, we notice:
* The number of page views for the Italian articles which mention Matteo Renzi is considerably higher than for those that are written in Portuguese.
* The only article that is present in both the top ranking is Partito Democratico (Italia).
* It seems that the pages differ in the content: the Portuguese ones are more related to topics that regard the international politics rather the Italians that refer to politics, journalists and public figures.
3. Make comparisons
We now move ahead exploring the data that we preprocessed and trying to figure out something interesting.
* We take a look at the number of mentions received in each article. In this contest, it may be possible that Matteo Renzi received more than one mention just because of the presence of references. For instance on this page, if you look up for Matteo Renzi, you will find 2 mentions but one of those just refers to the first. For the moment we do not address this issue.
The DataFrame below- obtained using article_mentions function in this library- shows the number of mentions that Matteo Renzi has received in each article according to both for the Italian and Portuguese corpora. The DataFrames are sorted by the number of mentions so that we get the pages where Matteo Renzi is more "popular".
Can you write Python code for it?

# Italian df of mentions per page
df_it_mentions = article_mentions('Corpus/wiki_ita_Matteo_Renzi.json', 'Matteo Renzi')

# Sort the df by the number of mentions and see the top 5
df_it_mentions = df_it_mentions.sort_values('Number of mentions', ascending = False)

# Show results
df_it_mentions.head(5)

# Portuguese df of mentions per page
df_pt_mentions = article_mentions('Corpus/wiki_port_Matteo_Renzi.json', 'Matteo Renzi')

# Sort the df by the number of mentions and see the top 5
df_pt_mentions = df_pt_mentions.sort_values('Number of mentions', ascending = False)

# Show results
df_pt_mentions.head(5)


I am working on the file "Mention/Mention_example.ipynb".
The first task is:
In this direction, one aspect that can be considered is the following:

Define how important is Matteo Renzi in the articles that mention him. It requires defining the concept of importance. Intuitively, we would say that higher is the number of mentions more is the importance of our object in the article. Moreover, it may be useful to weight the number of mentions according to the number of words in the article.
$$I_{string} = \frac{M}{|D|}$$
Where I is the importance, M is the number of mentions and D the number of words in the document. In this way, whether an article cited Renzi once but it is made up just by a few lines, the string of interest will result more significant.

Moreover, another aspect should be considered, especially when there is only one mention:
* The string (i.e. Matteo Renzi) is a pointer to its main page (i.e. Matteo Renzi -> Matteo Renzi). Whether the pointer is present we can imagine that the figure is more important than a page where there is no a hyperlink.
Another thing that can be visualized is the realtionship between the Number of mentions and the Pageviews. In order to do that we first merge the two pageviews and mentions DataFrames.
Can you write Python code for it?

# Merge pageviews and mentions DataFrames for IT
df_it_mension_pageview = pd.merge(df_it_mentions, ranked_df_ita, on=['Title'])

# Show it
df_it_mension_pageview.sample(5)

# Merge pageviews and mentions DataFrames for PT
df_pt_mension_pageview = pd.merge(df_pt_mentions, ranked_df_port, on=['Title'])

# Show it
df_pt_mension_pageview.sample(5)


I am working on the file "Mention/Mention_example.ipynb".
The first task is:
Perform the join on the Matches and plot the results.
Can you write Python code for it?

# Join the two dfs on the correspondence tuples
matches_mention = pd.merge(df_match_it_mentions, df_match_pt_mentions, on = 'Matches', suffixes = ('_IT','_PT'))

# Show result
matches_mention.head()

# bar_plot(df, 'Matches', 'Number of mentions_IT', 'Number of mentions_P', 'IT', 'PT', 'Compare IT and PT mentions',
# 'Article','No. mentions', 'color-bar-prova')
tls.embed('https://plot.ly/~crimenghini/38')


I am working on the file "Mention/Mention_example.ipynb".
The first task is:
The articles are filtered according to the presence of the mention to Matteo Renzi, those in Italian have been stored in a .json file whose each line corresponds to a page (title, text). The same holds for the articles in Portuguese. The two corpora are automatically stored in the folder Corpus.
                              {"title": "title_1", "text": "text_1"}
                                                    ...
                                                    ...
                              {"title": "title_n", "text": "text_n"}

2. Rank articles according to November pageviews
Once the data has been filtered, we proceed with a simple analysis of the pageviews. In particular, using the article_df_from_json function, all the article titles are extracted from the corpus and then stored in a DataFrame.
Can you write Python code for it?

# Get the df for the Italian articles
df_it_titles = article_df_from_json('Corpus/wiki_ita_Matteo_Renzi.json')

# Get the df for the Portuguese articles
df_pt_titles = article_df_from_json('Corpus/wiki_port_Matteo_Renzi.json')


I am working on the file "Mention/Mention_example.ipynb".
The first task is:
The dictionaries that match the PT and IT titles are re-defined taking into account the fact that some IT do not mention Renzi.
Can you write Python code for it?

# Re-define the two dictionaries
dict_italian_mentions = {k:v for k,v in dict_italian.items() if v in it_titles_with_mention}
# Define the inverted
inverted_dict_italian_mentions = {v : k for k, v in dict_italian.items()}

# Create the list of titles for PT articles according to the IT that don't mention Renzi
pt_titles_with_mention = list(dict_italian_mentions.keys())


I am working on the file "Mention/Mention_example.ipynb".
The first task is:
Then, we create a unique DataFrame which contains the mentions in IT an PT articles for the tuple of articles.
Can you write Python code for it?

# Create df for IT mentions
df_match_it_mentions = df_it_mentions.query('Title in @it_titles_with_mention').sort_values('Number of mentions', ascending = False)

# Create df for PT mentions
df_match_pt_mentions = df_pt_mentions.query('Title in @pt_titles_with_mention').sort_values('Number of mentions', ascending = False)


I am working on the file "Merge URLs and tweets.ipynb".
The first task is:
Extract tweet features
Can you write Python code for it?

def extract_meta_features(x):
    u_data = x["user"]
    u_url = u_data['url']
    if u_url is not None:
        u_url = u_data['entities']['url']['urls'][0]['expanded_url']
    return (x["id"],
            x['created_at'],
            x['retweet_count'],
            x['favorite_count'],
            x['in_reply_to_status_id'] is not None,
            'quoted_status' in x and x['quoted_status'] is not None,
            len(x['entities']['hashtags']),
            len(x['entities']['urls']),
            len(x['entities']['user_mentions']),
            0 if 'media' not in x['entities'] else len(x['entities']['media']), # Has photos
            u_data['id'],
            u_data[u'created_at'],
            u_data[u'listed_count'],
            u_data[u'favourites_count'],
            u_data[u'followers_count'],
            u_data[u'friends_count'],
            u_data[u'statuses_count'],
            u_data[u'verified'],
            u_data[u'location'].replace('\r', ''),
            u_data[u'name'].replace('\r',''),
            u_url
           )

extract_meta_features(data[0])

df_meta = pd.DataFrame((extract_meta_features(d) for d in data),
                      columns=["t_id", "t_created", "t_retweets",
                              "t_favorites", "t_is_reply", "t_is_quote",
                              "t_n_hashtags", "t_n_urls", "t_n_mentions",
                              "t_n_media",
                               "u_id", "u_created",
                               "u_n_listed", "u_n_favorites", "u_n_followers",
                               "u_n_friends", "u_n_statuses",
                               "u_is_verified", "u_location", "u_name", "u_url"
                              ])
df_meta.head()

df_meta.dtypes

df_meta[df_meta.u_url.apply(lambda x: x is not None)]["u_url"].head()

df_meta.to_csv("TID_META.txt", sep="\t", index=False, encoding='utf-8')
! head TID_META.txt

df_meta[df_meta.u_url.apply(lambda x: x is not None)]["u_url"].shape

df_meta.shape


I am working on the file "Merge URLs and tweets.ipynb".
The first task is:
Load expanded data
Can you write Python code for it?

df = pd.read_csv("URL_CAT_MAPPINGS.txt", sep="\t")
df.head()

df['URL_EXP_SUCCESS'] = (df.EXPANDED_STATUS < 2)
df.head()

URL_DICT = dict(zip(df[df.URL_CATS != 'UNK'].URL, df[df.URL_CATS != 'UNK'].URL_CATS))
URL_MAPS = dict(zip(df.URL, df.URL_DOMAIN))
URL_EXP_SUCCESS = dict(zip(df.URL, df.URL_EXP_SUCCESS))
len(URL_DICT), df.shape, len(URL_MAPS), len(URL_EXP_SUCCESS)

df.URL.head().values

URL_MAPS['http://bit.ly/1SqTn5d']

found_urls = 0
twitter_urls = 0
total_urls = 0
tid_mapped_urls = []
url_types = defaultdict(int)
for d in data:
    if 'urls' in d['entities']:
            m_entities = d['entities']['urls']
            for m in m_entities:
                total_urls += 1
                m = m['expanded_url']
                m_cats = "UNK"
                if m in URL_DICT:
                    found_urls += 1
                    m_cats = URL_DICT[m]
                elif m.startswith("https://twitter.com") or m.startswith("http://twitter.com"):
                    found_urls += 1
                    twitter_urls += 1
                    m_cats = "socialmedia|twitter"
                else:
                    m_type = "failed_url"
                    if URL_EXP_SUCCESS[m]:
                        m_type = URL_MAPS.get(m, "None.com")
                    """
                    m_type = m.split("/", 3)[2]
                    #m_type = m_type.split("/", 3)[2]
                    if m_type.startswith("www."):
                        m_type = m_type[4:]
                    """
                    url_types[m_type] += 1
                tid_mapped_urls.append((d["id"], m, m_cats))
print "Data: %s, Total: %s, Found: %s, Twitter: %s" % (len(data), total_urls, found_urls, twitter_urls)
url_types = Counter(url_types)
url_types.most_common(10)

url_types.most_common(50)

sum(url_types.values())

tid_mapped_urls[:10]

df_mapped_cats = pd.DataFrame(tid_mapped_urls, columns=["TID", "URL", "CATS"])
df_mapped_cats.head()

df_mapped_cats.to_csv("TID_URL_CATS.txt", sep="\t", index=False)
! head TID_URL_CATS.txt


I am working on the file "Merge URLs and tweets.ipynb".
The first task is:
Run code to get all URLs
```
with open("all_urls.txt", "wb+") as fp:
    for url in sorted(filter(lambda x: x[1] != 'twitter.com',
            unique_urls),
                      key=lambda x: url_types[x[1]],
                     reverse=True):
        print >> fp, "%s\t%s\t%s" % (url[0], url[1], url_types[url[1]])
! head all_urls.txt
```
Can you write Python code for it?

len(data)

data[0].keys()

data[0][u'source']

data[0][u'is_quote_status']

data[0][u'quoted_status']['text']

data[0]['text']

count_quoted = 0
has_coordinates = 0
count_replies = 0
language_ids = defaultdict(int)
count_user_locs = 0
user_locs = Counter()
count_verified = 0
for d in data:
    count_quoted += d.get('is_quote_status', 0)
    coords = d.get(u'coordinates', None)
    repl_id = d.get(u'in_reply_to_status_id', None)
    has_coordinates += (coords is not None)
    count_replies += (repl_id is not None)
    loc = d['user'].get('location', u'')
    count_verified += d['user']['verified']
    if loc != u'':
        count_user_locs += 1
        user_locs.update([loc])
    language_ids[d['lang']] += 1

print count_quoted, has_coordinates, count_replies, count_user_locs, count_verified
print("Of {} tweets, {} have coordinates, while {} have user locations, comprising of {} unique locations".format(
        len(data), has_coordinates, count_user_locs, len(user_locs)
    ))

user_locs.most_common(10)

len(data)

data[0]['user']


I am working on the file "Merge-k-sorted-lists.ipynb".
The first task is:
Tests
Can you write Python code for it?

import random

def random_sorted_list(size):
    return sorted([random.randint(0, 100) for _ in range(size)])

def issorted(alist):
    return alist == sorted(alist)

for size in [10, 20, 30]:
    for k in range(2, 20):
        lists = [random_sorted_list(size) for _ in range(k)]
        merged_list = merge(*lists)
        assert issorted(merged_list)


I am working on the file "Mergesort.ipynb".
The first task is:
Let's try out our code:
Can you write Python code for it?

lst = [5, 2, 1, 20, 10, 3, 4, 80, 0]
sorted_list = mergesort(lst)
print(sorted_list)


I am working on the file "Messy Sensor Data - A Programmer's Cleaning Guide.ipynb".
The first task is:
Irregular data frequency
Our data has odd timestamps from time to time, For example:
Can you write Python code for it?

# One section where the data has weird timestamps ...
indexed_weather_observations[1800:1806]


I am working on the file "Messy Sensor Data - A Programmer's Cleaning Guide.ipynb".
The first task is:
Let wind be numbers
To process wind direction, we need them in numbers.
 * North wind (↓) is 0 degrees, going clockwise ⟳.
 * East wind (←) is 90 degrees.
We need to transform each direction label "N", "NNE", "NE", "ENE", "E", etc., to numbers on the column.
Series.apply(): Transforms each entry with a function.
Transforming DataFrames
Can you write Python code for it?

# Translate wind direction to degrees
wind_directions = {
     'N':   0. , 'NNE':  22.5, 'NE':  45. , 'ENE':  67.5 ,
     'E':  90. , 'ESE': 112.5, 'SE': 135. , 'SSE': 157.5 ,
     'S': 180. , 'SSW': 202.5, 'SW': 225. , 'WSW': 247.5 ,
     'W': 270. , 'WNW': 292.5, 'NW': 315. , 'NNW': 337.5 }

# Replace wind directions column with a new number column
# `get()` accesses values safely from dictionary
indexed_weather_observations['Wind dir'] = \
    indexed_weather_observations['Wind dir'].apply(wind_directions.get)

# Display some entries
indexed_weather_observations.head()


I am working on the file "Messy Sensor Data - A Programmer's Cleaning Guide.ipynb".
The first task is:
Messy Sensor Data:
A Programmer's Cleaning Guide

@Xavier_Ho, #pyconau
<small>Feel free to tag me on Twitter for questions and comments.</small>
Related #pyconau talks
This talk is a tutorial (thanks @evildmp!) for cleaning up messy data with pandas. You might also find these other ones useful:

Using Python in a Data Hackathon - Tennessee Leeuwenburg
Visualising data with Python - Clare Sloggett

Slides will be available online.
There will be time for questions at the end - happy to navigate specific problems you may have.
Motivation: healthy water

Aquatic microorganisms thrive and die with seasonal temperatures.

Cyanobacteria, or "blue-green algae", produces oxygen in water.

Too much cyanobacteria in water is harmful to consume.

We model cyanobacteria in water with weather data.

In short: we can predict and track cyanobacteria, and keep our water healthy.
Let's take a look at our weather sensor data...
In this talk, we use pandas for data wrangling.
You can install it with your favourite package manager (instructions).
$ conda install pandas
To begin:
 * pandas.read_csv(): Opens a CSV file as a DataFrame, like a table.
 * DataFrame.head(): Displays the first 5 entries.
Read a CSV file
Can you write Python code for it?

import pandas as pd

# Open a comma-separated values (CSV) file as a DataFrame
weather_observations = pd.read_csv('observations/Canberra_observations.csv')

# Print the first 5 entries
weather_observations.head()


I am working on the file "Messy Sensor Data - A Programmer's Cleaning Guide.ipynb".
The first task is:
Note: NaN in the table above means empty, not the floating-point number value.
Can you write Python code for it?

# For consistency between slides
weather_observations = pd.read_csv('observations/Canberra_observations.csv',
                                   sep='\t',
                                   parse_dates={'Datetime': ['Date', 'Time']},
                                   dayfirst=True,
                                   infer_datetime_format=True,
                                   na_values=['-']
                                  )


I am working on the file "Messy Sensor Data - A Programmer's Cleaning Guide.ipynb".
The first task is:
Small steps in data cleaning
The data seems complete, but still has some issues:

Each day includes midnight, and another midnight next day.
Order starts at end of day and goes backwards in time.

pandas offers some functions to help us out:
 * DataFrame.drop_duplicates(): Delete duplicated items.
 * DataFrame.sort_values(): Rearrange in order.
 * DataFrame.set_index(): Specify a column to use as index.
Keep order in our data
Can you write Python code for it?

# Remove duplicated items with the same date and time
no_duplicates = weather_observations.drop_duplicates('Datetime', keep='last')

# Sorting is ascending by default, or chronological order
# sorted_dataframe = no_duplicates.sort_values('Datetime')

# Use `Datetime` as our DataFrame index
# indexed_weather_observations = sorted_dataframe.set_index('Datetime')

# Display some entries
no_duplicates.head()

# For consistency
no_duplicates = weather_observations.drop_duplicates('Datetime', keep='last')
sorted_dataframe = no_duplicates.sort_values('Datetime')
indexed_weather_observations = sorted_dataframe.set_index('Datetime')


I am working on the file "Messy Sensor Data - A Programmer's Cleaning Guide.ipynb".
The first task is:
The file seems to be tab seperated. There are dates, and some empty items.
Can we read it more clearly?
pandas.read_csv() is very versatile with keyword arguments.

sep: The separator between columns.

parse_dates: Treat one or more columns like dates.

dayfirst: Use DD.MM.YYYY format, not month first.

infer_datetime_format: Tell pandas to guess the date format.

na_values: Specify values to be treated as empty.

Read a CSV file the 🐼🐼 way
Can you write Python code for it?

# Supply pandas with some hints about the file to read
weather_observations = pd.read_csv('observations/Canberra_observations.csv',
#                                    sep='\t',
#                                    parse_dates={'Datetime': ['Date', 'Time']},
#                                    dayfirst=True,
#                                    infer_datetime_format=True,
#                                    na_values=['-'],
                                  )
# Display some entries
weather_observations.head()


I am working on the file "Messy Sensor Data - A Programmer's Cleaning Guide.ipynb".
The first task is:
There seems to be gaps in our dataset between 4th Jan and 9th Jan.
Gaps in the data
Weather sensors sometimes drop out.
But for modelling purposes, we need a gap-free dataset.
Series.interpolate(): Fill in empty values based on index.
Interpolate and fill empty rows
Can you write Python code for it?

# Interpolate data to fill empty values
for column in regular_observations.columns:
    regular_observations[column].interpolate('time', inplace=True)

# Display some interpolated entries
regular_observations[1633:1638]


I am working on the file "Messy Sensor Data - A Programmer's Cleaning Guide.ipynb".
The first task is:
We can solve this. DataFrame.asfreq() forces a specific frequency on the index, discarding and filling the rest.
Setting data frequency
Let's set each data point to be every 30 minutes.
Can you write Python code for it?

# Force the index to be every 30 minutes
regular_observations = indexed_weather_observations.asfreq('30min')

# Same section at different indices since setting its frequency :)
regular_observations[1633:1638]


I am working on the file "Meta Kaggle - Team sizes of competition winners.ipynb".
The first task is:
In this notebook, I am exploring the Meta Kaggle data (https://www.kaggle.com/c/meta-kaggle). The dataset is available in both csv and sqlite database formats.
There are 10 tables:
* Users
* Teams
* TeamMemberships
* Competitions
* Submissions
* CopetitionHostSegments
* RewardTypes
* Sites
* ValidationSets
* EvaluationAlgorithms
Getting started: import modules and connect to the SQLite database
Can you write Python code for it?

import pandas as pd
import sqlite3
import matplotlib
%matplotlib inline
matplotlib.style.use('ggplot')

cnx = sqlite3.connect('database.sqlite')


I am working on the file "Meta Kaggle - Team sizes of competition winners.ipynb".
The first task is:
Is there any relationship between user ranking and the number of competitions entered?
Let us explore this by plotting the rank vs. number of competitions entered for all users. First, we need a table with these 2 fields. We can construct this by joining the Teams and TeamMemberships tables.
Pandas has a function read_sql_query to execute a SQL query. The output is a data frame with the column names corresponding to sql-table column names.
Can you write Python code for it?

rank_vs_ncomps = pd.read_sql_query("""
SELECT U.Ranking AS rank, COUNT(M.TeamId) as num_comps
FROM Users AS U
JOIN TeamMemberships AS M ON U.Id = M.UserId
GROUP BY U.Id, U.Ranking
""", cnx)

ax = rank_vs_ncomps.plot(x='rank', y='num_comps',
        logx=True, logy= True,
        kind='scatter')
ax.set_xlabel('log10(User Rank)', fontsize=14)
ax.set_ylabel('log10(# competitions entered)', fontsize = 14)


I am working on the file "Meta Kaggle - Team sizes of competition winners.ipynb".
The first task is:
Plot the number of new users vs. month/year of registration:
Can you write Python code for it?

ax = reg_by_monyear.plot(x='Time', y='num',
                    title = 'Number of new registrations by time',
                    figsize=(8,4))
ax.set_xlabel('Month/Year of Registration', fontsize=14)
ax.set_ylabel('Number of registrants', fontsize = 14)


I am working on the file "Meta Kaggle - Team sizes of competition winners.ipynb".
The first task is:
So, most winning teams consisted of only one individual
Explore the number of registrations by time
Let us plot the number of users by their registration date month-by-month. The data is available in the Users table. We could do this by using SQL queries, but this time, I will do this by manipulating the pandas dataframe. Note the difference in the dataframe returned from sqlite3 and that returned by pd.read_csv()
Get a dataframe from the SQLite Users table :
Can you write Python code for it?

users_sqlite = pd.read_sql_query("""
SELECT *
FROM Users
""", cnx)

users_sqlite.info()


I am working on the file "Meta Kaggle - Team sizes of competition winners.ipynb".
The first task is:
We have used the log-log scale for both the x- and y-axes so that we could see more clearly the standing of top-ranking individuals. From this plot, it looked like practice contributes to user standing.
Do competitions winners work individually or collaboratively?
Let us explore the sizes of all teams that ever took the 1st place in a competition. To do this, we construct a table with team and team size as its fields using the Teams and TeamMemberships tables.
Can you write Python code for it?

team_top3 = pd.read_sql_query("""
SELECT team_size, rank, COUNT(*) AS num_teams
FROM
(
    SELECT T.Id AS team_id, T.Ranking AS rank, COUNT(M.UserId) AS team_size
    FROM Teams AS T
    JOIN TeamMemberships AS M ON T.Id = M.TeamId
    WHERE T.Ranking IN (1, 2, 3)
    GROUP BY T.Id, T.Ranking
)
GROUP BY team_size, rank
""", cnx)

team_top3['team_size'] = team_top3['team_size'].map(lambda x : x if (x<5) else 5)

team_top3 = team_top3.groupby(['team_size', 'rank'], as_index=False).sum()

ax = team_top3[team_top3['rank'] == 1].plot(kind='bar',
                                       x='team_size',
                                       y='num_teams',
                                      title='Size of winning (1st place) teams')
ax.set_xticklabels(['1','2','3','4','>=5'], rotation='horizontal')
ax.set_xlabel('Team Size', fontsize=14)
ax.set_ylabel('Number of Teams', fontsize = 14)


I am working on the file "Metagenomes/IdentifyingStrains/strains_depth_statistics.ipynb".
The first task is:
Note that we have reduced our matrix from having 11,054 entries with all the zeros to only having 1,397 entries now!
plot the data.
This just provides a quick overview of the data
Can you write Python code for it?

ax = dfa.plot('A_read_depth', 'A_strains', kind='scatter')
ax.set(ylabel="# strains", xlabel="read depth")


I am working on the file "Metagenomes/IdentifyingStrains/strains_depth_statistics.ipynb".
The first task is:
Removing the outliers
Remember the nine outliers above? If we use the data set where we have removed them, we can compare the r2 value for that dataset.
Can you write Python code for it?

model = sm.OLS(dfas['A_strains'], dfas['A_read_depth']).fit()
predictions = model.predict(dfas['A_read_depth'])
model.summary()


I am working on the file "Metagenomes/IdentifyingStrains/strains_depth_statistics.ipynb".
The first task is:
When we zoom in, this doesn't look like a strong correlation. Note that there are a lot of data points here compared to the whole data set. In the data set excluding (0,0) we had 1,397 entries, and now we have 1,386 entries, so we only removed 9 values!
Linear regression
What is the correlation between these two data sets. Note that we use the data with all the non-zero's removed (1,397 data points).
As a reminder, the statsmodels OLS uses y ~ x
Can you write Python code for it?

model = sm.OLS(dfa['A_strains'], dfa['A_read_depth']).fit()
predictions = model.predict(dfa['A_read_depth'])
model.summary()


I am working on the file "Meteorology-P-GridSequence-1.ipynb".
The first task is:
Advance the model to the end, saving the model time and output P values (converted back to mm/hr for convenience) at each step:
Can you write Python code for it?

time = [met.get_current_time()]
flux = [bprecip.max() * mps_to_mmph]
count = 1
while met.get_current_time() < met.get_end_time():
    met.update(met.get_time_step()*count)
    time.append(met.get_current_time())
    flux.append(met.get_value('atmosphere_water__precipitation_leq-volume_flux').max() * mps_to_mmph)
    count += 1


I am working on the file "Meteorology-P-GridSequence-1.ipynb".
The first task is:
Initialize the model.
Can you write Python code for it?


I am working on the file "Meteorology-P-GridSequence-1.ipynb".
The first task is:
The initial model precipitation volume flux is the first value from precip_rates.txt:
Can you write Python code for it?

bprecip = met.get_value('atmosphere_water__precipitation_leq-volume_flux')
print type(bprecip)
print bprecip.size
print bprecip.shape
bprecip * mps_to_mmph


I am working on the file "Meteorology-P-GridSequence-2.ipynb".
The first task is:
Advance the model to the end, saving the model time and output P values (converted back to mm/hr for convenience) at each step:
Can you write Python code for it?


I am working on the file "Meteorology-P-GridSequence-2.ipynb".
The first task is:
Check the time and flux values (noting that I've included the time = 0.0 value here):
Can you write Python code for it?


I am working on the file "Meteorology-P-GridSequence-2.ipynb".
The first task is:
Import the Babel-wrapped Meteorology component and create an instance:
Can you write Python code for it?

from cmt.components import Meteorology
met = Meteorology()


I am working on the file "Meteorology-P-TimeSeries.ipynb".
The first task is:
Advance the model to the end, saving the model time and output P values (converted back to mm/hr for convenience) at each step:
Can you write Python code for it?


I am working on the file "Meteorology-P-TimeSeries.ipynb".
The first task is:
BMI component
Import the BMI Meteorology component and create an instance:
Can you write Python code for it?

from topoflow.components.met_base import met_component
m = met_component()


I am working on the file "Meteorology-P-TimeSeries.ipynb".
The first task is:
Check the file:
Can you write Python code for it?


I am working on the file "Meteorology-P-TimeSeries.ipynb".
The first task is:
Check the time and flux values:
Can you write Python code for it?


I am working on the file "Meteorology-P-TimeSeries.ipynb".
The first task is:
Initialize the model. A value of snow depth h_snow is needed for the model to update.
Can you write Python code for it?

m.initialize('./input/meteorology-1.cfg')
m.h_snow = 0.0  # Needed for update


I am working on the file "Meteorology-P-TimeSeries.ipynb".
The first task is:
Next, write the values to a file to the input directory, where it's expected by the cfg file:
Can you write Python code for it?


I am working on the file "Meteorology-P-TimeSeries.ipynb".
The first task is:
Programmatically create a file holding the precipitation rate time series. This will mimic what I'll need to do in WMT, where I'll have access to the model time step and run duration. Start by defining the precipitation rate values:
Can you write Python code for it?

import numpy as np
n_steps = 10  # can get from cfg file
precip_rates = np.linspace(5, 20, num=n_steps, endpoint=False)
precip_rates


I am working on the file "Meteorology-P-TimeSeries.ipynb".
The first task is:
Unlike the scalar case, there's an output volume flux of precipitation:
Can you write Python code for it?

print precip * mps_to_mmph  # note that this is a reference, so it'll take the current value of `P`


I am working on the file "Meteorology-Qn_LW.ipynb".
The first task is:
Advance the model by one time step:
Can you write Python code for it?

bm.update(time_step)
print '\nCurrent time: {} s'.format(bm.get_current_time())


I am working on the file "Meteorology-Qn_LW.ipynb".
The first task is:
Again, check the calculated value of Qn_LW:
Can you write Python code for it?

Q = bm.get_value('land_surface_net-longwave-radiation__energy_flux')
print type(Q)
print Q.size
print Q.shape
Q


I am working on the file "Meteorology-Qn_LW.ipynb".
The first task is:
As above, Qn_LW is initialized to an np.zeros array with the dimensions of the grid. However, the Bocca implementation flattens the array:
Can you write Python code for it?

bQ = bm.get_value('land_surface_net-longwave-radiation__energy_flux')  # `Qn_LW` internally
print type(bQ)
print bQ.size
print bQ.shape
bQ


I am working on the file "Meteorology-Qn_LW.ipynb".
The first task is:
Check the calculated value of Qn_LW:
Can you write Python code for it?


Q = m.get_value('land_surface_net-longwave-radiation__energy_flux')
print type(Q)
print Q.size
print Q.shape
Q


I am working on the file "Meteorology-Qn_LW.ipynb".
The first task is:
Get the model time step (which has been broadcast over the grid):
Can you write Python code for it?

time_step = bm.get_value('model__time_step')  # `dt` internally
time_step = time_step.max()
print time_step


I am working on the file "Meteorology-Qn_LW.ipynb".
The first task is:
The calculated value of Qn_LW matches that of the BMI case above, but it has been broadcast over the grid. This is what we expect.
Advance the model by a second time step:
Can you write Python code for it?

bm.update(2*time_step)
print '\nCurrent time: {} s'.format(bm.get_current_time())


I am working on the file "Meteorology-Qn_LW.ipynb".
The first task is:
The initial Qn_LW array has been converted into a scalar.
The Qn_LW calculation starts on line 1631 of met_base.py:
```python
--------------------------------
Compute Qn_LW for this time
--------------------------------
T_air_K  = self.T_air  + self.C_to_K
T_surf_K = self.T_surf + self.C_to_K
LW_in    = self.em_air  * self.sigma * (T_air_K) 4.0
LW_out   = self.em_surf * self.sigma * (T_surf_K) 4.0
LW_out   = LW_out + ((1.0 - self.em_surf) * LW_in)
self.Qn_LW = (LW_in - LW_out)   # [W m-2]
```
The temperature and emissivity values are set to scalars in the cfg file, so Qn_LW is also a scalar.
Result: A scalar value of Qn_LW is calculated from scalar values of temperature and emissivity, which seems legit.
Babel-wrapped component
Import the Babel-wrapped Meteorology component, create an instance, and initialize the component:
Can you write Python code for it?

from cmt.components import Meteorology
bm = Meteorology()
bm.initialize('./input/meteorology.cfg')


I am working on the file "Meteorology-Qn_LW.ipynb".
The first task is:
There's no input value for Qn_LW; rather, it's calculated from temperature and emissivity values. It's initialized to an np.zeros array with the dimensions of the grid (line 951 of met_base.py):
Can you write Python code for it?

Q = m.get_value('land_surface_net-longwave-radiation__energy_flux')  # `Qn_LW` internally
print type(Q)
print Q.size
print Q.shape
Q


I am working on the file "Meteorology-SnowDegreeDay-Grid.ipynb".
The first task is:
Finalize the components:
Can you write Python code for it?


I am working on the file "Meteorology-SnowDegreeDay-Grid.ipynb".
The first task is:
Initialize the components with cfg files that, for simplicity, use the same time step and run duration:
Can you write Python code for it?


I am working on the file "Meteorology-SnowDegreeDay-Grid.ipynb".
The first task is:
Plot snow depth versus time.
Can you write Python code for it?

%matplotlib inline
from matplotlib import pyplot as plt

plt.plot(time[1:], snow_depth[1:])
plt.title('Snow depth versus time')
plt.xlabel('Time [s]')
plt.ylabel('Snow depth [m]')


I am working on the file "Meteorology-SnowDegreeDay-Grid.ipynb".
The first task is:
Run the coupled models to completion. In each time step, perform the following actions:

Get variables from Meteorology; set into SnowDegreeDay
Advance SnowDegreeDay
Get variables from SnowDegreeDay; set into Meteorology
Advance Meteorology
Can you write Python code for it?

count = 1
while met.get_current_time() < met.get_end_time():
    T_air = met.get_value('atmosphere_bottom_air__temperature')
    P_snow = met.get_value('atmosphere_water__snowfall_leq-volume_flux')
    T_surf = met.get_value('land_surface__temperature')
    rho_H2O = met.get_value('water-liquid__mass-per-volume_density')
    sno.set_value('atmosphere_bottom_air__temperature', T_air)
    sno.set_value('atmosphere_water__snowfall_leq-volume_flux', P_snow)
    sno.set_value('land_surface__temperature', T_surf)
    sno.set_value('water-liquid__mass-per-volume_density', rho_H2O)

    sno.update(sno.get_time_step()*count)

    rho_snow = sno.get_value('snowpack__z_mean_of_mass-per-volume_density')
    h_snow = sno.get_value('snowpack__depth')
    h_swe = sno.get_value('snowpack__liquid-equivalent_depth')
    SM = sno.get_value('snowpack__melt_volume_flux')
    met.set_value('snowpack__z_mean_of_mass-per-volume_density', rho_snow)
    met.set_value('snowpack__depth', h_snow)
    met.set_value('snowpack__liquid-equivalent_depth', h_swe)
    met.set_value('snowpack__melt_volume_flux', SM)

    met.update(met.get_time_step()*count)

    time.append(met.get_current_time())
    snow_depth.append(sno.get_value('snowpack__depth').max())
    air_temp.append(met.get_value('atmosphere_bottom_air__temperature').max())

    count += 1

print time

print snow_depth

print air_temp


I am working on the file "Meteorology-SnowDegreeDay-Grid.ipynb".
The first task is:
Store initial values of time, snow depth, and air temperature:
Can you write Python code for it?

time = [met.get_current_time()]
snow_depth = [sno.get_value('snowpack__depth').max()]
air_temp = [met.get_value('atmosphere_bottom_air__temperature').max()]


I am working on the file "Meteorology-SnowDegreeDay.ipynb".
The first task is:
Finalize the components:
Can you write Python code for it?


I am working on the file "Meteorology-SnowDegreeDay.ipynb".
The first task is:
Initialize the components with cfg files that, for simplicity, use the same time step and run duration:
Can you write Python code for it?

%cd input

met.initialize('meteorology-2.cfg')
sno.initialize('snow_degree_day-1.cfg')


I am working on the file "Meteorology-SnowDegreeDay.ipynb".
The first task is:
Plot snow depth versus time.
Can you write Python code for it?


I am working on the file "Meteorology-SnowDegreeDay.ipynb".
The first task is:
Store initial values of time, snow depth, and air temperature:
Can you write Python code for it?


I am working on the file "MiamiData01-intro.ipynb".
The first task is:
Fetch open data
Next, we need to import the data.  I accessed the 311 open data at Miami-Dade County’s Open Data Portal.
I saved it as a .csv file.
Can you write Python code for it?

!ls data/

dat = pd.read_csv('data/311_Service_Requests_-_Miami-Dade_County.csv')


I am working on the file "MiamiData01-intro.ipynb".
The first task is:
It has almost 400,000 rows!
Can you write Python code for it?

len(dat.City)

dat.head()

dat.columns

plt.plot(dat['Goal Days'], dat['Actual Completed Days'], '.', alpha=0.1)
plt.xlim(0, 30)
plt.ylim(0, 30)
plt.title('Does 311 meet its goals?')
plt.xlabel('Goal (Days)')
plt.ylabel('Actual (Days)')


I am working on the file "MiamiData01-intro.ipynb".
The first task is:
That looks better.  The red-dashed line is the 1:1 line.  Above this line, 311 did not meet its goal to complete the task, below this line 311 completed the task faster than its goal.  It sort of looks like most 311 tasks are completed below the 1:1 line.  But let's just compute it.
Can you write Python code for it?

dat.columns

diff = dat['Actual Completed Days'] - dat['Goal Days']

good_diff = diff[diff == diff]

plt.hist(good_diff, range=(-100, 100), bins=100)
plt.yscale('linear')
plt.xlabel('Actual - Goal')
plt.ylabel('$N$')

n_good = np.sum(good_diff <= 0)
n_bad = np.sum(good_diff > 0)
n_tot = len(good_diff)

n_good, n_bad, n_tot

print "Miami 311 meets their goal {:.1f}% of the time.".format(n_good*100.0/n_tot)


I am working on the file "Microentradas - Evitar ciertas etiquetas en la leyenda en Matplotlib.ipynb".
The first task is:
Como véis, en la gráfica anterior hay varios problemas pero como esta es una MicroEntrada solo nos vamos a centrar en el problema de las etiquetas repetidas en la leyenda.
¿Cómo podríamos evitar el meter tantas veces una etiqueta repetida?
Mi problema es que el bucle es o podría ser 'infinito' y tengo que inicializar las etiquetas de alguna forma. Si miro en esta respuesta encontrada en Stackoverflow dice que en la documentación se indica que "If label attribute is empty string or starts with “_”, those artists will be ignored." pero si busco aquí o en el enlace que indican en la respuesta en Stackoverflow no veo esa funcionalidad indicada en ningún sitio. Eso es porque aparecía en la versión 1.3.1 pero luego desapareció... Sin embargo podemos seguir usando esa funcionalidad aunque actualmente no esté documentada:
Can you write Python code for it?

plt.figure(figsize = (12, 6))
for i in range(10):
    x = np.arange(i * 10, i * 10 + 10)
    y_var1 = np.random.randint(1, 5, 10)
    y_var2 = np.random.randint(5, 8, 10)
    plt.plot(x, y_var1, color = 'k', label = 'variable1' if i == 0 else "_esto_no_se_pintará")
    plt.plot(x, y_var2, color = 'g', label = 'variable2' if i == 0 else "_esto_tampoco")
    plt.legend()
    plt.ylim(0, 9)


I am working on the file "Midterm Report/Midway report.ipynb".
The first task is:
ERCOT Installed Wind Capacity
This helps explain the figure above. There was lots of wind installed over the course of 2015, which could have helped bump up the output.
Can you write Python code for it?


I am working on the file "Midterm Report/Midway report.ipynb".
The first task is:
ERCOT Load and Wind Generation Over Time
These figures show monthly average load and wind generation over each year. Load (on the left) follows a predictable pattern with peak demand in the summer months. Wind generation (on the right) is a little messier. Over most years there is a dip in the summer, but we don't see the same dip in 2015.
Can you write Python code for it?


I am working on the file "Midterm Report/Midway report.ipynb".
The first task is:
Hourly Fossil Power Plant Generation (EPA data)
This figure shows the hourly gross load from three sample plants over the last 6 months of 2015. These three sample plants happen to show a range of different sizes and behaviors.
The left and middle subplots represent coal plants with 1 and 2 units, all of which have minimum operating loads. We hope that aggregating facilities into groups will allow us to ignore shutoff below the minimum load - we care about the change in group generation rather than if an individual power plant goes from off to on.
The plant on the right consists of two natural gas combustion turbines, which can quickly turn on and ramp up. It never appears to hit its maximum generation of ~250MW.
Can you write Python code for it?


I am working on the file "Midterm.ipynb".
The first task is:
Midterm
Chinmai Raman
3/22/2016
$x_{n+1} = rx_n(1-x_n)$ for $x_0$ in $[0,1]$ and $r$ in $[2.9,4]$
Problem 1
$x_0$ = 0.5, $r$ = 2.5
Can you write Python code for it?


I am working on the file "Midterm.ipynb".
The first task is:
Problem 4
The following are graphs zoomed into specific regions of the graph above:
Zoom graph 1
Can you write Python code for it?


I am working on the file "Midterm.ipynb".
The first task is:
The function is not graphable at r = 2.5
$x_0$  = 0.5,  $r$  = 3.2
Can you write Python code for it?


I am working on the file "Midterm.ipynb".
The first task is:
The sequence approaches an oscillation between 0.799 (at odd indices of x) and 0.513 (at even indices of x)
$x_0$ = 0.5,  $r$  = 3.5
Can you write Python code for it?


I am working on the file "Midterm.ipynb".
The first task is:
The sequence approaches an oscillation between two sets of two numbers: 0.501 (at indices 4n), 0.875 (at indices 4n+1), 0.383 (at indices 4n+2), and 0.827 (at indices 4n+3) where n is a whole number
Varying initial condition $x_0$ in the range $[0,1]$ with r = 3.2
Can you write Python code for it?


I am working on the file "Midterm.ipynb".
The first task is:
The sequence seems to converge to an oscillation between 16 values, approximately: (0.5078907479202178, 0.8922777178672167, 0.34314194567165146, 0.8046621163285997, 0.5611361517008182, 0.8791566643101159, 0.37927759935593724, 0.8404711840783617, 0.4786633609398336, 0.8908747497660097, 0.3470644400834328, 0.8090002508114317, 0.5516322766445462, 0.8829827655903487, 0.3688673985009418, 0.8311111397419982). It could be argued that the sequence oscillates between thirty-two points, but the values were two fine to compare adequately by eye. There seems to be an variation in every oscillation at index 4n+k ($0<=k<=7$ and n is a whole number) between eight values. These oscillations of oscillations are larger at r = 3.57 than at r = 3.5699 and r = 3.5441. Again, the periodicity of the oscillations is greater as r increases.
Can you write Python code for it?


I am working on the file "Midterm.ipynb".
The first task is:
The sequence seems to converge to an oscillation between 8 values, approximately: (0.5229, 0.8842, 0.3630, 0.8195, 0.5243, 0.8839, 0.3636, 0.8201). It could be argued that the sequence oscillates between four points, but there is a slight variation in the 3rd or 4th decimal place in the values of the sequence at the indices 4n, 4n+1, 4n+2, 4n+3. There seems to be an variation in every oscillation at index 4n+k ($0<=k<=3$ and n is a whole number) between two values.
Can you write Python code for it?


I am working on the file "Midterm.ipynb".
The first task is:
The sequences both converge to the same 4 values (0.501, 0.875, 0.383, and 0.827) at the same speed. At a fixed r, the value of $x_0$ does not have an impact on the values that the sequence converges to.
Problem 2
$r$ = 3.5441
Can you write Python code for it?


I am working on the file "Midterm.ipynb".
The first task is:
Varying $x_0$ with a constant $r$ does not change the values that the sequence converges to and oscillates between for a large N
$r$ = 3.57
Can you write Python code for it?


I am working on the file "Midterm.ipynb".
The first task is:
Zoom graph 2
Can you write Python code for it?


I am working on the file "Midterm/.ipynb_checkpoints/AlgorithmsEx03-checkpoint.ipynb".
The first task is:
Character counting and entropy
Write a function char_probs that takes a string and computes the probabilities of each character in the string:

First do a character count and store the result in a dictionary.
Then divide each character counts by the total number of character to compute the normalized probabilties.
Return the dictionary of characters (keys) and probabilities (values).
Can you write Python code for it?

def char_probs(s):
    """Find the probabilities of the unique characters in the string s.

    Parameters
    ----------
    s : str
        A string of characters.

    Returns
    -------
    probs : dict
        A dictionary whose keys are the unique characters in s and whose values
        are the probabilities of those characters.
    """

    alphabet = 'abcdefghijklmnopqrstuvwxyz'

    dic = {}
    for i in alphabet:
        if i in s:
            dic[i] = s.count(i)/len(s)
    return dic

test1 = char_probs('aaaa')
assert np.allclose(test1['a'], 1.0)
test2 = char_probs('aabb')
assert np.allclose(test2['a'], 0.5)
assert np.allclose(test2['b'], 0.5)
test3 = char_probs('abcd')
assert np.allclose(test3['a'], 0.25)
assert np.allclose(test3['b'], 0.25)
assert np.allclose(test3['c'], 0.25)
assert np.allclose(test3['d'], 0.25)


I am working on the file "Midterm/.ipynb_checkpoints/AlgorithmsEx03-checkpoint.ipynb".
The first task is:
The entropy is a quantiative measure of the disorder of a probability distribution. It is used extensively in Physics, Statistics, Machine Learning, Computer Science and Information Science. Given a set of probabilities $P_i$, the entropy is defined as:
$$H = - \Sigma_i P_i \log_2(P_i)$$
In this expression $\log_2$ is the base 2 log (np.log2), which is commonly used in information science. In Physics the natural log is often used in the definition of entropy.
Write a funtion entropy that computes the entropy of a probability distribution. The probability distribution will be passed as a Python dict: the values in the dict will be the probabilities.
To compute the entropy, you should:

First convert the values (probabilities) of the dict to a Numpy array of probabilities.
Then use other Numpy functions (np.log2, etc.) to compute the entropy.
Don't use any for or while loops in your code.
Can you write Python code for it?

def entropy(d):
    """Compute the entropy of a dict d whose values are probabilities."""
    ps = np.array([d[x] for x in d])
    H = -sum(ps*np.log2(ps))
    return H

assert np.allclose(entropy({'a': 0.5, 'b': 0.5}), 1.0)
assert np.allclose(entropy({'a': 1.0}), 0.0)


I am working on the file "Midterm/AlgorithmsEx03.ipynb".
The first task is:
Character counting and entropy
Write a function char_probs that takes a string and computes the probabilities of each character in the string:

First do a character count and store the result in a dictionary.
Then divide each character counts by the total number of character to compute the normalized probabilties.
Return the dictionary of characters (keys) and probabilities (values).
Can you write Python code for it?


I am working on the file "Midterm/AlgorithmsEx03.ipynb".
The first task is:
Use IPython's interact function to create a user interface that allows you to type a string into a text box and see the entropy of the character probabilities of the string.
Can you write Python code for it?

def calc(s):
    print(entropy(char_probs(s)))

interact(calc,s='Your Text');

assert True # use this for grading the pi digits histogram


I am working on the file "Midterm/InteractEx06.ipynb".
The first task is:
Use interact with plot_fermidist to explore the distribution:

For mu use a floating point slider over the range $[0.0,5.0]$.
for kT use a floating point slider over the range $[0.1,10.0]$.
Can you write Python code for it?


I am working on the file "Midterm/InteractEx06.ipynb".
The first task is:
Write a function plot_fermidist(mu, kT) that plots the Fermi distribution $F(\epsilon)$ as a function of $\epsilon$ as a line plot for the parameters mu and kT.

Use enegies over the range $[0,10.0]$ and a suitable number of points.
Choose an appropriate x and y limit for your visualization.
Label your x and y axis and the overall visualization.
Customize your plot in 3 other ways to make it effective and beautiful.
Can you write Python code for it?

def plot_fermidist(mu, kT):
    fig, ax = plt.subplots(figsize=(15,4))
    plt.plot(np.linspace(0,10.0,100),fermidist(np.linspace(0,10.0,100), mu, kT))
    ax.spines['right'].set_visible(False)
    ax.spines['top'].set_visible(False)
    ax.yaxis.set_ticks_position('left')
    ax.xaxis.set_ticks_position('bottom')
    plt.xlim(0,10);
    plt.ylim(0,1.2);
    plt.title('Fermidistribution as a Function of Energy');
    plt.xlabel('Energy Levels');
    plt.ylabel('Fermidistribution');
    plt.tick_params(axis='y', direction='inout', length=5)
    plt.tick_params(axis='x', direction='inout', length=8)

plot_fermidist(4.0, 1.0);

assert True # leave this for grading the plot_fermidist function


I am working on the file "Milestone Project 1- Walkthrough Steps Workbook.ipynb".
The first task is:
Step 10: Here comes the hard part! Use while loops and the functions you've made to run the game!
Can you write Python code for it?

print('Welcome to Tic Tac Toe!')

while True:
    # Set the game up here
    theBoard = [' '] * 10
    player1_marker, player2_marker = player_input() #tuple unpacking
    turn = choose_first()
    print(turn + 'will go first!')

    game_on = True
    while game_on:
        #Player 1 Turn
        if turn == 'Player 1': # janky
            display_board(theBoard)
            position = player_choice(theBoard)
            place_marker(theBoard, player1_marker, position)

            if win_check(theBoard, player1_marker):
                display_board(theBoard)
                print('Congrats, {pl}, has won the game!'.format(pl=turn))
                game_on = False
            else:
                turn = 'Player 2' # janky

        # Player2's turn.
        if turn == 'Player 2':
            display_board(theBoard)
            position = player_choice(theBoard)
            place_marker(theBoard, player2_marker, position)

            if win_check(theBoard, player2_marker):
                display_board(theBoard)
                print('Congrats, {pl}, has won the game!'.format(pl=turn))
                game_on = False
            else:
                turn = 'Player 1' # janky

        if not replay():
            break


I am working on the file "Milestone Project 1- Walkthrough Steps Workbook.ipynb".
The first task is:
Step 1: Write a function that can print out a board. Set up your board as a list, where each index 1-9 corresponds with a number on a number pad, so you get a 3 by 3 board representation.
Can you write Python code for it?

from IPython.display import clear_output
def display_board(board):
    clear_output()
    # implementation is really janky in tut


I am working on the file "Milestone Project 1- Walkthrough Steps Workbook.ipynb".
The first task is:
Step 3: Write a function that takes, in the board list object, a marker ('X' or 'O'), and a desired position (number 1-9) and assigns it to the board.
Can you write Python code for it?

def place_marker(board, marker, position):
    board[position] = marker


I am working on the file "Milestone Project 1- Walkthrough Steps Workbook.ipynb".
The first task is:
Step 4: Write a function that takes in a board and a mark (X or O) and then checks to see if that mark has won.
Can you write Python code for it?

def win_check(board,mark):
    # lots of manual checks in tut
    pass


I am working on the file "Milestone Project 1- Walkthrough Steps Workbook.ipynb".
The first task is:
Step 7: Write a function that checks if the board is full and returns a boolean value. True if full, False otherwise.
Can you write Python code for it?

def full_board_check(board):
    # janky implement using a 1D array
    for i in range(1, 10):
        if space_check(board, i):
            return False

    return True


I am working on the file "Milestone Project 1- Walkthrough Steps Workbook.ipynb".
The first task is:
Step 8: Write a function that asks for a player's next position (as a number 1-9) and then uses the function from step 6 to check if its a free position. If it is, then return the position for later use.
Can you write Python code for it?

def player_choice(board):
    position = ''
    while position not in '1 2 3 4 5 6 7 8 9'.split() or not space_check(board[int(position)]):
        position = raw_input('Choose your next position: (1-9)')
    return position


I am working on the file "Milestone Project 1- Walkthrough Steps Workbook.ipynb".
The first task is:
Step 9: Write a function that asks the player if they want to play again and returns a boolean True if they do want to play again.
Can you write Python code for it?

def replay():
    # what's interesting is the str func chaining
    return raw_input('Do you want to play again? Enter Yes or No').lower().startswith('y')


I am working on the file "Milestone_3_v4.ipynb".
The first task is:
About 82% of the genres are predicted correctly using PCA with 327 PCs. This is pretty comparable to the RFC above that uses the full dimension dataset. Perhaps the slight loss in accuracy is offset by the reduction in time it takes to fit the random forest, although this is a subjective judgment.
Model Performance by genre
It's important to consider how the different models are making their mistakes, so we decided to break down the prediction accuracy by genre for the LR and RFC.
Can you write Python code for it?

#visualize by-genre accuracy of predictions
def plot_acc_by_genre(pred, y_test, clf_str=''):

    preds = pd.DataFrame(pred)
    acc = []

    for i in range(7):
        a = np.array(y_test[[i]])
        b = np.array(preds[[i]])
        acc.append(np.count_nonzero(a==b) / preds.shape[0] * 100)

    plt.figure(figsize=(10,5))
    index = [0,1,2,3,4,5,6]
    plt.bar(index,acc,align="center")
    plt.xticks(index, ('Action', 'Drama', 'Comedy', 'Family', 'Romance', 'Documentary', 'Horror'))
    plt.ylim([0,100])
    plt.grid(axis='y')
    plt.title(clf_str + ' Accuracy by Genre')
    plt.ylabel('Accuracy')
    plt.show()

plot_acc_by_genre(lr_pred, y_test, 'Logistic Regression')

plot_acc_by_genre(rf_pred, y_test, 'Random Forest')


I am working on the file "Milestone_3_v4.ipynb".
The first task is:
As we can see, ading all our language and country predictors as indicator variables leads to a 1% improvement in classifier performance.  This is not a huge gain, but perhaps worth the added overhead.
Next, we wondered whether the different "basic" features had roughly similar importance for RFC predictions.  We evaluated this by looking at the RFC importance scores.
Can you write Python code for it?

importances = rfc_mdl.feature_importances_
std = np.std([tree.feature_importances_ for tree in rfc_mdl.estimators_],
             axis=0)
indices = np.argsort(importances)[::-1]

# Plot the feature importances of the forest
plt.figure()
plt.title("Feature importances")
plt.bar(range(X_train.shape[1]), importances[indices],
       color="r", yerr=std[indices], align="center")
plt.xticks(range(X_train.shape[1]), indices)
plt.xlim([-1, 15])
plt.show()


I am working on the file "Milestone_3_v4.ipynb".
The first task is:
Let's begin by loading and examining our raw dataset, containing data obtained through the TMDB API and saved previously as a CSV file.
Can you write Python code for it?

%cd ./Train_Data_Version_1/
features = pd.read_csv('features_V1.csv')

features.head(5)


I am working on the file "Milestone_3_v4.ipynb".
The first task is:
Model fitting and comparison
Now that we've cleaned up our data sufficiently, it's finally time to begin model fitting and comparison.  The first step is to split the full dataset into training and testing sets.
Can you write Python code for it?

#split our data into training and testing
X_train, X_test, y_train, y_test = train_test_split(feat2, labels2, test_size=0.33, random_state=42)

# ----------------------------
# Standardize features
scaler = preprocessing.StandardScaler().fit(X_train)
# Call: scaler.transform(X_train) to scale
# ----------------------------


I am working on the file "Milestone_3_v4.ipynb".
The first task is:
Next we'll load our previously dummy-coded genre labels and verify they look as expected.
Can you write Python code for it?

labels = pd.read_csv('multilabels.csv')

labels.head(5)


I am working on the file "Milestone_3_v4.ipynb".
The first task is:
There are a decent number of movies that are missing data for some of the features.  Since we can't fit our models with NaN values present, we can choose to either remove these movies, or impute values for the missing features.  Let's see how many movies we lose by dropping observations with any missing values.
Can you write Python code for it?

# Count missing data to see how many movies we lose by dropping NAs:
features.shape[0] - features.dropna().shape[0]


I am working on the file "Milestone_3_v4.ipynb".
The first task is:
This means that roughly 80% of our genre labels are predicted correctly for the test set when using the dataset with imputed data. As this performance is worse than leaving erroneous value in place, we will continue to test other methods of handling missing / nonsensical data.
Next, we wondered whether including information about languages and production location significantly improved predictions.
Can you write Python code for it?

#build several dfs with different predictors
a = X_train.ix[:,0:9] #just the basics
b = X_train.ix[:,0:132] #include original language
c = X_train.ix[:,0:365] #include production countries
d = X_train #include spoken language

e = X_test.ix[:,0:9] #just the basics
f = X_test.ix[:,0:132] #include original language
g = X_test.ix[:,0:365] #include production countries
h = X_test #include spoken language

#run models on each df
dfs_train = [a,b,c,d]
dfs_test = [e,f,g,h]
loss = []

for i in range(4):
    mdl = RFC(n_estimators=120, max_depth=60, class_weight = 'balanced', n_jobs=-1).fit(dfs_train[i],y_train)
    pred = mdl.predict(dfs_test[i])
    loss.append(hamming_loss(y_test, pred))

print loss

plt.figure(figsize=(10,5))
index = [0,1,2,3]
plt.bar(index,loss,align="center")
plt.xticks(index, ('Basics', '+ Original language', '+ Production Countries', '+ Spoken Language'))
plt.ylim([0,0.2])
plt.grid(axis='y')
plt.title('Random Forest with Different Attributes of Films')
plt.ylabel('Hamming loss')
plt.show()


I am working on the file "Milestone_3_v4.ipynb".
The first task is:
We also need to worry about features that have erroneous or nonsensical values, e.g. a budget of $0.00.  These erroneous values could have a negative influence on the model, if they're not dealt with.  So we'll create a new dataset with erroneous values replaced by the mean, with which we'll compare the non-altered dataset later.
Can you write Python code for it?

#make a second df with imputed mean values for nonsensical data in quantitative variables
feat3 = feat2.iloc[:,0:9].values
feat3 = feat3.astype('float')
feat3[feat3 == 0] = np.nan
imp = Imputer(missing_values='NaN', strategy='mean', axis=0)
feat3 = imp.fit_transform(feat3)

feat3 = pd.DataFrame(feat3, columns = list(feat2)[0:9])
feat3 = pd.concat([feat3, feat2.iloc[:,10:520]], axis=1).dropna()
feat3.head()

# Pop labels off of imputed dataset
labels3 = feat3.ix[:,512:519]
feat3 = feat3.drop(feat3.columns[[512, 513, 514, 515, 516, 517, 518]], axis=1)
feat2 = feat2.drop(feat2.columns[[513, 514, 515, 516, 517, 518, 519]], axis=1)


I am working on the file "Milestone_3_v4.ipynb".
The first task is:
We can see that 90% of the variance is explained by the first 327 PCs, so we will fit our LR using the first 327 PCs and compare the results to the model using the entire dataset above.
Can you write Python code for it?

# reduce data using PCA
X_train_reduced = pca.transform(scaler.transform(X_train))
print X_train_reduced.shape
X_test_reduced = pca.transform(scaler.transform(X_test))
print X_test_reduced.shape

# Now the same LR as above but with 327 principal components
lr_mdl_pca = OneVsRestClassifier(LogisticRegression(class_weight='balanced', C=100, solver='sag'), n_jobs=-1)
lr_mdl_pca.fit(X_train_reduced[:,0:var90pcs], y_train)

# Test LR performance
lr_pred_pca = lr_mdl_pca.predict(X_test_reduced[:,0:var90pcs])
hamming_loss(y_test, lr_pred_pca)


I am working on the file "Milestone_3_v4.ipynb".
The first task is:
We can see that the Hamming loss is pretty similar to that achieved using the full data set, only slightly worse.  This indicates that the features aren't very compressible, i.e. information is distributed widely across features. It is possible that we will will see gains by implementing a more flexible classifier, so we decided to fit an RFC, and compare results with those above.
Random Forest Classifier
As with the LR, we pre-tuned our RFC using the code in the Appendix and apply those values here.
Can you write Python code for it?

#run a tuned RFC on full data set and calculate Hamming loss
# **tuning parameters chosen previously using code in the Appendix
rfc_mdl = RFC(n_estimators=120, max_depth=60, class_weight ='balanced', n_jobs=-1).fit(X_train,y_train)
rf_pred = rfc_mdl.predict(X_test)
hamming_loss(y_test, rf_pred)


I am working on the file "Milestone_3_v4.ipynb".
The first task is:
We can see that the NB model appears to do a decent job right out of the box, with a Hamming loss of 0.31.  This means that roughly 69% of our genre labels are predicted correctly for the test set.  Since each genre assignment is a 50/50 choice, this would seem to be a large improvement over a random classifier.
We thought perhaps performance could be improved by tuning the NB model.  There isn't really a good way to 'tune' an NB classifier, but we can calibrate the class probabilities and look at how varying the class threshold affects our loss on the test set.
Can you write Python code for it?

from sklearn.calibration import CalibratedClassifierCV

nb_sigmoid = CalibratedClassifierCV(GaussianNB(), method='sigmoid')
nb_mdl = OneVsRestClassifier(nb_sigmoid, n_jobs=-1)
nb_mdl.fit(X_train, y_train)

nb_cal_proba = nb_mdl.predict_proba(X_test)

loss = []
# Choose optimal threshold by manual ROC
for thresh in np.linspace(0.01, 0.99, 99):

    pred = np.array(nb_cal_proba > thresh).astype(int)
    loss.append(hamming_loss(y_test, pred))

plt.plot(np.linspace(0.01, 0.99, 99), loss)
plt.xlabel('threshold'); plt.ylabel('Hamming Loss')

print 'Minumum Hamming loss: {loss}'.format(loss=min(loss))
print 'Loss when zero: {loss}'.format(loss=hamming_loss(y_test, np.zeros(np.shape(y_test))))


I am working on the file "Milestone_3_v4.ipynb".
The first task is:
We can see that we achieve a significantly better Hamming score with RFC (0.17) than with NB or LR.  This means that roughly 83% of our genre labels are predicted correctly for the test set.
Can you write Python code for it?

#calculate the overall accuracy of our predictions
rfc_mdl.score(X_test, y_test)


I am working on the file "MillikanOilDrop/.ipynb_checkpoints/Analysis-checkpoint.ipynb".
The first task is:
Air Viscosity
Can you write Python code for it?

Resistances = np.array([2.065, 2.011, 1.957, 1.959])*1e6 #From thermistor. Used to find temperature. Ohms

temperatures = np.array([24, 25, 26, 26]) #From Thermistor table. Celcius

rho = 885 #kg/m^3. Density of oil drop.
g = 9.80 #gravity

def viscosity(T):
    #Returns the viscosity of air for a given temp in celsius
    # Viscosity units [Ns/m^2]
    return (1.8 + (T-15)/209)*1e-5

viscosities = np.array([viscosity(entry) for entry in temperatures])
viscosities


I am working on the file "MillikanOilDrop/.ipynb_checkpoints/Analysis-checkpoint.ipynb".
The first task is:
Charge to Mass Ratio
e/m = (1.648 +/- 0.287)x10^11 C/kg
Can you write Python code for it?

# From a previous lab
eperm = 1.648*1e11 #C/kg
eperm_u = .287*1e11

m = Charge_Best/eperm
m

dm = np.sqrt( (delta_e/Charge_Best)**2 + (eperm_u/eperm)**2 )*m
dm

print('m = (%.3f +/- %.3f)x10^(-31) Kg' % (m*1e31,dm*1e31))


I am working on the file "MillikanOilDrop/.ipynb_checkpoints/Analysis-checkpoint.ipynb".
The first task is:
Correction Factor
Can you write Python code for it?

def gamma(r):
    b = .082*1e-7
    return (1+b/r)**(-3/2)

gammas = np.array([gamma(entry) for entry in radii])
gammas


I am working on the file "MillikanOilDrop/.ipynb_checkpoints/Analysis-checkpoint.ipynb".
The first task is:
Down Charge
Can you write Python code for it?


I am working on the file "MillikanOilDrop/.ipynb_checkpoints/Analysis-checkpoint.ipynb".
The first task is:
Down Uncer
Can you write Python code for it?


I am working on the file "MillikanOilDrop/.ipynb_checkpoints/Analysis-checkpoint.ipynb".
The first task is:
Drop Charge
Can you write Python code for it?

def down_charge(vd,vf,gamma,mass):
    return gamma*mass*g*((vd/vf) - 1)/E

def delta_qd(qu,dm, m, dvu, vu, dvf, vf):
    return np.sqrt((dm/m)**2 + (dvu/(vu-vf))**2 + ((vu/vf)*dvf/(vf-vu))**2)*qu

qd0 = down_charge(td0_speed, speeds[0], gammas[0], masses[0])
qd1 = down_charge(td1_speed, speeds[1], gammas[1], masses[1])
qd2 = down_charge(td2_speed, speeds[2], gammas[2], masses[2])
qd3 = down_charge(td3_speed, speeds[3], gammas[3], masses[3])
qd = np.array([qd0,qd1,qd2,qd3])
qd*1e19

qd0_u = delta_qd(qd0,masses_u[0],masses[0],td0_speed_u, td0_speed, speeds_u[0], speeds[0])
qd1_u = delta_qd(qd1,masses_u[1],masses[1],td1_speed_u, td1_speed, speeds_u[1], speeds[1])
qd2_u = delta_qd(qd2,masses_u[2],masses[2],td2_speed_u, td2_speed, speeds_u[2], speeds[2])
qd3_u = delta_qd(qd3,masses_u[3],masses[3],td3_speed_u, td3_speed, speeds_u[3], speeds[3])
qd_u = np.array([qd0_u,qd1_u,qd2_u,qd3_u])
qd_u*1e19


I am working on the file "MillikanOilDrop/.ipynb_checkpoints/Analysis-checkpoint.ipynb".
The first task is:
Drop Charge
Downward Movement
Can you write Python code for it?

df['Run 1']

# Fall time in seconds when V = -
td = np.array([df[key]['t_d (s)'] for key in keys])
td

#### tu0: Time Up 0'th drop.
#### Average Time
#### Std of Times
#### Resulting Speed
#### Speed Uncertainty

td0 = td[0] #All values here are about the same
avg_td0 = np.mean(td0)
std_td0 = np.std(td0)
td0_speed = .5*1e-3/avg_td0
td0_speed_u = .5*1e-3/(avg_td0**2)*std_td0/np.sqrt(len(td0))

td1 = np.array([td[1]]) #All values here are about the same
avg_td1 = np.mean(td1)
std_td1 = np.std(td1)
td1_speed = .5*1e-3/avg_td1
td1_speed_u = .5*1e-3/(avg_td1**2)*std_td1/np.sqrt(len(td1))

td2 = np.array([td[2]]) #All values here are about the same
avg_td2 = np.mean(td2)
std_td2 = np.std(td2)
td2_speed = .5*1e-3/avg_td2
td2_speed_u = .5*1e-3/(avg_td2**2)*std_td2/np.sqrt(len(td2))

td3 = td[3] #All values here are about the same
avg_td3 = np.mean(td3)
std_td3 = np.std(td3)
td3_speed = .5*1e-3/avg_td3
td3_speed_u = .5*1e-3/(avg_td3**2)*std_td3/np.sqrt(len(td3))

avg_td0

std_td0/np.sqrt(len(td0))


I am working on the file "MillikanOilDrop/.ipynb_checkpoints/Analysis-checkpoint.ipynb".
The first task is:
Drop Radius
Can you write Python code for it?

def radius(speed,visc):
    return np.sqrt(9*visc*speed/(2*g*rho))

radii = np.array([radius(speeds[i],viscosities[i]) for i in np.arange(4)]) #Blob radius in meters
radii


I am working on the file "MillikanOilDrop/.ipynb_checkpoints/Analysis-checkpoint.ipynb".
The first task is:
Droplet Mass
Can you write Python code for it?

def mass(r):
    return (4/3)*np.pi*rho*r**3

masses = np.array([mass(entry) for entry in radii])
masses


I am working on the file "MillikanOilDrop/.ipynb_checkpoints/Analysis-checkpoint.ipynb".
The first task is:
E Field
Can you write Python code for it?

d = 7.63*1e-3 #Electrode spacer thickness in meters
V = 301.8 #Volts. Constant through all measurements

E = V/d #V/m
E


I am working on the file "MillikanOilDrop/.ipynb_checkpoints/Analysis-checkpoint.ipynb".
The first task is:
Fall Speed
All measurements were of the time to travel .5mm
Can you write Python code for it?

# Units of m/s
speeds = np.array([(.5*1e-3)/entry for entry in avg_tf ])
speeds


I am working on the file "MillikanOilDrop/.ipynb_checkpoints/Analysis-checkpoint.ipynb".
The first task is:
Mask points that lie outside 2.5$\sigma$
Can you write Python code for it?

tf = np.array([ma.masked_outside(entry, np.mean(entry) - 2.5*np.std(entry), np.mean(entry) + 2.5*np.std(entry))
                     for entry in tf])


I am working on the file "MillikanOilDrop/.ipynb_checkpoints/Analysis-checkpoint.ipynb".
The first task is:
Plank Constant
Can you write Python code for it?


I am working on the file "MillikanOilDrop/.ipynb_checkpoints/Analysis-checkpoint.ipynb".
The first task is:
Read-In
Can you write Python code for it?

df = pd.read_excel('Data.xlsx', sheetname=None)

df['Run 1']

keys = ['Run 1', 'Run 2', 'Run 3', 'Run 4']

# Fall time in seconds when V = 0
tf = np.array([df[key]['t_f (s)'] for key in keys])
tf

# Average Fall times for each run (seconds)
avg_tf = np.array([np.mean(tf[i]) for i in np.arange(4)])
avg_tf

# Standard Deviation Fall times for each run (seconds)
std_tf = np.array([np.std(tf[i]) for i in np.arange(4)])
std_tf


I am working on the file "MillikanOilDrop/.ipynb_checkpoints/Analysis-checkpoint.ipynb".
The first task is:
Uncertainty
Can you write Python code for it?

# Standard deviation of the mean (seconds)
tf_u = np.array([std_tf[i]/np.sqrt(len(tf[i])) for i in np.arange(4)])

tf_u


I am working on the file "MillikanOilDrop/.ipynb_checkpoints/Analysis-checkpoint.ipynb".
The first task is:
Upward Movement
Can you write Python code for it?

# Rise time in seconds when V = +
tu = np.array([df[key]['t_u (s)'] for key in keys])
tu

#### tu0: Time Up 0'th drop.
#### Average Time
#### Std of Times
#### Resulting Speed
#### Speed Uncertainty

tu0 = ma.masked_array(tu[0],mask=[0,0,0,0,0,0,0,0,0,1])
avg_tu0 = np.mean(tu0)
std_tu0 = np.std(tu0)
tu0_speed = .5*1e-3/avg_tu0
tu0_speed_u = .5*1e-3/(avg_tu0**2)*std_tu0/np.sqrt(len(tu0))

tu1 = ma.masked_array(tu[1],mask=[0,0,1,1,0,0,0,0,0,0])
avg_tu1 = np.mean(tu1)
std_tu1 = np.std(tu1)
tu1_speed = .5*1e-3/avg_tu1
tu1_speed_u = .5*1e-3/(avg_tu1**2)*std_tu1/np.sqrt(len(tu1))

tu2_1 = ma.masked_array(tu[2],mask=[0,1,1,0,1,0,0,0,0,0])
avg_tu2_1 = np.mean(tu2_1)
std_tu2_1 = np.std(tu2_1)
tu2_1_speed = .5*1e-3/avg_tu2_1
tu2_1_speed_u = .5*1e-3/(avg_tu2_1**2)*std_tu2_1/np.sqrt(len(tu2_1))

tu2_2 = ma.masked_array(tu[2],mask=[1,0,0,1,0,1,1,1,1,1])
avg_tu2_2 = np.mean(tu2_2)
std_tu2_2 = np.std(tu2_2)
tu2_2_speed = .5*1e-3/avg_tu2_2
tu2_2_speed_u = .5*1e-3/(avg_tu2_2**2)*std_tu2_2/np.sqrt(len(tu2_2))

tu3 = tu[3]
avg_tu3 = np.mean(tu3)
std_tu3 = np.std(tu3)
tu3_speed = .5*1e-3/avg_tu3
tu3_speed_u = .5*1e-3/(avg_tu3**2)*std_tu3/np.sqrt(len(tu3))

#This data can't be used unless the free fall time data is analyzed
tu4 = np.array([4.20, 3.51, 3.44, 3.21, 3.56, 3.27, 3.25]) #This data from an extra column not entered into excel
avg_tu4 = np.mean(tu4)
std_tu4 = np.std(tu4)
tu4_speed = .5*1e-3/avg_tu4
tu4_speed_u = .5*1e-3/(avg_tu4**2)*std_tu4/np.sqrt(len(tu4))

avg_tu0

std_tu0/np.sqrt(len(tu0))

def up_charge(vu,vf,gamma,mass):
    return gamma*mass*g*((vu/vf) + 1)/E

qu0 = up_charge(tu0_speed, speeds[0], gammas[0], masses[0])
qu1 = up_charge(tu1_speed, speeds[1], gammas[1], masses[1])
qu2_1 = up_charge(tu2_1_speed, speeds[2], gammas[2], masses[2])
qu2_2 = up_charge(tu2_2_speed, speeds[2], gammas[2], masses[2])
qu3 = up_charge(tu3_speed, speeds[3], gammas[3], masses[3])
qu = np.array([qu0,qu1,qu2_1,qu2_2,qu3])
qu*1e19


I am working on the file "MillikanOilDrop/.ipynb_checkpoints/Analysis-checkpoint.ipynb".
The first task is:
Upwards Uncer
Can you write Python code for it?


I am working on the file "MillikanOilDrop/.ipynb_checkpoints/Analysis-checkpoint.ipynb".
The first task is:
\begin{equation}
    v = \frac{h}{t}
\end{equation}
\begin{equation}
    \delta v = \frac{h}{t^2} \delta t
\end{equation}
Can you write Python code for it?

#Units of m/s
speeds_u = np.array([((.5*1e-3)/avg_tf[i]**2)*tf_u[i] for i in np.arange(4)])
speeds_u


I am working on the file "MillikanOilDrop/Analysis.ipynb".
The first task is:
Air Viscosity
Can you write Python code for it?