I am working on the file "FDMS/TME3/Model_V5.ipynb".
The first task is:
Predict on testset
Can you write Python code for it?

%%time
filename = "data/reduced_test_5000.csv"
#filename = "data/test.csv"
test = pd.read_csv(filename)
test = test.set_index('Id')

features_columns = list([u'Ref', u'Ref_5x5_10th',
       u'Ref_5x5_50th', u'Ref_5x5_90th', u'RefComposite',
       u'RefComposite_5x5_10th', u'RefComposite_5x5_50th',
       u'RefComposite_5x5_90th', u'RhoHV', u'RhoHV_5x5_10th',
       u'RhoHV_5x5_50th', u'RhoHV_5x5_90th', u'Zdr', u'Zdr_5x5_10th',
       u'Zdr_5x5_50th', u'Zdr_5x5_90th', u'Kdp', u'Kdp_5x5_10th',
       u'Kdp_5x5_50th', u'Kdp_5x5_90th'])

def getX(raw):
    selected_columns = list([ u'minutes_past',u'radardist_km', u'Ref', u'Ref_5x5_10th',
       u'Ref_5x5_50th', u'Ref_5x5_90th', u'RefComposite',
       u'RefComposite_5x5_10th', u'RefComposite_5x5_50th',
       u'RefComposite_5x5_90th', u'RhoHV', u'RhoHV_5x5_10th',
       u'RhoHV_5x5_50th', u'RhoHV_5x5_90th', u'Zdr', u'Zdr_5x5_10th',
       u'Zdr_5x5_50th', u'Zdr_5x5_90th', u'Kdp', u'Kdp_5x5_10th',
       u'Kdp_5x5_50th', u'Kdp_5x5_90th'])

    data = raw[selected_columns]

    docX= []
    for i in data.index.unique():
        if isinstance(data.loc[i],pd.core.series.Series):
            m = [data.loc[i].as_matrix()]
            docX.append(m)
        else:
            m = data.loc[i].as_matrix()
            docX.append(m)
    X = np.array(docX)
    return X

#%%time
#X=getX(test)

#tmp = []
#for i in X:
#    tmp.append(len(i))
#tmp = np.array(tmp)
#sns.countplot(tmp,order=range(tmp.min(),tmp.max()+1))
#plt.title("Number of ID per number of observations\n(On test dataset)")
#plt.plot()

#testFull = test.dropna()
testNoFullNan = test.loc[test[features_columns].dropna(how='all').index.unique()]

%%time
X=getX(testNoFullNan)  # 1min
#XX = [np.array(t).mean(0) for t in X]  # 10s

XX=[]
for t in X:
    nm = np.nanmean(t,0)
    for idx,j in enumerate(nm):
        if np.isnan(j):
            nm[idx]=global_means[idx]
    XX.append(nm)
XX=np.array(XX)

# rescale to clip min at 0 (for non negative matrix factorization)
XX_rescaled=XX[:,:]-np.min(XX,0)

%%time
W =...


I am working on the file "FDMS/TME3/Model_V5.ipynb".
The first task is:
We prepare 3 subsets
Fully filled data only
Used at first to evaluate the models and no worry about features completion
Can you write Python code for it?

#noAnyNan = raw.loc[raw[features_columns].dropna(how='any').index.unique()]
noAnyNan = raw.dropna()

noFullNan = raw.loc[raw[features_columns].dropna(how='all').index.unique()]

fullNan = raw.drop(raw[features_columns].dropna(how='all').index)

print(len(raw))
print(len(noAnyNan))
print(len(noFullNan))
print(len(fullNan))


I am working on the file "FDMS/TME3/Model_V5.ipynb".
The first task is:
We regroup the data by ID
Can you write Python code for it?

# We select all features except for the minutes past,
# because we ignore the time repartition of the sequence for now

features_columns = list([u'Ref', u'Ref_5x5_10th',
       u'Ref_5x5_50th', u'Ref_5x5_90th', u'RefComposite',
       u'RefComposite_5x5_10th', u'RefComposite_5x5_50th',
       u'RefComposite_5x5_90th', u'RhoHV', u'RhoHV_5x5_10th',
       u'RhoHV_5x5_50th', u'RhoHV_5x5_90th', u'Zdr', u'Zdr_5x5_10th',
       u'Zdr_5x5_50th', u'Zdr_5x5_90th', u'Kdp', u'Kdp_5x5_10th',
       u'Kdp_5x5_50th', u'Kdp_5x5_90th'])

def getXy(raw):
    selected_columns = list([ u'minutes_past',u'radardist_km', u'Ref', u'Ref_5x5_10th',
       u'Ref_5x5_50th', u'Ref_5x5_90th', u'RefComposite',
       u'RefComposite_5x5_10th', u'RefComposite_5x5_50th',
       u'RefComposite_5x5_90th', u'RhoHV', u'RhoHV_5x5_10th',
       u'RhoHV_5x5_50th', u'RhoHV_5x5_90th', u'Zdr', u'Zdr_5x5_10th',
       u'Zdr_5x5_50th', u'Zdr_5x5_90th', u'Kdp', u'Kdp_5x5_10th',
       u'Kdp_5x5_50th', u'Kdp_5x5_90th'])

    data = raw[selected_columns]

    docX, docY = [], []
    for i in data.index.unique():
        if isinstance(data.loc[i],pd.core.series.Series):
            m = [data.loc[i].as_matrix()]
            docX.append(m)
            docY.append(float(raw.loc[i]["Expected"]))
        else:
            m = data.loc[i].as_matrix()
            docX.append(m)
            docY.append(float(raw.loc[i][:1]["Expected"]))
    X , y = np.array(docX) , np.array(docY)
    return X,y


I am working on the file "FDMS/TME3/Model_V5.ipynb".
The first task is:
max prof 24
nb trees 84
min sample per leaf 17
min sample to split 51
Can you write Python code for it?

svr = SVR(kernel='rbf', C=800.0)

%%time
srv = svr.fit(X_train,y_train)

print(svr.score(X_train,y_train))
print(svr.score(X_test,y_test))

err = (svr.predict(X_train)-y_train)**2
err.sum()/len(err)

err = (svr.predict(X_test)-y_test)**2
err.sum()/len(err)

%%time
svr_score = cross_val_score(svr, XX, y, cv=5)
print("Score: %s\nMean: %.03f"%(svr_score,svr_score.mean()))


I am working on the file "FDMS/TME3/Model_V6.ipynb".
The first task is:

Can you write Python code for it?

t = []
for i in XX:
    t.append(np.count_nonzero(~np.isnan(i)) / float(i.size))
pd.DataFrame(np.array(t)).describe()

import pickle

with open('data/knn.pkl', 'wb') as fid:
    pickle.dump(knn, fid)
with open('data/etreg.pkl', 'wb') as fid:
    pickle.dump(etreg, fid)
with open('data/rfr.pkl', 'wb') as fid:
    pickle.dump(rfr, fid)
with open('data/xgbr.pkl', 'wb') as fid:
    pickle.dump(xgbr, fid)
with open('data/gbr.pkl', 'wb') as fid:
    pickle.dump(gbr, fid)

# load it again
#with open('my_dumped_classifier.pkl', 'rb') as fid:
#    gnb_loaded = pickle.load(fid)

#knn,etreg,rfr,xgbr,gbr


knn = KNeighborsRegressor(n_neighbors=6,weights='distance',algorithm='ball_tree')

#parameters = {'weights':('distance','uniform'),'algorithm':('auto', 'ball_tree', 'kd_tree', 'brute')}
parameters = {'n_neighbors':range(1,10,1)}
grid_knn = grid_search.GridSearchCV(knn, parameters,scoring=manualScorer)

%%time
grid_knn.fit(X_train,y_train)

print(grid_knn.grid_scores_)
print("Best: ",grid_knn.best_params_)

knn = grid_knn.best_estimator_

%%time
knn= knn.fit(XX,y)

knn= knn.fit(X_train,y_train)

print(knn.score(X_train,y_train))
print(knn.score(X_test,y_test))

err = (knn.predict(X_train)-y_train)**2
err.sum()/len(err)

err = (knn.predict(X_test)-y_test)**2
err.sum()/len(err)


etreg = ExtraTreesRegressor(n_estimators=200, max_depth=None, min_samples_split=1, random_state=0)

parameters = {'n_estimators':range(100,200,20)}
grid_rf = grid_search.GridSearchCV(etreg, parameters,n_jobs=2,scoring=manualScorer)

%%time
grid_rf.fit(X_train,y_train)

print(grid_rf.grid_scores_)
print("Best: ",grid_rf.best_params_)

grid_rf.best_params_

#etreg = grid_rf.best_estimator_

%%time
etreg = etreg.fit(XX,y)

%%time
etreg = etreg.fit(X_train,y_train)

print(etreg.score(X_train,y_train))
print(etreg.score(X_test,y_test))

err = (etreg.predict(X_train)-y_train)**2
err.sum()/len(err)

err = (etreg.predict(X_test)-y_test)**2
err.sum()/len(err)


I am working on the file "FDMS/TME3/Model_V6.ipynb".
The first task is:
On fully filled dataset
Can you write Python code for it?


I am working on the file "FDMS/TME3/Model_V6.ipynb".
The first task is:
Per wikipedia, a value of more than 421 mm/h is considered "Extreme/large hail"
If we encounter the value 327.40 meter per hour, we should probably start building Noah's ark
Therefor, it seems reasonable to drop values too large, considered as outliers
Can you write Python code for it?

# Considering that the gauge may concentrate the rainfall, we set the cap to 1000
# Comment this line to analyse the complete dataset
l = len(raw)
raw = raw[raw['Expected'] < 300]  #1000
print("Dropped %d (%0.2f%%)"%(l-len(raw),(l-len(raw))/float(l)*100))

raw.head(5)

raw.describe()


I am working on the file "FDMS/TME3/Model_V6.ipynb".
The first task is:
Predicitons
As a first try, we make predictions on the complete data, and return the 50th percentile and uncomplete and fully empty data
Can you write Python code for it?

%%time
#X,y=getXy(noAnyNan)
X,y=getXy(noFullNan)

#%%time
#XX = [np.array(t).mean(0) for t in X]
#XX = np.array([np.append(np.nanmean(np.array(t),0),(np.array(t)[1:] - np.array(t)[:-1]).sum(0) ) for t in X])

%%time
global_means = np.nanmean(noFullNan,0)

XX=[]
for t in X:
    nm = np.nanmean(t,0)
    for idx,j in enumerate(nm):
        if np.isnan(j):
            nm[idx]=global_means[idx]
    XX.append(nm)
XX=np.array(XX)

# rescale to clip min at 0 (for non negative matrix factorization)
XX_rescaled=XX[:,:]-np.min(XX,0)

%%time
nmf = NMF(max_iter=1000)
W = nmf.fit_transform(XX_rescaled)
#H = nn.components_

# used to fill fully empty datas
global_means = np.nanmean(noFullNan,0)

# reduce the sequence structure of the data and produce
# new hopefully informatives features
def addFeatures(X,mf=0):
    # used to fill fully empty datas
    #global_means = np.nanmean(X,0)

    XX=[]
    nbFeatures=float(len(X[0][0]))
    for idxt,t in enumerate(X):

        # compute means, ignoring nan when possible, marking it when fully filled with nan
        nm = np.nanmean(t,0)
        tt=[]
        for idx,j in enumerate(nm):
            if np.isnan(j):
                nm[idx]=global_means[idx]
                tt.append(1)
            else:
                tt.append(0)
        tmp = np.append(nm,np.append(tt,tt.count(0)/nbFeatures))

        # faster if working on fully filled data:
        #tmp = np.append(np.nanmean(np.array(t),0),(np.array(t)[1:] - np.array(t)[:-1]).sum(0) )

        # add the percentiles
        tmp = np.append(tmp,np.nanpercentile(t,10,axis=0))
        tmp = np.append(tmp,np.nanpercentile(t,50,axis=0))
        tmp = np.append(tmp,np.nanpercentile(t,90,axis=0))

        for idx,i in enumerate(tmp):
            if np.isnan(i):
                tmp[idx]=0

        # adding the dbz as a feature
        test = t
        try:
            taa=test[:,0]
        except TypeError:
            taa=[test[0][0]]
        valid_time = np.zeros_like(taa)
        valid_time[0] =...


I am working on the file "FDMS/TME3/Model_V6.ipynb".
The first task is:
Predict on testset
Can you write Python code for it?

%%time
#filename = "data/reduced_test_5000.csv"
filename = "data/test.csv"
test = pd.read_csv(filename)
test = test.set_index('Id')

features_columns = list([u'Ref', u'Ref_5x5_10th',
       u'Ref_5x5_50th', u'Ref_5x5_90th', u'RefComposite',
       u'RefComposite_5x5_10th', u'RefComposite_5x5_50th',
       u'RefComposite_5x5_90th', u'RhoHV', u'RhoHV_5x5_10th',
       u'RhoHV_5x5_50th', u'RhoHV_5x5_90th', u'Zdr', u'Zdr_5x5_10th',
       u'Zdr_5x5_50th', u'Zdr_5x5_90th', u'Kdp', u'Kdp_5x5_10th',
       u'Kdp_5x5_50th', u'Kdp_5x5_90th'])

def getX(raw):
    selected_columns = list([ u'minutes_past',u'radardist_km', u'Ref', u'Ref_5x5_10th',
       u'Ref_5x5_50th', u'Ref_5x5_90th', u'RefComposite',
       u'RefComposite_5x5_10th', u'RefComposite_5x5_50th',
       u'RefComposite_5x5_90th', u'RhoHV', u'RhoHV_5x5_10th',
       u'RhoHV_5x5_50th', u'RhoHV_5x5_90th', u'Zdr', u'Zdr_5x5_10th',
       u'Zdr_5x5_50th', u'Zdr_5x5_90th', u'Kdp', u'Kdp_5x5_10th',
       u'Kdp_5x5_50th', u'Kdp_5x5_90th'])

    data = raw[selected_columns]

    docX= []
    for i in data.index.unique():
        if isinstance(data.loc[i],pd.core.series.Series):
            m = [data.loc[i].as_matrix()]
            docX.append(m)
        else:
            m = data.loc[i].as_matrix()
            docX.append(m)
    X = np.array(docX)
    return X

#%%time
#X=getX(test)

#tmp = []
#for i in X:
#    tmp.append(len(i))
#tmp = np.array(tmp)
#sns.countplot(tmp,order=range(tmp.min(),tmp.max()+1))
#plt.title("Number of ID per number of observations\n(On test dataset)")
#plt.plot()

#testFull = test.dropna()
testNoFullNan = test.loc[test[features_columns].dropna(how='all').index.unique()]

%%time
test_X=getX(testNoFullNan)  # 1min
#XX = [np.array(t).mean(0) for t in X]  # 10s

test_XX=[]
for t in test_X:
    nm = np.nanmean(t,0)
    for idx,j in enumerate(nm):
        if np.isnan(j):
            nm[idx]=global_means[idx]
    test_XX.append(nm)
test_XX=np.array(test_XX)

# rescale to clip min at 0 (for non negative matrix...


I am working on the file "FDMS/TME3/Model_V6.ipynb".
The first task is:
We regroup the data by ID
Can you write Python code for it?


I am working on the file "FDMS/TME3/Model_V6.ipynb".
The first task is:
max prof 24
nb trees 84
min sample per leaf 17
min sample to split 51
Can you write Python code for it?


I am working on the file "FDMS/TME3/Model_V7.ipynb".
The first task is:

Can you write Python code for it?

gbr = GradientBoostingRegressor(loss='ls', learning_rate=0.1, n_estimators=900,
                                subsample=1.0, min_samples_split=2, min_samples_leaf=1, max_depth=4, init=None,
                                random_state=None, max_features=None, alpha=0.5,
                                verbose=0, max_leaf_nodes=None, warm_start=False)

%%time
gbr = gbr.fit(X_train,y_train)
#os.system('say "終わりだ"') #its over!

#parameters = {'max_depth':range(2,5,1),'alpha':[0.5,0.6,0.7,0.8,0.9]}
#parameters = {'subsample':[0.2,0.4,0.5,0.6,0.8,1]}
#parameters = {'subsample':[0.2,0.5,0.6,0.8,1],'n_estimators':[800,1000,1200]}
#parameters = {'max_depth':range(2,4,1)}
parameters = {'n_estimators':[400,800,1100]}
#parameters = {'loss':['ls', 'lad', 'huber', 'quantile'],'alpha':[0.3,0.5,0.8,0.9]}
#parameters = {'learning_rate':[0.1,0.5,0.9]}

grid_gbr = grid_search.GridSearchCV(gbr, parameters,n_jobs=2,scoring=manualScorer)

%%time
grid_gbr = grid_gbr.fit(X_train,y_train)

print(grid_gbr.grid_scores_)
print("Best: ",grid_gbr.best_params_)

print(gbr.score(X_train,y_train))
print(gbr.score(X_test,y_test))


# used for the crossvalidation
def manualScorer(estimator, X, y):
    err = (estimator.predict(X_test)-y_test)**2
    return -err.sum()/len(err)


I am working on the file "FDMS/TME3/Model_V7.ipynb".
The first task is:
13.765.202 lines in train.csv
8.022.757 lines in test.csv

Few words about the dataset
Predictions is made in the USA corn growing states (mainly Iowa, Illinois, Indiana) during the season with the highest rainfall (as illustrated by Iowa for the april to august months)
The Kaggle page indicate that the dataset have been shuffled, so working on a subset seems acceptable
The test set is not a extracted from the same data as the training set however, which make the evaluation trickier
Load the dataset
Can you write Python code for it?

%%time
#filename = "data/train.csv"
filename = "data/reduced_train_100000.csv"
#filename = "data/reduced_train_1000000.csv"
raw = pd.read_csv(filename)
raw = raw.set_index('Id')

raw.columns

raw['Expected'].describe()


I am working on the file "FDMS/TME3/Model_V7.ipynb".
The first task is:
Extra Trees Regressor
Similar to random forest, slightly faster and performed better here
Can you write Python code for it?

etreg = ExtraTreesRegressor(n_estimators=200, max_depth=None, min_samples_split=1, random_state=0)

parameters = {'n_estimators':range(100,200,20)}
grid_rf = grid_search.GridSearchCV(etreg, parameters,n_jobs=2,scoring=manualScorer)

%%time
grid_rf.fit(X_train,y_train)

print(grid_rf.grid_scores_)
print("Best: ",grid_rf.best_params_)

#etreg = grid_rf.best_estimator_

%%time
etreg = etreg.fit(X_train,y_train)

print(etreg.score(X_train,y_train))
print(etreg.score(X_test,y_test))


I am working on the file "FDMS/TME3/Model_V7.ipynb".
The first task is:
Fully empty data
Can't do much with those, therefor we isolate them and decide what value to predict using the data analysis we made during the Data Visualization
Can you write Python code for it?

fullNan = raw.drop(raw[features_columns].dropna(how='all').index)

print("Complete dataset:",len(raw))
print("Fully filled:    ",len(noAnyNan))
print("Partly filled:   ",len(noFullNan))
print("Fully empty:     ",len(fullNan))


I am working on the file "FDMS/TME3/Model_V7.ipynb".
The first task is:
Gradient Boosting Regressor
Can you write Python code for it?

# the dbz feature does not influence xgbr so much
xgbr = xgb.XGBRegressor(max_depth=6, learning_rate=0.1, n_estimators=700, silent=True,
                        objective='reg:linear', nthread=-1, gamma=0, min_child_weight=1,
                        max_delta_step=0, subsample=1, colsample_bytree=1, colsample_bylevel=1,
                        reg_alpha=0, reg_lambda=1, scale_pos_weight=1, base_score=0.5,
                        seed=0, missing=None)

%%time
xgbr = xgbr.fit(X_train,y_train)

# without the nmf features
# print(xgbr.score(X_train,y_train))
## 0.993948231144
# print(xgbr.score(X_test,y_test))
## 0.613931733332

# with nmf features
print(xgbr.score(X_train,y_train))
print(xgbr.score(X_test,y_test))


I am working on the file "FDMS/TME3/Model_V7.ipynb".
The first task is:
Here for legacy
Can you write Python code for it?

in_dim = len(XX[0])
out_dim = 1

model = Sequential()
# Dense(64) is a fully-connected layer with 64 hidden units.
# in the first layer, you must specify the expected input data shape:
# here, 20-dimensional vectors.
model.add(Dense(128, input_shape=(in_dim,)))
model.add(Activation('tanh'))
model.add(Dropout(0.5))
model.add(Dense(1, init='uniform'))
model.add(Activation('linear'))

#sgd = SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True)
#model.compile(loss='mean_squared_error', optimizer=sgd)

rms = RMSprop()
model.compile(loss='mean_squared_error', optimizer=rms)

#model.fit(X_train, y_train, nb_epoch=20, batch_size=16)
#score = model.evaluate(X_test, y_test, batch_size=16)

prep = []
for i in y_train:
    prep.append(min(i,20))

prep=np.array(prep)
mi,ma = prep.min(),prep.max()
fy = (prep-mi) / (ma-mi)
#my = fy.max()
#fy = fy/fy.max()

model.fit(np.array(X_train), fy, batch_size=10, nb_epoch=10, validation_split=0.1)

pred = model.predict(np.array(X_test))*ma+mi

err = (pred-y_test)**2
err.sum()/len(err)

r = random.randrange(len(X_train))
print("(Train) Prediction %0.4f, True: %0.4f"%(model.predict(np.array([X_train[r]]))[0][0]*ma+mi,y_train[r]))

r = random.randrange(len(X_test))
print("(Test)  Prediction %0.4f, True: %0.4f"%(model.predict(np.array([X_test[r]]))[0][0]*ma+mi,y_test[r]))


I am working on the file "FDMS/TME3/Model_V7.ipynb".
The first task is:
Knn
We tried this one because of the Kaggle challenge on Taxi, as the analysis on sequence may contain some similarities
Can you write Python code for it?

knn = KNeighborsRegressor(n_neighbors=6,weights='distance',algorithm='ball_tree')

#parameters = {'weights':('distance','uniform'),'algorithm':('auto', 'ball_tree', 'kd_tree', 'brute')}
parameters = {'n_neighbors':range(1,10,1)}
grid_knn = grid_search.GridSearchCV(knn, parameters,scoring=manualScorer)

%%time
grid_knn.fit(X_train,y_train)

print(grid_knn.grid_scores_)
print("Best: ",grid_knn.best_params_)

knn = grid_knn.best_estimator_

knn= knn.fit(X_train,y_train)

print(knn.score(X_train,y_train))
print(knn.score(X_test,y_test))


I am working on the file "FDMS/TME3/Model_V7.ipynb".
The first task is:
Now let's add some features.
We start by averaging on each row every sequence of measures, and we use a Non-negative matrix factorisation to produce a new feature indicating the location of each measures in a latent space. Hopefully this will allow us to predict similar sequences the same way.
("RuntimeWarning: Mean of empty slice" is normal)
Can you write Python code for it?

# used to fill fully empty datas
global_means = np.nanmean(noFullNan,0)

XX=[]
for t in X:
    nm = np.nanmean(t,0)
    for idx,j in enumerate(nm):
        if np.isnan(j):
            nm[idx]=global_means[idx]
    XX.append(nm)
XX=np.array(XX)

# rescale to clip min at 0 (for non negative matrix factorization)
XX_rescaled=XX[:,:]-np.min(XX,0)

%%time
nmf = NMF(max_iter=5000)
W = nmf.fit_transform(XX_rescaled)
#H = nn.components_


I am working on the file "FDMS/TME3/Model_V7.ipynb".
The first task is:
Now we add other simpler features:
   * mean
   * number of NaNs
   * flag for each row only filled with NaN
   * percentiles (10%, 50%, 90%)
   * dBZ    (not Dragon Ball Z)
Can you write Python code for it?

# reduce the sequence structure of the data and produce
# new hopefully informatives features
def addFeatures(X,mf=0):
    # used to fill fully empty datas
    #global_means = np.nanmean(X,0)

    XX=[]
    nbFeatures=float(len(X[0][0]))
    for idxt,t in enumerate(X):

        # compute means, ignoring nan when possible, marking it when fully filled with nan
        nm = np.nanmean(t,0)
        tt=[]
        for idx,j in enumerate(nm):
            if np.isnan(j):
                nm[idx]=global_means[idx]
                tt.append(1)
            else:
                tt.append(0)
        tmp = np.append(nm,np.append(tt,tt.count(0)/nbFeatures))

        # faster if working on fully filled data:
        #tmp = np.append(np.nanmean(np.array(t),0),(np.array(t)[1:] - np.array(t)[:-1]).sum(0) )

        # add the percentiles
        tmp = np.append(tmp,np.nanpercentile(t,10,axis=0))
        tmp = np.append(tmp,np.nanpercentile(t,50,axis=0))
        tmp = np.append(tmp,np.nanpercentile(t,90,axis=0))

        for idx,i in enumerate(tmp):
            if np.isnan(i):
                tmp[idx]=0

        # adding the dbz as a feature
        test = t
        try:
            taa=test[:,0]
        except TypeError:
            taa=[test[0][0]]
        valid_time = np.zeros_like(taa)
        valid_time[0] = taa[0]
        for n in xrange(1,len(taa)):
            valid_time[n] = taa[n] - taa[n-1]
        valid_time[-1] = valid_time[-1] + 60 - np.sum(valid_time)
        valid_time = valid_time / 60.0

        sum=0
        try:
            column_ref=test[:,2]
        except TypeError:
            column_ref=[test[0][2]]
        for dbz, hours in zip(column_ref, valid_time):
            # See: https://en.wikipedia.org/wiki/DBZ_(meteorology)
            if np.isfinite(dbz):
                mmperhr = pow(pow(10, dbz/10)/200, 0.625)
                sum = sum + mmperhr * hours

        if not(mf is 0):
            tmp = np.append(tmp,mf[idxt])

       ...


I am working on the file "FDMS/TME3/Model_V7.ipynb".
The first task is:
Numerous features, with a lot of variability.
We regroup the data by ID
Can you write Python code for it?

# We select all features except for the minutes past,
# because we ignore the time repartition of the sequence for now

features_columns = list([u'Ref', u'Ref_5x5_10th',
       u'Ref_5x5_50th', u'Ref_5x5_90th', u'RefComposite',
       u'RefComposite_5x5_10th', u'RefComposite_5x5_50th',
       u'RefComposite_5x5_90th', u'RhoHV', u'RhoHV_5x5_10th',
       u'RhoHV_5x5_50th', u'RhoHV_5x5_90th', u'Zdr', u'Zdr_5x5_10th',
       u'Zdr_5x5_50th', u'Zdr_5x5_90th', u'Kdp', u'Kdp_5x5_10th',
       u'Kdp_5x5_50th', u'Kdp_5x5_90th'])

def getXy(raw):
    selected_columns = list([ u'minutes_past',u'radardist_km', u'Ref', u'Ref_5x5_10th',
       u'Ref_5x5_50th', u'Ref_5x5_90th', u'RefComposite',
       u'RefComposite_5x5_10th', u'RefComposite_5x5_50th',
       u'RefComposite_5x5_90th', u'RhoHV', u'RhoHV_5x5_10th',
       u'RhoHV_5x5_50th', u'RhoHV_5x5_90th', u'Zdr', u'Zdr_5x5_10th',
       u'Zdr_5x5_50th', u'Zdr_5x5_90th', u'Kdp', u'Kdp_5x5_10th',
       u'Kdp_5x5_50th', u'Kdp_5x5_90th'])

    data = raw[selected_columns]

    docX, docY = [], []
    for i in data.index.unique():
        if isinstance(data.loc[i],pd.core.series.Series):
            m = [data.loc[i].as_matrix()]
            docX.append(m)
            docY.append(float(raw.loc[i]["Expected"]))
        else:
            m = data.loc[i].as_matrix()
            docX.append(m)
            docY.append(float(raw.loc[i][:1]["Expected"]))
    X , y = np.array(docX) , np.array(docY)
    return X,y


I am working on the file "FDMS/TME3/Model_V7.ipynb".
The first task is:
Our data look like this:
Can you write Python code for it?


I am working on the file "FDMS/TME3/Model_V7.ipynb".
The first task is:
Per wikipedia, a value of more than 421 mm/h is considered "Extreme/large hail"
If we encounter the value 327.40 meter per hour, we should probably start building Noah's ark
Therefor, it seems reasonable to drop values too large, considered as outliers
Can you write Python code for it?

# Considering that the gauge may concentrate the rainfall, we set the cap to 1000
# Comment this line to analyse the complete dataset
l = len(raw)
raw = raw[raw['Expected'] < 300]  #1000
print("Dropped %d (%0.2f%%)"%(l-len(raw),(l-len(raw))/float(l)*100))


I am working on the file "FDMS/TME3/Model_V7.ipynb".
The first task is:
Predict on testset
Can you write Python code for it?

%%time
filename = "data/reduced_test_5000.csv"
#filename = "data/test.csv"
test = pd.read_csv(filename)
test = test.set_index('Id')

features_columns = list([u'Ref', u'Ref_5x5_10th',
       u'Ref_5x5_50th', u'Ref_5x5_90th', u'RefComposite',
       u'RefComposite_5x5_10th', u'RefComposite_5x5_50th',
       u'RefComposite_5x5_90th', u'RhoHV', u'RhoHV_5x5_10th',
       u'RhoHV_5x5_50th', u'RhoHV_5x5_90th', u'Zdr', u'Zdr_5x5_10th',
       u'Zdr_5x5_50th', u'Zdr_5x5_90th', u'Kdp', u'Kdp_5x5_10th',
       u'Kdp_5x5_50th', u'Kdp_5x5_90th'])

def getX(raw):
    selected_columns = list([ u'minutes_past',u'radardist_km', u'Ref', u'Ref_5x5_10th',
       u'Ref_5x5_50th', u'Ref_5x5_90th', u'RefComposite',
       u'RefComposite_5x5_10th', u'RefComposite_5x5_50th',
       u'RefComposite_5x5_90th', u'RhoHV', u'RhoHV_5x5_10th',
       u'RhoHV_5x5_50th', u'RhoHV_5x5_90th', u'Zdr', u'Zdr_5x5_10th',
       u'Zdr_5x5_50th', u'Zdr_5x5_90th', u'Kdp', u'Kdp_5x5_10th',
       u'Kdp_5x5_50th', u'Kdp_5x5_90th'])

    data = raw[selected_columns]

    docX= []
    for i in data.index.unique():
        if isinstance(data.loc[i],pd.core.series.Series):
            m = [data.loc[i].as_matrix()]
            docX.append(m)
        else:
            m = data.loc[i].as_matrix()
            docX.append(m)
    X = np.array(docX)
    return X

#%%time
#X=getX(test)

#tmp = []
#for i in X:
#    tmp.append(len(i))
#tmp = np.array(tmp)
#sns.countplot(tmp,order=range(tmp.min(),tmp.max()+1))
#plt.title("Number of ID per number of observations\n(On test dataset)")
#plt.plot()

#testFull = test.dropna()
testNoFullNan = test.loc[test[features_columns].dropna(how='all').index.unique()]

%%time
X=getX(testNoFullNan)  # 1min
#XX = [np.array(t).mean(0) for t in X]  # 10s

XX=[]
for t in X:
    nm = np.nanmean(t,0)
    for idx,j in enumerate(nm):
        if np.isnan(j):
            nm[idx]=global_means[idx]
    XX.append(nm)
XX=np.array(XX)

# rescale to clip min at 0 (for non negative matrix factorization)
XX_rescaled=XX[:,:]-np.min(XX,0)

%%time
W =...


I am working on the file "FDMS/TME3/Model_V7.ipynb".
The first task is:
Random Forest
Ladies and gentlemens, the winning solution of 80% of Kaggles's challenges
Can you write Python code for it?

rfr = RandomForestRegressor(n_estimators=200, criterion='mse', max_depth=None, min_samples_split=2,
                            min_samples_leaf=1, min_weight_fraction_leaf=0.0, max_features='auto',
                            max_leaf_nodes=None, bootstrap=True, oob_score=False, n_jobs=-1,
                            random_state=None, verbose=0, warm_start=False)

%%time
rfr = rfr.fit(X_train,y_train)

print(rfr.score(X_train,y_train))
print(rfr.score(X_test,y_test))


I am working on the file "FDMS/TME3/Model_V7.ipynb".
The first task is:
We prepare 3 subsets
Fully filled data only
Used at first to evaluate the models and no worry about features completion
Can you write Python code for it?

#noAnyNan = raw.loc[raw[features_columns].dropna(how='any').index.unique()]
noAnyNan = raw.dropna()


I am working on the file "FDMS/TME3/Submit_V0.ipynb".
The first task is:

Can you write Python code for it?

filename = "data/reduced_test_5000.csv"
test = pd.read_csv(filename)
test = test.set_index('Id')

features_columns = list([u'Ref', u'Ref_5x5_10th',
       u'Ref_5x5_50th', u'Ref_5x5_90th', u'RefComposite',
       u'RefComposite_5x5_10th', u'RefComposite_5x5_50th',
       u'RefComposite_5x5_90th', u'RhoHV', u'RhoHV_5x5_10th',
       u'RhoHV_5x5_50th', u'RhoHV_5x5_90th', u'Zdr', u'Zdr_5x5_10th',
       u'Zdr_5x5_50th', u'Zdr_5x5_90th', u'Kdp', u'Kdp_5x5_10th',
       u'Kdp_5x5_50th', u'Kdp_5x5_90th'])

def getX(raw):
    selected_columns = list([ u'radardist_km', u'Ref', u'Ref_5x5_10th',
       u'Ref_5x5_50th', u'Ref_5x5_90th', u'RefComposite',
       u'RefComposite_5x5_10th', u'RefComposite_5x5_50th',
       u'RefComposite_5x5_90th', u'RhoHV', u'RhoHV_5x5_10th',
       u'RhoHV_5x5_50th', u'RhoHV_5x5_90th', u'Zdr', u'Zdr_5x5_10th',
       u'Zdr_5x5_50th', u'Zdr_5x5_90th', u'Kdp', u'Kdp_5x5_10th',
       u'Kdp_5x5_50th', u'Kdp_5x5_90th'])

    data = raw[selected_columns]

    docX= []
    for i in data.index.unique():
        if isinstance(data.loc[i],pd.core.series.Series):
            m = [data.loc[i].as_matrix()]
            docX.append(m)
        else:
            m = data.loc[i].as_matrix()
            docX.append(m)
    X = np.array(docX)
    return X

X=getX(test)

tmp = []
for i in X:
    tmp.append(len(i))
tmp = np.array(tmp)
sns.countplot(tmp,order=range(tmp.min(),tmp.max()+1))
plt.title("Number of ID per number of observations\n(On test dataset)")
plt.plot()

#print("Average gauge observation in mm: %0.2f"%y.mean())

etreg.predict(X_test)

testFull = test.dropna()

X=getX(testFull)
XX = [np.array(t).mean(0) for t in X]

pd.DataFrame(etreg.predict(XX)).describe()

predFull = zip(testFull.index.unique(),etreg.predict(XX))

b = np.empty(len(a))
b.fill(3.14)

zip(a,b)

predFull[:10]

testNan = test.drop(test[features_columns].dropna(how='all').index)

tmp = np.empty(len(testNan))
tmp.fill(0.445000)   # 50th percentile of full Nan dataset
predNan =...


I am working on the file "FDMS/TME3/Submit_V0.ipynb".
The first task is:
13.765.202 lines in train.csv
8.022.757 lines in test.csv

Load the dataset
Can you write Python code for it?

%%time
filename = "data/reduced_train_100000.csv"
#filename = "data/reduced_test_100000.csv"
raw = pd.read_csv(filename)
raw = raw.set_index('Id')

raw['Expected'].describe()


I am working on the file "FDMS/TME3/Submit_V0.ipynb".
The first task is:
How much observations is there for each ID ?
Can you write Python code for it?

X,y=getXy(raw)

tmp = []
for i in X:
    tmp.append(len(i))
tmp = np.array(tmp)
sns.countplot(tmp,order=range(tmp.min(),tmp.max()+1))
plt.title("Number of ID per number of observations\n(On complete dataset)")
plt.plot()

print("Average gauge observation in mm: %0.2f"%y.mean())


I am working on the file "FDMS/TME3/Submit_V0.ipynb".
The first task is:
Now let's do the analysis on different subsets:
On fully filled dataset
Can you write Python code for it?

#noAnyNan = raw.loc[raw[features_columns].dropna(how='any').index.unique()]
noAnyNan = raw.dropna()

noAnyNan.isnull().sum()

X,y=getXy(noAnyNan)

tmp = []
for i in X:
    tmp.append(len(i))
tmp = np.array(tmp)
sns.countplot(tmp,order=range(tmp.min(),tmp.max()+1))
plt.title("Number of ID per number of observations\n(On fully filled dataset)")
plt.plot()

print("Average gauge observation in mm: %0.2f"%y.mean())

pd.DataFrame(y).describe()

noFullNan = raw.loc[raw[features_columns].dropna(how='all').index.unique()]

noFullNan.isnull().sum()

X,y=getXy(noFullNan)

tmp = []
for i in X:
    tmp.append(len(i))
tmp = np.array(tmp)
sns.countplot(tmp,order=range(tmp.min(),tmp.max()+1))
plt.title("Number of ID per number of observations\n(On partly filled dataset)")
plt.plot()

print("Average gauge observation in mm: %0.2f"%y.mean())

pd.DataFrame(y).describe()

fullNan = raw.drop(raw[features_columns].dropna(how='all').index)

fullNan.isnull().sum()

X,y=getXy(fullNan)

tmp = []
for i in X:
    tmp.append(len(i))
tmp = np.array(tmp)
sns.countplot(tmp,order=range(tmp.min(),tmp.max()+1))
plt.title("Number of ID per number of observations\n(On fully empty dataset)")
plt.plot()

print("Average gauge observation in mm: %0.2f"%y.mean())

pd.DataFrame(y).describe()


I am working on the file "FDMS/TME3/Submit_V0.ipynb".
The first task is:
Per wikipedia, a value of more than 421 mm/h is considered "Extreme/large hail"
If we encounter the value 327.40 meter per hour, we should probably start building Noah's ark
Therefor, it seems reasonable to drop values too large, considered as outliers
Can you write Python code for it?

# Considering that the gauge may concentrate the rainfall, we set the cap to 1000
# Comment this line to analyse the complete dataset
l = len(raw)
raw = raw[raw['Expected'] < 1000]
print("Dropped %d (%0.2f%%)"%(l-len(raw),(l-len(raw))/float(l)*100))

raw.head(5)


I am working on the file "FDMS/TME3/Submit_V0.ipynb".
The first task is:
Predicitons
As a first try, we make predictions on the complete data, and return the 50th percentile and uncomplete and fully empty data
Can you write Python code for it?

etreg = ExtraTreesRegressor(n_estimators=100, max_depth=None, min_samples_split=1, random_state=0)

X,y=getXy(noAnyNan)
XX = [np.array(t).mean(0) for t in X]

split = 0.2
ps = int(len(XX) * (1-split))
X_train = XX[:ps]
y_train = y[:ps]
X_test = XX[ps:]
y_test = y[ps:]

%%time
etreg.fit(X_train,y_train)

%%time
et_score = cross_val_score(etreg, XX, y, cv=5)
print("Score: %s\tMean: %.03f"%(et_score,et_score.mean()))

err = (etreg.predict(X_test)-y_test)**2
err.sum()/len(err)

r = random.randrange(len(X_train))
print(r)
print(etreg.predict(X_train[r]))
print(y_train[r])

r = random.randrange(len(X_test))
print(r)
print(etreg.predict(X_test[r]))
print(y_test[r])


I am working on the file "FDMS/TME3/Submit_V0.ipynb".
The first task is:
Quick analysis for the sparsity by column
Can you write Python code for it?

l = float(len(raw["minutes_past"]))
comp = [[1-raw[i].isnull().sum()/l , i] for i in raw.columns]
comp.sort(key=lambda x: x[0], reverse=True)

sns.barplot(zip(*comp)[0],zip(*comp)[1],palette=sns.cubehelix_palette(len(comp), start=.5, rot=-.75))
plt.title("Percentage of non NaN data")
plt.show()


I am working on the file "FDMS/TME3/Submit_V0.ipynb".
The first task is:
Strangely we notice that the less observations there is, the more it rains on average
However more of the expected rainfall fall below 0.5
What prediction should we make if there is no data?
Can you write Python code for it?

print("%d observations" %(len(raw)))
#print("%d fully filled, %d partly filled, %d fully empty"
#      %(len(noAnyNan),len(noFullNan),len(raw)-len(noFullNan)))
print("%0.1f%% fully filled, %0.1f%% partly filled, %0.1f%% fully empty"
      %(len(noAnyNan)/float(len(raw))*100,
        len(noFullNan)/float(len(raw))*100,
        (len(raw)-len(noFullNan))/float(len(raw))*100))


I am working on the file "FDMS/TME3/Submit_V1.ipynb".
The first task is:

Can you write Python code for it?

%%time
#filename = "data/reduced_test_5000.csv"
filename = "data/test.csv"
test = pd.read_csv(filename)
test = test.set_index('Id')

features_columns = list([u'Ref', u'Ref_5x5_10th',
       u'Ref_5x5_50th', u'Ref_5x5_90th', u'RefComposite',
       u'RefComposite_5x5_10th', u'RefComposite_5x5_50th',
       u'RefComposite_5x5_90th', u'RhoHV', u'RhoHV_5x5_10th',
       u'RhoHV_5x5_50th', u'RhoHV_5x5_90th', u'Zdr', u'Zdr_5x5_10th',
       u'Zdr_5x5_50th', u'Zdr_5x5_90th', u'Kdp', u'Kdp_5x5_10th',
       u'Kdp_5x5_50th', u'Kdp_5x5_90th'])

def getX(raw):
    selected_columns = list([ u'radardist_km', u'Ref', u'Ref_5x5_10th',
       u'Ref_5x5_50th', u'Ref_5x5_90th', u'RefComposite',
       u'RefComposite_5x5_10th', u'RefComposite_5x5_50th',
       u'RefComposite_5x5_90th', u'RhoHV', u'RhoHV_5x5_10th',
       u'RhoHV_5x5_50th', u'RhoHV_5x5_90th', u'Zdr', u'Zdr_5x5_10th',
       u'Zdr_5x5_50th', u'Zdr_5x5_90th', u'Kdp', u'Kdp_5x5_10th',
       u'Kdp_5x5_50th', u'Kdp_5x5_90th'])

    data = raw[selected_columns]

    docX= []
    for i in data.index.unique():
        if isinstance(data.loc[i],pd.core.series.Series):
            m = [data.loc[i].as_matrix()]
            docX.append(m)
        else:
            m = data.loc[i].as_matrix()
            docX.append(m)
    X = np.array(docX)
    return X

X=getX(test)

tmp = []
for i in X:
    tmp.append(len(i))
tmp = np.array(tmp)
sns.countplot(tmp,order=range(tmp.min(),tmp.max()+1))
plt.title("Number of ID per number of observations\n(On test dataset)")
plt.plot()

testFull = test.dropna()

%%time
X=getX(testFull)  # 1min
XX = [np.array(t).mean(0) for t in X]  # 10s

pd.DataFrame(etreg.predict(XX)).describe()

predFull = zip(testFull.index.unique(),etreg.predict(XX))

testNan = test.drop(test[features_columns].dropna(how='all').index)

tmp = np.empty(len(testNan))
tmp.fill(0.445000)   # 50th percentile of full Nan dataset
predNan = zip(testNan.index.unique(),tmp)

testLeft = test.drop(testNan.index.unique()).drop(testFull.index.unique())

tmp =...


I am working on the file "FDMS/TME3/Submit_V1.ipynb".
The first task is:
13.765.202 lines in train.csv
8.022.757 lines in test.csv

Few words about the dataset
Predictions is made in the USA corn growing states (mainly Iowa, Illinois, Indiana) during the season with the highest rainfall (as illustrated by Iowa for the april to august months)
The Kaggle page indicate that the dataset have been shuffled, so working on a subset seems acceptable
The test set is not a extracted from the same data as the training set however, which make the evaluation trickier
Load the dataset
Can you write Python code for it?

%%time
filename = "data/reduced_train_100000.csv"
#filename = "data/reduced_train_100000.csv"
raw = pd.read_csv(filename)
raw = raw.set_index('Id')

raw['Expected'].describe()


I am working on the file "FDMS/TME3/Submit_V1.ipynb".
The first task is:
How much observations is there for each ID ?
Can you write Python code for it?

%%time
X,y=getXy(raw)

tmp = []
for i in X:
    tmp.append(len(i))
tmp = np.array(tmp)
sns.countplot(tmp,order=range(tmp.min(),tmp.max()+1))
plt.title("Number of ID per number of observations\n(On complete dataset)")
plt.plot()

print("Average gauge observation in mm: %0.2f"%y.mean())


I am working on the file "FDMS/TME3/Submit_V1.ipynb".
The first task is:
Now let's do the analysis on different subsets:
On fully filled dataset
Can you write Python code for it?


I am working on the file "FDMS/TME3/Submit_V1.ipynb".
The first task is:
Per wikipedia, a value of more than 421 mm/h is considered "Extreme/large hail"
If we encounter the value 327.40 meter per hour, we should probably start building Noah's ark
Therefor, it seems reasonable to drop values too large, considered as outliers
Can you write Python code for it?

# Considering that the gauge may concentrate the rainfall, we set the cap to 1000
# Comment this line to analyse the complete dataset
l = len(raw)
raw = raw[raw['Expected'] < 1000]
print("Dropped %d (%0.2f%%)"%(l-len(raw),(l-len(raw))/float(l)*100))

raw.head(5)

raw.describe()


I am working on the file "FDMS/TME3/Submit_V1.ipynb".
The first task is:
Predicitons
As a first try, we make predictions on the complete data, and return the 50th percentile and uncomplete and fully empty data
Can you write Python code for it?

etreg = ExtraTreesRegressor(n_estimators=100, max_depth=None, min_samples_split=1, random_state=0)

%%time
X,y=getXy(noAnyNan)

%%time
#XX = [np.array(t).mean(0) for t in X]
XX = [np.append(np.nanmean(np.array(np.array(t)),0),(np.array(t)[1:] - np.array(t)[:-1]).sum(0) ) for t in X]

XX[0]

%%time
etreg = etreg.fit(XX,y)

split = 0.2
ps = int(len(XX) * (1-split))
X_train = XX[:ps]
y_train = y[:ps]
X_test = XX[ps:]
y_test = y[ps:]

from sklearn import svm

svr = svm.SVR()

svr.fit(X_train,y_train)

%%time
svr_score = cross_val_score(svr, XX, y, cv=5)
print("Score: %s\tMean: %.03f"%(svr_score,svr_score.mean()))

err = (svr.predict(X_test)-y_test)**2
err.sum()/len(err)

%%time
etreg.fit(X_train,y_train)

%%time
et_score = cross_val_score(etreg, XX, y, cv=5)
print("Score: %s\tMean: %.03f"%(et_score,et_score.mean()))

%%time
et_score = cross_val_score(etreg, XX, y, cv=5)
print("Score: %s\tMean: %.03f"%(et_score,et_score.mean()))

err = (etreg.predict(X_test)-y_test)**2
err.sum()/len(err)

err = (etreg.predict(X_test)-y_test)**2
err.sum()/len(err)

r = random.randrange(len(X_train))
print(r)
print(etreg.predict(X_train[r]))
print(y_train[r])

r = random.randrange(len(X_test))
print(r)
print(etreg.predict(X_test[r]))
print(y_test[r])


I am working on the file "FDMS/TME3/Submit_V1.ipynb".
The first task is:
Strangely we notice that the less observations there is, the more it rains on average
However more of the expected rainfall fall below 0.5
What prediction should we make if there is no data?
Can you write Python code for it?

print("%d observations" %(len(raw)))
#print("%d fully filled, %d partly filled, %d fully empty"
#      %(len(noAnyNan),len(noFullNan),len(raw)-len(noFullNan)))
print("%0.1f%% fully filled, %0.1f%% partly filled, %0.1f%% fully empty"
      %(len(noAnyNan)/float(len(raw))*100,
        len(noFullNan)/float(len(raw))*100,
        (len(raw)-len(noFullNan))/float(len(raw))*100))

raw[raw["Expected"] < 1]["Expected"].hist(bins=10)


I am working on the file "FDMS/TME3/Submit_V1.ipynb".
The first task is:
We see that except for the fixed features minutes_past, radardist_km and Expected the dataset is mainly sparse.
Let's transform the dataset to conduct more analysis
We regroup the data by ID
Can you write Python code for it?

# We select all features except for the minutes past,
# because we ignore the time repartition of the sequence for now

features_columns = list([u'Ref', u'Ref_5x5_10th',
       u'Ref_5x5_50th', u'Ref_5x5_90th', u'RefComposite',
       u'RefComposite_5x5_10th', u'RefComposite_5x5_50th',
       u'RefComposite_5x5_90th', u'RhoHV', u'RhoHV_5x5_10th',
       u'RhoHV_5x5_50th', u'RhoHV_5x5_90th', u'Zdr', u'Zdr_5x5_10th',
       u'Zdr_5x5_50th', u'Zdr_5x5_90th', u'Kdp', u'Kdp_5x5_10th',
       u'Kdp_5x5_50th', u'Kdp_5x5_90th'])

def getXy(raw):
    selected_columns = list([ u'radardist_km', u'Ref', u'Ref_5x5_10th',
       u'Ref_5x5_50th', u'Ref_5x5_90th', u'RefComposite',
       u'RefComposite_5x5_10th', u'RefComposite_5x5_50th',
       u'RefComposite_5x5_90th', u'RhoHV', u'RhoHV_5x5_10th',
       u'RhoHV_5x5_50th', u'RhoHV_5x5_90th', u'Zdr', u'Zdr_5x5_10th',
       u'Zdr_5x5_50th', u'Zdr_5x5_90th', u'Kdp', u'Kdp_5x5_10th',
       u'Kdp_5x5_50th', u'Kdp_5x5_90th'])

    data = raw[selected_columns]

    docX, docY = [], []
    for i in data.index.unique():
        if isinstance(data.loc[i],pd.core.series.Series):
            m = [data.loc[i].as_matrix()]
            docX.append(m)
            docY.append(float(raw.loc[i]["Expected"]))
        else:
            m = data.loc[i].as_matrix()
            docX.append(m)
            docY.append(float(raw.loc[i][:1]["Expected"]))
    X , y = np.array(docX) , np.array(docY)
    return X,y


I am working on the file "FDMS/TME4/TME4_FiltrageCollaboratif-Copy1.ipynb".
The first task is:

Can you write Python code for it?

# NMF by alternative non-negative least squares using projected gradients
# Author: Chih-Jen Lin, National Taiwan University
# Python/numpy translation: Anthony Di Franco

from numpy import *
from numpy.linalg import norm
from time import time
from sys import stdout

def nmf(V,Winit,Hinit,tol,timelimit,maxiter):
    """
    (W,H) = nmf(V,Winit,Hinit,tol,timelimit,maxiter)
    W,H: output solution
    Winit,Hinit: initial solution
    tol: tolerance for a relative stopping condition
    timelimit, maxiter: limit of time and iterations
    """

    W = Winit; H = Hinit; initt = time();

    gradW = dot(W, dot(H, H.T)) - dot(V, H.T)
    gradH = dot(dot(W.T, W), H) - dot(W.T, V)
    initgrad = norm(r_[gradW, gradH.T])
    print 'Init gradient norm %f' % initgrad
    tolW = max(0.001,tol)*initgrad
    tolH = tolW

    for iter in xrange(1,maxiter):
        # stopping condition
        projnorm = norm(r_[gradW[logical_or(gradW<0, W>0)],
                                     gradH[logical_or(gradH<0, H>0)]])
        if projnorm < tol*initgrad or time() - initt > timelimit: break

        (W, gradW, iterW) = nlssubprob(V.T,H.T,W.T,tolW,1000)
        W = W.T
        gradW = gradW.T

        if iterW==1: tolW = 0.1 * tolW

        (H,gradH,iterH) = nlssubprob(V,W,H,tolH,1000)
        if iterH==1: tolH = 0.1 * tolH

        if iter % 10 == 0: stdout.write('.')

    print '\nIter = %d Final proj-grad norm %f' % (iter, projnorm)
    return (W,H)

def nlssubprob(V,W,Hinit,tol,maxiter):
    """
    H, grad: output solution and gradient
    iter: #iterations used
    V, W: constant matrices
    Hinit: initial solution
    tol: stopping tolerance
    maxiter: limit of iterations
    """

    H = Hinit
    WtV = dot(W.T, V)
    WtW = dot(W.T, W)

    alpha = 1; beta = 0.1;
    for iter in xrange(1, maxiter):
        grad = dot(WtW, H) - WtV
        projgrad = norm(grad[logical_or(grad < 0, H >0)])
        if projgrad < tol: break

    # search step size
    for inner_iter in xrange(1,20):
        Hn = H - alpha*grad
      ...


I am working on the file "FDMS/TME4/TME4_FiltrageCollaboratif-Copy1.ipynb".
The first task is:
Collaboritive Filtering classes
Can you write Python code for it?

class BaselineMeanUser:
    def __init__(self):
        self.users={}
        self.movies={}
    def fit(self,train):
        for user in train:
            note=0
            for movie in train[user]:
                note+=train[user][movie]
            note=note/len(train[user])
            self.users[user]=note

    def predict(self,user,movie):
        return self.users[user]
    def score(self,X):
        nb_movies = len(get_moove(X))
        score = 0.0
        for user in X:
            for movie in X[user]:
                score += (self.predict(user,movie) - X[user][movie])**2
        return score/nb_movies

class BaselineMeanMovie:
    def __init__(self):
        self.users={}
        self.movies={}
    def fit(self,train):
        movies = get_moove(train)
        for movie in movies:
            note=0
            cpt=0
            for user in train:
                try:
                    note+=train[user][movie]
                    cpt+=1
                except KeyError:
                    pass
            note=note/cpt
            self.movies[movie]=note

    def predict(self,user,movie):
        return self.movies[movie]
    def score(self,X):
        nb_movies = len(get_moove(X))
        score = 0.0
        for user in X:
            for movie in X[user]:
                score += (self.predict(user,movie) - X[user][movie])**2
        return score/nb_movies

baseline_mu= BaselineMeanUser()
baseline_mm= BaselineMeanMovie()

baseline_mu.fit(train)
baseline_mm.fit(train)

print("score baseline mean user  ",baseline_mu.score(test))
print("score baseline mean movie ",baseline_mm.score(test))


I am working on the file "FDMS/TME4/TME4_FiltrageCollaboratif.ipynb".
The first task is:

Can you write Python code for it?

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

tag_headers = ['user_id', 'movie_id', 'tag', 'timestamp']
tags = pd.read_table('data/ml-10M/tags.dat', sep='::', header=None, names=tag_headers)

rating_headers = ['user_id', 'movie_id', 'rating', 'timestamp']
ratings = pd.read_table('data/ml-10M/ratings.dat', sep='::', header=None, names=rating_headers)

movie_headers = ['movie_id', 'title', 'genres']
movies = pd.read_table('data/ml-10M/movies.dat',
                       sep='::', header=None, names=movie_headers)
movie_titles = movies.title.tolist()

movies.head()

ratings.head()

tags.head()

df = movies.join(ratings, on=['movie_id'], rsuffix='_r').join(tags, on=['movie_id'], rsuffix='_t')
del df['movie_id_r']
del df['user_id_t']
del df['movie_id_t']
del df['timestamp_t']

df.head()

rp = df.pivot_table(columns=['movie_id'],index=['user_id'],values='rating')
rp.head()

rp = rp.fillna(0); # Replace NaN
rp.head()

Q = rp.values

Q

Q.shape

W = Q>0.5
W[W == True] = 1
W[W == False] = 0
# To be consistent with our Q matrix
W = W.astype(np.float64, copy=False)

W

lambda_ = 0.1
n_factors = 100
m, n = Q.shape
n_iterations = 20

X = 5 * np.random.rand(m, n_factors)
Y = 5 * np.random.rand(n_factors, n)

def get_error(Q, X, Y, W):
    return np.sum((W * (Q - np.dot(X, Y)))**2)

errors = []
for ii in range(n_iterations):
    X = np.linalg.solve(np.dot(Y, Y.T) + lambda_ * np.eye(n_factors),
                        np.dot(Y, Q.T)).T
    Y = np.linalg.solve(np.dot(X.T, X) + lambda_ * np.eye(n_factors),
                        np.dot(X.T, Q))
    if ii % 100 == 0:
        print('{}th iteration is completed'.format(ii))
    errors.append(get_error(Q, X, Y, W))
Q_hat = np.dot(X, Y)
print('Error of rated movies: {}'.format(get_error(Q, X, Y, W)))

%matplotlib inline

plt.plot(errors);
plt.ylim([0, 20000]);

def print_recommendations(W=W, Q=Q, Q_hat=Q_hat, movie_titles=movie_titles):
    #Q_hat -= np.min(Q_hat)
    #Q_hat[Q_hat < 1] *= 5
    Q_hat -= np.min(Q_hat)
    Q_hat *= float(5) /...


I am working on the file "FDMS/TME4/TME4_FiltrageCollaboratif.ipynb".
The first task is:
Collaboritive Filtering classes
Can you write Python code for it?

class BaselineMeanUser:
    def __init__(self):
        self.users={}
        self.movies={}
    def fit(self,train):
        for user in train:
            note=0
            for movie in train[user]:
                note+=train[user][movie]
            note=note/len(train[user])
            self.users[user]=round(note)

    def predict(self,user,movie):
        return self.users[user]
    def score(self,X):
        nb_movies = len(get_moove(X))
        score = 0
        for user in X:
            for movie in X[user]:
                if(self.predict(user,movie)==X[user][movie]):
                    score+=1
        return float(score)/nb_movies

class BaselineMeanMovie:
    def __init__(self):
        self.users={}
        self.movies={}
    def fit(self,train):
        movies = get_moove(train)
        for movie in movies:
            note=0
            cpt=0
            for user in train:
                try:
                    note+=train[user][movie]
                    cpt+=1
                except KeyError:
                    pass
            note=note/cpt
            self.movies[movie]=round(note)

    def predict(self,user,movie):
        return self.movies[movie]
    def score(self,X):
        nb_movies = len(get_moove(X))
        score = 0
        for user in X:
            for movie in X[user]:
                if(self.predict(user,movie)==X[user][movie]):
                    score+=1
        return float(score)/nb_movies

baseline_mu= BaselineMeanUser()
baseline_mm= BaselineMeanMovie()

baseline_mu.fit(train)
baseline_mm.fit(train)

print("score baseline mean user  ",baseline_mu.score(test))
print("score baseline mean movie ",baseline_mm.score(test))


I am working on the file "FDMS/TME4/TME4_FiltrageCollaboratif.ipynb".
The first task is:
Collect data
Can you write Python code for it?

def loadMovieLens(path='./data/movielens'):
    #Get movie titles
    movies={}
    for line in open(path+'/u.item'):
        id,title=line.split('|')[0:2]
        movies[id]=title

    # Load data
    prefs={}
    for line in open(path+'/u.data'):
        (user,movieid,rating,ts)=line.split('\t')
        prefs.setdefault(user,{})
        prefs[user][movies[movieid]]=float(rating)

    return prefs

data = loadMovieLens("data/ml-100k")


I am working on the file "FDMS/TME4/TME4_FiltrageCollaboratif.ipynb".
The first task is:
Explore data
Can you write Python code for it?


I am working on the file "FDMS/TME4/TME4_FiltrageCollaboratif.ipynb".
The first task is:
Part that allows to clean train and test
We don't want to have user in test set which are not in train test, the same for the movies so we delete them
Can you write Python code for it?

def get_moove(data):
    moove = {}
    for u in data:
        for m in data[u]:
            moove[m]=0
    return moove

def get_youser(data):
    youser = {}
    for u in data:
        youser[u]=0
    return youser

def clean(d1,d2):
    to_erase = {}
    for i in d1:
        try:
            d2[i]
        except KeyError:
            to_erase[i]=0
    for i in d2:
        try:
            d1[i]
        except KeyError:
            to_erase[i]=0
    return to_erase
def _remove_users(test,rem):
    for i in rem:
        try:
            del test[i]
        except KeyError:
            pass
def _remove_movies(test,rem):
    for i in test:
        for j in rem:
            try:
                del test[i][j]
            except KeyError:
                pass

mooveToRemoove = clean(get_moove(train),get_moove(test))
youserToRemoove = clean(get_youser(train),get_youser(test))
_remove_users(test,youserToRemoove)
_remove_movies(test,mooveToRemoove)


I am working on the file "FDMS/TME4/TME4_FiltrageCollaboratif_V2-Copy3.ipynb".
The first task is:

Can you write Python code for it?


#self.lamb*np.linalg.norm(self.theta)

#from scipy import linalg

def nmf(X, latent_features, max_iter=100, eps = 1e-5, printevery=100):

    print "NMF with %d latent features, %d iterations."%(latent_features, max_iter)

    #lamb = 0.2

    #X = copy.deepcopy(rawMatrix)
    #with np.errstate(all='ignore'):
    #avg_m = X.sum(0)/(X != 0).sum(0)
    avg_u = X.sum(1)/(X != 0).sum(1)
    #diff_m = avg_m - avg_m.mean()
    diff_u = avg_u - avg_u.mean()
    print(avg_u.mean())
    #X = X - diff_m
    for idxi,i in enumerate(X):
        for idxj,j in enumerate(i):
            if X[idxi,idxj]!=0:
                X[idxi,idxj]+=diff_u[idxi]

    # mask used to ignore null element (coded by zero)
    mask = np.sign(X)

    # randomly initialized matrix
    rows, columns = X.shape
    A = np.random.rand(rows, latent_features)

    Y = np.random.rand(latent_features, columns)
    # Not used as I couldn't find significant improvments
    #Y = linalg.lstsq(A, X)[0]  # initializing that way as recommanded in a blog post
    #Y = np.maximum(Y, eps)     # avoiding too low values

    masked_X = mask * X
    masktest = np.sign(rawMatrixTest)    # used for prints
    masktrain = np.sign(rawMatrixTrain)  # used for prints

    prev_A = A
    prev_Y = Y

    #diff_u = (avg_u - avg_u.mean().T).T
    #(np.array([1,5])-mat.T).T

    for i in range(1, max_iter + 1):

        top = np.dot(masked_X, Y.T)
        esti = np.dot((mask * np.dot(A, Y)), Y.T)
        #esti = esti - diff_u
        bottom = esti + eps
        #print("val",np.shape(top/bottom))
        A *= top / bottom

        top = np.dot(A.T, masked_X)
        esti = np.dot(A.T, mask * np.dot(A, Y))
        #esti = esti - diff_m
        bottom = esti + eps
        #print("lav",np.shape(top/bottom))
        tb = top / bottom
        #print(np.linalg.norm(tb))
        no = np.linalg.norm(tb)
        Y *= tb
        #Y *= (0.9 * tb) + ( 0.1 * ( tb + (1/no) )  )

        """
        if i % 10 == 0:
            diff = np.abs(Y -...


R = rawMatrixTrain

def matrix_factorization(R, K, steps=100, eps=0.0001, beta=0.02, decay=0.95):
    N,M = np.shape(R)
    P = np.random.rand(N,K)
    #P = np.maximum(P, eps)

    #Q = np.random.rand(M,K).T
    Q = linalg.lstsq(P, R)[0]
    Q = np.maximum(Q, eps)

    #masked_X = mask * X
    #X_est_prev = dot(A, Y)

    #mask = np.sign(R)
    #masked_R = mask * R

    masktest = np.sign(rawMatrixTest)
    masktrain = np.sign(rawMatrixTrain)

    for step in xrange(1,steps+1):
        #"""
        for i in xrange(len(R)):
            for j in xrange(len(R[i])):
                if R[i][j] > 0:
                    eij = R[i][j] - np.dot(P[i,:],Q[:,j])
                    P[i] = P[i] + eps * (2 * eij * Q.T[j] - beta * P[i])
                    #Q[i] = P[i] + eps * (2 * eij * Q.T[j] - beta * P[i])
                    Q.T[j] = Q.T[j] + eps * (2 * eij * P[i] - beta * Q.T[j])
                    #for k in xrange(K):
                    #    P[i][k] = P[i][k] + eps * (2 * eij * Q[k][j] - beta * P[i][k])
                        #Q[k][j] = Q[k][j] + eps * (2 * eij * P[i][k] - beta * Q[k][j])
        """
        ###
        top = np.dot(masked_R, Q.T)
        bottom = (np.dot((mask * np.dot(P, Q)), Q.T)) + eps

        P *= top / bottom

        P = np.maximum(P, eps)
        # print 'A',  np.round(A, 2)

        top = np.dot(P.T, masked_R)
        bottom = np.dot(P.T, mask * np.dot(P, Q)) + eps
        Q *= top / bottom
        Q = np.maximum(Q, eps)
        # print 'Y', np.round(Y, 2)
        """
        if step%5:
            eps=eps*decay

        if step % 10 == 0 or step == 1 or step == steps:

            X_est = dot(P, Q)
            q = masktest*X_est - rawMatrixTest
            q_train = masktrain*X_est - rawMatrixTrain
            print "Iteration %d, Err %.05f, Err on train %.05f"%( step, (q*q).sum()/ masktest.sum(), (q_train*q_train).sum()/ masktest.sum() )

    return P, Q.T

%%time
K = 10
nP, nQ = matrix_factorization(R, K,...


I am working on the file "FDMS/TME4/TME4_FiltrageCollaboratif_V2-Copy3.ipynb".
The first task is:
/!\ 18 movies have no ratings at all
so we get a divide by zero warning. Ignored with:
Can you write Python code for it?

with np.errstate(all='ignore'):
    avg_m = rawMatrix.sum(0)/(rawMatrix != 0).sum(0)
    avg_u = rawMatrix.sum(1)/(rawMatrix != 0).sum(1)

np.shape(t)

tt = t[0]

tt


I am working on the file "FDMS/TME4/TME4_FiltrageCollaboratif_V2-Copy3.ipynb".
The first task is:
Content example
Can you write Python code for it?


I am working on the file "FDMS/TME4/TME4_FiltrageCollaboratif_V2-Copy3.ipynb".
The first task is:
Loading the data
Can you write Python code for it?

def loadMovieLens(path='./data/movielens'):
    #Get movie titles
    movies={}
    rev_movies={}
    for idx,line in enumerate(open(path+'/u.item')):
        idx,title=line.split('|')[0:2]
        movies[idx]=title
        rev_movies[title]=idx

    # Load data
    prefs={}
    for line in open(path+'/u.data'):
        (user,movieid,rating,ts)=line.split('\t')
        prefs.setdefault(user,{})
        prefs[user][movies[movieid]]=float(rating)

    return prefs,rev_movies

data,movies = loadMovieLens("data/ml-100k")


I am working on the file "FDMS/TME4/TME4_FiltrageCollaboratif_V2-Copy3.ipynb".
The first task is:
Matrix used for fast evaluation
Can you write Python code for it?

evalArrayAll = getRawArray(data)
evalArrayTest = getRawArray(test)

evalArrayTest[:10,:10]


I am working on the file "FDMS/TME4/TME4_FiltrageCollaboratif_V2-Copy3.ipynb".
The first task is:
Non-negative Matrix Factorization
Fast implementation using numpy's matrix processing. Sadly, I can't avoid the overfitting starting after few iterations.
Can you write Python code for it?

#from scipy import linalg

def nmf(X, latent_features, max_iter=100, eps = 1e-5,printevery=100):

    print "NMF with %d latent features, %d iterations."%(latent_features, max_iter)

    # mask used to ignore null element (coded by zero)
    mask = np.sign(X)

    # randomly initialized matrix
    rows, columns = X.shape
    A = np.random.rand(rows, latent_features)

    Y = np.random.rand(latent_features, columns)
    # Not used as I couldn't find significant improvments
    #Y = linalg.lstsq(A, X)[0]  # initializing that way as recommanded in a blog post
    #Y = np.maximum(Y, eps)     # avoiding too low values

    masked_X = mask * X
    masktest = np.sign(rawMatrixTest)    # used for prints
    masktrain = np.sign(rawMatrixTrain)  # used for prints

    for i in range(1, max_iter + 1):

        top = np.dot(masked_X, Y.T)
        bottom = (np.dot((mask * np.dot(A, Y)), Y.T)) + eps
        A *= top / bottom

        top = np.dot(A.T, masked_X)
        bottom = np.dot(A.T, mask * np.dot(A, Y)) + eps
        Y *= top / bottom

        # evaluation
        if i % printevery == 0 or i == 1 or i == max_iter:
            X_est = np.dot(A, Y)
            q = masktest*X_est - rawMatrixTest
            q_train = masktrain*X_est - rawMatrixTrain
            print "Iteration %d, Err %.05f, Err train %.05f"%( i, (q*q).sum()/ masktest.sum(), (q_train*q_train).sum()/ masktest.sum() )

    return A, Y

%%time
A,Y = nmf(rawMatrixTrain,100,eps = 1e-5,max_iter=5,printevery=1)
resMatrix = A.dot(Y)


I am working on the file "FDMS/TME4/TME4_FiltrageCollaboratif_V2-Copy3.ipynb".
The first task is:
Raw matrix used for convenience and clarity.
Structure like scipy sparse matrix or python dictionnaries may be used for speedup.
Complete dataset
Can you write Python code for it?

rawMatrix = np.zeros((len(data.keys()),1682))
for u in data:
    for m in data[u]:
        rawMatrix[int(u)-1][int(movies[m])-1] = data[u][m]

print(np.shape(rawMatrix))
rawMatrix[:10,:10]


I am working on the file "FDMS/TME4/TME4_FiltrageCollaboratif_V2-Copy3.ipynb".
The first task is:
This is due to the high sparsity of the matrix.
We can of course reduce the features matrix size to avoid overfitting, but that will limit further improvments.
Can you write Python code for it?

%%time
A,Y = nmf(rawMatrixTrain,1,eps = 1e-5,max_iter=100,printevery=20)
resMatrix = A.dot(Y)


I am working on the file "FDMS/TME4/TME4_FiltrageCollaboratif_V2-Copy3.ipynb".
The first task is:
Train and test dataset
Can you write Python code for it?

rawMatrixTrain = np.zeros((len(data.keys()),1682))
for u in train:
    for m in train[u]:
        rawMatrixTrain[int(u)-1][int(movies[m])-1] = train[u][m]

rawMatrixTest = np.zeros((len(data.keys()),1682))
for u in test:
    for m in test[u]:
        rawMatrixTest[int(u)-1][int(movies[m])-1] = test[u][m]


I am working on the file "FDMS/TME4/TME4_FiltrageCollaboratif_V2-Copy3.ipynb".
The first task is:
split used for convenience on the average by movie baseline
Can you write Python code for it?


I am working on the file "FDMS/TME4/TME4_FiltrageCollaboratif_V2-Copy4.ipynb".
The first task is:

Can you write Python code for it?

class tSNE():
    def __init__(self,perp, nIter, lr, moment, dim=2):
        self.perp = perp # entre 5 et 50
        self.nIter = nIter
        self.lr = lr
        self.moment = moment
        self.dim = dim
    def fit(self,data):
        nEx = np.shape(data)[0]
        # Matrice des distances de ||xi - xj||² #
        normx = np.sum(data**2,1)
        normx = np.reshape(normx, (1, nEx))
        distancex = normx + normx.T - 2 * data.dot(data.T)
        # Calcul des sigma ---------------------------------------------------------------#
        lperp = np.log2(self.perp)
        # initialisation bornes pour la recherche dichotomique #
        sup = np.ones((nEx,1)) * np.max(distancex)
        inf = np.zeros((nEx,1))
        self.sigma = (sup + inf) / 2.
        # recherche dichotomique #
        stop = False
        while not stop:
            # Calculer la matrice des p(i|j)
            self.pcond = np.exp(-distancex / (2. * (self.sigma**2)))
            self.pcond = self.pcond / np.sum(self.pcond - np.eye(nEx),1).reshape(nEx,1)
            # Calculer l'entropie de p(i|j)
            entropy = - np.sum(self.pcond * np.log2(self.pcond), 0)
            # Mise a jour des bornes
              # Si il faut augmenter sigma
            up = entropy < lperp
            inf[up,0] = self.sigma[up,0]
              # Si il faut baisser sigma
            down = entropy > lperp
            sup[down,0] = self.sigma[down,0]
            # Mise a jour de sigma et condition d'arrêt
            old = self.sigma
            self.sigma = ((sup + inf) / 2.)
            if np.max(np.abs(old - self.sigma)) < 1e-5:
                stop = True
                #print np.exp(entropy)
                #print self.sigma.T
        #--------------------------------------------------------------------------#
        #initialiser y
        self.embeddings = np.zeros((self.nIter+2, nEx, self.dim))
        self.embeddings[1] = np.random.randn(nEx, self.dim) * 1e-4
       ...


I am working on the file "FDMS/TME4/TME4_FiltrageCollaboratif_V2-Copy4.ipynb".
The first task is:
Baseline: mean by user
Can you write Python code for it?

class baselineMeanUser:
    def __init__(self):
        self.users={}
    def fit(self,train):
        for user in train.keys():
            note=0.0
            for movie in train[user].keys():
                note+=train[user][movie]
            note=note/len(train[user])
            self.users[user]=note

    def predict(self,users):
        return [self.users[u] for u in users]

baseline_mu= baselineMeanUser()
baseline_mu.fit(train)
pred = baseline_mu.predict(evalArrayTest[:,0])
print("Mean Error %0.6f" %(
        (np.array(pred) - np.array(evalArrayTest[:,2], float)) ** 2).mean())

class baselineMeanMovie:
    def __init__(self):
        self.movies={}
    def fit(self,train):
        for movie in train.keys():
            note=0.0
            for user in train[movie].keys():
                note+=train[movie][user]
            note=note/len(train[movie])
            self.movies[movie]=note

    def predict(self,movies):
        res=[]
        for m in movies:
            try:
                res.append(self.movies[m])
            except:
                res.append(3)
        return res

baseline_mm= baselineMeanMovie()
baseline_mm.fit(m_train)
pred = baseline_mm.predict(evalArrayTest[:,1])
print("Mean Error %0.6f" %(
        (np.array(pred) - np.array(evalArrayTest[:,2], float)) ** 2).mean())


I am working on the file "FDMS/TME4/TME4_FiltrageCollaboratif_V2-Copy4.ipynb".
The first task is:
Content example
Can you write Python code for it?


I am working on the file "FDMS/TME4/TME4_FiltrageCollaboratif_V2-Copy4.ipynb".
The first task is:
Despite good results in few seconds on this dataset, this can only get us so far.
We then have to add regularization to the cost function.
Evaluation
Can you write Python code for it?

## This class is used to make predictions
class evalMF:
    def __init__(self,resMatrix,dicU,dicI):
        self.resMatrix=resMatrix
        self.dicU = dicU
        self.dicI = dicI
    def fit(self):
        pass

    def predict(self,user,movie):
        return self.resMatrix[int(user)-1][int(self.dicI[movie])-1]

mf = evalMF(resMatrix,data,movies)

# np.array([ (float(ra[2]) - mf.predict(ra[0],ra[1]))**2 for ra in evalArrayTest]).mean()
# faster evaluation
masqueTest=np.sign(rawMatrixTest)
q = masqueTest*resMatrix - rawMatrixTest
(q*q).sum()/ masqueTest.sum()


I am working on the file "FDMS/TME4/TME4_FiltrageCollaboratif_V2-Copy4.ipynb".
The first task is:
Non-negative Matrix Factorization
Fast implementation using numpy's matrix processing.
Can you write Python code for it?


I am working on the file "FDMS/TME4/TME4_FiltrageCollaboratif_V2-Copy4.ipynb".
The first task is:
This is due to the high sparsity of the matrix.
We can of course reduce the features matrix size to avoid overfitting, but that will limit further improvments.
Can you write Python code for it?


I am working on the file "FDMS/TME4/TME4_FiltrageCollaboratif_V2-Copy4.ipynb".
The first task is:
Train and test dataset
Can you write Python code for it?


I am working on the file "FDMS/TME4/TME4_FiltrageCollaboratif_V2-Copy4.ipynb".
The first task is:
We have not been very successful with incorporating the bias and L1 into that implementation...
We build a simpler model below, and then add the regularization and bias.
Can you write Python code for it?

class FactoMatriceBiais():
    def __init__(self, k, epsilon=1e-3, nbIter=2000, lamb=0.5):
        self.k = k
        self.lamb = lamb
        self.epsilon = epsilon
        self.nbIter = nbIter

    def fit(self, trainUsers, trainItems, triplet):

        self.p = {}
        self.q = {}
        self.bu = {} #biais sur les utilisateurs
        self.bi = {} #biais sur les items
        self.mu = np.random.random() * 2 - 1

        for j in range(len(triplet)): # On initialise les cases vides en random
            u = triplet[j][0]
            i = triplet[j][1]
            if not u in self.p:
                self.p[u] = np.random.rand(1,self.k) # matrice ligne pour un users
                self.bu[u] = np.random.rand() * 2 - 1
            if not i in self.q:
                self.q[i] = np.random.rand(self.k,1) # matrice colonne pour un item
                self.bi[i] = np.random.rand() * 2 - 1
        loss = []
        for it in range(self.nbIter):
            ind = np.random.randint(len(triplet))
            u = triplet[ind][0]
            i = triplet[ind][1]

            tmp = trainUsers[u][i] - (self.mu + self.bi[i] + self.bu[u] +self.p[u].dot(self.q[i])[0][0])
            self.p[u] = (1 - self.lamb * self.epsilon) * self.p[u] + self.epsilon * 2 * tmp * self.q[i].transpose()
            self.bu[u] = (1 - self.lamb * self.epsilon) * self.bu[u] + self.epsilon * 2 * tmp
            self.q[i] = (1 - self.lamb * self.epsilon) * self.q[i] + self.epsilon * 2 * tmp * self.p[u].transpose()
            self.bi[i] = (1 - self.lamb * self.epsilon) * self.bi[i] + self.epsilon * 2 * tmp
            self.mu = (1 - self.lamb * self.epsilon) * self.mu + self.epsilon * 2 * tmp

            loss.append(tmp*tmp) # erreur sans régularisation
            #loss.append(tmp**2 + self.lamb *(np.linalg.norm(self.p[u]).sum()**2 + np.linalg.norm(self.q[i]).sum()**2))

            if ((it)%(self.nbIter*0.2) == 0) :
                print "itération : " , it
                print "loss : ",...


I am working on the file "FDMS/TME4/TME4_FiltrageCollaboratif_V2-Copy4.ipynb".
The first task is:
We usualy see an important difference between users, so we need to take the bias into account.
Can you write Python code for it?

summ=0
for i in data["1"]:
    summ+=(float(data["1"][i]) - mf.predict("1",i))**2
summ/len(data["1"])

summ=0
for i in data["3"]:
    summ+=(float(data["3"][i]) - mf.predict("3",i))**2
summ/len(data["3"])


I am working on the file "FDMS/TME4/TME4_FiltrageCollaboratif_V2-Copy4.ipynb".
The first task is:
split used for convenience on the average by movie baseline
Can you write Python code for it?


I am working on the file "FDMS/TME5/TME4_5_FiltrageCollaboratif_tsne.ipynb".
The first task is:
But its not very informative...
Let's add some colors by ratings
Can you write Python code for it?

me = rawMatrix.mean(1)
X_ini = np.vstack([rawMatrix[np.where(np.logical_and(me>i, me<=j))]
               for i,j in [(0,2),(2,3),(3,4),(4,5)]])
cols = np.hstack(  [ np.round( me[ np.where(np.logical_and(me>i, me<=j)) ] )
               for i,j in [(0,2),(2,3),(3,4),(4,5)] ] )

%%time
X = mod.fit_transform(X_ini)

palette = np.array(sns.color_palette("hls", 6))

f = plt.figure(figsize=(8, 8))
ax = plt.subplot(aspect='equal')
sc = ax.scatter(X[:,0], X[:,1], lw=0, s=40,
                c=palette[cols.astype(np.int)])
plt.xlim(-25, 25)
plt.ylim(-25, 25)
ax.axis('off')
ax.axis('tight')

txts = []
for i in range(2,6):
    xtext, ytext = np.median(X[cols == i, :], axis=0)
    txt = ax.text(xtext, ytext, str(i), fontsize=24)
    txts.append(txt)

#plt.plot(mod.embedding_[12][0],mod.embedding_[12][1], 'bv')

plt.show()


I am working on the file "FDMS/TME5/TME4_5_FiltrageCollaboratif_tsne.ipynb".
The first task is:
For reference, let's compare it with sklearn's TSNE
Can you write Python code for it?

from sklearn.manifold import TSNE

mod = TSNE(random_state=1337)

%%time
X = mod.fit_transform(X_ini)

palette = np.array(sns.color_palette("hls", 10))

# We create a scatter plot.
f = plt.figure(figsize=(8, 8))
ax = plt.subplot(aspect='equal')
sc = ax.scatter(X[:,0], X[:,1], lw=0, s=40,
                c=palette[cols.astype(np.int)])
plt.xlim(-25, 25)
plt.ylim(-25, 25)
ax.axis('off')
ax.axis('tight')

#plt.plot(mod.embedding_[12][0],mod.embedding_[12][1], 'bv')

plt.show()


I am working on the file "FDMS/TME5/TME4_5_FiltrageCollaboratif_tsne.ipynb".
The first task is:
It produce similar results, albeit faster, as expected.
Can you write Python code for it?

rawMatrix = np.zeros((943,1682))
for u in data:
    for m in data[u]:
        rawMatrix[int(u)-1][int(movies[m])-1] = data[u][m]


I am working on the file "FDMS/TME5/TME4_5_FiltrageCollaboratif_tsne.ipynb".
The first task is:
Let's look at some predictions
Can you write Python code for it?


I am working on the file "FDMS/TME5/TME4_5_FiltrageCollaboratif_tsne.ipynb".
The first task is:
Loading the data
Can you write Python code for it?


I am working on the file "FDMS/TME5/TME4_5_FiltrageCollaboratif_tsne.ipynb".
The first task is:
Matrix used for fast evaluation
Can you write Python code for it?

def getRawArray(data):
    rawArray = []
    for u in data.keys():
        for i in data[u].keys():
            rawArray.append([u,i,data[u][i]])
    return rawArray

def getDataByUsers(rawArray) :
    usr = {}
    for t in rawArray:
        if not t[0] in usr.keys():
            usr[t[0]] = {}
        usr[t[0]][t[1]] = float(t[2])
    return usr

def getDataByItems(rawArray) :
    itm = {}
    for t in rawArray:
        if not t[1] in itm.keys():
            itm[t[1]] = {}
        itm[t[1]][t[0]] = float(t[2])
    return itm

# Split l'ensemble des rawArrays
def splitTrainTest(rawArray, testProp) :
    perm = np.random.permutation(rawArray)
    splitIndex = int(testProp * len(rawArray))
    return perm[splitIndex:], perm[:splitIndex]

# supprime des données de test les données inconnus en train
def deleteUnknowData(rawArray_test, trainUsers, trainItems) :
    to_Del = []
    for i,t in enumerate(rawArray_test):
        if not t[0] in trainUsers:
            to_Del.append(i)
        elif not t[1] in trainItems:
            to_Del.append(i)
    return np.delete(rawArray_test, to_Del, 0)

%%time

rawArray = getRawArray(data)

arrayTrain, arrayTest = splitTrainTest(rawArray , 0.2)

trainUsers = getDataByUsers(arrayTrain)
trainItems = getDataByItems(arrayTrain)

arrayTest = deleteUnknowData(arrayTest, trainUsers, trainItems)

testUsers = getDataByUsers(arrayTest)
testItems = getDataByItems(arrayTest)


I am working on the file "FDMS/TME5/TME4_5_FiltrageCollaboratif_tsne.ipynb".
The first task is:
Predict unknown data and fill the others with the right number
Can you write Python code for it?

%%time
rawMatrix = np.zeros((943,1682))
for u in data:
    for m in movies.keys():
        try:
            rawMatrix[int(u)-1][int(movies[m])-1] = model.predict([[u,m]])
        except KeyError:
            rawMatrix[int(u)-1][int(movies[m])-1] = 3
for u in data:
    for m in data[u]:
        rawMatrix[int(u)-1][int(movies[m])-1] = data[u][m]

rawMatrix[:5,:5]


I am working on the file "FDMS/TME5/TME4_5_FiltrageCollaboratif_tsne.ipynb".
The first task is:
Produced visualisation
Can you write Python code for it?

palette = np.array(sns.color_palette("hls", 4))

f = plt.figure(figsize=(8, 8))
ax = plt.subplot(aspect='equal')
sc = ax.scatter(X[:,0], X[:,1], lw=0, s=40)
plt.xlim(-25, 25)
plt.ylim(-25, 25)
ax.axis('off')
ax.axis('tight')

plt.show()


I am working on the file "FDMS/TME5/TME4_5_FiltrageCollaboratif_tsne.ipynb".
The first task is:
Raw matrix are used for convenience and clarity.
Structure like scipy sparse matrix or python dictionnaries may be used for speedup.
Complete dataset
Can you write Python code for it?

rawMatrix = np.zeros((943,1682))
for u in data:
    for m in data[u]:
        rawMatrix[int(u)-1][int(movies[m])-1] = data[u][m]

print(np.shape(rawMatrix))
rawMatrix[:5,:5]


I am working on the file "FDMS/TME5/TME4_5_FiltrageCollaboratif_tsne.ipynb".
The first task is:
Splitting data between train/test
We avoid to let unseen data form the train set in the test set.
We also try to minimise the dataset reduction by splitting on each user.
Can you write Python code for it?

def getRawArray(data):
    d = []
    for u in data.keys():
        for i in data[u].keys():
            d.append([u,i,data[u][i]])
    return np.array(d)
# splitting while avoiding to reduce the dataset too much
def split_train_test(data,percent_test):
    test={}
    train={}
    movie={}
    for u in data.keys():
        test.setdefault(u,{})
        train.setdefault(u,{})
        for movie in data[u]:
            #print(data[u][movie])
            if (random()<percent_test):
                test[u][movie]=data[u][movie]
            else:
                train[u][movie]=data[u][movie]
    return train, test
def split_train_test_by_movies(data,percent_test):
    test={}
    train={}
    movie={}
    for u in data.keys():
        for movie in data[u]:
            if (random()<percent_test):
                try:
                    test[movie][u]=data[u][movie]
                except KeyError:
                    test.setdefault(movie,{})
                    test[movie][u]=data[u][movie]
            else:
                try:
                    train[movie][u]=data[u][movie]
                except KeyError:
                    train.setdefault(movie,{})
                    train[movie][u]=data[u][movie]
    return train, test


I am working on the file "FDMS/TME5/TME4_5_FiltrageCollaboratif_tsne.ipynb".
The first task is:
This is due to the high sparsity of the matrix.
We can of course reduce the features matrix size to avoid overfitting, but that will limit further improvments:
Can you write Python code for it?


I am working on the file "FDMS/TME5/TME4_5_FiltrageCollaboratif_tsne.ipynb".
The first task is:
Train and test dataset
Can you write Python code for it?


I am working on the file "FDMS/TME5/TME4_5_FiltrageCollaboratif_tsne.ipynb".
The first task is:
We have not been very successful with incorporating the bias and L1 into that implementation...
We build a simpler model below, and then add the regularization and bias.
Can you write Python code for it?

class mf():
    def __init__(self, k, eps=1e-3, nIter=2000, lambd=0.5):
        self.k = k
        self.lambd = lambd
        self.eps = eps
        self.nIter = nIter

    def fit(self, trainUsers, trainItems, rawArray):
        print "MF with %d latent features, %d iterations."%(self.k, self.nIter)

        printevery = np.round(self.nIter/10,decimals=-4)

        self.p = {}
        self.q = {}
        self.bu = {}
        self.bi = {}
        self.mu = np.random.random() * 2 - 1

        for j in range(len(rawArray)):
            u = rawArray[j][0]
            i = rawArray[j][1]
            if not u in self.p:
                self.p[u] = np.random.rand(1,self.k)
                self.bu[u] = np.random.rand() * 2 - 1
            if not i in self.q:
                self.q[i] = np.random.rand(self.k,1)
                self.bi[i] = np.random.rand() * 2 - 1
        loss = []
        for it in range(self.nIter):
            ind = np.random.randint(len(rawArray))
            u = rawArray[ind][0]
            i = rawArray[ind][1]

            # thx to A & A !
            tmp = trainUsers[u][i] - (self.mu + self.bi[i] + self.bu[u] +self.p[u].dot(self.q[i])[0][0])
            self.p[u] = (1 - self.lambd * self.eps) * self.p[u] + self.eps * 2 * tmp * self.q[i].transpose()
            self.bu[u] = (1 - self.lambd * self.eps) * self.bu[u] + self.eps * 2 * tmp
            self.q[i] = (1 - self.lambd * self.eps) * self.q[i] + self.eps * 2 * tmp * self.p[u].transpose()
            self.bi[i] = (1 - self.lambd * self.eps) * self.bi[i] + self.eps * 2 * tmp
            self.mu = (1 - self.lambd * self.eps) * self.mu + self.eps * 2 * tmp

            loss.append(tmp*tmp)

            if it % printevery == 0 or it == self.nIter-1:
                print "Iteration %s, Err %.05f, True %.05f"%( str(it).ljust(8), np.mean(loss) , model.score(arrayTest))

        return self.p, self.q

    def predict(self, rawArray_test):
        pred = np.zeros(len(rawArray_test))
        for ind,t in enumerate(rawArray_test):
           ...


I am working on the file "FDMS/TME5/TME4_5_FiltrageCollaboratif_tsne.ipynb".
The first task is:
We see that it quickly get better than the baseline.
However, we see below that it overfit after that:
Can you write Python code for it?

%%time
A,Y = nmf(rawMatrixTrain,50,eps = 1e-5,max_iter=500,printevery=100)
resMatrix = A.dot(Y)


I am working on the file "FDMS/TME5/TME4_5_FiltrageCollaboratif_tsne.ipynb".
The first task is:
cleaning
18 movies have no ratings at all
Can you write Python code for it?

def deleteUnseenInTest(train,test):
    for k in test.keys():
        try:
            train[k]
        except KeyError:
            test.pop(k,None)

def deleteUnknowData(triplet_test, trainUsers, trainItems) :
    to_Del = []
    for i,t in enumerate(triplet_test):
        if not t[0] in trainUsers:
            to_Del.append(i)
        elif not t[1] in trainItems:
            to_Del.append(i)
    return np.delete(triplet_test, to_Del, 0)

deleteUnseenInTest(train,test)
deleteUnseenInTest(m_train,m_test)


I am working on the file "FDMS/TME5/TME4_5_FiltrageCollaboratif_tsne.ipynb".
The first task is:
thanks to Professeur Chen
Can you write Python code for it?


I am working on the file "FDMS/TME5/sklearn_t-SNE.ipynb".
The first task is:
For high-dimensional sparse data it is helpful to first reduce the dimensions to 50 dimensions with TruncatedSVD and then perform t-SNE. This will usually improve the visualization.
Can you write Python code for it?

from sklearn.decomposition import TruncatedSVD

X_reduced = TruncatedSVD(n_components=50, random_state=0).fit_transform(vectors)

X_embedded = TSNE(n_components=2, perplexity=40, verbose=2).fit_transform(X_reduced)

fig = figure(figsize=(10, 10))
ax = axes(frameon=False)
setp(ax, xticks=(), yticks=())
subplots_adjust(left=0.0, bottom=0.0, right=1.0, top=0.9,
                wspace=0.0, hspace=0.0)
scatter(X_embedded[:, 0], X_embedded[:, 1],
        c=newsgroups.target, marker="x")


I am working on the file "FDMS/TME5/sklearn_t-SNE.ipynb".
The first task is:
High-dimensional sparse data: the 20 newsgroups dataset
In high-dimensional and nonlinear domains, PCA is not applicable any more and many other manifold learning algorithms do not yield good visualizations either because they try to preserve the global data structure.
Can you write Python code for it?

from sklearn.datasets import fetch_20newsgroups
from sklearn.feature_extraction.text import TfidfVectorizer

categories = ['alt.atheism', 'talk.religion.misc', 'comp.graphics', 'sci.space']
newsgroups = fetch_20newsgroups(subset="train", categories=categories)
vectors = TfidfVectorizer().fit_transform(newsgroups.data)

print(repr(vectors))


I am working on the file "FDMS/TME5/sklearn_t-SNE.ipynb".
The first task is:
MNIST dataset
Can you write Python code for it?

from sklearn.datasets import fetch_mldata

# Load MNIST dataset
mnist = fetch_mldata("MNIST original")
X, y = mnist.data / 255.0, mnist.target

# Create subset and reduce to first 50 dimensions
indices = arange(X.shape[0])
random.shuffle(indices)
n_train_samples = 5000
X_pca = PCA(n_components=50).fit_transform(X)
X_train = X_pca[indices[:n_train_samples]]
y_train = y[indices[:n_train_samples]]

# Plotting function
matplotlib.rc('font', **{'family' : 'sans-serif',
                         'weight' : 'bold',
                         'size'   : 18})
matplotlib.rc('text', **{'usetex' : True})

def plot_mnist(X, y, X_embedded, name, min_dist=10.0):
    fig = figure(figsize=(10, 10))
    ax = axes(frameon=False)
    title("\\textbf{MNIST dataset} -- Two-dimensional "
          "embedding of 70,000 handwritten digits with %s" % name)
    setp(ax, xticks=(), yticks=())
    subplots_adjust(left=0.0, bottom=0.0, right=1.0, top=0.9,
                    wspace=0.0, hspace=0.0)
    scatter(X_embedded[:, 0], X_embedded[:, 1],
            c=y, marker="x")

    if min_dist is not None:
        from matplotlib import offsetbox
        shown_images = np.array([[15., 15.]])
        indices = arange(X_embedded.shape[0])
        random.shuffle(indices)
        for i in indices[:5000]:
            dist = np.sum((X_embedded[i] - shown_images) ** 2, 1)
            if np.min(dist) < min_dist:
                continue
            shown_images = np.r_[shown_images, [X_embedded[i]]]
            imagebox = offsetbox.AnnotationBbox(
                offsetbox.OffsetImage(X[i].reshape(28, 28),
                                      cmap=cm.gray_r), X_embedded[i])
            ax.add_artist(imagebox)

X_train_embedded = TSNE(n_components=2, perplexity=40, verbose=2).fit_transform(X_train)

plot_mnist(X[indices[:n_train_samples]], y_train, X_train_embedded,
           "t-SNE", min_dist=20.0)


I am working on the file "FDMS/TME5/sklearn_t-SNE.ipynb".
The first task is:
t-Distributed Stochastic Neighbor Embedding (t-SNE) in sklearn
t-SNE is a tool for data visualization. It reduces the dimensionality of data to 2 or 3 dimensions so that it can be plotted easily. Local similarities are preserved by this embedding.
t-SNE converts distances between data in the original space to probabilities. First, we compute conditional probabilites
$$p_{j|i} = \frac{\exp{(-d(\boldsymbol{x}i, \boldsymbol{x}_j) / (2 \sigma_i^2)})}{\sum{i \neq k} \exp{(-d(\boldsymbol{x}i, \boldsymbol{x}_k) / (2 \sigma_i^2)})}, \quad p{i|i} = 0,$$
which will be used to generate joint probabilities
$$p_{ij} = \frac{p_{j|i} + p_{i|j}}{2N}.$$
The $\sigma_i$ will be determined automatically. This procedure can be influenced by setting the perplexity of the algorithm.
A heavy-tailed distribution will be used to measure the similarities in the embedded space
$$q_{ij} = \frac{(1 + ||\boldsymbol{y}i - \boldsymbol{y}_j)||^2)^{-1}}{\sum{k \neq l} (1 + ||\boldsymbol{y}_k - \boldsymbol{y}_l)||^2)^{-1}},$$
and then we minimize the Kullback-Leibler divergence
$$KL(P|Q) = \sum_{i \neq j} p_{ij} \log \frac{p_{ij}}{q_{ij}}$$
between both distributions with gradient descent (and some tricks). Note that the cost function is not convex and multiple runs might yield different results.
More information can be found in these resources and in the documentation from t-SNE:

Website (Implementations, FAQ, etc.): t-Distributed Stochastic Neighbor Embedding
Original paper: Visualizing High-Dimensional Data Using t-SNE
Can you write Python code for it?

from sklearn.manifold import TSNE

help(TSNE)


I am working on the file "FDMS/TME6/TME6_Reco.ipynb".
The first task is:

Can you write Python code for it?

masqueTest=np.sign(rawMatrixTest)
q = masqueTest*resMatrix - rawMatrixTest
(q*q).sum()/ masqueTest.sum()

mf = evalMF(resMatrix,data,movies)

print data["1"]["Akira (1988)"]
print mf.predict("1","Akira (1988)")
print data["1"]["All Dogs Go to Heaven 2 (1996)"]
print mf.predict("1","All Dogs Go to Heaven 2 (1996)")

len(rawMatrixTest)

t = []
c = 10
for idxi,i in enumerate(rawMatrixTest):
    for idxj,j in enumerate(i):
        if rawMatrixTest[idxi][idxj] != 0:
            t.append( (resMatrix[idxi][idxj] - float(rawMatrixTest[idxi][idxj]))**2 )
            if c>0:
                print(rawMatrixTest[idxi][idxj],resMatrix[idxi][idxj])
                c-=1
np.array(t).mean()


I am working on the file "FDMS/TME6/TME6_Reco.ipynb".
The first task is:
/!\ 18 movies have no ratings at all
so we get a divide by zero warning. Ignored with:
Can you write Python code for it?

with np.errstate(all='ignore'):
    avg_m = rawMatrix.sum(0)/(rawMatrix != 0).sum(0)
    avg_u = rawMatrix.sum(1)/(rawMatrix != 0).sum(1)


I am working on the file "FDMS/TME6/TME6_Reco.ipynb".
The first task is:
Baseline: mean by user
Can you write Python code for it?

class baselineMeanUser:
    def __init__(self):
        self.users={}
    def fit(self,train):
        for user in train.keys():
            note=0.0
            for movie in train[user].keys():
                note+=train[user][movie]
            note=note/len(train[user])
            self.users[user]=note

    def predict(self,users):
        return [self.users[u] for u in users]

baseline_mu= baselineMeanUser()
baseline_mu.fit(train)
pred = baseline_mu.predict(evalArrayTest[:,0])
print("Mean Error %0.6f" %(
        (np.array(pred) - np.array(evalArrayTest[:,2], float)) ** 2).mean())

class baselineMeanMovie:
    def __init__(self):
        self.movies={}
    def fit(self,train):
        for movie in train.keys():
            note=0.0
            for user in train[movie].keys():
                note+=train[movie][user]
            note=note/len(train[movie])
            self.movies[movie]=note

    def predict(self,movies):
        res=[]
        for m in movies:
            try:
                res.append(self.movies[m])
            except:
                res.append(3)
        return res

baseline_mm= baselineMeanMovie()
baseline_mm.fit(m_train)
pred = baseline_mm.predict(evalArrayTest[:,1])
print("Mean Error %0.6f" %(
        (np.array(pred) - np.array(evalArrayTest[:,2], float)) ** 2).mean())


I am working on the file "FDMS/TME6/TME6_Reco.ipynb".
The first task is:
Content example
Can you write Python code for it?


I am working on the file "FDMS/TME6/TME6_Reco.ipynb".
The first task is:
Despite good results in few seconds on this dataset, this can only get us so far.
We then have to add regularization to the cost function.
Evaluation
Can you write Python code for it?

## This class is used to make predictions
class evalMF:
    def __init__(self,resMatrix,dicU,dicI):
        self.resMatrix=resMatrix
        self.dicU = dicU
        self.dicI = dicI
    def fit(self):
        pass

    def predict(self,user,movie):
        return self.resMatrix[int(user)-1][int(self.dicI[movie])-1]

mf = evalMF(resMatrix,data,movies)

# np.array([ (float(ra[2]) - mf.predict(ra[0],ra[1]))**2 for ra in evalArrayTest]).mean()
# faster evaluation
masqueTest=np.sign(rawMatrixTest)
q = masqueTest*resMatrix - rawMatrixTest
(q*q).sum()/ masqueTest.sum()

print data["1"]["Akira (1988)"]
print mf.predict("1","Akira (1988)")
print data["1"]["I.Q. (1994)"]
print mf.predict("1","I.Q. (1994)")


I am working on the file "FDMS/TME6/TME6_Reco.ipynb".
The first task is:
Loading the data
Can you write Python code for it?

def loadMovieLens(path='./data/movielens'):
    #Get movie titles
    movies={}
    rev_movies={}
    for idx,line in enumerate(open(path+'/u.item')):
        idx,title=line.split('|')[0:2]
        movies[idx]=title
        rev_movies[title]=idx

    # Load data
    prefs={}
    for line in open(path+'/u.data'):
        (user,movieid,rating,ts)=line.split('\t')
        prefs.setdefault(user,{})
        prefs[user][movies[movieid]]=float(rating)

    return prefs,rev_movies

data,movies = loadMovieLens("data/ml-100k")


I am working on the file "FDMS/TME6/TME6_Reco.ipynb".
The first task is:
Non-negative Matrix Factorization
Fast implementation using numpy's matrix processing.
Can you write Python code for it?

#from scipy import linalg

def nmf(X, latent_features, max_iter=100, eps = 1e-5,printevery=100):

    print "NMF with %d latent features, %d iterations."%(latent_features, max_iter)

    # mask used to ignore null element (coded by zero)
    mask = np.sign(X)

    # randomly initialized matrix
    rows, columns = X.shape
    A = np.random.rand(rows, latent_features)

    Y = np.random.rand(latent_features, columns)
    # Not used as I couldn't find significant improvments
    #Y = linalg.lstsq(A, X)[0]  # initializing that way as recommanded in a blog post
    #Y = np.maximum(Y, eps)     # avoiding too low values

    masked_X = mask * X
    masktest = np.sign(rawMatrixTest)    # used for prints
    masktrain = np.sign(rawMatrixTrain)  # used for prints

    for i in range(1, max_iter + 1):

        top = np.dot(masked_X, Y.T)
        bottom = (np.dot((mask * np.dot(A, Y)), Y.T)) + eps
        A *= top / bottom

        top = np.dot(A.T, masked_X)
        bottom = np.dot(A.T, mask * np.dot(A, Y)) + eps
        Y *= top / bottom

        # evaluation
        if i % printevery == 0 or i == 1 or i == max_iter:
            X_est = np.dot(A, Y)
            q = masktest*X_est - rawMatrixTest
            q_train = masktrain*X_est - rawMatrixTrain
            print "Iteration %d, Err %.05f, Err train %.05f"%( i, (q*q).sum()/ masktest.sum(), (q_train*q_train).sum()/ masktest.sum() )

    return A, Y

%%time
A,Y = nmf(rawMatrixTrain,100,eps = 1e-5,max_iter=5,printevery=1)
resMatrix = A.dot(Y)


I am working on the file "FDMS/TME6/TME6_Reco.ipynb".
The first task is:
Raw matrix used for convenience and clarity.
Structure like scipy sparse matrix or python dictionnaries may be used for speedup.
Complete dataset
Can you write Python code for it?

rawMatrix = np.zeros((len(data.keys()),1682))
for u in data:
    for m in data[u]:
        rawMatrix[int(u)-1][int(movies[m])-1] = data[u][m]

print(np.shape(rawMatrix))
rawMatrix[:10,:10]


I am working on the file "FDMS/TME6/TME6_Reco.ipynb".
The first task is:
Splitting data between train/test
We avoid to let unseen data form the train set in the test set.
We also try to minimise the dataset reduction by splitting on each user.
Can you write Python code for it?

def getRawArray(data):
    d = []
    for u in data.keys():
        for i in data[u].keys():
            d.append([u,i,data[u][i]])
    return np.array(d)

# splitting while avoiding to reduce the dataset too much
def split_train_test(data,percent_test):
    test={}
    train={}
    movie={}
    for u in data.keys():
        test.setdefault(u,{})
        train.setdefault(u,{})
        for movie in data[u]:
            #print(data[u][movie])
            if (random()<percent_test):
                test[u][movie]=data[u][movie]
            else:
                train[u][movie]=data[u][movie]
    return train, test

def split_train_test_by_movies(data,percent_test):
    test={}
    train={}
    movie={}
    for u in data.keys():
        for movie in data[u]:
            if (random()<percent_test):
                try:
                    test[movie][u]=data[u][movie]
                except KeyError:
                    test.setdefault(movie,{})
                    test[movie][u]=data[u][movie]
            else:
                try:
                    train[movie][u]=data[u][movie]
                except KeyError:
                    train.setdefault(movie,{})
                    train[movie][u]=data[u][movie]
    return train, test

percent_test=0.2
train,test=split_train_test(data,percent_test)


I am working on the file "FDMS/TME6/TME6_Reco.ipynb".
The first task is:
This is due to the high sparsity of the matrix.
We can of course reduce the features matrix size to avoid overfitting, but that will limit further improvments.
Can you write Python code for it?

%%time
A,Y = nmf(rawMatrixTrain,1,eps = 1e-5,max_iter=100,printevery=20)
resMatrix = A.dot(Y)


I am working on the file "FDMS/TME6/TME6_Reco.ipynb".
The first task is:
Train and test dataset
Can you write Python code for it?

rawMatrixTrain = np.zeros((len(data.keys()),1682))
for u in train:
    for m in train[u]:
        rawMatrixTrain[int(u)-1][int(movies[m])-1] = train[u][m]

rawMatrixTest = np.zeros((len(data.keys()),1682))
for u in test:
    for m in test[u]:
        rawMatrixTest[int(u)-1][int(movies[m])-1] = test[u][m]


I am working on the file "FDMS/TME6/TME6_Reco.ipynb".
The first task is:
Various atempts to incoporate the bias and the L1 regularization can be found below.
However, we have not been very successful with them yet...
A simpler yet slower model can be found at the bottom of the page, in which the bias and L1 regularization can be added easly.
Can you write Python code for it?

#self.lamb*np.linalg.norm(self.theta)

#from scipy import linalg

def nmf(X, latent_features, max_iter=100, eps = 1e-5, printevery=100):

    print "NMF with %d latent features, %d iterations."%(latent_features, max_iter)

    #lamb = 0.2

    ## User and Item bais

    #X = copy.deepcopy(rawMatrix)
    #with np.errstate(all='ignore'):
    #avg_m = X.sum(0)/(X != 0).sum(0)
    #avg_u = X.sum(1)/(X != 0).sum(1)
    #diff_m = avg_m - avg_m.mean()
    #diff_u = avg_u - avg_u.mean()
    #print(avg_u.mean())
    #X = X - diff_m
    #for idxi,i in enumerate(X):
    #    for idxj,j in enumerate(i):
    #        if X[idxi,idxj]!=0:
    #            X[idxi,idxj]+=diff_u[idxi]

    mask = np.sign(X)

    rows, columns = X.shape
    A = np.random.rand(rows, latent_features)

    Y = np.random.rand(latent_features, columns)
    # Not used as I couldn't find significant improvments
    #Y = linalg.lstsq(A, X)[0]  # initializing that way as recommanded in a blog post
    #Y = np.maximum(Y, eps)     # avoiding too low values

    masked_X = mask * X
    masktest = np.sign(rawMatrixTest)    # used for prints
    masktrain = np.sign(rawMatrixTrain)  # used for prints

    prev_A = A
    prev_Y = Y

    #diff_u = (avg_u - avg_u.mean().T).T
    #(np.array([1,5])-mat.T).T

    for i in range(1, max_iter + 1):

        top = np.dot(masked_X, Y.T)
        esti = np.dot((mask * np.dot(A, Y)), Y.T)
        #esti = esti - diff_u
        bottom = esti + eps
        #print("val",np.shape(top/bottom))
        A *= top / bottom

        top = np.dot(A.T, masked_X)
        esti = np.dot(A.T, mask * np.dot(A, Y))
        #esti = esti - diff_m
        bottom = esti + eps
        #print("lav",np.shape(top/bottom))
        tb = top / bottom
        #print(np.linalg.norm(tb))
        no = np.linalg.norm(tb)
        #Y *= tb
        Y *= (0.9 * tb) + ( 0.1 * ( tb + (1/no) )  )

        """
        ## Regularization
        if i % 10 == 0:
            diff = np.abs(Y - prev_Y)
            diff...


I am working on the file "FDMS/TME6/TME6_Reco.ipynb".
The first task is:
We see that it quickly get better than the baseline.
However, we see below that it overfit after that:
Can you write Python code for it?

%%time
A,Y = nmf(rawMatrixTrain,100,eps = 1e-5,max_iter=500,printevery=100)
resMatrix = A.dot(Y)


I am working on the file "FDMS/TME6/TME6_Reco.ipynb".
The first task is:
We usualy see an important difference between users, so we need to take the bias into account.
Can you write Python code for it?

summ=0
for i in data["1"]:
    summ+=(float(data["1"][i]) - mf.predict("1",i))**2
summ/len(data["1"])

summ=0
for i in data["3"]:
    summ+=(float(data["3"][i]) - mf.predict("3",i))**2
summ/len(data["3"])


I am working on the file "FDMS/TME6/TME6_Reco.ipynb".
The first task is:
cleaning
Can you write Python code for it?

def deleteUnseenInTest(train,test):
    for k in test.keys():
        try:
            train[k]
        except KeyError:
            test.pop(k,None)

deleteUnseenInTest(train,test)
deleteUnseenInTest(m_train,m_test)


I am working on the file "FDMS/TME6/TME6_Reco.ipynb".
The first task is:
split used for convenience on the average by movie baseline
Can you write Python code for it?