I am working on the file "Bitcoin Anomaly Analysis.ipynb".
The first task is:
D3 Network Visualization
Can you write Python code for it?

json_date_format = '%Y-%m-%dT%H:%M:%SZ'
# df['date'] = df.date.dt.strftime(json_date_format)

for scc in nx.strongly_connected_components(G_di):
    if len(scc) > 5:
        intersect = np.intersect1d(list(scc), anomalies_id)
        if intersect.size > 0:
            G_sub = G_di.subgraph(scc)

#G_sub_json = json_graph.node_link_data(G_sub)

#with open('../d3/json/network.json', 'w') as json_file:
#    json.dump(G_sub_json, json_file)

anomalies = (pred != 1)
np.concatenate((df[anomalies].from_user, df[anomalies].to_user))

anomalies_pairs = zip(df[anomalies].from_user, df[anomalies].to_user)

for i, j in anomalies_pairs:
    neigh = G_di.neighbors(i)
    neigh += G_di.neighbors(j)
    nodes = neigh + [i, j]
    if len(nodes) > 10 and len(nodes) < 20:
        G_sub = nx.subgraph(G_di, nodes)

from networkx.readwrite import json_graph
import json

anomalies_pairs = zip(df[anomalies].from_user, df[anomalies].to_user)

for e in G_sub.edges_iter():
    if e[:2] in anomalies_pairs:
        G_sub.edge[e[0]][e[1]]['type'] = 'licensing'
    else:
        G_sub.edge[e[0]][e[1]]['type'] = 'suit'

G_sub_json = json_graph.node_link_data(G_sub)

with open('../d3/json/network.json', 'w') as json_file:
    json.dump(G_sub_json, json_file)

G_di = nx.from_pandas_dataframe(df,
                             source='from_user', target='to_user',
                             edge_attr=['txn_key', 'amount', 'date'],
                             create_using=nx.DiGraph()
                            )

from IPython.display import IFrame
IFrame('./d3/html/network.html', width=1000, height=500)


I am working on the file "Bitcoin Anomaly Analysis.ipynb".
The first task is:
Isolation Forest for anomaly detection
Can you write Python code for it?

from sklearn.ensemble import IsolationForest

not_train_cols = ['txn_key', 'from_user', 'to_user', 'date']
X_train = df_feat[ [col for col in df_feat.columns if col not in not_train_cols] ].values

clf = IsolationForest(n_estimators=100,
                      contamination=0.01,
                      n_jobs=-1, random_state=42)
clf.fit(X_train)

clf.threshold_

pred = clf.predict(X_train)
anomalies = (pred != 1)


I am working on the file "Bitcoin Anomaly Analysis.ipynb".
The first task is:
Transaction Features to use

Number of transaction under this key
Transaction amount, total amount under this key
From equals to?
Number of unique from/to under this key
Transaction date
Year
Month
Day
Day of week
Day of year
Hour
Minute

Second

From/to in/out (unique) degree

From/to clustering coefficient
From/to in/out transaction frequency
All
Within ±12 hours
From/to transaction volume
All
Within ±12 hours
From/to first transaction date
From/to average in/out transaction amount
From/to average time between in/out transactions

Build graphs from transaction data - Undirected, directed and multi-directed
Can you write Python code for it?

import networkx as nx

# for features only defined in undirected graph
G = nx.from_pandas_dataframe(df,
                             source='from_user', target='to_user',
                             edge_attr=['txn_key', 'amount', 'date'],
                             create_using=nx.Graph()
                            )

# unique links between users
G_di = nx.from_pandas_dataframe(df,
                             source='from_user', target='to_user',
                             edge_attr=['txn_key', 'amount', 'date'],
                             create_using=nx.DiGraph()
                            )

# the full graph
G_mdi = nx.from_pandas_dataframe(df,
                             source='from_user', target='to_user',
                             edge_attr=['txn_key', 'amount', 'date'],
                             create_using=nx.MultiDiGraph()
                            )

# transaction feature maps
count_by_key = df.groupby('txn_key').size()
amount_by_key = df.groupby('txn_key').amount.sum()
ufrom_by_key = df.groupby('txn_key').from_user.agg(pd.Series.nunique)
uto_by_key = df.groupby('txn_key').to_user.agg(pd.Series.nunique)

# user feature maps
in_txn_count = df.groupby('to_user').size()
in_key_count = df.groupby('to_user').txn_key.agg(pd.Series.nunique)

out_txn_count = df.groupby('from_user').size()
out_key_count = df.groupby('from_user').txn_key.agg(pd.Series.nunique)

total_in_txn_amt = df.groupby('to_user').amount.sum()
total_out_txn_amt = df.groupby('from_user').amount.sum()

avg_in_txn_amt = df.groupby('to_user').amount.mean()
avg_out_txn_amt = df.groupby('from_user').amount.mean()

from_fst_txn_date = df.groupby('from_user').date.min()

df_feat = df.assign(
    # transaction features
    count_by_key = df.txn_key.map(count_by_key),
    amount_by_key = df.txn_key.map(amount_by_key),
    from_eq_to = df.from_user == df.to_user,
    ufrom_by_key = df.txn_key.map(ufrom_by_key),
    uto_by_key = df.txn_key.map(uto_by_key),

    # transaction date features
    date_year =...


I am working on the file "Bitcoin_Trading_Bot/Saved Predictions.ipynb".
The first task is:
Helper Functions for plotting and predicting
Can you write Python code for it?

def plot_results(predicted_data, true_data):
    fig=plt.figure(figsize=(18, 12), dpi= 80, facecolor='w', edgecolor='k')
    ax = fig.add_subplot(111)
    ax.plot(true_data, label='True Data')
    plt.plot(predicted_data, label='Prediction')
    plt.legend()
    plt.show()

def predict_sequences_multiple(model, data, window_size, prediction_len):
    #Predict sequence of 50 steps before shifting prediction run forward by 50 steps
    prediction_seqs = []
    for i in range(int(len(data)/prediction_len)):
        curr_frame = data[i*prediction_len]
        predicted = []
        for j in range(prediction_len):
            predicted.append(model.predict(curr_frame[np.newaxis,:,:])[0,0])
            curr_frame = curr_frame[1:]
            curr_frame = np.insert(curr_frame, [window_size-1], predicted[-1], axis=0)
        prediction_seqs.append(predicted)
    return prediction_seqs

def plot_results_multiple(predicted_data, true_data, prediction_len):
    fig=plt.figure(figsize=(18, 12), dpi= 80, facecolor='w', edgecolor='k')
    ax = fig.add_subplot(111)
    ax.plot(true_data, label='True Data')
    #Pad the list of predictions to shift it in the graph to it's correct start
    for i, data in enumerate(predicted_data):
        padding = [None for p in range(i * prediction_len)]
        plt.plot(padding + data, label='Prediction')
        plt.legend()
    plt.show()

true_values = []
def generator_strip_xy(data_gen, true_values):
    for x, y in data_gen_test:
        true_values += list(y)
        yield x


I am working on the file "Bitcoin_Trading_Bot/Saved Predictions.ipynb".
The first task is:
Load the saved model and weights and predict point-by-point (t+1) predictions generatively
Can you write Python code for it?

model = lstm.load_network('data/model_saved.h5')
predictions = model.predict_generator(
    generator_strip_xy(data_gen_test, true_values),
    steps=steps_test
)

#Save our predictions
with h5py.File(configs['model']['filename_predictions'], 'w') as hf:
    dset_p = hf.create_dataset('predictions', data=predictions)
    dset_y = hf.create_dataset('true_values', data=true_values)

plot_results(predictions[:800], true_values[:800])


I am working on the file "Bitcoin_Trading_Bot/Saved Predictions.ipynb".
The first task is:
Predict multiple points (t+N) ahead to get a trend line
Can you write Python code for it?

#Reload the data-generator
data_gen_test = dl.generate_clean_data(
    configs['data']['filename_clean'],
    batch_size=800,
    start_index=ntrain
)
data_x, true_values = next(data_gen_test)
window_size = 50 #numer of steps to predict into the future

#We are going to cheat a bit here and just take the next 400 steps from the testing generator and predict that data in its whole
predictions_multiple = predict_sequences_multiple(
    model,
    data_x,
    data_x[0].shape[0],
    window_size
)

plot_results_multiple(predictions_multiple, true_values, window_size)


I am working on the file "BivariateNormal.ipynb".
The first task is:
Definim functia care din vectorul probabilitatilor pr, matricea mediilor si array-ul 3D al matricilor de covarianta
evalueaza densitatea de probabilitate a mixturii in punctele unei grile din plan:
Can you write Python code for it?

def mixtureDensity(x,y, pr, med, Sigma):# x este array-ul absciselor nodurilor grilei,
                                        #iar y al ordonatelor
   pos=np.empty(x.shape + (2,))# daca x.shape este (m,n) atunci pos.shape este (m,n,2)
   pos[:, :, 0] = x; pos[:, :, 1] = y
   z=np.zeros(x.shape)
   for i in range(4) :
        z=z+pr[i]*Nd.pdf(pos, mean=med[i,:], cov=Sigma[:,:, i])
   return z


I am working on the file "BivariateNormal.ipynb".
The first task is:
Evident, distributia de probabilitate a pozitiei lui Piqué in teren este o mixtura de mai multe distributii.
Zona mai intens colorata in rosu este cea in care jucatorul a revenit mai des.
In acest document se pot vedea heatmap-urile tuturor jucatorilor din meciul Argentina-Germania.
Pentru optimizarea paginilor WEB se genereaza heatmap-ul click-urilor vizitatorilor
In recunoasterea formelor se pune problema inversa: dintr-un nor de puncte, ce se interpreteaza a fi valori de observatie asupra unei mixturi
(Gaussiene) de K distributii,  se estimeaza conform algoritmului EM (Expectation Maximization)
probabilitatile din definitia mixturii,
punctele ce sunt centrii norilor (adica vectorii medii pentru fiecare distributie bivariata componenta) si elementele matricilor de covarianta. Prezentarea algoritmului aici.
Can you write Python code for it?

from IPython.core.display import HTML
def  css_styling():
    styles = open("./custom.css", "r").read()
    return HTML(styles)
css_styling()


I am working on the file "BivariateNormal.ipynb".
The first task is:
Functia np.histogram2D calculeaza cate puncte sunt generate in fiecare subdreptunghi rezultat din divizarea
unui dreptunghi ce contine punctele generate, prin  50 intervale de subdivizune pe orizontala si 65 pe verticala (bins=[50, 65]).
Setand
 normed=true se calculeaza  probabilitatea ca un punct generat sa cada intr-un astfel de subdreptunghi, ca fiind nr punctelor generate in subdreptunghi supra nr total de puncte. Apoi colorarea subdreptunghiurilor se face mapand intervalul $[0,prob_{max}]$ al probabilitatilor inregistrate, pe nuantele paletei de culori
numita Blues.  Patratele mai intens colorate au probabilitatea experimentala de vizita mai mare, iar cele colorate cu light blue au o probabilitate mai redusa.
Imaginea distributiei de probabilitate astfel generata se numeste heatmap.
Notam ca,
pentru a vizualiza un punct de  coordonate carteziene (x,y), in apelul functiei np.histogram2d, se includ ca argumente
punctele de coordonate pixel $(y,x)$.
In cazul nostru am scris argumentele in aceasta ordine: pts[:,1], pts[:,0].
A se vedea in cursul de Algebra-Geometrie legatura dintre coordonatele pixel si coordonatele carteziene ale unui punct dintr-o imagine
Sa vizualizam  contourplot-ul  si heatmap-ul cu o paleta (colormap) ce nu  consta ca si Blues, folosit mai sus, doar din nuante ale aceleiasi culori.
Lista colormap-urilor din matplotlib este afisata aici:
Can you write Python code for it?

plt.rcParams['figure.figsize'] = (6.0, 4.0)

fig4 = plt.figure(4)
plt.contourf(x, y, V.pdf(pos), cmap=cm.PiYG)
plt.colorbar()

fig5 = plt.figure(5)
heatmap1=plt.imshow(Hmasked, cmap='PiYG', origin='lower',interpolation='nearest', extent=extent)
plt.colorbar()


I am working on the file "BivariateNormal.ipynb".
The first task is:
Mixturi Gaussiene 2D
Avand k distributii de probabilitate Gauss, 2D, de  densitati de probabilitate
 $f_1$, $f_2, \ldots, f_k$, atunci o combinatie convexa a lor cu probabilitatile: $p_1, p_2,\ldots, p_k$, $\sum_{j=1}^kp_k=1$:
$$f=p_1f_1+p_2f_2+\cdots+p_kf_k$$
este o densitate de probabilitate numita  mixtura Gaussiana 2D.
Sa ilustram  o mixtura Gaussiana ce este combinatia a 4 densitati de probabilitate Gaussiene 2D.
Mediile si abaterile standard ale coordonatelor vectorilor aleatori Gaussieni  $(X_i, Y_i)$, $i=0,1,2,3$, le dam in cate un array de 4 linii si 2 coloane, fiecare linie reprezentand vectorul mediilor, respectiv al abaterilor standard ale  distributiilor Gauss 2D, corespunzatoare.
Coeficientii de corelatie $\rho(X_i, Y_i)$, $i=0,1,2,3$, ii dam intr-un vector de 4 coordonate:
Can you write Python code for it?

pr=[0.2, 0.4, 0.15, 0.25]# lista probabilitatilor din definitia mixturii
med=np.array([[0.5,0.5], [0.7,4.3], [2.9, 3.4],[2.5,2]])# mediile
sig=np.array([[0.2, 0.25],   [0.5,0.8],  [0.2375, 0.4125 ],[0.35,0.53]]) # abaterile standard
rho=np.array([0.0, -0.67,-0.76,0.5])# coeficientii de corelatie

def Covariance(m, s, r):# functie ce genereaza matricea de cov a unui vector (X,Y)~N(m,Sigma)
  covar= r*s[0]*s[1]
  return np.array([[s[0]**2, covar], [covar, s[1]**2]])


I am working on the file "BivariateNormal.ipynb".
The first task is:
Observam ca estimatorul lui $\rho$ este suficient de bun. Axa mare a norului eliptic este
de panta pozitiva, pentru ca $\rho(X,Y)>0$.
Apeland acum functia sb.joinplot, cu  cuvantul cheie kind setat pe kde, este afisata o aproximatie a contourplot-ului
densitatii de probabilitate a vectorului aleator normal distribuit, $V=(X,Y)$, iar sus si lateral dreapta, aproximatii ale densitatii lui $X$, respectiv $Y$:
Can you write Python code for it?

sb.jointplot(pts[:,0], pts[:,1], size=6, kind="kde");


I am working on the file "BivariateNormal.ipynb".
The first task is:
Observam ca si in contourplot
zonele corespunzatoare intersectiei cu plane $z=h$, de cota h mai ridicata, sunt mai intens colorate si respectiv sunt colorate
cu nuante  din ce in ce mai light, pe masura ce cota $h$ descreste.
Simulam acum vectorul aleator V si  vom vedea ca  punctele generate vor fi dispuse intr-un disc eliptic la fel orientat
si aproximativ la fel colorat ca acest contourplot:
Can you write Python code for it?

pts=V.rvs(size=5000)# generam 5000 de puncte ca observatii asupra vectorului aleator

print type(pts)

fig3=plt.figure(3)
H, xedges, yedges = np.histogram2d(pts[:,1], pts[:,0], bins=[50,65], normed=True)
Hmasked = np.ma.masked_where(H==0,H) #mascam pixelii a caror valoare este 0
extent = [ xedges[0], xedges[-1],  yedges[0], yedges[-1]]
heatmap=plt.imshow(Hmasked, cmap='Blues', origin='lower',interpolation='nearest', extent=extent)
plt.colorbar()


I am working on the file "BivariateNormal.ipynb".
The first task is:
S-a definit array-ul 3D pos, pentru a corespunde modalitatii de implementare a densitatii
unei distributii normale multivariate.
Pentru a alege un dreptunghi  de definitie pentru mixtura, exploatam inegalitatea lui Cebasev conform careia:
    $P(m-k\sigma<X<m+k\sigma)>1/k^2$. Si anume identificam variabila $X_{jx}$ ce are media minima, respectiv variabila $X_{Jx}$ de medie maxima, intre toate variabilele $X_0, X_1, X_2, X_3$
    si procedam analog si pentru variabilele $Y_0, Y_1, Y_2, Y_3$:
Can you write Python code for it?

mmx=np.min(med[:,0] )
mMx=np.max(med[:,0])#cea mai mica si cea mai mare medie pt X_0, X_1, X_2, X_3
mmy=np.min(med[:,1] )
mMy=np.max(med[:,1])
jx=np.argmin(med[:,0])
Jx=np.argmin(-1.0*med[:,0])
jy=np.argmin(med[:,1])
Jy=np.argmin(-1.0*med[:,1])

plt.rcParams['figure.figsize'] = (10.0, 6.0)
k=3.5
x=np.linspace(mmx-k*sig[jx][0], mMx+k*sig[Jx][0], 100)
print 'x initial', x.shape
y=np.linspace(mmy-k*sig[jy][1], mMx+k*sig[Jy][1], 100)
x,y=np.meshgrid(x,y)# din punctele de divizune pe Ox si Oy se constituie grila 2D
print 'x din grila', x.shape#x este acumarray 2D. Elementele sale sunt abscisele nodurilor grilei
z=mixtureDensity(x,y,pr,  med, Sigma)
fig6=plt.figure(6)
ax = fig6.add_subplot(111, projection='3d')
ax.set_xlabel('x')
ax.set_ylabel('y')
ax.set_title('Densitatea de probabilitate a mixturii Gaussiene')
ax.plot_surface(x, y, z, rstride=1, cstride=1, cmap=cm.Blues,
        linewidth=0, antialiased=False)


I am working on the file "BivariateNormal.ipynb".
The first task is:
Sa generam contourplot-ul mixturii:
Can you write Python code for it?

plt.rcParams['figure.figsize'] = (6.0, 4.0)
fig8=plt.figure(8)
plt.contourf(x, y, z, cmap=cm.Blues)
plt.colorbar()


I am working on the file "BivariateNormal.ipynb".
The first task is:
Sa simulam acum aceasta mixtura:
Can you write Python code for it?

def  simDiscrete(pr):
    k=0
    F=pr[0]
    u=np.random.random()
    while(u>F):
        k+=1
        F=F+pr[k]
    return k

def  GaussianMixture2D(N, pr, med,Sigma):
   dis=[simDiscrete(pr) for i in xrange(N)]#genereaza N observatii asupra distr discrete

   pts=np.empty((N,2), dtype=float)# pts  contine punctele  generate simuland  mixtura
   n=len(pr)
   for k in range(n):

       I=[j for j in range(N) if dis[j] == k]# lista indicilor elementelor listei dis, egale
                                             #cu k (k=0, 1,  ..., n-1)
       s=len(I)
       ptsk=Nd.rvs(size=s, mean=med[k], cov=Sigma[:,:,k])#generam atatea valori din ia distr. f_k
                                                     #cat este lungimea lui I
       pts[I,:]=ptsk # in pozitiile I copiem valorile generate
   return pts

N=10000
pts=GaussianMixture2D(N,pr, med, Sigma)

plt.rcParams['figure.figsize'] = (6.0, 4.0)
fig9=plt.figure(9)
H, xedges, yedges = np.histogram2d(pts[:,1], pts[:,0], bins=[75,75], normed=True)
Hmasked = np.ma.masked_where(H==0,H)
extent = [ xedges[0], xedges[-1],  yedges[0], yedges[-1]]
heatmap=plt.imshow(Hmasked, cmap='Blues', origin='lower',interpolation='nearest', extent=extent)
plt.colorbar()


I am working on the file "BivariateNormal.ipynb".
The first task is:
Vizualizarea densitatii de probabilitate prin contourplot si a distributiei experimentale prin heatmap
Sa desenam graficul densitatii de probabilitate avectorului aleator $V=(X,Y)$. Fiind o suprafata,
importam  in plus instrumentele de grafica 3D:
Can you write Python code for it?

from mpl_toolkits.mplot3d import Axes3D
from matplotlib import cm# modulul matplotlib.cm contine colormap-urile

plt.rcParams['figure.figsize'] = (10.0, 6.0)
k=3
x=np.linspace(m[0]-k*s[0], m[0]+k*s[0], 100)
y=np.linspace(m[1]-k*s[1], m[1]+k*s[1], 100)
x,y=np.meshgrid(x,y)
pos=np.empty(x.shape + (2,))
pos[:, :, 0] = x; pos[:, :, 1] = y
z=V.pdf(pos)

fig1 = plt.figure(1)
ax = fig1.add_subplot(111, projection='3d')
ax.set_xlabel('x')
ax.set_ylabel('y')
surf = ax.plot_surface(x, y, z, rstride=1, cstride=1, cmap=cm.Blues,
        linewidth=0, antialiased=True)


I am working on the file "BivariateNormal.ipynb".
The first task is:
kde  inseamna kernel density estimation. Exista cativa algoritmi folositi in statistica si machine learning
care estimeaza densitatea de probabilitate a unei variabile aleatoare (vector aleator)  din observatii asupra acestora.
Exemple de aplicatii ale heatmap-urilor
Nu doar distributiile normale bivariate se vizualizeaza prin heatmap-uri, ci orice alta distributie 2D.
De exemplu dupa campionatul mondial din 2010 pe site-ul FIFA s-au afisat heatmap-urile color ale pozitiei jucatorilor in teren in timpul fiecarui meci.
Ilustram mai jos heatmap-ul asociat jucatorului Piqué:
Can you write Python code for it?

from IPython.display import Image

Image(filename='Imags/heatmap_pique.jpg')#imaginea a fost postata pana de curand
#aici:
#http://www.fifa.com/worldcup/archive/southafrica2010/statistics/players/player=216973/
#heatmap.html


I am working on the file "Black Box Tutorial.ipynb".
The first task is:
Feature Extraction
Can you write Python code for it?

# Apply SIFT to extract features
cg.extract_features(method='sift', extractor_parameters={'nfeatures': 1000})


I am working on the file "Black Box Tutorial.ipynb".
The first task is:
The graph method extract_features uses OpenCV in the background to calcluate features for every node in the graph using the detector methods SIFT, SURF, ORB or FAST. Here, you can see we are using SIFT. Every node now contains a pandas dataframe containing keypoint information. Along with the dataframe, there is descriptor information and image metadata.
We can see this be ploting a single node (i.e. a single image):
Can you write Python code for it?


I am working on the file "Black Box Tutorial.ipynb".
The first task is:
The graph object
Everything in Autocnet operates on a NetworkX graph object as the underlying structure. Every node is an image and each edge contains data expressing the relationship bertween every image (e.g. feature matches), in other words, edges connect overlapping images and describe their relationship. Here, we populate the graph from the input file and store it into cg. This can be either an ISIS cube, GDAL supported file, or an adjacency matrix (the least ideal option).
TODO: Find better input file example
Can you write Python code for it?


I am working on the file "Black Box Tutorial.ipynb".
The first task is:
We can see more by printing node data and the first few keypoints from the pandas dataframe:
Can you write Python code for it?

print(cg.node[0])

print("\n\n", cg.node[0].keypoints[:5])


I am working on the file "Black-Box Stochastic Variational Inference.ipynb".
The first task is:
Define our log posterior, and instance the dataset.
Can you write Python code for it?


I am working on the file "Black-Box Stochastic Variational Inference.ipynb".
The first task is:
Defining a simple 1D regression problem, so we can visualise the output.
Can you write Python code for it?

# Specify inference problem by its unnormalized log-posterior.
rbf = lambda x: norm.pdf(x, 0, 1)
sq = lambda x: np.sin(x)
num_weights, predictions, logprob = \
    make_nn_funs(layer_sizes=[1, 10, 10, 1], L2_reg=0.01,
                 noise_variance = 0.01, nonlinearity=rbf)


I am working on the file "Black-Box Stochastic Variational Inference.ipynb".
The first task is:
Defining our target distribution.
Can you write Python code for it?

%%output size=250
Z = isocontours(target)
target_distribution = hv.Image(Z)
hv.operation.contours(target_distribution, levels=np.linspace(0,max(Z.ravel()),10))


I am working on the file "Black-Box Stochastic Variational Inference.ipynb".
The first task is:
Here's our intractable distribution, it even says it right there, and we have a graph of it coming up!
Can you write Python code for it?

# Build variational objective.
objective, gradient, unpack_params = \
    black_box_variational_inference(log_posterior, D, num_samples=2000)


I am working on the file "Black-Box Stochastic Variational Inference.ipynb".
The first task is:
Just imports. Whe have to import numpy through autograd, hence the autograd.numpy as np, to have access to automatic differentiation for all of our numpy code.
Can you write Python code for it?

# %load ../examples/optimizers.py
from __future__ import absolute_import

import autograd.numpy as np
from builtins import range

def adam(grad, x, callback=None, num_iters=100,
         step_size=0.001, b1=0.9, b2=0.999, eps=10**-8):
    """Adam as described in http://arxiv.org/pdf/1412.6980.pdf.
    It's basically RMSprop with momentum and some correction terms."""
    m = np.zeros(len(x))
    v = np.zeros(len(x))
    for i in range(num_iters):
        g = grad(x, i)
        if callback: callback(x, i, g)
        m = (1 - b1) * g      + b1 * m  # First  moment estimate.
        v = (1 - b2) * (g**2) + b2 * v  # Second moment estimate.
        mhat = m / (1 - b1**(i + 1))    # Bias correction.
        vhat = v / (1 - b2**(i + 1))
        x -= step_size*mhat/(np.sqrt(vhat) + eps)
    return x


I am working on the file "Black-Box Stochastic Variational Inference.ipynb".
The first task is:
Looking at some of the outputs from the network before training. Obviously, the match to the dataset above isn't great.
Can you write Python code for it?

print("Optimizing variational parameters...")
variational_params = adam(gradient, init_var_params,
                          step_size=0.1, num_iters=1000, callback=callback)


I am working on the file "Black-Box Stochastic Variational Inference.ipynb".
The first task is:
Now we call the same black_box_variational_inference function from above, so we're using an isotropic Gaussian variational distribution on all the weights in our network. Then, we initialise all the variational parameters we're going to need.
Can you write Python code for it?

def sample_predictions(params, inputs):
    # Sample functions from posterior.
    #rs = npr.RandomState(0)
    mean, log_std = unpack_params(params)
    sample_weights = rs.randn(10, num_weights) * np.exp(log_std) + mean
    return predictions(sample_weights, inputs)

%%output size=200
preds = sample_predictions(init_var_params, inputs)
hv.Overlay([hv.Curve(zip(inputs.ravel(), p.ravel())) for p in preds])


I am working on the file "Black-Box Stochastic Variational Inference.ipynb".
The first task is:
Now we can graph the isotropic approximation next to the intractable distribution and see that we've not really got a perfect approximation, but at least it's doing its best, right?
Bayesian Neural Network
Next, fitting a Bayesian neural network in only a few more lines:
Can you write Python code for it?

def make_nn_funs(layer_sizes, L2_reg, noise_variance, nonlinearity=np.tanh):
    """These functions implement a standard multi-layer perceptron,
    vectorized over both training examples and weight samples."""
    shapes = list(zip(layer_sizes[:-1], layer_sizes[1:]))
    num_weights = sum((m+1)*n for m, n in shapes)

    def unpack_layers(weights):
        num_weight_sets = len(weights)
        for m, n in shapes:
            yield weights[:, :m*n]     .reshape((num_weight_sets, m, n)),\
                  weights[:, m*n:m*n+n].reshape((num_weight_sets, 1, n))
            weights = weights[:, (m+1)*n:]

    def predictions(weights, inputs):
        """weights is shape (num_weight_samples x num_weights)
           inputs  is shape (num_datapoints x D)"""
        inputs = np.expand_dims(inputs, 0)
        for W, b in unpack_layers(weights):
            outputs = np.einsum('mnd,mdo->mno', inputs, W) + b
            inputs = nonlinearity(outputs)
        return outputs

    def logprob(weights, inputs, targets):
        log_prior = -L2_reg * np.sum(weights**2, axis=1)
        preds = predictions(weights, inputs)
        log_lik = -np.sum((preds - targets)**2, axis=1)[:, 0] / noise_variance
        return log_prior + log_lik

    return num_weights, predictions, logprob


I am working on the file "Black-Box Stochastic Variational Inference.ipynb".
The first task is:
Our callback function while we're running the updates. It's just going to print out the progress as we loop around in the adam function above.
Can you write Python code for it?

target = lambda x: np.exp(log_posterior(x, 1))


I am working on the file "Black-Box Stochastic Variational Inference.ipynb".
The first task is:
Running the function we defined above...
Can you write Python code for it?

def isocontours(func, xlimits=[-2, 2], ylimits=[-4, 2], numticks=512):
    x = np.linspace(*xlimits, num=numticks)
    y = np.linspace(*ylimits, num=numticks)
    X, Y = np.meshgrid(x, y)
    zs = func(np.concatenate([np.atleast_2d(X.ravel()), np.atleast_2d(Y.ravel())]).T)
    Z = zs.reshape(X.shape)
    return Z


I am working on the file "Black-Box Stochastic Variational Inference.ipynb".
The first task is:
Sample from our variational distribution, and then use this to define a lower bound (a variational lower bound) on the marginal likelihood. Then, take a gradient of this stochastic estimate of the lower bound so we can perform gradient updates to the parameters we used to sample from the variational distribution; this is black box variational inference in 23 lines minus comments.
Can you write Python code for it?

# Specify an inference problem by its unnormalized log-posterior.
D = 2
def log_posterior(x, t):
    """An example 2D intractable distribution:
    a Gaussian evaluated at zero with a Gaussian prior on the log-variance."""
    mu, log_sigma = x[:, 0], x[:, 1]
    prior       = norm.logpdf(log_sigma, 0, 1.35)
    likelihood  = norm.logpdf(mu,        0, np.exp(log_sigma))
    return prior + likelihood


I am working on the file "Black-Box Stochastic Variational Inference.ipynb".
The first task is:
Then we can optimise the parameters with no callback, so we see no progress, and it should finish fairly quickly.
Can you write Python code for it?

%%output size=200
mean, log_std = unpack_params(variational_params)
variational_contour = lambda x: mvn.pdf(x, mean, np.diag(np.exp(2*log_std)))
Z = isocontours(variational_contour)
variational_distribution = hv.Image(Z)
hv.operation.contours(variational_distribution, levels=np.linspace(0,max(Z.ravel()),10))+target_distribution


I am working on the file "Black-Box Stochastic Variational Inference.ipynb".
The first task is:
These nested functions are kind of complicated. One of the strange parts above is the unpack_layers function. This is a quirk of how autograd works, and the fact that they wanted to write this in as few lines as possible. The grad function of autograd only takes the gradient of a single numpy array argument to a function (you state which argument, or it defaults to the first), so in order to only call grad once they have to unpack the weights from a single large array. Alternatively, they could have had all of these parameters in lists and iterated over the grad calls (this is kind of what Lasagne does, keeping everything in lists which are assumed to stay in the correct order). But, they already wrote the black_box_variational_inference function above, and they're not going to rewrite it.
In predictions, we're getting the output of the network for a given input and set of weights. They do this in as few lines as possible using the completely opaque np.einsum, which is a matrix operation in Einstein notation. This seems like a really useful thing to understand well, but, like regular expressions, not something you're going to remember without regular use.
Finally, we describe the log probability, which is just an L2 regularised squared error loss function.
Can you write Python code for it?

def build_toy_dataset(n_data=40, noise_std=0.1):
    D = 1
    rs = npr.RandomState(0)
    inputs  = np.concatenate([np.linspace(0, 2, num=n_data/2),
                              np.linspace(6, 8, num=n_data/2)])
    targets = np.cos(inputs) + rs.randn(n_data) * noise_std
    inputs = (inputs - 4.0) / 4.0
    inputs  = inputs.reshape((len(inputs), D))
    targets = targets.reshape((len(targets), D))
    return inputs, targets


I am working on the file "Black-Box Stochastic Variational Inference.ipynb".
The first task is:
This is going to give us an image with the contours of a distribution. Going to use it to graph the intractable distribution from above.
Can you write Python code for it?

def callback(params, t, g):
    print("Iteration {} lower bound {}".format(t, -objective(params, t)))

    plt.cla()
    target_distribution = lambda x : np.exp(log_posterior(x, t))
    plot_isocontours(ax, target_distribution)

    mean, log_std = unpack_params(params)
    variational_contour = lambda x: mvn.pdf(x, mean, np.diag(np.exp(2*log_std)))
    plot_isocontours(ax, variational_contour)
    plt.draw()
    plt.pause(1.0/30.0)


I am working on the file "Black-Box Stochastic Variational Inference.ipynb".
The first task is:
This isn't really necessary, we could just use vanilla SGD, but this is probably slightly better. Defines the adaptive gradient updates of the parameters, and the optimisation loop. Optionally, we can add a callback function.
Can you write Python code for it?

def black_box_variational_inference(logprob, D, num_samples):
    """Implements http://arxiv.org/abs/1401.0118, and uses the
    local reparameterization trick from http://arxiv.org/abs/1506.02557"""

    def unpack_params(params):
        # Variational dist is a diagonal Gaussian.
        mean, log_std = params[:D], params[D:]
        return mean, log_std

    def gaussian_entropy(log_std):
        return 0.5 * D * (1.0 + np.log(2*np.pi)) + np.sum(log_std)

    rs = npr.RandomState(0)
    def variational_objective(params, t):
        """Provides a stochastic estimate of the variational lower bound."""
        mean, log_std = unpack_params(params)
        samples = rs.randn(num_samples, D) * np.exp(log_std) + mean
        lower_bound = gaussian_entropy(log_std) + np.mean(logprob(samples, t))
        return -lower_bound

    gradient = grad(variational_objective)

    return variational_objective, gradient, unpack_params


I am working on the file "Black-Box Stochastic Variational Inference.ipynb".
The first task is:
You don't mind adding 1000 lines to the notebook output do you? Should be able to see the lower bound consistently increasing.
Can you write Python code for it?

%output size=350
plot_inputs = np.linspace(-4,4,200).reshape(-1,1)
preds = sample_predictions(variational_params, plot_inputs)
hv.Overlay([hv.Curve(zip(plot_inputs.ravel(), p.ravel())) for p in preds])*hv.Points(zip(inputs,targets))


I am working on the file "BlackBoxModel02.ipynb".
The first task is:
Calcul pour toutes les périodes :
prends du temps
Can you write Python code for it?

# init
df['T_theo'] = 0
df['eta_M'], df['h_M'] = 0, 0

# valeur initiale
T_zero = df['T_int'][ df['T_int'].first_valid_index() ]

for grp_id in Groupes:
    print( '%i, ' % grp_id, end='' )
    data_day = df_byday.get_group(  grp_id )

    # debug cas où aucun donnée exp. :
    if np.isnan( T_zero ):
        T_int = data_day['T_int']
        if np.isnan(  T_int ).all():
            T_zero = 0
        else:
            T_zero = data_day['T_int'][ data_day['T_int'].first_valid_index() ]

    # estimation
    params, res = fit_a_day( data_day, T_zero )

    # save
    df.loc[ data_day.index, 'T_theo'] =  res
    df.loc[ data_day.index, 'eta_M'] =  params[1]
    df.loc[ data_day.index, 'h_M'] =  params[0]

    # valeur initiale pour l'étape suivante
    T_zero = res[-1]

print('done')

df[['T_int', 'T_theo']].plot( figsize=(14, 5)  );

df[['T_int', 'T_theo']].plot( figsize=(14, 5)  );

df[['T_int', 'temperature', 'T_theo']].plot( figsize=(14, 5)  );

df[['flux_tot']].plot( figsize=(14, 5)  );

plt.figure( figsize=(14, 5) )
plt.subplot( 2, 1, 1 )
plt.plot( df[['h_M']] ); plt.ylabel('h_M');

plt.subplot( 2, 1, 2 )
plt.plot( df[['eta_M']], 'r' ); plt.ylabel('eta_M');


I am working on the file "BlackBoxModel02.ipynb".
The first task is:
Découpage en période jour / nuit
Can you write Python code for it?

""" Estimation des périodes de jour et de nuit à partir du flux solaire
"""

df['isnight'] = ( df['flux_tot'] == 0 ).astype(int)

# Permet de numéroter les périodes :
nights_days = df['isnight'].diff().abs().cumsum()
nights_days[0] = 0

df_byday  = df.groupby(nights_days)

Groupes = [ int( k ) for k in df_byday.groups.keys() ]

df_byday['temperature'].plot( figsize=(14, 3) );

df_byday['T_int'].plot( figsize=(14, 3) );


I am working on the file "BlackBoxModel02.ipynb".
The first task is:
Et le flux solaire, calculé pour mon appartement, et projété suivant la surface et l'orientation de mes fenètres (Velux):
Can you write Python code for it?

# Flux solaire sur les vitres:
df[['flux_tot']].plot( figsize=(14, 4) ); plt.ylabel('Watt');


I am working on the file "BlackBoxModel02.ipynb".
The first task is:
Modèle boite noire 02
L'idée est d'estimer les paramètres du modèle à partir de la mesure expérimentale de la température intérieure ($T$) et des données météo. On n'obtient pas forcement une valeur physique significative pour ces coefficients, mais cela permet d'estimer et de comparer les variations de jour en jour, et donc à priori de voir l'effet du comportement de l'habitant (ouverture des fenêtres et rideaux  en été).
Le modèle 'boite-noire'
On souhaite le modèle le plus simple possible, avec seulement deux coefficients : la résistance thermique $h$ avec l'extérieure, une masse thermique $M$ et un flux de chaleur externe $\eta\Phi(t)$.
Schéma électrique équivalent :

et l'équation différentielle corespondante:
$$
 \frac{dT}{dt} = \frac{h}{M} \,\left[ T_{ext}(t) - T \right] + \frac{\eta}{M} \, \Phi(t)
$$
$T(t)$ est la température à l'intérieure de l'appartement.
$T_{ext}(t)$ est la température extérieure donnée par la météo.
$\Phi(t)$ est le flux solaire (en Watt) sur les surfaces vitrées.
Il y a deux paramètres inconnues, tous les deux normés par la masse thermique $M$:
 * $\eta$ qui correspond à l'absorption des rayons solaire, normalement entre 0 et 1.
 * $h$ qui correspond à l'isolation avec l'air extérieur (en W/K).

$M$, enfin, est la masse thermique de l'appartement (en J/K) ... Elle détermine le temps de réponse caractérique du système, et est pour cette raison difficile à estimer parce que non unique.

Chargement des données
Données obtenues avec le notebook get_data_and_preprocess.ipynb.
Can you write Python code for it?

df_full  = pd.read_pickle( 'weatherdata.pck' )

df = df_full[['T_int', 'temperature', 'flux_tot', 'windSpeed']].copy()


I am working on the file "BlackBoxModel02.ipynb".
The first task is:
On a l'enregistrement de la température intérieure ($T_{int}$) et la température extérieure ('temperature') :
Can you write Python code for it?

df[['T_int', 'temperature']].plot( figsize=(14, 4) ); plt.ylabel('°C');


I am working on the file "BlackBoxModel02.ipynb".
The first task is:
Rq: Facteur 100 et 1e-6 pour avoir des valeurs proches de l'unité, et le même ordre de grandeur entre $\Phi$ et $\Delta T$... pour l'optimisation
Test sur les données entières :
Can you write Python code for it?

params =  ( 3, 3 )

res = apply_model( df, 30, params )

plt.figure( figsize=(14, 4) )
plt.plot( res )
plt.plot( df['T_int'].as_matrix() ) ;


I am working on the file "BlackBoxModel02.ipynb".
The first task is:
Résidus
Can you write Python code for it?

R = df['T_int'] - df['T_theo']

R.plot( figsize=(14, 5), style='k'  ); plt.ylabel('°C');

# Plot variation relative temp.

plt.figure( );
for grp_id in Groupes:
    data = df_byday.get_group(  grp_id )

    if np.isnan( data['T_int'] ).all():
        continue

    T_int = data['T_int'].as_matrix()
    Tmin, Tmax = T_int[ ~ np.isnan( T_int ) ].min(), T_int[ ~ np.isnan( T_int ) ].max()
    if  data['isnight'].all():
        T_int = T_int - Tmax
    else:
        T_int = T_int - Tmin
    plt.plot( T_int )

# Corrélation T_ext <-> Phi
from scipy.stats import pearsonr
norm = lambda X: (X - X.min())/(X.max() - X.min())

coeffs_cor, groupes_id = [], []

plt.figure( );
for grp_id in Groupes[2:]:
    data = df_byday.get_group(  grp_id )

    if data['isnight'].all():
        continue

    T_ext = data['temperature'].as_matrix()
    phi = data['flux_tot'].as_matrix()

    #T_ext, phi = norm(T_ext), norm(phi)

    plt.plot( T_ext, phi, '.' )

    coeffs_cor.append( pearsonr(T_ext, phi)[0] )
    groupes_id.append( grp_id )
#plt.axis('equal')

plt.plot( coeffs_cor ) ;

sorted( zip( groupes_id, coeffs_cor ), key=lambda x:x[1] )

data = df_byday.get_group(  Groupes[13] )

T_int = data['T_int'].as_matrix()
T_ext = data['temperature'].as_matrix()
phi = data['flux_tot'].as_matrix()

plt.plot( norm(phi) )
plt.plot( norm(T_ext) );


I am working on the file "BlackBoxModel02.ipynb".
The first task is:
Solveur équation diff. (ODE)
L'équation est résolue en temps avec odeint de scipy (doc, OdePack).
Can you write Python code for it?

from scipy.integrate import odeint

def get_dTdt(  T, t, params, get_Text, get_Phi ):
    """ dérivé de la température p/r au temps
        params :   [ h/M  ,  eta/M ]
        get_Text, get_Phi: fonctions d'interpolation
    """

    T_ext  = get_Text( t )
    phi = get_Phi( t )

    dTdt = params[0] * ( T_ext - T ) + params[1] / 100 * phi

    return 1e-6*dTdt

def apply_model( data, T_start, params, full_output=False ):

    data_dict = data.to_dict(orient='list')
    time_sec = data.index.astype(np.int64) // 1e9    # conversion en secondes

    # construction des fonctions d'interpolations:
    get_Text = lambda t: np.interp( t, time_sec, data_dict['temperature'] )
    get_Phi  = lambda t: np.interp( t, time_sec, data_dict['flux_tot'] )

    T_theo = odeint(get_dTdt, T_start, time_sec, args=(params, get_Text, get_Phi  ), \
                full_output=full_output, h0=30*60)
    # h0 : pas de temps initial utilisé par le solveur

    return T_theo.flatten()


I am working on the file "BlackBoxModel02.ipynb".
The first task is:
Tracé pour une période (debug)
Can you write Python code for it?

len( Groupes )

data = df_byday.get_group(  Groupes[1] )

T_int = data['T_int'].interpolate().as_matrix()
T_zero = T_int[ ~ np.isnan( T_int ) ][0]

params, res = fit_a_day( data, T_zero )

print( data.index[0] )
print( params )

plt.figure( figsize=(14, 5) )
plt.plot(data.index,  res, '--', label='T_theo' )
plt.plot(data.index, T_int, label='T_int' );
plt.plot(data.index, data['temperature'].as_matrix() , label='T_ext' );

plt.plot(data.index, data['flux_tot'].as_matrix()/100 + 20,  label='~ Flux'  );
plt.legend();


I am working on the file "Blog Post Content/.ipynb_checkpoints/Airport Waiting Time-checkpoint.ipynb".
The first task is:
And this is the pattern for the scanner
Can you write Python code for it?

FRISK_PATTERN = sp.poisson.rvs(FRISK_RATE,size=SIM_TIME)
FRISK_LIST = []
for index, item in enumerate(FRISK_PATTERN):
    FRISK_LIST += [index]*item

arrivalfig = figure()
arrivalplot = plot(TIMEAXIS,FRISK_PATTERN,'ro-')
ylabel('People Leaving at time t from frisking counter')
xlabel("Time (minutes)")
show()


I am working on the file "Blog Post Content/.ipynb_checkpoints/Airport Waiting Time-checkpoint.ipynb".
The first task is:
Critical to note that this ignores the queuing and assumes that xx people are processed at each time interval at the counter. This will be used in conjunction with the scanner output to choose the bottle neck at each point in time
Can you write Python code for it?

EXIT_NUMER = zip(FRISK_PATTERN,SCAN_PATTERN)
EXIT_NUMBER = [min(k) for k in EXIT_NUMER]

plot(EXIT_NUMBER,'o')
show()
EXIT_PATTERN = []


I am working on the file "Blog Post Content/.ipynb_checkpoints/Airport Waiting Time-checkpoint.ipynb".
The first task is:
For this simulation, we'll be using numpy and scipy for their statistical and matrix math prowess and matplotlib as our primary plotting tool
Can you write Python code for it?

#Everything will me modeled as a Poisson Process

SIM_TIME = 180

QUEUE_ARRIVAL_RATE = 15

N_SCANNERS =4

SCANNER_BAG_CHECKING_RATE = 3 #Takes 20 seconds to put your bag on Scanner

FRISK_MACHINES_PER_SCANNER = 3 #Number of people checking machine per scanner

N_FRISK_MACHINES = N_SCANNERS*FRISK_MACHINES_PER_SCANNER

FRISK_CHECKING_RATE = 2 #Half a minute per frisk

SCANNER_RATE = SCANNER_BAG_CHECKING_RATE*N_SCANNERS

FRISK_RATE = FRISK_CHECKING_RATE*N_FRISK_MACHINES

FRISK_ARRIVAL_RATE = SCANNER_RATE


I am working on the file "Blog Post Content/.ipynb_checkpoints/Airport Waiting Time-checkpoint.ipynb".
The first task is:
Minimum number of processed people between the scanners and the frisking is the bottleneck at any given time, and this will be the exit rate at any given time.
Can you write Python code for it?

for index, item in enumerate(EXIT_NUMBER):
    EXIT_PATTERN += [index]*item

RESIDUAL_ARRIVAL_PATTERN = ARRIVAL_LIST[0:len(EXIT_PATTERN)]
WAIT_TIMES = [m-n for m,n in zip(EXIT_PATTERN,RESIDUAL_ARRIVAL_PATTERN)]

#print EXIT_PATTERN

'''
for i,val in EXIT_PATTERN:
    WAIT_TIMES += [ARRIVAL_PATTERN(i) - val]
'''

plot(WAIT_TIMES,'r-')
ylabel('Wait times for people entering the queue')
xlabel("Order of entering the queue")
show()


I am working on the file "Blog Post Content/.ipynb_checkpoints/Airport Waiting Time-checkpoint.ipynb".
The first task is:
We're taking the arrivals at each of the time intervals, generated by a poisson function and storing the number of people who have arrived at each minute.
The ARRIVAL_LIST variable is used to calculate the entry time of each of the people in the queue. This will be later used to assess overall wait time for people in the queue.
The time axis is used to help plot results as X-axis variable
Can you write Python code for it?

SCAN_PATTERN = sp.poisson.rvs(SCANNER_RATE,size=SIM_TIME)
SCAN_LIST = []
for index, item in enumerate(SCAN_PATTERN):
    SCAN_LIST += [index]*item

arrivalfig = figure()
arrivalplot = plot(TIMEAXIS,SCAN_PATTERN,'o-')
ylabel('People arrived at time t for the scanner')
xlabel("Time (minutes)")
show()


I am working on the file "Blog Post Content/Airport Waiting Time.ipynb".
The first task is:
And this is the pattern for the scanner
Can you write Python code for it?


I am working on the file "Blog Post Content/Airport Waiting Time.ipynb".
The first task is:
Critical to note that this ignores the queuing and assumes that xx people are processed at each time interval at the counter. This will be used in conjunction with the scanner output to choose the bottle neck at each point in time
Can you write Python code for it?

EXIT_NUMER = zip(FRISK_PATTERN,SCAN_PATTERN)
EXIT_NUMBER = [min(k) for k in EXIT_NUMER]

#plot(EXIT_NUMBER,'o')
#show()
EXIT_PATTERN = []

for index, item in enumerate(EXIT_NUMBER):
    EXIT_PATTERN += [index]*item


I am working on the file "Blog Post Content/Airport Waiting Time.ipynb".
The first task is:
For this simulation, we'll be using numpy and scipy for their statistical and matrix math prowess and matplotlib as our primary plotting tool
Can you write Python code for it?


I am working on the file "Blog Post Content/Airport Waiting Time.ipynb".
The first task is:
Minimum number of processed people between the scanners and the frisking is the bottleneck at any given time, and this will be the exit rate at any given time.
Can you write Python code for it?

RESIDUAL_ARRIVAL_PATTERN = ARRIVAL_LIST[0:len(EXIT_PATTERN)]
WAIT_TIMES = [m-n for m,n in zip(EXIT_PATTERN,RESIDUAL_ARRIVAL_PATTERN)]

#print EXIT_PATTERN

'''
for i,val in EXIT_PATTERN:
    WAIT_TIMES += [ARRIVAL_PATTERN(i) - val]
'''

plot(WAIT_TIMES,'r-')
ylabel('Wait times for people entering the queue')
xlabel("Order of entering the queue")
ylim([0,40])
show()


I am working on the file "BlogPosts/Average_precision/Average_precision_post.ipynb".
The first task is:
For every value of recall greater than 0, the precision is exactly 0.1. Rather than interpolating linearly between adjacent operating points, we instead use a step function consisting of horizontal and vertical lines. Notice that it no longer matters how we define the precision when recall is exactly 0 since there is no area under that point.
The intuition behind step-wise interpolation
This may seem a little arbitrary, but I hope I can convince you that this is the natural way to interpolate these points.
Imagine that, instead of assigning a score of 0.5 to each of the examples, our classifier is actually producing a long floating point number, but we only see that number after it has been rounded to some arbitrary precision. So perhaps the scores are [0.500001, 0.499992, …]. Since we can't recover the actual scores, we can hallucinate them by adding some tiny amount of noise to the scores. This is equivalent to insisting that the classifier produce a strict ranking by arbitrarily breaking ties.
Now if we plot the operating points without any interpolation, we see that we approximate the shape in the previous figure:
Can you write Python code for it?

some_noise = np.random.normal(
    loc=0, scale=0.1, size=len(constant_preds))

noisy_preds = constant_preds + some_noise

plot_recall_precision_from_predictions(
    obs, noisy_preds)


I am working on the file "BlogPosts/Average_precision/Average_precision_post.ipynb".
The first task is:
However, this ignores the recall values associated with each threshold. To see why we should care, let's invent a new dataset consisting of 5 positive examples and 3 negative examples, and two models which assign scores as follows:
<img src="fig/table2.png" />
Both assign the same labels for thresholds less than 0.5 or more than 0.8, so let's consider a threshold in between. Both models have a precision of 0.5, since half of the positively labelled examples are actually negative. But model 1 finds 2 of the 5 positives, and model 2 identifies only 1 of them. The models seems different, and yet their average precision is the same.
Interpolated average precision
Since the thresholds define a series of (recall, precision) points (called operating points), it's natural to plot them on a precision–recall graph. Here are the operating points derived from the oranges and apples example above:
Can you write Python code for it?

%matplotlib inline
ranked_predictions = [1,1,0,1,0,1,1,0,0,0]
p, r = operating_points(ranked_predictions)
plot_recall_precision(p, r)


I am working on the file "BlogPosts/Average_precision/Average_precision_post.ipynb".
The first task is:
If we continue adding observations where the positive and negative data are in the same ratio, the operating points get closer and closer to the horizontal line above:
Can you write Python code for it?

many_obs, many_constant_preds = generate_data_and_constant_predictions(
    n=100000, frac_positive=0.1)

much_noise = np.random.normal(
    loc=0, scale=0.1, size=len(many_constant_preds))

noisy_preds = many_constant_preds + much_noise

plot_recall_precision_from_predictions(many_obs, noisy_preds)


I am working on the file "BlogPosts/Average_precision/Average_precision_post.ipynb".
The first task is:
If we trusted this measure, we'd conclude that the do-nothing model was better than a model that can achieve a true positive rate of 10% at the expense of a false positive rate of only 0.1%. With uneven datasets like the one we're working on here (10% positives), it's incredibly difficult to add an operating point above the line joining the end points.
We can instead use the step interpolation strategy, which comes from comparing these graphs:
Can you write Python code for it?

compare_recall_precisions_from_predictions(
    obs,
    OrderedDict([['Constant prediction', constant_preds],
                 ['Improved prediction', better_preds]]),
    interpolation='step',
    title='Step-wise interpolation')


I am working on the file "BlogPosts/Average_precision/Average_precision_post.ipynb".
The first task is:
Notice that there are always two operating points that are independent of our scores:

The first corresponds to a threshold above the highest score: that means we label all of the examples as negatives, and so the recall is trivially 0 and we just defined the precision to be 1.

The second corresponds to a threshold below the lowest score: that means we predict 1 for all of the examples. In this case, recall is 1, and precision is the underlying rate of positives in the dataset.

If a model assigns the same score to all of the examples, then these are the only operating points we see (because the labels are all the same for all thresholds):
Can you write Python code for it?

obs, constant_preds = generate_data_and_constant_predictions(
    n=100, frac_positive=0.1)

plot_recall_precision_from_predictions(obs, constant_preds)


I am working on the file "BlogPosts/Average_precision/Average_precision_post.ipynb".
The first task is:
Now let's invent a second model which simply rounds the first model's scores to the nearest tenth. So a score of 0.5321 is mapped to 0.5. We can use linear interpolation to compare the interpolated average precision for the two models:
Can you write Python code for it?

compare_recall_precisions_from_predictions(
    y, OrderedDict([['Unrounded', scores],
                    ['Rounded',   np.round(scores, 1)]]),
    interpolation='linear',
    title='Linear interpolation')


I am working on the file "BlogPosts/Average_precision/Average_precision_post.ipynb".
The first task is:
Now the second model is preferred to the first. (We should note that the area is still very low: given the unevenness of the classes, this is inevitable without a very good model or a very easy task!)
Hammering home the point
In case you need further convincing, let's train a model that classifies a heavily-unbalanced dataset (only 5% of examples have a positive label). This model achieves an ROC AUC score of around 0.6 — good but not spectacular:
Can you write Python code for it?

y, scores, roc_auc = train_model_and_evaluate(
    n_dim=50, n_samples=5000, frac_positive=0.05, mixing_factor=0.02)

print("Model achieves ROC AUC of {:4.4f}".format(roc_auc))


I am working on the file "BlogPosts/Average_precision/Average_precision_post.ipynb".
The first task is:
Precision and recall
Consider a classifier whose job is to distinguish between photographs of apples and photographs of oranges. We'll encode an orange as a positive example and an apple as a negative example. The classifier assigns scores between 0 and 1 like this:

We've added a dotted blue line at 0.38, which we'll call the threshold for oranginess: anything with a score more than 0.38 is labelled as an orange, and anything with a score less than 0.38 is labelled an apple. We can interpret this as splitting the available data into two bins, B+ and B−. Unfortunately this isn't a very good classifier: we've found 4 out of the 5 oranges, but we've also labelled 2 out of the 5 apples as oranges. These two measures are called recall and precision, and ideally we want both to be high.

Precision: the fraction of objects in B+ that are in fact positive.
Recall: the fraction of true positives correctly assigned to B+.

By selecting different thresholds, we can weigh up precision and recall: a higher threshold is likely to give us higher precision, and a lower threshold will give us a higher recall.
Since there are only 10 examples here, there are only 11 interesting places we can put the threshold: between some pair of them, or above or below all of the scores. Here's a table showing all of the precision and recall scores we could get by moving the threshold.

It's worth paying attention to the top row's precision: we've defined 0/0 = 1 which seems ridiculous until you consider that, out of the 0 examples we labelled as positives, all 0 were in fact positive — we didn't make a single mistake! If this seems unprincipled, accept it as a definition for now: we'll come back to it.
If we have a spread of scores, then as we increase the threshold from 0 to 1, we will incrementally improve the recall by moving positive examples from the negative bin B− to the positive bin...
Can you write Python code for it?


I am working on the file "BlogPosts/Average_precision/Average_precision_post.ipynb".
The first task is:
TL;DR Interpolated average precision is a common metric for classification tasks. However, interpolating linearly between operating points, as in scikit-learn's implementation, systematically rewards models that assign few discrete scores when there are more negative than positive examples. We propose a step-wise interpolation strategy that addresses this problem and reflects the literature, and we include code to compute average precision scores using this interpolation.

Precision and recall
Average precision
Interpolated average precision
Proposal: step-wise interpolation
The intuition behind step-wise interpolation
Scikit-learn
Hammering home the point
Code
Can you write Python code for it?

from average_precision_post_code import *


I am working on the file "BlogPosts/Average_precision/Average_precision_post.ipynb".
The first task is:
This suggests that there's a way to get a precision ≈ 0.8 with a recall of 0.2. But consider how we usually think about interpolation: it's a process that assumes we simply haven't made enough observations to fill the gaps, and so we predict where a point would appear if we observed something at that x-value. Interpolation should reflect what a scatter plot would look like in the limit of infinite data. But this isn't the case here: no matter how much data we have, we'll still only have two possible precision values: 1.0 and 0.1.
So if we can't just draw a straight line, what should the graph look like such that we can compute a meaningful area?
Proposal: step-wise interpolation
Let's think about the precision we can actually achieve if we demand a recall of 0.2. We can't choose a threshold that gives a recall of exactly 0.2, so we should instead ask "What precision can I achieve if I recall at least 20% of the true positives?" With this do-nothing classifier the answer now is easy: 0.1, which we achieve when we assign every point to B+. This reflects the definition used in, for example, Manning et al's Information Retrieval.
What would this look like?
Can you write Python code for it?

plot_recall_precision_from_predictions(
    obs, constant_preds,
    interpolation='step',
    title='Step-wise interpolation')


I am working on the file "BlogPosts/Average_precision/further_testing.ipynb".
The first task is:
Area calculation tests
Can you write Python code for it?

measure_area([1,2,3,4], [5,3,1,7],
             interpolation='step',
             interpolation_direction='right')

measure_area([1,2,3,4], [5,3,1,7],
             interpolation='step',
             interpolation_direction='left')

measure_area([1,5,6,10], [2, 3.5, 4, 5],
             interpolation='step',
             interpolation_direction='left')

measure_area([1,5,6,10], [2, 3.5, 4, 5],
             interpolation='step',
             interpolation_direction='right')

measure_area([1,5,6,10], [2, 3.5, 4, 5],
             interpolation='linear')

measure_area([1,2,3,4], [5,3,1,4],
             interpolation='linear')


I am working on the file "BlogPosts/Average_precision/further_testing.ipynb".
The first task is:
Pretty good classifier
Can you write Python code for it?

y_close = [np.random.normal(loc=i, scale=len(y_true)/3) for i in range(len(y_true), 0, -1)]
compare_interpolations_from_scores(y_true, y_close)


I am working on the file "BlogPosts/Average_precision/further_testing.ipynb".
The first task is:
Random score
Can you write Python code for it?

y_rand = np.random.random(len(y_true))
compare_interpolations_from_scores(y_true, y_rand)


I am working on the file "BlogPosts/Average_precision/further_testing.ipynb".
The first task is:
Rounded versions of random score
Can you write Python code for it?

y_round = np.round(y_rand, 1)
compare_interpolations_from_scores(y_true, y_round)


I am working on the file "BlogPosts/Average_precision/further_testing.ipynb".
The first task is:
Test of the warnings
Can you write Python code for it?

rap.average_precision_score(y_true, y_const)

rap.average_precision_score(y_true, y_const, interpolation='step')

rap.average_precision_score(y_true, y_rand)

rap.average_precision_score(y_true, y_rand, interpolation='step')


I am working on the file "BlogPosts/Average_precision/further_testing.ipynb".
The first task is:
Test of the warnings
Testing
Area calculation tests
Testing in the context of modelling
Can you write Python code for it?

import matplotlib.pyplot as plt
import matplotlib
%matplotlib inline
import numpy as np
from sklearn.metrics.base import _average_binary_score
from sklearn.metrics import precision_recall_curve
import warnings

import roam_average_precision as rap

matplotlib.style.use('../../src/roam.mplstyle')

y_true = [1,1,0,0,0,0,0,0,0]
y_const = 0.5 * np.ones(len(y_true))
y_rand = np.random.random(len(y_true))


I am working on the file "BlogPosts/Average_precision/further_testing.ipynb".
The first task is:
Testing
The measure_area helper function so you can easily visualize arbitrary examples to try to find problems with the auc implementation. The caption tells you the area calculated using the same interpolation as in the figure.
Can you write Python code for it?

def measure_area(x, y, **kwargs):
    fig, ax = plt.subplots(1,1, figsize=(7,4))
    plot_area(ax, x, y, **kwargs)
    plt.show()

def plot_area(ax, x, y,
              reorder=False,
              interpolation=None,
              interpolation_direction='right'):
    direction = 1
    if reorder:
        # reorder the data points according to the x axis and using y to
        # break ties
        order = np.lexsort((y, x))
        x, y = x[order], y[order]
    else:
        dx = np.diff(x)
        if np.any(dx < 0):
            if np.all(dx <= 0):
                direction = -1
            else:
                raise ValueError("Reordering is not turned on, and "
                                 "the x array is not increasing: %s" % x)
    ax.scatter(x, y, marker='o', linewidths=0, s=25)
    if interpolation == 'linear':
        x_long = x
        y_long = y

    elif interpolation == 'step':
        if direction == -1:
            x, y = list(reversed(x)), list(reversed(y))

        if interpolation_direction == 'right':
            y_long = [v for v in y for _ in (0, 1)][1:]
            x_long = [v for v in x for _ in (0, 1)][:-1]
        elif interpolation_direction == 'left':
            y_long = [v for v in y for _ in (0, 1)][:-1]
            x_long = [v for v in x for _ in (0, 1)][1:]
        else:
            raise ValueError
    if max(x) < 1.1:
        ax.set_xticks(np.arange(-1, max(x)+1, 0.1))
    else:
        ax.set_xticks(np.arange(-1, max(x)+1, 1.0))
    if max(y) < 1.1:
        ax.set_yticks(np.arange(-1, max(y)+1, 0.1))
    else:
        ax.set_yticks(np.arange(-1, max(y)+1, 1.0))

    ax.plot(x_long, y_long)
    area = rap.auc(x, y, interpolation=interpolation,
               interpolation_direction=interpolation_direction)
    ax.fill_between(x_long, 0, y_long, alpha=0.2,
                    label='Area = {:5.4f}'.format(area))
    ax.legend(loc='upper center', bbox_to_anchor=(0.5, 1.2),
          ncol=3, fancybox=True, shadow=True)


I am working on the file "BlogPosts/Categorical_variables_in_tree_models/categorical_variables_post.ipynb".
The first task is:
Artificial dataset
For our artificial data experiments, we define a categorical variable $c$ which takes values from $C^+$ or $C^-$ and a normally-distributed continuous variable $z \sim N(10, 3^2)$, and specify that
$$
y =
\begin{cases}
    1, & \text{if } c\in C^+ \text{ or } z>10 \
    0, & \text{otherwise }
\end{cases}
$$
To make this more challenging, we'll create some further continuous variables $x_i$ that have varying correlations with $y$.
Let's generate a dataset with $|C^+| = |C^-| = 100$, $100$ additional weakly correlated features, and $10,000$ samples:
Can you write Python code for it?

data_categorical, data_onehot = generate_dataset(
    num_x=100, n_samples=10000, n_levels=200)


I am working on the file "BlogPosts/Categorical_variables_in_tree_models/categorical_variables_post.ipynb".
The first task is:
Could a dataset like this arise is practice? Well, perhaps knowing that somebody comes from some subset of states is enough to give good confidence of a positive outcome, but in the remaining states, we would need to consider the value of some second variable. This is the sort of basic relationship encoded in the artificial dataset.
Artificial data bake-off
We'll first focus our discussion on single decision trees to keep things simple, and then extend the results to random forests. We conduct two kinds of analysis:

A baseline model that doesn't include the categorical variable $c$.

A model that includes $c$.

This allows us to intuitively quantify the value of $c$ for the prediction problem. For each experiment, we'll train and evaluate a tree 10 times and average the results.
Scikit-learn's DecisionTreeClassifier
Scikit-learn can process only the one-hot-encoded version. Here's the baseline evaluation without $c$:
Can you write Python code for it?

results_no_c = evaluate_sklearn_model(
    data_onehot,
    feature_names=get_feature_names(data_onehot, include_c=False),
    target_col='y',
    model=DecisionTreeClassifier())

print_auc_mean_std(results_no_c)


I am working on the file "BlogPosts/Categorical_variables_in_tree_models/categorical_variables_post.ipynb".
The first task is:
Finally, let's see what happens if we use H2O with the one-hot encoded data:
Can you write Python code for it?

h2o_results_with_c_onehot = evaluate_h2o_model(
    data_onehot,
    feature_names=get_feature_names(data_onehot, include_c=True),
    target_col='y',
    model=H2ODecisionTree())

print_auc_mean_std(h2o_results_with_c_onehot)


I am working on the file "BlogPosts/Categorical_variables_in_tree_models/categorical_variables_post.ipynb".
The first task is:
In fact, all of the $x$-based ones have higher importance than all of the $c$-based ones.
H2O's decision trees
H2O doesn't have a basic decision tree, but rather only random forests. To facilitate a direct comparison with scikit-learn, we wrote a little wrapper class called H2ODecisionTree, which specifies a single-tree forest using all the available features, which is equivalent to a single decision tree. As before, we first evaluate the model without $c$:
Can you write Python code for it?

h2o_results_no_c = evaluate_h2o_model(
    data_categorical,
    feature_names=get_feature_names(data_categorical, include_c=False),
    target_col='y',
    model=H2ODecisionTree())

print_auc_mean_std(h2o_results_no_c)


I am working on the file "BlogPosts/Categorical_variables_in_tree_models/categorical_variables_post.ipynb".
The first task is:
In stark contrast to the scikit-learn models, the variable $c$ has the largest feature importance, just as our data-generation procedure leads us to expect:
Can you write Python code for it?


I am working on the file "BlogPosts/Categorical_variables_in_tree_models/categorical_variables_post.ipynb".
The first task is:
Let's also test a scikit-learn LogisticRegressionCV classifier, to compare a linear classifier with the tree-based ones:
Can you write Python code for it?

sklearn_regression_results = evaluate_sklearn_model(
    data_onehot,
    feature_names=get_feature_names(data_onehot, include_c=True),
    target_col='y',
    model=LogisticRegressionCV())

print_auc_mean_std(sklearn_regression_results)


I am working on the file "BlogPosts/Categorical_variables_in_tree_models/categorical_variables_post.ipynb".
The first task is:
On the left, the $x_{i}$ variables have no contribution to make. Their noisy quality is not disruptive. On the right, however, $x_{i}$  was chosen too early. Individually, these features can be highly informative, but they do not guarantee purity, so choosing them too early can result in branches that remain impure.
Here's the top of the one-hot-encoded version; you can see all the new  variables derived from $c$ tacked on as the final set of sparse columns.
Can you write Python code for it?


I am working on the file "BlogPosts/Categorical_variables_in_tree_models/categorical_variables_post.ipynb".
The first task is:
TL;DR Decision tree models can handle categorical variables without one-hot encoding them. However, popular implementations of decision trees (and random forests) differ as to whether they honor this fact. We show that one-hot encoding can seriously degrade tree-model performance. Our primary comparison is between H2O (which honors categorical variables) and scikit-learn (which requires them to be one-hot encoded).
Unnecessary one-hot encoding
Many real-world datasets include a mix of continuous and categorical variables. The defining property of the latter is that they do not permit a total ordering. A major advantage of decision tree models and their ensemble counterparts, random forests, is that they are able to operate on both continuous and categorical variables directly. In contrast, most other popular models (e.g., generalized linear models, neural networks) must instead transform categorical variables into some numerical analog, usually by one-hot encoding them to create a new dummy variable for each level of the original variable:
$$
    \begin{array}{c}
    \hline
    \textbf{Speciality} \
    \hline
    \textrm{Cardiology} \
    \textrm{Neurology} \
    \textrm{Neurology} \
    \textrm{Cardiology} \
    \textrm{Gerontology} \
    \hline
    \end{array}
    \Longrightarrow
    \begin{array}{c c c}
    \hline
    \textbf{Speciality:Cardiology} & \textbf{Specialty:Neurology} & \textbf{Specialty:Gerontology} \
    \hline
    1 & 0 & 0 \
    0 & 1 & 0 \
    0 & 1 & 0 \
    1 & 0 & 0 \
    0 & 0 & 1 \
    \hline
    \end{array}
$$
One-hot encoding can lead to a huge increase in the dimensionality of the feature representations. For example, one-hot encoding U.S. states adds 49 dimensions to the intuitive feature representation. In addition, one-hot encoding erases important structure in the underlying representation by splitting a single feature into many separate ones. (The naming convention used above, and by many software packages, can be misleading: the three features on the right are completely...
Can you write Python code for it?

from tree_categorical_variables import *


I am working on the file "BlogPosts/Categorical_variables_in_tree_models/categorical_variables_post.ipynb".
The first task is:
The extra feature led to a modest improvement in AUC, although nowhere near the perfect performance that we were expecting. In addition, none of the $c$-based variables are among the top features as ranked by Gini feature importance:
Can you write Python code for it?


I am working on the file "BlogPosts/Categorical_variables_in_tree_models/categorical_variables_post.ipynb".
The first task is:
This produces categorical and one-hot-encoded versions of the dataset. Here's a look at the categorical version:
Can you write Python code for it?


I am working on the file "BlogPosts/Categorical_variables_in_tree_models/categorical_variables_post.ipynb".
The first task is:
Without $c$, the performance is very similar to scikit-learn's. But when H2O has access to $c$, it achieves an almost-perfect AUC:
Can you write Python code for it?

h2o_results_with_c = evaluate_h2o_model(
    data_categorical,
    feature_names=get_feature_names(data_categorical, include_c=True),
    target_col='y',
    model=H2ODecisionTree())

print_auc_mean_std(h2o_results_with_c)


I am working on the file "BlogPosts/Categorical_variables_in_tree_models/categorical_variables_post.ipynb".
The first task is:
You'll notice that $y=1$ whenever either $c$ starts with A or $z>10$. A simple decision tree should be able to perfectly predict the outcome variable by splitting first on $c$ and then on $z$, as illustrated on the left in the following diagram:
Can you write Python code for it?

from IPython.display import SVG, display

display(SVG("fig/Decision tree visualization.svg"))


I am working on the file "BlogPosts/Decorator_mixing/decorator_mixing.ipynb".
The first task is:
Example of using both: Providing a process_time
I write a lot of data pipelines, and it can be useful to have a process be consistent about all the times it uses (e.g., to set a creation_time field for a record). A @cachedproperty is a nice way to ensure we always get the same time back!
Can you write Python code for it?

from datetime import datetime
from typing import Iterator

class ProcessTimeProvider(ABC):
    """Abstract interface for providing times for data pipelines + other processes"""

    @cachedproperty
    @abstractmethod
    def process_time(self) -> datetime:
        pass

class LocalProcessTimeProvider(ProcessTimeProvider):

    @cachedproperty
    def process_time(self) -> datetime:
        return datetime.now()

class UTCProcessTimeProvider(ProcessTimeProvider):

    @cachedproperty
    def process_time(self) -> datetime:
        return datetime.utcnow()


I am working on the file "BlogPosts/Decorator_mixing/decorator_mixing.ipynb".
The first task is:
For the sake of completeness, let's see an example of how these might work. The code below counts the # of times the 🦊 character appears in a text.
Can you write Python code for it?

class FoxMention:

    def __init__(self, offset: int, creation_time: datetime):
        self.offset = offset
        self.creation_time = creation_time

    def __repr__(self):
        return '<FoxMention(offset={}, creation_time={})>'.format(self.offset, self.creation_time)

class FoxExtractionProcess(UTCProcessTimeProvider):
    """Counts 🦊s!"""

    def extract_foxes(self, text) -> Iterator[FoxMention]:
        for offset, character in enumerate(text):
            if character == '🦊':
                yield FoxMention(
                    offset=offset,
                    creation_time=self.process_time
                )

fox_extractor = FoxExtractionProcess()
text = 'The quick brown 🦊 jumps over the lazy 🦊'
for fox_mention in fox_extractor.extract_foxes(text):
    print(fox_mention)


I am working on the file "BlogPosts/Decorator_mixing/decorator_mixing.ipynb".
The first task is:
It looks like the cachedproperty decorator has changed the top-level process_time into a cachedproperty object, which contains a func attribute that is the original process_time object! Is that where __isabstractmethod__ could be hiding?
Can you write Python code for it?


I am working on the file "BlogPosts/Decorator_mixing/decorator_mixing.ipynb".
The first task is:
Overview
Background
Intro to @abstractmethod
Intro to @cachedproperty
Example of using both: Providing a process time

The Problem
Going deeper: How @abstractmethod works
Looking closer at @cachedproperty

A Fix!
A More Generic Solution

Overview
Recently I ran into an issue where two of my favorite Python decorators, @abstractmethod and @cachedproperty, weren't playing well together. This post covers that specific issue, as well as more general ways to avoid this problem when writing your own decorators.
Background
Intro to @abstractmethod
The @abstractmethod decorator from Python 3's built-in abc module is a nice way to enforce proper implementation of an interface. It will raise a TypeError when you call __init__ for a class that has unimplemented abstract methods.
Can you write Python code for it?

from abc import abstractmethod, ABC

class AbstractPouncer(ABC):

    @abstractmethod
    def pounce(self):
        pass

class Fox(AbstractPouncer):

    def pounce(self):
        self.crouch()
        self.leap()
        self.attack()

    def crouch(self):
        print('Crouch crouch crouch...')

    def leap(self):
        print('Wheeee!')

    def attack(self):
        print('I GOTCHU 🦊')

try:
    AbstractPouncer()
except TypeError as e:
    print('TypeError:', e)

fox = Fox()
fox.pounce()


I am working on the file "BlogPosts/Decorator_mixing/decorator_mixing.ipynb".
The first task is:
So if __isabstractmethod__ isn't in ProcessTimeProvider.process_time, what happened to it? Let's inspect the process_time method a little more.
Can you write Python code for it?


I am working on the file "BlogPosts/Decorator_mixing/decorator_mixing.ipynb".
The first task is:
Voilà! Now we can declare abstract cached properties with all the benefits of Python's abc module. As noted above, this is now fixed in the newest version of boltons.
A More Generic Solution
This problem likely affects all sorts of of decorators, not just cachedproperty, as well as other attributes beyond __isabstractmethod__. Luckily, Python's functools actually has a built-in solution, @wraps, for helping wrapper functions maintain important attributes of the original function:
Can you write Python code for it?

from functools import wraps

def bad_decorator(func):
    def wrapper(num):
        return func(num)

    return wrapper

def good_decorator(func):
    @wraps(func)
    def wrapper(num):
        return func(num)

    return wrapper

class BadTwoAdder(ABC):
    @bad_decorator
    @abstractmethod
    def bad_add_two(num):
        """Add two to a number"""
        return 2 + num

class GoodTwoAdder(ABC):
    @good_decorator
    @abstractmethod
    def good_add_two(num):
        """Add two to a number"""
        return 2 + num


I am working on the file "BlogPosts/Decorator_mixing/decorator_mixing.ipynb".
The first task is:
Wait – isn't ProcessTimeProvider an abstract class that hasn't had all its methods implemented? As we learned above, this should throw a TypeError when we try to instantiate the abstract class. This was baffling to me at first, and I had to understand a little more about how Python decorators and abstract methods are implemented to fix it.
Going deeper: How @abstractmethod works
The current implementation of @abstractmethod is surprisingly simple:
def abstractmethod(funcobj):
    funcobj.__isabstractmethod__ = True
    return funcobj
It takes the function it is decorating, and sets an attribute (yes, functions can have attributes!) called __isabstractmethod__ to True. All right, let's check for that:
Can you write Python code for it?

print(getattr(AbstractPouncer.pounce, '__isabstractmethod__', None))
print(getattr(ProcessTimeProvider.process_time, '__isabstractmethod__', None))


I am working on the file "BlogPosts/Decorator_mixing/decorator_mixing.ipynb".
The first task is:
We're onto something! In AbstractPouncer.pounce, which didn't have a @cachedproperty annotation, we see __isabstractmethod__ is set, as expected. So what's happening in ProcessTimeProvider.process_time? It must be related to @cachedproperty. Let's look at the implementation again.
Looking closer at @cachedproperty
Can you write Python code for it?

class cachedproperty:

    def __init__(self, func):
        self.__doc__ = getattr(func, '__doc__')
        self.func = func

    def __get__(self, obj, objtype=None):
        if obj is None:
            return self
        value = obj.__dict__[self.func.__name__] = self.func(obj)
        return value

    def __repr__(self):
        cn = self.__class__.__name__
        return '<%s func=%s>' % (cn, self.func)


I am working on the file "BlogPosts/Hyperparameter_tuning_comparison/Hyperparameter_tuning_comparison_post.ipynb".
The first task is:
And here's an example assessment:
Can you write Python code for it?

X_small, y_small = small_dataset

sklearn_results = assess(
    X_small,
    y_small,
    skopt_gp_search,
    LogisticRegression,
    param_grid=maxent_skopt_grid,
    xval_indices=get_cross_validation_indices(X_small, y_small),
    dataset_name='small')

prepare_summary(sklearn_results)


I am working on the file "BlogPosts/Hyperparameter_tuning_comparison/Hyperparameter_tuning_comparison_post.ipynb".
The first task is:
For experiments of this size, grid search is basically infeasible given our rigorous evaluation set-up, so we don't do it. Furthermore, sampling half of the settings from that grid is likely to be too costly. Thus, we again specify that just 100 settings will be sampled, for a run that is more comparable to what we're allowing Hyperopt and scikit-optimize:
Can you write Python code for it?

partial_experimental_run = [
    (random_search, param_grid, {'sampsize': 100}),
    (hyperopt_search, hyperopt_grid, {'max_evals': 100}),
    (skopt_gp_search, skopt_grid,  {'n_calls': 100}),
    (skopt_forest_search, skopt_grid, {'n_calls': 100}),
    (skopt_gbrt_search, skopt_grid, {'n_calls': 100})]

large_results = run_experiments(
    partial_experimental_run, large_dataset)

prepare_summaries(large_results).round(3)


I am working on the file "BlogPosts/Hyperparameter_tuning_comparison/Hyperparameter_tuning_comparison_post.ipynb".
The first task is:
Here, all the models perform about the same, but Hyperopt and scikit-optimize show their value in the time it takes to achieve these results: they are about five times faster than random_search and ten times faster than grid_search.
Random search with fewer samples
The above results are unrealistic in the sense that they assume we have sufficient resources and time to do full grid search and take very large samples from the grid for random search. In practice (as we move to having tens or hundreds of thousands of examples), we're unlikely to have these luxuries. It's thus worth seeing what happens if we place random search more on a par with the other methods by taking just 100 samples from the grid, using the same dataset of 1000 examples and 100 features. Here's what we find:
Can you write Python code for it?

limited_random_search_run = [
    (random_search, param_grid, {'sampsize': 100})]

limited_random_search_results = run_experiments(
    limited_random_search_run, small_dataset)

prepare_summaries(limited_random_search_results).round(3)


I am working on the file "BlogPosts/Hyperparameter_tuning_comparison/Hyperparameter_tuning_comparison_post.ipynb".
The first task is:
Search is then a process of sampling from these spaces.
Finally, scikit-optimize asks you to specify just the upper and lower bounds of the space to be searched:
Can you write Python code for it?

skopt_grid = {
    'max_depth': (4, 12),
    'learning_rate': (0.01, 0.5),
    'n_estimators': (20, 200),
    'objective' : Categorical(('multi:softprob',)),
    'gamma': (0, 0.5),
    'min_child_weight': (1, 5),
    'subsample': (0.1, 1),
    'colsample_bytree': (0.1, 1)}


I am working on the file "BlogPosts/Hyperparameter_tuning_comparison/Hyperparameter_tuning_comparison_post.ipynb".
The first task is:
The requirements for this code are a bit finicky, so see the README.md for installation and usage notes.
Baseline methods
We consider two common baseline approaches to finding the best hyperparameters:

Grid search: an exhaustive search of every combination of every setting of the hyperparameters. This method is guaranteed to find the best settings in the (discrete version of the) search space, but it is simply not tractable for large parameter spaces.

Randomized search: this method samples from the full grid. It doesn't offer the same guarantees as grid search, but it has been shown to be extremely effective in practice.

Packages
We assess two packages that support more sophisticated approaches. Both of them rely on sequential model-based optimization: at the start, they sample a small number of points from the full hyperparameter space and run evaluations using them. They then alternate between training regression models on the resulting (setting, loss) pairs and using those models to identify effective settings to add to the regression model data.

Hyperopt: this package explores the hyperparameter space strategically using a tree of Parzen estimators, a Bayesian approach.

Scikit-optimize: this new package has three algorithms – forest_minimize (decision-tree regression search), gbrt_minimize (gradient-boosted-tree regression search), and gp_minimize (Gaussian process regression search).

There are two other prominent Python packages that we have not included in our evaluation: Spearmint and MOE. The version of Spearmint under active development has a license that does not permit commercial work, and MOE has grown into the start-up SigOpt.
Data
For this post, we rely on scikit-learn's (amazingly flexible) make_classification method to create artificial datasets of various sizes. Our function artificial_dataset is a wrapper that permits fiddling with just a few of its many knobs. For instance, this creates a small dataset:
Can you write Python code for it?

_ = artificial_dataset(
    n_samples=100, n_features=10, n_classes=3)


I am working on the file "BlogPosts/Hyperparameter_tuning_comparison/Hyperparameter_tuning_comparison_post.ipynb".
The first task is:
The results for accuracy are kind of a wash, as the following plot shows; the confidence interval bars overlap a lot, creating a messy display, but that actually helps to convey that the numerical differences are not especially reliable.
Can you write Python code for it?

plot_representation_size_accuracy(
    representation_size_results, include_cis=True)


I am working on the file "BlogPosts/Hyperparameter_tuning_comparison/Hyperparameter_tuning_comparison_post.ipynb".
The first task is:
This is a tuple (X, y) where X is the matrix of feature representations and y is the target.
Hyperparameter space
In this post, we use the XGBoost XGBClassifier as our base model. This a good choice for the post because it has a lot of hyperparameters to tune, but our main motivation is to keep the discussion manageable. As we noted above, the associated code supports experiments with many others, including all the scikit-learn models. (See below for an example.)
For all the approaches we study, the basis for search is a dictionary mapping parameter names to values. However, the values need to be specified differently in each case. We've set them up so that they are as aligned as possible in terms of the space of values they define.
For the two baseline methods, grid search and randomized search, we specify the full list of values to be explored for each hyperparameter:
Can you write Python code for it?


I am working on the file "BlogPosts/Hyperparameter_tuning_comparison/Hyperparameter_tuning_comparison_post.ipynb".
The first task is:
This one-line table can be thought of as an extension of the above. Here, we see clearly that the strong showing of random search (and grid search) was due to their brute-force approach. Sampling 1,093 settings is much different from sampling just a 100 when it comes to happening by chance upon a good model! In sum, Hyperopt and Scikit-optimize earn their keep not only in terms of speed but also in terms of expected performance.
A larger dataset
The following defines a dataset with 10K examples and just 20 features, which might correspond to a situation in which one has a small number of highly informative predictors derived from structured information:
Can you write Python code for it?

large_dataset = artificial_dataset(
    n_samples=10000, n_features=20, n_classes=3)


I am working on the file "BlogPosts/Modern_TensorFlow/modern-tensorflow.ipynb".
The first task is:
A word about TensorFlow model serving
Exporting an inference graph and the serving signature is "simple" (though the serving_fn interface could be improved). The cool thing is that once you have your tf.estimator and your serving_input_fn, you can just use tensorflow_serving and get a RESTful API serving your model!
Serving interface
Can you write Python code for it?

def serving_input_fn():
    tokens = tf.placeholder(
        dtype=tf.int32, shape=[None, None], name="tokens")
    sequence_length = tf.size(tokens)
    features = {'tokens': tokens, 'sequence_length': sequence_length}
    return tf.estimator.export.ServingInputReceiver(
        features=features, receiver_tensors=tokens)

estimator.export_savedmodel('export', serving_input_fn)


I am working on the file "BlogPosts/Modern_TensorFlow/modern-tensorflow.ipynb".
The first task is:
Batch
Can you write Python code for it?

dataset = dataset.batch(batch_size=3)

for el in dataset:
    print(el)


I am working on the file "BlogPosts/Modern_TensorFlow/modern-tensorflow.ipynb".
The first task is:
Contents
See the README.md for an overview of this notebook and its goals.

Eager execution
tf.data: feeding data into the graph
Placeholders (before)
Dataset from np.array
Dataset from text file
Dataset from custom generator
tf.data: Dataset Transforms
Shuffle
Repeat
Map
Batch
Padded batch
NLP: preprocessing in Tensorflow
Tokenizing by white space in TensorFlow
Lookup token index from vocab file in TensorFlow
Full Example
Task and Data
Graph (test with eager execution)
Model (tf.estimator)
Before: custom model classes
Now:  tf.estimator
input_fn
model_fn
Instantiate and train your Estimator
TensorBoard, train_and_evaluate, predict etc.

A word about TensorFlow model serving
Serving interface
Docker Image
Pull existing image
Run
Rest API POST with curl

Setup
Can you write Python code for it?

from distutils.version import LooseVersion
import sys

if LooseVersion(sys.version) < LooseVersion('3.4'):
    raise Exception('You need python>=3.4, but you have {}'.format(sys.version))

# Standard
from pathlib import Path

# External
import numpy as np
import tensorflow as tf

if LooseVersion(tf.__version__) < LooseVersion('1.9'):
    raise Exception('You need tensorflow>=1.9, but you have {}'.format(tf.__version__))


I am working on the file "BlogPosts/Modern_TensorFlow/modern-tensorflow.ipynb".
The first task is:
Dataset from custom generator
The best of both worlds, perfect for NLP
It will allow you do put all your logic in pure python, in your generator_fn, before feeding it to the Graph.
Can you write Python code for it?

def generator_fn():
    for _ in range(2):
        yield b'Hello world'

dataset = (tf.data.Dataset.from_generator(
    generator_fn,
    output_types=(tf.string),  # Define type and shape of your generator_fn output
    output_shapes=()))         # like you would have for your `placeholders`

for el in dataset:
    print(el)


I am working on the file "BlogPosts/Modern_TensorFlow/modern-tensorflow.ipynb".
The first task is:
Full Example
Task and Data
The tokens_generator returns list of ids. We map even/odd numbers to 2 different ids.
The labels_generator returns list of label ids. We want to predict if a token is
- a word (label 0)
- an odd number (label 1)
- an even number (label 2)
Can you write Python code for it?

# We tokenize by white space and assign these ids
tok_to_idx = {'hello': 0, 'world': 1, '<odd>': 2, '<even>': 3}

def tokens_generator():
    with Path(path).open() as f:
        for line in f:
            # Tokenize by white space
            tokens = line.strip().split()
            token_ids = []
            for tok in tokens:
                # Look for digits
                if tok.isdigit():
                    if int(tok) % 2 == 0:
                        tok = '<even>'
                    else:
                        tok = '<odd>'
                token_ids.append(tok_to_idx.get(tok.lower(), len(tok_to_idx)))
            yield (token_ids, len(token_ids))

def get_label(token_id):
    if token_id == 2:
        return 1
    elif token_id == 3:
        return 2
    else:
        return 0

def labels_generator():
    for token_ids, _ in tokens_generator():
        yield [get_label(tok_id) for tok_id in token_ids]

dataset = tf.data.Dataset.from_generator(
    tokens_generator,
    output_types=(tf.int32, tf.int32),
    output_shapes=([None], ()))
for el in dataset:
    print(el)


I am working on the file "BlogPosts/Modern_TensorFlow/modern-tensorflow.ipynb".
The first task is:
Graph (test with eager execution)
Let's build a model that predicts the classes 0, 1 and 2 above.
Test our graph logic here, with eager_execution activated.
Can you write Python code for it?

batch_size = 4
vocab_size = 4
dim = 100

shapes = ([None], ())
defaults = (0, 0)
# The last sentence is longer: need padding
dataset = dataset.padded_batch(
    batch_size, shapes, defaults)

# Define all variables (In eager execution mode, have to be done just once)
# Otherwise you would create new variable at each loop iteration!
embeddings = tf.get_variable('embeddings', shape=[vocab_size, dim])
lstm_cell = tf.contrib.rnn.LSTMCell(100)
dense_layer = tf.layers.Dense(3, activation=tf.nn.relu)

for tokens, sequence_length in dataset:
    token_embeddings = tf.nn.embedding_lookup(embeddings, tokens)
    lstm_output, _ = tf.nn.dynamic_rnn(
        lstm_cell, token_embeddings, dtype=tf.float32, sequence_length=sequence_length)
    logits = dense_layer(lstm_output)
    print(logits.shape)


I am working on the file "BlogPosts/Modern_TensorFlow/modern-tensorflow.ipynb".
The first task is:
Here, X, h, y are nodes of the computational graph. But you can actually get the value of these nodes!
In the past you would have written
python
X = tf.placeholder(dtype=tf.float32, shape=[2, 4])
h = tf.layers.dense(X, 2, activation=tf.nn.relu)
y = tf.nn.softmax(h)
with tf.Session() as sess:
    sess.run(tf.global_variables_initializer())
    sess.run(y, feed_dict={X: np.random.normal(size=[2, 4])})
tf.data: feeding data into the graph
tf.placeholders is replaced by tf.data.Dataset.
Placeholders (before)
python
x = tf.placeholder(dtype=tf.int32, shape=[None, 5])
with tf.Session() as sess:
    x_eval = sess.run(x, feed_dict={x: x_np})
    print(x_eval)
Dataset from np.array
Below is a simple example where we have a np.array, one row = one example.
Can you write Python code for it?

x_np = np.array([[i]*5 for i in range(10)])
x_np


I am working on the file "BlogPosts/Modern_TensorFlow/modern-tensorflow.ipynb".
The first task is:
Lookup token index from vocab file in TensorFlow
You're probably used to performing the lookup token -> token_idx outside TensorFlow. However, tf.contrib.lookup provides exactly this functionality. It's fast, and when exporting the model for serving, it will consider your vocab.txt file as a model's resource and keep it with the model!
Can you write Python code for it?

# One lexeme per line
path_vocab = 'vocab.txt'
with Path(path_vocab).open() as f:
    for idx, line in enumerate(f):
        print(idx, ' -> ', line.strip())


I am working on the file "BlogPosts/Modern_TensorFlow/modern-tensorflow.ipynb".
The first task is:
Map
Note: while map is super handy when working with images (TensorFlow has a lot of image preprocessing functions and efficiency is crucial), it's not as practical for NLP, because you're now working with tensors. We found it easier to write the most of the preprocessing logic in python, in a generator_fn, before feeding it to the graph.
Can you write Python code for it?

dataset = dataset.map(
    lambda t: tf.string_split([t], delimiter=' ').values,
    num_parallel_calls=4)  # Multithreading

for el in dataset:
    print(el)


I am working on the file "BlogPosts/Modern_TensorFlow/modern-tensorflow.ipynb".
The first task is:
Notice that a 4 has been appended at the end of the first row.
And much more: prefetch, zip, concatenate, skip, take etc.
See the documentation.
Note: the recommended standard workflow is

shuffle
repeat (repeat after shuffle so that one epoch = all the examples)
map, using the num_parallel_calls argument to get multithreading for free.
batch or padded_batch
prefetch (will prefetch data on the GPU so that it doesn't suffer from any data starvation – and only use 80% of your expensive GPU).

NLP: preprocessing in Tensorflow
Tokenizing by white space in TensorFlow
This is an example of why using map is kind of annoying in NLP. It works, but it's not as easy as just using .split() or any other python code.
Can you write Python code for it?

def tf_tokenize(t):
    return tf.string_split([t], delimiter=' ').values

dataset = tf.data.TextLineDataset(['test.txt'])
dataset = dataset.map(tf_tokenize)
for el in dataset:
    print(el)


I am working on the file "BlogPosts/Modern_TensorFlow/modern-tensorflow.ipynb".
The first task is:
Padded batch
In NLP, we usually work with sentences of different length. When building your batch, we need to 'pad', i.e. add some fake elements at the end of the shorter sentences. You can perform this operation easily in TensorFlow.
Here is a dummy example:
Can you write Python code for it?

def generator_fn():
    yield [1, 2]
    yield [1, 2, 3]

dataset = tf.data.Dataset.from_generator(
    generator_fn,
    output_types=(tf.int32),
    output_shapes=([None]))

dataset = dataset.padded_batch(
    batch_size=2,
    padded_shapes=([None]),
    padding_values=(4))  # Optional, if not set will default to 0

for el in dataset:
    print(el)


I am working on the file "BlogPosts/Modern_TensorFlow/modern-tensorflow.ipynb".
The first task is:
Repeat
Repeat your dataset to perform multiple epochs!
Can you write Python code for it?

dataset = dataset.repeat(2)  # 2 epochs

for el in dataset:
    print(el)


I am working on the file "BlogPosts/Modern_TensorFlow/modern-tensorflow.ipynb".
The first task is:
TensorBoard, train_and_evaluate, predict etc.
Now, the estimator is trained, serialized to disk etc. You also have access to TensorBoard. (Lots of stuff for free, without having to write boilerplate code!)
To access tensorboard :
tensorboard --logdir model
Check evaluate, train_and_evaluate... documentation.
Example with early stopping, where we run evaluation every 2 minutes (120 seconds).
python
hook = tf.contrib.estimator.stop_if_no_increase_hook(
        estimator, 'accuracy', 500, min_steps=8000, run_every_secs=120)
train_spec = tf.estimator.TrainSpec(input_fn=input_fn, hooks=[hook])
eval_spec = tf.estimator.EvalSpec(input_fn=input_fn, throttle_secs=120)
tf.estimator.train_and_evaluate(estimator, train_spec, eval_spec)
Can you write Python code for it?

# Iterate over the 2 first elements of the (shuffled) dataset and yield predictions
# You need to write variants of your input_fn for eval / predict modes
for idx, predictions in enumerate(estimator.predict(input_fn)):
    print(predictions['preds'])
    if idx > 0:
        break


I am working on the file "BlogPosts/Modern_TensorFlow/modern-tensorflow.ipynb".
The first task is:
The following does just the same as above, but now el is a tf.Tensor of dtype=tf.string!
Can you write Python code for it?

dataset = tf.data.TextLineDataset([path])
for el in dataset:
    print(el)


I am working on the file "BlogPosts/Modern_TensorFlow/modern-tensorflow.ipynb".
The first task is:
To use it in TensorFlow:
Can you write Python code for it?

# The last idx (2) will be used for unknown words
lookup_table = tf.contrib.lookup.index_table_from_file(
    path_vocab, num_oov_buckets=1)

for el in dataset:
    print(lookup_table.lookup(el))


I am working on the file "BlogPosts/Modern_TensorFlow/modern-tensorflow.ipynb".
The first task is:
We create a Dataset from this array.
This dataset is a node of the graph.
Each time you query its value, it will move to the next row of the underlying np.array.
Can you write Python code for it?

dataset = tf.data.Dataset.from_tensor_slices(x_np)

for el in dataset:
    print(el)


I am working on the file "BlogPosts/Modern_TensorFlow/modern-tensorflow.ipynb".
The first task is:
input_fn
A callable that returns a dataset that yields tuples of features, labels
Can you write Python code for it?

def input_fn():
    # Create datasets for features and labels
    dataset_tokens = tf.data.Dataset.from_generator(
        tokens_generator,
        output_types=(tf.int32, tf.int32),
        output_shapes=([None], ()))
    dataset_output = tf.data.Dataset.from_generator(
        labels_generator,
        output_types=(tf.int32),
        output_shapes=([None]))

    # Zip features and labels in one Dataset
    dataset = tf.data.Dataset.zip((dataset_tokens, dataset_output))

    # Shuffle, repeat, batch and prefetch
    shapes = (([None], ()), [None])
    defaults = ((0, 0), 0)
    dataset = (dataset
               .shuffle(10)
               .repeat(100)
               .padded_batch(4, shapes, defaults)
               .prefetch(1))

    # Dataset yields tuple of features, labels
    return dataset