I am working on the file "Statistics_block2.ipynb".
The first task is:
Comparison between 3 priors
Can you write Python code for it?

Nobs = 1 #Change this parameter to see how it shapes the graph
nu_grid = np.linspace(Nobs-4*np.sqrt(Nobs), Nobs+4*np.sqrt(Nobs), 1000)

# Part II
gaussian_II = stat.norm(Nobs, np.sqrt(Nobs))

# Part III
N_III = 10**6
poisson_III = stat.poisson(Nobs)
mc_sample_III = poisson_III.rvs(N_III)

# Part IV lin
log_nu_posterior_lin = -nu_grid + Nobs*np.log(nu_grid) - np.log(msc.factorial(Nobs))
nu_posterior_lin = np.exp(log_nu_posterior_lin)

# Part V lin
N_V_lin=10**7
nu_prior_lin = np.random.random(N_V_lin)*(Nobs+10*np.sqrt(Nobs))
poisson_V_lin = stat.poisson(nu_prior_lin)
N_mc_lin = poisson_V_lin.rvs(N_V_lin) #We'll draw a sample from N_mc
mc_sample_V_lin = nu_prior_lin[N_mc_lin==Nobs]
print len(mc_sample_V_lin)

# Part V.E) lin
mu_VE_lin = np.mean(mc_sample_V_lin)
sigma_VE_lin = np.std(mc_sample_V_lin)
gaussian_VE_lin = stat.norm(mu_VE_lin, sigma_VE_lin)

# Part IV log
log_nu_posterior_log = -nu_grid + (Nobs-1)*np.log(nu_grid) - np.log(msc.factorial(Nobs-1))
nu_posterior_log = np.exp(log_nu_posterior_log)

# Part V log
N_V_log=5*10**7
nu_lim = Nobs+10*np.sqrt(Nobs)
# For Nobs=1 it is better to increase the value, which may also require to increase N_V_log
nu_lim = Nobs+10000*np.sqrt(Nobs)
r_prior_log = np.random.random(N_V_log)
nu_prior_log = np.exp(2*r_prior_log*np.log(nu_lim))/nu_lim
poisson_V_log = stat.poisson(nu_prior_log)
N_mc_log = poisson_V_log.rvs(N_V_log) #We'll draw a sample from N_mc
mc_sample_V_log = nu_prior_log[N_mc_log==Nobs]
print len(mc_sample_V_log)

# Part V.E) log
mu_VE_log = np.mean(mc_sample_V_log)
sigma_VE_log = np.std(mc_sample_V_log)
gaussian_VE_log = stat.norm(mu_VE_log, sigma_VE_log)

# Part IV - Jeffrey's prior (we're using the log so that P(nu) doesn't blow up)
log_nu_posterior_jef = -nu_grid + (Nobs-1./2.)*np.log(nu_grid) - np.log(spc.gamma(Nobs+1./2.))
nu_posterior_jef = np.exp(log_nu_posterior_jef)

# Part V jef
N_V_jef = 10**7
nu_lim = Nobs+10*np.sqrt(Nobs)
r_prior_jef = np.random.random(N_V_jef)
nu_prior_jef = 4*nu_lim**2*r_prior_jef**2
poisson_V_jef =...


I am working on the file "Statistics_block2.ipynb".
The first task is:
Con $\rho=0.999$, el plot muestra una correlación mucho más grande respecto x1 y x2 (es casi una linea recta en diagonal). Si $\rho=1$, x1 = x2.
Part II
The PDF of a multi-dimensional (2D) gaussian is:
$$f(x_{1},x_{2})=\frac{1}{2\pi\sqrt{\det V}}e^{-\frac{1}{2}\left(\begin{array}{cc}
x_{1}-\mu_{1} & x_{2}-\mu_{2}\end{array}\right)V^{-1}\left(\begin{array}{c}
x_{1}-\mu_{1}\
x_{2}-\mu_{2}
\end{array}\right)}=\frac{1}{2\pi\sigma_{1}\sigma_{2}\sqrt{1-\rho^{2}}}e^{-\frac{1}{2}\text{A}}$$
where
$$
A\equiv\left(\begin{array}{cc}
x_{1}-\mu_{1} & x_{2}-\mu_{2}\end{array}\right)V^{-1}\left(\begin{array}{c}
x_{1}-\mu_{1}\
x_{2}-\mu_{2}
\end{array}\right)
$$
and V is the covariance matrix:
$$
V=\left[\begin{array}{cc}
\sigma_{1}^{2} & \rho\sigma_{1}\sigma_{2}\
\rho\sigma_{1}\sigma_{2} & \sigma_{2}^{2}
\end{array}\right]
$$
The inverse of V is:
$$
V^{-1}=\frac{1}{\det V}\left[\begin{array}{cc}
\sigma_{2}^{2} & -\rho\sigma_{1}\sigma_{2}\
-\rho\sigma_{1}\sigma_{2} & \sigma_{1}^{2}
\end{array}\right]
$$
The determinant of V is:
$$
\det V=\sigma_{1}^{2}\sigma_{2}^{2}-\rho^{2}\sigma_{1}^{2}\sigma_{2}^{2}=\sigma_{1}^{2}\sigma_{2}^{2}\left(1-\rho^{2}\right)
$$
Calculate A:
$$A=\left(\begin{array}{cc}
x_{1}-\mu_{1} & x_{2}-\mu_{2}\end{array}\right)V^{-1}\left(\begin{array}{c}
x_{1}-\mu_{1}\
x_{2}-\mu_{2}
\end{array}\right)=
 \frac{1}{\det V}\left(\begin{array}{cc}
x_{1}-\mu_{1} & x_{2}-\mu_{2}\end{array}\right)\left(\begin{array}{c}
\sigma_{2}^{2}\left(x_{1}-\mu_{1}\right)-\rho\sigma_{1}\sigma_{2}\left(x_{2}-\mu_{2}\right)\
\sigma_{1}^{2}\left(x_{2}-\mu_{2}\right)-\rho\sigma_{1}\sigma_{2}\left(x_{1}-\mu_{1}\right)
\end{array}\right)\
 =\frac{1}{\det V}\left[\sigma_{2}^{2}\left(x_{1}-\mu_{1}\right)^{2}-\rho\sigma_{1}\sigma_{2}\left(x_{1}-\mu_{1}\right)\left(x_{2}-\mu_{2}\right)+\sigma_{1}^{2}\left(x_{2}-\mu_{2}\right)^{2}-\rho\sigma_{1}\sigma_{2}\left(x_{1}-\mu_{1}\right)\left(x_{2}-\mu_{2}\right)\right]\
 =\frac{1}{\det V}\left{...
Can you write Python code for it?

def try_mu(muold,step):
    return np.random.normal(loc=muold,scale=step)

def g_ratio(munew,muold,D,rhoD2,det,s1,s2,rho):
    gnew = (s1*(D[:,0]-munew[0]))**2+(s2*(D[:,1]-munew[1]))**2-rhoD2 #rhoD2=2*rho*s1*s2*x1*x2
    gold = (s1*(D[:,0]-muold[0]))**2+(s2*(D[:,1]-muold[1]))**2-rhoD2
    return np.exp(-1./(2*det)*sum(gnew-gold))

N3_III = 10**6
step = 0.5 # this is an initial guess
mus = np.ones((2,N3_III+1)) #the 1, 1 values are an initial guess

s1 = 1
s2 = 2
rho = 0.999
rhoD2 = 2*rho*s1*s2*D[:,0]*D[:,1]

det = (s1*s2*np.sqrt(1-rho**2))**2

for i in xrange(N3_III):
    if i%5000==0:
        print i
    mutry = try_mu(mus[:,i],step)
    #print mutry
    A = min(1, g_ratio(mutry,mus[:,i],D,rhoD2,det,s1,s2,rho))
    #print A
    eta = np.random.random() #Generate a number from the uniform distribution
    if A>eta:
        mus[:,i+1]=mutry
    else:
        mus[:,i+1]=mus[:,i]
print mus

plt.plot(mus[0,5000:],mus[1,5000:],'.')
print np.mean(mus[0,5000:]),np.mean(mus[1,5000:])
print mus[:,-1]


I am working on the file "Statistics_block2.ipynb".
The first task is:
Part V
Let's now construc the Bayesian posterior pdf for $\nu$ using a MC simulation:
 1. Generate $10^7$ values for $\nu$ drawn from a uniform distribution between 0 and a large number (for instance, $N_{obs}+10\sqrt{N_{obs}}$). This is our prior pdf.
 2. For each value of $\nu$ generate one value of $N$ according to a Poisson distribution with parameter $\nu$
 3. If $N=N_{obs}$ keep the value of $\nu$. Otherwise, discard.
 4. Plot the histogram of the values of $\nu$ that have been kept. Is it a Gaussian?
 5. Compute the sample mean and the variance from the histogram of $\nu$. Plot a Gaussian of these mean and variance. Compare to section V.4.
[Oriol]: Note that $N_{obs}+10\sqrt{N_{obs}}\approx \mu+10\sigma$
Can you write Python code for it?

N=10**7
nu_rndm = np.random.random(N)*(Nobs+10*np.sqrt(Nobs))
poisson = stat.poisson(nu_rndm)
N_mc = poisson.rvs(N) #We'll draw a sample from N_mc
nu_mc_sample = nu_rndm[N_mc==Nobs] # Solo nos quedamos con los valores de nu en nu_rndm con los que al generar la Poisson,
                                    # nos han dado, en el primer intento, que el de N_mc es igual a N_obs


I am working on the file "Statistics_block2.ipynb".
The first task is:
Part VI
Repeat for $N_{obs}=10$
Part VII
Repeat for $N_{obs}=1$
Parts II-VII
Las dos celdas que siguen sirven para haver los apartados II-V, VI i VII de manera sintetizada.
[Oriol]: Debajo, los plots se pueden pesonalizar a gusto, jugando con el numero y posición de los subplots y en qué subplot está cada gráfico. Los tres numeros de los subplots definen una cuadrícula y una posición en esa cuadrícula. La posición dentro de la cuadrícula tiene que ser menor o igual al producto de los dos primeros números. Para ejemplos podéis mirar los plots tal y como están ahora.
Can you write Python code for it?

Nobs = 10
nu_grid = np.linspace(Nobs-4*np.sqrt(Nobs), Nobs+6*np.sqrt(Nobs), 1000)

# Part II
gaussian_II = stat.norm(Nobs, np.sqrt(Nobs))

# Part III
N_III = 10**6
poisson_III = stat.poisson(Nobs)
mc_sample_III = poisson_III.rvs(N_III)

# Part IV
log_nu_posterior_lin = -nu_grid + Nobs*np.log(nu_grid) - np.log(msc.factorial(Nobs))
nu_posterior_lin = np.exp(log_nu_posterior_lin)

# Part V
N_V_lin=10**7
nu_prior_lin = np.random.random(N_V_lin)*(Nobs+10*np.sqrt(Nobs))
poisson_V_lin = stat.poisson(nu_prior_lin)
N_mc_lin = poisson_V_lin.rvs(N_V_lin) #We'll draw a sample from N_mc
mc_sample_V_lin = nu_prior_lin[N_mc_lin==Nobs]
print len(mc_sample_V_lin)

# Part V.E)
mu_VE_lin = np.mean(mc_sample_V_lin)
sigma_VE_lin = np.std(mc_sample_V_lin)
gaussian_VE_lin = stat.norm(mu_VE_lin, sigma_VE_lin)

rng = [nu_grid[0], nu_grid[-1]]
Nbins = 12
Nbins = np.arange(int(nu_grid[0]),int(2*nu_grid[-1])) # For the case Nobs = 1, to have bins between integer values.
                             # For other Nobs, it's bettter to comment it.

fig = plt.figure(figsize=(12,12))
ax1 = fig.add_subplot(221)
ax2 = fig.add_subplot(222)
ax3 = fig.add_subplot(212)

ax1.plot(nu_grid,gaussian_II.pdf(nu_grid), label='Gaussian fit to freq. estimator')
ax1.hist(mc_sample_III, normed=1, range=rng, bins=Nbins, label='MC frequentist estimator')
ax1.plot(nu_grid,nu_posterior_lin, label='Linear prior')

ax2.plot(nu_grid,nu_posterior_lin, label='Linear prior')
ax2.hist(mc_sample_V_lin, normed=1, range=rng, bins=Nbins, label='MC linear prior')
ax2.plot(nu_grid,gaussian_VE_lin.pdf(nu_grid), label='Gaussian fit to MC linear prior')

ax3.plot(nu_grid,gaussian_II.pdf(nu_grid), '--', label='Gaussian fit to freq. estimator')
ax3.hist(mc_sample_III, normed=1, range=rng, bins=Nbins, histtype="step", label='MC frequentist estimator')
ax3.plot(nu_grid,nu_posterior_lin, label='Linear prior')
ax3.hist(mc_sample_V_lin, normed=1, range=rng, bins=Nbins, histtype="step", label='MC linear prior')
ax3.plot(nu_grid,gaussian_VE_lin.pdf(nu_grid), ':', label='Gaussian fit to...


I am working on the file "Statistics_block2.ipynb".
The first task is:
Part VIII
Repeat steps IV to VII using now a prior for $\nu$ uniform in $\log\nu$. Compare to the previous results.
A constant prior in $\log\nu$ implies:
$$ \Pi(\nu)d\log\nu=c d\log\nu=\frac{c}{\nu} d\nu$$
In our case:
$$
\Pi(\nu)=
\begin{cases}
0 & \nu<0 \
\frac{c}{\nu} & \nu>0
\end{cases}
$$
The likelihood is the same as for the linear case:
$$L(N ;\nu)=\frac{\nu^{N}}{N!}e^{-\nu}$$
Therefore, the normalization factor becomes:
$$
\int_{-\infty}^{\infty} L(N ;\nu) \Pi(\nu) d\nu = \int_{0}^{\infty} \frac{\nu^{N}}{N!}e^{-\nu} \frac{c}{\nu} d\nu = \frac{c}{N!} \int_{0}^{\infty} \nu^{N-1}e^{-\nu}d\nu = \frac{c}{N!} (N-1)! = \frac{c}{N}
$$
Therefore, the posterior probability is:
$$
P(\nu;N)=\frac{L(N ;\nu)\Pi(\nu)}{\int_{-\infty}^{\infty} L(N ;\nu) \Pi(\nu) d\nu}=
\begin{cases}
0 & \nu<0 \
\frac{\nu^{N}}{N!}e^{-\nu}\frac{c}{\nu}\frac{N}{c}=\frac{\nu^{N-1}}{(N-1)!}e^{-\nu} & \nu>0
\end{cases}
$$
However, like in the linear case, due to the huge powers and exponentials, the computations will be done with the logarithm of $P(\nu;N)$ for $\nu>0$ and eventually exponentiated to plot the real value.
$$ \log\big(P(\nu;N)\rvert_{\nu>0}\big)=-\nu+(N-1)\log(\nu)-\log\big((N-1)!\big)$$
Simulating with an improper prior
For the linear case, we have easily solved the problem of the improper prior because generating a uniform sample is really easy, and the only trick used was to make it between 0 and $N_{obs}+10\sqrt{N_{obs}}$. Now, generating  a sample following this prior is not that simple, and it has to be done either by try-reject or inverse sampling.
Due to the shape of the pdf filling a ridiculously small area of any square containing it, the try-reject was discarded.
Inversion sampling does not require a finite box, but in this case, as it does require the cdf and it is an improper pdf, the range of the prior has been limited to $\nu\in[{\nu_{LIM}}^{-1},\nu_{LIM}]$, being $\nu_{LIM}=N_{obs}+10\sqrt{N_{obs}}$. Therefore:
$$\int_{{\nu_{LIM}}^{-1}}^{\nu_{LIM}} \frac{c}{\nu} d\nu = c \Big[\log \nu...
Can you write Python code for it?

Nobs = 1
nu_grid = np.linspace(Nobs-5*np.sqrt(Nobs), Nobs+6*np.sqrt(Nobs), 1000)

# Part II
gaussian_II = stat.norm(Nobs, np.sqrt(Nobs))

# Part III
N_III = 10**6
poisson_III = stat.poisson(Nobs)
mc_sample_III = poisson_III.rvs(N_III)

# Part IV
log_nu_posterior_log = -nu_grid + (Nobs-1)*np.log(nu_grid) - np.log(msc.factorial(Nobs-1))
nu_posterior_log = np.exp(log_nu_posterior_log)

# Part V
N_V_log=10**6
nu_lim = Nobs+10*np.sqrt(Nobs)
# For Nobs=1 it is better to increase the value
nu_lim = Nobs+1000*np.sqrt(Nobs)
r_prior_log = np.random.random(N_V_log)
nu_prior_log = np.exp(2*r_prior_log*np.log(nu_lim))/nu_lim
poisson_V_log = stat.poisson(nu_prior_log)
N_mc_log = poisson_V_log.rvs(N_V_log) #We'll draw a sample from N_mc
mc_sample_V_log = nu_prior_log[N_mc_log==Nobs]

# Part V.E)
mu_VE_log = np.mean(mc_sample_V_log)
sigma_VE_log = np.std(mc_sample_V_log)
gaussian_VE_log = stat.norm(mu_VE_log, sigma_VE_log)

rng = [nu_grid[0], nu_grid[-1]]
Nbins = 12
Nbins = np.arange(int(nu_grid[0]),int(2*nu_grid[-1])) # For the case Nobs = 1, to have bins between integer values.
                             # For other Nobs, it's bettter to comment it.

fig = plt.figure(figsize=(12,12))
ax1 = fig.add_subplot(221)
ax2 = fig.add_subplot(222)
ax3 = fig.add_subplot(212)

ax1.plot(nu_grid,gaussian_II.pdf(nu_grid), label='Gaussian fit to freq. estimator')
ax1.hist(mc_sample_III, normed=1, range=rng, bins=Nbins, label='MC frequentist estimator')
ax1.plot(nu_grid,nu_posterior_log, label='Log prior')

ax2.plot(nu_grid,nu_posterior_log, label='Log prior')
ax2.hist(mc_sample_V_log, normed=1, range=rng, bins=Nbins, label='MC log prior')
ax2.plot(nu_grid,gaussian_VE_log.pdf(nu_grid), label='Gaussian fit to MC log prior')

ax3.plot(nu_grid,gaussian_II.pdf(nu_grid), '--', label='Gaussian fit to freq. estimator')
ax3.hist(mc_sample_III, normed=1, histtype="step", range=rng, bins=Nbins, label='MC frequentist estimator')
ax3.plot(nu_grid,nu_posterior_log, label='Log prior')
ax3.hist(mc_sample_V_log, normed=1, histtype="step", range=rng,...


I am working on the file "Stefania_files/tagging-full-systematic.ipynb".
The first task is:
Add max, sum among PIDs
Can you write Python code for it?

from itertools import combinations
PIDs = {'k': data_track.PIDNNk.values,
        'e': data_track.PIDNNe.values,
        'mu': data_track.PIDNNm.values,
       }
for (pid_name1, pid_values1), (pid_name2, pid_values2) in combinations(PIDs.items(), 2):
    data_track['max_PID_{}_{}'.format(pid_name1, pid_name2)] = numpy.maximum(pid_values1, pid_values2)
    data_track['sum_PID_{}_{}'.format(pid_name1, pid_name2)] = pid_values1 + pid_values2


I am working on the file "Stefania_files/tagging-full-systematic.ipynb".
The first task is:
Compute full scheme for different random states to obtain quality with full systematic
Can you write Python code for it?

numpy.random.seed(13)

random_states = numpy.random.randint(1, high=10e3, size=30)

random_states

D2_step = []
tagging_efficiency_step = []
tagging_efficiency_delta_step = []
auc_step = []

for step, random_state in zip(range(len(random_states)), random_states):
    tagging_efficiency, tagging_efficiency_delta, D2, auc = tagging(tt_base, mn_base, track_logistic=True,
                                                                    vertex_logistic=False,
                                                                    random_state=random_state)
    D2_step.append(D2)
    tagging_efficiency_step.append(tagging_efficiency)
    tagging_efficiency_delta_step.append(tagging_efficiency_delta)
    auc_step.append(auc)


I am working on the file "Stefania_files/tagging-full-systematic.ipynb".
The first task is:
DT, MN base
Can you write Python code for it?

from sklearn.isotonic import IsotonicRegression
from sklearn.linear_model import LogisticRegression

tt_base = DecisionTrainClassifier(learning_rate=0.02, n_estimators=1500, depth=6, pretransform_needed=True,
                                  max_features=15, loss=LogLossFunction(regularization=100))
mn_base = MatrixNetSkyGridClassifier(connection='skygrid', user_name='antares',
                                     iterations=1000, regularization=0.02, sync=False)

from utils import prepare_B_data_for_given_part, calibrate_probs, calculate_auc_with_and_without_untag_events

def tagging(base_estimator_track, base_estimator_vertex,
            track_logistic=False, vertex_logistic=False, random_state=None):
    # track training
    folding_track = FoldingClassifier(base_estimator_track, n_folds=2,
                                      random_state=random_state, ipc_profile='ssh-ipy',
                                      features=features_track)
    folding_track.fit_lds(data_track_sw_passed_lds)

    # compute B data from track data
    Bdata_tracks = prepare_B_data_for_given_part(folding_track,
                                                [data_track_sw_passed, data_track_sw_not_passed],
                                                 logistic=track_logistic, random_state=random_state)

    # vertex training
    folding_vertex = FoldingClassifier(base_estimator_vertex, n_folds=2, random_state=random_state,
                                       features=features_vertex)
    folding_vertex.fit_lds(data_vertex_sw_passed_lds)

    Bdata_vertex = prepare_B_data_for_given_part(folding_vertex,
                                                  [data_vertex_sw_passed, data_vertex_sw_not_passed],
                                                  logistic=vertex_logistic,
                                                  sign_part_column='signVtx', part_name='vertex',
                                                 random_state=random_state)

    # Combine track and vertex
    Bdata =...


I am working on the file "Stefania_files/tagging-full-systematic.ipynb".
The first task is:
Leave muons, kaons, electrons
Can you write Python code for it?

threshold_kaon = 0.7
threshold_muon = 0.4
threshold_electron = 0.6
cut_pid = """
            ((PIDNNk > {trk})
                | (PIDNNm > {trm})
                | (PIDNNe > {tre})) """.format(trk=threshold_kaon, trm=threshold_muon,
                                               tre=threshold_electron).replace("\n", "")

data_track = data_track.query(cut_pid)

get_events_statistics(data_track)


I am working on the file "Stefania_files/tagging-full-systematic.ipynb".
The first task is:
Main idea:
find tracks/vertices, which can help reconstruct the sign of B if you know track sign.
label = signB * signTrack
* the highest output means that this is same sign B as track/vertex
* the lowest output means that this is opposite sign B as track/vertex
Define features
Can you write Python code for it?

features_track = list(set(data_track.columns) - {'run', 'event', 'i', 'signB', 'signTrack', 'N_sig_sw',
                                                 'Bmass', 'mult',
                                                 'PIDNNp', 'PIDNNpi', 'label', 'thetaMin',
                                                 'Dist_phi', event_id_column,
                                                 'mu_cut', 'e_cut', 'K_cut'})
features_track

features_vertex =  ['mult', 'nnkrec', 'ptB', 'vflag', 'ipsmean', 'ptmean', 'vcharge',
                     'svm', 'svp', 'BDphiDir', 'svtau', 'docamax']


I am working on the file "Stefania_files/tagging-full-systematic.ipynb".
The first task is:
Remove rows with NAN from data
Can you write Python code for it?

data_track = data_track_nan.dropna()
len(data_track_nan), len(data_track), get_events_statistics(data_track)


I am working on the file "Stefania_files/tagging-full-systematic.ipynb".
The first task is:
Result quality with full systematic error
Can you write Python code for it?

from utils import result_table

result = result_table(numpy.mean(tagging_efficiency_step), numpy.mean(tagging_efficiency_delta_step),
                      D2_step, auc_step, name='NEW + full systematic')
result

result = result_table(numpy.mean(tagging_efficiency_step), numpy.mean(tagging_efficiency_delta_step),
                      D2_step, auc_step, name='NEW + full systematic')
result.to_csv('img/new-tagging-systematic.csv', header=True, index=False)


I am working on the file "Stefania_files/tagging-full-systematic.ipynb".
The first task is:
define label = signB * signTrack

if > 0 (same sign) - label 1
if < 0 (different sign) - label 0
Can you write Python code for it?

data_track['label'] = (data_track.signB.values * data_track.signTrack.values > 0) * 1


I am working on the file "Stefania_files/tagging-full-systematic.ipynb".
The first task is:
vertex data
Can you write Python code for it?

import root_numpy
data_vertex = pandas.DataFrame(root_numpy.root2array('datasets/1016_vtx.root'))

data_vertex[event_id_column] = data_vertex.runNum.apply(str) + '_' + (data_vertex.evtNum.apply(int)).apply(str)
# reconstructing sign of B
data_vertex['signB'] = data_vertex.tagAnswer * (2 * data_vertex.iscorrect - 1)
# assure sign is +1 or -1
data_vertex['signVtx'] = (data_vertex.signVtx.values > 0) * 2 - 1
data_vertex['label'] = (data_vertex.signVtx.values * data_vertex.signB.values > 0) * 1


I am working on the file "Stefania_files/track-based-tagging-OS.ipynb".
The first task is:
Add diff_pt and cos(diff_phi)
Can you write Python code for it?

# add different between max pt in event and pt for each track
def add_diff_pt(data):
    max_pt = group_max(data[event_id_column].values.astype(str), data.partPt.values)
    data.loc[:, 'diff_pt'] = max_pt - data['partPt'].values

# max is computing max over tracks in the same event for saome data
def group_max(groups, data):
    # computing unique integer id for each group
    assert len(groups) == len(data)
    _, event_id = numpy.unique(groups, return_inverse=True)
    max_over_event = numpy.zeros(max(event_id) + 1) - numpy.inf
    numpy.maximum.at(max_over_event, event_id, data)
    return max_over_event[event_id]

# add diff pt
add_diff_pt(data)
# add cos(diff_phi)
data.loc[:, 'cos_diff_phi'] = numpy.cos(data.diff_phi.values)


I am working on the file "Stefania_files/track-based-tagging-OS.ipynb".
The first task is:
Add max, sum among PIDs
Can you write Python code for it?

from itertools import combinations
PIDs = {'k': data.PIDNNk.values,
        'e': data.PIDNNe.values,
        'mu': data.PIDNNm.values,
       }
for (pid_name1, pid_values1), (pid_name2, pid_values2) in combinations(PIDs.items(), 2):
    data.loc[:, 'max_PID_{}_{}'.format(pid_name1, pid_name2)] = numpy.maximum(pid_values1, pid_values2)
    data.loc[:, 'sum_PID_{}_{}'.format(pid_name1, pid_name2)] = pid_values1 + pid_values2


I am working on the file "Stefania_files/track-based-tagging-OS.ipynb".
The first task is:
Apply ghost prob cut
Can you write Python code for it?

initial_cut = '(ghostProb < 0.4)'
data = data.query(initial_cut)
os_selection = (data.IPs.values > 3) * ((abs(data.diff_eta.values) > 0.6) | (abs(data.diff_phi.values) > 0.825))
data = data[os_selection]

get_events_statistics(data)


I am working on the file "Stefania_files/track-based-tagging-OS.ipynb".
The first task is:
Choose most probable B-events
Can you write Python code for it?

_, take_indices = numpy.unique(data[event_id_column], return_index=True)

figure(figsize=[15, 5])

subplot(1, 2, 1)
hist(data.Bmass.values[take_indices], bins=100)
title('B mass hist')
xlabel('mass')

subplot(1, 2, 2)
hist(data.N_sig_sw.values[take_indices], bins=100, normed=True)
title('sWeights hist')
xlabel('signal sWeights')
plt.savefig('img/Bmass_OS.png' , format='png')


I am working on the file "Stefania_files/track-based-tagging-OS.ipynb".
The first task is:
Comparison table of different models
Can you write Python code for it?

pandas.set_option('display.precision', 8)
result = pandas.concat(models)
result.index = result.name
result.drop('name', axis=1)


I am working on the file "Stefania_files/track-based-tagging-OS.ipynb".
The first task is:
DT
Can you write Python code for it?

tt_base = DecisionTrainClassifier(learning_rate=0.02, n_estimators=3000, depth=6,
                                  max_features=15, loss=LogLossFunction(regularization=100))
tt_folding = FoldingGroupClassifier(SklearnClassifier(tt_base), n_folds=2, random_state=11,
                                    train_features=features, group_feature='group_column',
                                    parallel_profile = 'threads-2')
%time tt_folding.fit_lds(data_sw_passed_lds)
pass

import cPickle
with open('models/dt_OS.pkl', 'w') as f:
    cPickle.dump(tt_folding, f)

comparison_report = ClassificationReport({'tt': tt_folding}, data_sw_passed_lds)

comparison_report.compute_metric(RocAuc())

comparison_report.roc()

lc = comparison_report.learning_curve(RocAuc(), steps=1)

lc

for est in tt_folding.estimators:
    est.estimators = est.estimators[:1000]

comparison_report.feature_importance()


I am working on the file "Stefania_files/track-based-tagging-OS.ipynb".
The first task is:
Define B-like events for training
Events with low sWeight still will be used only to test quality.
Can you write Python code for it?

sweight_threshold = 1.
data_sw_passed = data[data.N_sig_sw > sweight_threshold]
data_sw_not_passed = data[data.N_sig_sw <= sweight_threshold]
get_events_statistics(data_sw_passed)

_, take_indices = numpy.unique(data_sw_passed[event_id_column], return_index=True)

figure(figsize=[15, 5])
subplot(1, 2, 1)
hist(data_sw_passed.Bmass.values[take_indices], bins=100)
title('B mass hist for sWeight > 1 selection')
xlabel('mass')

subplot(1, 2, 2)
hist(data_sw_passed.N_sig_sw.values[take_indices], bins=100, normed=True)
title('sWeights hist for sWeight > 1 selection')
xlabel('signal sWeights')
plt.savefig('img/Bmass_selected_OS.png' , format='png')

hist(data_sw_passed.diff_pt.values, bins=100)
pass


I am working on the file "Stefania_files/track-based-tagging-OS.ipynb".
The first task is:
Implementing best tracking
Can you write Python code for it?

from utils import prepare_B_data_for_given_part

Bdata_prepared = prepare_B_data_for_given_part(tt_folding, [data_sw_passed, data_sw_not_passed], logistic=True)

Bdata_prepared.to_csv('models/Bdata_tracks_OS.csv', header=True, index=False)


I am working on the file "Stefania_files/track-based-tagging-OS.ipynb".
The first task is:
Leave not muons, kaons, electrons; Filter pions, protons
Can you write Python code for it?

threshold_kaon = 0.
threshold_muon = 0.
threshold_electron = 0.
threshold_pion = 0.5
threshold_proton = 0.5
cut_pid = " ((PIDNNk > {trk}) | (PIDNNm > {trm}) | (PIDNNe > {tre})) & (PIDNNpi < {trpi}) & (PIDNNp < {trp}) "
cut_pid = cut_pid.format(trk=threshold_kaon, trm=threshold_muon, tre=threshold_electron, trpi=threshold_pion,
                         trp=threshold_proton)

data = data.query(cut_pid)

get_events_statistics(data)


I am working on the file "Stefania_files/track-based-tagging-OS.ipynb".
The first task is:
Main idea:
find tracks, which can help reconstruct the sign of B if you know track sign.
label = signB * signTrack
* the highest output means that this is same sign B as track
* the lowest output means that this is opposite sign B than track
Define features
Can you write Python code for it?

features = list(set(data.columns) - {'index', 'run', 'event', 'i', 'signB', 'signTrack', 'N_sig_sw', 'Bmass', 'mult',
                                     'PIDNNp', 'PIDNNpi', 'label', 'thetaMin', 'Dist_phi', event_id_column,
                                     'mu_cut', 'e_cut', 'K_cut', 'ID', 'diff_phi', 'group_column'})
features


I am working on the file "Stefania_files/track-based-tagging-OS.ipynb".
The first task is:
PIDs histograms
Can you write Python code for it?

figure(figsize=[15, 4])
for i, column in enumerate(['PIDNNm', 'PIDNNe', 'PIDNNk']):
    subplot(1, 3, i + 1)
    hist(data_sw_passed[column].values, bins=60, range=(0, 1), label=column)
    legend()


I am working on the file "Stefania_files/track-based-tagging-OS.ipynb".
The first task is:
Remove rows with NAN from data
Can you write Python code for it?

data = data_nan.dropna()
len(data_nan), len(data), get_events_statistics(data)


I am working on the file "Stefania_files/track-based-tagging-OS.ipynb".
The first task is:
define label = signB * signTrack

if > 0 (same sign) - label 1
if < 0 (different sign) - label 0
Can you write Python code for it?

data.loc[:, 'label'] = (data.signB.values * data.signTrack.values > 0) * 1

','.join(data.columns)


I am working on the file "Stefania_files/track-based-tagging-OS.ipynb".
The first task is:
pt
Can you write Python code for it?

hist(data_sw_passed.diff_pt.values, bins=60, normed=True)
pass


I am working on the file "Stefania_files/track-based-tagging-PID-less-FeaturesExperiments_axelr.ipynb".
The first task is:
Add diff_pt and cos(diff_phi)
Can you write Python code for it?

# add different between max pt in event and pt for each track
def add_diff_pt(data):
    max_pt = group_max(data[event_id_column].values.astype(str), data.partPt.values)
    data.loc[:, 'diff_pt'] = max_pt - data['partPt'].values

# max is computing max over tracks in the same event for some data (column)
def group_max(groups, data):
    # computing unique integer id for each group
    assert len(groups) == len(data)
    _, event_id = numpy.unique(groups, return_inverse=True)
    max_over_event = numpy.zeros(max(event_id) + 1) - numpy.inf
    numpy.maximum.at(max_over_event, event_id, data)
    return max_over_event[event_id]

# add diff pt
add_diff_pt(data)
# add cos(diff_phi)
data.loc[:, 'cos_diff_phi'] = numpy.cos(data.diff_phi.values)


I am working on the file "Stefania_files/track-based-tagging-PID-less-FeaturesExperiments_axelr.ipynb".
The first task is:
Add max, sum among PIDs
Can you write Python code for it?


I am working on the file "Stefania_files/track-based-tagging-PID-less-FeaturesExperiments_axelr.ipynb".
The first task is:
Alex experiments
Renaming
Can you write Python code for it?

names_mapping = {
    'Bmass'    : 'sig_mass',
    'EOverP'   : 'part_e_over_p',
    'IP'       : 'part_ip',
    'IPPU'     : 'part_ippu',
    'IPerr'    : 'part_ip_err',
    'IPs'      : 'part_ip_significance',
    'PIDNNe'   : 'part_pid_e',
    'PIDNNk'   : 'part_pid_k',
    'PIDNNm'   : 'part_pid_m',
    'PIDNNp'   : 'part_pid_p',
    'PIDNNpi'  : 'part_pid_pi',
    'diff_eta' : 'both_diff_eta',
    'diff_phi' : 'both_diff_phi',
    'eta'      : 'part_eta',
    'ghostProb': 'part_pid_ghost',
    'nnkrec'   : 'n_reconstructed_vertices',
    'partP'    : 'part_p',
    'partPt'   : 'part_pt',
    'partlcs'  : 'part_lcs',
    'proj'     : 'both_proj',
    'ptB'      : 'sig_pt',
    'signB'    : 'sig_sign',
    'etaB'     : 'sig_eta',
    'signTrack': 'part_sign',
    'veloch'   : 'part_veloch',
}

# _, delegates = numpy.unique(data[event_id_column], return_index=True)
# columns_old = [old_name for old_name, new_name in names_mapping.items() if 'sig_' in new_name]
# columns_new = [new_name for old_name, new_name in names_mapping.items() if 'sig_' in new_name]

# short_data = data[columns_old].iloc[delegates, :]
# short_data = short_data.rename(columns=names_mapping)

# short_data.head()

# from decisiontrain import DecisionTrainClassifier
# clf = FoldingGroupClassifier(DecisionTrainClassifier(n_estimators=1000, depth=4), n_folds=2)
# clf.fit(short_data[['sig_pt', 'sig_eta', 'sig_mass']], (short_data['sig_sign'] + 1) // 2)

# report = clf.test_on(short_data[['sig_pt', 'sig_eta', 'sig_mass']], (short_data['sig_sign'] + 1) // 2)

# report.learning_curve(RocAuc(), steps=1)


I am working on the file "Stefania_files/track-based-tagging-PID-less-FeaturesExperiments_axelr.ipynb".
The first task is:
Apply ghost prob cut
Can you write Python code for it?

initial_cut = '(ghostProb < 0.4)'
data = data.query(initial_cut)

get_events_statistics(data)


I am working on the file "Stefania_files/track-based-tagging-PID-less-FeaturesExperiments_axelr.ipynb".
The first task is:
Calibration
Can you write Python code for it?

from utils import get_result_with_bootstrap_for_given_part

models = []

models.append(get_result_with_bootstrap_for_given_part(tagging_efficiency, tagging_efficiency_delta, tt_folding,
                                                      [data_sw_passed, data_sw_not_passed], 'tt-iso', logistic=False))

models.append(get_result_with_bootstrap_for_given_part(tagging_efficiency, tagging_efficiency_delta, tt_folding,
                                                      [data_sw_passed, data_sw_not_passed], 'tt-log', logistic=True))

models.append(get_result_with_bootstrap_for_given_part(tagging_efficiency, tagging_efficiency_delta, tt_folding,
                                                      [data_sw_passed, data_sw_not_passed], 'tt-iso-normed',
                                                       logistic=False, normed_signs=True))

models.append(get_result_with_bootstrap_for_given_part(tagging_efficiency, tagging_efficiency_delta, tt_folding,
                                                      [data_sw_passed, data_sw_not_passed], 'tt-log-normed',
                                                       logistic=True, normed_signs=True))


I am working on the file "Stefania_files/track-based-tagging-PID-less-FeaturesExperiments_axelr.ipynb".
The first task is:
Choose most probable B-events
Can you write Python code for it?

_, take_indices = numpy.unique(data[event_id_column], return_index=True)

figure(figsize=[15, 5])

subplot(1, 2, 1)
hist(data.Bmass.values[take_indices], bins=100)
title('B mass hist')
xlabel('mass')

subplot(1, 2, 2)
hist(data.N_sig_sw.values[take_indices], bins=100, normed=True)
title('sWeights hist')
xlabel('signal sWeights')
plt.savefig('img/Bmass_less_PID.png' , format='png')


I am working on the file "Stefania_files/track-based-tagging-PID-less-FeaturesExperiments_axelr.ipynb".
The first task is:
Comparison table of different models
Can you write Python code for it?

pandas.set_option('display.precision', 8)
result = pandas.concat(models)
result.index = result.name
result.drop('name', axis=1)

pandas.set_option('display.precision', 8)
result = pandas.concat(models)
result.index = result.name
result.drop('name', axis=1)

result.ix[:2, :].to_csv('img/new-tagging_full_tracks.csv', index=False, header=True)


I am working on the file "Stefania_files/track-based-tagging-PID-less-FeaturesExperiments_axelr.ipynb".
The first task is:
Define B-like events for training
Events with low sWeight still will be used only to test quality.
Can you write Python code for it?

sweight_threshold = 1.
data_sw_passed = data[data.N_sig_sw > sweight_threshold]
data_sw_not_passed = data[data.N_sig_sw <= sweight_threshold]
get_events_statistics(data_sw_passed)

_, take_indices = numpy.unique(data_sw_passed[event_id_column], return_index=True)

figure(figsize=[15, 5])
subplot(1, 2, 1)
hist(data_sw_passed.Bmass.values[take_indices], bins=100)
title('B mass hist for sWeight > 1 selection')
xlabel('mass')

subplot(1, 2, 2)
hist(data_sw_passed.N_sig_sw.values[take_indices], bins=100, normed=True)
title('sWeights hist for sWeight > 1 selection')
xlabel('signal sWeights')
plt.savefig('img/Bmass_selected_less_PID.png' , format='png')

hist(data_sw_passed.diff_pt.values, bins=100)
pass


I am working on the file "Stefania_files/track-based-tagging-PID-less-FeaturesExperiments_axelr.ipynb".
The first task is:
Main idea:
find tracks, which can help reconstruct the sign of B if you know track sign.
label = signB * signTrack
* the highest output means that this is same sign B as track
* the lowest output means that this is opposite sign B than track
Define features
Can you write Python code for it?


I am working on the file "Stefania_files/track-based-tagging-PID-less-FeaturesExperiments_axelr.ipynb".
The first task is:
PID pairs scatters
Can you write Python code for it?

figure(figsize=[15, 16])
bins = 60
step = 3
for i, (feature1, feature2) in enumerate(combinations(['PIDNNk', 'PIDNNm', 'PIDNNe', 'PIDNNp', 'PIDNNpi'], 2)):
    subplot(4, 3, i + 1)
    Z, (x, y) = numpy.histogramdd(data_sw_passed[[feature1, feature2]].values, bins=bins, range=([0, 1], [0, 1]))
    pcolor(numpy.log(Z).T, vmin=0)
    xlabel(feature1)
    ylabel(feature2)
    xticks(numpy.arange(bins, step), x[::step]), yticks(numpy.arange(bins, step), y[::step])
plt.savefig('img/PID_selected_less_PID.png' , format='png')


I am working on the file "Stefania_files/track-based-tagging-PID-less-FeaturesExperiments_axelr.ipynb".
The first task is:
PIDs histograms
Can you write Python code for it?


I am working on the file "Stefania_files/track-based-tagging-PID-less-FeaturesExperiments_axelr.ipynb".
The first task is:
Remove rows with NAN from data
Can you write Python code for it?

data = data_nan.dropna()
len(data_nan), len(data), get_events_statistics(data)

del data_nan

for column in data.columns:
    if (data[column].dtype == 'int64') and (data[column].max() < 2 ** 30):
        data[column] = data[column].astype('int32')
    if data[column].dtype == 'float64':
        data[column] = data[column].astype('float32')


I am working on the file "Stefania_files/track-based-tagging-PID-less-FeaturesExperiments_axelr.ipynb".
The first task is:
Simplified training
Can you write Python code for it?

old_features = list(names_mapping.keys()) + ['label', 'group_column', 'N_sig_sw', 'event_id']
data_trimmed = data[data.N_sig_sw > 1.][old_features].rename(columns=names_mapping)


I am working on the file "Stefania_files/track-based-tagging-PID-less-FeaturesExperiments_axelr.ipynb".
The first task is:
Train to distinguish same sign vs opposite sign
Can you write Python code for it?

from rep.estimators import XGBoostClassifier

data_sw_passed_lds = LabeledDataStorage(data_sw_passed, data_sw_passed.label, data_sw_passed.N_sig_sw.values)


I am working on the file "Stefania_files/track-based-tagging-PID-less-FeaturesExperiments_axelr.ipynb".
The first task is:
Комбинируем умно
Combining formula
$$
p_r / (1 - p_r) = \prod_i (p_i / (1 - p_i))^{sign track}
$$
$$
lpr = logit(p_r) \qquad lp_i = logit(p_i)
$$
$$
p_r / (1 - p_r) = e^{lpr}
$$
$$
p_i / (1 - p_i) = e^{lpi}
$$
$$
e^{lpr} = \prod_i e^{lpi * sign track}
$$
$$
lpr = \sum_i lpi * sign track
$$
Calculating tagging efficiency ($\epsilon_{tag}$)
$$N (\text{passed selection}) = \sum_{\text{passed selection}} sw_i$$
$$N (\text{all events}) = \sum_{\text{all events}} sw_i,$$
where $sw_i$ - sPLot weight (sWeight for signal)
$$\epsilon_{tag} = \frac{N (\text{passed selection})} {N (\text{all events})}$$
$$\Delta\epsilon_{tag} = \frac{\sqrt{N (\text{passed selection})}} {N (\text{all events})}$$
Can you write Python code for it?

N_B_passed = float(get_events_number(data))
tagging_efficiency = N_B_passed / get_N_B_events()
tagging_efficiency_delta = sqrt(N_B_passed) / get_N_B_events()
tagging_efficiency, tagging_efficiency_delta

hist(data.diff_pt.values, bins=100)
pass


I am working on the file "Stefania_files/track-based-tagging-PID-less-FeaturesExperiments_axelr.ipynb".
The first task is:
Объединяем на коленке
Can you write Python code for it?

from sklearn.linear_model import LogisticRegression

# track_predictions.min(), track_predictions.max()

calibrator = FoldingGroupClassifier(LogisticRegression(), n_folds=2,
                                    train_features=['p'],
                                    group_feature='group_column', )

X_ = pandas.DataFrame(dict(p=track_predictions, group_column=data_trimmed.group_column))

calibrator.fit(X_, data_trimmed.label)

calibrated_predictions = logit(calibrator.predict_proba(X_)[:, 1])

_, group_indices = numpy.unique(data_trimmed.group_column, return_inverse=True)

# event_predictions = numpy.bincount(group_indices, weights=data_trimmed['part_sign'].values * calibrated_predictions)
event_predictions = numpy.bincount(group_indices, weights=data_trimmed['part_sign'].values * track_predictions)

event_predictions /= numpy.sqrt(numpy.bincount(group_indices) + 1)

sig_sign = numpy.bincount(group_indices, weights=data_trimmed['sig_sign'].values) / numpy.bincount(group_indices)

# event_predictions = numpy.bincount(group_indices, weights=data_trimmed['part_sign'] * track_predictions)

print roc_auc_score(sig_sign, event_predictions, sample_weight=data['N_sig_sw'])
print roc_auc_score(data_trimmed.label, track_predictions)
print roc_auc_score(data_trimmed.label, calibrated_predictions)

data_trimmed.label.values[:20], (data_trimmed.sig_sign * data_trimmed.part_sign).values[:20]

def test_quality_on_subset_of_features(train_features):
    clf = FoldingGroupClassifier(DecisionTrainClassifier(n_estimators=3000, n_threads=14, learning_rate=0.03), n_folds=2,
                                 train_features=train_features,
                                 group_feature='group_column', )
    %time clf.fit(data_trimmed, data_trimmed['label'])
    pprint(sorted(zip(clf.estimators[0].feature_importances_, train_features)))
    report = clf.test_on(data_trimmed, data_trimmed['label'])
    report.learning_curve(RocAuc(), steps=2).plot()
    track_predictions = logit(clf.predict_proba(data_trimmed)[:, 1])
   ...


I am working on the file "Stefania_files/track-based-tagging-PID-less-FeaturesExperiments_axelr.ipynb".
The first task is:
Распределение тегирующих треков
Can you write Python code for it?

from scipy.special import logit
track_predictions = logit(clf.predict_proba(data_trimmed)[:, 1])

from utils import group_max
max_pred = group_max(data_trimmed.group_column, track_predictions)
min_pred = -group_max(data_trimmed.group_column, -track_predictions)

roc_auc_score(data_trimmed.sig_sign, track_predictions)

plt.figure(figsize=[20, 40])
plotted_columns = data_trimmed.columns
plotted_columns = plotted_columns[data_trimmed.dtypes != 'O']

for i, column in enumerate(plotted_columns, 1):
    plt.subplot(9, 4, i)
    plot_margins = numpy.percentile(data_trimmed[column].values, [0.5, 99.5])
    plot_config = dict(alpha=0.4, bins=30, normed=True, range=plot_margins)
    if 'pid' in column:
        plot_config['range'] = (0.1, 1)
    plt.hist(data_trimmed[column].values, label='all', **plot_config)
    plt.hist(data_trimmed[column].values[max_pred == track_predictions], label='ss', **plot_config)
    plt.hist(data_trimmed[column].values[min_pred == track_predictions], label='os', **plot_config)
    plt.title(column)
    plt.legend(loc='best')


I am working on the file "Stefania_files/track-based-tagging-SS.ipynb".
The first task is:
Add diff_pt and cos(diff_phi)
Can you write Python code for it?

# add different between max pt in event and pt for each track
def add_diff_pt(data):
    max_pt = group_max(data[event_id_column].values.astype(str), data.partPt.values)
    data.loc[:, 'diff_pt'] = max_pt - data['partPt'].values

# max is computing max over tracks in the same event for saome data
def group_max(groups, data):
    # computing unique integer id for each group
    assert len(groups) == len(data)
    _, event_id = numpy.unique(groups, return_inverse=True)
    max_over_event = numpy.zeros(max(event_id) + 1) - numpy.inf
    numpy.maximum.at(max_over_event, event_id, data)
    return max_over_event[event_id]

# add diff pt
add_diff_pt(data)
# add cos(diff_phi)
data.loc[:, 'cos_diff_phi'] = numpy.cos(data.diff_phi.values)


I am working on the file "Stefania_files/track-based-tagging-SS.ipynb".
The first task is:
Apply ghost prob cut, leave  Same Side (SS) tracks
Can you write Python code for it?

initial_cut = '(ghostProb < 0.4)'
data = data.query(initial_cut)
ss_selection = (data.IPs.values < 3) * (abs(data.diff_eta.values) < 0.6) * (abs(data.diff_phi.values) < 0.825)
data = data[ss_selection]

get_events_statistics(data)


I am working on the file "Stefania_files/track-based-tagging-SS.ipynb".
The first task is:
Calculating tagging efficiency ($\epsilon_{tag}$)
$$N (\text{passed selection}) = \sum_{\text{passed selection}} sw_i$$
$$N (\text{all events}) = \sum_{\text{all events}} sw_i,$$
where $sw_i$ - sPLot weight (sWeight for signal)
$$\epsilon_{tag} = \frac{N (\text{passed selection})} {N (\text{all events})}$$
$$\Delta\epsilon_{tag} = \frac{\sqrt{N (\text{passed selection})}} {N (\text{all events})}$$
Can you write Python code for it?


I am working on the file "Stefania_files/track-based-tagging-SS.ipynb".
The first task is:
Calibration
Can you write Python code for it?

from utils import get_result_with_bootstrap_for_given_part

models = []

models.append(get_result_with_bootstrap_for_given_part(tagging_efficiency, tagging_efficiency_delta, tt_folding,
                                                      [data_sw_passed, data_sw_not_passed], 'tt-iso', logistic=False))

models.append(get_result_with_bootstrap_for_given_part(tagging_efficiency, tagging_efficiency_delta, tt_folding,
                                                      [data_sw_passed, data_sw_not_passed], 'tt-log', logistic=True))


I am working on the file "Stefania_files/track-based-tagging-SS.ipynb".
The first task is:
Choose most probable B-events
Can you write Python code for it?

_, take_indices = numpy.unique(data[event_id_column], return_index=True)

figure(figsize=[15, 5])

subplot(1, 2, 1)
hist(data.Bmass.values[take_indices], bins=100)
title('B mass hist')
xlabel('mass')

subplot(1, 2, 2)
hist(data.N_sig_sw.values[take_indices], bins=100, normed=True)
title('sWeights hist')
xlabel('signal sWeights')
plt.savefig('img/Bmass_SS.png' , format='png')


I am working on the file "Stefania_files/track-based-tagging-SS.ipynb".
The first task is:
DT
Can you write Python code for it?

tt_base = DecisionTrainClassifier(learning_rate=0.02, n_estimators=3000, depth=6,
                                  max_features=15, loss=LogLossFunction(regularization=100))
tt_folding = FoldingGroupClassifier(SklearnClassifier(tt_base), n_folds=2, random_state=11,
                                    train_features=features, group_feature=event_id_column)
%time tt_folding.fit_lds(data_sw_passed_lds)
pass

import cPickle
with open('models/dt_SS.pkl', 'w') as f:
    cPickle.dump(tt_folding, f)

comparison_report = ClassificationReport({'tt': tt_folding}, data_sw_passed_lds)

comparison_report.compute_metric(RocAuc())

comparison_report.roc()

lc = comparison_report.learning_curve(RocAuc(), steps=100)

lc

for est in tt_folding.estimators:
    est.estimators = est.estimators[:2000]

comparison_report.feature_importance()


I am working on the file "Stefania_files/track-based-tagging-SS.ipynb".
The first task is:
Define B-like events for training
Events with low sWeight still will be used only to test quality.
Can you write Python code for it?

sweight_threshold = 1.
data_sw_passed = data[data.N_sig_sw > sweight_threshold]
data_sw_not_passed = data[data.N_sig_sw <= sweight_threshold]
get_events_statistics(data_sw_passed)

_, take_indices = numpy.unique(data_sw_passed[event_id_column], return_index=True)

figure(figsize=[15, 5])
subplot(1, 2, 1)
hist(data_sw_passed.Bmass.values[take_indices], bins=100)
title('B mass hist for sWeight > 1 selection')
xlabel('mass')

subplot(1, 2, 2)
hist(data_sw_passed.N_sig_sw.values[take_indices], bins=100, normed=True)
title('sWeights hist for sWeight > 1 selection')
xlabel('signal sWeights')
plt.savefig('img/Bmass_selected_SS.png' , format='png')

hist(data_sw_passed.diff_pt.values, bins=100)
pass


I am working on the file "Stefania_files/track-based-tagging-SS.ipynb".
The first task is:
Implementing best tracking
Can you write Python code for it?

from utils import prepare_B_data_for_given_part

Bdata_prepared = prepare_B_data_for_given_part(tt_folding, [data_sw_passed, data_sw_not_passed], logistic=True)

Bdata_prepared.to_csv('models/Bdata_tracks_SS.csv', header=True, index=False)


I am working on the file "Stefania_files/track-based-tagging-SS.ipynb".
The first task is:
Leave not muons, kaons, electrons, pions, protons
Can you write Python code for it?

threshold_kaon = 0.
threshold_muon = 0.
threshold_electron = 0.
threshold_pion = 0.
threshold_proton = 0.
cut_pid = " ( (PIDNNk > {trk}) | (PIDNNm > {trm}) | (PIDNNe > {tre}) | (PIDNNpi > {trpi}) | (PIDNNp > {trp})) "
cut_pid = cut_pid.format(trk=threshold_kaon, trm=threshold_muon, tre=threshold_electron, trpi=threshold_pion,
                         trp=threshold_proton)

data = data.query(cut_pid)

get_events_statistics(data)


I am working on the file "Stefania_files/track-based-tagging-SS.ipynb".
The first task is:
Main idea:
find tracks, which can help reconstruct the sign of B if you know track sign.
label = signB * signTrack
* the highest output means that this is same sign B as track
* the lowest output means that this is opposite sign B than track
Define features
Can you write Python code for it?

features = list(set(data.columns) - {'index', 'run', 'event', 'i', 'signB', 'signTrack', 'N_sig_sw', 'Bmass', 'mult',
                                     'PIDNNp', 'PIDNNpi', 'label', 'thetaMin', 'Dist_phi', event_id_column,
                                     'mu_cut', 'e_cut', 'K_cut', 'ID', 'diff_phi'})
features


I am working on the file "Stefania_files/track-based-tagging-SS.ipynb".
The first task is:
PIDs histograms
Can you write Python code for it?

figure(figsize=[15, 4])
for i, column in enumerate(['PIDNNm', 'PIDNNe', 'PIDNNk']):
    subplot(1, 3, i + 1)
    hist(data_sw_passed[column].values, bins=60, range=(0, 1), label=column)
    legend()


I am working on the file "Stefania_files/track-based-tagging-SS.ipynb".
The first task is:
Reading initial data
Can you write Python code for it?

import root_numpy
data_nan = pandas.DataFrame(root_numpy.root2array('datasets/tracks.root', 'tracks'))

data_nan.head()

event_id_column = 'event_id'
event_id = data_nan.run.apply(str) + '_' + data_nan.event.apply(str)
data_nan[event_id_column] = numpy.unique(event_id, return_inverse=True)[1]

get_events_statistics(data_nan)

get_N_B_events()


I am working on the file "Stefania_files/track-based-tagging-SS.ipynb".
The first task is:
pt
Can you write Python code for it?

hist(data_sw_passed.diff_pt.values, bins=60, normed=True)
pass


I am working on the file "Stefania_files/track-based-tagging-track-sign-usage.ipynb".
The first task is:
Add diff_pt and cos(diff_phi)
Can you write Python code for it?


I am working on the file "Stefania_files/track-based-tagging-track-sign-usage.ipynb".
The first task is:
Add max, sum among PIDs
Can you write Python code for it?

from itertools import combinations
PIDs = {'k': data.PIDNNk.values,
        'e': data.PIDNNe.values,
        'mu': data.PIDNNm.values,
       }
for (pid_name1, pid_values1), (pid_name2, pid_values2) in combinations(PIDs.items(), 2):
    data.loc[:, 'max_PID_{}_{}'.format(pid_name1, pid_name2)] = numpy.maximum(pid_values1, pid_values2)
    data.loc[:, 'sum_PID_{}_{}'.format(pid_name1, pid_name2)] = pid_values1 + pid_values2


I am working on the file "Stefania_files/track-based-tagging-track-sign-usage.ipynb".
The first task is:
Calibration
Can you write Python code for it?

from utils import get_result_with_bootstrap_for_given_part

models = []

models.append(get_result_with_bootstrap_for_given_part(tagging_efficiency, tagging_efficiency_delta, tt_folding,
                                                      [data_sw_passed, data_sw_not_passed], 'tt-iso', logistic=False))

models.append(get_result_with_bootstrap_for_given_part(tagging_efficiency, tagging_efficiency_delta, tt_folding,
                                                      [data_sw_passed, data_sw_not_passed], 'tt-log', logistic=True))

models.append(get_result_with_bootstrap_for_given_part(tagging_efficiency, tagging_efficiency_delta, tt_folding,
                                                      [data_sw_passed, data_sw_not_passed], 'tt-iso-normed',
                                                       logistic=False, normed_signs=True))

models.append(get_result_with_bootstrap_for_given_part(tagging_efficiency, tagging_efficiency_delta, tt_folding,
                                                      [data_sw_passed, data_sw_not_passed], 'tt-log-normed',
                                                       logistic=True, normed_signs=True))


I am working on the file "Stefania_files/track-based-tagging-track-sign-usage.ipynb".
The first task is:
Choose most probable B-events
Can you write Python code for it?

_, take_indices = numpy.unique(data[event_id_column], return_index=True)

figure(figsize=[15, 5])

subplot(1, 2, 1)
hist(data.Bmass.values[take_indices], bins=100)
title('B mass hist')
xlabel('mass')

subplot(1, 2, 2)
hist(data.N_sig_sw.values[take_indices], bins=100, normed=True)
title('sWeights hist')
xlabel('signal sWeights')
#plt.savefig('img/Bmass_less_PID.png' , format='png')


I am working on the file "Stefania_files/track-based-tagging-track-sign-usage.ipynb".
The first task is:
Comparison table of different models
Can you write Python code for it?

pandas.set_option('display.precision', 8)
result = pandas.concat(models)
result.index = result.name
result.drop('name', axis=1)


I am working on the file "Stefania_files/track-based-tagging-track-sign-usage.ipynb".
The first task is:
Import
Can you write Python code for it?

import pandas
import numpy
from folding_group import FoldingGroupClassifier
from rep.data import LabeledDataStorage
from rep.report import ClassificationReport
from rep.report.metrics import RocAuc

from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import roc_curve, roc_auc_score

from utils import get_N_B_events, get_events_number, get_events_statistics


I am working on the file "Stefania_files/track-based-tagging-track-sign-usage.ipynb".
The first task is:
Leave not muons, kaons, electrons, protons, pions
Can you write Python code for it?


I am working on the file "Stefania_files/track-based-tagging-track-sign-usage.ipynb".
The first task is:
PID pairs scatters
Can you write Python code for it?

figure(figsize=[15, 16])
bins = 60
step = 3
for i, (feature1, feature2) in enumerate(combinations(['PIDNNk', 'PIDNNm', 'PIDNNe', 'PIDNNp', 'PIDNNpi'], 2)):
    subplot(4, 3, i + 1)
    Z, (x, y) = numpy.histogramdd(data_sw_passed[[feature1, feature2]].values, bins=bins, range=([0, 1], [0, 1]))
    pcolor(numpy.log(Z).T, vmin=0)
    xlabel(feature1)
    ylabel(feature2)
    xticks(numpy.arange(bins, step), x[::step]), yticks(numpy.arange(bins, step), y[::step])
# plt.savefig('img/PID_selected_less_PID.png' , format='png')


I am working on the file "Stefania_files/track-based-tagging-track-sign-usage.ipynb".
The first task is:
PIDs histograms
Can you write Python code for it?


I am working on the file "Stefania_files/track-based-tagging-track-sign-usage.ipynb".
The first task is:
Train to distinguish same sign vs opposite sign
Can you write Python code for it?

from hep_ml.decisiontrain import DecisionTrainClassifier
from hep_ml.losses import LogLossFunction
from rep.estimators import SklearnClassifier

data_sw_passed_lds = LabeledDataStorage(data_sw_passed, data_sw_passed.label, data_sw_passed.N_sig_sw.values)


I am working on the file "Stefania_files/track-based-tagging-track-sign-usage.ipynb".
The first task is:
count of tracks
Can you write Python code for it?

_, n_tracks = numpy.unique(data_sw_passed[event_id_column], return_counts=True)
hist(n_tracks, bins=100)
title('Number of tracks')
# plt.savefig('img/tracks_number_less_PID.png' , format='png')


I am working on the file "Stefania_files/track-based-tagging-track-sign-usage.ipynb".
The first task is:
define label = signB * signTrack

if > 0 (same sign) - label 1
if < 0 (different sign) - label 0
Can you write Python code for it?

data.loc[:, 'label'] = (data.signB.values * data.signTrack.values > 0) * 1

', '.join(data.columns)


I am working on the file "Stefania_files/track-based-tagging-track-sign-usage.ipynb".
The first task is:
pt
Can you write Python code for it?


I am working on the file "Stefania_files/track-tagging.ipynb".
The first task is:
Add diff_pt and cos(diff_phi)
Can you write Python code for it?

from utils import add_diff_pt
# add diff pt
add_diff_pt(data)
# add cos(diff_phi)
data['cos_diff_phi'] = numpy.cos(data.diff_phi.values)


I am working on the file "Stefania_files/track-tagging.ipynb".
The first task is:
Calibration
Can you write Python code for it?

from utils import get_result_with_bootstrap_for_given_part

result = get_result_with_bootstrap_for_given_part(tagging_efficiency, tagging_efficiency_delta, tt_folding,
                                                 [data_sw_passed, data_sw_not_passed], 'tt-log', get_N_B_events(),
                                                 logistic=True, n_calibrations=30)
result

import utils
reload(utils)

from utils import get_result_with_bootstrap_for_given_part

result = get_result_with_bootstrap_for_given_part(tagging_efficiency, tagging_efficiency_delta, tt_folding,
                                                 [data_sw_passed, data_sw_not_passed], 'tt-log', get_N_B_events(),
                                                 logistic=True, n_calibrations=1)
result

result.to_csv('img/tracks.csv', index=False, header=True)


I am working on the file "Stefania_files/track-tagging.ipynb".
The first task is:
Import
Can you write Python code for it?


I am working on the file "Stefania_files/track-tagging.ipynb".
The first task is:
Leave not muons, kaons, electrons, protons, pions
Can you write Python code for it?


I am working on the file "Stefania_files/track-tagging.ipynb".
The first task is:
MC trained algorithm testing
Can you write Python code for it?

from utils import estimate_algorithm

import cPickle
with open('models/dt_MC.pkl', 'r') as f:
    tt_folding_MC = cPickle.load(f)
with open('models/calibrator_tracks_MC.pkl', 'r') as f:
    calibrator_tracks_MC = cPickle.load(f)
with open('models/calibrator_B_MC.pkl', 'r') as f:
    calibrator_B_MC = cPickle.load(f)

p_MC = tt_folding_MC.predict_proba(data)[:, 1]
roc_auc_score(data.label, p_MC, sample_weight=data.N_sig_sw.values.astype(float64))

estimate_algorithm(tt_folding_MC, calibrator_tracks_MC, calibrator_B_MC, data, get_N_B_events())

estimate_algorithm(tt_folding_MC, calibrator_tracks_MC, calibrator_B_MC, data, get_N_B_events(), calib_part_itself=True,
                   calib_itself=True)


I am working on the file "Stefania_files/track-tagging.ipynb".
The first task is:
Main idea:
find tracks, which can help reconstruct the sign of B if you know track sign.
label = signB * signTrack
* the highest output means that this is same sign B as track
* the lowest output means that this is opposite sign B than track
Define features
Can you write Python code for it?

features = list(set(data.columns) - {'index', 'run', 'event', 'i', 'signB', 'signTrack', 'N_sig_sw', 'Bmass', 'mult',
                                     'PIDNNp', 'PIDNNpi', 'label', 'thetaMin', 'Dist_phi', event_id_column,
                                     'mu_cut', 'e_cut', 'K_cut', 'ID', 'diff_phi', 'group_column'})
features


I am working on the file "Stefania_files/track-tagging.ipynb".
The first task is:
PID pairs scatters
Can you write Python code for it?

figure(figsize=[15, 16])
bins = 60
step = 3
for i, (feature1, feature2) in enumerate(combinations(['PIDNNk', 'PIDNNm', 'PIDNNe', 'PIDNNp', 'PIDNNpi'], 2)):
    subplot(4, 3, i + 1)
    Z, (x, y) = numpy.histogramdd(data_sw_passed[[feature1, feature2]].values, bins=bins, range=([0, 1], [0, 1]))
    pcolor(numpy.log(Z).T, vmin=0)
    xlabel(feature1)
    ylabel(feature2)
    xticks(numpy.arange(bins, step), x[::step]), yticks(numpy.arange(bins, step), y[::step])
plt.savefig('img/PID_selected_less_PID.png' , format='png')


I am working on the file "Stefania_files/track-tagging.ipynb".
The first task is:
Reading initial data
Can you write Python code for it?

import root_numpy
data_nan = pandas.DataFrame(root_numpy.root2array('datasets/data/csv/JPsiK/Tracks.root'))

data_nan.head()

event_id_column = 'event_id'
event_id = data_nan.run.apply(str) + '_' + data_nan.event.apply(str)
data_nan['group_column'] = numpy.unique(event_id, return_inverse=True)[1]
data_nan[event_id_column] = event_id

get_events_statistics(data_nan)

get_N_B_events()


I am working on the file "Stefania_files/track-tagging.ipynb".
The first task is:
Remove rows with NAN from data
Can you write Python code for it?

data = data_nan.dropna()
len(data_nan), len(data), get_events_statistics(data)


I am working on the file "Stefania_files/track-tagging.ipynb".
The first task is:
Train to distinguish same sign vs opposite sign
Can you write Python code for it?

from decisiontrain import DecisionTrainClassifier
from rep.estimators import SklearnClassifier
from hep_ml.losses import LogLossFunction

data_sw_passed_lds = LabeledDataStorage(data_sw_passed, data_sw_passed.label.values, data_sw_passed.N_sig_sw.values)


I am working on the file "Stefania_files/track-tagging.ipynb".
The first task is:
count of tracks
Can you write Python code for it?

figure(figsize=(20, 6))

subplot(1, 2, 1)
_, n_tracks = numpy.unique(data_sw_passed[event_id_column], return_counts=True)
hist(n_tracks, bins=100)
title('Number of tracks for events with sWeight > 1')

subplot(1, 2, 2)
_, n_tracks_all = numpy.unique(data[event_id_column], return_counts=True)
hist(n_tracks_all, bins=106)
title('Number of tracks')

plt.savefig('img/tracks_number_less_PID.png' , format='png')


I am working on the file "Stefania_files/track-tagging.ipynb".
The first task is:
pt
Can you write Python code for it?

hist(data_sw_passed.diff_pt.values, bins=60, normed=True)
pass


I am working on the file "Stefania_files/vertex-based-new-loss.ipynb".
The first task is:
Calibrating results $p(\text{vrt same sign}|B)$ and combining them
Can you write Python code for it?

models = []

from utils import get_result_with_bootstrap_for_given_part

models.append(get_result_with_bootstrap_for_given_part(tagging_efficiency, tagging_efficiency_delta, tt_folding,
                                                      [data_sw_passed, data_sw_not_passed],
                                                      logistic=True, name="tt-log",
                                                      sign_part_column='signVtx', part_name='vertex'))

models.append(get_result_with_bootstrap_for_given_part(tagging_efficiency, tagging_efficiency_delta, tt_folding,
                                                      [data_sw_passed, data_sw_not_passed],
                                                      logistic=False, name="tt-iso",
                                                      sign_part_column='signVtx', part_name='vertex'))


I am working on the file "Stefania_files/vertex-based-new-loss.ipynb".
The first task is:
Comparison of different models
Can you write Python code for it?


I am working on the file "Stefania_files/vertex-based-new-loss.ipynb".
The first task is:
Define B-like events for training and others for prediction
Can you write Python code for it?

sweight_threshold = 1.
data_sw_passed = data[data.N_sig_sw > sweight_threshold]
data_sw_not_passed = data[data.N_sig_sw <= sweight_threshold]
get_events_statistics(data_sw_passed)


I am working on the file "Stefania_files/vertex-based-new-loss.ipynb".
The first task is:
Define features
Can you write Python code for it?

features =  ['mult', 'nnkrec', 'ptB', 'vflag', 'ipsmean', 'ptmean', 'vcharge',
             'svm', 'svp', 'BDphiDir', 'svtau', 'docamax']


I am working on the file "Stefania_files/vertex-based-new-loss.ipynb".
The first task is:
Find good vtx to define sign B
trying to guess sign of B based on sign of vtx. If the guess is good, the vtx will be used on next step to train classifier.
2-folding random forest selection for right tagged events
Can you write Python code for it?

data_sw_passed_lds = LabeledDataStorage(data_sw_passed, data_sw_passed.label, data_sw_passed.N_sig_sw)


I am working on the file "Stefania_files/vertex-based-new-loss.ipynb".
The first task is:
Import
Can you write Python code for it?

import pandas
import numpy

from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import roc_curve, roc_auc_score

from rep.metaml import FoldingClassifier
from rep.data import LabeledDataStorage
from rep.report import ClassificationReport
from rep.report.metrics import RocAuc

from utils import get_N_B_events, get_events_number, get_events_statistics


I am working on the file "Stefania_files/vertex-based-new-loss.ipynb".
The first task is:
Reading initial data
Can you write Python code for it?

import root_numpy
data = pandas.DataFrame(root_numpy.root2array('datasets/1016_vtx.root'))


I am working on the file "Stefania_files/vertex-based-new-loss.ipynb".
The first task is:
Report for all vtx
Can you write Python code for it?

report = ClassificationReport({'tt': tt_folding}, data_sw_passed_lds)

report.learning_curve(RocAuc())

report.compute_metric(RocAuc())

report.roc()


I am working on the file "Stefania_files/vertex-based-new-loss.ipynb".
The first task is:
Training on all vtx
in this case we don't use preselection with RandomForest
DT full
Can you write Python code for it?

from hep_ml.decisiontrain import DecisionTrainClassifier
from hep_ml.losses import LogLossFunction

from hep_ml.losses import HessianLossFunction
from hep_ml.commonutils import check_sample_weight
from scipy.special import expit

class LogLossFunctionTagging(HessianLossFunction):
    """Logistic loss function (logloss), aka binomial deviance, aka cross-entropy,
       aka log-likelihood loss.
    """
    def fit(self, X, y, sample_weight):
        self.sample_weight = check_sample_weight(y, sample_weight=sample_weight,
                                                 normalize=True, normalize_by_class=True)
        self.initial_pred = numpy.log(X['initial_pred'].values)
        self.y_signed = 2 * y - 1
        self.minus_y_signed = - self.y_signed
        self.y_signed_times_weights = self.y_signed * self.sample_weight
        HessianLossFunction.fit(self, X, y, sample_weight=self.sample_weight)
        return self

    def __call__(self, y_pred):
        y_pred = y_pred + self.initial_pred
        return numpy.sum(self.sample_weight * numpy.logaddexp(0, self.minus_y_signed * y_pred))

    def negative_gradient(self, y_pred):
        y_pred = y_pred + self.initial_pred
        return self.y_signed_times_weights * expit(self.minus_y_signed * y_pred)

    def hessian(self, y_pred):
        y_pred = y_pred + self.initial_pred
        expits = expit(y_pred)
        return self.sample_weight * expits * (1 - expits)

    def prepare_tree_params(self, y_pred):
        y_pred = y_pred + self.initial_pred
        return self.y_signed * expit(self.minus_y_signed * y_pred), self.sample_weight

tt_base = DecisionTrainClassifier(learning_rate=0.02, n_estimators=1500, depth=6,
                                  max_features=8, loss=LogLossFunctionTagging(regularization=100), train_features=features)
tt_folding = FoldingClassifier(tt_base, n_folds=2, random_state=11,
                               features=features + ['initial_pred'])
%time tt_folding.fit_lds(data_sw_passed_lds)
pass

from scipy.special import expit,...


I am working on the file "Stefania_files/vertex-based-tagging.ipynb".
The first task is:
Calibrating results $p(\text{vrt same sign}|B)$ and combining them
Can you write Python code for it?

models = []

from utils import get_result_with_bootstrap_for_given_part

models.append(get_result_with_bootstrap_for_given_part(tagging_efficiency, tagging_efficiency_delta, tt_folding,
                                                      [data_sw_passed, data_sw_not_passed],
                                                      logistic=True, name="tt-log",
                                                      sign_part_column='signVtx', part_name='vertex'))

models.append(get_result_with_bootstrap_for_given_part(tagging_efficiency, tagging_efficiency_delta, tt_folding,
                                                      [data_sw_passed, data_sw_not_passed],
                                                      logistic=False, name="tt-iso",
                                                      sign_part_column='signVtx', part_name='vertex'))

models.append(get_result_with_bootstrap_for_given_part(tagging_efficiency, tagging_efficiency_delta, tt_folding_rf,
                                                      [data_sw_passed_selected,
                                                       data_sw_passed_not_selected,
                                                       data_sw_not_passed],
                                                      logistic=True, name="rf-tt-log",
                                                      sign_part_column='signVtx', part_name='vertex'))

models.append(get_result_with_bootstrap_for_given_part(tagging_efficiency,
                                                       tagging_efficiency_delta, tt_folding_rf,
                                                      [data_sw_passed_selected,
                                                       data_sw_passed_not_selected,
                                                       data_sw_not_passed],
                                                      logistic=False, name="rf-tt-iso",
                                                      sign_part_column='signVtx', part_name='vertex'))


I am working on the file "Stefania_files/vertex-based-tagging.ipynb".
The first task is:
Comparison of different models
Can you write Python code for it?


I am working on the file "Stefania_files/vertex-based-tagging.ipynb".
The first task is:
DT for good vtx
Can you write Python code for it?

from hep_ml.decisiontrain import DecisionTrainClassifier
from hep_ml.losses import LogLossFunction

tt_base = DecisionTrainClassifier(learning_rate=0.02, n_estimators=9000, depth=6, pretransform_needed=True,
                                  max_features=8, loss=LogLossFunction(regularization=100))
tt_folding_rf = FoldingClassifier(tt_base, n_folds=2, random_state=11, ipc_profile='ssh-ipy',
                                   features=features)
%time tt_folding_rf.fit_lds(data_sw_passed_selected_lds)
pass


I am working on the file "Stefania_files/vertex-based-tagging.ipynb".
The first task is:
Define B-like events for training and others for prediction
Can you write Python code for it?


I am working on the file "Stefania_files/vertex-based-tagging.ipynb".
The first task is:
Define features
Can you write Python code for it?


I am working on the file "Stefania_files/vertex-based-tagging.ipynb".
The first task is:
Define label
label = signB * signVtx > 0
* same sign of B and vtx -> label = 1
* opposite sign of B and vtx -> label = 0
Can you write Python code for it?

event_id_column = 'event_id'
data[event_id_column] = data.runNum.apply(str) + '_' + (data.evtNum.apply(int)).apply(str)
# reconstructing sign of B
data['signB'] = data.tagAnswer * (2 * data.iscorrect - 1)
# assure sign is +1 or -1
data['signVtx'] = (data.signVtx.values > 0) * 2 - 1
data['label'] = (data.signVtx.values * data.signB.values > 0) * 1

data.head()

get_events_statistics(data)

N_pass = get_events_number(data)
tagging_efficiency = 1. * N_pass / get_N_B_events()
tagging_efficiency_delta = sqrt(N_pass) / get_N_B_events()
print tagging_efficiency, tagging_efficiency_delta


I am working on the file "Stefania_files/vertex-based-tagging.ipynb".
The first task is:
Import
Can you write Python code for it?


I am working on the file "Stefania_files/vertex-based-tagging.ipynb".
The first task is:
ROC AUC
Can you write Python code for it?


I am working on the file "Stefania_files/vertex-based-tagging.ipynb".
The first task is:
Reading initial data
Can you write Python code for it?


I am working on the file "Stefania_files/vertex-based-tagging.ipynb".
The first task is:
Report for all vtx
Can you write Python code for it?


I am working on the file "Stefania_files/vertex-based-tagging.ipynb".
The first task is:
Report for selected vtx
Can you write Python code for it?

report = ClassificationReport({'tt': tt_folding_rf}, data_sw_passed_selected_lds)

report.learning_curve(RocAuc())

report.compute_metric(RocAuc())


I am working on the file "Stefania_files/vertex-based-tagging.ipynb".
The first task is:
Select good vtx
leaving for training only those events that were not-so poorly predicted by RandomForest.
Can you write Python code for it?

mask = ((flat_ss(probs[:, 1]) < 0.6) & (data_sw_passed.label == 0)) | \
       ((flat_os(probs[:, 1]) > 0.2) & (data_sw_passed.label == 1))
data_sw_passed_selected = data_sw_passed[mask]
data_sw_passed_not_selected = data_sw_passed[~mask]

get_events_statistics(data_sw_passed_selected)

data_sw_passed_selected_lds = LabeledDataStorage(data_sw_passed_selected,
                                                 data_sw_passed_selected.label,
                                                 data_sw_passed_selected.N_sig_sw)


I am working on the file "Stefania_files/vertex-based-tagging.ipynb".
The first task is:
Training on all vtx
in this case we don't use preselection with RandomForest
DT full
Can you write Python code for it?

from hep_ml.decisiontrain import DecisionTrainClassifier
from hep_ml.losses import LogLossFunction
tt_base = DecisionTrainClassifier(learning_rate=0.02, n_estimators=1500, depth=6, pretransform_needed=True,
                                  max_features=8, loss=LogLossFunction(regularization=100))
tt_folding = FoldingClassifier(tt_base, n_folds=2, random_state=11, ipc_profile='ssh-ipy',
                               features=features)
%time tt_folding.fit_lds(data_sw_passed_lds)
pass


I am working on the file "Step Through Variables and Data Types.ipynb".
The first task is:
Above, Python has inferred the type of the variable pizza to be an integer. Since strings can only be concatenated with other strings, our print statement generates an error. There are two ways we can resolve the error:

Declare the pizzas variable as type string (str) on assignment or
Re-cast the pizzas variable as a string within the print statement.
Can you write Python code for it?

customer = 'Carol'
pizzas = str(2)
print(customer + ' ordered ' + pizzas + ' pizzas.')

customer = 'Carol'
pizzas = 2
print(customer + ' ordered ' + str(pizzas) + ' pizzas.')


I am working on the file "Step Through Variables and Data Types.ipynb".
The first task is:
Lists in Python are:

mutable - the list and list items can be changed
ordered - list items keep the same "place" in the list

Ordered here does not mean sorted. The list below is printed with the numbers in the order we added them to the list, not in numeric order:
Can you write Python code for it?

ordered = [3, 2, 7, 1, 19, 0]
ordered

# There is a 'sort' method for sorting list items as needed:

ordered.sort()
ordered


I am working on the file "Step Through Variables and Data Types.ipynb".
The first task is:
The ability to reassign variable values becomes important when iterating through groups of objects for batch processing or other purposes. In the example below, the value of b is dynamically updated every time the while loop is executed:
Can you write Python code for it?

a = 5
b = 10
while b > a:
    print("b="+str(b))
    b = b-1


I am working on the file "Step Through Variables and Data Types.ipynb".
The first task is:
The above output for typed_list may seem odd. Referring to the documentation, we see that the argument to the type constructor is an iterable, which according to the documentation is "An object capable of returning its members one at a time." In our construtor statement above
```
Using the type constructor
constructed_list = list('purple')
```
the word 'purple' is the object - in this case a word - that when used to construct a list returns its members (individual letters) one at a time.
Compare the outputs below:
Can you write Python code for it?

constructed_list_int = list(123)

constructed_list_str = list('123')
constructed_list_str


I am working on the file "Step Through Variables and Data Types.ipynb".
The first task is:
There are cases when we may want to declare the data type, for example to assign a different data type from the default that will be inferred. Concatenating strings provides a good example.
Can you write Python code for it?

customer = 'Carol'
pizzas = 2
print(customer + ' ordered ' + pizzas + ' pizzas.')


I am working on the file "Step Through Variables and Data Types.ipynb".
The first task is:
Try It Yourself
Go to the section 4.4. Numeric Types in the Python 3 documentation at https://docs.python.org/3.4/library/stdtypes.html. The table in that section describes different operators - try some!
What is the difference betwee## Variables
Variables allow us to store values for later use. n the different division operators (/, //, and %)?
Variables
Variables allow us to store values for later use.
Can you write Python code for it?


I am working on the file "Step Through Variables and Data Types.ipynb".
The first task is:
Variables can be reassigned.
Can you write Python code for it?


I am working on the file "StepSize.ipynb".
The first task is:
Notice that there is large truncation error at large step sizes (starting from the left).  Then there is a downward decrease in error with a slope of 1 and 2 for forward and central respectively as expected (corresponding to error of $\mathcal{O}(h)$ and $\mathcal{O}(h^2)$).  If not for finite precision arithmetic these lines would continue indefinitely.  However, in a computer, subtractive cancellation starts becoming significant and the error starts increasing again.  At very small step sizes are error is 100% (the methods predict derivatives of 0).  Also note, as discussed, that the optimal step size for central diference is a bit larger than the optimal step size for forward difference.  For this function they are around $10^{-8}$ and $10^{-5}$.
Let's now try the same thing, but using the complex step method.  Almost everything can stay the same except that we need to import from cmath rather than math.  Doing this makes all of the functions we use (sin, cos, exp) defined for complex numbers and not just real numbers.  In Matlab, the functions are already overloaded for complex number automatically.
The formula using the complex step method is:
$$\text{Complex Step:}\quad f^\prime(x) \approx \frac{\operatorname{Im}[f(x + ih)]}{h} $$
Can you write Python code for it?

from cmath import sin, cos, exp

# initialize
grad_cs = np.zeros(n)

# loop through each step size
for i in range(n):
    h = step_size[i]
    grad_cs[i] = f(x + complex(0, h)).imag / h


I am working on the file "StepSize.ipynb".
The first task is:
Same as before, we will compute the error then plot the result from all three methods.
Can you write Python code for it?

# compute error
error_cs = np.abs((grad_cs - grad_exact)/grad_exact)

# the error is below machine precision in some cases so just add epsilon error so it shows on plot
error_cs[error_cs == 0] = 1e-16

plt.figure()
plt.loglog(step_size, error_fd, '.-', label='forward')
plt.loglog(step_size, error_cd, '.-', label='central')
plt.loglog(step_size, error_cs, '.-', label='complex')
plt.gca().set_ylim(ymin=1e-18, ymax=1e1)
ticks = np.arange(-1, -26, -3)
plt.xticks(10.0**ticks)
plt.gca().invert_xaxis()
plt.legend(loc='center right')
plt.xlabel('step size')
plt.ylabel('relative error')
plt.show()


I am working on the file "StepSize.ipynb".
The first task is:
Using a for loop we will now test out each of those step sizes.
Can you write Python code for it?

# loop through and try all the different starting points
for i in range(n):
    h = step_size[i]
    grad_fd[i] = (f(x + h) - f(x))/h
    grad_cd[i] = (f(x + h) - f(x-h))/(2*h)


I am working on the file "StepSize.ipynb".
The first task is:
We have an analytical solution for the gradient, so let's compute the relative error.
Can you write Python code for it?

# compute relative error compared to the exact solution
grad_exact = g(x)
error_fd = np.abs((grad_fd - grad_exact)/grad_exact)
error_cd = np.abs((grad_cd - grad_exact)/grad_exact)


I am working on the file "StepSize.ipynb".
The first task is:
We now plot the results on a loglog scale.  The x-axis shows the different step sizes we tried and the y-axis the relative error as compared to the exact gradients.  Note that I've inverted the x-axis so that smaller step sizes are on the right.
Can you write Python code for it?

plt.style.use('ggplot')
plt.figure()
plt.loglog(step_size, error_fd, '.-', label='forward')
plt.loglog(step_size, error_cd, '.-', label='central')
plt.gca().set_ylim(ymin=1e-18, ymax=1e1)
ticks = np.arange(-1, -26, -3)
plt.xticks(10.0**ticks)
plt.gca().invert_xaxis()
plt.legend(loc='center right')
plt.xlabel('step size')
plt.ylabel('relative error')
plt.show()


I am working on the file "Steppers.ipynb".
The first task is:
Steppers
Various steppers which are used to numerically solve differential equations are derived and defined in this notebook
The function below numerically solves an equation of the form
$$\frac{dy}{dt}=f(y,t)$$
by approximating using the Euler method
$$
\begin{align}
y_{n+1}&=y_{n}+h  \dfrac{dy}{dt}\
y_{n+1}&=y_{n}+h  f(y_n,t_n)
\end{align}
$$
Where $y$ is the state vector of the system
The derivation is as follows
Can you write Python code for it?

#Solves an equation of the form dy/dt=f(y,t) using the Euler method
#f is the function which returns the derivative of the state vector of the system
#y0 is the initial state of the system
#t is an array of times at which f will be evaluated

def Euler(f, y0, t, args=()):
    n    = len(t)                 #Find number of steps
    y0   = np.asarray(y0)         #Convert input state to numpy array
    y    = np.zeros((n, len(y0))) #Initialise array to hold state at each timestep
    y[0] = y0                     #Set initial state of system

    for i in range(n-1):
        h      = t[i+1]-t[i]
        y[i+1] = y[i] + h * f(y[i], t[i], *args) #*args passes all members of args to function
    return y


I am working on the file "Steppers.ipynb".
The first task is:
The function below numerically solves an equation of the form
$$\frac{dy}{dt}=f(y,t)$$
by approximating using the Backward Euler method.
$$
\begin{align}
y_{n+1}&=y_{n}+h  f(y_{n+1},t_{n+1})
\end{align}
$$
This approximation yields an algebraic equation for $y_{n+1}$ in which $y_{n+1}$ appears on both sides. This is because the Backward Euler method is an implicit method.
The implicit equation must be solved by fixed point iteration (as used in this case) or some version of the Newton-Rhapson method.
The the function $f(y,t)$ must be evaluated multiple times each step with implicit methods. However implicit methods are also more stable than explicit methods.
Can you write Python code for it?

#Solves an equation of the form dy/dt=f(y,t) using the backward Euler method
#f is the function which returns the derivative of the state vector of the system
#y0 is the initial state of the system
#t is an array of times at which f will be evaluated

def BackwardEuler(f, y0, t, args=()):
    n    = len(t)                 #Find number of steps
    y0   = np.asarray(y0)         #Convert input state to numpy array
    y    = np.zeros((n, len(y0))) #Initialise array to hold state at each timestep
    y[0] = y0                     #Set initial state of system

    for i in range(n-1):
        h      = t[i+1]-t[i]
        #Fixed point iteration to approximate y[i+1]
        y[i+1] = y[i]
        for j in range(10):
            y[i+1] = y[i] + h * f(y[i+1], t[i+1], *args)  #*args passes all members of args to function
    return y


I am working on the file "Steppers.ipynb".
The first task is:
The function below numerically solves an equation of the form
$$\frac{dy}{dt}=f(y,t)$$
by approximating using the improved Euler method (Heun's method)
$$
\begin{align}
y_{n+1}&=y_{n}+\frac{h}{2}[f(y_n, t_n) + f(y_{n+1},t_{n+1})]
\end{align}
$$
This method leads to a cancelation of errors as the forward and backward approximations of the derivative are averaged
Can you write Python code for it?

#Solves an equation of the form dy/dt=f(y,t) using the improved Euler method, also know as Heun's Method
#f is the function which returns the derivative of the state vector of the system
#y0 is the initial state of the system
#t is an array of times at which f will be evaluated

def ImprovedEuler(f, y0, t, args=()):
    n    = len(t)                 #Find number of steps
    y0   = np.asarray(y0)         #Convert input state to numpy array
    y    = np.zeros((n, len(y0))) #Initialise array to hold state at each timestep
    y[0] = y0                     #Set initial state of system

    for i in range(n-1):
        h      = t[i+1]-t[i]
        k1     = f(y[i],     t[i],    *args) #Forward evaluation
        k2     = f(y[i]+h*k1,t[i]+h,  *args) #Backward evaluation
        y[i+1] = y[i] + h*(k1+k2)/2          #Central evaluation
    return y


I am working on the file "Stiffness_eigenvalues.ipynb".
The first task is:
Analytic integration
The stiffness matrix is obtained integrating the product of the interpolator-derivatives (displacement-to-strains) matrix with the constitutive tensor and itself, i.e.
$$\begin{align}
K &= \int\limits_{-1}^{1}\int\limits_{-1}^{1} K_\text{int} dr\, ds\
 &= \int\limits_{-1}^{1}\int\limits_{-1}^{1} B^T C\, B\, dr\, ds \enspace .
\end{align}$$
Can you write Python code for it?

K = zeros(8,8)
for i in range(8):
    for j in range(8):
        K[i,j] = integrate(K_int[i,j], (r,-1,1), (s,-1,1))

simplify(K/K_factor)


I am working on the file "Stiffness_eigenvalues.ipynb".
The first task is:
Being the stiffness matrix integrand
$$K_\text{int} = B^T C B$$
Can you write Python code for it?


I am working on the file "Stiffness_eigenvalues.ipynb".
The first task is:
Thus, the interpolation matrix renders
Derivatives interpolation matrix
Can you write Python code for it?

dHdr = zeros(2,4)
for i in range(4):
    dHdr[0,i] = diff(N[i],r)
    dHdr[1,i] = diff(N[i],s)

jaco = eye(2)  # Jacobian matrix, identity for now
dHdx = jaco*dHdr

B = zeros(3,8)
for i in range(4):
    B[0, 2*i] = dHdx[0, i]
    B[1, 2*i+1] = dHdx[1, i]
    B[2, 2*i] = dHdx[1, i]
    B[2, 2*i+1] = dHdx[0, i]

B


I am working on the file "Stiffness_eigenvalues.ipynb".
The first task is:
We can check some numerical vales for $E=1$ Pa and $\nu=1/3$
Can you write Python code for it?

K_num = K.subs([(E, 1), (nu, S(1)/3)])

K_num.eigenvects()


I am working on the file "StoDIG/Facies_classification_StoDIG.ipynb".
The first task is:
Convolutional Neural Network
We build a CNN with the following layers:

Dropout layer on input
One 1D convolutional layer, with MaxPooling
Series of Dropout & Fully-Connected layers, of parameterizable length
Can you write Python code for it?

# Set parameters
input_dim = 24
output_dim = 9
n_per_batch = 128
epochs = 200

def dnn_model(init_dropout_rate=0.5, main_dropout_rate=0.45, hidden_dim_1=192, hidden_dim_2=96, max_norm=10, n_dense=3, sigma=0.0, nb_conv=32):
    # Define the model
    model = Sequential()
    model.add(Dropout(init_dropout_rate, input_shape=(conv_length,input_dim,)))
    model.add(Convolution1D(nb_conv, conv_length, border_mode='same', activation='relu', input_shape=(conv_length,input_dim), input_length=conv_length))
    model.add(MaxPooling1D(pool_length=2, stride=None, border_mode='same'))
    model.add(Flatten())
    model.add(Dropout(main_dropout_rate, input_shape=(nb_conv*conv_length,)))
    model.add(Dense(hidden_dim_1, input_dim=nb_conv*conv_length, init='uniform', activation='relu', W_constraint=maxnorm(max_norm)))
    for i in range(n_dense):
        if (i==1):
            model.add(Dropout(main_dropout_rate, input_shape=(hidden_dim_1,)))
            model.add(Dense(hidden_dim_2, input_dim=hidden_dim_1, init='uniform', activation='relu', W_constraint=maxnorm(max_norm)))
        else:
            model.add(Dropout(main_dropout_rate, input_shape=(hidden_dim_2,)))
            model.add(Dense(hidden_dim_2, input_dim=hidden_dim_2, init='uniform', activation='relu', W_constraint=maxnorm(max_norm)))
    model.add(Dense(output_dim, init='normal', activation='softmax'))

    optimizerNadam = Nadam(lr=0.002, beta_1=0.9, beta_2=0.999, epsilon=1e-08, schedule_decay=0.004)
    model.compile(loss='categorical_crossentropy', optimizer=optimizerNadam, metrics=['accuracy'])
    return model


I am working on the file "StoDIG/Facies_classification_StoDIG.ipynb".
The first task is:
Data Augmentation

It is physically reasonable to expect 1st and 2nd order derivatives of logs to play an important role in determining facies. To save the CNN the effort of learning convolution kernels to represent these features to the rest of the Neural Network we compute them here (for training and validation data). Further, we expand the input data to be acted on by the convolutional layer.
Can you write Python code for it?

conv_length = 7

# Reproducibility
np.random.seed(7)
# Load data

def addGradients(input):
    output = input
    for i in range(8):
        grad    = np.gradient(output[:,i])
        gradT   = np.reshape(grad,(grad.size,1))
        output  = np.concatenate((output, gradT), axis=1)

        grad2   = np.gradient(grad)
        grad2T  = np.reshape(grad2,(grad2.size,1))
        output  = np.concatenate((output, grad2T), axis=1)

    return output

def expand_dims(input):
    r = int((conv_length-1)/2)
    l = input.shape[0]
    n_input_vars = input.shape[1]
    output = np.zeros((l, conv_length, n_input_vars))
    for i in range(l):
        for j in range(conv_length):
            for k in range(n_input_vars):
                output[i,j,k] = input[min(i+j-r,l-1),k]
    return output

X_train = np.empty((0,conv_length,24), dtype=float)
X_test  = np.empty((0,conv_length,24), dtype=float)

wellId = 0.0
for i in range(10):
    X_train_subset = X_train_in[X_train_in[:, 8] == wellId][:,0:8]
    X_train_subset = addGradients(X_train_subset)
    X_train_subset = expand_dims(X_train_subset)
    X_train = np.concatenate((X_train,X_train_subset),axis=0)
    wellId = wellId + 1.0

for i in range(2):
    X_test_subset = X_test_in[X_test_in[:, 8] == wellId][:,0:8]
    X_test_subset = addGradients(X_test_subset)
    X_test_subset = expand_dims(X_test_subset)
    X_test = np.concatenate((X_test,X_test_subset),axis=0)
    wellId = wellId + 1.0

print(X_train.shape)
print(X_test.shape)

# Obtain labels
y_labels = np.zeros((len(y_train),1))
for i in range(len(y_train)):
    y_labels[i] = np.argmax(y_train[i])
y_labels = y_labels.astype(int)


I am working on the file "StoDIG/Facies_classification_StoDIG.ipynb".
The first task is:
Data ingest

We load the training and testing data to preprocess it for further analysis, filling the missing data values in the PE field with zero and proceeding to normalize the data that will be fed into our model.
Can you write Python code for it?

filename = 'train_test_data.csv'
data = pd.read_csv(filename)
data.head(12)

# Set 'Well Name' and 'Formation' fields as categories
data['Well Name'] = data['Well Name'].astype('category')
data['Formation'] = data['Formation'].astype('category')

# Fill missing values and normalize for 'PE' field
data['PE'] = data['PE'].fillna(value=0)
mean_pe = data['PE'].mean()
std_pe = data['PE'].std()
data['PE'] = (data['PE']-mean_pe)/std_pe

# Normalize the rest of fields (GR, ILD_log10, DelthaPHI, PHIND,NM_M,RELPOS)
correct_facies_labels = data['Facies'].values
feature_vectors = data.drop(['Formation'], axis=1)
well_labels = data[['Well Name', 'Facies']].values
data_vectors = feature_vectors.drop(['Well Name', 'Facies'], axis=1).values
scaler = preprocessing.StandardScaler().fit(data_vectors)
scaled_features = scaler.transform(data_vectors)
data_out = np.hstack([well_labels, scaled_features])


I am working on the file "StoDIG/Facies_classification_StoDIG.ipynb".
The first task is:
In order to avoid overfitting, we evaluate our model by running a 5-fold stratified cross-validation routine.
Can you write Python code for it?

# Cross Validation
def cross_validate():
    t0 = time.time()
    estimator = KerasClassifier(build_fn=dnn_model, nb_epoch=epochs, batch_size=n_per_batch, verbose=0)
    skf = StratifiedKFold(n_splits=5, shuffle=True)
    results_dnn = cross_val_score(estimator, X_train, y_train, cv= skf.get_n_splits(X_train, y_train))
    t1 = time.time()
    print("Cross Validation time = %d" % (t1-t0) )
    print(' Cross Validation Results')
    print( results_dnn )
    print(np.mean(results_dnn))

cross_validate()


I am working on the file "StoDIG/Facies_classification_StoDIG.ipynb".
The first task is:
Prediction

To predict the STUART and CRAWFORD blind wells we do the following:
Set up a plotting function to display the logs & facies.
Can you write Python code for it?

# 1=sandstone  2=c_siltstone   3=f_siltstone
# 4=marine_silt_shale 5=mudstone 6=wackestone 7=dolomite
# 8=packstone 9=bafflestone
facies_colors = ['#F4D03F', '#F5B041','#DC7633','#6E2C00', '#1B4F72','#2E86C1', '#AED6F1', '#A569BD', '#196F3D']

#facies_color_map is a dictionary that maps facies labels
#to their respective colors
facies_color_map = {}
for ind, label in enumerate(facies_labels):
    facies_color_map[label] = facies_colors[ind]

def label_facies(row, labels):
    return labels[ row['Facies'] -1]

def make_facies_log_plot(logs, facies_colors):
    #make sure logs are sorted by depth
    logs = logs.sort_values(by='Depth')
    cmap_facies = colors.ListedColormap(
            facies_colors[0:len(facies_colors)], 'indexed')

    ztop=logs.Depth.min(); zbot=logs.Depth.max()

    cluster=np.repeat(np.expand_dims(logs['Facies'].values,1), 100, 1)

    f, ax = plt.subplots(nrows=1, ncols=6, figsize=(8, 12))
    ax[0].plot(logs.GR, logs.Depth, '-g')
    ax[1].plot(logs.ILD_log10, logs.Depth, '-')
    ax[2].plot(logs.DeltaPHI, logs.Depth, '-', color='0.5')
    ax[3].plot(logs.PHIND, logs.Depth, '-', color='r')
    ax[4].plot(logs.PE, logs.Depth, '-', color='black')
    im=ax[5].imshow(cluster, interpolation='none', aspect='auto',
                    cmap=cmap_facies,vmin=1,vmax=9)

    divider = make_axes_locatable(ax[5])
    cax = divider.append_axes("right", size="20%", pad=0.05)
    cbar=plt.colorbar(im, cax=cax)
    cbar.set_label((17*' ').join([' SS ', 'CSiS', 'FSiS',
                                'SiSh', ' MS ', ' WS ', ' D  ',
                                ' PS ', ' BS ']))
    cbar.set_ticks(range(0,1)); cbar.set_ticklabels('')

    for i in range(len(ax)-1):
        ax[i].set_ylim(ztop,zbot)
        ax[i].invert_yaxis()
        ax[i].grid()
        ax[i].locator_params(axis='x', nbins=3)

    ax[0].set_xlabel("GR")
    ax[0].set_xlim(logs.GR.min(),logs.GR.max())
    ax[1].set_xlabel("ILD_log10")
    ax[1].set_xlim(logs.ILD_log10.min(),logs.ILD_log10.max())
   ...


I am working on the file "StoDIG/Facies_classification_StoDIG.ipynb".
The first task is:
Split data into training data and blind data, and output as Numpy arrays
Can you write Python code for it?

def preprocess(data_out):

    data = data_out
    well_data = {}
    well_names = ['SHRIMPLIN', 'ALEXANDER D', 'SHANKLE', 'LUKE G U', 'KIMZEY A', 'CROSS H CATTLE',
                  'NOLAN', 'Recruit F9', 'NEWBY', 'CHURCHMAN BIBLE', 'STUART', 'CRAWFORD']
    for name in well_names:
        well_data[name] = [[], []]

    for row in data:
        well_data[row[0]][1].append(row[1])
        well_data[row[0]][0].append(list(row[2::]))

    chunks = []
    chunks_test = []
    chunk_length = 1
    chunks_facies = []
    wellID=0.0
    for name in well_names:

        if name not in ['STUART', 'CRAWFORD']:
            test_well_data = well_data[name]
            log_values = np.array(test_well_data[0])
            facies_values =  np.array(test_well_data[1])
            for i in range(log_values.shape[0]):
                toAppend = np.concatenate((log_values[i:i+1, :], np.asarray(wellID).reshape(1,1)), axis=1)
                chunks.append(toAppend)
                chunks_facies.append(facies_values[i])
        else:
            test_well_data = well_data[name]
            log_values = np.array(test_well_data[0])
            for i in range(log_values.shape[0]):
                toAppend = np.concatenate((log_values[i:i+1, :], np.asarray(wellID).reshape(1,1)), axis=1)
                chunks_test.append(toAppend)

        wellID = wellID + 1.0

    chunks_facies = np.array(chunks_facies, dtype=np.int32)-1
    X_ = np.array(chunks)
    X = np.zeros((len(X_),len(X_[0][0]) * len(X_[0])))
    for i in range(len(X_)):
        X[i,:] = X_[i].flatten()

    X_test = np.array(chunks_test)
    X_test_out = np.zeros((len(X_test),len(X_test[0][0]) * len(X_test[0])))
    for i in range(len(X_test)):
        X_test_out[i,:] = X_test[i].flatten()
    y = np_utils.to_categorical(chunks_facies)
    return X, y, X_test_out

X_train_in, y_train, X_test_in = preprocess(data_out)


I am working on the file "StoDIG/Facies_classification_StoDIG.ipynb".
The first task is:
We train the CNN and evaluate it on precision/recall.
Can you write Python code for it?

# Load the model
t0 = time.time()
model_dnn = dnn_model()
model_dnn.summary()
t1 = time.time()
print("Load time = %d" % (t1-t0) )

#Train model
t0 = time.time()
model_dnn.fit(X_train, y_train, batch_size=n_per_batch, nb_epoch=epochs, verbose=0)
t1 = time.time()
print("Train time = %d seconds" % (t1-t0) )

# Predict Values on Training set
t0 = time.time()
y_predicted = model_dnn.predict( X_train , batch_size=n_per_batch, verbose=2)
t1 = time.time()
print("Test time = %d seconds" % (t1-t0) )

# Print Report

# Format output [0 - 8 ]
y_ = np.zeros((len(y_train),1))
for i in range(len(y_train)):
    y_[i] = np.argmax(y_train[i])

y_predicted_ = np.zeros((len(y_predicted), 1))
for i in range(len(y_predicted)):
    y_predicted_[i] = np.argmax( y_predicted[i] )

# Confusion Matrix
conf = confusion_matrix(y_, y_predicted_)

def accuracy(conf):
    total_correct = 0.
    nb_classes = conf.shape[0]
    for i in np.arange(0,nb_classes):
        total_correct += conf[i][i]
    acc = total_correct/sum(sum(conf))
    return acc

adjacent_facies = np.array([[1], [0,2], [1], [4], [3,5], [4,6,7], [5,7], [5,6,8], [6,7]])
facies_labels = ['SS', 'CSiS', 'FSiS', 'SiSh', 'MS','WS', 'D','PS', 'BS']

def accuracy_adjacent(conf, adjacent_facies):
    nb_classes = conf.shape[0]
    total_correct = 0.
    for i in np.arange(0,nb_classes):
        total_correct += conf[i][i]
        for j in adjacent_facies[i]:
            total_correct += conf[i][j]
    return total_correct / sum(sum(conf))

# Print Results
print ("\nModel Report")
print ("-Accuracy: %.6f" % ( accuracy(conf) ))
print ("-Adjacent Accuracy: %.6f" % ( accuracy_adjacent(conf, adjacent_facies) ))
print ("\nConfusion Matrix")
display_cm(conf, facies_labels, display_metrics=True, hide_zeros=True)


I am working on the file "StoDIG/Facies_classification_StoDIG_2.ipynb".
The first task is:
Data Augmentation

We expand the input data to be acted on by the convolutional layer.
Can you write Python code for it?

conv_domain = 11

# Reproducibility
np.random.seed(7)
# Load data

def expand_dims(input):
    r = int((conv_domain-1)/2)
    l = input.shape[0]
    n_input_vars = input.shape[1]
    output = np.zeros((l, conv_domain, n_input_vars))
    for i in range(l):
        for j in range(conv_domain):
            for k in range(n_input_vars):
                output[i,j,k] = input[min(i+j-r,l-1),k]
    return output

X_train = np.empty((0,conv_domain,8), dtype=float)
X_test  = np.empty((0,conv_domain,8), dtype=float)

wellId = 0.0
for i in range(10):
    X_train_subset = X_train_in[X_train_in[:, 8] == wellId][:,0:8]
    X_train_subset = expand_dims(X_train_subset)
    X_train = np.concatenate((X_train,X_train_subset),axis=0)
    wellId = wellId + 1.0

for i in range(2):
    X_test_subset = X_test_in[X_test_in[:, 8] == wellId][:,0:8]
    X_test_subset = expand_dims(X_test_subset)
    X_test = np.concatenate((X_test,X_test_subset),axis=0)
    wellId = wellId + 1.0

print(X_train.shape)
print(X_test.shape)

# Obtain labels
y_labels = np.zeros((len(y_train),1))
for i in range(len(y_train)):
    y_labels[i] = np.argmax(y_train[i])
y_labels = y_labels.astype(int)