I am working on the file "HDDM/Within_models_from_tutorial.ipynb".
The first task is:
Working with real data
Now let's move to the real data
Can you write Python code for it?

# load data
df1a = hddm.load_csv('df1a.v.hddm_stim.csv')
df = df1a

# randomly chose 10 participants from the dataset
df_subj = df['subj_idx'].unique()
random.seed(10)
df_test_list = []
for i in range(10):
    pos = random.randint(0, (len(df_subj)-1))
    df_test_list.append(df_subj[pos])

df_test = df[df['subj_idx'].isin(df_test_list)]

def z_link_func(x, data=mydata):
    stim = (dmatrix('0 + C(s, [[1], [-1]])',
                    {'s':data.stimulus.loc[x.index]},
                    return_type="dataframe")
           )
    return 1/(1+np.exp(-np.multiply(x.to_frame(), stim)))

z_reg = {'model': 'z ~ 1 + C(condition)', 'link_func': z_link_func}
v_reg = {'model': 'v ~ 1 + C(condition)', 'link_func': lambda x:x}
a_reg = {'model': 'a ~ 1 + C(condition)', 'link_func': lambda x:x}

reg_descr = [z_reg, v_reg]

m_reg = hddm.HDDMRegressor(mydata, reg_descr, include='z')

m_reg.sample(5000, burn=200, dbname='within_effect.db', db='pickle')

m_within_subj = hddm.HDDMRegressor(df1a, "v ~ C(match, val, Treatment(’Mismatch.Neutral’))")


I am working on the file "HDDM/hddm_bwsubs_simulation.ipynb".
The first task is:
Plot the correlations
Can you write Python code for it?

colors = np.where(data_group["condition"]== 'level1','r','b')
fig = plt.figure()
fig.set_size_inches(5, 4)
plt.scatter(data_group.z_x,data_group.a_population, c=colors) # predicted versus observed a
plt.xlabel('z_x')
plt.ylabel('simulated a')


I am working on the file "HDDM/hddm_bwsubs_simulation.ipynb".
The first task is:
Recovery of parameters
Can you write Python code for it?

# a_reg_within = {'model': 'a ~ 1+x + C(condition)', 'link_func': lambda x: x}
# for including and estimating within subject effects of  condition

# try main effect of within and interaction between within and between-subject
a_reg = {'model': 'a ~ 1 + z_x:condition + C(condition)', 'link_func': lambda x: x}

v_reg = {'model': 'v ~ 1 + z_x:condition + C(condition)', 'link_func': lambda x: x}
reg_comb = [a_reg, v_reg]
# m_reg = hddm.HDDMRegressor(data_group, reg_comb, group_only_regressors=['true'])

# Do not estimate individual subject parameters for all regressors.
m_reg = hddm.HDDMRegressor(data_group, a_reg, group_only_regressors=['true'])
m_reg.find_starting_values()
m_reg.sample(2000, burn=500, dbname='a_bwsubs_int.db', db='pickle')
m_reg.save('a_bwsubs_int')

m_reg.print_stats()  # check values of reg coefficients against the generated ones

m_reg.plot_posteriors()

#load from file and examine recovery
m_reg = hddm.load('a_bwsubs_int')
data_group = pd.read_csv('data_group_int.csv')

#look at correlation of recovered parameter with original
subjdf = data_group.groupby(['subj_idx','condition']).first().reset_index()

## check for residual correlation with x
a_int_recovered =[]

# obtain mean a intercept parameter for subjects
from scipy import stats
for i in range(0,(1+max(x_range))*subjs_per_bin):
    a='a_Intercept_subj.'
    a+=str(i)
    a+='.0'
    p1=m_reg.nodes_db.node[a]
    a_int_recovered.append(p1.trace().mean())
    a_int_recovered.append(p1.trace().mean()) # repeat the intercept one more time for the level2

#obtain mean a regression weight for z_x
#a_x_recovered = m_reg.nodes_db.node['a_z_x'].trace().mean()

#obtain mean a regression weight for z_x  (slope)

a_x_recovered_1 = m_reg.nodes_db.node['a_z_x:condition[level1]'].trace().mean()
a_x_recovered_2 = m_reg.nodes_db.node['a_z_x:condition[level2]'].trace().mean()

#compute predicted a parameter as a function of subject intercept and between-subjects regressor
# subjdf['a_int_recovered'] =...


I am working on the file "HDDM/hddm_bwsubs_simulation.ipynb".
The first task is:
load from file and examine recovery
Can you write Python code for it?

#load from file and examine recovery
m_reg = hddm.load('a_bwsubs_model_t200')
data_group = pd.read_csv('data_group.csv')

#look at correlation of recovered parameter with original
subjdf = data_group.groupby('subj_idx').first().reset_index()

## check for residual correlation with x
a_int_recovered =[]

#obtain mean a intercept parameter for subjects
from scipy import stats
for i in range(0,(1+max(x_range))*subjs_per_bin):
    a='a_Intercept_subj.'
    a+=str(i)
    a+='.0'
    p1=m_reg.nodes_db.node[a]
    a_int_recovered.append(p1.trace().mean())

#obtain mean a regression weight for z_x
a_x_recovered = m_reg.nodes_db.node['a_z_x'].trace().mean()

#compute predicted a parameter as a function of subject intercept and between-subjects regressor
subjdf.apred = a_int_recovered + a_x_recovered * subjdf.z_x

# correlation of recovered a with population a. r = .97 here: good!
fig = plt.figure()
fig.set_size_inches(5, 4)
plt.scatter(subjdf.apred,subjdf.a_population) # predicted versus observed a
plt.xlabel('recovered a')
plt.ylabel('simulated a')
plt.savefig('correlation of simulated and recovered a params.png', dpi=300, format='png')
print('Pearson correlation between a_x and x')
print(stats.pearsonr(subjdf.apred,subjdf.a_population)) # correlation between predicted a value and population a value

# correlation of subject intercept with between-subjects regressor
# should be zero correlation if entirely accounted for by x regressor -
# this correlation should instead be positive if x is removed from the model fit
fig = plt.figure()
fig.set_size_inches(5, 4)
plt.scatter(subjdf.x,a_int_recovered)
plt.xlabel('between subjs regressor x')
plt.ylabel('recovered subject a intercept')
plt.show()
plt.savefig('residual correlation between subj intercept and x.png', dpi=300, format='png')
print('Pearson correlation between a_x and x')
print(stats.pearsonr(a_int_recovered,subjdf.x))


I am working on the file "HEVRP_Gurobi.ipynb".
The first task is:
$2^{da}$ restricción
$\sum_{i=1}^n x_{i,j}^k - \sum_{l=1}^nx_{j,l}^k = 0  \hspace{1cm} j=1,...,n; \hspace{0.2cm} k=1,...,T$
Can you write Python code for it?

m.addConstrs( (x_1.sum('*',j) - x_1.sum(j,'*') == 0 for j in id_nodos), "2da_1")
m.addConstrs( (x_2.sum('*',j) - x_2.sum(j,'*') == 0 for j in id_nodos), "2da_2")
m.addConstrs( (x_3.sum('*',j) - x_3.sum(j,'*') == 0 for j in id_nodos), "2da_3")


I am working on the file "HEVRP_Gurobi.ipynb".
The first task is:
$3^{ra}$ restricción
$\sum_{i=1}^n u_{i,j} - \sum_{l=1}^n u_{j,l}= q_j  \hspace{1cm} j=1,...,n$
Can you write Python code for it?

m.addConstrs( (u.sum('*',j) - u.sum(j,'*') == q[j] for j in id_clientes), "3ra")


I am working on the file "HEVRP_Gurobi.ipynb".
The first task is:
$4^{ta}$ restricción
$u_{n-1,j}\leq \sum_{k=1}^T Q_k \cdot x_{n-1,j}^k   \hspace{1cm} j=1,...,n$
Can you write Python code for it?

m.addConstrs( (u[n_nodos-1,j] <= capacidades[0]*x_1[n_nodos-1,j] + capacidades[1]*x_2[n_nodos-1,j]+ capacidades[2]*x_3[n_nodos-1,j] for j in id_clientes), "4ta")


I am working on the file "HEVRP_Gurobi.ipynb".
The first task is:
Especificamos las restricciones

$1^{ra}$ restricción
$\sum_{k=1}^T \sum_{i=1}^n x_{i,j}^k=1\hspace{1cm} j=1,...,n$
Can you write Python code for it?

m.addConstrs(  (x_1.sum('*',j) + x_2.sum('*',j) +x_3.sum('*',j)  == 1 for j in id_clientes), "1ra")


I am working on the file "HEVRP_Gurobi.ipynb".
The first task is:
Especificamos modelo

La formulación según detallado en (Gheysen, Golden & Assad, 1984) es la siguiente:
Se consideran $n$ clientes donde el depósito se encuentra en la locación $n-1$ e indexadas con $i = 1,...., n$
Los siguientes entradas se asumen disponibles:
T = numeros de vehiculos
$Q_k$ = capacidad del vehículo $k$; ($Q_1 < Q_2 <....<Q_T$)
$q_j$ = demanda del cliente $j$
$d_{ij}$ = distancia de viaje entre cliente $i$ y cliente $j$.
Creamos el modelo de optimización de gurobipy llamado "VRP" y la almacenamos en la variable m.
Can you write Python code for it?


I am working on the file "HEVRP_Gurobi.ipynb".
The first task is:
Mostramos resultados
Can you write Python code for it?

for v in m.getVars():
    if v.x > 0 :
        print('%s %g' % (v.varName, v.x))


I am working on the file "HEVRP_Gurobi.ipynb".
The first task is:
Opcional: Podemos especificar los cores que queremos usar para resolver el problema
Can you write Python code for it?


I am working on the file "HEVRP_Gurobi.ipynb".
The first task is:
Resolvemos
Can you write Python code for it?


I am working on the file "HG-GAN.ipynb".
The first task is:
Testing function: reducing_data [works]
Can you write Python code for it?

tmp = reducing_data(neuron_list[0:3], pruning_number=10)


I am working on the file "HG-GAN.ipynb".
The first task is:
Testing function: separate [works]
Can you write Python code for it?

geo, morph = separate(tmp)

print morph[0]
print morph[1]
print morph[2]

geo[2][0:3,9]


I am working on the file "HIV_Age_Advancement.ipynb".
The first task is:
All HIV Combined
Can you write Python code for it?

#Do not import
fig, ax = subplots(figsize=(5,4))
plot_regression(a2, p2, ax=ax)
fig.tight_layout()


I am working on the file "HIV_Age_Advancement.ipynb".
The first task is:
Association of age residuals with HIV status.
Can you write Python code for it?


I am working on the file "HIV_Age_Advancement.ipynb".
The first task is:
Comparing short vs. long term infected HIV+ subjects
Can you write Python code for it?

kruskal_pandas(duration.ix[ti(hiv == 'HIV+')], (pred_c - age))


I am working on the file "HIV_Age_Advancement.ipynb".
The first task is:
Dropping 6 patients with Low Concordance

2/6 have bad probes on the sites used in the models
Can you write Python code for it?

diff = ((pred_adj - pred_horvath_adj) / ((pred_adj + pred_horvath_adj) * .5)).abs()
diff = diff.groupby(level=0).first()
diff.name = 'Absolute difference in models'

#Do not import
fig, axs = subplots(1,2, figsize=(10,4))
diff.hist(ax=axs[0])
axs[0].set_xlabel(diff.name)
violin_plot_pandas(duration, diff, ax=axs[1])
fig.tight_layout()

in_model = detection_p[detection_p.level_0.isin(hannum_model.index.union(horvath_model.index))]
(diff).ix[in_model.Sample_Name.unique()].dropna().order()

pt = ti(diff < .2)

o = ti(diff > .2)
o

#Do not import
fig, axs = subplots(1,2, figsize=(9,4), sharex=True, sharey=True)
plot_regression(pred_adj.ix[ti(hiv=='HIV-')].ix[pt], pred_horvath_adj, ax=axs[0])
plot_regression(pred_adj.ix[ti(hiv=='HIV+')].ix[pt], pred_horvath_adj, ax=axs[1])
series_scatter(pred_adj.ix[ti(hiv=='HIV-')].ix[o], pred_horvath_adj, ax=axs[1],
               color=colors[0], ann=None)
series_scatter(pred_adj.ix[ti(hiv=='HIV+')].ix[o], pred_horvath_adj, ax=axs[0],
               color=colors[0], ann=None)

fig.tight_layout()
fig.savefig(FIGDIR + 'hiv_model_agreement_filter.png', dpi=300)

#Do not import
v = detection_p.groupby('Sample_Name').size()
v.name = '# probes with poor detection'
series_scatter(diff, v)

detection_p[detection_p.Sample_Name.isin(o)].groupby('Sample_Name').size().order()

#pt = ti(((pred_adj - pred_horvath_adj)).abs().dropna() < 10)


I am working on the file "HIV_Age_Advancement.ipynb".
The first task is:
Hannum Model
Can you write Python code for it?

pred = run_hannum_model(df_hiv)

#Do not import
model_plot(pred)

get_error(age, pred, denominator='x', groups=hiv)


I am working on the file "HIV_Age_Advancement.ipynb".
The first task is:
Horvath Model
Can you write Python code for it?

pred_horvath = run_horvath_model(df_hiv_n)

#Do not import
model_plot(pred_horvath)

reg = linear_regression(age, pred_horvath.ix[ti(duration=='Control')])
pred_horvath_adj = (pred_horvath - reg['intercept']) / reg['slope']

#Do not import
model_plot(pred_horvath_adj)

get_error(age, pred_horvath_adj, denominator='x', groups=hiv)


I am working on the file "HIV_Age_Advancement.ipynb".
The first task is:
Looking at Predicted Time of Onset

The idea of age acceleration, only really makes sense in this context as a person should age normally until the onset of the disease
Can you write Python code for it?

p2 = (pred_c - (clinical.age - (clinical['estimated duration hiv (months)'] / 12.)))
p2.name = 'Biological Time Since Onset'
a2 = (clinical['estimated duration hiv (months)'] / 12.)
a2.name = 'Actual Time Since Onset'
a2 = a2.ix[pt].dropna()
p2 = p2.ix[pt].dropna()


I am working on the file "HIV_Age_Advancement.ipynb".
The first task is:
Mean age resisuals broken down by HIV status. Not that HIV- is ~0 by construction as the full dataset is normalized by the controls.
Can you write Python code for it?


I am working on the file "HIV_Age_Advancement.ipynb".
The first task is:
Merging Hannum and Horvath
Without filter.
Can you write Python code for it?

pred_c = (pred_horvath_adj + pred_adj) / 2
pred_c = pred_c.ix[duration.index]
pred_c.name = 'Predicted Age (Combined)'
reg = linear_regression(age, pred_c.ix[ti(duration=='Control')])
pred_c = (pred_c - reg['intercept']) / reg['slope']

#Do not import
model_plot(pred_c)

get_error(age, pred_c, denominator='x', groups=hiv)


I am working on the file "HIV_Age_Advancement.ipynb".
The first task is:
Preforming a linear adjustment on the control data.
Can you write Python code for it?

reg = linear_regression(age, pred.ix[ti(duration=='Control')])
pred_adj = (pred - reg['intercept']) / reg['slope']

#Do not import
model_plot(pred_adj)

get_error(age, pred_adj, denominator='x', groups=hiv)


I am working on the file "HIV_Age_Advancement.ipynb".
The first task is:
Quality Control

Look at detection p-value and concordance of the models
Can you write Python code for it?

#Do not import
fig, axs = subplots(1,2, figsize=(9,4), sharex=True, sharey=True)
plot_regression(pred_adj.ix[ti(hiv=='HIV-')], pred_horvath_adj, ax=axs[0])
plot_regression(pred_adj.ix[ti(hiv=='HIV+')], pred_horvath_adj, ax=axs[1])
fig.tight_layout()

get_error(pred_adj, pred_horvath_adj, groups=hiv)


I am working on the file "HIV_Age_Advancement.ipynb".
The first task is:
Short and Long Duration Split
Can you write Python code for it?

#Do not import
fig, axs = subplots(1,2, figsize=(10,4))
plot_regression(a2.ix[ti(duration=='HIV Short')], p2, ax=axs[0])
axs[0].set_xbound(0,5)
plot_regression(a2.ix[ti(duration=='HIV Long')], p2, ax=axs[1])


I am working on the file "HIV_Age_Advancement.ipynb".
The first task is:
With QC filter
Can you write Python code for it?

pred_c = (pred_horvath_adj + pred_adj) / 2
pred_c = pred_c.ix[duration.index].ix[pt]
pred_c.name = 'Predicted Age (Combined)'
reg = linear_regression(age, pred_c.ix[ti(duration=='Control')])
pred_c = (pred_c - reg['intercept']) / reg['slope']

#Do not import
model_plot(pred_c)

get_error(age, pred_c, denominator='x', groups=hiv)

#Do not import
fig, axs = subplots(1,3, figsize=(15,4), sharex=True, sharey=True)
plot_regression(age, pred_c.ix[ti(duration=='Control')], ax=axs[0])
plot_regression(age, pred_c.ix[ti(duration=='HIV Short')], ax=axs[1])
plot_regression(age, pred_c.ix[ti(duration=='HIV Long')], ax=axs[2])
axs[2].set_xlim(20,75)
axs[2].set_ylim(20,75);

get_error(age, pred_c, denominator='x', groups=duration)


I am working on the file "HIV_Age_Advancement_Confounders.ipynb".
The first task is:
Look at Confounders
Data from the labs
Can you write Python code for it?

l2 = (labs.ix[:, labs.dtypes.isin([dtype('int64'), dtype('float64')])]
      .dropna(1, how='all'))
l3 = labs.ix[:, ti(labs.apply(lambda s: len(s.unique()), axis=0) < 6)]

spearman_pandas(residual, np.log2(l2['CD4/CD8 ratio']))

pearson_pandas(residual, np.log2(l2['CD4/CD8 ratio']))

spearman_pandas(residual.ix[ti(duration=='HIV Long')],
                np.log2(l2['CD4/CD8 ratio']))

spearman_pandas(resid_adj.ix[ti(duration=='HIV Short')],
                np.log2(l2['CD4/CD8 ratio']))

spearman_pandas(resid_adj, np.log2(l2['CD4/CD8 ratio']))

series_scatter(residual, np.log2(l2['CD4/CD8 ratio']))

l2 = (labs.ix[:, labs.dtypes.isin([dtype('int64'), dtype('float64')])]
      .dropna(1, how='all'))
l3 = labs.ix[:, ti(labs.apply(lambda s: len(s.unique()), axis=0) < 6)]
keepers = labs.index.difference(['RG065','RG175','RG279','RA182','RM285'])
keepers = keepers.intersection(duration.index)
l2 = l2.ix[keepers]
l3 = l3.ix[keepers]

duration.name = 'duration'

violin_plot_pandas(combine(labs['LLQ PLASMA'] == '>LLQ', duration=='HIV Long'),
                   age, order=['neither','duration','both','LLQ PLASMA'])

violin_plot_pandas(combine(labs['LLQ PLASMA'] == '>LLQ', duration=='HIV Long'),
                   age_advancement, order=['neither','duration','both','LLQ PLASMA'])

series_scatter(np.log(labs['rnvalue PLASMA'][labs['LLQ PLASMA'] == '>LLQ']),
               age_advancement)

screen_feature(age_advancement, pearson_pandas, l2.T, align=False).head()

bins = np.floor(age_advancement / 5.)
bins = bins.clip(-1,2)
spearman_pandas(bins, l2.MCV)

fig, axs = subplots(1,2, figsize=(6,4))
bins = np.floor(age_advancement / 5.)
bins = bins.clip(-1,2).map({-1: '< 0', 0:'0-5', 1:'5+', 2:'5+'})
box_plot_pandas(bins, l2.MCV, order=['< 0','0-5','5+'], ax=axs[0])
box_plot_pandas(bins, l2['age'], order=['< 0','0-5','5+'], ax=axs[1])
for ax in axs:
    prettify_ax(ax)
fig.tight_layout()

fig, ax = subplots(figsize=(5,4))
series_scatter(age_advancement, l2.MCV, ax=ax, color=colors[3],
              ...


I am working on the file "HIV_Age_Advancement_Confounders.ipynb".
The first task is:
Looking at Predicted Time of Onset

The idea of age acceleration, only really makes sense in this context as a person should age normally until the onset of the disease
Can you write Python code for it?

fig, ax = subplots(figsize=(5,4))
plot_regression(a2, p2, ax=ax)
fig.tight_layout()


I am working on the file "HIV_Age_Advancement_Confounders.ipynb".
The first task is:
Looking at residuals of model fit with cell composition

This is very conservative
Can you write Python code for it?

hiv = (duration != 'Control').astype(float)
hiv.name = 'HIV'
age.name = 'chron_age'
pred_c.name = 'bio_age'
df = process_factors([residual, hiv, pred_c, age, cell_counts.NK,
                      cell_counts.CD4T, cell_counts.CD8T,
                      cell_counts.Bcell, cell_counts.Mono,
                      cell_counts.Gran])
fmla = robjects.Formula('bio_age ~ chron_age + NK + CD4T + CD8T + '
                        'Bcell + Mono + Gran')
m = robjects.r.lm(fmla, df)
s = robjects.r.summary(m)
print '\n\n'.join(str(s).split('\n\n')[-3:])

1.4299 / 2.3176

hiv = (duration != 'Control').astype(float)
hiv.name = 'HIV'
age.name = 'chron_age'
pred_c.name = 'bio_age'
df = process_factors([residual, hiv, pred_c, age, cell_counts.NK,
                      cell_counts.CD4T, cell_counts.CD8T,
                      cell_counts.Bcell, cell_counts.Mono,
                      cell_counts.Gran])
fmla = robjects.Formula('residual ~ chron_age + NK + CD4T + CD8T + '
                        'Bcell + Mono + Gran')
m = robjects.r.lm(fmla, df)
s = robjects.r.summary(m)
print '\n\n'.join(str(s).split('\n\n')[-3:])

rmse = lambda v: (v ** 2).mean() ** .5

v = robjects.r.residuals(m)
r2 = pd.Series(pandas2ri.ri2py(v), index=list(v.names[0]))
r2.name = 'residual'
hiv = (duration != 'Control').astype(float)
hiv.name = 'HIV'
df = process_factors([r2, hiv, pred_c, cell_counts.NK, cell_counts.CD4T,
                      cell_counts.CD8T, cell_counts.Bcell, cell_counts.Mono,
                      cell_counts.Gran])
fmla = robjects.Formula('residual ~ HIV')
m1 = robjects.r.lm(fmla, df)
s = robjects.r.summary(m1)
print '\n\n'.join(str(s).split('\n\n')[-3:])


I am working on the file "HIV_Age_Advancement_Confounders.ipynb".
The first task is:
Modeling residuals of aging model with HIV and cell composition

Here we have to use the estimated cell counts as we do not have blood work for the controls
Can you write Python code for it?

hiv = (duration != 'Control').astype(float)
hiv.name = 'HIV'
age.name = 'chron_age'
pred_c.name = 'bio_age'

hiv = (duration != 'Control').astype(float)
hiv.name = 'HIV'
df = process_factors([residual, hiv, age, cell_counts.NK, cell_counts.CD4T,
                      cell_counts.CD8T, cell_counts.Bcell, cell_counts.Mono,
                      cell_counts.Gran], standardize=False)
fmla = robjects.Formula('residual ~ chron_age + HIV + NK + CD4T + CD8T + '
                        'Bcell + Mono + Gran')
m = robjects.r.lm(fmla, df)
s = robjects.r.summary(m)
print '\n\n'.join(str(s).split('\n\n')[-3:])

hiv = (duration != 'Control').astype(float)
hiv.name = 'HIV'
df = process_factors([residual, hiv, pred_c, age, cell_counts.NK,
                      cell_counts.CD4T, cell_counts.CD8T,
                      cell_counts.Bcell, cell_counts.Mono,
                      cell_counts.Gran], standardize=False)
fmla = robjects.Formula('residual ~ bio_age + HIV + NK')
m = robjects.r.lm(fmla, df)
s = robjects.r.summary(m)
print '\n\n'.join(str(s).split('\n\n')[-3:])


I am working on the file "HIV_Age_Advancement_Confounders.ipynb".
The first task is:
Multivariate modeling of confounders
Here we are looking at biological age, MCV, and NK cell count.  We contructed a similar model with monocyte count as well but found that it did not add to the model fit.
Can you write Python code for it?

age_adj.name = 'age_advancement'
hiv = (duration != 'Control').astype(float)
hiv.name = 'HIV'

age.name = 'bio_age'
duration_t = clinical['estimated duration hiv (months)'] / 12.
duration.name = 'duration'
monocytes = labs['Monocyte %']
monocytes.name = 'monocytes'

df = process_factors([age_advancement, duration, age, age_at_dx,
                      l2.MCV, l2.MCH, cell_counts.NK, cell_counts.CD4T,
                      monocytes], standardize=True)
fmla = robjects.Formula('age_advancement ~ bio_age + MCV + NK')
m = robjects.r.lm(fmla, df)
s = robjects.r.summary(m)
print '\n\n'.join(str(s).split('\n\n')[-3:])


I am working on the file "HIV_Age_Advancement_Confounders.ipynb".
The first task is:
While we see a significant effect of NK cell concentration with increasing age advancment, this does not seem to be specific to HIV+ patients.
Can you write Python code for it?

fig, ax = subplots(1,1, figsize=(4,3))
rr = cell_counts.NK
k = pred_c.index
hiv = duration != 'Control'

sns.regplot(*match_series(residual.ix[k], rr.ix[ti(hiv==0)]),
            ax=ax, label='HIV+')
sns.regplot(*match_series(residual.ix[k], rr.ix[ti(hiv>0)]),
            ax=ax, label='Control')
prettify_ax(ax)


I am working on the file "HK_Variance_Frames_Running.ipynb".
The first task is:
Set all the operating parameters to the default values:
Can you write Python code for it?

def set_fpe_defaults(fpe):
    "Set the FPE to the default operating parameters and return a list of the default values"
    defaults = {}
    for k in range(len(fpe.ops.address)):
        if fpe.ops.address[k] is None:
            continue
        fpe.ops.address[k].value = fpe.ops.address[k].default
        defaults[fpe.ops.address[k].name] = fpe.ops.address[k].default
    return defaults
set_fpe_defaults(fpe1)


I am working on the file "HLT2-TreesPruning.ipynb".
The first task is:
BBDT-5, 6
Can you write Python code for it?

ef_base_bbdt5 = MatrixNetSkyGridClassifier(train_features=variables, user_name='antares',
                                          connection='skygrid',
                                          iterations=5000, sync=False, intervals=5)
ef_base_bbdt5.fit(train, train['signal'])

ef_base_bbdt6 = MatrixNetSkyGridClassifier(train_features=variables, user_name='antares',
                                          connection='skygrid',
                                          iterations=5000, sync=False, intervals=6)
ef_base_bbdt6.fit(train, train['signal'])


I am working on the file "HLT2-TreesPruning.ipynb".
The first task is:
Base model with 5000 trees
Can you write Python code for it?

ef_base = MatrixNetSkyGridClassifier(train_features=variables, user_name='antares',
                                     connection='skygrid',
                                     iterations=5000, sync=False)
ef_base.fit(train, train['signal'])


I am working on the file "HLT2-TreesPruning.ipynb".
The first task is:
Calculate thresholds on classifiers
Can you write Python code for it?

thresholds = dict()
test_bck = test[test['signal'] == 0]
RATE = [2500., 4000.]
events_pass = dict()
for name, cl in estimators.items():
    prob = cl.predict_proba(test_bck)
    thr, result = calculate_thresholds(test_bck, prob, total_test_bck_events, rates=RATE)
    for rate, val in result.items():
        events_pass['{}-{}'.format(rate, name)] = val[1]
    thresholds[name] = thr
    print name, result


I am working on the file "HLT2-TreesPruning.ipynb".
The first task is:
Classification report using events
Can you write Python code for it?

plots = OrderedDict()
for key, value in estimators.items():
    plots[key] = plot_roc_events(value, test[test['signal'] == 1], test[test['signal'] == 0], key)

bbdt_plots = plots.copy()
bbdt_plots.pop('Prunned MN')
bbdt_plots.pop('Prunned MN + forest')

from rep.plotting import FunctionsPlot
FunctionsPlot(bbdt_plots).plot(new_plot=True, xlim=(0.02, 0.06), ylim=(0.65, 0.82))
plot([1. * events_pass['2500.0-base MN'] / statistic_length(ds_test_bck)['Events']] * 2,
     [0., 1], 'b--', label='rate: 2.5 kHz')
plot([1. * events_pass['4000.0-base MN'] / statistic_length(ds_test_bck)['Events']] * 2,
     [0., 1], 'g--', label='rate: 4. kHz')
lgd = legend(loc='upper center', fontsize=16, bbox_to_anchor=(0.5, 1.3), ncol=3)
title('ROC for events (training decays)', fontsize=20)
xlabel('FRP, background events efficiency', fontsize=20)
ylabel('TPR, signal events efficiency', fontsize=20)

from rep.plotting import FunctionsPlot
FunctionsPlot(plots).plot(new_plot=True, xlim=(0.02, 0.06), ylim=(0.65, 0.82))
plot([1. * events_pass['2500.0-base MN'] / statistic_length(ds_test_bck)['Events']] * 2,
     [0., 1], 'b--', label='rate: 2.5 kHz')
plot([1. * events_pass['4000.0-base MN'] / statistic_length(ds_test_bck)['Events']] * 2,
     [0., 1], 'g--', label='rate: 4. kHz')
lgd = legend(loc='upper center', fontsize=16, bbox_to_anchor=(0.5, 1.4), ncol=3)
title('ROC for events (training decays)', fontsize=20)
xlabel('FRP, background events efficiency', fontsize=20)
ylabel('TPR, signal events efficiency', fontsize=20)


I am working on the file "HLT2-TreesPruning.ipynb".
The first task is:
Counting events and svrs,
that passed L0 and GoodGenB preselection (this data was generated by skim)
Can you write Python code for it?

print 'Signal', statistic_length(signal_data)
print 'Bck', statistic_length(bck_data)

total_bck_events = statistic_length(bck_data)['Events'] + empty_events[bck_train_mode_name]
total_signal_events_by_mode = dict()
for mode in sig_train_modes_names:
    total_signal_events_by_mode[mode] = statistic_length(signal_data[signal_data['mode'] == mode])['Events'] + empty_events[mode]


I am working on the file "HLT2-TreesPruning.ipynb".
The first task is:
Counting events and svrs,
which passed pass_nbody (equivalent Mike's preselections for nbody selection)
Can you write Python code for it?

# hlt2 nbody selection
signal_data = signal_data[(signal_data['pass_nbody'] == 1) & (signal_data['mcor'] <= 10e3)]
bck_data = bck_data[(bck_data['pass_nbody'] == 1) & (bck_data['mcor'] <= 10e3)]

print 'Signal', statistic_length(signal_data)
print 'Bck', statistic_length(bck_data)

total_signal_events_by_mode_presel = dict()
for mode in sig_train_modes_names:
    total_signal_events_by_mode_presel[mode] = statistic_length(signal_data[signal_data['mode'] == mode])['Events']
total_bck_events_presel = statistic_length(bck_data)['Events']


I am working on the file "HLT2-TreesPruning.ipynb".
The first task is:
DIfferent rates
Can you write Python code for it?

thresholds = OrderedDict()
RATE = [2000., 2500., 3000., 3500., 4000.]
for name, cl in estimators.items():
    prob = cl.predict_proba(ds_test_bck)
    thr, result = calculate_thresholds(ds_test_bck, prob, total_test_bck_events, rates=RATE)
    thresholds[name] = thr
    print name, result

train_modes_eff, statistic = result_statistic({'base MN': estimators['base MN']}, sig_train_modes_names,
                                              test[test['signal'] == 1],
                                              thresholds, RATE, total_test_signal_events)

order_rate = OrderedDict()
for j in numpy.argsort([i[1] for i in train_modes_eff.keys()]):
    order_rate[train_modes_eff.keys()[j]] = train_modes_eff.values()[j]

from rep.plotting import BarComparePlot
BarComparePlot(order_rate).plot(new_plot=True, figsize=(18, 6), ylabel='efficiency', fontsize=18)
lgd = legend(bbox_to_anchor=(0.5, 1.2), loc='upper center', ncol=5, fontsize=18)
# plt.savefig('rates.pdf' , format='pdf', bbox_extra_artists=(lgd,), bbox_inches='tight')


I am working on the file "HLT2-TreesPruning.ipynb".
The first task is:
Define all total events in test samples
(which passed just l0 and goodgenB) using also empty events. Suppose that events which didn't pass pass_nboby also were equal randomly divided into training and test samples
Can you write Python code for it?

total_test_bck_events = (total_bck_events - total_bck_events_presel) // 2 + statistic_length(ds_test_bck)['Events']
total_test_signal_events = dict()
for mode in sig_train_modes_names:
    total_not_passed_signal = total_signal_events_by_mode[mode] - total_signal_events_by_mode_presel[mode]
    total_test_signal_events[mode] = total_not_passed_signal // 2 + \
        statistic_length(ds_test_signal[ds_test_signal['mode'] == mode])['Events']

print 'Bck total test events:', total_test_bck_events
'Signal total test events:', total_test_signal_events

import cPickle
if os.path.exists('models/prunned.pkl'):
    with open('models/prunned.pkl', 'r') as file_pr:
        estimators = cPickle.load(file_pr)


I am working on the file "HLT2-TreesPruning.ipynb".
The first task is:
Define variables
Can you write Python code for it?

variables = ["n", "mcor", "chi2", "eta", "fdchi2", "minpt", "nlt16", "ipchi2", "n1trk", "sumpt"]


I am working on the file "HLT2-TreesPruning.ipynb".
The first task is:
Final efficiencies for each mode
Can you write Python code for it?

train_modes_eff, statistic = result_statistic(estimators, sig_train_modes_names,
                                              test[test['signal'] == 1],
                                              thresholds, RATE, total_test_signal_events)

from rep.plotting import BarComparePlot
xticks_labels = ['$B^0 \\to K^*\mu^+\mu^-$', "$B^0 \\to D^+D^-$", "$B^0 \\to D^- \mu^+ \\nu_{\mu}$",
                 '$B^+ \\to \pi^+ K^-K^+$', '$B^0_s \\to \psi(1S) K^+K^-\pi^+\pi^-$', '$B^0_s \\to D_s^-\pi^+$']
for r in RATE:
    new_dict = []
    for key, val in train_modes_eff.iteritems():
        if (key[0] in {'base MN', 'Prunned MN', 'BBDT MN special',
                       'base MN + forest', 'Prunned MN + forest', 'BBDT MN special + forest'}) and r == key[1]:
            new_dict.append((key, val))
    new_dict = dict(new_dict)
    BarComparePlot(new_dict).plot(new_plot=True, figsize=(24, 8), ylabel='efficiency', fontsize=22)
    xticks(3 + 11 * numpy.arange(6), xticks_labels, rotation=0)
    lgd = legend(bbox_to_anchor=(0.5, 1.3), loc='upper center', ncol=2, fontsize=22)
# plt.savefig('hlt2-experiments.pdf' , format='pdf', bbox_extra_artists=(lgd,), bbox_inches='tight')

from rep.plotting import BarComparePlot
for r in RATE:
    new_dict = []
    for key, val in train_modes_eff.iteritems():
        if r == key[1]:
            new_dict.append((key, val))
    new_dict = dict(new_dict)
    BarComparePlot(new_dict).plot(new_plot=True, figsize=(24, 8), ylabel='efficiency', fontsize=22)
    lgd = legend(bbox_to_anchor=(0.5, 1.3), loc='upper center', ncol=2, fontsize=22)
# plt.savefig('hlt2-experiments.pdf' , format='pdf', bbox_extra_artists=(lgd,), bbox_inches='tight')


I am working on the file "HLT2-TreesPruning.ipynb".
The first task is:
Matrixnet training
Can you write Python code for it?

from rep_ef.estimators import MatrixNetSkyGridClassifier


I am working on the file "HLT2-TreesPruning.ipynb".
The first task is:
Minimize log_loss
он же BinomialDeviance
Can you write Python code for it?


I am working on the file "HLT2-TreesPruning.ipynb".
The first task is:
Prepare train/test splitting
Divide events which passed alll preselections into two equal parts randomly
Can you write Python code for it?

ds_train_signal, ds_train_bck, ds_test_signal, ds_test_bck = prepare_data(signal_data, bck_data, 'unique')


I am working on the file "HLT2-TreesPruning.ipynb".
The first task is:
Pruning
Can you write Python code for it?

from rep.data import LabeledDataStorage
from rep.report import ClassificationReport
report = ClassificationReport({'base': ef_base}, LabeledDataStorage(test, test['signal']))

report.roc()


I am working on the file "HLT2-TreesPruning.ipynb".
The first task is:
test: counting events and svrs
Can you write Python code for it?

print 'Signal', statistic_length(ds_test_signal)
print 'Bck', statistic_length(ds_test_bck)

test = pandas.concat([ds_test_bck, ds_test_signal])


I am working on the file "HLT2-TreesPruning.ipynb".
The first task is:
train: counting events and svrs
Can you write Python code for it?

print 'Signal', statistic_length(ds_train_signal)
print 'Bck', statistic_length(ds_train_bck)

train = pandas.concat([ds_train_bck, ds_train_signal])


I am working on the file "HMC.ipynb".
The first task is:
Inference
As we said earlier, we will resort to MCMA using Hamiltonian Monte Carlo. To give a bit of context for this method following quote from the reference maybe useful:

Hamiltonian dynamics has a physical interpretation that can provide useful intuitions. In
two dimensions, we can visualize the dynamics as that of a frictionless puck that slides over a surface of varying height. The state of this system consists of the position of the puck, given by a 2D vector $q$, and the momentum of the puck (its mass times its velocity), given by a 2D vector $p$. The potential energy, $U(q)$, of the puck is proportional to the height of the surface at its current position, and its kinetic energy, $K(p)$, is equal to $|p|2/(2m)$, where m is the mass of the puck. On a level part of the surface, the puck moves at a constant velocity, equal to $p/m$. If it encounters a rising slope, the puck’s momentum allows it to continue, with its kinetic energy decreasing and its potential energy increasing, until the kinetic energy (and hence p) is zero, at which point it will slide back down (with kinetic energy increasing and potential energy decreasing).
In non-physical MCMC applications of Hamiltonian dynamics, the position will correspond
to the variables of interest. The potential energy will be minus the log of the probability
density for these variables. Momentum variables, one for each position variable, will
be introduced artificially.

Following the paper we learn that there are two quantities that we need for sampling using this method, 1) negative log likelihood $L$ and 2) derivative of negative of log likelihood with respect to parameter of interest, here $\mu$ and $\Sigma$, denoted by $\frac{\partial L}{\partial \mu}$ and $\frac{\partial L}{\partial \Sigma}$ respectively.
Mean
In this section we derive the equations for inference of the mean, assuming covariance matrix is known. Based on earlier explanation we can derive the two quantities of interest using following formulas:
\begin{align}
  L &= \sum_i^N ...
Can you write Python code for it?

def logEnergy_mu(X,mu, cov):
    cov = np.reshape(cov, (X.shape[1],X.shape[1]))
    cov_inv = np.linalg.inv(cov)
    sum_ = 0   #0.5*(mu).T.dot(mu) uncomment this to add a normal prior.
    _, logdet = np.linalg.slogdet(cov)
    sum_ += X.shape[0]*logdet + ((X.T-mu).T.dot(cov_inv)*(X.T-mu).T).sum()
    return sum_

def grad_logEnergy_mu(X,mu, cov):
    cov = np.reshape(cov, (X.shape[1],X.shape[1]))
    sum_ = 0#-mu
    cov_inv = np.linalg.inv(cov)
    sum_ += -np.dot(cov_inv,X.T-mu).sum(axis=1, keepdims = True)
    return sum_

def p_sampler_mu():
    return np.random.normal(0,1, size= (X.shape[1],1))

U_mu = lambda mu: logEnergy_mu(samples, mu, cov)

U_der_mu = lambda mu: grad_logEnergy_mu(samples, mu, cov)

current_q = np.array([[-1.0],[-3.0]])
eps = 0.001
sample_gen = 2000
hist_val = []
hist_prob = []
hist_accept = []
for i in range(sample_gen):
    current_q, prob, accept = hmc.hmc(U_mu, U_der_mu, eps, L = 30, current_q = current_q, p_sampler=p_sampler_mu )
    hist_val.append(current_q)
    hist_prob.append(prob[0])
    hist_accept.append(accept)
    if i % 500 == 0:
        print(i)
        print(prob[0], accept)
        print(current_q)
hist_prob = np.concatenate(hist_prob)
hist_prob[hist_prob>1] = 1
hist_accept = np.array(hist_accept)
hist_val = np.concatenate(hist_val,axis=1).T

import matplotlib.gridspec as gridspec
fig = plt.figure(figsize=(6*3,6))
gs = gridspec.GridSpec(2, 1)
ax1 = fig.add_subplot(gs[0])
ax1.plot(np.abs(hist_val[:,0]-mu_[0].flatten()),alpha=0.7,label="$\mu_1$",linewidth=2)
ax1.plot(np.abs(hist_val[:,1]-mu_[1].flatten()),alpha=0.7,label="$\mu_2$",linewidth=2)
ax1.set_ylabel("absolute error")
ax1.set_xlabel("iteration")
ax1.set_yscale("log")
ax1.legend(loc=1,ncol=2)

ax2 = fig.add_subplot(gs[1],sharex=ax1)
ax2.plot(hist_prob,linewidth=2)
ax2.set_ylabel("P(accept)")
ax2.set_xlabel("iteration")

#plt.show()


I am working on the file "HMC.ipynb".
The first task is:
Putting everything together
Gibbs sampling style
In this section we use inference in previous sections to infere both parameters $\mu$ and $\Sigma$ simultaneously. In our first approach we alternate between parameters assuming th eother is fixed.
Can you write Python code for it?

current_mu = np.array([[-1.0],[3.0]])
current_cov = np.array([[0.1,.0],[0.0,1.0]]).reshape((4,1))
eps = 0.001
sample_gen = 2000
hist_val_mu = []
hist_prob_mu = []
hist_accept_mu = []
hist_val_cov = []
hist_prob_cov = []
hist_accept_cov = []

for i in range(sample_gen):
    U_mu = lambda mu: logEnergy_mu(samples, mu, current_cov)
    U_der_mu = lambda mu: grad_logEnergy_mu(samples, mu, current_cov)
    U_cov = lambda cov: logEnergy_cov(samples, current_mu, cov)
    U_der_cov = lambda cov: grad_logEnergy_cov(samples, current_mu, cov)

    current_mu, prob_mu, accept_mu = hmc.hmc(U_mu, U_der_mu, eps, L = 30, current_q = current_mu,p_sampler= p_sampler_mu )
    hist_val_mu.append(current_mu)
    hist_prob_mu.append(prob_mu[0])
    hist_accept_mu.append(accept_mu)
    current_cov, prob_cov, accept_cov = hmc.hmc(U_cov, U_der_cov, eps, L = 30, current_q = current_cov,p_sampler= p_sampler_cov )
    hist_val_cov.append(current_cov)
    hist_prob_cov.append(prob_cov[0])
    hist_accept_cov.append(accept_cov)
    if i % 500 == 0:
        print(i)
        print("mu:",prob_mu[0], accept_mu,current_mu)
        print("cov:",prob_cov[0], accept_cov,current_cov)
hist_prob_mu = np.concatenate(hist_prob_mu)
hist_prob_mu[hist_prob_mu>1] = 1
hist_accept_mu = np.array(hist_accept_mu)
hist_val_mu = np.concatenate(hist_val_mu,axis=1).T

hist_prob_cov = np.concatenate(hist_prob_cov)
hist_prob_cov[hist_prob_cov>1] = 1
hist_accept_cov = np.array(hist_accept_cov)
hist_val_cov = np.concatenate(hist_val_cov,axis=1).T


I am working on the file "HN.ipynb".
The first task is:
Fitting a TF-IDF matrix
See the documentation for Tf-idf term weighting for more details. The Wikipedia article on TF-IDF is also a good source.
Can you write Python code for it?

from sklearn.feature_extraction.text import TfidfVectorizer

vectoriser = TfidfVectorizer(max_df=0.5, min_df=1, stop_words='english', use_idf=True)
tfidf_matrix = vectoriser.fit_transform(hn['title'])
feature_names = vectoriser.get_feature_names()


I am working on the file "HN.ipynb".
The first task is:
Load Word2Vec model
In this case we are loading a 300 dimension word2vec model that has been pre-trained on the Google News corpus.
This means if we ask the model for a specific word (and the word is known to the model), we will get an ordered list of 300 numbers that represent that word.
Can you write Python code for it?

from gensim.models import Word2Vec

MODEL_DIM = 300
WORD2VEC_MODEL = "GoogleNews-vectors-negative300.bin"
model = Word2Vec.load_word2vec_format(WORD2VEC_MODEL, binary=True)


I am working on the file "HN.ipynb".
The first task is:
Step 2
The 2D version of the sentence vectors are now plotted as a scatter plot using Plotly.js.
Note: depending on the random_state parameter of the tsne_model the graph will change with each run. See the documentation for more details.
Can you write Python code for it?

from plotly.offline import download_plotlyjs, init_notebook_mode, iplot
from plotly.graph_objs import *
import colorlover as cl

init_notebook_mode(connected=True) # inject plotly.js into the notebook

trace = Scatter(
    x = X_2d[:, 0],
    y = X_2d[:, 1],
    mode = "markers",
    text = hn_small['title'].values.tolist()
    )

iplot({
        "data": [trace],
        "layout": Layout(title="HN")
    })


I am working on the file "HN.ipynb".
The first task is:
Visualising the list of title vectors
Step 1
Reduce the dimensionality of the vectors from 300 to 2 so that they can be plotted on a 2D surface.
For this process the Barnes-Hut implementation of the T-SNE algorithm is used.
Can you write Python code for it?

from sklearn.manifold import TSNE

tnse_model = TSNE(n_components=2, method="barnes_hut")
X_2d  = tnse_model.fit_transform(np.array(hn_small['vector'].values.tolist()))


I am working on the file "HOPS/.ipynb_checkpoints/hops2cf-checkpoint.ipynb".
The first task is:
Join the individual CF phenomena into one dataset.
Can you write Python code for it?

cubes = iris.cube.CubeList([u_cube, v_cube])


I am working on the file "HOPS/.ipynb_checkpoints/hops2cf-checkpoint.ipynb".
The first task is:
Now the phenomena.
NOTE: You don't need the broadcast_to trick if saving more than 1 time step.
Here I just wanted the single time snapshot to have the time dimension to create a full example.
Can you write Python code for it?

import numpy as np

u = vbaro[itime, :, :, 0]
u_cube = iris.cube.Cube(np.broadcast_to(u, (1,) + u.shape),
                        units=vbaro.units,
                        long_name=vbaro.long_name,
                        var_name='u',
                        standard_name='barotropic_eastward_sea_water_velocity',
                        dim_coords_and_dims=[(time, 0)],
                        aux_coords_and_dims=[(latitude, (1, 2)),
                                             (longitude, (1, 2))])

v = vbaro[itime, :, :, 1]
v_cube = iris.cube.Cube(np.broadcast_to(v, (1,) + v.shape),
                        units=vbaro.units,
                        long_name=vbaro.long_name,
                        var_name='v',
                        standard_name='barotropic_northward_sea_water_velocity',
                        dim_coords_and_dims=[(time, 0)],
                        aux_coords_and_dims=[(latitude, (1, 2)),
                                             (longitude, (1, 2))])


I am working on the file "HOPS/.ipynb_checkpoints/hops2cf-checkpoint.ipynb".
The first task is:
Save the CF-compliant file!
Can you write Python code for it?

iris.save(cubes, 'hops.nc')

!ncdump -h hops.nc


I am working on the file "HOPS/.ipynb_checkpoints/hops2cf-checkpoint.ipynb".
The first task is:
Using iris to create the CF object.
NOTE: ideally lon, lat should be DimCoord like time and not AuxCoord,
but iris refuses to create 2D DimCoord. Not sure if CF enforces that though.
First the Coordinates.
FIXME: change to a full time slice later!
Can you write Python code for it?

itime = -1

import iris
iris.FUTURE.netcdf_no_unlimited = True

time = iris.coords.DimCoord(vtime[itime],
                            var_name='time',
                            long_name=vtime.long_name,
                            standard_name='longitude',
                            units=vtime.units)

longitude = iris.coords.AuxCoord(coords[:, :, 0],
                                 var_name='vlat',
                                 standard_name='longitude',
                                 units='degrees')

latitude = iris.coords.AuxCoord(coords[:, :, 1],
                                var_name='vlon',
                                standard_name='latitude',
                                units='degrees')


I am working on the file "HOPS/.ipynb_checkpoints/hops_velocity3-checkpoint.ipynb".
The first task is:
Join the individual CF phenomena into one dataset.
Can you write Python code for it?

u_cube = u_cubes.concatenate_cube()
v_cube = v_cubes.concatenate_cube()

cubes = iris.cube.CubeList([u_cube, v_cube])


I am working on the file "HOPS/.ipynb_checkpoints/hops_velocity3-checkpoint.ipynb".
The first task is:
Now the phenomena.
NOTE: You don't need the broadcast_to trick if saving more than 1 time step.
Here I just wanted the single time snapshot to have the time dimension to create a full example.
Can you write Python code for it?

vbaro.shape

import numpy as np

u_cubes = iris.cube.CubeList()
v_cubes = iris.cube.CubeList()

for k in range(vbaro.shape[0]):  # vbaro.shape[0]
    time = iris.coords.DimCoord(vtime[k],
                                var_name='time',
                                long_name=vtime.long_name,
                                standard_name='time',
                                units=vtime.units)

    u = vbaro[k, :, :, 0]
    u_cubes.append(iris.cube.Cube(np.broadcast_to(u, (1,) + u.shape),
                                  units=vbaro.units,
                                  long_name=vbaro.long_name,
                                  var_name='u',
                                  standard_name='barotropic_eastward_sea_water_velocity',
                                  dim_coords_and_dims=[(time, 0), (lon, 1), (lat, 2)],
                                  aux_coords_and_dims=[(latitude, (1, 2)),
                                                       (longitude, (1, 2))]))

    v = vbaro[k, :, :, 1]
    v_cubes.append(iris.cube.Cube(np.broadcast_to(v, (1,) + v.shape),
                                  units=vbaro.units,
                                  long_name=vbaro.long_name,
                                  var_name='v',
                                  standard_name='barotropic_northward_sea_water_velocity',
                                  dim_coords_and_dims=[(time, 0), (lon, 1), (lat, 2)],
                                  aux_coords_and_dims=[(longitude, (1, 2)),
                                                       (latitude, (1, 2))]))


I am working on the file "HOPS/.ipynb_checkpoints/hops_velocity3-checkpoint.ipynb".
The first task is:
Save the CF-compliant file!
Can you write Python code for it?


I am working on the file "HOPS/hops2cf.ipynb".
The first task is:
Extract lon, lat variables from vgrid2 and u, v variables from vbaro.
The goal is to split the joint variables into individual CF compliant phenomena.
Can you write Python code for it?

vtime = nc['time']
coords = nc['vgrid2']
vbaro = nc['vbaro']


I am working on the file "HOPS/hops2cf.ipynb".
The first task is:
Join the individual CF phenomena into one dataset.
Can you write Python code for it?


I am working on the file "HOPS/hops2cf.ipynb".
The first task is:
Now the phenomena.
NOTE: You don't need the broadcast_to trick if saving more than 1 time step.
Here I just wanted the single time snapshot to have the time dimension to create a full example.
Can you write Python code for it?


I am working on the file "HOPS/hops_velocity3.ipynb".
The first task is:
Extract lon, lat variables from vgrid2 and u, v variables from vbaro.
The goal is to split the joint variables into individual CF compliant phenomena.
Can you write Python code for it?


I am working on the file "HOPS/hops_velocity3.ipynb".
The first task is:
Join the individual CF phenomena into one dataset.
Can you write Python code for it?


I am working on the file "HOPS/hops_velocity3.ipynb".
The first task is:
Save the CF-compliant file!
Can you write Python code for it?


I am working on the file "HOPS/hops_velocity_try-Copy1.ipynb".
The first task is:
Join the individual CF phenomena into one dataset.
Can you write Python code for it?


I am working on the file "HOPS/hops_velocity_try.ipynb".
The first task is:
Extract lon, lat variables from vgrid2 and u, v variables from vbaro.
The goal is to split the joint variables into individual CF compliant phenomena.
Can you write Python code for it?


I am working on the file "HOPS/hops_velocity_try.ipynb".
The first task is:
Join the individual CF phenomena into one dataset.
Can you write Python code for it?


I am working on the file "HOPS/hops_velocity_try.ipynb".
The first task is:
Now the phenomena.
NOTE: You don't need the broadcast_to trick if saving more than 1 time step.
Here I just wanted the single time snapshot to have the time dimension to create a full example.
Can you write Python code for it?

import numpy as np

u_cubes = iris.cube.CubeList()
v_cubes = iris.cube.CubeList()

for k in range(2):  # vbaro.shape[0]
    time = iris.coords.DimCoord(vtime[k],
                                var_name='time',
                                long_name=vtime.long_name,
                                standard_name='time',
                                units=vtime.units)

    u = vbaro[k, :, :, 0]
    u_cubes.append(iris.cube.Cube(np.broadcast_to(u, (1,) + u.shape),
                                  units=vbaro.units,
                                  long_name=vbaro.long_name,
                                  var_name='u',
                                  standard_name='barotropic_eastward_sea_water_velocity',
                                  dim_coords_and_dims=[(time, 0), (lon, 1), (lat, 2)],
                                  aux_coords_and_dims=[(latitude, (1, 2)),
                                                       (longitude, (1, 2))]))

    v = vbaro[k, :, :, 1]
    v_cubes.append(iris.cube.Cube(np.broadcast_to(v, (1,) + v.shape),
                                  units=vbaro.units,
                                  long_name=vbaro.long_name,
                                  var_name='v',
                                  standard_name='barotropic_northward_sea_water_velocity',
                                  dim_coords_and_dims=[(time, 0), (lon, 1), (lat, 2)],
                                  aux_coords_and_dims=[(longitude, (1, 2)),
                                                       (latitude, (1, 2))]))


I am working on the file "HOPS/hops_velocity_try.ipynb".
The first task is:
Save the CF-compliant file!
Can you write Python code for it?


I am working on the file "HOW TO USE THE LIBRARY.ipynb".
The first task is:
Make sure in the following cell that the directories are correct.
Only the fcs must exist and have fcs data on it as the other two will be created in the process.
The directory path syntaxis varies between OS
2. Save directory locations.
Can you write Python code for it?

fcs_dir = './fcs sample data/'      # This is the directory of the .fcs files in this case
csv_dir = 'csv_files/'               # This is where the converted data will be stored
merged_csv_dir = 'merged_data/'      # This is where the final merged dataframe will be stored
merged_filename = 'data'            # This  is  the name


I am working on the file "HOW TO USE THE LIBRARY.ipynb".
The first task is:
The next line of code converts fcs files in the fcs directory to csv files in the csv directory.
3. Convert fcs files to csv files.
Can you write Python code for it?


I am working on the file "HOW TO USE THE LIBRARY.ipynb".
The first task is:
The next line of code takes the csv files, merges them into a single dataframe and saves it in the specified directory.
4. Merge the data into the same dataframe.
Can you write Python code for it?


I am working on the file "HRDIC_data_vizualization_AZ31.ipynb".
The first task is:
Interactive visualization
Can you write Python code for it?

f,ax=plt.subplots(1,1,figsize=(9,5))
image_map=ax.imshow(du11_cropped,
                    vmin=0,
                    vmax=0.09,
                    cmap='viridis',
                    interpolation='bilinear')
plt.colorbar(image_map);


I am working on the file "HRDIC_data_vizualization_AZ31.ipynb".
The first task is:
The dataset is loaded with the command hrdic.DeformationMapFast which requires a data directory and filename as inputs. Below we have used './' for current directory and the name of the datafile in Zenodo. The datafile is available as a text file (AZ31.txt) or a NumpPy binary file (AZ31.npy). The '.npy' file loads faster but there is also an alternative function for reading the '.txt' file instead.
Can you write Python code for it?

#using the .txt is much slower than the binary .npy
#def_map=hrdic.DeformationMap('./','AZ31.txt')
def_map=hrdic.DeformationMapFast('./','AZ31.npy')


I am working on the file "HRDIC_data_vizualization_AZ31.ipynb".
The first task is:
The edges of the map can be cropped to remove edge artifacts:
Can you write Python code for it?


I am working on the file "HRDIC_data_vizualization_AZ31.ipynb".
The first task is:
The interactive view of the displacement gradient components:
Can you write Python code for it?

interactive(hrdic.scrub, fig=fixed(image_map), fmap=fixed(def_map),
            component=('du11','du22','du12','du21','Max shear'),
            colourmap= ('viridis','bwr'),
            cmin=(-0.5,1.9,0.01),
            cmax=(0.0,2.0,0.01))


I am working on the file "HTML_and_JSON_processing.ipynb".
The first task is:
Examples with BeautifulSoup library
The BeautifulSoup is library dedicated to simplify scraping information from HTML pages. It is an DOM parser. The sample data named Example1 from this tutorial repo are used next example:
Can you write Python code for it?

from bs4 import BeautifulSoup

# path to data
path = "data/example1.html"

# template for printing the output
sentence = "{} {} is {} years old."

# load data
with open(path, 'r') as datafile:
    sample_html = datafile.read()

# create tree
soup = BeautifulSoup(sample_html, "html.parser")

# get title and print it
title = soup.find("title")
print(title.text, "\n")

# select all rows in table
table = soup.find("table",  {"id": "main_table"})
table_rows = table.findAll("tr")

# iterate over table and print results
for row in table_rows:
    first_name = row.find("td", {"class": "first_name"})
    last_name = row.find("td", {"class": "last_name"})
    age = row.find("td", {"class": "age"})
    if first_name and last_name and age:
        print(sentence.format(first_name.text, last_name.text, age.text))


I am working on the file "HTML_and_JSON_processing.ipynb".
The first task is:
If you need just the <i>key number</i> value from the text. And it is sure that:

information appers only once in the text

information will not change the form (words, word order ...)

You can use following approach.
Can you write Python code for it?

# unclean way
target_start = sample_html.find("the key number is ") + len("the key number is")
target_end = sample_html[target_start:].find(".") + target_start
print(sample_html[target_start:target_end])


I am working on the file "HTML_and_JSON_processing.ipynb".
The first task is:
JSON support in Requests library
The Requests library can convert the HTTP JSON reponse directly to Python standard format (dictionary/list). See following example.
Can you write Python code for it?

import requests

r = requests.get("http://api.open-notify.org/iss-now.json")
obj = r.json()

print(obj)


I am working on the file "HTML_and_JSON_processing.ipynb".
The first task is:
Note, that the output is string. In similar way you can unpack the message back to Python standard list/dictionary. Example follows.
Can you write Python code for it?

# unpack message
message = json.loads(js_message)

# show result
print(type(message))
print(message)


I am working on the file "HTML_and_JSON_processing.ipynb".
The first task is:
Second example prints just content of paragraph with major_content class:
Can you write Python code for it?

import xml.etree.ElementTree as ET

tree = ET.fromstring(sample_html)

tree.findall("./body/p[@class='major_content']")[0].text


I am working on the file "HTML_and_JSON_processing.ipynb".
The first task is:
Simle example of usage follows. Following parser print out encountered tags and data.
Can you write Python code for it?

# from HTMLParser import HTMLParser # Python 2.7
from html.parser import HTMLParser

class TestHTMLParser(HTMLParser):

    def handle_starttag(self, tag, attrs):
        print("Tag start:", tag)

    def handle_endtag(self, tag):
        print("Tag end:", tag)

    def handle_data(self, data):
        print("Tag data:", data)

# instantiate the parser and fed in some HTML
parser = TestHTMLParser(convert_charrefs=True)
parser.feed(sample_html)


I am working on the file "HTML_and_JSON_processing.ipynb".
The first task is:
The goal of this second parser is to get content from paragraph with class: major_content.
Can you write Python code for it?

class Test2HTMLParser(HTMLParser):

    def __init__(self):
        HTMLParser.__init__(self, convert_charrefs=True)
        self.recording = False

    def handle_starttag(self, tag, attrs):
        if tag == "p" and "major_content" in dict(attrs).values():
            self.recording = True

    def handle_endtag(self, tag):
        self.recording = False

    def handle_data(self, data):
        if self.recording:
            print(data)

# instantiate the parser and fed in some HTML
parser2 = Test2HTMLParser()
parser2.feed(sample_html)


I am working on the file "HW0.ipynb".
The first task is:
Call the x.max function again, but use the axis keyword to print the maximum of each row in x.
Can you write Python code for it?


I am working on the file "HW0.ipynb".
The first task is:
Here's a way to quickly simulate 500 coin "fair" coin tosses (where the probabily of getting Heads is 50%, or 0.5)
Can you write Python code for it?

x = np.random.binomial(500, .5)
x = np.random.binomial?
print ("number of heads:", x)

x = np.random.binomial


I am working on the file "HW0.ipynb".
The first task is:
If any of these libraries are missing or out of date, you will need to install them and restart IPython
Hello matplotlib
The notebook integrates nicely with Matplotlib, the primary plotting package for python. This should embed a figure of a sine wave:
Can you write Python code for it?

#this line prepares IPython for working with matplotlib
%matplotlib inline

# this actually imports matplotlib
import matplotlib.pyplot as plt

x = np.linspace(0, 10, 30)  #array of 30 points from 0 to 10
y = np.sin(x)
z = y + np.random.normal(size=30) * .2
plt.plot(x, y, 'ro-', label='A sine wave')
plt.plot(x, z, 'b-', label='Noisy sine')
plt.legend(loc = 'lower right')
plt.xlabel("X axis")
plt.ylabel("Y axis")


I am working on the file "HW0.ipynb".
The first task is:
Next, write a function that simulates the contestant's guesses for nsim simulations. Call this function simulate_guess. The specs:
Can you write Python code for it?

"""
Function
--------
simulate_guess

Return any strategy for guessing which door a prize is behind. This
could be a random strategy, one that always guesses 2, whatever.

Parameters
----------
nsim : int
    The number of simulations to generate guesses for

Returns
-------
guesses : array
    An array of guesses. Each guess is a 0, 1, or 2

Example
-------
>>> print simulate_guess(5)
array([0, 0, 0, 0, 0])
"""
#your code here
def simulate_guess(nsim):
    #compute here
    answer =np.random.random_integers(0,1,nsim)
    return answer
#your code here
print (simulate_guess(10))


I am working on the file "HW0.ipynb".
The first task is:
Next, write a function, goat_door, to simulate randomly revealing one of the goat doors that a contestant didn't pick.
Can you write Python code for it?

"""
Function
--------
goat_door

Simulate the opening of a "goat door" that doesn't contain the prize,
and is different from the contestants guess

Parameters
----------
prizedoors : array
    The door that the prize is behind in each simulation
guesses : array
    THe door that the contestant guessed in each simulation

Returns
-------
goats : array
    The goat door that is opened for each simulation. Each item is 0, 1, or 2, and is different
    from both prizedoors and guesses

Examples
--------
>>> print goat_door(np.array([0, 1, 2]), np.array([1, 1, 1]))
>>> array([2, 2, 0])
"""
def goat_door(prizedoors, guesses):

    #strategy: generate random answers, and
    #keep updating until they satisfy the rule
    #that they aren't a prizedoor or a guess
    result = np.random.randint(0, 3, prizedoors.size)

    while True:
        bad = (result == prizedoors) | (result == guesses)
        if not bad.any():
            return result
        result[bad] = np.random.randint(0, 3, bad.sum())

print (goat_door(np.array([0, 1, 2,1]), np.array([1, 1, 1,1])))


I am working on the file "HW0.ipynb".
The first task is:
Now, put it together. Simulate 10000 games where contestant keeps his original guess, and 10000 games where the contestant switches his door after a  goat door is revealed. Compute the percentage of time the contestant wins under either strategy. Is one strategy better than the other?
Can you write Python code for it?

#your code here

#simulate prizedoors, guesses, goatdoors
prizedoors = simulate_prizedoor(10000)
guesses = simulate_guess(10000)
goatdoors = goat_door(prizedoors, guesses)
#compute win percent if keep original
print("Win Percentage - keep original")
print(win_percentage(guesses,prizedoors))
#compute win percent if switch guess
switchedguesses = switch_guess(guesses,goatdoors)
print("Win Percentage - switch guess")
print(win_percentage(switchedguesses,prizedoors))


I am working on the file "HW0.ipynb".
The first task is:
Print the maximum, minimum, and mean of the array. This does not require writing a loop. In the code cell below, type x.m<TAB>, to find built-in operations for common array statistics like this
Can you write Python code for it?

#your code here
print (x.max())
print (x.min())
print (x.mean())


I am working on the file "HW0.ipynb".
The first task is:
Repeat this simulation 500 times, and use the plt.hist() function to plot a histogram of the number of Heads (1s) in each simulation
Can you write Python code for it?

#your code here
heads = []
for i in range(1,500):
    heads.append(np.random.binomial(500, .5))
plt.hist(heads, bins =500)


I am working on the file "HW01-Intro_to_Pandas/Data Wrangling with Pandas.ipynb".
The first task is:
A common data analysis procedure is the split-apply-combine operation, which groups subsets of data together, applies a function to each of the groups, then recombines them into a new data table.
For example, we may want to aggregate our data with with some function.

*(figure taken from "Python for Data Analysis", p.251)*

We can aggregate in Pandas using the aggregate (or agg, for short) method:
Can you write Python code for it?


I am working on the file "HW01-Intro_to_Pandas/Data Wrangling with Pandas.ipynb".
The first task is:
A logical way to specify these numerically is to change them to integer values, perhaps using "Placebo" as a baseline value. If we create a dict with the original values as keys and the replacements as values, we can pass it to the map method to implement the changes.
Can you write Python code for it?

treatment_map = {'Placebo': 0, '5000U': 1, '10000U': 2}

cdystonia['treatment'] = cdystonia.treat.map(treatment_map)
cdystonia.treatment


I am working on the file "HW01-Intro_to_Pandas/Data Wrangling with Pandas.ipynb".
The first task is:
Also, if to_datetime() has problems parsing any particular date/time format, you can pass the spec in using the format= argument.
The read_* functions now have an optional parse_dates argument that try to convert any columns passed to it into datetime format upon import:
Can you write Python code for it?

segments = pd.read_csv("Data/AIS/transit_segments.csv", parse_dates=['st_time', 'end_time'])

segments.dtypes


I am working on the file "HW01-Intro_to_Pandas/Data Wrangling with Pandas.ipynb".
The first task is:
Alternately, if we simply want to replace particular values in a Series or DataFrame, we can use the replace method.
An example where replacement is useful is dealing with zeros in certain transformations. For example, if we try to take the log of a set of values:
Can you write Python code for it?

vals = pd.Series([float(i)**10 for i in range(10)])
vals

np.log(vals)


I am working on the file "HW01-Intro_to_Pandas/Data Wrangling with Pandas.ipynb".
The first task is:
Alternately, we can transform the data, using a function of our choice with the transform method:
Can you write Python code for it?

normalize = lambda x: (x - x.mean())/x.std()

cdystonia_grouped.transform(normalize).head()


I am working on the file "HW01-Intro_to_Pandas/Data Wrangling with Pandas.ipynb".
The first task is:
Alternatively, one can specify custom quantiles to act as cut points:
Can you write Python code for it?

quantiles = pd.qcut(segments.seg_length, [0, 0.01, 0.05, 0.95, 0.99, 1])
quantiles[:30]


I am working on the file "HW01-Intro_to_Pandas/Data Wrangling with Pandas.ipynb".
The first task is:
By default the Categorical type represents an unordered categorical.
Can you write Python code for it?


I am working on the file "HW01-Intro_to_Pandas/Data Wrangling with Pandas.ipynb".
The first task is:
Compare this ordering with the original:
Can you write Python code for it?


I am working on the file "HW01-Intro_to_Pandas/Data Wrangling with Pandas.ipynb".
The first task is:
Concatenating along axis=1 will concatenate column-wise, but respecting the indices of the two DataFrames.
Can you write Python code for it?

pd.concat([mb1, mb2], axis=1).shape

pd.concat([mb1, mb2], axis=1).head()


I am working on the file "HW01-Intro_to_Pandas/Data Wrangling with Pandas.ipynb".
The first task is:
Data aggregation and GroupBy operations
One of the most powerful features of Pandas is its GroupBy functionality. On occasion we may want to perform operations on groups of observations within a dataset. For exmaple:

aggregation, such as computing the sum of mean of each group, which involves applying a function to each group and returning the aggregated results
slicing the DataFrame into groups and then doing something with the resulting slices (e.g. plotting)
group-wise transformation, such as standardization/normalization
Can you write Python code for it?


I am working on the file "HW01-Intro_to_Pandas/Data Wrangling with Pandas.ipynb".
The first task is:
Data transformation
There are a slew of additional operations for DataFrames that we would collectively refer to as "transformations" which include tasks such as removing duplicate values, replacing values, and grouping values.
Dealing with duplicates
We can easily identify and remove duplicate values from DataFrame objects. For example, say we want to removed ships from our vessels dataset that have the same name:
Can you write Python code for it?


I am working on the file "HW01-Intro_to_Pandas/Data Wrangling with Pandas.ipynb".
The first task is:
Date/Time data handling
Date and time data are inherently problematic. There are an unequal number of days in every month, an unequal number of days in a year (due to leap years), and time zones that vary over space. Yet information about time is essential in many analyses, particularly in the case of time series analysis.
The datetime built-in library handles temporal information down to the nanosecond.
Can you write Python code for it?

from datetime import datetime

now = datetime.now()
now

now.day

now.weekday()


I am working on the file "HW01-Intro_to_Pandas/Data Wrangling with Pandas.ipynb".
The first task is:
For a simple cross-tabulation of group frequencies, the crosstab function (not a method) aggregates counts of data according to factors in rows and columns. The factors may be hierarchical if desired.
Can you write Python code for it?


I am working on the file "HW01-Intro_to_Pandas/Data Wrangling with Pandas.ipynb".
The first task is:
For random sampling, DataFrame and Series objects have a sample method that can be used to draw samples, with or without replacement:
Can you write Python code for it?

vessels.sample(n=10)

vessels.sample(n=10, replace=True)


I am working on the file "HW01-Intro_to_Pandas/Data Wrangling with Pandas.ipynb".
The first task is:
For this dataset, it makes sense to create a hierarchical index based on the patient and observation:
Can you write Python code for it?

cdystonia2 = cdystonia.set_index(['patient','obs'])
cdystonia2.head()

cdystonia2.index.is_unique


I am working on the file "HW01-Intro_to_Pandas/Data Wrangling with Pandas.ipynb".
The first task is:
Having a custom data type for dates and times is convenient because we can perform operations on them easily. For example, we may want to calculate the difference between two times:
Can you write Python code for it?

my_age = now - datetime(1970, 1, 1)
my_age

print(type(my_age))
my_age.days/365


I am working on the file "HW01-Intro_to_Pandas/Data Wrangling with Pandas.ipynb".
The first task is:
Hence, we get a considerable speedup simply by using the appropriate dtype for our data.
Discretization
Pandas' cut function can be used to group continuous or countable data in to bins. Discretization is generally a very bad idea for statistical analysis, so use this function responsibly!
Lets say we want to bin the ages of the cervical dystonia patients into a smaller number of groups:
Can you write Python code for it?


I am working on the file "HW01-Intro_to_Pandas/Data Wrangling with Pandas.ipynb".
The first task is:
However, an ordering can be imposed. The order is lexical by default, but will assume the order of the listed categories to be the desired order.
Can you write Python code for it?

cdystonia.treat.cat.categories = ['Placebo', '5000U', '10000U']

cdystonia.treat.cat.as_ordered().head()


I am working on the file "HW01-Intro_to_Pandas/Data Wrangling with Pandas.ipynb".
The first task is:
However, since there are multiple taxonomic units with the same class, our index is no longer unique:
Can you write Python code for it?


I am working on the file "HW01-Intro_to_Pandas/Data Wrangling with Pandas.ipynb".
The first task is:
However, the index is no longer unique, due to overlap between the two DataFrames.
Can you write Python code for it?


I am working on the file "HW01-Intro_to_Pandas/Data Wrangling with Pandas.ipynb".
The first task is:
If we are only interested in taxa that are included in both DataFrames, we can specify a join=inner argument.
Can you write Python code for it?

pd.concat([mb1, mb2], axis=1, join='inner').head()


I am working on the file "HW01-Intro_to_Pandas/Data Wrangling with Pandas.ipynb".
The first task is:
If we concatenate along axis=0 (the default), we will obtain another data frame with the the rows concatenated:
Can you write Python code for it?


I am working on the file "HW01-Intro_to_Pandas/Data Wrangling with Pandas.ipynb".
The first task is:
If we wanted to use the second table to fill values absent from the first table, we could use combine_first.
Can you write Python code for it?


I am working on the file "HW01-Intro_to_Pandas/Data Wrangling with Pandas.ipynb".
The first task is:
If we wish, we can easily aggregate according to multiple keys:
Can you write Python code for it?


I am working on the file "HW01-Intro_to_Pandas/Data Wrangling with Pandas.ipynb".
The first task is:
In addition to datetime there are simpler objects for date and time information only, respectively.
Can you write Python code for it?

from datetime import date, time

time(3, 24)

date(1970, 9, 3)


I am working on the file "HW01-Intro_to_Pandas/Data Wrangling with Pandas.ipynb".
The first task is:
In addition, time zone information can be applied:
Can you write Python code for it?


I am working on the file "HW01-Intro_to_Pandas/Data Wrangling with Pandas.ipynb".
The first task is:
In such situations, we can replace the zero with a value so small that it makes no difference to the ensuing analysis. We can do this with replace.
Can you write Python code for it?

vals = vals.replace(0, 1e-6)
np.log(vals)


I am working on the file "HW01-Intro_to_Pandas/Data Wrangling with Pandas.ipynb".
The first task is:
In this case, the default inner join is suitable; we are not interested in observations from either table that do not have corresponding entries in the other.
Notice that mmsi field that was an index on the vessels table is no longer an index on the merged table.
Here, we used the merge function to perform the merge; we could also have used the merge method for either of the tables:
Can you write Python code for it?


I am working on the file "HW01-Intro_to_Pandas/Data Wrangling with Pandas.ipynb".
The first task is:
In this section, we will manipulate data collected from ocean-going vessels on the eastern seaboard. Vessel operations are monitored using the Automatic Identification System (AIS), a safety at sea navigation technology which vessels are required to maintain and that uses transponders to transmit very high frequency (VHF) radio signals containing static information including ship name, call sign, and country of origin, as well as dynamic information unique to a particular voyage such as vessel location, heading, and speed.
The International Maritime Organization’s (IMO) International Convention for the Safety of Life at Sea requires functioning AIS capabilities on all vessels 300 gross tons or greater and the US Coast Guard requires AIS on nearly all vessels sailing in U.S. waters. The Coast Guard has established a national network of AIS receivers that provides coverage of nearly all U.S. waters. AIS signals are transmitted several times each minute and the network is capable of handling thousands of reports per minute and updates as often as every two seconds. Therefore, a typical voyage in our study might include the transmission of hundreds or thousands of AIS encoded signals. This provides a rich source of spatial data that includes both spatial and temporal information.
For our purposes, we will use summarized data that describes the transit of a given vessel through a particular administrative area. The data includes the start and end time of the transit segment, as well as information about the speed of the vessel, how far it travelled, etc.
Can you write Python code for it?

segments = pd.read_csv("Data/AIS/transit_segments.csv")
segments.head()


I am working on the file "HW01-Intro_to_Pandas/Data Wrangling with Pandas.ipynb".
The first task is:
Its also possible to group by one or more levels of a hierarchical index. Recall cdystonia2, which we created with a hierarchical index:
Can you write Python code for it?

cdystonia2.head(10)

cdystonia2.groupby(level='obs', axis=0)['twstrs'].mean()


I am working on the file "HW01-Intro_to_Pandas/Data Wrangling with Pandas.ipynb".
The first task is:
Let's transform these data into decades, beginnnig with individuals in their 20's and ending with those in their 80's:
Can you write Python code for it?


I am working on the file "HW01-Intro_to_Pandas/Data Wrangling with Pandas.ipynb".
The first task is:
Merging and joining DataFrame objects
Now that we have the vessel transit information as we need it, we may want a little more information regarding the vessels themselves. In the data/AIS folder there is a second table that contains information about each of the ships that traveled the segments in the segments table.
Can you write Python code for it?

vessels = pd.read_csv("Data/AIS/vessel_information.csv", index_col='mmsi')
vessels.head()

[v for v in vessels.type.unique() if v.find('/')==-1]

vessels.type.value_counts()


I am working on the file "HW01-Intro_to_Pandas/Data Wrangling with Pandas.ipynb".
The first task is:
Note that you can easily combine discretiztion with the generation of indicator variables shown above:
Can you write Python code for it?


I am working on the file "HW01-Intro_to_Pandas/Data Wrangling with Pandas.ipynb".
The first task is:
Notice that c_ and r_ are not really functions at all, since it is performing some sort of indexing operation, rather than being called. They are actually class instances, but they are here behaving mostly like functions. Don't think about this too hard; just know that they are there.

This operation is also called binding or stacking.
With Pandas' indexed data structures, there are additional considerations as the overlap in index values between two data structures affects how they are concatenate.
Lets import two microbiome datasets, each consisting of counts of microorganiams from a particular patient. We will use the first column of each dataset as the index.
Can you write Python code for it?

mb1 = pd.read_excel('Data/microbiome/MID1.xls', 'Sheet 1', index_col=0, header=None)
mb2 = pd.read_excel('Data/microbiome/MID2.xls', 'Sheet 1', index_col=0, header=None)
mb1.shape, mb2.shape

mb1.head()


I am working on the file "HW01-Intro_to_Pandas/Data Wrangling with Pandas.ipynb".
The first task is:
Notice that these are 8-bit integers, which are essentially single bytes of data, making memory usage lower.
There is also a performance benefit. Consider an operation such as calculating the total segment lengths for each ship in the segments table (this is also a preview of pandas' groupby operation!):
Can you write Python code for it?

%time segments.groupby(segments.name).seg_length.sum().sort_values(ascending=False, inplace=False).head()

segments['name'] = segments.name.astype('category')

%time segments.groupby(segments.name).seg_length.sum().sort_values(ascending=False, inplace=False).head()


I am working on the file "HW01-Intro_to_Pandas/Data Wrangling with Pandas.ipynb".
The first task is:
Pandas also has a custom NA value for missing datetime objects, NaT.
Can you write Python code for it?


I am working on the file "HW01-Intro_to_Pandas/Data Wrangling with Pandas.ipynb".
The first task is:
The challenge, however, is that several ships have travelled multiple segments, so there is not a one-to-one relationship between the rows of the two tables. The table of vessel information has a one-to-many relationship with the segments.
In Pandas, we can combine tables according to the value of one or more keys that are used to identify rows, much like an index. Using a trivial example:
Can you write Python code for it?

df1 = pd.DataFrame(dict(id=range(4), age=np.random.randint(18, 31, size=4)))
df2 = pd.DataFrame(dict(id=list(range(3))+list(range(3)),
                        score=np.random.random(size=6)))

df1

df2

pd.merge(df1, df2)


I am working on the file "HW01-Intro_to_Pandas/Data Wrangling with Pandas.ipynb".
The first task is:
The dateutil package includes a parser that attempts to detect the format of the date strings, and convert them automatically.
Can you write Python code for it?

from dateutil.parser import parse

parse(segments.st_time.ix[0])


I am working on the file "HW01-Intro_to_Pandas/Data Wrangling with Pandas.ipynb".
The first task is:
The index of these data is the unique biological classification of each organism, beginning with domain, phylum, class, and for some organisms, going all the way down to the genus level.
Can you write Python code for it?


I am working on the file "HW01-Intro_to_Pandas/Data Wrangling with Pandas.ipynb".
The first task is:
The outer join above yields the union of the two tables, so all rows are represented, with missing values inserted as appropriate. One can also perform right and left joins to include all rows of the right or left table (i.e. first or second argument to merge), but not necessarily the other.
Looking at the two datasets that we wish to merge:
Can you write Python code for it?


I am working on the file "HW01-Intro_to_Pandas/Data Wrangling with Pandas.ipynb".
The first task is:
The parentheses indicate an open interval, meaning that the interval includes values up to but not including the endpoint, whereas the square bracket is a closed interval, where the endpoint is included in the interval. We can switch the closure to the left side by setting the right flag to False:
Can you write Python code for it?


I am working on the file "HW01-Intro_to_Pandas/Data Wrangling with Pandas.ipynb".
The first task is:
This behavior can be overridden by specifying a suffixes argument, containing a list of the suffixes to be used for the columns of the left and right columns, respectively.
Concatenation
A common data manipulation is appending rows or columns to a dataset that already conform to the dimensions of the exsiting rows or colums, respectively. In NumPy, this is done either with concatenate or the convenience "functions" c_ and r_:
Can you write Python code for it?

np.concatenate([np.random.random(5), np.random.random(5)])

np.r_[np.random.random(5), np.random.random(5)]

np.c_[np.random.random(5), np.random.random(5)]


I am working on the file "HW01-Intro_to_Pandas/Data Wrangling with Pandas.ipynb".
The first task is:
This dataset includes repeated measurements of the same individuals (longitudinal data). Its possible to present such information in (at least) two ways: showing each repeated measurement in their own row, or in multiple columns representing multiple measurements.
The stack method rotates the data frame so that columns are represented in rows:
Can you write Python code for it?


I am working on the file "HW01-Intro_to_Pandas/Data Wrangling with Pandas.ipynb".
The first task is:
This grouped dataset is hard to visualize
Can you write Python code for it?


I am working on the file "HW01-Intro_to_Pandas/Data Wrangling with Pandas.ipynb".
The first task is:
Though most of the transits appear to be short, there are a few longer distances that make the plot difficult to read. This is where a transformation is useful:
Can you write Python code for it?


I am working on the file "HW01-Intro_to_Pandas/Data Wrangling with Pandas.ipynb".
The first task is:
To complement this, unstack pivots from rows back to columns.
Can you write Python code for it?


I am working on the file "HW01-Intro_to_Pandas/Data Wrangling with Pandas.ipynb".
The first task is:
Using the string methods split and join we can create an index that just uses the first three classifications: domain, phylum and class.
Can you write Python code for it?

class_index = mb1.index.map(lambda x: ' '.join(x.split(' ')[:3]))

mb_class = mb1.copy()
mb_class.index = class_index


I am working on the file "HW01-Intro_to_Pandas/Data Wrangling with Pandas.ipynb".
The first task is:
Using this sequence as an argument to the take method results in a reordered DataFrame:
Can you write Python code for it?


I am working on the file "HW01-Intro_to_Pandas/Data Wrangling with Pandas.ipynb".
The first task is:
Value replacement
Frequently, we get data columns that are encoded as strings that we wish to represent numerically for the purposes of including it in a quantitative analysis. For example, consider the treatment variable in the cervical dystonia dataset:
Can you write Python code for it?


I am working on the file "HW01-Intro_to_Pandas/Data Wrangling with Pandas.ipynb".
The first task is:
We can also create a hierarchical index based on keys identifying the original tables.
Can you write Python code for it?

pd.concat([mb1, mb2], keys=['patient1', 'patient2']).head()

pd.concat([mb1, mb2], keys=['patient1', 'patient2']).index.is_unique


I am working on the file "HW01-Intro_to_Pandas/Data Wrangling with Pandas.ipynb".
The first task is:
We can also perform the same replacement that we used map for with replace:
Can you write Python code for it?

cdystonia2.treat.replace({'Placebo': 0, '5000U': 1, '10000U': 2})


I am working on the file "HW01-Intro_to_Pandas/Data Wrangling with Pandas.ipynb".
The first task is:
We can convert this to a category type either by the Categorical constructor, or casting the column using astype:
Can you write Python code for it?

pd.Categorical(cdystonia.treat)

cdystonia['treat'] = cdystonia.treat.astype('category')

cdystonia.treat.describe()


I am working on the file "HW01-Intro_to_Pandas/Data Wrangling with Pandas.ipynb".
The first task is:
We can re-establish a unique index by summing all rows with the same class, using groupby:
Can you write Python code for it?


I am working on the file "HW01-Intro_to_Pandas/Data Wrangling with Pandas.ipynb".
The first task is:
We can see that although there are date/time fields in the dataset, they are not in any specialized format, such as datetime.
Can you write Python code for it?


I am working on the file "HW01-Intro_to_Pandas/Data Wrangling with Pandas.ipynb".
The first task is:
we see that there is a mmsi value (a vessel identifier) in each table, but it is used as an index for the vessels table. In this case, we have to specify to join on the index for this table, and on the mmsi column for the other.
Can you write Python code for it?

segments_merged = pd.merge(vessels, segments, left_index=True, right_on='mmsi')

segments_merged.head()


I am working on the file "HW01-Intro_to_Pandas/Intro to Pandas.ipynb".
The first task is:
15 records contains those values
Operations
DataFrame and Series objects allow for several operations to take place either on a single object, or between two or more objects.
For example, we can perform arithmetic on the elements of two objects, such as combining baseball statistics across years. First, let's (artificially) construct two Series, consisting of home runs hit in years 2006 and 2007, respectively:
Can you write Python code for it?

hr2006 = baseball.loc[baseball.year==2006, 'hr']
hr2006.index = baseball.player[baseball.year==2006]

hr2007 = baseball.loc[baseball.year==2007, 'hr']
hr2007.index = baseball.player[baseball.year==2007]

hr2007


I am working on the file "HW01-Intro_to_Pandas/Intro to Pandas.ipynb".
The first task is:
A DataFrame has a second index, representing the columns:
Can you write Python code for it?


I am working on the file "HW01-Intro_to_Pandas/Intro to Pandas.ipynb".
The first task is:
Above, Pandas recognized NA and an empty field as missing data.
Can you write Python code for it?


I am working on the file "HW01-Intro_to_Pandas/Intro to Pandas.ipynb".
The first task is:
Additionally, the order of the set of indices in a hierarchical MultiIndex can be changed by swapping them pairwise:
Can you write Python code for it?


I am working on the file "HW01-Intro_to_Pandas/Intro to Pandas.ipynb".
The first task is:
Alternately, if we want to process our data in reasonable chunks, the chunksize argument will return an iterable object that can be employed in a data processing loop. For example, our microbiome data are organized by bacterial phylum, with 14 patients represented in each:
Can you write Python code for it?

pd.read_csv("Data/microbiome.csv", chunksize=14)

data_chunks = pd.read_csv("Data/microbiome.csv", chunksize=14)

mean_tissue = pd.Series({chunk.Taxon[0]: chunk.Tissue.mean() for chunk in data_chunks})

mean_tissue


I am working on the file "HW01-Intro_to_Pandas/Intro to Pandas.ipynb".
The first task is:
Alternatively, you can break ties via one of several methods, such as by the order in which they occur in the dataset:
Can you write Python code for it?


I am working on the file "HW01-Intro_to_Pandas/Intro to Pandas.ipynb".
The first task is:
But note, we cannot use the attribute indexing method to add a new column:
Can you write Python code for it?


I am working on the file "HW01-Intro_to_Pandas/Intro to Pandas.ipynb".
The first task is:
By default, dropna drops entire rows in which one or more values are missing.
Can you write Python code for it?


I am working on the file "HW01-Intro_to_Pandas/Intro to Pandas.ipynb".
The first task is:
Clearly, sum is more meaningful for some columns than others. For methods like mean for which application to string variables is not just meaningless, but impossible, these columns are automatically exculded:
Can you write Python code for it?


I am working on the file "HW01-Intro_to_Pandas/Intro to Pandas.ipynb".
The first task is:
Critically, the labels are used to align data when used in operations with other Series objects:
Can you write Python code for it?


I am working on the file "HW01-Intro_to_Pandas/Intro to Pandas.ipynb".
The first task is:
Data can also be sorted by any index level, using sortlevel:
Can you write Python code for it?


I am working on the file "HW01-Intro_to_Pandas/Intro to Pandas.ipynb".
The first task is:
Data summarization
We often wish to summarize data in Series or DataFrame objects, so that they can more easily be understood or compared with similar data. The NumPy package contains several functions that are useful here, but several summarization or reduction methods are built into Pandas data structures.
Can you write Python code for it?


I am working on the file "HW01-Intro_to_Pandas/Intro to Pandas.ipynb".
The first task is:
Exercise 1
Try out these commands to see what they return:

data.head()
data.tail(3)
data.shape
Can you write Python code for it?

data.head() # returns the first (5 by default) rows of data.

data.tail(3) # returns the 3 last rows of data

data.shape # returns the dimension of data (nbr rows, nbr cols)


I am working on the file "HW01-Intro_to_Pandas/Intro to Pandas.ipynb".
The first task is:
Exercise 4
Calculate on base percentage for each player, and return the ordered series of estimates.
$$OBP = \frac{H + BB + HBP}{AB + BB + HBP + SF}$$
define the function and appy it.
Can you write Python code for it?

def on_base_perc(pl):
    nominator = pl['h'] + pl['bb'] + pl['hbp'] #H+BB+HBP
    denom = pl['ab'] + pl['bb'] +pl['hbp'] +pl['sf']
    if(denom == 0) : # If the denom == 0 we can not compute nominator/denom, hence we retrun NaN
        return np.NaN
    return nominator / denom
baseball.apply(on_base_perc, axis=1).round(3)


I am working on the file "HW01-Intro_to_Pandas/Intro to Pandas.ipynb".
The first task is:
For a DataFrame, we can sort according to the values of one or more columns using the by argument of sort_values:
Can you write Python code for it?

baseball[['player','sb','cs']].sort_values(ascending=[False,True],
                                           by=['sb', 'cs']).head(10)


I am working on the file "HW01-Intro_to_Pandas/Intro to Pandas.ipynb".
The first task is:
For a more concise (and readable) syntax, we can use the new query method to perform selection on a DataFrame. Instead of having to type the fully-specified column, we can simply pass a string that describes what to select. The query above is then simply:
Can you write Python code for it?


I am working on the file "HW01-Intro_to_Pandas/Intro to Pandas.ipynb".
The first task is:
Hierarchical indexing
In the baseball example, I was forced to combine 3 fields to obtain a unique index that was not simply an integer value. A more elegant way to have done this would be to create a hierarchical index from the three fields.
Can you write Python code for it?

baseball_h = baseball.set_index(['year', 'team', 'player'])
baseball_h.head(10)


I am working on the file "HW01-Intro_to_Pandas/Intro to Pandas.ipynb".
The first task is:
Hierarchical indices can be created on either or both axes. Here is a trivial example:
Can you write Python code for it?

frame = pd.DataFrame(np.arange(12).reshape(( 4, 3)),
                  index =[['a', 'a', 'b', 'b'], [1, 2, 1, 2]],
                  columns =[['Ohio', 'Ohio', 'Colorado'], ['Green', 'Red', 'Green']])

frame


I am working on the file "HW01-Intro_to_Pandas/Intro to Pandas.ipynb".
The first task is:
If an index is not specified, a default sequence of integers is assigned as the index. A NumPy array comprises the values of the Series, while the index is a pandas Index object.
Can you write Python code for it?


I am working on the file "HW01-Intro_to_Pandas/Intro to Pandas.ipynb".
The first task is:
If we only want to import a small number of rows from, say, a very large data file we can use nrows:
Can you write Python code for it?


I am working on the file "HW01-Intro_to_Pandas/Intro to Pandas.ipynb".
The first task is:
If we plan on modifying an extracted Series, its a good idea to make a copy.
Can you write Python code for it?

vals = data.value.copy()
vals[5] = 1000
vals


I am working on the file "HW01-Intro_to_Pandas/Intro to Pandas.ipynb".
The first task is:
If we wish to access columns, we can do so either by dict-like indexing or by attribute:
Can you write Python code for it?

data['patient']

data.patient

type(data.value)

data[['value']]


I am working on the file "HW01-Intro_to_Pandas/Intro to Pandas.ipynb".
The first task is:
Importing data
A key, but often under-appreciated, step in data analysis is importing the data that we wish to analyze. Though it is easy to load basic data structures into Python using built-in tools or those provided by packages like NumPy, it is non-trivial to import structured data well, and to easily convert this input into a robust data structure:
genes = np.loadtxt("genes.csv", delimiter=",", dtype=[('gene', '|S10'), ('value', '<f4')])

Pandas provides a convenient set of functions for importing tabular data in a number of formats directly into a DataFrame object. These functions include a slew of options to perform type inference, indexing, parsing, iterating and cleaning automatically as data are imported.
Let's start with some more bacteria data, stored in csv format.
Can you write Python code for it?


I am working on the file "HW01-Intro_to_Pandas/Intro to Pandas.ipynb".
The first task is:
In a DataFrame we can slice along either or both axes:
Can you write Python code for it?


I am working on the file "HW01-Intro_to_Pandas/Intro to Pandas.ipynb".
The first task is:
In addition to using loc to select rows and columns by label, pandas also allows indexing by position using the iloc attribute.
So, we can query rows and columns by absolute position, rather than by name:
Can you write Python code for it?


I am working on the file "HW01-Intro_to_Pandas/Intro to Pandas.ipynb".
The first task is:
Index objects are immutable:
Can you write Python code for it?


I am working on the file "HW01-Intro_to_Pandas/Intro to Pandas.ipynb".
The first task is:
Indexing and Selection
Indexing works analogously to indexing in NumPy arrays, except we can use the labels in the Index object to extract values in addition to arrays of integers.
Can you write Python code for it?

# Sample Series object
hits = baseball_newind.h
hits

# Numpy-style indexing
hits[:3]

# Indexing by label
hits[['womacto01CHN2006','schilcu01BOS2006']]


I am working on the file "HW01-Intro_to_Pandas/Intro to Pandas.ipynb".
The first task is:
Its important to note that the Series returned when a DataFrame is indexted is merely a view on the DataFrame, and not a copy of the data itself. So you must be cautious when manipulating this data:
Can you write Python code for it?

vals = data.value
vals

vals[5] = 0
vals


I am working on the file "HW01-Intro_to_Pandas/Intro to Pandas.ipynb".
The first task is:
Missing data
The occurence of missing data is so prevalent that it pays to use tools like Pandas, which seamlessly integrates missing data handling so that it can be dealt with easily, and in the manner required by the analysis at hand.
Missing data are represented in Series and DataFrame objects by the NaN floating point value. However, None is also treated as missing, since it is commonly used as such in other contexts (e.g. NumPy).
Can you write Python code for it?

foo = pd.Series([np.nan, -3, None, 'foobar'])
foo

foo.isnull()


I am working on the file "HW01-Intro_to_Pandas/Intro to Pandas.ipynb".
The first task is:
Most real-world data is incomplete, with values missing due to incomplete observation, data entry or transcription error, or other reasons. Pandas will automatically recognize and parse common missing data indicators, including NA and NULL.
Can you write Python code for it?

!cat Data/microbiome_missing.csv

pd.read_csv("Data/microbiome_missing.csv").head(20)


I am working on the file "HW01-Intro_to_Pandas/Intro to Pandas.ipynb".
The first task is:
Note that it is probably faster to first filter the values bigger than 1000 as it filters more values out.
Specifying a Series as a new columns cause its values to be added according to the DataFrame's index:
Can you write Python code for it?

treatment = pd.Series([0]*4 + [1]*2)
treatment

data['treatment'] = treatment
data


I am working on the file "HW01-Intro_to_Pandas/Intro to Pandas.ipynb".
The first task is:
Notice that because of the mix of string and integer (and NaN) values, the dtype of the array is object. The dtype will automatically be chosen to be as general as needed to accomodate all the columns.
Can you write Python code for it?


I am working on the file "HW01-Intro_to_Pandas/Intro to Pandas.ipynb".
The first task is:
Notice that fillna by default returns a new object with the desired filling behavior, rather than changing the Series or  DataFrame in place (in general, we like to do this, by the way!).
We can alter values in-place using inplace=True.
Can you write Python code for it?


I am working on the file "HW01-Intro_to_Pandas/Intro to Pandas.ipynb".
The first task is:
Notice that read_csv automatically considered the first row in the file to be a header row.
We can override default behavior by customizing some the arguments, like header, names or index_col.
Can you write Python code for it?


I am working on the file "HW01-Intro_to_Pandas/Intro to Pandas.ipynb".
The first task is:
Notice that the id index is not sequential. Say we wanted to populate the table with every id value. We could specify and index that is a sequence from the first to the last id numbers in the database, and Pandas would fill in the missing data with NaN values:
Can you write Python code for it?

id_range = range(baseball.index.values.min(), baseball.index.values.max())
baseball.reindex(id_range).head()


I am working on the file "HW01-Intro_to_Pandas/Intro to Pandas.ipynb".
The first task is:
Notice that we specified the id column as the index, since it appears to be a unique identifier. We could try to create a unique index ourselves by combining player and year:
Can you write Python code for it?

player_id = baseball.player + baseball.year.astype(str)
baseball_newind = baseball.copy()
baseball_newind.index = player_id
baseball_newind.head()


I am working on the file "HW01-Intro_to_Pandas/Intro to Pandas.ipynb".
The first task is:
Notice the DataFrame is sorted by column name. We can change the order by indexing them in the order we desire:
Can you write Python code for it?


I am working on the file "HW01-Intro_to_Pandas/Intro to Pandas.ipynb".
The first task is:
Notice this is different than with Series, where dict-like indexing retrieved a particular element (row).
If we want access to a row in a DataFrame, we index its loc attribute.
Can you write Python code for it?


I am working on the file "HW01-Intro_to_Pandas/Intro to Pandas.ipynb".
The first task is:
NumPy's math functions and other operations can be applied to Series without losing the data structure.
Can you write Python code for it?

# natural logarithm
np.log(bacteria)
# log base 10
np.log10(bacteria)


I am working on the file "HW01-Intro_to_Pandas/Intro to Pandas.ipynb".
The first task is:
Operations can also be broadcast between rows or columns.
For example, if we subtract the maximum number of home runs hit from the hr column, we get how many fewer than the maximum were hit by each player:
Can you write Python code for it?