I am working on the file "Old Code/Miscellaneous/Masimo Analysis (Old).ipynb".
The first task is:
Set Date and Time of ROP Exam
Can you write Python code for it?


I am working on the file "Old Code/NIRS Analysis - Template150918.ipynb".
The first task is:
Export Data to CSV
Make sure the file name is changed!!!! ROP###Avgs_NIRS.csv
Can you write Python code for it?

#export out all averages as a csv file with only

import csv
class excel_tab(csv.excel):
        delimiter = '\t'
csv.register_dialect("excel_tab", excel_tab)

with open('ROP009_NIRSAvgs.csv', 'w') as f:    #CHANGE CSV FILE NAME
    writer = csv.writer(f, dialect=excel_tab)
    writer.writerow(avg0)
    for i1 in r1:
        writer.writerow(df1[i1:(i1+win1)].mean())
    for i2 in r2:
        writer.writerow(df2[i2:(i2+win2)].mean())
    for i3 in r3:
        writer.writerow(df3[i3:(i3+win3)].mean())
    for i4 in r4:
        writer.writerow(df4[i4:(i4+win4)].mean())
    for i5 in r5:
        writer.writerow(df5[i5:(i5+win5)].mean())


I am working on the file "Old Code/NIRS151111.ipynb".
The first task is:
Data Recording Time Length
Can you write Python code for it?

print "Data Recording Time!"
print '*' * 10
print "Pre-Exam Data Recording Length\t", X - Y # start of exam - first data point
print "Post-Exam Data Recording Length\t", Q - Z #last data point - end of exam
print "Total Data Recording Length\t", Q - Y #last data point - first data point


I am working on the file "Old Code/NIRS151111.ipynb".
The first task is:
Export Data to CSV
Make sure the file name is changed!!!! ROP###Avgs_NIRS.csv
Can you write Python code for it?

#export out all averages as a csv file with only

import csv
class excel_tab(csv.excel):
        delimiter = '\t'
csv.register_dialect("excel_tab", excel_tab)

with open('ROP013_NIRSAvgs.csv', 'w') as f:    #CHANGE CSV FILE NAME
    writer = csv.writer(f, dialect=excel_tab)
    writer.writerow(avg0)
    for i1 in r1:
        writer.writerow(df1[i1:(i1+win1)].mean())
    for i2 in r2:
        writer.writerow(df2[i2:(i2+win2)].mean())
    for i3 in r3:
        writer.writerow(df3[i3:(i3+win3)].mean())
    for i4 in r4:
        writer.writerow(df4[i4:(i4+win4)].mean())
    for i5 in r5:
        writer.writerow(df5[i5:(i5+win5)].mean())


I am working on the file "Old Code/ROPFinal160302.ipynb".
The first task is:
Average Every 15 Mins Hour 2-3 After ROP Exam
Can you write Python code for it?


I am working on the file "Old Code/ROPFinal160302.ipynb".
The first task is:
Average Every 5 Mins Hour 1-2 After ROP Exam
Can you write Python code for it?


I am working on the file "Old Code/ROPFinal160302.ipynb".
The first task is:
Baseline Average Calculation
From first point of data collection to the first eye drops
Can you write Python code for it?

print 'Baseline Averages\n', 'NIRS :\t', avg0NIRS, '\nPI :\t',avg0PI, '\nSpO2 :\t',avg0O2,'\nPR :\t',avg0PR,


I am working on the file "Old Code/ROPFinal160302.ipynb".
The first task is:
First Eye Drop Avg Every 10 Sec For 5 Minutes
Can you write Python code for it?


I am working on the file "Old Code/ROPFinal160302.ipynb".
The first task is:
Second Eye Drop Avg Every 10 Sec For 5 Minutes
Can you write Python code for it?


I am working on the file "Old Code/ROPFinal160302.ipynb".
The first task is:
Third Eye Drop Avg Every 10 Sec For 5 Minutes
Can you write Python code for it?


I am working on the file "Old/visualisation.ipynb".
The first task is:
actual brain visualisation widget
Can you write Python code for it?

from ipywidgets import widgets
from ipywidgets import interact,fixed
from IPython.display import display

def rebuild_nii_compute(data,nim,num):
    """
    precompute to make widget faster
    """

    #data = np.load('Mean_Vec.npy')
    a = data[:,num].copy()
    #nim = nib.load('cc400_roi_atlas.nii')
    imdat=nim.get_data()
    imdat_new = imdat.copy()

    for n, i in enumerate(np.unique(imdat)):
        if i != 0:
            imdat_new[imdat == i] = a[n-1] * 100000 # scaling factor. Could also try to get float values in nifti...

    nim_out = nib.Nifti1Image(imdat_new, nim.get_affine(), nim.get_header())
    nim_out.set_data_dtype('float32')
    # to save:
    #nim_out.to_filename('Gradient_'+ str(num) +'_res.nii')

    #nilearn.plotting.plot_epi(nim_out)
    return nim_out,imdat_new

def rebuild_nii_plot(num,nims,cutc_x,cutc_y,cutc_z):
    """
    simple plotting function for widget
    """
    cut_coords = (cutc_x,cutc_y,cutc_z)
    fig = nilearn.plotting.plot_epi(nims[num],cut_coords=cut_coords)
    plt.show()

def rebuild_nii_plot_new(num,imdats_new,cutc_x,cutc_y,cutc_z,cmap,figsize):
    """
    maybe using matplotlib is faster
    """
    cut_coords = (cutc_x,cutc_y,cutc_z)
    fig,axes=plt.subplots(1,3,figsize=figsize)
    imdat = imdats_new[num]
    axes[0].imshow(imdat[:,cutc_y,:].T,cmap=cmap,origin='lower')
    axes[0].set_title('y')
    axes[1].imshow(imdat[cutc_x,:,:].T,cmap=cmap,origin='lower')
    axes[1].set_title('x')
    axes[2].imshow(imdat[:,:,cutc_z].T,cmap=cmap,origin='lower')
    axes[2].set_title('z')
    plt.show()

#prepare data
data = np.load('Mean_Vec.npy')
nim = nib.load('cc400_roi_atlas.nii')
imdat=nim.get_data()
print(data.shape,imdat.shape)

#precompute to speed up widget
nims = []
imdats = []
for j in range(data.shape[1]):
    nim_new,imdat_new = rebuild_nii_compute(data,nim,j)
    nims.append(nim_new)
    imdats.append(imdat_new)

#testing plotting function
nilearn.plotting.plot_epi(nims[0]) #automatic choice of...


I am working on the file "Old/visualisation.ipynb".
The first task is:
another attempt with a pure matplotlib version of the plot function
Can you write Python code for it?

coords_max = imdat.shape
figsize=(15,10)
i = interact(rebuild_nii_plot_new,
             num=(0,9),
             imdats_new=fixed(imdats),
             cutc_x=(0,coords_max[0]-1),
             cutc_y=(0,coords_max[1]-1),
             cutc_z=(0,coords_max[2]-1),
             cmap = ['jet','viridis','gray'],
             figsize=fixed(figsize)
            )


I am working on the file "Old/visualisation.ipynb".
The first task is:
here is another very similar application.
dataset from https://github.com/datacarpentry/python-neuroimaging-lesson/tree/master/neuroimaging/data
Can you write Python code for it?

epi_img = nib.load('/home/bjoern/brainhack/python-neuroimaging-lesson/neuroimaging/data/someones_epi.nii.gz')
epi_data = epi_img.get_data()

def plot_projection(data,axis,pos0,pos1,pos2,cmap,figsize):
    if axis=='x':
        imdat = data[pos0,:,:]
    elif axis=='y':
        imdat = data[:,pos1,:]
    elif axis=='z':
        imdat = data[:,:,pos2]

    fig,ax = plt.subplots(figsize=figsize)
    ax.imshow(imdat,cmap=cmap, origin="lower")
    plt.show()

#some settings for widget
data = epi_data
data.shape
figsize=(6,6)

i = interact(plot_projection,
             data=fixed(data),
             axis=['x','y','z'],
             pos0=(0,data.shape[0]-1),
             pos1=(0,data.shape[1]-1),
             pos2=(0,data.shape[2]-1),
             cmap = ['gray','jet','viridis'],
            figsize=fixed(figsize))

def show_slices_grid(slices,gridsize):
    """ Function to display row of image slices """
    fig, axes = plt.subplots(gridsize, len(slices)//gridsize+1,figsize=(12,12))
    for i, slice in enumerate(slices):
        row = i//gridsize
        column = i%gridsize
        axes[row][column].imshow(slice.T, cmap="gray", origin="lower")
    fig.tight_layout()

slices = [epi_data[:,:,i] for i in range(epi_data.shape[2])]
show_slices_grid(slices,6)


I am working on the file "OldSummerStudentsCourse/2016/notebooks/FillHistogram_Example_py.ipynb".
The first task is:
Open a file which is located on the web. No type is to be specified for "f".
Can you write Python code for it?


I am working on the file "OldSummerStudentsCourse/2016/notebooks/FillHistogram_Example_py.ipynb".
The first task is:
Optional: activate the JavaScript visualisation to produce interactive plots.
Can you write Python code for it?


I am working on the file "OldSummerStudentsCourse/2016/notebooks/TTreeAccess_Example_py.ipynb".
The first task is:
Loop over the TTree called "events" in the file. It is accessed with the dot operator.
Same holds for the access to the branches: no need to set them up - they are just accessed by name, again with the dot operator.
Can you write Python code for it?

maxPt=-1
for event in f.events:
    maxPt=-1
    for track in event.tracks:
        pt = track.Pt()
        if pt > maxPt: maxPt = pt
    if event.evtNum % 100 == 0:
        print "Processing event number %i" %event.evtNum
        print "Max pt is %f" %maxPt


I am working on the file "OldSummerStudentsCourse/2016/notebooks/TTreeAccess_Example_py.ipynb".
The first task is:
Open a file which is located on the web. No type is to be specified for "f".
Can you write Python code for it?


I am working on the file "OldSummerStudentsCourse/2017/examples/notebooks/FillHistogram_Example_py.ipynb".
The first task is:
Loop over the TTree called "events" in the file. It is accessed with the dot operator.
Same holds for the access to the branches: no need to set them up - they are just accessed by name, again with the dot operator.
Can you write Python code for it?

h = ROOT.TH1F("TracksPt","Tracks;Pt [GeV/c];#",128,0,64)
for event in f.events:
    for track in event.tracks:
        h.Fill(track.Pt())
c = ROOT.TCanvas()
h.Draw()
c.Draw()


I am working on the file "OldSummerStudentsCourse/2017/examples/notebooks/FillHistogram_Example_py.ipynb".
The first task is:
Open a file which is located on the web. No type is to be specified for "f".
Can you write Python code for it?


I am working on the file "OldSummerStudentsCourse/2017/examples/notebooks/TTreeAccess_Example_py.ipynb".
The first task is:
Loop over the TTree called "events" in the file. It is accessed with the dot operator.
Same holds for the access to the branches: no need to set them up - they are just accessed by name, again with the dot operator.
Can you write Python code for it?


I am working on the file "OldSummerStudentsCourse/2017/examples/notebooks/TTreeAccess_Example_py.ipynb".
The first task is:
Open a file which is located on the web. No type is to be specified for "f".
Can you write Python code for it?


I am working on the file "Old_Runs/july1st.ipynb".
The first task is:
Let's see what our chains look like by producing trace plots:
Can you write Python code for it?

#set up subplots for chain plotting
axes=['ax'+str(i) for i in range(ndim)]
fig, (axes) = plt.subplots(ndim, figsize=(10,60))

plt.tight_layout()

for i in range(0,ndim):
    if i<int(ndim/2):
        axes[i].set(ylabel='d%i' % (i+1))
    else:
        axes[i].set(ylabel='c%i' % (i-5))

#plot traces for each parameter
for i in range(0,ndim):
    sns.tsplot(traces_cold[i],ax=axes[i])


I am working on the file "Old_Runs/july1st.ipynb".
The first task is:
Now we are going to use the seaborn distplot function to plot histograms of the last half of the traces for each parameter.
Can you write Python code for it?

#set up subplots for histogram plotting
axes=['ax'+str(i) for i in range(ndim)]
fig, (axes) = plt.subplots(ndim, figsize=(10,60))

plt.tight_layout()

for i in range(0,ndim):
    if i<int(ndim/2):
        axes[i].set(ylabel='d%i' % (i+1))
    else:
        axes[i].set(ylabel='c%i' % (i-5))

#plot traces for each parameter
for i in range(0,ndim):
    sns.distplot(traces_cold[i],ax=axes[i],hist=True,norm_hist=False)


I am working on the file "Old_Runs/july1st.ipynb".
The first task is:
Now we want to see how similar the parameters at different steps are. To do this, we draw one thousand random samples from the last half of the chain and plot the reddening profile corresponding to those parameters in light blue. Then, we plot the "best fit" reddening profile corresponding to the 50th quantile parameters (essentially the median of the last half of the chains). In all cases, we take the average of the CO for all nside 128 pixels at each slice. As you can see, drawing random samples from the last half of the chain produce reddening profiles essentially identical to the best fit values.
Can you write Python code for it?

from dustcurve import plot_posterior

ratio=0.06
plot_posterior.plot_samples(np.asarray(post_all),np.linspace(4,19,120),np.linspace(0,7,700),quantile_50,traces_cold,ratio,gv.unique_co,y_range=[0,2],vmax=20,normcol=False)


I am working on the file "Old_Runs/tutorial_6slices.ipynb".
The first task is:
Let's see what our chains look like by producing trace plots:
Can you write Python code for it?


I am working on the file "Old_Runs/tutorial_6slices.ipynb".
The first task is:
Now let's overplot the reddening profiles corresponding to our most probable parameters on top of the stacked stellar posterior surfaces. We show two options. The first plot has been normalized so that a) each individual stellar posterior array sums to one and b) each distance column in the stacked posterior array contains the same amount of "ink." The second plot just normalized so each individual stellar posterior array sums to 1. Each colored reddening profile corresponds to a single pixel in the Dame et al. 2001 CO spectral cube. There are stars in ~175 Dame et al. 2001 CO pixels in our region of interest.
Can you write Python code for it?

from dustcurve import pixclass

post_all=np.empty((0,700,120))

for i in range(0,len(gv.unique_post)):
    post_all=np.vstack((post_all,gv.unique_post[i]))

#from dustcurve import plot_posterior
#plot the reddening profile over the stacked, normalized stellar posterior surfaces
#normcol=True, normsurf=True
#plot_posterior.plot_posterior(np.asarray(post_all),np.linspace(4,19,120),np.linspace(0,7,700),quantile_50,ratio,gv.unique_co,y_range=[0,2],vmax=0.03,normcol=True)
#plot_posterior.plot_posterior(np.asarray(post_all),np.linspace(4,19,120),np.linspace(0,7,700),quantile_50,ratio,gv.unique_co,y_range=[0,2],vmax=10.0,normcol=False)


I am working on the file "Old_Runs/tutorial_6slices.ipynb".
The first task is:
Now we want to see how similar the parameters at different steps are. To do this, we draw one thousand random samples from the last half of the chain and plot the reddening profile corresponding to those parameters in light blue. Then, we plot the "best fit" reddening profile corresponding to the 50th quantile parameters (essentially the median of the last half of the chains). We are normalizing the surfaces in two different ways, as we did above.
Can you write Python code for it?

#from dustcurve import plot_posterior
#from dustcurve import globalvars as gv
#import numpy as np

#plot_posterior.plot_samples(np.asarray(post_all),np.linspace(4,19,120),np.linspace(0,7,700),quantile_50,traces_cold,ratio,gv.unique_co,y_range=[0,2],vmax=0.03,normcol=True)
#plot_posterior.plot_samples(np.asarray(post_all),np.linspace(4,19,120),np.linspace(0,7,700),quantile_50,traces_cold,ratio,gv.unique_co,y_range=[0,2],vmax=20,normcol=False)


I am working on the file "Old_Runs/tutorial_6slices.ipynb".
The first task is:
The sampler is done running, so now let's check out the results. We are going to print out our mean acceptance fraction across all walkers for the coldest temperature chain.
We are also going to discard the first half of each walker's chain as burn-in; to change the number of steps to burn off, simply edit the 3rd dimension of sampler.chain[0,:,n:,:] and input your desired value of n. Next, we are going to compute and print out the 50th, 16th, and 84th percentile of the chains for each distance parameter, using the "quantile" attribute of a pandas dataframe object. The 50th percentile measurement represents are best guess for the each distance parameter, while the difference between the 16th and 50th gives us a lower limit and the difference between the 50th and 84th percentile gives us an upper limit:
Can you write Python code for it?

#Extract the coldest [beta=1] temperature chain from the sampler object; discard first half of samples as burnin
samples_cold = sampler.chain[0,:,int(.5*nsteps):,:]
traces_cold = samples_cold.reshape(-1, ndim).T

#check out acceptance fraction:
print("Our mean acceptance fraction for the coldest chain is %.2f" % np.mean(sampler.acceptance_fraction[0]))

#find best fit values for each of the 24 parameters (12 d's and 12 c's)
theta=pd.DataFrame(traces_cold)
quantile_50=theta.quantile(.50, axis=1).values
quantile_84=theta.quantile(.84, axis=1).values
quantile_16=theta.quantile(.16, axis=1).values

upperlim=quantile_84-quantile_50
lowerlim=quantile_50-quantile_16

#print out distances
for i in range(0,int(len(quantile_50)/2)):
    print('d%i: %.3f + %.3f - %.3f' % (i+1,quantile_50[i],upperlim[i], lowerlim[i]))

#print out coefficients
for i in range(int(len(quantile_50)/2), int(len(quantile_50))):
    print('c%i: %.3f + %.3f - %.3f' % (i+1-int(len(quantile_50)/2),quantile_50[i],upperlim[i], lowerlim[i]))


I am working on the file "Old_Runs/tutorial_6slices_sig0.01.ipynb".
The first task is:
Let's see what our chains look like by producing trace plots:
Can you write Python code for it?


I am working on the file "Old_Runs/tutorial_6slices_sig0.01.ipynb".
The first task is:
Now let's overplot the reddening profiles corresponding to our most probable parameters on top of the stacked stellar posterior surfaces. We show two options. The first plot has been normalized so that a) each individual stellar posterior array sums to one and b) each distance column in the stacked posterior array contains the same amount of "ink." The second plot just normalized so each individual stellar posterior array sums to 1. Each colored reddening profile corresponds to a single pixel in the Dame et al. 2001 CO spectral cube. There are stars in ~175 Dame et al. 2001 CO pixels in our region of interest.
Can you write Python code for it?

from dustcurve import pixclass

post_all=np.empty((0,700,120))

for file in fnames:
    pixObj=pixclass.PixStars('/n/fink1/czucker/Data/'+file)
    post_array=np.asarray(pixObj.get_p()[:,:,:])
    post_all=np.vstack((post_all,post_array))

unique_co,unique_post,ratio=ldata

from dustcurve import plot_posterior
#plot the reddening profile over the stacked, normalized stellar posterior surfaces
#normcol=True, normsurf=True
plot_posterior.plot_posterior(np.asarray(post_all),np.linspace(4,19,120),np.linspace(0,7,700),quantile_50,ratio,unique_co,y_range=[0,2],vmax=0.03,normcol=True)
plot_posterior.plot_posterior(np.asarray(post_all),np.linspace(4,19,120),np.linspace(0,7,700),quantile_50,ratio,unique_co,y_range=[0,2],vmax=10.0,normcol=False)


I am working on the file "Old_Runs/tutorial_6slices_sig0.01.ipynb".
The first task is:
Now let's save the results to a file!
Can you write Python code for it?

import h5py
#Save the results of the sampler:
#our pixels of choice
output='2degrees_june21_0.01sig.h5'

fwrite = h5py.File("/n/fink1/czucker/Output/"+str(output), "w")
chaindata = fwrite.create_dataset("/chains", sampler.chain.shape, dtype='f')
chaindata[:,:,:,:]=sampler.chain

probdata = fwrite.create_dataset("/probs", sampler.lnprobability.shape, dtype='f')
probdata[:,:,:]=sampler.lnprobability

fwrite.close()


I am working on the file "Old_Runs/tutorial_6slices_sig0.01.ipynb".
The first task is:
Now we want to see how similar the parameters at different steps are. To do this, we draw one thousand random samples from the last half of the chain and plot the reddening profile corresponding to those parameters in light blue. Then, we plot the "best fit" reddening profile corresponding to the 50th quantile parameters (essentially the median of the last half of the chains). We are normalizing the surfaces in two different ways, as we did above.
Can you write Python code for it?


I am working on the file "One-to-One.ipynb".
The first task is:
1) Normal Matching
マッチング過程
M0, M1はF0にプロポーズし、M2はF2にプロポーズをする。
このとき、M2は誰とも競合せずF2と付き合うことが出来るが、M0とM1は競合する。F0からすると、M1よりM0の方が好みなので、M1を断りM0と付き合うという選択を行う。
最後に、残ったM1は残ったF1にプロポーズして付き合う。
マッチング結果
M0 - F0
M1 - F1
M2 - F2
問題点
M1は第三希望のF1と付き合い、F2は第二希望のM2と付き合うことになっている。しかし、M1とF2が付き合ってしまえば、M1は第二希望の人、F2は第一希望の人と付き合うことができ、双方の効用が増加する。故に、このときM1とF2はこのマッチング結果を破棄して駆け落ちするインセンティブが生じてしまうので、「不安定」なマッチングであったと言える。
Can you write Python code for it?

prop_matched, resp_matched = mf.BOS(prop_prefs, resp_prefs)

prop_matched

resp_matched

mf.Graph(prop_matched, resp_matched)


I am working on the file "One-to-One.ipynb".
The first task is:
2) Deffered Acceptance
マッチング過程
M0, M1はF0にプロポーズし、M2はF2にプロポーズをする。
このとき、M2は誰とも競合せずF2と付き合うことが出来るが、M0とM1は競合する。F0からすると、M1よりM0の方が好みなので、M1を断りM0と付き合うという選択を行う。
残ったM1は第二希望のF2にプロポーズをする。F2はM2からプロポーズを受けて付き合おうとしていたが、M2よりM1の方が好みなので、M2を断りM1と付き合うという選択を行う。
残ったM2は第二希望のF0にプロポーズをする。F0はM0からプロポーズを受けて付き合おうとしていたが、M0よりM2の方が好みなので、M0を断りM2と付き合うという選択を行う。
残ったM0は第二希望のF1にプロポーズをして、誰とも競合せずに付き合う。
マッチング結果
M0 - F1
M1 - F2
M2 - F0
このアルゴリズムでは駆け落ちするペアは存在せず「安定」なマッチングが実現され、上記のアルゴリズムで生じた問題を回避することが出来た。
Can you write Python code for it?

prop_matched, resp_matched = mf.DA(prop_prefs, resp_prefs)

prop_matched

resp_matched

mf.Graph(prop_matched, resp_matched)


I am working on the file "One_Topic_Driver_Example.ipynb".
The first task is:
Convert to Panda DataFrame and then plot.
Can you write Python code for it?

df = pd.DataFrame(data=evolution, index=None, columns=None)

fig = plt.figure()
ax = df.plot(legend=False)
ax.set_xlabel("Time")
ax.set_ylabel("Opinions")
ax.set_title('Evolution of Opinions')
plt.show()


I am working on the file "One_Topic_Driver_Example.ipynb".
The first task is:
Import Game Modules From a Given Path
User have to edit the path and put the correct one on his/her machine.
Can you write Python code for it?

import sys
# search path for modules
sys.path.append('/Users/hn/Documents/GitHub/PyOpinionGame/')

import opiniongame.config as og_cfg
import opiniongame.IO as og_io
import opiniongame.coupling as og_coupling
import opiniongame.state as og_state
import opiniongame.adjacency as og_adj
import opiniongame.selection as og_select
import opiniongame.potentials as og_pot
import opiniongame.core as og_core
import opiniongame.stopping as og_stop
import opiniongame.opinions as og_opinions


I am working on the file "One_Topic_Driver_Example.ipynb".
The first task is:
Plot the numpy array
Can you write Python code for it?

fig1 = plt.figure()
plt.plot(evolution)
plt.xlabel('Time')
plt.ylabel('Opinionds')
plt.title('Evolution of Opinions')
plt.show()


I am working on the file "One_Topic_Driver_Example.ipynb".
The first task is:
Reshape the history
Since here we have just one topic, the matrix can be converted to a 2D array, rather than 3.
This makes the plotting job easier:
Can you write Python code for it?

time, population_size, no_of_topics = rdict['history'].shape
evolution = rdict['history'].reshape(time, population_size)


I am working on the file "One_Topic_Driver_Example.ipynb".
The first task is:
Run The Game
Can you write Python code for it?

#
# run the game
#
state = og_core.run_until_convergence(config, state, ufuncs)

# Extract and save the history of the game
rdict = {}
rdict['history'] = state.history
og_io.saveMatrix('output.mat', rdict)


I am working on the file "One_Topic_Driver_Example.ipynb".
The first task is:
Set up inline matplotlib
This tells Python to plot the figure right in here, in the notebook.
Can you write Python code for it?

%matplotlib inline
#rcParams['figure.figsize'] = 5, 4
sb.set_style('whitegrid')


I am working on the file "One_Topic_Driver_Example.ipynb".
The first task is:
User Defined States Can go in the following cell:
Can you write Python code for it?

# user_weights =
# user_initialOpinions =
# user_adj =

state = og_state.WorldState(adj=default_adj,
                            couplingWeights=default_weights,
                            initialOpinions=default_initialOpinions,
                            initialHistorySize=100,
                            historyGrowthScale=2)
state.validate()
ufuncs = og_cfg.UserFunctions(og_select.FastPairSelection,
                              og_stop.iterationStop,
                              og_pot.createTent(tau))


I am working on the file "One_Topic_Driver_Example.ipynb".
The first task is:
seed PRNG: must do this before any random numbers are ever sampled during default generation
Can you write Python code for it?

print("SEEDING PRNG: "+str(config.startingseed))
np.random.seed(config.startingseed)


I am working on the file "OnroadLanesDetector.ipynb".
The first task is:
Constants with image/video pathes for testing, pipeline initialization
Can you write Python code for it?

image1 = "input/test_images/solidWhiteCurve.jpg"
image2 = "input/test_images/solidWhiteRight.jpg"
image3 = "input/test_images/solidYellowCurve.jpg"
image4 = "input/test_images/solidYellowCurve2.jpg"
image5 = "input/test_images/solidYellowLeft.jpg"
image6 = "input/test_images/whiteCarLaneSwitch.jpg"

video1 = "input/test_videos/challenge.mp4"
video2 = "input/test_videos/solidYellowLeft.mp4"
video3 = "input/test_videos/solidWhiteRight.mp4"

detection_area_filter = DetectionAreaFilter()

max_distance_delta = 40  # max distance between lines (rho1 - rho2) in polar coordinate system
max_angle_delta = np.radians(4)  # max angle between lines (theta1 - theta2) in polar coordinate system
threshold = 3  # min amount of lines in set filter
coordinate_sorter = CoordinateSorter(max_distance_delta, max_angle_delta, threshold)

img_processor = ImageProcessor(detection_area_filter, coordinate_sorter)

showImage(image4)
#playVideo(video1)


I am working on the file "OnroadLanesDetector.ipynb".
The first task is:
Here is the result of getColorMask function, that turns all white and yellow objects into white ones and makes  other colored objects black:

And the result of applyDetectionArea function, that creates new b/w image with applyed trapezium shaped mask out of given b/w image:

Then goes Canny edge detection:
Can you write Python code for it?

def getEdges(image, low_threshold=50, high_threshold=150):
    edges = cv2.Canny(image, low_threshold, high_threshold)
    return edges


I am working on the file "OnroadLanesDetector.ipynb".
The first task is:
Next class is responsible for filtering out lanes detection area:
Can you write Python code for it?

class DetectionAreaFilter:

    def __init__(self):
        self._lower_yellow = np.array([20, 0, 170], dtype=np.uint8)
        self._upper_yellow = np.array([55, 255, 255], dtype=np.uint8)

        self._lower_white = np.array([0, 0, 220], dtype=np.uint8)
        self._upper_white = np.array([255, 25, 255], dtype=np.uint8)

        self._ignore_mask_color = 255

    def getColorMask(self, hsv_image):
        mask_yellow = cv2.inRange(hsv_image, self._lower_yellow, self._upper_yellow)
        mask_white = cv2.inRange(hsv_image, self._lower_white, self._upper_white)

        mask = cv2.add(mask_white, mask_yellow)
        return mask

    def applyDetectionArea(self, bw_image, width_adjustment=60, height_adjustment=65):
        im_height = bw_image.shape[0]
        im_half_height = im_height // 2
        im_width = bw_image.shape[1]
        im_half_width = im_width // 2

        area_left_bottom = (0, im_height)
        area_left_top = (im_half_width - width_adjustment, im_half_height + height_adjustment)
        area_right_top = (im_half_width + width_adjustment, im_half_height + height_adjustment)
        area_right_bottom = (im_width, im_height)

        detection_area = [area_left_bottom, area_left_top, area_right_top, area_right_bottom]
        vertices = np.array([detection_area], dtype=np.int32)

        mask = np.zeros_like(bw_image)
        cv2.fillPoly(mask, vertices, self._ignore_mask_color)

        masked_image = cv2.bitwise_and(bw_image, mask)
        return masked_image


I am working on the file "OnroadLanesDetector.ipynb".
The first task is:
Result of getLaneLines transformed into lines

Then goes line grop detection, that accepts getLaneLines result as input parameter.
CoordinateSorter class does line groups detection and median line calculation for each found group.
Influence it's input parameters on 'sort' function behavior could be described with help of Gherkin language:
CoordinateSorter(max_distance_delta, max_angle_delta, threshold)

 Scenario: 'max_distance_delta' and 'max_angle_delta' parameters allow to control line group detection
   Given: 5 lines have been given to sort
     And: it is possible to create a chain 'chain_1' of lines line1, line2, line3
   Where: distance between links is less (or equal) then (max_distance_delta, max_angle_delta)
     And: it is possible to create a chain 'chain_2' of lines line4, line5
   Where: distance between links is less (or equal) then (max_distance_delta, max_angle_delta)
     And: distance between chain_1 and chain_2 edges is more than (max_distance_delta, max_angle_delta)
    Then: chain_1 and chain_2 considered as two separate lines

 Scenario: 'threshold' parameter allows to filter out noise lines
   Given: threshold = 4, set of lines
    When: sorter found 3 groups of lines
     And: the first set of lines contains 10 lines, second - 5 lines
     But: the third set of lines contains 3 lines
    Then: the third considered as noise and will not be presented in sorting result

Resulting line is calculate as median of all lines in a group
Can you write Python code for it?

class CoordinateSorter:

    def __init__(self, max_distance_delta, max_angle_delta, threshold):
        if max_angle_delta < 0:
            raise ValueError("[max_angle_delta] must be positive number")

        if max_angle_delta < 0:
            raise ValueError("[max_angle_delta] must be positive number")

        if threshold < 1 or type(threshold) != int:
            raise ValueError("[threshold] expected to be integer greater then or equal to 1")

        self._max_point_distance = (max_distance_delta, max_angle_delta)
        self._min_points_amount = threshold

    def _sortPointsByDistance(self, points_dict):
        set_list = list()

        for key, value in points_dict.items():
            indexes_set = set()
            set_list.append(indexes_set)
            indexes_set.add(key)

            for inner_key, inner_value in points_dict.items():
                point_distance = abs(np.subtract(value, inner_value))

                if point_distance[0] <= self._max_point_distance[0] \
                        and point_distance[1] <= self._max_point_distance[1]:
                    indexes_set.add(inner_key)

        return set_list

    def _splitOnGroups(self, set_list_source):

        sorted_source = list(set_list_source)
        sorted_source.sort(key=len, reverse=True)

        extremums = list()

        def find_extremums(ordered_list_of_set_items):
            if len(ordered_list_of_set_items) == 0:
                return

            first_extremum = ordered_list_of_set_items[0]
            items_for_further_sorting = list()

            for dot_set in ordered_list_of_set_items:
                if dot_set.issubset(first_extremum):
                    continue
                else:
                    if len(first_extremum.intersection(dot_set)):
                        first_extremum = first_extremum.union(dot_set)
                    else:
                        items_for_further_sorting.append(dot_set)

            extremums.append(first_extremum)
           ...


I am working on the file "OnroadLanesDetector.ipynb".
The first task is:
Result of lines sorting function in Cartesian coordinate system:

Drawing of lines with needed length:
Can you write Python code for it?

def convert(rho, theta, y_min, y_max):

    def create_point(y):
        x = (rho - y*np.sin(theta))/np.cos(theta)
        return int(x), int(y)

    d1 = create_point(y_max)
    d2 = create_point(y_min)

    return d1, d2

def drawLines(polar_coordinates_array, image, color, line_weight = 10):

    y_max = image.shape[0]
    y_min = int(y_max * 2 / 3)

    lines = [convert(rho, theta, y_min, y_max) for rho, theta in polar_coordinates_array]

    for d1, d2 in lines:
        cv2.line(image, d1, d2, color, line_weight)


I am working on the file "OnroadLanesDetector.ipynb".
The first task is:
The result of Canny edge detection:

Lines detetection with help of Hough transform method
Can you write Python code for it?

def getLaneLines(edges):
    deg = np.pi/180
    lines = cv2.HoughLines(edges, 1, 1*deg, 40)

    if lines is None:
        return np.array([])

    points_array = list()
    for line in lines:
        for rho, theta in line:
            points_array.append((rho, theta))

    return np.array(points_array)


I am working on the file "OnroadLanesDetector.ipynb".
The first task is:
Video processing entry point
Can you write Python code for it?

def playVideo(file_path):
    video_path = getPathFor(file_path)
    video = cv2.VideoCapture(video_path)
    print("About to start video playback...")

    while video.isOpened():
        _, bgr_frame = video.read()

        if not isinstance(bgr_frame, np.ndarray):
            # workaround to handle end of video stream.
            break

        frame = img_processor.processFrame(bgr_frame)
        cv2.imshow("output", frame)

        key = cv2.waitKey(1) & 0xFF
        # stop video on ESC key pressed
        if key == 27:
            break

    print("Video has been closed successfully.")
    video.release()
    cv2.destroyAllWindows()


I am working on the file "Open Hluttaw API Examples.ipynb".
The first task is:
Committees
Can you write Python code for it?

#List all committees

query = 'classification:Committee'
r = requests.get('http://api.openhluttaw.org/en/search/organizations?q='+query)
pages = r.json()['num_pages']

committees = []
for page in range(1,pages+1):
    r = requests.get('http://api.openhluttaw.org/en/search/organizations?q='+query+'&page='+str(page))
    orgs = r.json()['results']
    for org in orgs:
        committees.append(org)

for committee in committees:
    print committee['name']

import json

json_export = []
for committee in committees:
    json_export.append({'name':committee['name'],
                        'id':committee['id']})

print json.dumps(json_export,sort_keys=True, indent=4)

#Looking up a specific committee will list down all members and persons details, a committee is just
#organization class same as party, same as upper house, lower house

#"name": "Amyotha Hluttaw Local and Overseas Employment Committee"
r = requests.get('http://api.openhluttaw.org/en/organizations/9f3448056d2b48e1805475a45a4ae1ed')
committee = r.json()['result']

#List committee members

# missing on behalf_of expanded for organizations https://github.com/Sinar/popit_ng/issues/200
for member in committee['memberships']:
    print member['person']['id']
    print member['person']['name']
    print member['person']['image']


I am working on the file "Open Hluttaw API Examples.ipynb".
The first task is:
Party
Can you write Python code for it?

#List all committees

query = 'classification:Party'
r = requests.get('http://api.openhluttaw.org/en/search/organizations?q='+query)
pages = r.json()['num_pages']

parties = []
for page in range(1,pages+1):
    r = requests.get('http://api.openhluttaw.org/en/search/organizations?q='+query+'&page='+str(page))
    orgs = r.json()['results']
    for org in orgs:
        parties.append(org)

# BUG in https://github.com/Sinar/popit_ng/issues/197
# use JSON party lookup below to lookup values directly on client side

for party in parties:
    print party['name']


I am working on the file "Open Hluttaw API Examples.ipynb".
The first task is:
[
    {
        "id": "fd24165b8e814a758cd1098dc7a9038a",
        "name": "National League for Democracy"
    },
    {
        "id": "9462adf5cffa41c386e621fee28c59eb",
        "name": "Union Solidarity and Development Party"
    },
    {
        "id": "7997379fe27c4e448af522c85e306bfb",
        "name": "\"Wa\" Democratic Party"
    },
    {
        "id": "90e4903937bf4b8ba9185157dde06345",
        "name": "Kokang Democracy and Unity Party"
    },
    {
        "id": "2d2c795149c74b6f91cdea8caf28e968",
        "name": "Zomi Congress for Democracy"
    },
    {
        "id": "b366273152a84d579c4e19b14d36c0b5",
        "name": "Ta'Arng Palaung National Party"
    },
    {
        "id": "f2189158953e4d9e9296efeeffe7cf35",
        "name": "National Unity Party"
    },
    {
        "id": "d53d27fef3ac4b2bb4b7bf346215f626",
        "name": "Pao National Organization"
    },
    {
        "id": "2f0c09d5eb05432d8fcf247b5cb1885f",
        "name": "Mon National Party"
    },
    {
        "id": "dc69205c7eb54a7aaf68b3d2e3d9c23e",
        "name": "Rakhine National Party"
    },
    {
        "id": "e67bf2cdb4ff4ce89167cba3a514a6df",
        "name": "Shan Nationalities League for Democracy"
    },
    {
        "id": "a7a1ac9d2f20470d87e556af41dfaa19",
        "name": "Lisu National Development Party"
    },
    {
        "id": "8cc2d69bed8743bbaa229b164afecf9a",
        "name": "Independent"
    },
    {
        "id": "016a8ad7b40343ba96e0c03f47019680",
        "name": "Arakan National Party"
    },
    {
        "id": "6e76561e385946e0a3761d4f25293912",
        "name": "The Taaung (Palaung) National Party"
    },
    {
        "id": "63ec5681df974c67b7a217873fa9cdf5",
        "name": "Kachin Democratic Party"
    }
]
Can you write Python code for it?

#Listing people by party in org Amyotha 897739b2831e41109713ac9d8a96c845
#Pyithu org id would be 7f162ebef80e4a4aba12361ea1151fce
#We list by membership and specific organization_id and on_behalf_of_id of parties above

#Amyotha Members represented by Arakan National Party
query = 'organization_id:897739b2831e41109713ac9d8a96c845 AND on_behalf_of_id:016a8ad7b40343ba96e0c03f47019680'
r = requests.get('http://api.openhluttaw.org/en/search/memberships?q='+query)
pages = r.json()['num_pages']

memberships = []
for page in range(1,pages+1):
    r = requests.get('http://api.openhluttaw.org/en/search/memberships?q='+query+'&page='+str(page))
    members = r.json()['results']
    for member in members:
        memberships.append(member)

for member in memberships:
    print member['post']['label']
    print member['person']['id']
    print member['person']['name']
    print member['person']['image']


I am working on the file "OpenAI Reinforcement Learning Agent Plays Atari Games.ipynb".
The first task is:
Create a frame for environment
Can you write Python code for it?

def _process_frame(frame, conf):
    frame = frame[conf["crop1"]:conf["crop2"] + 160, :160]
    frame = resize(rgb2gray(frame), (80, conf["dimension2"]))
    frame = resize(frame, (80, 80))
    frame = np.reshape(frame, [1, 80, 80])
    return frame


I am working on the file "OpenAI Reinforcement Learning Agent Plays Atari Games.ipynb".
The first task is:
Environment, setting up the openAI and Universe
Create the atari environment function
Can you write Python code for it?

def atari_env(env_id, env_conf):
    env = gym.make(env_id)
    if len(env.observation_space.shape) > 1:
        env = Vectorize(env)
        env = AtariRescale(env, env_conf)
        env = NormalizedEnv(env)
        env = Unvectorize(env)

    return env


I am working on the file "OpenAI Reinforcement Learning Agent Plays Atari Games.ipynb".
The first task is:
Function to load arguments into play
Can you write Python code for it?

def loadarguments():
    global env_conf
    global env
    global setup_json
    global shared_model
    global saved_state
    global optimizer
    global torch

    undo_logger_setup()

    torch.set_default_tensor_type('torch.FloatTensor')
    torch.manual_seed(args['seed'])

    setup_json = read_config(args['EC'])

    env_conf = setup_json[args['config']]

    for i in setup_json.keys():
        if i in args['ENV']:
            env_conf = setup_json[i]
    env = atari_env(args['ENV'], env_conf)

    shared_model = A3Clstm(env.observation_space.shape[0], env.action_space)
    if args['L']:
        saved_state = torch.load(
            '{0}{1}.dat'.format(args['LMD'], args['ENV']))
        shared_model.load_state_dict(saved_state)
    shared_model.share_memory()

    if args['SO']:
        if args['OPT'] == 'RMSprop':
            optimizer = SharedRMSprop(shared_model.parameters(), lr=args['LR'])
        if args['OPT'] == 'Adam':
            optimizer = SharedAdam(shared_model.parameters(), lr=args['LR'])
        if args['OPT'] == 'LrSchedAdam':
            optimizer = SharedLrSchedAdam(
                shared_model.parameters(), lr=args['LR'])
        optimizer.share_memory()
    else:
        optimizer = None


I am working on the file "OpenAI Reinforcement Learning Agent Plays Atari Games.ipynb".
The first task is:
Functions to run the model on the environment
Test
Function To test the model on a game/environ
Can you write Python code for it?

def test(args, shared_model, env_conf,render=False):
    log = {}
    setup_logger('{}_log'.format(args['ENV']),
                 r'{0}{1}_log'.format(args['LG'], args['ENV']))
    log['{}_log'.format(args['ENV'])] = logging.getLogger(
        '{}_log'.format(args['ENV']))
    d_args = args
    for k in d_args.keys():
        log['{}_log'.format(args['ENV'])].info('{0}: {1}'.format(k, d_args[k]))

    torch.manual_seed(args['seed'])
    env = atari_env(args['ENV'], env_conf)
    reward_sum = 0
    start_time = time.time()
    num_tests = 0
    reward_total_sum = 0
    player = Agent(None, env, args, None)
    player.model = A3Clstm(
        player.env.observation_space.shape[0], player.env.action_space)
    player.state = player.env.reset()
    player.state = torch.from_numpy(player.state).float()
    player.model.eval()

    while True:
        if player.done:
            player.model.load_state_dict(shared_model.state_dict())
        if render:
            env.render()
        player.action_test()
        reward_sum += player.reward

        if player.done:
            num_tests += 1
            player.current_life = 0
            reward_total_sum += reward_sum
            reward_mean = reward_total_sum / num_tests
            log['{}_log'.format(args['ENV'])].info(
                "Time {0}, episode reward {1}, episode length {2}, reward mean {3:.4f}".
                format(
                    time.strftime("%Hh %Mm %Ss",
                                  time.gmtime(time.time() - start_time)),
                    reward_sum, player.eps_len, reward_mean))

            if reward_sum > args['SSL']:
                player.model.load_state_dict(shared_model.state_dict())
                state_to_save = player.model.state_dict()
                torch.save(state_to_save, '{0}{1}.dat'.format(
                    args['SMD'], args['ENV']))

            reward_sum = 0
            player.eps_len = 0
            state = player.env.reset()
            time.sleep(60)
            player.state = torch.from_numpy(state).float()


I am working on the file "OpenAI Reinforcement Learning Agent Plays Atari Games.ipynb".
The first task is:
Helper Functions Section
Logging function
Can you write Python code for it?

def setup_logger(logger_name, log_file, level=logging.INFO):

    ''' Makes use of the logging module'''
    #Instantiates the logging class
    l = logging.getLogger(logger_name)

    #Formatter
    formatter = logging.Formatter('%(asctime)s : %(message)s')

    #file handler
    fileHandler = logging.FileHandler(log_file, mode='w')
    fileHandler.setFormatter(formatter)

    #streamhandler
    streamHandler = logging.StreamHandler()
    streamHandler.setFormatter(formatter)

    #add the above handles to the logger instance
    l.setLevel(level)
    l.addHandler(fileHandler)
    l.addHandler(streamHandler)


I am working on the file "OpenAI Reinforcement Learning Agent Plays Atari Games.ipynb".
The first task is:
Import OpenAI Universe environment and gym
Import Pytorch for model
Can you write Python code for it?

import gym
from universe import vectorized
from universe.wrappers import Unvectorize, Vectorize

from gym.spaces.box import Box
from gym.configuration import undo_logger_setup

import torch
from torch.multiprocessing import Process
import torch.nn.functional as F

import torch.optim as optim


I am working on the file "OpenAI Reinforcement Learning Agent Plays Atari Games.ipynb".
The first task is:
Model
Using Google DeepMind's Idea.
Research Paper: https://arxiv.org/pdf/1602.01783.pdf
Asynchronous Advantage Actor-Critic (A3C)

The A3C algorithm was released by Google’s DeepMind group earlier this year, and it made a splash by… essentially obsoleting DQN. It was faster, simpler, more robust, and able to achieve much better scores on the standard battery of Deep RL tasks. On top of all that it could work in continuous as well as discrete action spaces. Given this, it has become the go-to Deep RL algorithm for new challenging problems with complex state and action spaces
Medium Article explaining A3c reinforcement learning
The Actor-Critic Structure

Many workers training and learning concurrently, and then updates global network with gradients
Process Flow

Long Short Term Memory Recurrent Neural Nets
Implementing LSTM and A3C with Pytorch
Created as a module and then imported
Can you write Python code for it?


I am working on the file "OpenAI Reinforcement Learning Agent Plays Atari Games.ipynb".
The first task is:
Playing BeamRider (4000 episodes)
Input
Can you write Python code for it?


I am working on the file "OpenAI Reinforcement Learning Agent Plays Atari Games.ipynb".
The first task is:
Playing Breakout (3000 episodes)
Input
Can you write Python code for it?


I am working on the file "OpenAI Reinforcement Learning Agent Plays Atari Games.ipynb".
The first task is:
Playing SpaceInvader (10000 episodes)
Input
Can you write Python code for it?

args = {'LR': 0.001, "G":0.99, "T":1.00, "S":1,"W":8,"NS":20,"M":1000000,"ENV":'SpaceInvaders-v0',
         "EC":'./settings.json',"SO":True,"L":True,"SSL":20, "OPT":"Adam","CL":False,
         "LMD":'./modeldata/',"SMD":"./modeldata/","LG":'./log/', "seed":42,"config":"SpaceInvaders"
        }

loadarguments()


I am working on the file "OpenAI Reinforcement Learning Agent Plays Atari Games.ipynb".
The first task is:
Playing the Atari Games Section
The best part
If it gives tensor errors, just run the cells again and somehow it works the 2nd time. This is because we don't have the full Share optimizer data generated yet
Load model is disabled on default so that you can observe how it learns through iteration,  Set L to True if you want to load the trained models and see how well it performs
Playing PacMan (10000 episodes)
Input Parameters
Can you write Python code for it?


I am working on the file "OpenAI Reinforcement Learning Agent Plays Atari Games.ipynb".
The first task is:
Putting it altogether
List of Games, pick one here and then edit the environment accordingly
Choose an Atari game and it has to be a 4D Tensor Game
Or if you don't know what that means, just guess and check
Can you write Python code for it?


I am working on the file "OpenAI Reinforcement Learning Agent Plays Atari Games.ipynb".
The first task is:
Read Json Object
Can you write Python code for it?

def read_config(file_path):
    """Read JSON config."""
    #use the context manager
    with open(file_path, 'r') as f:
        json_object = json.load(f)

    return json_object


I am working on the file "OpenAI Reinforcement Learning Agent Plays Atari Games.ipynb".
The first task is:
Run This to Train
Also logs it into a file
Can you write Python code for it?

processes = []

p = Process(target=test, args=(args, shared_model, env_conf))
p.start()
processes.append(p)

time.sleep(0.1)
for rank in range(0, args['W']):
    p = Process(
        target=train, args=(rank, args, shared_model, optimizer, env_conf))
    p.start()
    processes.append(p)
for p in processes:
    p.join()


I am working on the file "OpenAI Reinforcement Learning Agent Plays Atari Games.ipynb".
The first task is:
Run this to Start
Can you write Python code for it?


I am working on the file "OpenAI Reinforcement Learning Agent Plays Atari Games.ipynb".
The first task is:
Share grads between 2 models
More on this later
Can you write Python code for it?

def ensure_shared_grads(model, shared_model):
    for param, shared_param in zip(model.parameters(),
                                   shared_model.parameters()):
        if shared_param.grad is not None:
            return
        shared_param._grad = param.grad


I am working on the file "OpenAI Reinforcement Learning Agent Plays Atari Games.ipynb".
The first task is:
Shared Memory and optimization algorithims
As Part of the A3C Network, multiple workers will be working together to update a global network
RMSprop
RMSprop is an unpublished, adaptive learning rate method proposed by Geoff Hinton in Lecture 6e of his Coursera Class.
RMSprop and Adadelta have both been developed independently around the same time stemming from the need to resolve Adagrad's radically diminishing learning rates. RMSprop in fact is identical to the first update vector of Adadelta
RMSprop as well divides the learning rate by an exponentially decaying average of squared gradients. Hinton suggests γ
to be set to 0.9, while a good default value for the learning rate η is 0.001.
Can you write Python code for it?

from SharedOptimizers import SharedRMSprop


I am working on the file "OpenCV.ipynb".
The first task is:
20 Jun 17 test case for Matplotlib bug https://github.com/matplotlib/matplotlib/issues/7215/
behavior fixed by importing pandas (installing not enough as nothing seems to upgrade) even though it is not called
Can you write Python code for it?

! pip install --upgrade pandas
import pandas

%matplotlib inline
import numpy as np
import datetime as dt
import matplotlib as mpl
from matplotlib import pyplot as plt
from matplotlib.dates import date2num, MonthLocator, WeekdayLocator, DateFormatter


I am working on the file "OpenCV.ipynb".
The first task is:
Added Friday afternoon 15 Apr 17
Can you write Python code for it?

! pip install --upgrade pandas
%matplotlib inline
import matplotlib as mpl
from matplotlib import pyplot as plt
from matplotlib.dates import date2num, MonthLocator, WeekdayLocator, DateFormatter
import datetime as dt
import numpy as np
import pandas as pd

count = (dt.datetime.today() - dt.datetime(2016,11,15)).days
count

dates = [dt.datetime(2016,11,15) + dt.timedelta(days=i) for i in xrange(count)]
type(dates)

import numpy as np
dates_np = np.arange(np.datetime64('2016-11-15','D'),np.datetime64(dt.datetime.today(),'D'))

dates_np

type1 = np.random.randint(0,5,count)
type2 = np.random.randint(0,5,count)
type3 = np.random.randint(0,7,count)

type(type1)

#plt.figure(figsize=(20,7))
#plt.title('Testing', fontsize=16)
#plt.xlabel('Date', fontsize=16)
#plt.ylabel('Frequency', fontsize=16)
fig, ax = plt.subplots(1,1)
p1 = plt.bar(dates_np, type1, width=1, label='Type 1')
p2 = plt.bar(dates_np, type2, bottom = type1, width=1, label='Type 2')
p3 = plt.bar(dates_np, type3, bottom = type1 + type2, width=1, label='Type 3')
ax.xaxis_date()
ax.xaxis.set_major_locator(MonthLocator())
ax.xaxis.set_minor_locator(WeekdayLocator())
ax.xaxis.set_major_formatter(DateFormatter('%b %y'))
ax.set_title('Testing', fontsize=16)
ax.set_xlabel('Date')
ax.set_ylabel('Frequency')
ax.set_xlim(dates_np[0],dates_np[-1])
fig.set_size_inches(17,6)
fig.autofmt_xdate()
fig.tight_layout()
plt.legend((p1[0],p2[0],p3[0]), ('First', 'Second','Third'))
plt.show()

type(dates_np[0]),type(type1[0])


I am working on the file "OpenCV.ipynb".
The first task is:
Added Friday afternoon 17 Mar 17
Can you write Python code for it?

%matplotlib inline
import cv2
from matplotlib import pyplot as plt
import matplotlib.cm as cm
image = cv2.imread("Screenshot_2016-02-23-12-47-43.png")
fig, ax = plt.subplots()
fig.set_size_inches(4, 4)
#ax.axis([1280, 1400, 400, 200])
image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
plt.imshow(image_rgb)
plt.show()

image_gray = cv2.cvtColor(image_rgb, cv2.COLOR_RGB2GRAY)
fig, ax = plt.subplots(ncols=1, nrows=1, figsize=(2, 2))
up_right_gray_target = image_gray[210:310, 1280:1400]
plt.imshow(up_right_gray_target, cmap = cm.gray)
plt.show()

image_gray = cv2.cvtColor(image_rgb, cv2.COLOR_RGB2GRAY)
fig, ax = plt.subplots(ncols=1, nrows=1, figsize=(2, 2))
low_left_gray_target = image_gray[2412:2512,65:165]
plt.imshow(low_left_gray_target, cmap = cm.gray)
plt.show()

image_gray = cv2.imread("Screenshot_2016-02-23-12-47-43.png",0)
#targets = [up_right_gray_target,low_left_gray_target]
targets = [up_right_gray_target]
for tgt in targets:
    w, h = tgt.shape[::-1]
    res = cv2.matchTemplate(image_gray,tgt,cv2.TM_CCOEFF)
    res1= cv2.matchTemplate(image_gray,tgt,cv2.TM_CCOEFF_NORMED)
    min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(res)
    top_left = max_loc
    bottom_right = (top_left[0] + w, top_left[1] + h)
    cv2.rectangle(image_gray,top_left, bottom_right, 255, 2)
#fig, ax = plt.subplots(ncols=1, nrows=1, figsize=(10, 14))
#plt.imshow(image_gray, cmap = cm.gray)
#plt.show()

plt.figure(figsize=(16,9))
plt.subplot(1,2,1)
plt.imshow(res,cmap=cm.gray)
plt.subplot(1,2,2)
plt.imshow(res1,cmap=cm.gray)
plt.show()


I am working on the file "OpenCV.ipynb".
The first task is:
Added Thursday afternoon 23 Mar 17
Can you write Python code for it?

%matplotlib inline
import cv2
from matplotlib import pyplot as plt
import matplotlib.cm as cm
image_gray = cv2.imread("Screenshot_2016-02-23-12-47-43.png",0)
up_right_gray_target = image_gray[210:310, 1280:1400]
#targets = [up_right_gray_target,low_left_gray_target]
targets = [up_right_gray_target]
for tgt in targets:
    w, h = tgt.shape[::-1]
    res_TM_CCOEFF = cv2.matchTemplate(image_gray,tgt,cv2.TM_CCOEFF)
    res_TM_CCOEFF_NORMED = cv2.matchTemplate(image_gray,tgt,cv2.TM_CCOEFF_NORMED)
    res_TM_SQDIFF = cv2.matchTemplate(image_gray,tgt,cv2.TM_SQDIFF)
    res_TM_SQDIFF_NORMED = cv2.matchTemplate(image_gray,tgt,cv2.TM_SQDIFF_NORMED)
    res_TM_CORR = cv2.matchTemplate(image_gray,tgt,cv2.TM_CCORR)
    res_TM_CORR_NORMED = cv2.matchTemplate(image_gray,tgt,cv2.TM_CCORR_NORMED)
    min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(res_TM_SQDIFF_NORMED)
#    top_left = max_loc
    top_left = min_loc
    bottom_right = (top_left[0] + w, top_left[1] + h)
    cv2.rectangle(image_gray,top_left, bottom_right, 255, 2)
fig, ax = plt.subplots(ncols=1, nrows=1, figsize=(10, 14))
plt.imshow(image_gray, cmap = cm.gray)
plt.show()

fig = plt.figure(figsize=(12,29))
ax1 = fig.add_subplot(321)
plt.title('CCOEFF')
plt.imshow(res_TM_CCOEFF,cmap=cm.gray)
plt.subplot(3,2,2)
plt.title('CCOEFF_NORMED')
plt.imshow(res_TM_CCOEFF_NORMED,cmap=cm.gray)
plt.subplot(3,2,3)
plt.title('TM_SQDIFF')
plt.imshow(res_TM_SQDIFF,cmap=cm.gray)
plt.subplot(3,2,4)
plt.title('TM_SQDIFF_NORMED')
plt.imshow(res_TM_SQDIFF_NORMED,cmap=cm.gray)
plt.subplot(3,2,5)
plt.title('TM_CORR')
plt.imshow(res_TM_CORR,cmap=cm.gray)
plt.subplot(3,2,6)
plt.title('TM_CORR_NORMED')
plt.imshow(res_TM_CORR_NORMED,cmap=cm.gray)
plt.show()


I am working on the file "OpenCV.ipynb".
The first task is:
In this case it's necessary to change the image space from BGR (Blue, Green, Red) to RGB, so the correct flag is cv2.COLOR_BGR2RGB
Can you write Python code for it?

t0 = t.time()
cv_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
t1 = t.time()
dt_cv = t1-t0
print "Conversion took %0.5f seconds" % dt_cv

plt.imshow(cv_rgb)
plt.show()


I am working on the file "OpenCV.ipynb".
The first task is:
below from from http://matplotlib.org/users/text_intro.html
Can you write Python code for it?

# -*- coding: utf-8 -*-
import matplotlib.pyplot as plt

fig = plt.figure()
fig.suptitle('bold figure suptitle', fontsize=14, fontweight='bold')

ax = fig.add_subplot(111)
fig.subplots_adjust(top=0.85)
ax.set_title('axes title')

ax.set_xlabel('xlabel')
ax.set_ylabel('ylabel')

ax.text(3, 8, 'boxed italics text in data coords', style='italic',
        bbox={'facecolor':'red', 'alpha':0.5, 'pad':10})

ax.text(2, 6, r'an equation: $E=mc^2$', fontsize=15)

ax.text(3, 2, u'unicode: Institut f\374r Festk\366rperphysik')

ax.text(0.95, 0.01, 'colored text in axes coords',
        verticalalignment='bottom', horizontalalignment='right',
        transform=ax.transAxes,
        color='green', fontsize=15)

ax.plot([2], [1], 'o')
ax.annotate('annotate', xy=(2, 1), xytext=(3, 4),
            arrowprops=dict(facecolor='black', shrink=0.05))

ax.axis([0, 10, 0, 10])

plt.show()


I am working on the file "OpenCV_Recognize.ipynb".
The first task is:
Crop the image to make an initial test case
Can you write Python code for it?

img_full = PIL_Image.open('res_mario.jpg')
img_half = img_full.crop((0,0,img_full.size[0]/2,img_full.size[1]))
img_half.save('mario_test1.jpg')
IpyImage(filename='mario_test1.jpg')


I am working on the file "OpenMS/OpenMS.ipynb".
The first task is:
After that, we suppress possible warnings which might be issued due to insecure connection.
Can you write Python code for it?

import requests
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning) # suppress warnings


I am working on the file "OpenMS/OpenMS.ipynb".
The first task is:
Feature linking
This process is used to match corresponding features across all the MS runs.
The method of spinning up is similar to that of feature finder. However, the container ("featurelinkerunlabeledqt") will be dependent on all of the FeatureFinderMetabo containers in the previous step. This means that "featurelinkerunlabeledqt" will only run if all the FeatureFinderMetabo processes successfully finish.
Can you write Python code for it?

featureLinkerDir="featureLinkerDir"
if not os.path.exists(featureLinkerDir):
    os.makedirs(featureLinkerDir)
url_featureLinker="https://admin:"+password+"@"+control+"/chronos/scheduler/dependency"
featureLinkerInput=' '.join(["/data/"+featureFinderDir+"/" + fileName for fileName in featureFinderoutNames])
featureLinkerOutput="featureLinkerResult.consensusXML"
containerNamefeatureLinker="featureLinker"
parents="%s"%(containerNamesfeatureFinder)
json_featureLinker="""
    {
        "parents" : %s,
        "cpus": "0.45",
        "mem": "256",
        "epsilon" : "PT10M",
        "name" : "%s",
        "container": {
            "type": "DOCKER",
            "image": "payamemami/featurelinkerunlabeledqt",
            "volumes": [{
                "hostPath": "/mnt/container-volumes/jupyter/%s",
                "containerPath": "/data",
                "mode": "RW"
             }]
        },
        "command" : "FeatureLinkerUnlabeledQT -in %s -out /data/%s/%s -ini /data/%s",
        "owner" : "payam.emami@medsci.uu.se"
    }
    """ % (parents.replace("\'","\""),containerNamefeatureLinker,workingDir,featureLinkerInput,featureLinkerDir,featureLinkerOutput,paramDirectory+"/featureLinkerParam.ini")
response=requests.post(url_featureLinker, headers = {'content-type' : 'application/json'}, data=json_featureLinker, verify=False)
print("HTTP response code featureFinder: " + str(response.status_code))


I am working on the file "OpenMS/OpenMS.ipynb".
The first task is:
Finally, we can see the plot!
Can you write Python code for it?

from IPython.display import Image
Image(filename=plotDir+"/"+plotOutput)


I am working on the file "OpenMS/OpenMS.ipynb".
The first task is:
Formatting the CSV file in R
The CSV file from the previous step is converted to an "Excel" like file. This step will only run if TextExporter has finished.
Can you write Python code for it?

convertToDecyderDir="convertToDecyderDir"
if not os.path.exists(convertToDecyderDir):
    os.makedirs(convertToDecyderDir)
url_convertToDecyder="https://admin:"+password+"@"+control+"/chronos/scheduler/dependency"
convertToDecyderInput=textExporterDir+"/"+textExporterOutput
convertToDecyderOutput=""
containerNameconvertToDecyder="converttodecyder"
textExporterOutputName="textExporterOutput.xls"
parents="%s"%(containerNameTextExporter)
json_textExporter="""
    {
        "parents" : ["%s"],
        "cpus": "0.45",
        "mem": "256",
        "epsilon" : "PT10M",
        "name" : "%s",
        "container": {
            "type": "DOCKER",
            "image": "payamemami/converttodecyder",
            "volumes": [{
                "hostPath": "/mnt/container-volumes/jupyter/%s",
                "containerPath": "/data",
                "mode": "RW"
             }]
        },
        "command" : "Rscript convert_to_decyder.R -in=/data/%s -out=/data/%s -name=%s",
        "owner" : "payam.emami@medsci.uu.se"
    }
    """ % (parents,containerNameconvertToDecyder,workingDir,convertToDecyderInput,convertToDecyderDir,textExporterOutputName)
response=requests.post(url_textExporter, headers = {'content-type' : 'application/json'}, data=json_textExporter, verify=False)
print("HTTP response code featureFinder: " + str(response.status_code))


I am working on the file "OpenMS/OpenMS.ipynb".
The first task is:
Next, we create two folders ("rawFiles" and "paramFiles") and download the files using the links provided in "data_list.txt" and "params.txt" to their corresponding folders.
Can you write Python code for it?

import csv
import os
rawDirectory="rawFiles"
if not os.path.exists(rawDirectory):
    os.makedirs(rawDirectory)
paramDirectory="paramFiles"
if not os.path.exists(paramDirectory):
    os.makedirs(paramDirectory)
param_path=[]
with open('params.txt','r') as f:
    reader=csv.reader(f,delimiter='\t')
    for row in reader:
        urllib.request.urlretrieve(row[1],paramDirectory+"/"+row[0])
        param_path.append(row)

data_path=[]
with open('data_list.txt','r') as f:
    reader=csv.reader(f,delimiter='\t')
    for row in reader:
        urllib.request.urlretrieve(row[1],rawDirectory+"/"+row[0])
        data_path.append(row)


I am working on the file "OpenMS/OpenMS.ipynb".
The first task is:
Using the formatted file, we plot the data using R functions from the ggplot package.
Can you write Python code for it?

plotDir="plotDir"
if not os.path.exists(plotDir):
    os.makedirs(plotDir)
url_plot="https://admin:"+password+"@"+control+"/chronos/scheduler/dependency"
plotInput=convertToDecyderDir+"/"+textExporterOutputName
plotOutput="plot.png"
containerNameplot="plotmsdata"
parents="%s"%(containerNameconvertToDecyder)
json_plot="""
    {
        "parents" : ["%s"],
        "cpus": "0.45",
        "mem": "256",
        "epsilon" : "PT10M",
        "name" : "%s",
        "container": {
            "type": "DOCKER",
            "image": "payamemami/plotmsdata",
            "volumes": [{
                "hostPath": "/mnt/container-volumes/jupyter/%s",
                "containerPath": "/data",
                "mode": "RW"
             }]
        },
        "command" : "Rscript plotMSData.R -in=/data/%s -out=/data/%s/%s -pattern=intensity_ -impute=T -plottype=BOX -width=20 -height=20 -imagetype=PNG -log=T",
        "owner" : "payam.emami@medsci.uu.se"
    }
    """ % (parents,containerNameplot,workingDir,plotInput,plotDir,plotOutput)
response=requests.post(url_plot, headers = {'content-type' : 'application/json'}, data=json_plot, verify=False)
print("HTTP response code featureFinder: " + str(response.status_code))


I am working on the file "OpenMS/OpenMS.ipynb".
The first task is:
We then download two text files (called "data_list.txt" and "params.txt") where "data_list.txt" contains the names of the raw MS files (mzML) and download links for each of the files.
In addition, since OpenMS tools reads parameters as ".ini" files, we also need to download these files from the links provided in "params.txt".
Can you write Python code for it?


I am working on the file "OpenStreetMap.ipynb".
The first task is:
1 - Data Preparation
Map Area: Saint-Joseph - Île de La Réunion
- http://www.openstreetmap.org/relation/1282272#map=12/-21.2918/55.6440
- http://overpass-api.de/api/map?bbox=55.4871,-21.4039,55.8009,-21.1796
Can you write Python code for it?

Image(filename='LaReunion.png', width=300, height=300)

from openstreetmap import audit, shape
from openstreetmap.data_gouv_fr import postalcode, fantoir

OSM_FILE = "data/Saint-Joseph.La-Reunion.osm"


I am working on the file "OpenStreetMap.ipynb".
The first task is:
1.1 - Auditing Data
1.1.1 - node and way xml tags
A quick look at the different 'tag' available in our data set
Can you write Python code for it?


I am working on the file "OpenStreetMap.ipynb".
The first task is:
2.2 - MongoDB Import
Can you write Python code for it?

# Drop previous database if any
!mongo OpenStreetMap --eval "db.dropDatabase()"

!mongoimport -d OpenStreetMap -c LaReunion --file data/Saint-Joseph.La-Reunion.osm.json


I am working on the file "OpenStreetMap.ipynb".
The first task is:
Check that we have the same count as the result of our python cleansing code
Can you write Python code for it?


I am working on the file "OpenStreetMap.ipynb".
The first task is:
Count the number of "xml node" imported
Can you write Python code for it?

pipeline = [
    {"$unwind": "$type"},
    {"$group": {"_id": "$type", "count": {"$sum": 1}}},
    {"$sort": SON([("count", -1), ("_id", -1)])}
]
nodes = pd.DataFrame(list(db.LaReunion.aggregate(pipeline)))
nodes.columns = ["node", "count"]
nodes.head()


I am working on the file "OpenStreetMap.ipynb".
The first task is:
Few tags are to be ignored for the rest of cleansing process as tag names contains problematic characters.
The auditing will focus on the following tags (to illustrate the different checks that could be put in place for cleansing) from the remaining tags:
- population, capacity, elevation, direction, phone, postal_code
- addr:city, addr:postcode, addr:housenumber, addr:street
other tags will be included "as-is".

note: City, Post Code, Street name and type will be checked against official data sources (see references in audit.py)
Can you write Python code for it?

# Official databases are filtered to limit the data for the area of interest aka 974**
# All tag values not compliants with the rules defined are listed in associated files under audit folder
# The diffirents files will be manually checked and updated for the next step in cleansing process
!openstreetmap/audit.py -i -o data/Saint-Joseph.La-Reunion.osm -f data/FANTOIR1016 -a 974 -u audit


I am working on the file "OpenStreetMap.ipynb".
The first task is:
From the auditing, the following problems have been identified:
1. Some street types are wrongly spelled or abbreviated
2. No street type information for some street (all street listed under one single entry 'None' in our audit results)
3. Street name case not consistent (Full upercase, lowercase or mix)
4. House number contain more than one number (grouping of several houses is not considered as an issue for the auditing process and the validation rule has been updated to take it into account)
5. Same issue for the phone (e.g.: "0692407639;0692407637")
6. Some city are wrongly spelled or case not consistent (Full upercase, lowercase or mix)
7. An extra space inside postcode value + an unknown postcode value: 97447 to be replaced by 97442 according to the city name
2 - From XML to JSON to MongoDB
2.1 - Create JSON file
Based on the cleansing identified in previous section, Some values are partially or fully updated during the JSON file generation from the data recovered from associated files under update folder
Can you write Python code for it?

!openstreetmap/shape.py -o data/Saint-Joseph.La-Reunion.osm -u update


I am working on the file "OpenStreetMap.ipynb".
The first task is:
Just peek the previously selected sample example from our python cleansing code
Can you write Python code for it?


I am working on the file "OpenStreetMap.ipynb".
The first task is:
One remaining issue: a 'nan' value that could not be updated without extra information
Can you write Python code for it?


I am working on the file "OpenStreetMap.ipynb".
The first task is:
The above code addressed most remaining issues not handled during the initial cleansing step, for the City and PostCode fields, what about street? What are the remaining issues?
Can you write Python code for it?

# To compare with the official database, some french characters have to be translated to their corresponding ASCII
# what a pity for a french official database
xtd = {ord(u'’'): u"'", ord(u'é'): u'e', ord(u'è'): u'e', ord(u'É'): u'E',}
def tr(x):
    return x.translate(xtd).upper()

ways_referential = fantoir.FANTOIR().ways("data/FANTOIR1016", "974")
addresses["CHECKED"] = addresses.street.dropna().apply(lambda x: tr(x) in ways_referential["FULL_NAME"].values)

# Generate a step 2 mapping file for manual street name update ... updated file to be moved to update folder
mapping = addresses[addresses.CHECKED == False]["street"].unique()
df = pd.DataFrame.from_dict({ "OLD": mapping, "NEW": mapping })
df.to_csv("audit/street_clean_step2.csv",
          encoding='utf-8',
          index=False,
          quoting=csv.QUOTE_ALL)


I am working on the file "OpenStreetMap.ipynb".
The first task is:
There is some missing data for all selected attributes (with one Nan for postcode)
3.1 - City
Let see if we can identify any problems with city attribute...  Can we recover the missing data for the city?

note: This will be possible if the postcode is not null... to crosscheck with the official postcal code database
Can you write Python code for it?

from openstreetmap.data_gouv_fr import postalcode
codes = postalcode.PostalCode("data/laposte_hexasmal.csv").localityByPostcode()
addresses.loc[:, "city"] = addresses.postcode.dropna().apply(lambda x: codes[int(x)].title())
addresses[(addresses.city.isnull() == True) & (addresses.postcode.isnull() == False)].city.count()

addresses.city.unique()


I am working on the file "OpenStreetMap.ipynb".
The first task is:
We can consider having 1 top contributor followed by 5 high contributors... let see the distribution of the contribution for all users
Can you write Python code for it?

pipeline = [
    { "$project": { "_id": False, "user": "$created.user" } },
    { "$group": { "_id": "$user", "count": { "$sum": 1 } } },
    { "$sort": SON([("count", -1), ("_id", -1)]) },
    { "$project": { "_id": "$count"} }
]
contributions = pd.DataFrame(list(db.LaReunion.aggregate(pipeline)))
contributions.describe().T


I am working on the file "OpenStreetMap.ipynb".
The first task is:
We have identify some rules to (partially) update the addresses, why not updating the mongo database...
 Postal Code and City
Can you write Python code for it?

for n in db.LaReunion.find({ "type": "node",
                            "address": { "$exists": True },
                            "address.postcode": { "$exists": True } }):
    postcode = n["address"]["postcode"].replace(' ', '')
    db.LaReunion.update_one({ "_id": n["_id"] }, { "$set": { "address.postcode":  postcode } }, upsert=False)

for n in db.LaReunion.find({ "type": "node",
                            "address": { "$exists": True },
                            "address.postcode": { "$exists": True } }):
    city = codes[int(n["address"]["postcode"])]
    db.LaReunion.update_one({ "_id": n["_id"] }, { "$set": { "address.city":  city } }, upsert=False)

pipeline = [
    { "$match": { "type": "node", "address": { "$exists": True }, "address.postcode": { "$exists": True } } },
    { "$project": { "_id": False,
                   "city": "$address.city",
                   "housenumber": "$address.housenumber",
                   "postcode": "$address.postcode",
                   "street": "$address.street" } },
    { "$limit": 10 }
]
pd.DataFrame(list(db.LaReunion.aggregate(pipeline)))

pretty(list(db.LaReunion.find({
        "type": "node",
        "address": { "$exists": True },
        "address.city": { "$exists": False },
        "address.postcode": { "$exists": False }}).limit(1)))


I am working on the file "OpenStreetMapsExample.ipynb".
The first task is:
And then download them.
Can you write Python code for it?

tiff_files = []
asset_type = "_3B_Visual"
# check if our scenes have been activated
if True: #scenes_are_active(activated):
    for to_download,name in zip(activated,to_get):
        # create the product name
        name = name + asset_type + ".tif"
        # if the product exists locally
        if( os.path.isfile(name) ):
            # do nothing
            print "We have scene {0} already, skipping...".format(name)
            tiff_files.append(name)
        elif to_download["status"] == "active":
            # otherwise download the product
            print "Downloading {0}....".format(name)
            fname = download_and_save(client,to_download)
            tiff_files.append(fname)
            print "Download done."
        else:
            print "Could not download, still activating"
else:
    print "Scenes aren't ready yet"

sorted(tiff_files)
print tiff_files


I am working on the file "OpenStreetMapsExample.ipynb".
The first task is:
Let's get going with Open Street Maps.

Open Street Maps is a huge and open collection of data about the Earth.
OSM is free to query. The interfaces are powerful, but hella cryptic.
Let's say we had a pixel in an image and we wanted to know what in the world was at that pixel.
We can use the Open Street Maps Nominatim function to look up what is there, like Google maps.
We can also use the OSM interface to find the 'nodes' near our pixel.
OSM Nominatim works through Lat Long values. To get these lat long values we are going to through UTM coordinates.
To get correct the UTM values we'll need to ask GDAL what our UDM zone is.
Can you write Python code for it?

infile = tiff_files[0]
# Open the file
gtif = gdal.Open(infile)
# Get the project reference object this knows the UTM zone
reff = gtif.GetProjectionRef()
# arr is the actual image data.
arr = gtif.ReadAsArray()
# Trans is our geo transfrom array.
trans = gtif.GetGeoTransform()
# print the ref object
print reff
# find our UTM zone
i = reff.find("UTM")
print reff[i:i+12]


I am working on the file "OpenStreetMapsExample.ipynb".
The first task is:
Let's put it all together

We're going to use the scenes we downloaded earlier as our input and build a little image for every park in San Francisco!
We just have to make a few file names and call the functions above.
If we really wanted to get fancy we could do this for every image that has our park and make a sick movie or lots of different types of a analysis.
Can you write Python code for it?

geo_json_files = []
tif_file_names = []
unique_park_names = list(set(park_df["name"].tolist()))
for name in list(unique_park_names):
    # Generate our file names
    geojson_name = "./parks/"+name.replace(" ","_")+".geojson"
    tif_name = "./parks/"+name.replace(" ","_")+".tif"
    # write geojson
    write_geojson_by_name(park_df,name,geojson_name)
    # write to park file
    crop_scenes_to_geojson(geojson_name,tiff_files,tif_name)
    # Save the results to lists
    geo_json_files.append(geojson_name)
    tif_file_names.append(tif_name)


I am working on the file "OpenStreetMapsExample.ipynb".
The first task is:
Let's review from last time.

We'll query the Planet API and get a list of scenes.
We'll then use pandas and shapely to clean up and filter the results
We'll then render the footprints of the good scenes over our AOI
Can you write Python code for it?

print AOIs[1]
myAOI = AOIs[1]["geometry"]

# build a query using the AOI and
# a cloud_cover filter that excludes 'cloud free' scenes

old = datetime.datetime(year=2017,month=1,day=1)

query = filters.and_filter(
    filters.geom_filter(myAOI),
    filters.range_filter('cloud_cover', lt=5),
    filters.date_range('acquired', gt=old)
)

# build a request for only PlanetScope imagery
request = filters.build_search_request(
    query, item_types=['PSScene3Band']
)

# if you don't have an API key configured, this will raise an exception
result = client.quick_search(request)
scenes = []
planet_map = {}
for item in result.items_iter(limit=500):
    planet_map[item['id']]=item
    props = item['properties']
    props["id"] = item['id']
    props["geometry"] = item["geometry"]
    props["thumbnail"] = item["_links"]["thumbnail"]
    scenes.append(props)
scenes = pd.DataFrame(data=scenes)
# now let's clean up the datetime stuff
# make a shapely shape from our aoi
portland = shape(myAOI)
footprints = []
overlaps = []
# go through the geometry from our api call, convert to a shape and calculate overlap area.
# also save the shape for safe keeping
for footprint in scenes["geometry"].tolist():
    s = shape(footprint)
    footprints.append(s)
    overlap = 100.0*(portland.intersection(s).area / portland.area)
    overlaps.append(overlap)
# take our lists and add them back to our dataframe
scenes['overlap'] = pd.Series(overlaps, index=scenes.index)
scenes['footprint'] = pd.Series(footprints, index=scenes.index)
# now make sure pandas knows about our date/time columns.
scenes["acquired"] = pd.to_datetime(scenes["acquired"])
scenes["published"] = pd.to_datetime(scenes["published"])
scenes["updated"] = pd.to_datetime(scenes["updated"])

scenes = scenes[scenes['overlap']>0.9]

print len(scenes)
# now let's clean up the datetime stuff
# make a shapely shape from our aoi
portland = shape(myAOI)
footprints = []
overlaps = []
# go through the geometry from our api call, convert to a shape and calculate overlap area.
# also save...


I am working on the file "OpenStreetMapsExample.ipynb".
The first task is:
Let's take a look at the first few parks!

matplotlib and tifs can be a bit heavy handed.
imma teach you a protip use image magick and the built in image display.
Use subprocess to tell imagemagick to convert tifs to jpg.
Then load and display the images.
WARNING: do not use imagemagick to modify geotiffs!
Can you write Python code for it?

magic = ["mogrify","-format", "jpg", "./parks/*.tif"]
subprocess.call(magic)
for p in tif_file_names[0:20]:
    print p
    display(Image(p.replace('tif','jpg')))


I am working on the file "OpenStreetMapsExample.ipynb".
The first task is:
Let's take a look at what this looks like in terms of images.
Can you write Python code for it?

imgs = [p[0] for p in paired]
for p in imgs[0:50]:
    print p[8:-4].replace("_"," ")
    display(Image(p.replace('tif','jpg')))


imgs = [p[0] for p in paired]
for p in imgs[0:35]:
    print p[8:-4].replace("_"," ")
    display(Image(p.replace('tif','jpg')))


I am working on the file "OpenStreetMapsExample.ipynb".
The first task is:
Not let's do some quick analytics -- your code goes here.

For completeness let's do some basic image processing.
For each of parks we are going to calculate the average "greeness" per pixel over the other two channels.
We do this as it controls for white pixels, like clouds.
Since there are black pixels we'll have to controll for that by only using only the non-black pixels.
We'll use numpy here, but scikit image and OpenCV have many more features.
It is also worth noting that the visual product is probably only useful for calculating areas. If you want to do real science use the Analytics products.
The real way to do this is to calculate a Normalized Difference Vegetation Index (NDVI) using the analytic product.
Here is an example of NDVI calculations.
Can you write Python code for it?

def load_image3(filename):
    """Return a 3D (r, g, b) numpy array with the data in the specified TIFF filename."""
    path = os.path.abspath(os.path.join('./', filename))
    if os.path.exists(path):
        with rasterio.open(path) as src:
            b,g,r,mask = src.read()
            return np.dstack([b, g, r])

def get_avg_greeness(filename):
    retVal = -1.0
    try:
        # load the image
        img = load_image3(filename)
        if img is not None:
            # add all the channels together, black pixels will still be zero
            # this isn't a perfect method but there are very few truly black spots
            # on eart
            black_like_my_soul = np.add(np.add(img[:,:,0],img[:,:,1]),img[:,:,2])
            # sum up the not black pixels
            not_black = np.count_nonzero(black_like_my_soul)
            # sum up all the green
            img = np.array(img,dtype='int16')
            total_green = np.sum(img[:,:,1]-((np.add(img[:,:,0],img[:,:,2])/2)))
            # calculate our metric
            if total_green != 0 and not_black > 0:
                retVal = total_green / float(not_black)
        return retVal
    except Exception as e:
        print e
        return -1.0

greens = [get_avg_greeness(f) for f in tif_file_names]
print greens

paired = zip(tif_file_names,greens)
paired.sort(key=(lambda tup: tup[1]))
paired.reverse()
labels = [p[0][8:-4].replace("_"," ") for p in paired]
data = [p[1] for p in paired]
plt.figure(figsize=(20,6))
xlocations = np.array(range(len(paired)))+0.5
width = 1
plt.bar(xlocations, data, width=width)
plt.yticks(range(-1,25,1))
plt.xticks(xlocations+ width/2, labels)
plt.xlim(0, xlocations[-1]+width*2)
plt.ylim(-2,np.max(data)+1)
plt.title("Greeness over Average Red and Blue Channel by Park")
plt.gca().get_xaxis().tick_bottom()
plt.gca().get_yaxis().tick_left()
xa = plt.gca()
xa.set_xticklabels(xa.xaxis.get_majorticklabels(), rotation=90)
plt.show()


def load_image3(filename):
    """Return a 3D (r, g, b) numpy array with the data in the specified TIFF filename."""
    path = os.path.abspath(os.path.join('./', filename))
    if os.path.exists(path):
        with rasterio.open(path) as src:
            b,g,r,mask = src.read()
            return np.dstack([b, g, r])

def get_avg_greeness(filename):
    retVal = -1.0
    try:
        # load the image
        img = load_image3(filename)
        if img is not None:
            # add all the channels together, black pixels will still be zero
            # this isn't a perfect method but there are very few truly black spots
            # on eart
            black_like_my_soul = np.add(np.add(img[:,:,0],img[:,:,1]),img[:,:,2])
            # sum up the not black pixels
            not_black = np.count_nonzero(black_like_my_soul)
            # sum up all the green
            img = np.array(img,dtype='int16')
            total_green = np.sum(img[:,:,1]-((np.add(img[:,:,0],img[:,:,2])/2)))
            # calculate our metric
            if total_green != 0 and not_black > 0:
                retVal = total_green / float(not_black)
        return retVal
    except Exception as e:
        print e
        return -1.0

greens = [get_avg_greeness(f) for f in tif_file_names]
print greens

paired = zip(tif_file_names,greens)
paired.sort(key=(lambda tup: tup[1]))
paired.reverse()
labels = [p[0][8:-4].replace("_"," ") for p in paired]
data = [p[1] for p in paired]
plt.figure(figsize=(20,6))
xlocations = np.array(range(len(paired)))+0.5
width = 1
plt.bar(xlocations, data, width=width)
plt.yticks(range(-1,25,1))
plt.xticks(xlocations+ width/2, labels)
plt.xlim(0, xlocations[-1]+width*2)
plt.ylim(-2,np.max(data)+1)
plt.title("Greeness over Average Red and Blue Per Park")
plt.gca().get_xaxis().tick_bottom()
plt.gca().get_yaxis().tick_left()
xa = plt.gca()
xa.set_xticklabels(xa.xaxis.get_majorticklabels(), rotation=90)
plt.show()


I am working on the file "OpenStreetMapsExample.ipynb".
The first task is:
Now let's find the big parks.

The pandas dataframe can have multiple enteries per park.
We can use the group by command to sum up these disparate areas.
Finally we'll output the results
Can you write Python code for it?

park_sz = park_df.groupby("name").sum()
park_sz = park_sz.sort_values(by='area',ascending=[0])
display(park_sz)


I am working on the file "OpenStreetMapsExample.ipynb".
The first task is:
Now let's plot over our slippy map.

We'll calculate a non-linear opacity per park and then use that for plotting.
Can you write Python code for it?

opacity_map = {}
gmax = np.max(greens)
gmin = np.min(greens)
# this is a nonlinear mapping
opacity = [np.clip((float(g**2)-gmin)/float(gmax-gmin),0,1) for g in greens]
for op,name in zip(opacity,imgs):
    opacity_map[name]=op

m = Map(
    center=center,
    zoom=zoom,
    default_tiles = planetMapTiles # Uncomment to use Planet.com basemap
)
dc = DrawControl(
    polygon = polygon,
    rectangle = rectangle
)
# Initialize an action counter variable
actionCount = 0
AOIs = {}

# Register the draw controls handler
def handle_draw(self, action, geo_json):
    # Increment the action counter
    global actionCount
    actionCount += 1
    # Remove the `style` property from the GeoJSON
    geo_json['properties'] = {}
    # Convert geo_json output to a string and prettify (indent & replace ' with ")
    geojsonStr = json.dumps(geo_json, indent=2).replace("'", '"')
    AOIs[actionCount] = json.loads(geojsonStr)

# Attach the draw handler to the draw controls `on_draw` event
dc.on_draw(handle_draw)
m.add_control(dc)
m

for p in portland_parks["features"]:
    t = "./parks/"+p["properties"]["name"].replace(" ","_") + ".tif"
    feat = {'geometry':p["geometry"],"properties":{
            'style':{'color': "#00FF00",'fillColor': "#00FF00",'fillOpacity': opacity_map[t],'weight': 1}},
            'type':u"Feature"}
    # convert to geojson
    gjson = GeoJSON(data=feat)
    # add it our map
    m.add_layer(gjson)
m


I am working on the file "OpenStreetMapsExample.ipynb".
The first task is:
Now let's query a point on our scene and see what OSM tells us.

First we'll define a pixel position
We'll use GDAL to open the scene and then map a pixel to UTM
We'll then convert the UTM value to Lat / Lon using the UTM region we found before.
Then we'll instantiate a Nominatim object and perform a revers lookup and print the results.
We'll then use the OSM Api to get node at this place.
Can you write Python code for it?

pos = [3000,1400] # this is the pixel we want info abou
ds = gdal.Open(infile)
# take the GDAL info and make it into UTM
my_utm = pixel2utm(ds,pos[0],pos[1])
# convert UTM into Lat Long
# need to figure out how to get zone info
my_lla = utm.to_latlon(my_utm[0],my_utm[1],10,"N")
# do the lat long look up from OSM
geolocator = Nominatim()
# reverse look up the are based on lat lon
location = geolocator.reverse("{0},{1}".format(my_lla[0],my_lla[1]))
# print location info
print location.address
print location.raw
# get the OSM ID info
osm_id = int(location.raw["place_id"])
print osm_id
# create an interface to the OSM API
MyApi = OsmApi()
# Look up our position
print MyApi.NodeGet(osm_id)


I am working on the file "OpenStreetMapsExample.ipynb".
The first task is:
Now to the meat of the problem.

Our goal is to get each park as a small image so we can analyze it.
We'll write a function to create a geojson file from our big geojson file
We'll also write a function that takes in our scene list, an input geojson file, and calls gdal warp to generate our small park image.
Can you write Python code for it?

def write_geojson_by_name(df,name,outfile):
    """
    Take in a dataframe, a park name, and an output file name
    Save the park's geojson to the specified file.
    """
    temp = df[df["name"]==name]
    to_write = {"type": "FeatureCollection",
                "features": temp["geojson"].tolist()}
    with open(outfile,'w') as fp:
        fp.write(json.dumps(to_write))
def crop_scenes_to_geojson(geojson,scenes,out_name):
    """
    Take in a geojson file, a list of scenes, and an output name
    Call gdal and warp the scenes to match the geojson file and save the results to outname.
    """
    commands = ["gdalwarp", # t
           "-t_srs","EPSG:3857",
           "-cutline",geojson,
           "-crop_to_cutline",
           "-tap",
            "-tr", "3", "3"
           "-overwrite"]
    for tiff in scenes:
        commands.append(tiff)
    commands.append(out_name)
    print " ".join(commands)
    subprocess.call(commands)


I am working on the file "OpenStreetMapsExample.ipynb".
The first task is:
Now we'll activate our scenes
Can you write Python code for it?

to_get = scenes["id"][0:7].tolist()
activated = []
# for each scene to get
for scene in to_get:
    # get the product
    product_types = get_products(client,scene)
    for p in product_types:
        # if there is a visual product
        if p == "visual": # p == "basic_analytic_dn"
            print "Activating {0} for scene {1}".format(p,scene)
            # activate the product
            _,product = activate_product(client,scene,product=p)
            activated.append(product)


I am working on the file "OpenStreetMapsExample.ipynb".
The first task is:
Now we'll add in our boiler plate activation code for reference.
Can you write Python code for it?

def get_products(client, scene_id, asset_type='PSScene3Band'):
    """
    Ask the client to return the available products for a
    given scene and asset type. Returns a list of product
    strings
    """
    out = client.get_assets_by_id(asset_type,scene_id)
    temp = out.get()
    return temp.keys()

def activate_product(client, scene_id, asset_type="PSScene3Band",product="analytic"):
    """
    Activate a product given a scene, an asset type, and a product.

    On success return the return value of the API call and an activation object
    """
    temp = client.get_assets_by_id(asset_type,scene_id)
    products = temp.get()
    if( product in products.keys() ):
        return client.activate(products[product]),products[product]
    else:
        return None

def download_and_save(client,product):
    """
    Given a client and a product activation object download the asset.
    This will save the tiff file in the local directory and return its
    file name.
    """
    out = client.download(product)
    fp = out.get_body()
    fp.write()
    return fp.name

def scenes_are_active(scene_list):
    """
    Check if all of the resources in a given list of
    scene activation objects is read for downloading.
    """
    retVal = True
    for scene in scene_list:
        if scene["status"] != "active":
            print "{} is not ready.".format(scene)
            return False
    return True


I am working on the file "OpenStreetMapsExample.ipynb".
The first task is:
Now we'll update our slippy map.

Just toss the aois in, just like our scene footprints.
Can you write Python code for it?

for p in portland_parks["features"]:
    feat = {'geometry':p["geometry"],"properties":{
            'style':{'color': "#00FF00",'fillColor': "#00FF00",'fillOpacity': 0.0,'weight': 1}},
            'type':u"Feature"}
    # convert to geojson
    gjson = GeoJSON(data=feat)
    # add it our map
    m.add_layer(gjson)
m


for p in sf_parks["features"]:
    feat = {'geometry':p["geometry"],"properties":{
            'style':{'color': "#00FF00",'fillColor': "#00FF00",'fillOpacity': 0.2,'weight': 1}},
            'type':u"Feature"}
    # convert to geojson
    gjson = GeoJSON(data=feat)
    # add it our map
    m.add_layer(gjson)
m


I am working on the file "OpenStreetMapsExample.ipynb".
The first task is:
Now, well, OSM is hard.

This is I wanted to show where to programaticaly query OSM for all sorts of data.
Turns out that it is a lot harder than it should be, especially if you want to work with GeoJson.
Out of scope for this talk, but let's punt.
OSM has a feature called Overpass. It is like the most convoluted Google maps ever using a very complex query language that I still don't grok.
We're going to use it to get all of the parks in Portland as GeoJSON using the web interface called Overpass Turbo.
Let's take a look at that.
Here's the query to run. Then export as GeoJSON
[bbox:{{bbox}}][timeout:1800];
way["leisure"="park"];map_to_area->.a;
foreach(
  (._;>;);
  is_in;
  way(pivot)["leisure"="park"];
  out geom;
);

Let's load up our park data

Load the file using GeoPandas (some syntactic sugar on Pandas).
Also load the raw json, and chunk out each park.
Update the area value because there is no value, not really useful except as a proxy measurement.
Update and sort our data frame.
Can you write Python code for it?

import geopandas as gpd
fname = "./portland_parks_small.geojson"
park_df = gpd.read_file(fname)
portland_parks = json.load(open(fname,'r'))
# raw geojson works better with GDAL
geojson = [p for p in portland_parks["features"]]
# no area out of the box
p = [p.area for p in park_df["geometry"].tolist()]
park_df["area"] = pd.Series(p)
park_df["geojson"] = pd.Series(geojson)
park_df.sort_values(['area',], ascending=[1])
park_df.head()
#len(park_df)
#print park_df["wikipedia"].dropna()


I am working on the file "OpenStreetMapsExample.ipynb".
The first task is:
Now, well, OSM is hard.

This is I wanted to show where to programaticaly query OSM for all sorts of data.
Turns out that it is a lot harder than it should be, especially if you want to work with GeoJson.
Out of scope for this talk, but let's punt.
OSM has a feature called Overpass. It is like the most convoluted Google maps ever using a very complex query language that I still don't grok.
We're going to use it to get all of the parks in San Francisco as GeoJSON using the web interface called Overpass Turbo.
Let's take a look at that.
Here's the query to run. Then export as GeoJSON
[bbox:{{bbox}}][timeout:1800];
way["leisure"="park"];map_to_area->.a;
foreach(
  (._;>;);
  is_in;
  way(pivot)["leisure"="park"];
  out geom;
);

Let's load up our park data

Load the file using GeoPandas (some syntactic sugar on Pandas).
Also load the raw json, and chunk out each park.
Update the area value because there is no value, not really useful except as a proxy measurement.
Update and sort our data frame.
Can you write Python code for it?

import geopandas as gpd
fname = "./sf_parks.geojson"
park_df = gpd.read_file(fname)
sf_parks = json.load(open(fname,'r'))
# raw geojson works better with GDAL
geojson = [p for p in sf_parks["features"]]
# no area out of the box
p = [p.area for p in park_df["geometry"].tolist()]
park_df["area"] = pd.Series(p)
park_df["geojson"] = pd.Series(geojson)
park_df.sort_values(['area',], ascending=[1])
park_df.head()
#len(park_df)
#print park_df["wikipedia"].dropna()


I am working on the file "Operationalizing Models with WML and Scikit-Learn.ipynb".
The first task is:
3.1.3: Split the data set for training and testing
As the target and feature columns has been defined, you can now split the data set into two sets that will be used for training the model and for testing the trained model.
Can you write Python code for it?

heart_train, heart_test, target_train, target_test = cross_validation.train_test_split(features_df, heart_data_df.loc[:,'diagnosed'], test_size=0.33, random_state=0)


I am working on the file "Operationalizing Models with WML and Scikit-Learn.ipynb".
The first task is:
3.1.4: Construct DMatix objects
DMatrix is the data interface provided by the XGBoost library. The training data and test data are converted as DMatrix objects to perform training and to make predictions. The DMatrix objects can be created from various data formats, such as Numpy arrays, Pandas data frames, or a Scipy sparse array. For more information about the DMatrix interface, see Python Package Introduction.
Next, prepare the DMatrix objects for training and testing based on the training and test data that was split above
Can you write Python code for it?

dm_train = xgb.DMatrix(heart_train, label=target_train)
dm_test = xgb.DMatrix(heart_test)


I am working on the file "Operationalizing Models with WML and Scikit-Learn.ipynb".
The first task is:
4.1: Save a XGBoost model in the Machine Learning Repository
In this subsection you will learn how to save a model artifact to your Watson Machine Learning instance by using the Watson Machine Learing repository Python client package.
Create an artifact and save it to the machine learning repository by running the following cells:
Can you write Python code for it?

# Check if props is mandatory
props1 = MetaProps({MetaNames.AUTHOR_NAME:"YOUR_name", MetaNames.AUTHOR_EMAIL:"Your_email@email.com"})

model_artifact = MLRepositoryArtifact(xgb_model, name='XGB_Heart_Disease_Detection', meta_props=props1)

saved_model = ml_repository_client.models.save(model_artifact)

saved_model_meta = saved_model.meta.get()
pprint.pprint(saved_model_meta)


I am working on the file "Operationalizing Models with WML and Scikit-Learn.ipynb".
The first task is:
4.2 Load the Booster from the saved model
Can you write Python code for it?

saved_model_meta['modelVersionHref']

loaded_artifact = ml_repository_client.models.version_from_href(saved_model_meta['modelVersionHref'])
loaded_xgb_model = loaded_artifact.model_instance()
print("Type of model: " + str(type(loaded_xgb_model)))

y_lpredict = loaded_xgb_model.predict(dm_test)
print(y_lpredict)


I am working on the file "Operationalizing Models with WML and Scikit-Learn.ipynb".
The first task is:
5. 2 Perform Prediction
Now, let us perform predictions on a new set of data using the model that is deployed in the scoring service.
Can you write Python code for it?

payload_scoring = {
   "values": [[64.0, 1.0, 4.0, 328.0, 263.0, 0.0, 0.0, 105.0, 1.0, 0.2, 2.0, 1.0, 7.0]]
}

response_scoring = requests.post(scoring_url, json=payload_scoring, headers=header)
pprint.pprint(response_scoring.text)


I am working on the file "Operationalizing Models with WML and Scikit-Learn.ipynb".
The first task is:
5.1.2 Deploy Model
We can now create the online deployment for the published model.
Can you write Python code for it?

payload_online = {"name": "xgb_heart_disease_v1", "description": "xgb_heart_disease", "type": "online"}
response_online = requests.post(endpoint_deployments, json=payload_online, headers=header)

print(response_online.text)

scoring_url = json.loads(response_online.text).get('entity').get('scoring_url')
print(scoring_url)


I am working on the file "Operationalizing Models with WML and Scikit-Learn.ipynb".
The first task is:
5.1: Deploy the model and create online scoring endpoint
Get the published_models URL from instance details.
Can you write Python code for it?

endpoint_instance = wml_credentials['url'] + "/v3/wml_instances/" + wml_credentials['instance_id']
header = {'Content-Type': 'application/json', 'Authorization': 'Bearer ' + mltoken}

response_get_instance = requests.get(endpoint_instance, headers=header)
print(response_get_instance)
print(response_get_instance.text)

endpoint_published_models = json.loads(response_get_instance.text).get('entity').get('published_models').get('url')
print(endpoint_published_models)


I am working on the file "Operationalizing Models with WML and Scikit-Learn.ipynb".
The first task is:

4. Persist the model
In this section store the XGBoost model in the Watson Machine Learning repository by using Watson Machine Learning repository service Python client libraries.
Can you write Python code for it?

from repository.mlrepository import MetaNames
from repository.mlrepository import MetaProps
from repository.mlrepositoryclient import MLRepositoryClient
from repository.mlrepositoryartifact import MLRepositoryArtifact


I am working on the file "Operationalizing Models with WML and Scikit-Learn.ipynb".
The first task is:

5. Deploy and score in a Cloud
To access the Watson Machine Learning REST APIs we require a Watson Machine Learning access token.
To create the WML access token, run the commands in the following cell:
Can you write Python code for it?

import urllib3, requests, json

headers = urllib3.util.make_headers(basic_auth='{}:{}'.format(wml_credentials['username'], wml_credentials['password']))
url = '{}/v3/identity/token'.format(wml_credentials['url'])
response = requests.get(url, headers=headers)
mltoken = json.loads(response.text).get('token')
header = {'Content-Type': 'application/json', 'Authorization': 'Bearer ' + mltoken}


I am working on the file "Operationalizing Models with WML and Scikit-Learn.ipynb".
The first task is:
Create a Booster by using the training data set, which is in the form of a DMatrix object.
Can you write Python code for it?