I am working on the file "MPI-LeipzigDataset.ipynb".
The first task is:
Now we have an easy way to see the range of the behavioral data available. A non-static version of the above code is below.
MRI data available
If you'd like to interactively work with find_subjects_data, a non-static version of the code is below:
Can you write Python code for it?

%matplotlib notebook

#Allow us to import python files in scripts
import sys
sys.path.append('./scripts')

import matplotlib.pyplot as plt
import numpy as np
from ipywidgets import interactive

import find_subjects_behavior_data as fsbd

#Arguments that would normally be passed through the command line call
behavior_files = [
    "MPI-Leipzig/behavioral_data_MPILMBB/phenotype/BDI.tsv",
    "MPI-Leipzig/behavioral_data_MPILMBB/phenotype/HADS.tsv",
    "MPI-Leipzig/behavioral_data_MPILMBB/phenotype/NEO.tsv"
]
behavior_keys = [
    "BDI_summary_sum",
    "HADS-D_summary_sum",
    "NEO_N"
]

#Get data using find_subject_data
subjects, complete_subjects, raw_data, complete_raw_data = fsbd.get_data(behavior_files, behavior_keys)

def draw_figure():
    fsbd.draw_figure(behavior_keys, raw_data, complete_raw_data)

interactive_plot = interactive(draw_figure)
interactive_plot


I am working on the file "MPI-LeipzigDataset.ipynb".
The first task is:
We can investigate what keys are available in any .tsv header by examining the corresponding .json file:
Can you write Python code for it?

%%bash

cat MPI-Leipzig/behavioral_data_MPILMBB/phenotype/BDI.json | head


I am working on the file "MPI/PR_MPI_Diffusion2d.ipynb".
The first task is:
N - slices
Can you write Python code for it?

%%writefile mpi004.py

from mpi4py import MPI

import numpy as np

comm = MPI.COMM_WORLD
rank = comm.Get_rank()
Nproc = comm.size

def numpy_diff2d(u,dx2,dy2,c):

    A = (1.0-2.0*(c/dx2+c/dy2))
    u[1:-1,1:-1] = A*u[1:-1,1:-1] + c/dy2*(u[2:,1:-1] + u[:-2,1:-1]) + \
                                    c/dx2*(u[1:-1,2:] + u[1:-1,:-2])

N = 16*128
Nx = N
Ny = N/Nproc

Niter=200
dx = 0.1
dy = 0.1
dx2 = dx*dx
dy2 = dy*dy
dt = 0.01
D = 0.2
c = D*dt

u = np.zeros([Ny, Nx])

if rank == 0:
    u[-2,u.shape[1]/2] = 1.0/np.sqrt(dx2*dy2)
    print "CLF = ",c/dx2,c/dy2

t0 = MPI.Wtime()

for i in range(Niter):

    if Nproc>1:
        if rank == 0:
            comm.Send([u[-2,:], MPI.FLOAT], dest=1)
        if rank >0  and rank < Nproc-1:
            comm.Recv([u[0,:], MPI.FLOAT], source=rank-1)
            comm.Send([u[-2,:], MPI.FLOAT], dest=rank+1)
        if rank == Nproc - 1:
            comm.Recv([u[0,:], MPI.FLOAT], source=Nproc-2)
            comm.Send([u[1,:], MPI.FLOAT], dest=Nproc-2)
        if rank >0  and rank < Nproc-1:
            comm.Recv([u[-1,:], MPI.FLOAT], source=rank+1)
            comm.Send([u[1,:], MPI.FLOAT], dest=rank-1)
        if rank == 0:
            comm.Recv([u[-1,:], MPI.FLOAT], source=1)

    #print rank

    comm.Barrier()
    numpy_diff2d(u,dx2,dy2,c)

t1 = MPI.Wtime()
print rank,t1-t0

#np.savez("udata%04d"%rank, u=u)
if Nproc>1:
    U = comm.gather(u[1:-1,1:-1])

    if rank==0:
         np.savez("Udata", U=U)

!mpirun -H gpu2,gpu3 python mpi004.py

!mpirun -n 4 python mpi004.py

data = np.load("Udata.npz")
plt.imshow(np.vstack(data['U']))
print data['U'].shape

a = np.arange(0,16).reshape(4,4)

b = a[:,2]
c = a[2,:]

np.may_share_memory(a,b),np.may_share_memory(a,c)

a.flags

b.flags

c.flags

a=np.array(range(6))

b = a[2:4]

b=666

print a

np.may_share_memory?


I am working on the file "MPI/PR_MPI_Jupyter.ipynb".
The first task is:
First Way:

write a file and do view.run("myfile.py")
Can you write Python code for it?

%%writefile mpi001.py

from mpi4py import MPI
import time
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
size = comm.Get_size()
time.sleep(5)
import os
a = 1.23 + rank
print ("OK, rank= ",rank,size,os.getpid())

ar = view.run('mpi001.py')

ar

ar.display_outputs()

ar = view.execute('mpi001.py')

view['a']


I am working on the file "MPI/PR_MPI_Jupyter.ipynb".
The first task is:
Second Way:

use magics: %%px ....

 important!
Can you write Python code for it?

view.activate()

%%px

from mpi4py import MPI
import time
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
size = comm.Get_size()
time.sleep(5)
import os
a = 1.23 + rank
print ("OK, rank= ",rank,size,os.getpid(),a)

%pxresult


I am working on the file "MPI/PR_MPI_Jupyter.ipynb".
The first task is:
Working directory in ipyparallel
If one wants to work in specyfic directory, then it is possible to set it using view.apply_sync commands:
Can you write Python code for it?

import ipyparallel as ipp
c = ipp.Client(profile='mpi')
print(c.ids)
view = c[:]
view.activate()

import os

print(view.apply_sync(os.getcwd))

view.map(os.chdir, ['ProgramowanieRownolegle/MPI']*len(c.ids))

print(view.apply_sync(os.getcwd))


I am working on the file "MPI/PR_MPI_Jupyter.ipynb".
The first task is:
Worth trying:

%%px --block
%%px --targets ::2
%autopx
Can you write Python code for it?

%%px --block

from mpi4py import MPI
import numpy as np

comm = MPI.COMM_WORLD
rank = comm.rank
print(comm.size,comm.rank)


I am working on the file "MPI/PR_MPI_Jupyter.ipynb".
The first task is:
this is equivalent to:
Can you write Python code for it?


I am working on the file "MPI/PR_MPI_game_of_life.ipynb".
The first task is:
3rd version  - using Sendrecv
Using Sendrecv one can avoid potential lock-outs.
Can you write Python code for it?

%%px --block

from mpi4py import MPI
import numpy as np

from game_of_life_utils import *

#number of procesess:

comm = MPI.COMM_WORLD
No_processes = comm.Get_size()
rank = comm.Get_rank()

#constants

Ni,Nj = No_processes*10,40
Niter = 225

#area:

if rank==0:

    X = np.zeros((Ni,Nj),dtype=np.bool)
    set_ic(X,unbounded,offset = (25,28))

else:
    X = None

# subdomains:
buf = np.empty((Ni//No_processes,Nj),dtype=np.bool)
X_local = np.empty((Ni//No_processes+2,Nj),dtype=np.bool)

comm.Scatter(X, buf )
X_local[1:-1,:] = buf

U_local_time = []
Xtime = []

L_bulk  = (1,slice(None))
L_ghost = (0,slice(None))
R_bulk  = (-2, slice(None))
R_ghost = (-1, slice(None))

for i in range(Niter):

    if rank%2 == 0:
        comm.Sendrecv(X_local[R_bulk], dest=(rank+1)%No_processes,sendtag=0,\
                      recvbuf=X_local[R_ghost],source=(rank+1)%No_processes)
        comm.Sendrecv(X_local[L_bulk], dest=(rank-1)%No_processes,sendtag=0, \
                      recvbuf=X_local[L_ghost],source=(rank-1)%No_processes)

    if rank%2 == 1:
        comm.Sendrecv(X_local[L_bulk], dest=(rank-1)%No_processes,sendtag=0,\
                      recvbuf=X_local[L_ghost],source=(rank-1)%No_processes)
        comm.Sendrecv(X_local[R_bulk], dest=(rank+1)%No_processes,sendtag=0,\
                      recvbuf=X_local[R_ghost],source=(rank+1)%No_processes)

    X_local = life_step_2(X_local)

    comm.Gather(X_local[1:-1],X )
    if rank==0:
        Xtime.append( X.copy() )

    U_local_time.append( X_local[1:-1].copy() )


I am working on the file "MPI/PR_MPI_game_of_life.ipynb".
The first task is:
Comparison of parallel and single process versions
Above parallel examples are set to use the same IC and glocal  board size as the single process one.
On rank=0 we gather the time evolution of global domain, therefore one can easily validate the parallel version.
Can you write Python code for it?

np.argsort(view['rank'])[0]

len( view['Xtime'][ np.argsort(view['rank'])[0] ] )


I am working on the file "MPI/PR_MPI_game_of_life.ipynb".
The first task is:
Game of life - serial version
bulk update
Here, certain initial conditions as well as update single process life_step_2 procedure are defined.
Note, that update is periodic in both spatial directions
Can you write Python code for it?


I am working on the file "MPI/PR_MPI_game_of_life.ipynb".
The first task is:
Xtime_parallel  will be a copy (in this notebook) of table of time snashots of global domain .
Xtime - keeps the results of a serial simulation
Can you write Python code for it?

Xtime_parallel =  view['Xtime'][ np.argsort(view['rank'])[0] ]

Ni, Nj = Xtime_parallel[0].shape
n = max(Ni,Nj)
if n<100:
    scale = 500//n
else:
    scale = 1

for i,X_ in enumerate(Xtime_parallel[::]):
    clear_output(wait=True)
    time.sleep(0.02)
    display(Im.fromarray(240*X_.astype(np.uint8)).resize((scale*Nj,scale*Ni)))
    print(i)


I am working on the file "MPI/PR_MPI_game_of_life.ipynb".
The first task is:
setting proper working directory
Engines started by ipyparallel have by default $HOME as working directory, since we import a file we might change it to current working directory of the project.
Can you write Python code for it?

import os
notebook_wd = os.getcwd()

import os
print(view.apply_sync(os.getcwd))

view.map(os.chdir, [notebook_wd]*len(c.ids))
print(view.apply_sync(os.getcwd))


I am working on the file "MPI/PR_MPI_game_of_life.ipynb".
The first task is:
validation

for each timestep we want identical results
below, the first number should be equal to number of timesteps
Can you write Python code for it?

sum([np.all(X_ == Xp_) for X_,Xp_ in zip(Xtime,Xtime_parallel)]), len(Xtime)

for X_,Xp_ in zip(Xtime,Xtime_parallel):
    print (np.all(X_ == Xp_))


I am working on the file "MPI/PR_MPI_game_of_life.ipynb".
The first task is:
version with glocal periodic boundaries
one can split the communication into even and odd nodes:
```python
if rank%2 == 0:
    comm.Send(X_local[R_bulk], dest=(rank+1)%No_processes)
    comm.Recv(X_local[R_ghost],source=(rank+1)%No_processes)
if rank%2 == 1:
    comm.Recv(X_local[L_ghost],source=(rank-1)%No_processes)
    comm.Send(X_local[L_bulk], dest=(rank-1)%No_processes)
if rank%2 == 1:
    comm.Send(X_local[R_bulk], dest=(rank+1)%No_processes)
    comm.Recv(X_local[R_ghost],source=(rank+1)%No_processes)
if rank%2 == 0:
    comm.Recv(X_local[L_ghost],source=(rank-1)%No_processes)
    comm.Send(X_local[L_bulk], dest=(rank-1)%No_processes)
```
Can you write Python code for it?

%%px --block

from mpi4py import MPI
import numpy as np

from game_of_life_utils import *

#number of procesess:

comm = MPI.COMM_WORLD
No_processes = comm.Get_size()
rank = comm.Get_rank()

#constants

Ni,Nj = No_processes*10,40
Niter = 225

#area:

if rank==0:

    X = np.zeros((Ni,Nj),dtype=np.bool)
    set_ic(X,unbounded,offset = (25,28))

else:
    X = None

# subdomains:
buf = np.empty((Ni//No_processes,Nj),dtype=np.bool)
X_local = np.empty((Ni//No_processes+2,Nj),dtype=np.bool)

comm.Scatter(X, buf )
X_local[1:-1,:] = buf

U_local_time = []
Xtime = []

L_bulk  = (1,slice(None))
L_ghost = (0,slice(None))
R_bulk  = (-2, slice(None))
R_ghost = (-1, slice(None))

for i in range(Niter):

    if rank%2 == 0:
        comm.Send(X_local[R_bulk], dest=(rank+1)%No_processes)
        comm.Recv(X_local[R_ghost],source=(rank+1)%No_processes)
        comm.Recv(X_local[L_ghost],source=(rank-1)%No_processes)
        comm.Send(X_local[L_bulk], dest=(rank-1)%No_processes)

    if rank%2 == 1:
        comm.Recv(X_local[L_ghost],source=(rank-1)%No_processes)
        comm.Send(X_local[L_bulk], dest=(rank-1)%No_processes)
        comm.Send(X_local[R_bulk], dest=(rank+1)%No_processes)
        comm.Recv(X_local[R_ghost],source=(rank+1)%No_processes)

    X_local = life_step_2(X_local)

    comm.Gather(X_local[1:-1],X )
    if rank==0:
        Xtime.append( X.copy() )

    U_local_time.append( X_local[1:-1].copy() )


I am working on the file "MPI/PR_MPI_p2p.ipynb".
The first task is:
Communicating  memorybuffers:  Send and Recv
Can you write Python code for it?

%%px --block

import numpy as np
from mpi4py import MPI

comm = MPI.COMM_WORLD
rank = comm.Get_rank()

a = np.zeros((2,2))
if rank == 0:
    a[:] = 2
    comm.Send(a[0,:], dest=1)
elif rank == 1:
    comm.Recv(a[0,:], source=0)

print ("OK,",rank,np.sum(a))

%%px --block

import numpy as np
from mpi4py import MPI

comm = MPI.COMM_WORLD
rank = comm.Get_rank()

a = np.zeros((2,2))
if rank == 0:
    a[:] = 2
    comm.Send(a[:,0], dest=1)
elif rank == 1:
    comm.Recv(a[:,0], source=0)

print ("OK,",rank,np.sum(a))


I am working on the file "MPI/PR_MPI_p2p.ipynb".
The first task is:
Contiguous memory buffers
Can you write Python code for it?

a = np.zeros((2,2))
a.flags

a[:,0].flags

a[0,:].flags

%%px --block

import numpy as np
from mpi4py import MPI

comm = MPI.COMM_WORLD
rank = comm.Get_rank()

a = np.zeros((2,2))
if rank == 0:
    a[:] = 2
    buf = a[:,0].copy()
    comm.Send(buf, dest=1)
elif rank == 1:
    buf = np.empty(2)
    comm.Recv(buf, source=0)
    a[:,0] = buf
    print ("OK,",np.sum(a))

import ipyparallel as ipp
c = ipp.Client(profile='mpi')
print(c.ids)
view = c[:]
view.activate()

%%px --block --target :3
print("OK")

%%px --block
print("OK")

%%px --block
import numpy as np
from mpi4py import MPI

comm = MPI.COMM_WORLD
rank = comm.Get_rank()

a = np.zeros((2,2))

if rank == 0:
    import os
    print(os.getcwd())


I am working on the file "MPI/PR_MPI_p2p.ipynb".
The first task is:
Parallel eigenvalues:
python
import numpy as np
%time np.max(np.real(np.linalg.eigvals(np.random.randn(400,400))))
A task: find a biggest entry in a random matrix:
Can you write Python code for it?

%time np.max(np.random.randn(5000,5000))

%%px --block
from mpi4py import MPI
import time
import numpy as np
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
size = comm.Get_size()
t = MPI.Wtime()
print("result =",np.max(np.random.randn(5000,5000//4)))
t = MPI.Wtime() - t
print(rank,":: execution time:",t)


I am working on the file "MPI/PR_MPI_p2p.ipynb".
The first task is:
Send and receive

We use rank to differentiate code between processors.
Note that mpi4py serializes arbitrary data before send.

Important!

In MPI for Python, the Send(), Recv() and Sendrecv()  can communicate memory buffers.
The variants send(), recv() and sendrecv() can communicate generic Python objects.
Can you write Python code for it?

%%px --block

from mpi4py import MPI

comm = MPI.COMM_WORLD
rank = comm.Get_rank()

data = None

if rank == 0:
    data = {'a': 7, 'b': 3.14}
    comm.send(data, dest=1)

elif rank == 1:
    data = comm.recv(source=0)

print("OK, rank= ",rank,"dane: ",data)


I am working on the file "MPI/PR_cython_numpy_fortran_diff2d.ipynb".
The first task is:
Benchmarks
Can you write Python code for it?

%%time
u = calc(1000,200,cython_diff2d,dx2,dy2,c)

%%time
u = calc(1000,200,numpy_diff2d,dx2,dy2,c)

%%time
u = calc(1000,200,numpy_diff2d_a,dx2,dy2,c)

%%time
u = calc(1000,200,numpy_diff2d_b,dx2,dy2,c)

N = 1000
fortran_source = """

subroutine fortran_diff2d(u, v, dx2, dy2, c)
real(8), intent(inout) :: u({0}, {1})
real(8), intent(inout) :: v({0}, {1})
real(8), intent(in) :: dx2, dy2, c
v(2:{0}-1,2:{1}-1) = u(2:{0}-1,2:{1}-1)+ c*( (u(3:,2:{1}-1)+u(:{0}-2,2:{1}-1))/dy2 + &
        (u(2:{0}-1,3:) + u(2:{0}-1,:{1}-2))/dx2)
end subroutine

""".format(N,N)

fp = open("myfile.f90", "w")
fp.write(fortran_source)
fp.close()

!cat myfile.f90

%%capture f2py.log
!f2py -c -m my_fortran_module myfile.f90

!ls -l

from my_fortran_module import fortran_diff2d

def calcF(N, Niter, func, dx2,dy2,c):
    u = np.zeros([N, N],order='F')
    v = np.zeros_like(u)
    u[u.shape[0]//2,u.shape[1]//2] = 1.0/np.sqrt(dx2*dy2)
    for i in range(Niter//2):
        func(u,v,dx2,dy2,c)
        func(v,u,dx2,dy2,c)
    return u

%%time
u = calcF(1000,200,fortran_diff2d,dx2,dy2,c)


I am working on the file "MPI/PR_cython_numpy_fortran_diff2d.ipynb".
The first task is:
Najszybsza wersja - wektorowy Fortran
Warto zauważyć, że:

nie zawsze jest taka duża różnica miedzy numpy/Cython/Fortran, sprawdź dla np. mapowania funkcji na tablicy wartości
numpy jest kilka razy wolniejsze bo przy każdym działaniu stosuje wartości pośrednie
istnieje możliwośc obejścia z pomocą modułu numexpr, który kompiluje całe wyrażenie na tablicy numpy, sprawdź!
wektorowy Fortran najefektywniej kompiluje wyrażenia typu $u_i+u_{i+1}$

Wniosek - warto stosować Fortran do pewnych operacji!
Automatyczna generacja kodu
Problem: kompilowane moduły nie mogą być ponownie ładowane.
Przeanalizujmy prostą metodę generacji kodu
Can you write Python code for it?

import subprocess
subprocess.check_output(["pwd"])

import subprocess
import importlib

counter = 12

def prepare_fortran_module(N=100):

    global counter
    fortran_source = """

subroutine fortran_diff2d(u, v, dx2, dy2, c)
real(8), intent(in) :: u({0}, {1})
real(8), intent(inout) :: v({0}, {1})
real(8), intent(in) :: dx2, dy2, c
v(2:{0}-1,2:{1}-1) = u(2:{0}-1,2:{1}-1)+ c*( (u(3:,2:{1}-1)+u(:{0}-2,2:{1}-1))/dy2 + &
        (u(2:{0}-1,3:) + u(2:{0}-1,:{1}-2))/dx2)
end subroutine

subroutine fortran_diff2d_a(u, dx2, dy2, c)
real(8), intent(inout) :: u({0}, {1})
real(8), intent(in) :: dx2, dy2, c
u(2:{0}-1,2:{1}-1) = u(2:{0}-1,2:{1}-1)+ c*( (u(3:,2:{1}-1)+u(:{0}-2,2:{1}-1))/dy2 + &
        (u(2:{0}-1,3:) + u(2:{0}-1,:{1}-2))/dx2)
end subroutine

""".format(N,N)

    fp = open("myfile.f90", "w")
    fp.write(fortran_source)
    fp.close()
    counter=counter+1
    try:
        output = subprocess.check_output(["f2py", "-c","-m", "fortran_module%05d"%counter, "myfile.f90"])
        m = importlib.import_module("fortran_module%05d"%counter)
    except:
        print ("problem z kompilacja!")
        return output

    return m

fortran_module = prepare_fortran_module(N=1000)

fortran_diff2d, fortran_diff2d_a = fortran_module.fortran_diff2d, fortran_module.fortran_diff2d_a

def calcF(N, Niter, func, dx2,dy2,c):
    u = np.zeros([N, N],order='F')
    v = np.zeros_like(u)
    u[u.shape[0]//2,u.shape[1]//2] = 1.0/np.sqrt(dx2*dy2)
    for i in range(Niter//2):
        func(u,v,dx2,dy2,c)
        func(v,u,dx2,dy2,c)
    return u


I am working on the file "MPI/PR_cython_numpy_fortran_diff2d.ipynb".
The first task is:
Parametry symulacji
Can you write Python code for it?


I am working on the file "MPI/PR_cython_numpy_fortran_diff2d.ipynb".
The first task is:
Walidacja wyników
Can you write Python code for it?

u = calc(N,125,numpy_diff2d_b,dx2,dy2,c)
plt.imshow(u)

u = calc(N,125,cython_diff2d,dx2,dy2,c)
plt.imshow(u)


I am working on the file "MPI/PR_cython_numpy_fortran_diff2d.ipynb".
The first task is:
teraz możemy skompilować wielokrotnie ten sam kod i załadować modluł
Can you write Python code for it?

N = 1000

fortran_module = prepare_fortran_module(N=N)
fortran_diff2d, fortran_diff2d_a = fortran_module.fortran_diff2d, fortran_module.fortran_diff2d_a

u = calcF(N,1225,fortran_diff2d,dx2,dy2,c)
plt.imshow(u)

def calcF_a(N, Niter, func, dx2,dy2,c):
    u = np.zeros([N, N],order='F')
    v = np.zeros_like(u)
    u[u.shape[0]//2,u.shape[1]//2] = 1.0/np.sqrt(dx2*dy2)
    for i in range(Niter):
        func(u,dx2,dy2,c)
    return u

u = calcF_a(1000,1225,fortran_diff2d_a,dx2,dy2,c)
plt.imshow(u)

%%time
u = calc(1000,200,numpy_diff2d_a,dx2,dy2,c)

%%time
u = calc(1000,200,cython_diff2d,dx2,dy2,c)

fortran_diff2d = prepare_fortran_module(N=1000).fortran_diff2d

%%time
u = calcF(1000,200,fortran_diff2d,dx2,dy2,c)

%%time
u = calcF_a(1000,200,fortran_diff2d_a,dx2,dy2,c)


I am working on the file "MPI/PR_cython_numpy_fortran_diff2d.ipynb".
The first task is:
znamy rozwiązanie równania dyfuzji na nieskończonym obszarze startujące z punktu
$$ u(x,0) = \delta (x)$$
$$ u(x,t) =  \frac{1}{4 \, \pi D t}e^{\left(-\frac{x^{2} + y^{2}}{4  D t}\right)}$$
Can you write Python code for it?

Lx,Ly = N/2*dx,N/2*dy
x  = np.linspace(-Lx,Lx,N)
y  = np.linspace(-Ly,Ly,N)

X,Y = np.meshgrid(x,y )
Niter = 125
t = dt*Niter
P = 1/(4*np.pi*D*t)*np.exp(-(X**2+Y**2)/(4*D*t) )
plt.contourf(X,Y,P)
plt.axes().set_aspect('equal')

u = calc(N,Niter,cython_diff2d,dx2,dy2,c)
np.sum(P)*dx2,np.sum(u)*dx2

plt.plot(X[X.shape[0]//2,:],P[X.shape[0]//2,:],'b')
plt.plot(X[X.shape[0]//2,:],u[X.shape[0]//2,:],'r')


I am working on the file "MPI/ipyparallel_gpu4.ipynb".
The first task is:
Configure mpi profile first:
 - http://ipyparallel.readthedocs.io/en/latest/process.html#parallel-process
Can you write Python code for it?

c = ipp.Client(profile='gpu4')

c.ids

view = c[:]

view.activate()

%px --block
import os

import os
os.environ

c[:].apply_sync(lambda : "Hello, World")


I am working on the file "MPI/ipyparallel_mpi_tests.ipynb".
The first task is:
Interesting might be:
python
c = Client('/path/to/my/ipcontroller-client.json', sshserver='me@myhub.example.com')
Can you write Python code for it?

import mpi4py

%%px

%%writefile psum.py
from mpi4py import MPI
import numpy as np

def psum(a):
    locsum = np.sum(a)
    rcvBuf = np.array(0.0,'d')
    MPI.COMM_WORLD.Allreduce([locsum, MPI.DOUBLE],
        [rcvBuf, MPI.DOUBLE],
        op=MPI.SUM)
    return rcvBuf

view = c[:]

view.activate()

view.run('psum.py')

view.scatter('a',np.arange(16,dtype='float'))

view['a']

%px totalsum = psum(a)

view['totalsum']

%%writefile psum.py
from mpi4py import MPI
import numpy as np

import time
def psum(a):
    locsum = np.sum(a)
    rcvBuf = np.array(0.0,'d')
    MPI.COMM_WORLD.Allreduce([locsum, MPI.DOUBLE],
        [rcvBuf, MPI.DOUBLE],
        op=MPI.SUM)
    return rcvBuf

rank = MPI.COMM_WORLD.Get_rank()
size = MPI.COMM_WORLD.Get_size()
if rank==0:
    a = np.arange(16,dtype='float')
else:
    a = None
a_local = np.empty(16/size,dtype='float')
MPI.COMM_WORLD.Scatter( a, a_local )
time.sleep(rank*0.1)
print(rank,":::",a_local)

totalsum = psum(a_local)
print(totalsum)

!mpirun -n 4 /opt/conda/envs/py27/bin/python psum.py


I am working on the file "MSE (Maximum Spacing Estimation).ipynb".
The first task is:
for this nice plot I took some inspiration from this stack overflow answer:
http://stackoverflow.com/questions/29779079/adding-a-scatter-of-points-to-a-boxplot-using-matplotlib
Basically, we put all the points at around their location in the boxplot, but we "move them around" a bit so that is easier for the eye to see how dense the areas in the boxplot look like. Is more of a way to make the plot easier to look at, yet the boxplot has pretty much all the information you will need in this case.
Can you write Python code for it?

xx=[]
yy=[]

for indx,samplesize in enumerate(allSamplesizes):
    yy.append(results['MSE'][samplesize])
    xx.append(np.random.normal(len(yy), 0.04, size=len(results['MSE'][samplesize])))
    yy.append(results['ML'][samplesize])
    xx.append(np.random.normal(len(yy), 0.04, size=len(results['MSE'][samplesize])))

print np.shape(yy)
print np.shape(xx)

import pylab as P

P.figure(figsize=(12,7)) #size in inches

bp = P.boxplot(yy)
P.plot(xx,yy, 'r.', alpha=0.15)

P.xticks([x+1 for x in range(len(yy))], [x.format(sz) for sz in allSamplesizes for x in ('MSE_{}','ML_{}')])
P.xlabel('# of samples given to the estimator')
P.ylabel('Value of estimated shape parameter')
P.title('MSE,ML: variation of estimated shape parameter of a Pareto I\n')

#horiz line with true shape value
P.plot([x+1 for x in range(len(yy))],realAlpha*np.ones(len(yy)),
       color='dodgerblue', ls='dashed',linewidth='3')

P.show()


I am working on the file "MSMs.ipynb".
The first task is:
Estimating the transition probability matrix
transition count matrix C(τ)
The transition count matrix C(τ), which denotes the number of observed transitions from state i at time t to state j at time t + τ, where τ is the lag time of the model.
* transition count matrix should be symmetric because all elementary transitions should be reversible under equilibrium condition. But the equilibrium sampling is almost never reached in simulations
* Markovian in physical sense: next conformation is simply a deterministic function of the system's current state.
* lag time or observation interval τ
* the Chapman-Kolmogorov equation: taking n steps with an MSM with a lag
* time of τ should be equivalent to an MSM with a lag time of nτ.
transition probability matrix T(τ)
the transition probability matrix T(τ) includes the probability of transitions from state i to state j in
a certain time interval τ is obtained by normalizing C(τ) with the sum of all transitions from state i.
* To ensure that the total population of all the states is conserved, a maximum likelihood estimate of the transition probability matrix that obeys the detailed balance is obtained.
* The eigenvalues/eigenvectors of the transition matrix gives information about the aggregate transitions between subsets of states in the model and what timescales these transitions occur on.
* The equilibrium populations of the individual states are estimated from the first eigenvector of the transition probability matrix.
* The timescales of the dynamical processes occurring on the conformational landscape of the protein can be obtained by estimating the eigenvalues of this matrix.
Can you write Python code for it?

from msmbuilder.cluster import KCenters
from msmbuilder.msm import MarkovStateModel

msmts0, msmts1 = {}, {}
lag_times = [1, 10, 20, 30, 40]
n_states = [4, 8, 16, 32, 64]

for n in n_states:
    msmts0[n] = []
    msmts1[n] = []
    for lag_time in lag_times:
        assignments = KCenters(n_clusters=n).fit_predict(sequences)
        msm = MarkovStateModel(lag_time=lag_time, verbose=False).fit(assignments)
        timescales = msm.timescales_
        msmts0[n].append(timescales[0])
        msmts1[n].append(timescales[1])
        print('n_states=%d\tlag_time=%d\ttimescales=%s' % (n, lag_time, timescales[0:2]))
    print()

%matplotlib inline
from matplotlib.pyplot import *

figure(figsize=(14,3))

for i, n in enumerate(n_states):
    subplot(1,len(n_states),1+i)
    plot(lag_times, msmts0[n])
    plot(lag_times, msmts1[n])
    if i == 0:
        ylabel('Relaxation Timescale')
    xlabel('Lag Time')
    title('%d states' % n)

show()


I am working on the file "MSS_Xmas_Trees/ml_seg_sub5_CRAW.ipynb".
The first task is:
Make a new prediction, with the best model on the full dataset B:
Can you write Python code for it?

clf.fit(go_B_PHASE_II,correctB)
predicted_blind_PHASE_II = clf.predict(features_blind_PHASE_II)
#out_f1=metrics.f1_score(correct_facies_labels, predicted_blind_PHASE_II, average = 'micro')
#print "f1 score on the prediction of blind"
#print out_f1


I am working on the file "MSS_Xmas_Trees/ml_seg_sub5_CRAW.ipynb".
The first task is:
PHASE Ib
Making several predictions using dataset A
PREPARE THE BLIND DATA FOR SERIAL MODELLING
Can you write Python code for it?

#
#blindwell='CHURCHMAN BIBLE'

#df0 = training_data0[training_data0['Well Name'] == blindwell]
#df1 = df0.drop(['Formation', 'Well Name', 'Depth','Facies'], axis=1)
#df1a=df0[(np.abs(stats.zscore(df1))<8).all(axis=1)]
#blind=magic(df1a)
#correct_facies_labels = blind['Facies'].values
#features_blind = blind.drop(['Formation', 'Well Name', 'Depth','Facies'], axis=1)
#pred_blind=0*correct_facies_labels

df0 = test_data[test_data['Well Name'] == 'CRAWFORD']
df1 = df0.drop(['Formation', 'Well Name', 'Depth'], axis=1)
df1a=df0[(np.abs(stats.zscore(df1))<8).all(axis=1)]
blind=magic(df1a)
pred_blind=0*predicted_regime7


I am working on the file "MSS_Xmas_Trees/ml_seg_sub5_CRAW.ipynb".
The first task is:
Phase 1a:
-Create predictions specifically for the most difficult facies
-at this stage we focus on TP and FP only

training for facies 9 specifically
Can you write Python code for it?

df0 = test_data[test_data['Well Name'] == 'CRAWFORD']
df1 = df0.drop(['Formation', 'Well Name', 'Depth'], axis=1)
df1a=df0[(np.abs(stats.zscore(df1))<8).all(axis=1)]
blind=magic(df1a)

df1a.head()

features_blind = blind.drop(['Formation', 'Well Name', 'Depth'], axis=1)

#============================================================
df0=training_data0.dropna()
df1 = df0.drop(['Formation', 'Well Name', 'Depth','Facies'], axis=1)
df1a=df0[(np.abs(stats.zscore(df1))<8).all(axis=1)]
all1=magic(df1a)
#X, y = make_balanced_binary(all1, 9,6)
X, y = make_balanced_binary(all1, 9,9)
#============================================================
correct_train=y

clf = RandomForestClassifier(max_depth = 6, n_estimators=600)
clf.fit(X,correct_train)

predicted_blind1 = clf.predict(features_blind)

predicted_regime9=predicted_blind1.copy()


I am working on the file "MSS_Xmas_Trees/ml_seg_sub5_CRAW.ipynb".
The first task is:
Through experimenting with the cluster size I've decided on 4 clusters.
This corresponds largely with the corresponding similarity in facies distribution
CRAWFORD is most similar to ALEXANDER and LUKE. This will called cluster 1. (The only ones with facies 1)
STUART is most similar to KIMZEY and NOLAN This will be called cluster 2
Collating the Data:
based on the regimes we determined
Can you write Python code for it?

# based on kmeans clustering
data=[]
df = training_data0[training_data0['Well Name'] == 'ALEXANDER D']
data.append(df)
df = training_data0[training_data0['Well Name'] == 'LUKE G U']
data.append(df)
Regime_1 = pd.concat(data, axis=0)
print len(Regime_1)

data=[]
df = training_data0[training_data0['Well Name'] == 'KIMZEY A']
data.append(df)
df = training_data0[training_data0['Well Name'] == 'NOLAN']
data.append(df)
Regime_2 = pd.concat(data, axis=0)
print len(Regime_2)

data=[]
df = training_data0[training_data0['Well Name'] == 'CHURCHMAN BIBLE']
data.append(df)
df = training_data0[training_data0['Well Name'] == 'SHRIMPLIN']
data.append(df)
df = training_data0[training_data0['Well Name'] == 'NEWBY']
data.append(df)
df = training_data0[training_data0['Well Name'] == 'Recruit F9']
data.append(df)
Regime_3 = pd.concat(data, axis=0)
print len(Regime_3)

data=[]
df = training_data0[training_data0['Well Name'] == 'SHANKLE']
data.append(df)
df = training_data0[training_data0['Well Name'] == 'CROSS H CATTLE']
data.append(df)
Regime_4 = pd.concat(data, axis=0)
print len(Regime_4)


I am working on the file "MSS_Xmas_Trees/ml_seg_sub5_CRAW.ipynb".
The first task is:
training for facies 5 specifically
Can you write Python code for it?

features_blind = blind.drop(['Formation', 'Well Name', 'Depth'], axis=1)

#============================================================
df0=training_data0.dropna()
df1 = df0.drop(['Formation', 'Well Name', 'Depth','Facies'], axis=1)
df1a=df0[(np.abs(stats.zscore(df1))<8).all(axis=1)]
all1=magic(df1a)
X, y = make_balanced_binary(all1, 5,13)
#============================================================

go_A=StandardScaler().fit_transform(X)
go_blind=StandardScaler().fit_transform(features_blind)
correct_train_A=binarify(y, 1)
#=============================================

clf = KNeighborsClassifier(n_neighbors=4,algorithm='brute')
clf.fit(go_A,correct_train_A)
predicted_blind1 = clf.predict(go_blind)

clf = KNeighborsClassifier(n_neighbors=5,leaf_size=10)
clf.fit(go_A,correct_train_A)
predicted_blind2 = clf.predict(go_blind)

clf = KNeighborsClassifier(n_neighbors=5)
clf.fit(go_A,correct_train_A)
predicted_blind3 = clf.predict(go_blind)

clf = tree.DecisionTreeClassifier()
clf.fit(go_A,correct_train_A)
predicted_blind4 = clf.predict(go_blind)

clf = tree.DecisionTreeClassifier()
clf.fit(go_A,correct_train_A)
predicted_blind5 = clf.predict(go_blind)

clf = tree.DecisionTreeClassifier()
clf.fit(go_A,correct_train_A)
predicted_blind6 = clf.predict(go_blind)

#####################################
predicted_blind=predicted_blind1+predicted_blind2+predicted_blind3+predicted_blind4+predicted_blind5+predicted_blind6
for ii in range(len(predicted_blind)):
    if predicted_blind[ii] > 4:
        predicted_blind[ii]=1
    else:
        predicted_blind[ii]=0
#####################################
print "-------"

#####################################

print "-------"
predicted_regime5=predicted_blind.copy()


I am working on the file "MSS_Xmas_Trees/ml_seg_sub5_CRAW.ipynb".
The first task is:
training for facies 7 specifically
Can you write Python code for it?

features_blind = blind.drop(['Formation', 'Well Name', 'Depth'], axis=1)

#============================================================
df0=training_data0.dropna()
df1 = df0.drop(['Formation', 'Well Name', 'Depth','Facies'], axis=1)
df1a=df0[(np.abs(stats.zscore(df1))<8).all(axis=1)]
all1=magic(df1a)
X, y = make_balanced_binary(all1, 7,13)
#============================================================

go_A=StandardScaler().fit_transform(X)
go_blind=StandardScaler().fit_transform(features_blind)
correct_train_A=binarify(y, 1)
#=============================================

clf = KNeighborsClassifier(n_neighbors=4,algorithm='brute')
clf.fit(go_A,correct_train_A)
predicted_blind1 = clf.predict(go_blind)

clf = KNeighborsClassifier(n_neighbors=5,leaf_size=10)
clf.fit(go_A,correct_train_A)
predicted_blind2 = clf.predict(go_blind)

clf = KNeighborsClassifier(n_neighbors=5)
clf.fit(go_A,correct_train_A)
predicted_blind3 = clf.predict(go_blind)

clf = tree.DecisionTreeClassifier()
clf.fit(go_A,correct_train_A)
predicted_blind4 = clf.predict(go_blind)

clf = tree.DecisionTreeClassifier()
clf.fit(go_A,correct_train_A)
predicted_blind5 = clf.predict(go_blind)

clf = tree.DecisionTreeClassifier()
clf.fit(go_A,correct_train_A)
predicted_blind6 = clf.predict(go_blind)

#####################################
predicted_blind=predicted_blind1+predicted_blind2+predicted_blind3+predicted_blind4+predicted_blind5+predicted_blind6
for ii in range(len(predicted_blind)):
    if predicted_blind[ii] > 5:
        predicted_blind[ii]=1
    else:
        predicted_blind[ii]=0

#####################################

print "-------"
predicted_regime7=predicted_blind.copy()
print sum(predicted_regime7)


I am working on the file "MSS_Xmas_Trees/ml_seg_sub5_STU.ipynb".
The first task is:
Add the initial predictions as features:
Can you write Python code for it?

go_B_PHASE_II=go_B.copy()
features_blind_PHASE_II=features_blind.copy()
for kk in range(len(pred_array)-1):
    name='prediction_' + str(kk)
    featB=pred_array[kk,:]
    go_B_PHASE_II[name]=featB
    feat=pred_blind[kk,:]
    features_blind_PHASE_II[name]=feat


I am working on the file "MSS_Xmas_Trees/ml_seg_sub5_STU.ipynb".
The first task is:
Create several predictions, varying the dataset and the technique
Can you write Python code for it?

pred_array=0*correctB
pred_blind=np.zeros(len(features_blind))

print "logistic regression"
clf = linear_model.LogisticRegression(class_weight='balanced')
pred_array, pred_blind=phaseI_model_scaled(regime_train1, correctA1, go_B, clf, pred_array, pred_blind, features_blind)
print "svm"
clf = svm.SVC(decision_function_shape='ovo',class_weight='balanced')
pred_array, pred_blind=phaseI_model_scaled(regime_train2, correctA2, go_B, clf, pred_array, pred_blind, features_blind)
print "svm lin"
clf = svm.LinearSVC(class_weight='balanced')
pred_array, pred_blind=phaseI_model_scaled(regime_train3, correctA3, go_B, clf, pred_array, pred_blind, features_blind)
print "svm3"
clf = svm.SVC(C = 100, cache_size=2400, class_weight=None, coef0=0.0,
  decision_function_shape=None, degree=3, gamma=0.01, kernel='rbf',
  max_iter=-1, probability=True, random_state=49, shrinking=True,
  tol=0.001, verbose=False)
pred_array, pred_blind=phaseI_model_scaled(regime_train4, correctA4, go_B, clf, pred_array, pred_blind, features_blind)
print "knn0"
clf = KNeighborsClassifier(n_neighbors=8)
pred_array, pred_blind=phaseI_model_scaled(regime_train4, correctA4, go_B, clf, pred_array, pred_blind, features_blind)
print "knn1"
clf = KNeighborsClassifier(n_neighbors=4,algorithm='brute')
pred_array, pred_blind=phaseI_model_scaled(regime_train5, correctA5, go_B, clf, pred_array, pred_blind, features_blind)
print "knn2"
clf = KNeighborsClassifier(n_neighbors=5,leaf_size=10,p=1)
pred_array, pred_blind=phaseI_model_scaled(regime_train6, correctA6, go_B, clf, pred_array, pred_blind, features_blind)
print "knn3"
clf = KNeighborsClassifier(n_neighbors=7,p=1)
pred_array, pred_blind=phaseI_model_scaled(regime_train7, correctA7, go_B, clf, pred_array, pred_blind, features_blind)
print "knn3"
clf = KNeighborsClassifier(n_neighbors=6,p=1)
pred_array, pred_blind=phaseI_model_scaled(regime_train1, correctA1, go_B, clf, pred_array, pred_blind, features_blind)
print "knn4"
clf =...


I am working on the file "MSS_Xmas_Trees/ml_seg_sub5_STU.ipynb".
The first task is:
Make a new prediction, with the best model on the full dataset B:
Can you write Python code for it?


I am working on the file "MSS_Xmas_Trees/ml_seg_sub5_STU.ipynb".
The first task is:
PHASE Ib
Making several predictions using dataset A
PREPARE THE BLIND DATA FOR SERIAL MODELLING
Can you write Python code for it?

#
#blindwell='CHURCHMAN BIBLE'

#df0 = training_data0[training_data0['Well Name'] == blindwell]
#df1 = df0.drop(['Formation', 'Well Name', 'Depth','Facies'], axis=1)
#df1a=df0[(np.abs(stats.zscore(df1))<8).all(axis=1)]
#blind=magic(df1a)
#correct_facies_labels = blind['Facies'].values
#features_blind = blind.drop(['Formation', 'Well Name', 'Depth','Facies'], axis=1)
#pred_blind=0*correct_facies_labels

df0 = test_data[test_data['Well Name'] == 'STUART']
df1 = df0.drop(['Formation', 'Well Name', 'Depth'], axis=1)
df1a=df0[(np.abs(stats.zscore(df1))<8).all(axis=1)]
blind=magic(df1a)
pred_blind=0*predicted_regime7


I am working on the file "MSS_Xmas_Trees/ml_seg_sub5_STU.ipynb".
The first task is:
PREPARE THE DATA FOR SERIAL MODELLING
This could be done smarter but basically manual at this point
selecting bias towards the REGIME the blind data has been classified as
for CHURCHMAN BIBLE  this is regime 3
For CRAWFORD this is regime 1
For STUART this is regime 2
Can you write Python code for it?

main_regime=regime2A_train
other1=regime1A_train
other2=regime3A_train
other3=regime4A_train

main_test=regime2A_test
other1_test=regime1A_test
other2_test=regime3A_test
other3_test=regime4A_test

tmp2=[regime1B_train, regime2B_train, regime3B_train, regime4B_train]
go_B= pd.concat(tmp2, axis=0)
correctB=np.concatenate((regime1B_test, regime2B_test, regime3B_test, regime4B_test))

#===================================================
tmp1=[main_regime, other1, other2, other3]
regime_train1= pd.concat(tmp1, axis=0)
correctA1=np.concatenate((main_test, other1_test, other2_test, other3_test))
#===================================================
tmp1=[main_regime, other2, other3]
regime_train2= pd.concat(tmp1, axis=0)
correctA2=np.concatenate((main_test, other2_test, other3_test))
#===================================================
tmp1=[main_regime, other1, other3]
regime_train3= pd.concat(tmp1, axis=0)
correctA3=np.concatenate((main_test, other1_test, other3_test))
#===================================================
tmp1=[main_regime, other1, other2]
regime_train4= pd.concat(tmp1, axis=0)
correctA4=np.concatenate((main_test, other1_test, other2_test))
#===================================================
tmp1=[main_regime, other1]
regime_train5= pd.concat(tmp1, axis=0)
correctA5=np.concatenate((main_test, other1_test))
#===================================================
tmp1=[main_regime, other2]
regime_train6= pd.concat(tmp1, axis=0)
correctA6=np.concatenate((main_test, other2_test))
#===================================================
tmp1=[main_regime, other3]
regime_train7= pd.concat(tmp1, axis=0)
correctA7=np.concatenate((main_test, other3_test))
#===================================================
tmp1=[main_regime]
regime_train8= pd.concat(tmp1, axis=0)
correctA8=main_test


I am working on the file "MSS_Xmas_Trees/ml_seg_sub5_STU.ipynb".
The first task is:
Phase II:
Stacking the predictions from phase Ib.
New predictions from data B

First prediction of B data without Phase I input:
Can you write Python code for it?

clf = RandomForestClassifier(max_depth = 15, n_estimators=1600,min_samples_leaf=15)
clf.fit(go_B,correctB)
predicted_blind_PHASE_I = clf.predict(features_blind)
#out_f1=metrics.f1_score(correct_facies_labels, predicted_blind_PHASE_I, average = 'micro')
#print "f1 score on the prediction of blind"
#print out_f1

predicted_blind_PHASE_I


I am working on the file "MSS_Xmas_Trees/ml_seg_sub5_STU.ipynb".
The first task is:
Split the data into 2 parts:
from A We will make initial predictions
from B we will make the final prediction(s)
Can you write Python code for it?

df0=Regime_1.dropna()
df1 = df0.drop(['Formation', 'Well Name', 'Depth','Facies'], axis=1)
df1a=df0[(np.abs(stats.zscore(df1))<8).all(axis=1)]
df2a=magic(df1a)

y=df2a['Facies'].values
X=df2a.drop(['Formation', 'Well Name', 'Depth','Facies'], axis=1)
regime1A_train, regime1B_train, regime1A_test, regime1B_test = train_test_split(X, y, test_size=0.5, random_state=42)

df0=Regime_2.dropna()
df1 = df0.drop(['Formation', 'Well Name', 'Depth','Facies'], axis=1)
df1a=df0[(np.abs(stats.zscore(df1))<8).all(axis=1)]
df2b=magic(df1a)
y= df2b['Facies'].values
X= df2b.drop(['Formation', 'Well Name', 'Depth','Facies'], axis=1)
regime2A_train, regime2B_train, regime2A_test, regime2B_test= train_test_split(X, y, test_size=0.5, random_state=42)

df0=Regime_3.dropna()
df1 = df0.drop(['Formation', 'Well Name', 'Depth','Facies'], axis=1)
df1a=df0[(np.abs(stats.zscore(df1))<8).all(axis=1)]
df2c=magic(df1a)
y=df2c['Facies'].values
X= df2c.drop(['Formation', 'Well Name', 'Depth','Facies'], axis=1)
regime3A_train, regime3B_train, regime3A_test, regime3B_test = train_test_split(X, y, test_size=0.5, random_state=42)

df0=Regime_4.dropna()
df1 = df0.drop(['Formation', 'Well Name', 'Depth','Facies'], axis=1)
df1a=df0[(np.abs(stats.zscore(df1))<8).all(axis=1)]
df2d=magic(df1a)
y=df2d['Facies'].values
X= df2d.drop(['Formation', 'Well Name', 'Depth','Facies'], axis=1)
regime4A_train, regime4B_train, regime4A_test, regime4B_test = train_test_split(X, y, test_size=0.5, random_state=42)


I am working on the file "MSS_Xmas_Trees/ml_seg_sub5_STU.ipynb".
The first task is:
TO DO- some more steps here
Permute facies based on earlier prediction:
Can you write Python code for it?


I am working on the file "MSS_Xmas_Trees/ml_seg_sub5_STU.ipynb".
The first task is:
This is the only feature engineering used:
Can you write Python code for it?

def magic(df):
    df1=df.copy()
    b, a = signal.butter(2, 0.2, btype='high', analog=False)
    feats0=['GR','ILD_log10','DeltaPHI','PHIND','PE','NM_M','RELPOS']
    #feats01=['GR','ILD_log10','DeltaPHI','PHIND']
    #feats01=['DeltaPHI']
    #feats01=['GR','DeltaPHI','PHIND']
    feats01=['GR',]
    feats02=['PHIND']
    #feats02=[]
    for ii in feats0:
        df1[ii]=df[ii]
        name1=ii + '_1'
        name2=ii + '_2'
        name3=ii + '_3'
        name4=ii + '_4'
        name5=ii + '_5'
        name6=ii + '_6'
        name7=ii + '_7'
        name8=ii + '_8'
        name9=ii + '_9'
        xx1 = list(df[ii])
        xx_mf= signal.medfilt(xx1,9)
        x_min1=np.roll(xx_mf, 1)
        x_min2=np.roll(xx_mf, -1)
        x_min3=np.roll(xx_mf, 3)
        x_min4=np.roll(xx_mf, 4)
        xx1a=xx1-np.mean(xx1)
        xx_fil = signal.filtfilt(b, a, xx1)
        xx_grad=np.gradient(xx1a)
        x_min5=np.roll(xx_grad, 3)
        #df1[name4]=xx_mf
        if ii in feats01:
            df1[name1]=x_min3
            df1[name2]=xx_fil
            df1[name3]=xx_grad
            df1[name4]=xx_mf
            df1[name5]=x_min1
            df1[name6]=x_min2
            df1[name7]=x_min4
            #df1[name8]=x_min5
            #df1[name9]=x_min2
        if ii in feats02:
            df1[name1]=x_min3
            df1[name2]=xx_fil
            df1[name3]=xx_grad
            #df1[name4]=xx_mf
            df1[name5]=x_min1
            #df1[name6]=x_min2
            #df1[name7]=x_min4
    return df1

#filename = 'training_data.csv'
filename = 'facies_vectors.csv'
training_data0 = pd.read_csv(filename)
filename = 'validation_data_nofacies.csv'
test_data = pd.read_csv(filename)

all_wells=training_data0['Well Name'].unique()
print all_wells

# what to do with the naans
training_data1=training_data0.copy()
me_tot=training_data1['PE'].median()
print me_tot
for well in all_wells:
    df=training_data0[training_data0['Well Name'] == well]
    print well
    print len(df)
   ...


I am working on the file "MSS_Xmas_Trees/ml_seg_sub5_STU.ipynb".
The first task is:
kmeans clustering to find natural clusters:
Can you write Python code for it?

frames = [data_all, data_test]
X = pd.concat(frames)
X

X1 = X.drop(['Well Name'], axis=1)

from sklearn.cluster import KMeans
kmeans = KMeans(n_clusters=2).fit(X1)
kmeans.labels_

kmeans = KMeans(n_clusters=4).fit(X1)
kmeans.labels_


I am working on the file "MSS_Xmas_Trees/ml_seg_sub5_STU.ipynb".
The first task is:
training for facies 5 specifically
Can you write Python code for it?

features_blind = blind.drop(['Formation', 'Well Name', 'Depth'], axis=1)

#============================================================
df0=training_data0.dropna()
df1 = df0.drop(['Formation', 'Well Name', 'Depth','Facies'], axis=1)
df1a=df0[(np.abs(stats.zscore(df1))<8).all(axis=1)]
all1=magic(df1a)
X, y = make_balanced_binary(all1, 5,10)
#X, y = make_balanced_binary(all1, 5,16)
#============================================================

go_A=StandardScaler().fit_transform(X)
go_blind=StandardScaler().fit_transform(features_blind)
correct_train_A=binarify(y, 1)
#=============================================

clf = KNeighborsClassifier(n_neighbors=4,algorithm='brute')
clf.fit(go_A,correct_train_A)
predicted_blind1 = clf.predict(go_blind)

clf = KNeighborsClassifier(n_neighbors=5,leaf_size=10)
clf.fit(go_A,correct_train_A)
predicted_blind2 = clf.predict(go_blind)

clf = KNeighborsClassifier(n_neighbors=5)
clf.fit(go_A,correct_train_A)
predicted_blind3 = clf.predict(go_blind)

clf = tree.DecisionTreeClassifier()
clf.fit(go_A,correct_train_A)
predicted_blind4 = clf.predict(go_blind)

clf = tree.DecisionTreeClassifier()
clf.fit(go_A,correct_train_A)
predicted_blind5 = clf.predict(go_blind)

clf = tree.DecisionTreeClassifier()
clf.fit(go_A,correct_train_A)
predicted_blind6 = clf.predict(go_blind)

#####################################
predicted_blind=predicted_blind1+predicted_blind2+predicted_blind3+predicted_blind4+predicted_blind5+predicted_blind6
for ii in range(len(predicted_blind)):
    if predicted_blind[ii] > 5:
        predicted_blind[ii]=1
    else:
        predicted_blind[ii]=0
#####################################
print "-------"

#####################################

print "-------"
predicted_regime5=predicted_blind.copy()
print(sum(predicted_regime5))


I am working on the file "MSS_Xmas_Trees/ml_seg_sub5_STU.ipynb".
The first task is:
training for facies 7 specifically
Can you write Python code for it?

features_blind = blind.drop(['Formation', 'Well Name', 'Depth'], axis=1)

#============================================================
df0=training_data0.dropna()
df1 = df0.drop(['Formation', 'Well Name', 'Depth','Facies'], axis=1)
df1a=df0[(np.abs(stats.zscore(df1))<8).all(axis=1)]
all1=magic(df1a)
X, y = make_balanced_binary(all1, 7,11)
X, y = make_balanced_binary(all1, 7,13)
#============================================================

go_A=StandardScaler().fit_transform(X)
go_blind=StandardScaler().fit_transform(features_blind)
correct_train_A=binarify(y, 1)
#=============================================

clf = KNeighborsClassifier(n_neighbors=4,algorithm='brute')
clf.fit(go_A,correct_train_A)
predicted_blind1 = clf.predict(go_blind)

clf = KNeighborsClassifier(n_neighbors=5,leaf_size=10)
clf.fit(go_A,correct_train_A)
predicted_blind2 = clf.predict(go_blind)

clf = KNeighborsClassifier(n_neighbors=5)
clf.fit(go_A,correct_train_A)
predicted_blind3 = clf.predict(go_blind)

clf = tree.DecisionTreeClassifier()
clf.fit(go_A,correct_train_A)
predicted_blind4 = clf.predict(go_blind)

clf = tree.DecisionTreeClassifier()
clf.fit(go_A,correct_train_A)
predicted_blind5 = clf.predict(go_blind)

clf = tree.DecisionTreeClassifier()
clf.fit(go_A,correct_train_A)
predicted_blind6 = clf.predict(go_blind)

#####################################
predicted_blind=predicted_blind1+predicted_blind2+predicted_blind3+predicted_blind4+predicted_blind5+predicted_blind6
for ii in range(len(predicted_blind)):
    if predicted_blind[ii] > 5:
        predicted_blind[ii]=1
    else:
        predicted_blind[ii]=0

#####################################

print "-------"
predicted_regime7=predicted_blind.copy()
print(sum(predicted_regime7))


I am working on the file "MSS_Xmas_Trees/ml_seg_sub5_STU2.ipynb".
The first task is:
Create several predictions, varying the dataset and the technique
Can you write Python code for it?

pred_array=0*correctB
pred_blind=np.zeros(len(features_blind))

print "logistic regression"
clf = linear_model.LogisticRegression(class_weight='balanced')
pred_array, pred_blind=phaseI_model_scaled(regime_train1, correctA1, go_B, clf, pred_array, pred_blind, features_blind)
print "svm"
clf = svm.SVC(decision_function_shape='ovo',class_weight='balanced')
pred_array, pred_blind=phaseI_model_scaled(regime_train2, correctA2, go_B, clf, pred_array, pred_blind, features_blind)
print "svm lin"
clf = svm.LinearSVC(class_weight='balanced')
pred_array, pred_blind=phaseI_model_scaled(regime_train3, correctA3, go_B, clf, pred_array, pred_blind, features_blind)
print "svm3"
clf = svm.SVC(C = 1,  class_weight=None, coef0=0.0,
  decision_function_shape=None, degree=3, gamma=0.0010, kernel='rbf',
  max_iter=-1, probability=True, random_state=49, shrinking=True,
  tol=0.001, verbose=False)
pred_array, pred_blind=phaseI_model_scaled(regime_train5, correctA5, go_B, clf, pred_array, pred_blind, features_blind)
# print "knn0"
# clf = KNeighborsClassifier(n_neighbors=8)
# pred_array, pred_blind=phaseI_model_scaled(regime_train4, correctA4, go_B, clf, pred_array, pred_blind, features_blind)
# print "knn1"
# clf = KNeighborsClassifier(n_neighbors=4,algorithm='brute')
# pred_array, pred_blind=phaseI_model_scaled(regime_train5, correctA5, go_B, clf, pred_array, pred_blind, features_blind)
print "knn2"
clf = KNeighborsClassifier(n_neighbors=7,leaf_size=10,p=1)
pred_array, pred_blind=phaseI_model_scaled(regime_train1, correctA1, go_B, clf, pred_array, pred_blind, features_blind)
#print "knn3"
# clf = KNeighborsClassifier(n_neighbors=7,p=1)
# pred_array, pred_blind=phaseI_model_scaled(regime_train2, correctA2, go_B, clf, pred_array, pred_blind, features_blind)
# print "knn3"
#clf = KNeighborsClassifier(n_neighbors=6,p=1)
#pred_array, pred_blind=phaseI_model_scaled(regime_train5, correctA5, go_B, clf, pred_array, pred_blind, features_blind)
print "knn4"
clf =...


I am working on the file "MSS_Xmas_Trees/ml_seg_sub5_STU2.ipynb".
The first task is:
Do this with a couple of predictions and average
To Do
Permute facies based on earlier prediction:
Can you write Python code for it?


I am working on the file "MSS_Xmas_Trees/ml_seg_sub5_STU2.ipynb".
The first task is:
Phase 1a:
-Create predictions specifically for the most difficult facies
-at this stage we focus on TP and FP only

training for facies 9 specifically
Can you write Python code for it?

# df0 = test_data[test_data['Well Name'] == 'CRAWFORD']
# df1 = df0.drop(['Formation', 'Well Name', 'Depth'], axis=1)
# df1a=df0[(np.abs(stats.zscore(df1))<8).all(axis=1)]
# blind=magic(df1a)

df0 = test_data[test_data['Well Name'] == 'STUART']
df1 = df0.drop(['Formation', 'Well Name', 'Depth'], axis=1)
df1a=df0[(np.abs(stats.zscore(df1))<8).all(axis=1)]
blind=magic(df1a)

df1a.head()

features_blind = blind.drop(['Formation', 'Well Name', 'Depth'], axis=1)

#============================================================
df0=training_data0.dropna()
df1 = df0.drop(['Formation', 'Well Name', 'Depth','Facies'], axis=1)
df1a=df0[(np.abs(stats.zscore(df1))<8).all(axis=1)]
all1=magic(df1a)
#X, y = make_balanced_binary(all1, 9,6)
for kk in range(1,3):
    X, y = make_balanced_binary(all1, 9,kk)
#============================================================
    correct_train=y

    #clf = RandomForestClassifier(max_depth = 6, n_estimators=1600)
    clf = RandomForestClassifier(max_depth = 6, n_estimators=800)
    clf.fit(X,correct_train)

    predicted_blind1 = clf.predict(features_blind)

    predicted_regime9=predicted_blind1.copy()
    print("kk is %d, nr of predictions for this regime is %d" % (kk, sum(predicted_regime9)))
    print "----------------------------------"


I am working on the file "MSS_Xmas_Trees/ml_seg_sub5_STU2.ipynb".
The first task is:
Phase II:
Stacking the predictions from phase Ib.
New predictions from data B

First prediction of B data without Phase I input:
Can you write Python code for it?

#clf = RandomForestClassifier(max_depth = 15, n_estimators=1600,min_samples_leaf=15)
clf = RandomForestClassifier(max_depth = 8, n_estimators=3000)
clf.fit(go_B,correctB)
predicted_blind_PHASE_I = clf.predict(features_blind)
#out_f1=metrics.f1_score(correct_facies_labels, predicted_blind_PHASE_I, average = 'micro')
#print "f1 score on the prediction of blind"
#print out_f1


I am working on the file "MSS_Xmas_Trees/ml_seg_sub5_STU2.ipynb".
The first task is:
This is the only feature engineering used:
Can you write Python code for it?


I am working on the file "MSS_Xmas_Trees/ml_seg_sub5_STU2.ipynb".
The first task is:
Through experimenting with the cluster size I've decided on 4 clusters.
This corresponds largely with the corresponding similarity in facies distribution
CRAWFORD is most similar to ALEXANDER and LUKE. This will called cluster 1. (The only ones with facies 1)
STUART is most similar to KIMZEY and NOLAN This will be called cluster 2
Collating the Data:
based on the regimes we determined
Can you write Python code for it?

# based on kmeans clustering
data=[]
df = training_data0[training_data0['Well Name'] == 'ALEXANDER D']
data.append(df)
df = training_data0[training_data0['Well Name'] == 'LUKE G U']
data.append(df)
df = training_data0[training_data0['Well Name'] == 'CROSS H CATTLE']
data.append(df)
Regime_1 = pd.concat(data, axis=0)
print len(Regime_1)

data=[]
df = training_data0[training_data0['Well Name'] == 'KIMZEY A']
data.append(df)
df = training_data0[training_data0['Well Name'] == 'NOLAN']
data.append(df)
df = training_data0[training_data0['Well Name'] == 'CHURCHMAN BIBLE']
data.append(df)
df = training_data0[training_data0['Well Name'] == 'SHANKLE']
data.append(df)
Regime_2 = pd.concat(data, axis=0)
print len(Regime_2)

data=[]

df = training_data0[training_data0['Well Name'] == 'SHRIMPLIN']
data.append(df)
df = training_data0[training_data0['Well Name'] == 'NEWBY']
data.append(df)
df = training_data0[training_data0['Well Name'] == 'Recruit F9']
data.append(df)
Regime_3 = pd.concat(data, axis=0)
print len(Regime_3)


I am working on the file "MSS_Xmas_Trees/ml_seg_sub5_STU2.ipynb".
The first task is:
training for facies 1 specifically
Can you write Python code for it?

features_blind = blind.drop(['Formation', 'Well Name', 'Depth'], axis=1)

#============================================================
df0=training_data0.dropna()
df1 = df0.drop(['Formation', 'Well Name', 'Depth','Facies'], axis=1)
df1a=df0[(np.abs(stats.zscore(df1))<8).all(axis=1)]
all1=magic(df1a)

#for kk in range(1,13):
for kk in range(2,5):
    X, y = make_balanced_binary(all1, 1,kk)
    #============================================================

    #=============================================
    go_A=StandardScaler().fit_transform(X)
    go_blind=StandardScaler().fit_transform(features_blind)
    correct_train_A=binarify(y, 1)

    clf = linear_model.LogisticRegression()
    clf.fit(go_A,correct_train_A)
    predicted_blind1 = clf.predict(go_blind)

    clf = KNeighborsClassifier(n_neighbors=5)
    clf.fit(go_A,correct_train_A)
    predicted_blind2 = clf.predict(go_blind)

    clf = svm.SVC(decision_function_shape='ovo')
    clf.fit(go_A,correct_train_A)
    predicted_blind3 = clf.predict(go_blind)

    clf = svm.LinearSVC()
    clf.fit(go_A,correct_train_A)
    predicted_blind4 = clf.predict(go_blind)

    #####################################
    predicted_blind=predicted_blind1+predicted_blind2+predicted_blind3+predicted_blind4
    for ii in range(len(predicted_blind)):
        if predicted_blind[ii] > 3:
            predicted_blind[ii]=1
        else:
            predicted_blind[ii]=0

    for ii in range(len(predicted_blind)):
        if predicted_blind[ii] == 1 and predicted_blind[ii-1] == 0 and predicted_blind[ii+1] == 0:
            predicted_blind[ii]=0
        if predicted_blind[ii] == 1 and predicted_blind[ii-1] == 0 and predicted_blind[ii+2] == 0:
            predicted_blind[ii]=0
        if predicted_blind[ii] == 1 and predicted_blind[ii-2] == 0 and predicted_blind[ii+1] == 0:
            predicted_blind[ii]=0
    #####################################

   ...


I am working on the file "MSS_Xmas_Trees/ml_seg_sub5_STU2.ipynb".
The first task is:
training for facies 5 specifically
Can you write Python code for it?

features_blind = blind.drop(['Formation', 'Well Name', 'Depth'], axis=1)

#============================================================
df0=training_data0.dropna()
df1 = df0.drop(['Formation', 'Well Name', 'Depth','Facies'], axis=1)
df1a=df0[(np.abs(stats.zscore(df1))<8).all(axis=1)]
all1=magic(df1a)
for kk in range(1,6):
#for kk in range(2,4):
    X, y = make_balanced_binary(all1, 5,kk)
    #X, y = make_balanced_binary(all1, 5,13)
    #============================================================

    go_A=StandardScaler().fit_transform(X)
    go_blind=StandardScaler().fit_transform(features_blind)
    correct_train_A=binarify(y, 1)
    #=============================================

    clf = KNeighborsClassifier(n_neighbors=4,algorithm='brute')
    clf.fit(go_A,correct_train_A)
    predicted_blind1 = clf.predict(go_blind)

    clf = KNeighborsClassifier(n_neighbors=5,leaf_size=10)
    clf.fit(go_A,correct_train_A)
    predicted_blind2 = clf.predict(go_blind)

    clf = KNeighborsClassifier(n_neighbors=5)
    clf.fit(go_A,correct_train_A)
    predicted_blind3 = clf.predict(go_blind)

    clf = tree.DecisionTreeClassifier()
    clf.fit(go_A,correct_train_A)
    predicted_blind4 = clf.predict(go_blind)

    clf = tree.DecisionTreeClassifier()
    clf.fit(go_A,correct_train_A)
    predicted_blind5 = clf.predict(go_blind)

    clf = tree.DecisionTreeClassifier()
    clf.fit(go_A,correct_train_A)
    predicted_blind6 = clf.predict(go_blind)

    #####################################
    predicted_blind=predicted_blind1+predicted_blind2+predicted_blind3+predicted_blind4+predicted_blind5+predicted_blind6
    for ii in range(len(predicted_blind)):
        if predicted_blind[ii] > 4:
            predicted_blind[ii]=1
        else:
            predicted_blind[ii]=0

    print "-------"
    predicted_regime5=predicted_blind.copy()
    print("kk is %d, nr of predictions for this regime is %d" % (kk,...


I am working on the file "MSS_Xmas_Trees/ml_seg_sub5_STU2a.ipynb".
The first task is:
Add the initial predictions as features:
Can you write Python code for it?


I am working on the file "MSS_Xmas_Trees/ml_seg_sub5_STU2a.ipynb".
The first task is:
Create several predictions, varying the dataset and the technique
Can you write Python code for it?


I am working on the file "MSS_Xmas_Trees/ml_seg_sub5_STU2a.ipynb".
The first task is:
Do this with a couple of predictions and average
To Do
Permute facies based on earlier prediction:
Can you write Python code for it?


I am working on the file "MSS_Xmas_Trees/ml_seg_sub5_STU2a.ipynb".
The first task is:
Make a new prediction, with the best model on the full dataset B:
Can you write Python code for it?

clf.fit(go_B_PHASE_II,correctB)
predicted_blind_PHASE_II = clf.predict(features_blind_PHASE_II)
#out_f1=metrics.f1_score(correct_facies_labels, predicted_blind_PHASE_II, average = 'micro')
#print "f1 score on the prediction of blind"
#print out_f1

print len(predicted_blind_PHASE_II)-np.count_nonzero(predicted_blind_PHASE_II==predicted_blind_PHASE_I)


I am working on the file "MSS_Xmas_Trees/ml_seg_sub5_STU2a.ipynb".
The first task is:
PHASE Ib
Making several predictions using dataset A
PREPARE THE BLIND DATA FOR SERIAL MODELLING
Can you write Python code for it?

df0 = test_data[test_data['Well Name'] == 'STUART']
df1 = df0.drop(['Formation', 'Well Name', 'Depth'], axis=1)
df1a=df0[(np.abs(stats.zscore(df1))<8).all(axis=1)]
blind=magic(df1a)
pred_blind=0*predicted_regime7


I am working on the file "MSS_Xmas_Trees/ml_seg_sub5_STU2a.ipynb".
The first task is:
Phase 1a:
-Create predictions specifically for the most difficult facies
-at this stage we focus on TP and FP only

training for facies 9 specifically
Can you write Python code for it?

# df0 = test_data[test_data['Well Name'] == 'CRAWFORD']
# df1 = df0.drop(['Formation', 'Well Name', 'Depth'], axis=1)
# df1a=df0[(np.abs(stats.zscore(df1))<8).all(axis=1)]
# blind=magic(df1a)

df0 = test_data[test_data['Well Name'] == 'STUART']
df1 = df0.drop(['Formation', 'Well Name', 'Depth'], axis=1)
df1a=df0[(np.abs(stats.zscore(df1))<8).all(axis=1)]
blind=magic(df1a)

df1a.head()

features_blind = blind.drop(['Formation', 'Well Name', 'Depth'], axis=1)

#============================================================
df0=training_data0.dropna()
df1 = df0.drop(['Formation', 'Well Name', 'Depth','Facies'], axis=1)
df1a=df0[(np.abs(stats.zscore(df1))<8).all(axis=1)]
all1=magic(df1a)
#X, y = make_balanced_binary(all1, 9,6)
for kk in range(4,5):
    X, y = make_balanced_binary(all1, 9,kk)
#============================================================
    correct_train=y

    #clf = RandomForestClassifier(max_depth = 6, n_estimators=1600)
    clf = RandomForestClassifier(max_depth = 6, n_estimators=800)
    clf.fit(X,correct_train)

    predicted_blind1 = clf.predict(features_blind)

    predicted_regime9=predicted_blind1.copy()
    print("kk is %d, nr of predictions for this regime is %d" % (kk, sum(predicted_regime9)))
    print "----------------------------------"


I am working on the file "MSS_Xmas_Trees/ml_seg_sub5_STU2a.ipynb".
The first task is:
Phase II:
Stacking the predictions from phase Ib.
New predictions from data B

First prediction of B data without Phase I input:
Can you write Python code for it?


I am working on the file "MSS_Xmas_Trees/ml_seg_sub5_STU2a.ipynb".
The first task is:
Split the data into 2 parts:
from A We will make initial predictions
from B we will make the final prediction(s)
Can you write Python code for it?

df0=Regime_1.dropna()
df1 = df0.drop(['Formation', 'Well Name', 'Depth','Facies'], axis=1)
df1a=df0[(np.abs(stats.zscore(df1))<8).all(axis=1)]
df2a=magic(df1a)

y=df2a['Facies'].values
X=df2a.drop(['Formation', 'Well Name', 'Depth','Facies'], axis=1)
regime1A_train, regime1B_train, regime1A_test, regime1B_test = train_test_split(X, y, test_size=0.5, random_state=42)

df0=Regime_2.dropna()
df1 = df0.drop(['Formation', 'Well Name', 'Depth','Facies'], axis=1)
df1a=df0[(np.abs(stats.zscore(df1))<8).all(axis=1)]
df2b=magic(df1a)
y= df2b['Facies'].values
X= df2b.drop(['Formation', 'Well Name', 'Depth','Facies'], axis=1)
regime2A_train, regime2B_train, regime2A_test, regime2B_test= train_test_split(X, y, test_size=0.5, random_state=42)

df0=Regime_3.dropna()
df1 = df0.drop(['Formation', 'Well Name', 'Depth','Facies'], axis=1)
df1a=df0[(np.abs(stats.zscore(df1))<8).all(axis=1)]
df2c=magic(df1a)
y=df2c['Facies'].values
X= df2c.drop(['Formation', 'Well Name', 'Depth','Facies'], axis=1)
regime3A_train, regime3B_train, regime3A_test, regime3B_test = train_test_split(X, y, test_size=0.5, random_state=42)

#df0=Regime_4.dropna()
#df1 = df0.drop(['Formation', 'Well Name', 'Depth','Facies'], axis=1)
#df1a=df0[(np.abs(stats.zscore(df1))<8).all(axis=1)]
#df2d=magic(df1a)
#y=df2d['Facies'].values
#X= df2d.drop(['Formation', 'Well Name', 'Depth','Facies'], axis=1)
#regime4A_train, regime4B_train, regime4A_test, regime4B_test = train_test_split(X, y, test_size=0.5, random_state=42)


I am working on the file "MSS_Xmas_Trees/ml_seg_sub5_STU2a.ipynb".
The first task is:
Through experimenting with the cluster size I've decided on 4 clusters.
This corresponds largely with the corresponding similarity in facies distribution
CRAWFORD is most similar to ALEXANDER and LUKE. This will called cluster 1. (The only ones with facies 1)
STUART is most similar to KIMZEY and NOLAN This will be called cluster 2
Collating the Data:
based on the regimes we determined
Can you write Python code for it?


I am working on the file "MSS_Xmas_Trees/ml_seg_sub5_STU2a.ipynb".
The first task is:
training for facies 1 specifically
Can you write Python code for it?


I am working on the file "MSS_Xmas_Trees/ml_seg_sub5_STU2a.ipynb".
The first task is:
training for facies 7 specifically
Can you write Python code for it?

features_blind = blind.drop(['Formation', 'Well Name', 'Depth'], axis=1)

#============================================================
df0=training_data0.dropna()
df1 = df0.drop(['Formation', 'Well Name', 'Depth','Facies'], axis=1)
df1a=df0[(np.abs(stats.zscore(df1))<8).all(axis=1)]
all1=magic(df1a)
for kk in range(1,13):
    X, y = make_balanced_binary(all1, 7,kk)
    #X, y = make_balanced_binary(all1, 7,13)
    #============================================================

    go_A=StandardScaler().fit_transform(X)
    go_blind=StandardScaler().fit_transform(features_blind)
    correct_train_A=binarify(y, 1)
    #=============================================

    clf = KNeighborsClassifier(n_neighbors=4,algorithm='brute')
    clf.fit(go_A,correct_train_A)
    predicted_blind1 = clf.predict(go_blind)

    clf = KNeighborsClassifier(n_neighbors=5,leaf_size=10)
    clf.fit(go_A,correct_train_A)
    predicted_blind2 = clf.predict(go_blind)

    clf = KNeighborsClassifier(n_neighbors=5)
    clf.fit(go_A,correct_train_A)
    predicted_blind3 = clf.predict(go_blind)

    clf = tree.DecisionTreeClassifier()
    clf.fit(go_A,correct_train_A)
    predicted_blind4 = clf.predict(go_blind)

    clf = tree.DecisionTreeClassifier()
    clf.fit(go_A,correct_train_A)
    predicted_blind5 = clf.predict(go_blind)

    clf = tree.DecisionTreeClassifier()
    clf.fit(go_A,correct_train_A)
    predicted_blind6 = clf.predict(go_blind)

    #####################################
    predicted_blind=predicted_blind1+predicted_blind2+predicted_blind3+predicted_blind4+predicted_blind5+predicted_blind6
    for ii in range(len(predicted_blind)):
        if predicted_blind[ii] > 5:
            predicted_blind[ii]=1
        else:
            predicted_blind[ii]=0

    #####################################
    print "-------"
    predicted_regime7=predicted_blind.copy()
    print("kk is %d, nr of predictions for this regime is...


I am working on the file "MSS_Xmas_Trees/ml_seg_try1.ipynb".
The first task is:
2) Prediction of Blind well
Can you write Python code for it?

#remove 1 well
blind = training_data[training_data['Well Name'] == 'SHANKLE']
training_data = training_data[training_data['Well Name'] != 'SHANKLE']

correct_facies_labels = training_data['Facies'].values
feature_vectors = training_data.drop(['Formation', 'Well Name', 'Depth','Facies','FaciesLabels'], axis=1)
scaler = preprocessing.StandardScaler().fit(feature_vectors)
scaled_features = scaler.transform(feature_vectors)

X_train, dum1, y_train, dum2 = train_test_split(scaled_features,  correct_facies_labels, test_size=0.2, random_state=0)
rf.fit(X_train, y_train)

# get the blind well
correct_facies_labels = blind['Facies'].values
feature_vectors = blind.drop(['Formation', 'Well Name', 'Depth','Facies','FaciesLabels'], axis=1)
scaler = preprocessing.StandardScaler().fit(feature_vectors)
scaled_features = scaler.transform(feature_vectors)

predicted_random_forest = rf.predict(scaled_features)
print "All training data different from test well"
print "prediction from random forest:"
print metrics.accuracy_score(correct_facies_labels, predicted_random_forest)

print "f1 score:"
print metrics.f1_score(correct_facies_labels, predicted_random_forest,average = 'weighted')


I am working on the file "MSS_Xmas_Trees/ml_seg_try1.ipynb".
The first task is:
Basic statistics by well:
Can you write Python code for it?

temp_4=training_data.groupby('Well Name')
#temp_4.describe()
#temp_5=training_data.groupby('Well Name').count()
#temp_5=training_data.groupby('Well Name').max()
temp_5=training_data.groupby('Well Name').mean()
temp_5


I am working on the file "MSS_Xmas_Trees/ml_seg_try1.ipynb".
The first task is:
First steps, reading in and exploring the data are the same as Brendon's steps:
Can you write Python code for it?

filename = 'training_data.csv'
training_data0 = pd.read_csv(filename)
training_data0.head()


I am working on the file "MSS_Xmas_Trees/ml_seg_try1.ipynb".
The first task is:
The prediction performs much much beter if the all data is included in the training,
compared to blind wells. Shouldn't be that much a surprise but doesn't this suggest some wells
are not representative of the others
Can you write Python code for it?

from sklearn.metrics import confusion_matrix
from classification_utilities import display_cm, display_adj_cm

#conf = confusion_matrix(correct_facies_labels, predicted_gradboost)
conf = confusion_matrix(correct_facies_labels, predicted_random_forest)

display_cm(conf, facies_labels, hide_zeros=True)


I am working on the file "MSS_Xmas_Trees/ml_seg_try1.ipynb".
The first task is:
This is the benchmark to beat : 0.44 using rf, (slightly higher for gradient boost)

3) Data exploration
Basic statistics by facies:
Can you write Python code for it?


I am working on the file "MSS_Xmas_Trees/ml_seg_try1.ipynb".
The first task is:
scale the data:
Can you write Python code for it?

scaler = preprocessing.StandardScaler().fit(feature_vectors)
scaled_features = scaler.transform(feature_vectors)

X_train, X_test, y_train, y_test = train_test_split(scaled_features,  correct_facies_labels, test_size=0.2, random_state=0)

rf = RandomForestClassifier(max_depth = 15,n_estimators=200,max_features=None)
#rf = RandomForestClassifier()
rf.fit(X_train, y_train)

predicted_random_forest = rf.predict(X_test)

print "prediction from random forest:"
print metrics.accuracy_score(list(y_test), predicted_random_forest)

print "f1 score:"
print metrics.f1_score(list(y_test), predicted_random_forest,average = 'weighted')

training_data=training_data0.copy()


I am working on the file "MVideo/training model_1(keras==1.2.2).ipynb".
The first task is:
Cнизижаем размерность
Can you write Python code for it?

p = PCA(n_components=100)

p.fit(X_train.reshape((X_train.shape[0], -1)))

X_train_transformed = p.transform(X_train.reshape((X_train.shape[0], -1)))

X_test_transformed = p.transform(X_test.reshape((X_test.shape[0], -1)))


I am working on the file "MVideo/training model_1(keras==1.2.2).ipynb".
The first task is:
Загрузка обработчика комментариев
Загружаем предобученную модель word to vec в обработчик комментариев (модель можно найти здесь, мы использовали модель размерностью 500 со словарем более 1 млн. слов).

В обработчике:
 - переводим текст в нижний регистр
 - разбиваем по словам, удаляя всё, кроме букв
 - исправляем орфографию, где это возможно, с помощью Яндекс.спеллер
 - нормализуем слова, приводя их к единственному числу, именительному падежу и т.д. и т.п.
 - еще раз иcправляем орфографию :)
 - переводим полученный массив слов с помощью обученной w2v модели в матрицу размерностью количество_слов х размер_вектора_слова - 100x500 (размер комментария выбран изходя из анализа данных в  "data preprocessing.ipynb")
Can you write Python code for it?

DIR = 'data/w2v_models/'
MODEL_NAME = 'tenth.norm-sz500-w7-cb0-it5-min5.w2v'
#MODEL_NAME = 'all.norm-sz100-w10-cb0-it1-min100.w2v'

# initialazing comment preprocessor
sp = SentenceProcessor(DIR + MODEL_NAME)
sp.stop_list = stop_words
sp.sample_len = 100

WORD_VECTOR_LENGTH = sp.w2v.vector_size # 500 each word
SENTENCE_LENGTH = sp.sample_len # 100 words

print ('Размерность коментария: {}x{}'.format(SENTENCE_LENGTH, WORD_VECTOR_LENGTH))


I am working on the file "MVideo/training model_1(keras==1.2.2).ipynb".
The first task is:
Классификатор 5 / не 5
Can you write Python code for it?

from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import BaggingClassifier
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.svm import SVC
from sklearn.decomposition import PCA
from sklearn.metrics import accuracy_score

def to_bool(y):
    y_bool = []
    for a in y:
        if a == 5.0:
            y_bool.append(1)
        else:
            y_bool.append(0)
    return y_bool

y_train_bool = to_bool(y_train)
y_test_bool = to_bool(y_test)


I am working on the file "MVideo/training model_1(keras==1.2.2).ipynb".
The first task is:
Модель № 1
Вход
Данные: Данные предоставленные МВидео (80% обучающая выборка/ 20% тестовая выборка)
Представление: word2vec на основе обученной модели gensim, размерность пространства: 500
Описание модели: 4 параллельные 1D свертки с разым размером окна по компоненте word2vec, с L2 регулиризацией, далее объединение результатов, GlobalVaxPooling, и один полносвязный слой с дропаутом, далее выход регрессии.
Параметры обучения: Оптимизатор: стохастический градиентный спуск, learning rate опредлялся как: 2 эпохи с 0.01, далее ~5 эпох с 0.001, далее ~5 эпох с 0.0001, далее ~5 эпох с 0.00005
Целевая метрика: MSE
Результат (целевые переменные не нормировались!): MSE(train)=0.8403, MSE(test)=0.9136
Загрузка данных
Для начала загружаем данные из *.csv файлов, данные были предварительно разбиты на обучающую и проверочную выборки (см. файл "data preprocessing.ipynb"). Кроме того, были загружены дополнительные данные для увеличения обучающей выборки(отзывы об интернет провайдерах).
Can you write Python code for it?

# загрузка из файла
reviews_test = pd.read_csv('data/reviews_test.csv', header=0, encoding='utf-8')
reviews_train = pd.read_csv('data/reviews_train.csv', header=0, encoding='utf-8')
reviews_internet = pd.read_csv('data/internet_reviews.csv', header=0, encoding='utf-8')

# обучающая выборка
X_train_raw = reviews_train.comment
y_train_raw = reviews_train.reting

# проверочная выборка
X_test_raw = reviews_test.comment
y_test_raw = reviews_test.reting

# дополнительная обучающая выборка
X_internet = reviews_internet.comment
y_internet = reviews_internet.rating


I am working on the file "MVideo/training model_1(keras==1.2.2).ipynb".
The first task is:
Обучаем классификатор
Can you write Python code for it?

clf = SVC(C=100.0)
clf.fit(X_train_transformed, y_train_bool)

y_pred = clf.predict(X_test_transformed)

accuracy_score(y_test_bool, y_pred)


I am working on the file "MVideo/training model_1(keras==1.2.2).ipynb".
The first task is:
Результаты
целевые переменные не нормировались!
Can you write Python code for it?

# метрики
errors = get_score(model, X_val_vec, y_val.values, sparse=40)


I am working on the file "MVideo/training model_2 (keras==2.0.8).ipynb".
The first task is:
Обучение модели
Can you write Python code for it?

from keras.models import Sequential
import keras
from keras.layers import Dense
from keras.layers import Flatten
from keras.layers import LocallyConnected1D, Conv1D, Dropout
from keras.layers import MaxPooling1D, GlobalMaxPooling1D
from keras.layers.recurrent import LSTM
from keras.preprocessing import sequence
from keras.optimizers import Adam, SGD
from keras.models import Model
from keras.layers.merge import concatenate
from keras import regularizers
from keras.layers import Input, Dense

input_1 = Input(shape=(100,500))
conv_1 = Conv1D(filters=256, kernel_size=3, activation='relu', kernel_regularizer=regularizers.l2(0.02))(input_1)
pool_1 = GlobalMaxPooling1D()(conv_1)

conv_2 = Conv1D(filters=256, kernel_size=5, activation='relu', kernel_regularizer=regularizers.l2(0.02))(input_1)
pool_2 = GlobalMaxPooling1D()(conv_2)

conv_3 = Conv1D(filters=512, kernel_size=7, activation='relu', kernel_regularizer=regularizers.l2(0.02))(input_1)
pool_3 = GlobalMaxPooling1D()(conv_3)

conv_4 = Conv1D(filters=512, kernel_size=9, activation='relu', kernel_regularizer=regularizers.l2(0.02))(input_1)
pool_4 = GlobalMaxPooling1D()(conv_4)

concat_1 = concatenate([pool_1, pool_2, pool_3, pool_4], axis=1)

dense_1 = Dense(300, activation='relu')(concat_1)
drop_1 = Dropout(0.5)(dense_1)

dense_4 = Dense(1, activation=None)(drop_1)

model = Model(inputs=input_1, outputs=dense_4)

model.summary()

sgd = SGD(lr=0.00005)
model.compile(loss='mean_squared_error', optimizer=sgd, metrics=['mse'])

model.fit(X_train_final, y_train_final, batch_size=10, epochs=15, validation_data=(X_test, y_test), shuffle=True,
          verbose=True)

model.save('trained_model_2(keras==2.0.8)')

import sklearn
from sklearn.metrics import mean_squared_error
from sklearn.metrics import mean_absolute_error
from sklearn.metrics import median_absolute_error
from sklearn.metrics import r2_score
import matplotlib.pyplot as plt

def get_score(model, x, y, plot=True, sparse=50):

    y_pred = model.predict(x)
    y_pred = np.clip(y_pred, 1.0, 5.0)

 ...


I am working on the file "MVideo/training model_2 (keras==2.0.8).ipynb".
The first task is:
Подготовка данных: перевод в матрицу, размером: "количество отзывов" х "количество слов в отзыве (обрезано или дополнено до 100)" х "размерность word2vec"
Can you write Python code for it?

X_train = []
y_train = []

for i in tqdm(range(len(X_train_raw))):
    sent = sentence_processor.process(X_train_raw[i])
    matrix = sentence_processor.convert2matrix(sent, sample_len=SENTENCE_LENGTH)
    if matrix.shape == (SENTENCE_LENGTH, VECTOR_SIZE):
        X_train.append(matrix)
        y_train.append(y_train_raw[i])

X_test = []
y_test = []

for i in tqdm(range(len(X_test_raw))):
    sent = sentence_processor.process(X_test_raw[i])
    matrix = sentence_processor.convert2matrix(sent, sample_len=SENTENCE_LENGTH)
    if matrix.shape == (SENTENCE_LENGTH, VECTOR_SIZE):
        X_test.append(matrix)
        y_test.append(y_test_raw[i])

X_train = np.array(X_train, dtype=np.float32)
X_test = np.array(X_test, dtype=np.float32)
y_train = np.array(y_train, dtype=np.float32)
y_test = np.array(y_test, dtype=np.float32)

reviews_internet = pd.read_csv('data/internet_reviews.csv', header=0, encoding='utf-8')

X_reviews_internet = reviews_internet.comment
y_reviews_internet = reviews_internet.rating

X_reviews_internet_ = []
y_reviews_internet_ = []

for i in tqdm(range(len(X_reviews_internet))):
    sent = sentence_processor.process(X_reviews_internet[i])
    matrix = sentence_processor.convert2matrix(sent, sample_len=SENTENCE_LENGTH)
    if matrix.shape == (SENTENCE_LENGTH, VECTOR_SIZE):
        X_reviews_internet_.append(matrix)
        y_reviews_internet_.append(y_reviews_internet[i])

X_reviews_internet_ = np.array(X_reviews_internet_, dtype=np.float32)
y_reviews_internet_ = np.array(y_reviews_internet_, dtype=np.float32)

X_train_final = np.concatenate((X_train, X_reviews_internet_), axis=0)
y_train_final = np.concatenate((y_train, y_reviews_internet_), axis=0)


I am working on the file "MVideo/training model_2 (keras==2.0.8).ipynb".
The first task is:
Результаты
Результаты на тестовой выборке (20% от исходных данных)
целевые переменные не нормировались!
Can you write Python code for it?

get_score(model, X_test, y_test, sparse=50)


I am working on the file "MXCH.ipynb".
The first task is:
Figure 1.A: Transverse Mercator
Can you write Python code for it?

# 4000 km east-west, 3000 km north-south, low resolution, Transverse Mercator
width = 4000000
height = 3000000
res = 'l'
proj = 'tmerc'

fig1a = plt.figure(figsize=(8, 6))

# Create basemaps for Mexico and Switzerland
m_mx = Basemap(width=width, height=height, resolution=res, projection=proj,
            lon_0=DF[0]+shift[0], lat_0=DF[1]+shift[1])
m_ch = Basemap(width=width, height=height, resolution=res, projection=proj,
            lon_0=TH[0]+shift[0], lat_0=TH[1]+shift[1])

# Draw coast lines
m_mx.drawcoastlines(color=cMX, linewidth=1)
m_ch.drawcoastlines(color=cCH, linewidth=1)

# Draw all country borders
m_mx.drawcountries(color=cMX, linewidth=1)
m_ch.drawcountries(color=cCH, linewidth=1)

# Fill continents and lakes
m_mx.fillcontinents(lake_color='none', color=cMX, alpha=.4)
m_ch.fillcontinents(lake_color='none', color=cCH, alpha=.4)

# Draw extra points for the Swiss and Mexican countries to highlight
CHE_adm0 = shapefile.Reader('data/basemap/CHE_adm0')
MEX_adm0 = shapefile.Reader('data/basemap/MEX_adm0')

chlonlat = np.array(CHE_adm0.shape().points)
mxlonlat = np.array(MEX_adm0.shape().points)

m_ch.plot(chlonlat[:, 0], chlonlat[:, 1] , '.', c=cCH, latlon=True, ms=1)
m_mx.plot(mxlonlat[:, 0], mxlonlat[:, 1], '.', c=cMX, latlon=True, ms=1)

# Draw scales to cross-check that the scales are the same
# (shift scale 1600 km to the west from centre, and a bit north/south)
smx = np.array(m_mx(DF[0], DF[1])) - np.array([1600000,  50000])
sch = np.array(m_ch(TH[0], TH[1])) - np.array([1600000, -350000])
ismx = m_mx(smx[0], smx[1], inverse='True')
isch = m_ch(sch[0], sch[1], inverse='True')

m_mx.drawmapscale(ismx[0], ismx[1], DF[0]+shift[0], DF[1]+shift[1],
                  500, barstyle='fancy', fillcolor2=cMX, fontsize=12)
m_ch.drawmapscale(isch[0], isch[1], TH[0]+shift[0], TH[1]+shift[1],
                  500, barstyle='fancy',  fillcolor2=cCH, fontsize=12)

# Add Mexico City and Thalheim AG
m_mx.plot(DF[0], DF[1], 'x', c='r', ms=10, mew=2, latlon=True, label='Thalheim')
m_ch.plot(TH[0], TH[1], '+',...


I am working on the file "MXCH.ipynb".
The first task is:
Figure 1.B: Orthographic
Can you write Python code for it?

fig1b = plt.figure(figsize=(8, 8))

# Create basemaps for Mexico and Switzerland
m_mx = Basemap(resolution='c', projection='ortho', lon_0=DF[0], lat_0=DF[1])
m_ch = Basemap(resolution='c', projection='ortho', lon_0=TH[0], lat_0=TH[1])

# Draw coast lines
m_mx.drawcoastlines(color=cMX, linewidth=1)
m_ch.drawcoastlines(color=cCH, linewidth=1)

# Draw all country borders
m_mx.drawcountries(color=cMX, linewidth=1)
m_ch.drawcountries(color=cCH, linewidth=1)

# Fill continents and lakes
m_mx.fillcontinents(lake_color='none', color=cMX, alpha=.4)
m_ch.fillcontinents(lake_color='none', color=cCH, alpha=.4)

plt.show()


I am working on the file "MXCH.ipynb".
The first task is:
Settings for both figures
I want that Mexico City and my home village Thalheim overlay (I took the coordinates from Wikipedia). The easiest way to achieve this is to use them as the centre of the map. However, I shift them subsequently by 2° to the south, as Mexico City is not in the
centre of Mexico.
Can you write Python code for it?

# Coordinates (from Wikipedia) and shift
DF = [-99.1333, 19.4333]
TH = [  8.1042, 47.4375]
shift = [0, 2]

# Colours
cMX = '#0000cc' # Blue tone
cCH = '#006600' # Green tone


I am working on the file "MX_BarrancasDelCobre.ipynb".
The first task is:
Profile from viewpoint down to Urique
Not used in blog, later added
Can you write Python code for it?

import numpy as np
import matplotlib.pyplot as plt
fig_p,ax = plt.subplots(figsize=(tm.cm2in([10.8, 5])))

# Switch off axis and ticks
ax.spines['right'].set_visible(False)
ax.spines['top'].set_visible(False)
ax.spines['left'].set_visible(False)
ax.spines['bottom'].set_visible(False)
ax.xaxis.set_ticks_position('bottom')
ax.yaxis.set_ticks_position('none')

# Get data
pdat = np.loadtxt('./data/Mexico/ProfileUrique.txt', skiprows=1)

# Ticks, hlines, axis
plt.yticks(np.arange(1,6)*500, ('500 m', '1000 m', '1500 m', '2000 m', '2500 m'))
plt.hlines([1000, 2000], -.5, 17, colors='.8')
plt.hlines([500, 1500, 2500], -.5, 17, colors='.8', lw=.5)
plt.axis([-.5, 17, 200, 2500])

# Sum up differences to get distance, distance starts now at every waypoint
distance = np.cumsum(pdat[:,4])/1000

# Plot data
plt.plot(distance, pdat[:, 2])
plt.show()


I am working on the file "MX_Oaxaca.ipynb".
The first task is:
2. Profile
Route drawn in Google Earth, exported, and subsequently converted in http://www.gpsvisualizer.com.
Can you write Python code for it?

fig_p,ax = plt.subplots(figsize=(tm.cm2in([10.8, 5])))

# Switch off axis and ticks
ax.spines['right'].set_visible(False)
ax.spines['top'].set_visible(False)
ax.spines['left'].set_visible(False)
ax.spines['bottom'].set_visible(False)
ax.xaxis.set_ticks_position('none')
ax.yaxis.set_ticks_position('none')

# Get data
pdat = np.loadtxt('./data/Mexico/OaxacaProfile.txt', delimiter='\t', skiprows=1)

# Plot City names and kilometers
plt.annotate('Mexiko Stadt', (0, 5500), horizontalalignment='center', rotation='vertical')
plt.annotate('Puebla (129 km)', (129, 6000), horizontalalignment='center', rotation='vertical')
plt.annotate('Tehuacan (252 km)', (252, 6000), horizontalalignment='center', rotation='vertical')
plt.annotate('Oaxaca (465 km)', (465, 5400), horizontalalignment='center', rotation='vertical')

# Ticks, hlines, axis
plt.xticks(())
plt.yticks((1000, 1500, 2000, 2500, 3000, 3500), ('1000 m', '', '2000 m', '', '3000 m', ''))
plt.hlines([1000, 2000, 3000], -100, 500, colors='.8')
plt.hlines([1500, 2500, 3500], -100, 500, colors='.8', lw=.5)
plt.axis([-20, 486, 500, 6000])

# Plat data
plt.plot(pdat[:,3], pdat[:,2])

# Save-path
#fpath = '../mexico.werthmuller.org/content/images/oaxaca/'
#plt.savefig(fpath+'Profile.png', bbox_inches='tight')

plt.show()


I am working on the file "MX_S&S_Trip.ipynb".
The first task is:
1. Map
Can you write Python code for it?

fig_x = tm.plt.figure(figsize=(tm.cm2in([11, 6])))

# Locations
MDF = [19.433333, -99.133333] # Mexico City
OAX = [16.898056, -96.414167] # Oaxaca
PES = [15.861944, -97.067222] # Puerto Escondido
ACA = [16.863611, -99.8825]   # Acapulco
PBL = [19., -97.883333]       # Puebla

# Create basemap
m_x = tm.Basemap(width=3500000, height=2300000, resolution='c',  projection='tmerc', lat_0=24, lon_0=-102)

# Plot image
###m_x.warpimage('./data/TravelMap/HYP_HR_SR_OB_DR/HYP_HR_SR_OB_DR.tif')

# Put a shade over non-Mexican countries
countries = ['USA', 'BLZ', 'GTM', 'HND', 'SLV', 'NIC', 'CUB']
tm.country(countries, m_x, fc='.8', ec='.3', lw=.5, alpha=.6)

# Fill states
fcs = 32*['none']
ecs = 32*['k']
lws = 32*[.2,]
tm.country('MEX', bmap=m_x, fc=fcs, ec=ecs, lw=lws, adm=1)
ecs = 32*['none']
#ecs[19] = 'r'
lws = 32*[1,]
tm.country('MEX', bmap=m_x, fc=fcs, ec=ecs, lw=lws, adm=1)

# Add arrows
tm.arrow(MDF, ACA, m_x, rad=.3)
tm.arrow(ACA, PES, m_x, rad=.3)
#tm.arrow(PES, OAX, m_x, rad=-.3)
tm.arrow(OAX, PBL, m_x, rad=.3)
#tm.arrow(PBL, MDF, m_x, rad=.3)

# Add visited cities
tm.city(OAX, 'Oaxaca', m_x, offs=[.6, 0])
tm.city(MDF, 'Mexiko-Stadt', m_x, offs=[-.6, .6], halign="right")
tm.city(PES, 'Puerto Escondido', m_x, offs=[-2, -1.5])
tm.city(ACA, 'Acapulco', m_x, offs=[-.8, 0], halign="right")
tm.city(PBL, 'Puebla', m_x, offs=[.6, .6])

# Save-path
#fpath = '../mexico.werthmuller.org/content/images/simon/'
#tm.plt.savefig(fpath+'MapSSTrip.png', bbox_inches='tight')
tm.plt.show()


I am working on the file "MX_Sprachenvielfalt.ipynb".
The first task is:
Analphabetismus in Prozent
Can you write Python code for it?

fig = plt.figure()
ax = fig.add_subplot(1, 1, 1)
lal = 3
ax.barh( 7.6, 17.8); ax.text(lal,  8, "17.8 %", va='center', ha='right')
ax.barh( 6.6, 16.7); ax.text(lal,  7, "16.7 %", va='center', ha='right')
ax.barh( 5.6, 16.3); ax.text(lal,  6, "16.3 %", va='center', ha='right')
ax.barh( 4.6, 11.4); ax.text(lal,  5, "11.4 %", va='center', ha='right')
ax.barh( 3.6, 10.4); ax.text(lal,  4, "10.4 %", va='center', ha='right')
ax.barh( 2.6, 10.2); ax.text(lal,  3, "10.2 %", va='center', ha='right')
ax.barh( 1.6, 10.2); ax.text(lal,  2, "10.2 %", va='center', ha='right')
ax.barh( 0.6,  6.9); ax.text(lal,  1, " 6.9 %", va='center', ha='right')

ax.xaxis.set_ticks_position('none')
ax.yaxis.set_ticks_position('none')

plt.xticks([5, 10, 15], ())
plt.yticks(np.arange(8)+1, ('Mexiko Durch.', 'Hidalgo', 'Michoacán', 'Puebla', 'Veracruz',
                             'Oaxaca', 'Guerrero', 'Chiapas'))

plt.title("Analphabetismus in Prozent")

#plt.savefig(fpath+'sprachenvielfalt/Analfabetismo.png', bbox_inches='tight')
plt.show()


I am working on the file "MX_Sprachenvielfalt.ipynb".
The first task is:
Karte Analphabetismus
Can you write Python code for it?

tm.setup_noxkcd(200)
fig_x = plt.figure(figsize=(tm.cm2in([11, 6])))

# Create basemap
m_x = Basemap(width=3500000, height=2300000, resolution='c',
              projection='tmerc', lat_0=24, lon_0=-102)
m_x.drawmapboundary(fill_color='#99ccff')

# Fill non-visited countries (fillcontinents does a bad job)
countries = ['USA', 'BLZ', 'GTM', 'HND', 'SLV', 'NIC', 'CUB']
tm.country(countries, m_x, fc='.8', ec='.5', lw=.5)

# Fill states
cols = np.array([3.3,  #  0 Aguascalientes
                 3.2,  #  1 Baja California Sur
                 2.6,  #  2 Baja California
                 8.3,  #  3 Campeche
                 17.8, #  4 Chiapas
                 3.7,  #  5 Chihuahua
                 2.6,  #  6 Coahuila
                 5.1,  #  7 Colima
                 2.1,  #  8 Distrito Federal
                 3.8,  #  9 Durango
                 8.2,  # 10 Guanajuato
                 16.7, # 11 Guerrero
                 10.2, # 12 Hidalgo
                 4.4,  # 13 Jalisco
                 4.4,  # 14 México
                 10.2, # 15 Michoacán
                 6.4,  # 16 Morelos
                 6.3,  # 17 Nayarit
                 2.2,  # 18 Nuevo León
                 16.3, # 19 Oaxaca
                 10.4, # 20 Puebla
                 6.3,  # 21 Querétaro
                 4.8,  # 22 Quintana Roo
                 7.9,  # 23 San Luis Potosí
                 5.0,  # 24 Sinaloa
                 3.0,  # 25 Sonora
                 7.1,  # 26 Tabasco
                 3.6,  # 27 Tamaulipas
                 5.2,  # 28 Tlaxcala
                 11.4, # 29 Veracruz
                 9.2,  # 30 Yucatán
                 5.5]) # 31 Zacatecas
cols = cm.Greens((cols-2.1)/(17.8-2.1))

fcs = 32*['']
for i in range(32):
    fcs[i] = colors.rgb2hex(cols[i,:])

tm.country('MEX', bmap=m_x, fc=fcs, ec='k', lw=.5, adm=1)

# Add visited cities
tm.city([0, 0], '', m_x)

# Save-path
#plt.savefig(fpath+'sprachenvielfalt/KarteAnalphabetismus.png', bbox_inches='tight')
plt.show()


I am working on the file "MX_Sprachenvielfalt.ipynb".
The first task is:
Karte indigene Sprachen
Can you write Python code for it?

tm.setup_noxkcd(200)
fig_x = plt.figure(figsize=(tm.cm2in([11, 6])))

# Create basemap
m_x = Basemap(width=3500000, height=2300000, resolution='c',
              projection='tmerc', lat_0=24, lon_0=-102)
m_x.drawmapboundary(fill_color='#99ccff')

# Fill non-visited countries (fillcontinents does a bad job)
countries = ['USA', 'BLZ', 'GTM', 'HND', 'SLV', 'NIC', 'CUB']
tm.country(countries, m_x, fc='.8', ec='.5', lw=.5)

# Fill states
cols = np.array([0.2,  #  0 Aguascalientes
                 1.8,  #  1 Baja California Sur
                 1.4,  #  2 Baja California
                 12.0, #  3 Campeche
                 27.3, #  4 Chiapas
                 3.5,  #  5 Chihuahua
                 0.2,  #  6 Coahuila
                 0.7,  #  7 Colima
                 1.5,  #  8 Distrito Federal
                 2.2,  #  9 Durango
                 0.3,  # 10 Guanajuato
                 15.2, # 11 Guerrero
                 14.8, # 12 Hidalgo
                 0.8,  # 13 Jalisco
                 2.7,  # 14 México
                 3.5,  # 15 Michoacán
                 1.9,  # 16 Morelos
                 5.2,  # 17 Nayarit
                 0.9,  # 18 Nuevo León
                 33.8, # 19 Oaxaca
                 11.5, # 20 Puebla
                 1.8,  # 21 Querétaro
                 16.2, # 22 Quintana Roo
                 10.6, # 23 San Luis Potosí
                 0.9,  # 24 Sinaloa
                 2.5,  # 25 Sonora
                 2.9,  # 26 Tabasco
                 0.8,  # 27 Tamaulipas
                 2.5,  # 28 Tlaxcala
                 9.3,  # 29 Veracruz
                 29.6, # 30 Yucatán
                 0.4]) # 31 Zacatecas
cols = cm.Greens((cols-0.2)/(33.8-.2))

fcs = 32*['']
for i in range(32):
    fcs[i] = colors.rgb2hex(cols[i,:])

tm.country('MEX', bmap=m_x, fc=fcs, ec='k', lw=.5, adm=1)

# Add visited cities
tm.city([0, 0], '', m_x)

# Save-path
#plt.savefig(fpath+'sprachenvielfalt/KarteEinheimischeSprachen.png', bbox_inches='tight')
plt.show()


I am working on the file "Machine Learning - Hands on Lab - Session #1 - Data Visualisation.ipynb".
The first task is:
1.3 Summary - Important Information
1.3.1 Train - Test Split Analysis
Can you write Python code for it?

training_min_date = df_train_users.date_account_created.min()
training_max_date = df_train_users.date_account_created.max()

print ("Training Date Range : [" + training_min_date +", " + training_max_date +"]")

# ===============

testing_min_date = df_test_users.date_account_created.min()
testing_max_date = df_test_users.date_account_created.max()

print ("Testing Date Range : [" + testing_min_date +", " + testing_max_date +"]")


I am working on the file "Machine Learning - Hands on Lab - Session #1 - Data Visualisation.ipynb".
The first task is:
1.4.1 Feature Analysis
Can you write Python code for it?

# How many rows in the DataFrame

row_count = len(df_train_users.index)
print ("Row Count = " + str(row_count))
print("\n")
# ===============================

# Print what's the size of the ID
field_length = df_train_users.id.astype(str).map(len)

id_maxlength = len(df_train_users.loc[field_length.argmax(), 'id'])
id_minlength = len(df_train_users.loc[field_length.argmin(), 'id'])

if (id_maxlength != id_minlength):
    print ("ID Length = [" + str(id_minlength) + ", " + str(id_maxlength) + "]")
else:
    print ("ID Length = " + str(id_maxlength))

print("\n")

# ===============================

# Count NaN Values for date_first_booking
NaN_Count_date_first_booking = df_train_users.date_first_booking.isnull().sum()
print ("date_first_booking NaN Count = " + str(NaN_Count_date_first_booking) \
+ " && " +  str("%.2f" % (float(NaN_Count_date_first_booking)/row_count*100)) +"%")

print("\n")
# ===============================

# Possible Values for gender

gender_repartition = df_train_users.gender.value_counts()
for gender, count in gender_repartition.iteritems():
    print ("Gender: " + gender + " && Count: " + str(count) + " && " \
           + str("%.2f" % (float(count)/row_count*100)) +"%")

print("\n")
# ===============================

# Count NaN Values for age
NaN_Count_age = df_train_users.age.isnull().sum()
print ("age NaN Count = " + str(NaN_Count_age)  + " && "  \
       +  str("%.2f" % (float(NaN_Count_age)/row_count*100)) +"%")

print("\n")
# ===============================

# Possible Values for signup_method

signup_method_repartition = df_train_users.signup_method.value_counts()
for method, count in signup_method_repartition.iteritems():
    print ("Method: " + method + " && Count: " + str(count) + " && " \
           + str("%.2f" % (float(count)/row_count*100)) +"%")

print("\n")
# ===============================

# Possible Values for language

language_repartition = df_train_users.language.value_counts()
for language, count in language_repartition.iteritems():
    print ("language: " +...


I am working on the file "Machine Learning - Hands on Lab - Session #1 - Data Visualisation.ipynb".
The first task is:
1.4.2 Output Analysis - Is the dataset balanced ?
1.4.2.1 Dummy Analysis
Let's plot the the bookings distribution by destination in order to see whether the dataset is balanced or not.
Can you write Python code for it?

dict_output = dict()

country_destination_repartition = df_train_users.country_destination.value_counts()

for country_destination, count in country_destination_repartition.iteritems():
    dict_output[country_destination] = float(count)/row_count*100

df_output = pd.DataFrame(list(dict_output.items()),
                      columns=['Country', 'Repartition'])

df_output.sort_values(by=['Repartition'], ascending=False).head(n=5)

p = bk.Bar(df_output, label=['Country'], values='Repartition', ylabel='Booking Proportion in %')

bk.show(p)


I am working on the file "Machine Learning - Hands on Lab - Session #1 - Data Visualisation.ipynb".
The first task is:
1.4.2.2 Same visualisation, with data grouped by Booking-year
Can you write Python code for it?

df_output_by_year_and_month = df_train_users.loc[:,['country_destination','date_account_created']]

df_output_by_year_and_month.loc[:,'year_account_created'] = df_output_by_year_and_month['date_account_created'].apply(
    lambda x: datetime.strptime(x, "%Y-%m-%d").strftime("%Y")
)

df_output_by_year_and_month.loc[:,'month_account_created'] = df_output_by_year_and_month['date_account_created'].apply(
    lambda x: datetime.strptime(x, "%Y-%m-%d").strftime("%m")
)

df_output_by_year_and_month.loc[:,'year_month_account_created'] = df_output_by_year_and_month['date_account_created'].apply(
    lambda x: datetime.strptime(x, "%Y-%m-%d").strftime("%Y - %m")
)

df_output_by_year_and_month.head(n=3)

tmp_df = df_output_by_year_and_month.loc[:,['country_destination','year_account_created']]
rslt = tmp_df.groupby("year_account_created")['country_destination'].value_counts()

list_year = [x[0] for x in rslt.index.values]
list_dest = [x[1] for x in rslt.index.values]
list_count = rslt.values

df_tmp = pd.DataFrame(
    {'Year': list_year,
     'Destination': list_dest,
     'Count': list_count
    }
)

df_tmp.sort_values(by=["Destination", "Year"]).head(n=5)

df_pivot = pd.pivot_table(df_tmp, values='Count', index=['Destination'], columns=['Year'],
                          aggfunc=np.sum, margins=True).astype(int)

total_by_years = df_pivot.loc[['All']]

for year, col in total_by_years.iteritems():
    subtotal_by_year = col[0]

    df_pivot[year] = df_pivot[year]/subtotal_by_year*100

df_pivot = df_pivot.sort_values(by="All", ascending=False).drop("All") # We sort by the column "All" and remove the line "All"
df_pivot

df_pivot2 = df_pivot.copy()

df_pivot2["Destination"] = df_pivot.index
df_pivot2 = pd.melt(df_pivot2, id_vars="Destination")
df_pivot2 = df_pivot2.drop(df_pivot2[df_pivot2.Year == "All"].index) # We remove the column 'All' from the plotting

df_pivot2.head(n=3)

p = bk.Bar(df_pivot2, label=['Destination'], values='value', group="Year", legend='top_right', ylabel='Booking Proportion in %')
bk.show(p)


I am working on the file "Machine Learning - Hands on Lab - Session #1 - Data Visualisation.ipynb".
The first task is:
2. Dataset Loading & Visualition
2.1 How does the expected "output" look like ?
Sample Submission File - sample_submission_NDF.csv
We load the csv file and visualise it straight away
Can you write Python code for it?

df_sample_submission = pd.read_csv("data/sample_submission_NDF.csv")

df_sample_submission.head(n=5) # Only display a few lines and not the whole dataframe


I am working on the file "Machine Learning - Hands on Lab - Session #1 - Data Visualisation.ipynb".
The first task is:
=> We can conclude that we only have data about the year 2015 !
1.2.4 train_users_2.csv
We load the csv file and visualise it straight away
Can you write Python code for it?

df_train_users = pd.read_csv("data/train_users_2.csv")
df_train_users.head(n=5) # Only display a few lines and not the whole dataframe


I am working on the file "Machine Learning - Hands on Lab - Session #1 - Data Visualisation.ipynb".
The first task is:
Displayed as followed, it is fairly hard to really see what is in this file.
Let's try to sort the file in regards with the first column ("age_bucket") and then by the third one ("gender")
Can you write Python code for it?

tmp_sorted = df_age_gender_bkts.sort_values(by=['age_bucket', 'gender'])
tmp_sorted.head(n=12) # Only display a few lines and not the whole dataframe