I am working on the file "FDMS/TME6/TME6_RecommendationSociale.ipynb".
The first task is:
u.data: Toutes les données de jugement au format "user id |  lm id | rating | timestamp",
avec les champs séparés par des tabulations

u.links: Les liens entre utilisateur du réseau.
Chaque ligne concerne liens sortants d'un utilisateur.
Une ligne commence par le numéro d'un utilisateur, suivi par tous ses successeurs dans le réseau, séparés par des tabulation.
Can you write Python code for it?

r_max = 5
def f(x):
    return (x-1)/(r_max-1)

def readData(name):
    a = np.array([i[:-2].split('\t')[:3] for i in open("recodata/"+name)])
    # mettre les notes a l'echelle
    a[:,2] = [f(float(i)) for i in a[:,2]]
    return a

def readLinks(name):
    ret = []
    for i in open("recodata/"+name):
        t = i[:-2].split('\t')
        a = t[0]
        for j in t[1:]:
            ret.append([a,j,1])
    return np.array(ret)

u_data = readData("u.data")
u1_train = readData("u1.train")
u1_test = readData("u1.test")
u2_train = readData("u2.train")
u2_test = readData("u2.test")
u3_train = readData("u3.train")
u3_test = readData("u3.test")
u4_train = readData("u4.train")
u4_test = readData("u4.test")
u5_train = readData("u5.train")
u5_test = readData("u5.test")

u_links = readLinks("u.links")

print(u_data[0])
print(u1_train[0])
print(u1_test[0])
print(u5_train[0])
print(u5_test[0])
print(u_links[0])

tmp=[]
for i in u_links:
    tmp.append(i[0])

users=u_data[:,0]
movies=u_data[:,1]

dicu={}
for i in users:
    dicu[i]=0
dicm={}
for i in movies:
    dicm[i]=0

len(dicu.keys())

len(dicm.keys())

rawMatrix = np.zeros((len(dicu.keys()),len(dicm.keys())))
for l in u_data:
    rawMatrix[int(l[0])-1][int(l[1])-1] = l[2]

rawMatrix[:7,:7]


I am working on the file "FDMS/TME8/TME8_Diffusion.ipynb".
The first task is:
Creation des liste de successeurs
Can you write Python code for it?

def getSuccsOfTab(tab):
    succ=[]
    for i in tab:
        succ_i=[]
        for j in tab:
            if(i[1]<j[1]):
                succ_i.append(j[0])
        succ.append([i[0],succ_i])
    return succ

def unique(liste):
    seen=set()
    seen_add =seen.add
    return [x for x in liste if not(x in seen or seen_add(x))]

succs_train = []
for line in data_train:
    succs_train.append(getSuccsOfTab(line))

succs_test = []
for line in data_test:
    succs_test.append(getSuccsOfTab(line))


I am working on the file "FDMS/TME8/TME8_Diffusion.ipynb".
The first task is:
Création des graphes
Can you write Python code for it?

#optimisation au niveau du temps possible
def getGraph(succs):
    graph={}
    for h in succs:
        for i in h:
            try:
                for j in i[1]:
                    graph[i[0]].append(j)
                graph[i[0]]=unique(graph[i[0]])
                graph[i[0]].sort()
            except KeyError:
                graph[i[0]]=i[1]
    return graph

graph_train = getGraph(succs_train)
graph_test = getGraph(succs_test)


I am working on the file "FDMS/TME8/TME8_Diffusion.ipynb".
The first task is:
Fonction d'apprentissage
Can you write Python code for it?

#ne fonctionne pas
def fitModele(data_train,nbIt=1,eps=1e-1):
    succs_train = []
    for line in data_train:
        succs_train.append(getSuccsOfTab(line))
    graph_train = getGraph(succs_train)

    #init
    graph_cur = getGraphWeightRandom(graph_train)

    #it=0
    #ep=100000
    #while((it<nbIt) & (ep>eps)):
        #probas_chapiteau=[getProbaOfList(getListPrec(d),graph_cur) for d in data_train]

    #loop for : utilisateur u vers v
    for m,u in enumerate(graph_cur):
        if((m % 10)==0):
            print(u)
        for v in graph_cur[u]:
            s=0
            nbUV=0
            nbnotUV=0
            for idx,d in enumerate(data_train[:3]):
                probas_chapiteau = getProbaOfList(getListPrec(d),graph_cur)
                if(existLinkUV(d,u,v)):
                    nbUV+=1
                    s+=graph_cur[u][v]*1.0/probas_chapiteau[v]
                else:
                    nbnotUV+=1
            graph_cur[u][v]= s*1.0/(nbUV+nbnotUV) # met a jour les poids

    return graph_cur

#pc = fitModele(data_train)

#devrait fonctionner
def fitModele2(data_train,nbIt=2,eps=1e-1):
    succs_train = []
    for line in data_train:
        succs_train.append(getSuccsOfTab(line))
    graph_train = getGraph(succs_train)

    #init
    theta_cur = getGraphWeightRandom(graph_train)

    for it in range(nbIt):
        dicoAS = constructAlldicosSucc(data_train)
        dicoAP = constructAlldicosPrec(data_train)
        sommeDico={}
        compteurDuvplus={}
        compteurDuvmoins={}
        for u in theta_cur:
            sommeDico[u]={}
            compteurDuvplus[u]={}
            compteurDuvmoins[u]={}
            for v in theta_cur[u]:
                sommeDico[u][v]=0.0
                compteurDuvplus[u][v]=0.0
                compteurDuvmoins[u][v]=0.0
        probas=[]
        for i in dicoAP:
            probas.append(getProbaOfDicoPrec(i,theta_cur))

        #A OPTIMISER
        for p in probas:
           ...


I am working on the file "FDMS/TME8/TME8_Diffusion.ipynb".
The first task is:
Inférence
Il s'agit à partir du graphe et d'une liste des premiers utilisateurs inféctés, de déduire quels seront les suivants.
Can you write Python code for it?

#ne fonctionne pas
def inference(listeT1, graph):
    listeUP=[]
    listeU=[]
    listeTmp=[]

    for u in listeT1:
        for v in graph[u]:
            if (np.random.rand()<graph[u][v]):
                if(v not in listeU):
                    listeU.append(v)
                    listeUP.append((v,graph[u][v]))
                    listeTmp.append(v)

    t=1
    while((len(listeTmp)>0) &(t<15)):
        listeTmp2=[]
        for u in listeTmp:
            for v in graph[u]:
                if (np.random.rand()<graph[u][v]):
                    if(v not in listeU):
                        listeU.append(v)
                        listeUP.append((v,graph[u][v]))
                        listeTmp2.append(v)
        listeTmp=copy.deepcopy(listeTmp2)
        t+=1

    listeUP.sort(key=lambda elem: elem[1])
    return  [i[0] for i in listeUP]

t1=[68.0]
print(inference(t1,pc))

#devrait fonctionner
def inference2(listeT1, graph,iteration):

    dicoProbaInfection={}
    for u in graph:
        dicoProbaInfection[u]=0

    for it in range(iteration):
        listeFinalInfected={}
        listeTM1={}

        #on affecte les personnes au temps T2
        for u in listeT1.keys():
            for v in graph[u]:
                if(np.random.rand()<graph[u][v]):
                    try:
                        listeT1[v]
                    except KeyError:
                        try:
                            listeFinalInfected[v]
                            #listeTM1[v]
                        except KeyError:
                            listeFinalInfected[v]=1
                            listeTM1[v]=1

        #on affecte les gens aux temps t3 et suivants
        t=1
        while(len(listeTM1.keys())>0 & t<20):
            listeTM1tmp={}
            for u in listeTM1.keys():
                for v in graph[u]:
                    try:
                        listeT1[v]
                    except KeyError:
                        try:
                    ...


I am working on the file "FDMS/TME8/TME8_Diffusion.ipynb".
The first task is:
La mean average precision
Elle permet d'évaluer notre modèle de diffusion en calculant la précision moyenne entre la liste de diffusion réelle et celle prédite par notre algorithme.
Can you write Python code for it?

def MAP(graph,data_test):
    s=0
    for d in data_test:
        listeD=[i[0] for i in d if (i[1]>1)]
        listeT1=[i[0] for i in d if (i[1]==1)]
        listeU=inference(listeT1,graph)
        s2=0
        for i,u in enumerate(listeU):
            intersection = [val for val in listeD if val in listeU[:i]]
            s2+=len(intersection)*1.0/(i+1)
        try:
            s+=s2/len(listeU)
        except ZeroDivisionError:
            pass
    return s/len(data_test)
MAP(pc,data_test)

MAP2(pc,data_test)

#devrait fonctionner
def MAP2(graph,data_test,it=2):
    s=0
    for d in tqdm(data_test):#boucle dataset

        listeD=[i[0] for i in d if (i[1]>1)]
        listeT1={}
        for i in d:
            if (i[1]==1):
                 listeT1[i[0]]=1
        dicoU=inference2(listeT1,graph,it)
        listeU = [i[0] for i in sorted(dicoU.items(), key=operator.itemgetter(1))[::-1]]

        s2=0
        for i,u in enumerate(listeU):
            intersection = [val for val in listeD if val in listeU[:i]]
            s2+=len(intersection)*1.0/(i+1)
        try:
            s+=s2/len(listeU)
        except ZeroDivisionError:
            pass
    return s/len(data_test)

MAP2(pc,data_test)


I am working on the file "FDMS/TME9/tme9.ipynb".
The first task is:
Le score des algorithmes "Static" sont semblables et représentent une bonne solution étant donné que leurs scores est proche de l'optimal.
Can you write Python code for it?

#Affichage des scores
plt.plot(traceRandom, label = "Random")
plt.plot(traceStaticBest, label = "StaticBest")
plt.plot(traceStaticBestCum, label = "StaticBestCum")
plt.plot(traceOptimal, label = "Optimal")
plt.xlabel("Visites")
plt.ylabel(u"Score cumulé")
plt.title(u"Scores cumulés")
plt.legend()
plt.show()


I am working on the file "FDMS/TME9/tme9.ipynb".
The first task is:
Lecture du fichier
Chaque ligne contient:
 numéro article -- représentation de l'article 5 dimensions -- taux de clics sur les publicité de 10 annonceurs
Can you write Python code for it?

f = open("./CTR.txt")
ctr=[]
for i in f.readlines():
    line = i.split(':')
    ctr.append([int(line[0]),[float(i) for i in line[1].split(';')],[float(i) for i in line[2][:-2].split(';')]])

#Affichage des données pour l'article 0
ctr[0]


I am working on the file "FDMS/TME9/tme9.ipynb".
The first task is:
Nous obtenons un score cumulé optimal de 1532.00 il correspond au fait qu'a chaque visite nous ayons choisit la publicité avec le plus fort taux de clic.
Les scores des autres baseline seront bien entendu en dessous, le but est de se rapprocher de ce score.
Can you write Python code for it?

#Score random
score,annonceur,traceRandom = baseRandom(ctr)
score.sum()


I am working on the file "FDMS/TME9/tme9.ipynb".
The first task is:
On remarque que le score de lin_UCB est meilleur que celui de UCB, et que ce score se rapproche de celui de static best.
Conclusion
Une simple baseline comme static best permet d'obtenir un bon score, malgré cela pour une prise en compte du contexte, il est nécessaire de recourir à un algorithme tel que linUCB. En revanche nous pouvons noter que UCB est aussi une bonne méthode bien que moins bonne que la static best.

Brouillon
premiere version de lin ucb ..
Can you write Python code for it?

def linUCB_V1(ctr,sigma):
    #nb annonceurs
    nb_a = len(ctr[0][2])

    #nb features par objet = d
    d = len(ctr[0][1])

    alpha = 1+np.sqrt(np.log(2/sigma)/2)
    A = np.array([np.identity(d) for i in np.arange(nb_a)])
    b = np.zeros((nb_a,d))

    Atmp = np.array([np.identity(d) for i in np.arange(nb_a)])
    btmp = np.zeros((nb_a,d))

    theta = np.zeros((nb_a,d))
    p=np.zeros((nb_a,d,d))

    for t,annonce in enumerate(ctr,1):

        objet=np.array(annonce[1])
        objetT=objet.T
        for idx,c in enumerate(annonce[2]):
            Atmp[idx] = objetT*objet+np.identity(d)
            btmp[idx] = objet*c
            theta[idx] = np.linalg.inv(Atmp[idx]).dot(btmp[idx])
            p[idx] = theta[idx].T*objet + alpha*np.sqrt(objetT*np.linalg.inv(Atmp[idx])*objet)
        argmax = np.argmax(p)
        A[argmax] = A[argmax] + objet * objetT
        b[argmax] = np.array(annonce[2][argmax])*objet
    return A,b,theta

A,b,theta= linUCB_V1(ctr,0.5)

score,annonceur,testTrace = predict(theta,ctr)
print(testTrace[-1])

plt.plot(traceRandom, label = "Random")
plt.plot(traceStaticBest, label = "StaticBest")
plt.plot(traceStaticBestCum, label = "StaticBestCum")
plt.plot(traceOptimal, label = "Optimal")
plt.plot(UCB, label = "UCB")
plt.plot(testTrace, label = "lin_UCB_V1")
plt.legend()
plt.show()


I am working on the file "FDMS/TME9/tme9.ipynb".
The first task is:
UCB
Developpement de l'article UCB.
La stratégie centrale est la suivante:
$π_t = arg max_i B_{t,T_i (t−1)}(i)$
avec
$ B_{t,s}(i) = \mu_{i,s} + \sqrt{\dfrac{2log t}{s}}$
Le principe est de choisir un annonceur ayant le plus de chance d'etre l'annonceur avec le plus fort taux de clics en fonction des annonces précédentes tout en prenant en compte le fait que l'on doive tester les autres annonceurs (d'où l'ajout du $+ \sqrt{\dfrac{2log t}{s}}$).
Avec cette méthode on évite de rester sur un bras sous-optimal.
Can you write Python code for it?

def UCB(ctr):
    score=[]
    annonceur=[]
    trace=[]
    nbAds=len(ctr[0][2])

    #poids de chaque ads
    weights= np.zeros(nbAds)

    #tableau compte nombre de fois qu'une ads a été choisie
    arrayS=np.zeros(nbAds)

    #initialisation 10 première lignes 10 ads
    t=0
    for it in np.arange(nbAds):
        t+=1
        arrayS[it]+=1
        weights[it]+=ctr[it][2][it]

    #algo UCB
    for a in ctr:
        #calcul du argmax de l'itération précédente
        mu=np.zeros(nbAds)
        for it in np.arange(nbAds):
            mu[it]+=weights[it]/arrayS[it]+np.square((2*np.log(t))/arrayS[it])
        argmax = np.argmax(np.array(mu))

        #maj des valeurs uniquement celle de l'argmax !!
        t+=1
        arrayS[argmax]+=1
        weights[argmax]+=a[2][argmax]

        score.append(a[2][argmax])
        annonceur.append(argmax)
        trace.append(np.array(score).sum())
    return np.array(score),np.array(annonceur),trace

#calcul des scores pour UCB
score,annonceur,UCB = UCB(ctr)
score.sum()

plt.plot(traceRandom, label = "Random")
plt.plot(traceStaticBest, label = "StaticBest")
plt.plot(traceStaticBestCum, label = "StaticBestCum")
plt.plot(traceOptimal, label = "Optimal")
plt.plot(UCB, label = "UCB")
plt.xlabel("Visites")
plt.ylabel(u"Score cumulé")
plt.title(u"Scores cumulés")
plt.legend()
plt.show()


I am working on the file "FDMS/TME_Dataiku/kaggle_whats_cooking/Model_V7.ipynb".
The first task is:
Few words about the dataset
Predictions is made in the USA corn growing states (mainly Iowa, Illinois, Indiana) during the season with the highest rainfall (as illustrated by Iowa for the april to august months)
The Kaggle page indicate that the dataset have been shuffled, so working on a subset seems acceptable
The test set is not a extracted from the same data as the training set however, which make the evaluation trickier
Load the dataset
Can you write Python code for it?

%%time
#filename = "data/train.csv"
filename = "data/train.json"
#filename = "data/reduced_train_1000000.csv"
raw = pd.read_json(filename)
#raw = raw.set_index('Id')

traindf = raw
traindf['ingredients_clean_string'] = [' , '.join(z).strip() for z in traindf['ingredients']]
traindf['ingredients_string'] = [' '.join([WordNetLemmatizer().lemmatize(re.sub('[^A-Za-z]', ' ', line)) for line in lists]).strip() for lists in traindf['ingredients']]

#
testdf = pd.read_json("data/train.json")
testdf['ingredients_clean_string'] = [' , '.join(z).strip() for z in testdf['ingredients']]
testdf['ingredients_string'] = [' '.join([WordNetLemmatizer().lemmatize(re.sub('[^A-Za-z]', ' ', line)) for line in lists]).strip() for lists in testdf['ingredients']]

corpustr = traindf['ingredients_string']
vectorizertr = TfidfVectorizer(stop_words='english',
                             ngram_range = ( 1 , 1 ),analyzer="word",
                             max_df = .57 , binary=False , token_pattern=r'\w+' , sublinear_tf=False)

tfidftr=vectorizertr.fit_transform(corpustr).todense()

#
corpusts = testdf['ingredients_string']

vectorizerts = TfidfVectorizer(stop_words='english')

#
tfidfts=vectorizertr.transform(corpusts)

predictors_tr = tfidftr

targets_tr = traindf['cuisine']

#
predictors_ts = tfidfts

raw.head()

tfidftr[0]

targets_tr[:5]

labels = pd.get_dummies(targets_tr)

labels[:5]

classifier = LogisticRegression()
classifier=classifier.fit(predictors_tr,targets_tr)

predictions=classifier.predict(predictors_ts)
testdf['cuisine'] = predictions
testdf = testdf.sort('id' , ascending=True)

#testdf[['id' , 'ingredients_clean_string' , 'cuisine' ]].to_csv("submission.csv")

%%time
classifier = GradientBoostingRegressor()
classifier=classifier.fit(predictors_tr,targets_tr)

predictions=classifier.predict(predictors_ts)
testdf['cuisine'] = predictions
testdf = testdf.sort('id' , ascending=True)

testdf[['id' , 'ingredients_clean_string' , 'cuisine' ]].to_csv("submission.csv")

raw.head()

raw['ingredients'][0]


I am working on the file "FDMS/TME_Dataiku/kaggle_whats_cooking/Model_V7.ipynb".
The first task is:
Notes
We tried different regressor model, like GBR, SVM, MLP, Random Forest and KNN as recommanded by the winning team of the Kaggle on taxi trajectories. So far GBR seems to be the best, slightly better than the RF.
The new features we exctracted only made a small impact on predictions but still improved them consistently.
We tried to use a LSTM to take advantage of the sequential structure of the data but it didn't work too well, probably because there is not enought data (13M lines divided per the average length of sequences (15), less the 30% of fully empty data)
Can you write Python code for it?

# from __future__ import exam_success
from __future__ import absolute_import
from __future__ import print_function

# Standard imports
%matplotlib inline
import os
import sklearn
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np
import random
import pandas as pd
import scipy.stats as stats

# Sk cheats
from sklearn.cross_validation import cross_val_score
from sklearn import grid_search
from sklearn.ensemble import RandomForestRegressor
from sklearn.ensemble import ExtraTreesRegressor
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.neighbors import KNeighborsRegressor
from sklearn.svm import SVR
#from sklearn.preprocessing import Imputer   # get rid of nan
from sklearn.decomposition import NMF        # to add features based on the latent representation
from sklearn.decomposition import ProjectedGradientNMF

# Faster gradient boosting
import xgboost as xgb

# For neural networks models
from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Activation
from keras.optimizers import SGD, RMSprop


I am working on the file "FEEG6016 Simulation and Modelling/01-Monte-Carlo-Lab-1.ipynb".
The first task is:
Accuracy
To check the accuracy of the method, let's apply this to calculate $\pi$.
The area of a circle of radius $2$ is $4\pi$, so the area of the quarter circle in $x, y \in [0, 2]$ is just $\pi$:
$$
\begin{equation}
  \pi = \int_0^2 \sqrt{4 - x^2} \, dx.
\end{equation}
$$
Check the convergence of the Monte Carlo integration with $N$. (I suggest using $N = 100 \times 2^i$ for $i = 0, \dots, 19$; you should find the error scales roughly as $N^{-1/2}$)
Can you write Python code for it?

def g(x):
    return (4-x**2)**0.5

Nlist = []
PiList = []

import matplotlib.pyplot as mplot

for i in range(0,20):
    #print(i, mc_integrate(g, (0,2), (0,8), N = 100*2**i))
    Nlist.append(100*2**i)
    PiList.append(mc_integrate(g, (0,2), (0,8), N = 100*2**i))

NlogList = []
abserrlogList = []

for i in range(len(Nlist)):
    NlogList.append(numpy.log(Nlist[i]))
    absError = abs(numpy.pi-PiList[i])
    abserrlogList.append(numpy.log(absError))

(p1, p0) = numpy.polyfit(NlogList, abserrlogList, 1) # y = p1*x + p0

print(p1, p0)

def fittedLine(x):
    return p1*x+p0

def create_plot_data(f, xmin, xmax, n):
    """Takes xmin, a min value of x, and xmax, a max value of x\n
    and creates a sequence xs, with values between xmin and xmax\n
    inclusive, with n elements. It also creates a sequence ys\n
    with the values of the function f at the corresponding\n
    values in the sequence xs. It then returns a tuple containing\n
    these 2 seqeuences i.e. (xs, ys)"""
    xsArray = numpy.linspace(start=xmin, stop=xmax, num=n)
    ysArray = numpy.zeros(n)
    i = 0
    for element in numpy.nditer(xsArray):
        #       print element
        ysArray[i] = f(element)
        i += 1
    return (list(xsArray), list(ysArray))

(xFit, yFit) = create_plot_data(fittedLine, 4, 19, 5)

#print(xyFit)
#print(NlogList, abserrlogList)

mplot.clf()
plot1 = mplot.plot(NlogList, abserrlogList, 'bo', label='')
plot2 = mplot.plot(xFit, yFit, '--r', label='Best fit Line: y={}*x+{}'.format(p1,p0))
mplot.xlabel("log(N)")
mplot.ylabel("log(|Error|)")
mplot.legend(prop={'size':12})
mplot.show()


I am working on the file "FEEG6016 Simulation and Modelling/01-Monte-Carlo-Lab-1.ipynb".
The first task is:
Accuracy
To check the accuracy of the method, let's apply this to calculate $\pi$.
The area of a circle of radius $2$ is $4\pi$, so the area of the quarter circle in $x, y \in [0, 2]$ is just $\pi$:
$$
\begin{equation}
  \pi = \int_0^2 \sqrt{4 - x^2} \, dx.
\end{equation}
$$
Check the convergence of the Monte Carlo integration with $N$. (I suggest using $N = 100 \times 2^i$ for $i = 0, \dots, 19$; you should find the error scales roughly as $N^{-1/2}$)
Mean Value Method
Monte Carlo integration is pretty inaccurate, as seen above: it converges slowly, and has poor accuracy at all $N$. An alternative is the mean value method, where we note that by definition the average value of $f$ over the interval $[a, b]$ is precisely the integral multiplied by the width of the interval.
Hence we can just choose our $N$ random points in $x$ as above, but now just compute
$$
\begin{equation}
  I \simeq \frac{b-a}{N} \sum_{i=1}^N f(x_i).
\end{equation}
$$
Can you write Python code for it?

def mv_integrate(f, domain_x, N = 10000):
    """
    Mean value Monte Carlo integration: to be completed
    """
    import numpy.random
    return I


I am working on the file "FEEG6016 Simulation and Modelling/01-Monte-Carlo-Lab-1.ipynb".
The first task is:
Integration
If we have an ugly function, say
$$
\begin{equation}
  f(x) = \sin^2 \left(\frac{1}{x (2-x)}\right),
\end{equation}
$$
then it can be very difficult to integrate. To see this, just do a quick plot.
Can you write Python code for it?

%matplotlib inline
import numpy
from matplotlib import pyplot
from matplotlib import rcParams
rcParams['font.family'] = 'serif'
rcParams['font.size'] = 16
rcParams['figure.figsize'] = (12,6)

from __future__ import division

def f(x):
    return numpy.sin(1.0/(x*(2.0-x)))**2

x = numpy.linspace(0.0, 2.0, 10000)
pyplot.plot(x, f(x))
pyplot.xlabel(r"$x$")
pyplot.ylabel(r"$\sin^2([x(x-2)]^{-1})$");


I am working on the file "FEEG6016 Simulation and Modelling/01-Monte-Carlo-Lab-1.ipynb".
The first task is:
Let's look at the accuracy of this method again applied to computing $\pi$.
Can you write Python code for it?

def g(x):
    return (4-x**2)**0.5

mv_integrate(g, (0, 2), 10000000)

Nlist2 = []
PiList2 = []

import matplotlib.pyplot as mplot

for i in range(0,20):
    #print(i, mv_integrate(g, (0,2), (0,8), N = 100*2**i))
    Nlist2.append(100*2**i)
    PiList2.append(mv_integrate(g, (0,2), N = 100*2**i))

NlogList2 = []
abserrlogList2 = []

for i in range(len(Nlist2)):
    NlogList2.append(numpy.log(Nlist2[i]))
    absError2 = abs(numpy.pi-PiList2[i])
    abserrlogList2.append(numpy.log(absError2))

(p1_2, p0_2) = numpy.polyfit(NlogList2, abserrlogList2, 1) # y = p1*x + p0

print(p1_2, p0_2)

def fittedLine_2(x):
    return p1_2*x+p0_2

(xFit2, yFit2) = create_plot_data(fittedLine_2, 4, 19, 5)

#print(xyFit)
#print(NlogList, abserrlogList)

mplot.clf()
plot1_2 = mplot.plot(NlogList2, abserrlogList2, 'bo', label='')
plot2_2 = mplot.plot(xFit2, yFit2, '--r', label='Best fit Line: y={}*x+{}'.format(p1_2,p0_2))
mplot.xlabel("log(N)")
mplot.ylabel("log(|Error|)")
mplot.legend(prop={'size':12})
mplot.show()


I am working on the file "FEEG6016 Simulation and Modelling/01-Monte-Carlo-Lab-1.ipynb".
The first task is:
Let's look at the accuracy of this method again applied to computing $\pi$.
The convergence rate is the same (only roughly, typically), but the Mean Value method is expected to be better in terms of its absolute error.
Dimensionality
Compared to standard integration methods (Gauss quadrature, Simpson's rule, etc) the convergence rate for Monte Carlo methods is very slow. However, there is one crucial advantage: as you change dimension, the amount of calculation required is unchanged, whereas for standard methods it grows geometrically with the dimension.
Try to compute the volume of an $n$-dimensional unit hypersphere, which is the object in $\mathbb{R}^n$ such that
$$
\begin{equation}
  \sum_{i=1}^n x_i^2 \le 1.
\end{equation}
$$
The volume of the hypersphere can be found in closed form, but can rapidly be computed using the Monte Carlo method above, by counting the $k$ points that randomly fall within the hypersphere and using the standard formula $I \simeq V k / N$.
Can you write Python code for it?

def mc_integrate_multid(f, domain, N = 10000):
    """
    Monte Carlo integration in arbitrary dimensions (read from the size of the domain): to be completed
    """

    return I

from scipy import special

def volume_hypersphere(ndim=3):
    return numpy.pi**(float(ndim)/2.0) / special.gamma(float(ndim)/2.0 + 1.0)


I am working on the file "FEEG6016 Simulation and Modelling/01-Monte-Carlo-Lab-1.ipynb".
The first task is:
Mean Value Method
Monte Carlo integration is pretty inaccurate, as seen above: it converges slowly, and has poor accuracy at all $N$. An alternative is the mean value method, where we note that by definition the average value of $f$ over the interval $[a, b]$ is precisely the integral multiplied by the width of the interval.
Hence we can just choose our $N$ random points in $x$ as above, but now just compute
$$
\begin{equation}
  I \simeq \frac{b-a}{N} \sum_{i=1}^N f(x_i).
\end{equation}
$$
Can you write Python code for it?

def mv_integrate(f, domain_x, N = 10000):
    """
    Mean value Monte Carlo integration: to be completed
    """
    import numpy.random
    a = domain_x[0]
    b = domain_x[1]
    domain_xrange = (domain_x[1]-domain_x[0])
    xArray = numpy.random.random(N) * (domain_xrange + domain_x[0])
    funcArray = f(xArray)
    sum_ = sum(funcArray)
    I = ((b-a)/N)*sum_
    return I


I am working on the file "FEEG6016 Simulation and Modelling/01-Monte-Carlo-Lab-1.ipynb".
The first task is:
So if we scattered (using a uniform random distribution) a large number of points within this box, the fraction of them falling below the curve is approximately the integral we want to compute, divided by the area of the box:
$$
\begin{equation}
  I = \int_a^b f(x) \, dx \quad \implies \quad I \simeq \frac{k A}{N}
\end{equation}
$$
where $N$ is the total number of points considered, $k$ is the number falling below the curve, and $A$ is the area of the box. We can choose the box, but we need $y \in [\min_{x \in [a, b]} (f(x)), \max_{x \in [a, b]} (f(x))] = [c, d]$, giving $A = (d-c)(b-a)$.
So let's apply this technique to the function above, where the box in $y$ is $[0,1]$.
Can you write Python code for it?

def mc_integrate(f, domain_x, domain_y, N = 10000):
    """
    Monte Carlo integration function: to be completed. Result, for the given f, should be around 1.46.
    """
    import numpy.random

    return I


def mc_integrate(f, domain_x, domain_y, N = 10000):
    """
    Monte Carlo integration function: to be completed. Result, for the given f, should be around 1.46.
    """
    import numpy.random
    domain_xrange = (domain_x[1]-domain_x[0])
    domain_yrange = (domain_y[1]-domain_y[0])
    randomXArray = numpy.random.random(N) * (domain_xrange + domain_x[0])
    randomYArray = numpy.random.random(N) * (domain_yrange + domain_y[0])
    funcValuesArray = f(randomXArray)
    #funcValuesArray = numpy.array([f(x) for x in randomXArray])
    #print(f(2))
    #print(randomXArray, randomYArray, funcValuesArray)
    boolArray = randomYArray <= funcValuesArray
    Hits = sum(boolArray)
    #print(randomXArray, randomYArray, funcValuesArray, boolArray, Hits)
    I=(Hits*domain_xrange*domain_yrange)/N
    return I

mc_integrate(lambda x: x**3, (0,2), (0,8), 1000000)

mc_integrate(f, (0,2), (0,1), 1000)

%timeit mc_integrate(f, [0,2], [0,2], 100000)


I am working on the file "FEEG6016 Simulation and Modelling/01-Monte-Carlo-Lab-1.ipynb".
The first task is:
The convergence rate is the same (only roughly, typically), but the Mean Value method is expected to be better in terms of its absolute error.
Dimensionality
Compared to standard integration methods (Gauss quadrature, Simpson's rule, etc) the convergence rate for Monte Carlo methods is very slow. However, there is one crucial advantage: as you change dimension, the amount of calculation required is unchanged, whereas for standard methods it grows geometrically with the dimension.
Try to compute the volume of an $n$-dimensional unit hypersphere, which is the object in $\mathbb{R}^n$ such that
$$
\begin{equation}
  \sum_{i=1}^n x_i^2 \le 1.
\end{equation}
$$
The volume of the hypersphere can be found in closed form, but can rapidly be computed using the Monte Carlo method above, by counting the $k$ points that randomly fall within the hypersphere and using the standard formula $I \simeq V k / N$.
Can you write Python code for it?

def mc_integrate_multid(f, domain, N = 10000):
    """
    Monte Carlo integration in arbitrary dimensions (read from the size of the domain): to be completed
    """
    import numpy.random
    dimensionality = len(domain) #dimensionality of the hypersphere - determined by the length of the domain list

    #initialisation of lists/arrays
    domain_range = numpy.zeros(shape=(dimensionality, 1))
    funcValuesArray = numpy.zeros(N)

    for i in range(0, dimensionality):
        domain_range[i] = (domain[i][1]-domain[i][0])  #calculates domain range for each dimension

    for i in range(0, N):
        XYZ = []
        for j in range(0, dimensionality):
            XYZ.append(float(numpy.random.rand() * (domain_range[j] + domain[j][0])))
        #print XYZ
        funcValuesArray[i] = f(numpy.array(XYZ)) #creates an N element list of function values at random points (x, y, z ...)
        #print(funcValuesArray[i])
    #print(funcValuesArray)

    zeroes = numpy.zeros(N)
    boolArray = funcValuesArray <= zeroes  #compares the array of function values with an of array of zeroes
    Hits = sum(boolArray) #sums the number of hits (i.e. points inside or on the surface of the hypersphere)
    hyperVolume = 1 #initialises hypervolume
    for i in range(0, dimensionality):
        hyperVolume *= domain_range[i] #calculates hypervolume of hypercuboid domain

    I = (Hits*hyperVolume)/N #calculates integral's value
    return float(I)#*2**dimensionality

def MakeHyperSphereOfRadiusR(R):
    def HyperSphere(ArrayOfDimensionalVariables):
        """defines a hypersphere of radius R with variables in the Array e.g. for a 3d
        sphere with radius R pass an array [x, y, z] and the radius R. The function returns a value
        which is <= 0 if the value is inside the function and >0 if it's outside the function"""
        #print(ArrayOfDimensionalVariables)
        return sum(ArrayOfDimensionalVariables**2)-R**2  #e.g. for 3d sphere for points inside or on the surface of the sphere...


I am working on the file "FEEG6016 Simulation and Modelling/01-Monte-Carlo-Lab-1.ipynb".
The first task is:
This is a plot of my calculated values of the Integral
Can you write Python code for it?

from scipy import special

def volume_hypersphere(ndim=3):
    return numpy.pi**(float(ndim)/2.0) / special.gamma(float(ndim)/2.0 + 1.0)*1**ndim

volume_hypersphere(ndim=5)

volList = []
for i in range(0,10):
    volList.append(volume_hypersphere(ndim=i+1))

mplot.clf()
plot1_3 = mplot.plot(dimensionList, integralList, 'bo', label='')
plot2_2 = mplot.plot(dimensionList, volList, 'ro', label='')
mplot.xlabel("dimensionality")
mplot.ylabel("Integral value")
#mplot.legend(prop={'size':12})
mplot.show()


I am working on the file "FEEG6016 Simulation and Modelling/01-Monte-Carlo-Lab-1.ipynb".
The first task is:
This is a plot of my calculated values of the integral in blue and the actual values in red
Can you write Python code for it?

print(volList, "\n\n" ,integralList)

volArray = numpy.array(volList)
integralArray = numpy.array(integralList)

errorArray = abs(volArray-integralArray)

mplot.clf()
plot1_3 = mplot.plot(dimensionList, errorArray, 'bo', label='')
#plot2_2 = mplot.plot(xFit2, yFit2, '--r', label='Best fit Line: y={}*x+{}'.format(p1_2,p0_2))
mplot.xlabel("dimensionality")
mplot.ylabel("Absolute Error in Integral value")
#mplot.legend(prop={'size':12})
mplot.show()


I am working on the file "FEEG6016 Simulation and Modelling/01-Monte-Carlo-Lab-1.ipynb".
The first task is:
This is a plot of my calculated values of the integral in blue and the actual values in red. This plot however uses the domain -2 to 2 in each dimension but calculated for the same unit hypersphere, the values diverge at higher dimensionality, why?
As dimensionality increases so does the effective hypervolume of the domain by a factor $2^{dimensionality}$, this means that the probability of a random point being inside the hypersphere is proportional to $2^{-dimensionality}$ so as dimensionality is increased the points are much less likely to fall inside the hypersphere and so you need a larger N, number of random points, for the answer to converge.
Now let us repeat this across multiple dimensions.
The errors clearly vary over a range, but the convergence remains roughly as $N^{-1/2}$ independent of the dimension; using other techniques such as Gauss quadrature would see the points required scaling geometrically with the dimension.
Importance sampling
Consider the integral (which arises, for example, in the theory of Fermi gases)
$$
\begin{equation}
  I = \int_0^1 \frac{x^{-1/2}}{e^x + 1} \, dx.
\end{equation}
$$
This has a finite value, but the integrand diverges as $x \to 0$. This may cause a problem for Monte Carlo integration when a single value may give a spuriously large contribution to the sum.
We can get around this by changing the points at which the integrand is sampled. Choose a weighting function $w(x)$. Then a weighted average of any function $g(x)$ can be
$$
\begin{equation}
  <g>_w = \frac{\int_a^b w(x) g(x) \, dx}{\int_a^b w(x) \, dx}.
\end{equation}
$$
As our integral is
$$
\begin{equation}
  I = \int_a^b f(x) \, dx
\end{equation}
$$
we can, by setting $g(x) = f(x) / w(x)$ get
$$
\begin{equation}
  I = \int_a^b f(x) \, dx = \left< \frac{f(x)}{w(x)} \right>_w \int_a^b w(x) \, dx.
\end{equation}
$$
This gives
$$
\begin{equation}
  I \simeq \frac{1}{N} \sum_{i=1}^N \frac{f(x_i)}{w(x_i)} \int_a^b w(x) \, dx,
\end{equation}
$$
where the points $x_i$ are now chosen from a non-uniform probability...
Can you write Python code for it?

def ic_integrate(f, domain_x, N = 10000):
    """
    Mean value Monte Carlo integration: to be completed
    Using w(x)=x**(-0.5)
    """
    import numpy.random
    def w(x):
        return x**(-0.5)

    a = domain_x[0]
    b = domain_x[1]
    lower_limit = (a**0.5)
    upper_limit = (b**0.5)
    domain_yrange = upper_limit-lower_limit
    yArray = numpy.random.random(N) * (domain_yrange + lower_limit) #y = unformly distributed random no.s
    xArray = yArray**(2)  #x = random numbers distributed with the probability distribution p(x)=x**(-0.5)/2
    #print(xArray)
    funcOverWeightingArray = f(xArray)/w(xArray)
    sum_ = sum(funcOverWeightingArray)
    #print(sum_, N, domain_x[0], domain_x[1])
    I = (sum_/N)*2*(b**0.5-a**0.5)
    return I

def f_fermi(x):
    return x**(-0.5)/(numpy.exp(x)+1)

ic_integrate(f_fermi, [0,1], N=1000000)


I am working on the file "FEEG6016 Simulation and Modelling/01-Monte-Carlo-Lab-1.ipynb".
The first task is:
We see that as the function oscillates infinitely often, integrating this with standard methods is going to be very inaccurate.
However, we note that the function is bounded, so the integral (given by the shaded area below) must itself be bounded - less than the total area in the plot, which is $2$ in this case.
Can you write Python code for it?

pyplot.fill_between(x, f(x))
pyplot.xlabel(r"$x$")
pyplot.ylabel(r"$\sin^2([x(x-2)]^{-1})$");


I am working on the file "FEEG6016 Simulation and Modelling/02-Monte-Carlo_Lab-2.ipynb".
The first task is:
Efficiency
Note that the sum over all particles scales as $n^2$ where $n$ is the number of particles. As the number of steps the algorithm will need to take will also scale as $n$, this makes the number of calculations at least as bad as $n^3$. This is expensive; if you try the naive approach then you'll have difficulty using more than 50 particles in a moderate time.
Instead we can note that, at each stage, the algorithm will move only one particle. Therefore, if we store not just the locations of the particles but also their pairwise separations, at each step we will only have to modify a small number of the separations. So we can store $r^2_{ij} = \vec{r}{ij} \cdot \vec{r}{ij}$ only, for $j > i$, and when perturbing particle $k$ we only need to update the separations $r^2_{ik}$ for $i<k$ and $r^2_{kj}$ for $k<j$.
This should significantly reduce the number of calculations done in each step.
In addition, note that for reasonable behaviour the acceptance rate should be $\sim 40\%$. This depends on the fractional perturbation distance $\Delta$; values $\sim 0.4$ are reasonable when $\rho \sim 0.1$, but values $\sim 0.02$ are reasonable when $\rho \sim 0.9$.
Results

Check that the energy has converged to a "constant" state.
Plot a histogram of the energies to show that they follow the Boltzmann distribution.
Can you write Python code for it?

%matplotlib inline
import numpy
from scipy import constants
from matplotlib import pyplot
from mpl_toolkits.mplot3d.axes3d import Axes3D
from matplotlib import rcParams
rcParams['font.family'] = 'serif'
rcParams['font.size'] = 16
rcParams['figure.figsize'] = (12,6)


I am working on the file "FEEG6016 Simulation and Modelling/02-Monte-Carlo_Lab-2.ipynb".
The first task is:
Point to note: an electronic copy of the Frenkel and Smit book is available through the library. This lab is based on case study 1 in chapter 3.4 of that book.
Lennard-Jones fluids
When computing the interactions between lots of bodies (atoms, molecules, planets, etc) we can either use the true potential or force between them, or we can approximate it with some potential (or force) that is easier (and usually cheaper) to calculate. The parameters of the potential can then be set to approximate the "real" features we're interested in.
In computational chemistry, one such approximation is the Lennard-Jones potential. Given two bodies separated by a distance $r$, the potential generated by those two bodies is
\begin{equation}
  U(r) = 4 \varepsilon \left[ \left( \frac{\sigma}{r} \right)^{12} - \left( \frac{\sigma}{r} \right)^{6} \right].
\end{equation}
Here $\varepsilon$ and $\sigma$ are parameters. When there are more than two bodies the total potential is the sum over all pairwise potentials.
In principle this generates a potential between particles that are separated by huge distances. Instead it is typical to truncate the potential: to pick a cut-off distance so that any particles separated by more than that distance do not contribute, and to correct for those small contributions.
Here we use a Lennard-Jones potential inside a box size $[0,L]^3$ with a cut-off $r_c = L/2$, with parameters set so that
\begin{equation}
U = \begin{cases} 4 \left[ \frac{1}{r^{12}} - \frac{1}{r^6} \right] & r < r_c \ 0 & r > r_c. \end{cases}
\end{equation}
Include tail corrections (that is, additional energy and pressure terms resulting from the particles outside the cutoff radius) as
\begin{align}
  U^{\text{tail}} & = \frac{8 \pi \rho}{3} \left[ \frac{1}{3} \frac{1}{r_c^9} - \frac{1}{r_c^3} \right] \
  p^{\text{tail}} & = \frac{16 \pi \rho^2}{3} \left[ \frac{2}{3} \frac{1}{r_c^9} - \frac{1}{r_c^3} \right].
\end{align}
For each configuration we need to compute the pressure using
$$
\begin{equation}
  p = \frac{\rho}{\beta} +...
Can you write Python code for it?


I am working on the file "FEEG6016 Simulation and Modelling/03-Molecular-Dynamics-Lab-1.ipynb".
The first task is:
Basics of Molecular Dynamics
Can you write Python code for it?

%matplotlib inline
import numpy
from matplotlib import pyplot
from mpl_toolkits.mplot3d.axes3d import Axes3D
from matplotlib import rcParams
rcParams['font.family'] = 'serif'
rcParams['font.size'] = 16
rcParams['figure.figsize'] = (12,6)


I am working on the file "FEEG6016 Simulation and Modelling/04-Molecular-Dynamics-Lab-2.ipynb".
The first task is:
Energy minimization
In lab 1 we worked with the Lennard-Jones potential. Other simple potentials include the harmonic potential
$$
\begin{equation}
  V = \frac{1}{2} k \left( r - r_{\text{eq}} \right)^2
\end{equation}
$$
with harmonic constant $k$ and equilibrium position $r_{\text{eq}}$, the Kratzer potential
$$
\begin{equation}
  V = D_0 \left( \frac{r-r_{\text{eq}}}{r} \right)^2
\end{equation}
$$
with dissociation energy $D_0$, and the Morse potential
$$
\begin{equation}
 V = D_0 \left[ 1 - e^{-\alpha (r - r_{\text{eq}})} \right]^2
\end{equation}
$$
where $\alpha$ is the Morse parameter.
Carbon Monoxide
For example, carbon monoxide (CO) consists of one carbon atom (atomic weight 12.011) and one oxygen atom (atomic weight 15.999) separated by 1.1283 Angstroms. This can be modelled by

a harmonic potential with $k = 2743.0$ and $r_{\text{eq}} = 1.1283$ Angstroms, or
a Kratzer potential with $D_0 = 258.9$ kcal/mol and the same equilibrium distance, or
a Morse potential with $\alpha = 2.302$ Angstroms${}^{-1}$, and the same dissociation energy and equilibrium distance.

Using standard minimization algorithms (e.g. scipy.optimize.minimize) show that, for suitable initial guesses, all of the above potentials have minima at the expected location.
By plotting the potentials, understand why the success of the algorithms changes.
Briefly test how changing the potential changes the number of steps required by the algorithm.
Also compute the force $\frac{dV}{dr}$ at the minimum point for each potential to check that it vanishes as expected.
Can you write Python code for it?

%matplotlib inline
import numpy
from matplotlib import pyplot
from mpl_toolkits.mplot3d.axes3d import Axes3D
from matplotlib import rcParams
rcParams['font.family'] = 'serif'
rcParams['font.size'] = 16
rcParams['figure.figsize'] = (12,6)
from scipy.optimize import minimize


I am working on the file "FEEG6016 Simulation and Modelling/05-Agents-Lab-1.ipynb".
The first task is:
Conway's Game of Life
A simple agent model is Conway's Game of Life, which is an example of Cellular automota. A two-dimensional square grid of cells are either "dead" or "alive". At each iteration, each cell checks its neighbours (including diagonals: each cell has 8 neighbours).

Any live cell with fewer than two live neighbours dies ("under-population")
Any live cell with two or three neighbours lives ("survival")
Any live cell with four or more neighbours dies ("over-population")
Any dead cell with exactly three neigbours lives ("reproduction")

At the boundaries of the grid periodic boundary conditions are imposed.
Write a function that takes a numpy array representing the grid. Test it on some of the standard example patterns. The matplotlib imshow function, and the matplotlib FuncAnimation function may help; if running in the notebook, the instructions on installing and using ffmpeg and html5 may also be useful.
Can you write Python code for it?

%matplotlib inline
import numpy
from matplotlib import pyplot, animation

from matplotlib import rcParams
rcParams['font.family'] = 'serif'
rcParams['font.size'] = 16
rcParams['figure.figsize'] = (12,6)

from __future__ import division

def conway_iteration(grid):
    """
    Take one iteration of Conway's game of life.

    Parameters
    ----------

    grid : array
        (N+2) x (N+2) numpy array representing the grid (1: live, 0: dead)

    """

    # Code to go here

    return grid

# Try the loaf - this is static

grid_loaf = numpy.array([[0,0,0,0,0,0,0,0],
                         [0,0,0,0,0,0,0,0],
                         [0,0,0,1,1,0,0,0],
                         [0,0,1,0,0,1,0,0],
                         [0,0,0,1,0,1,0,0],
                         [0,0,0,0,1,0,0,0],
                         [0,0,0,0,0,0,0,0],
                         [0,0,0,0,0,0,0,0]])

fig = pyplot.figure()
im = pyplot.imshow(grid_loaf[1:-1,1:-1], cmap=pyplot.get_cmap('gray'))

def init():
    im.set_array(grid_loaf[1:-1,1:-1])
    return im,

def animate(i):
    conway_iteration(grid_loaf)
    im.set_array(grid_loaf[1:-1,1:-1])
    return im,

# This will only work if you have ffmpeg installed

anim = animation.FuncAnimation(fig, animate, init_func=init, interval=50, frames=10, blit=True)

HTML(anim.to_html5_video())


I am working on the file "FEEG6016 Simulation and Modelling/06-Agents-Lab-2.ipynb".
The first task is:
Define a class called Agent. This should store the 2-dimensional ${\bf z} = (x, y)$ location of the individual, and its 2-dimensional velocity ${\bf v} = (v_x, v_y)$. It should have a method that updates its location over a timestep $\Delta t$ as
\begin{equation}
  {\bf z} = \begin{pmatrix} x \ y \end{pmatrix} \to \begin{pmatrix} x \ y \end{pmatrix} + \Delta t \begin{pmatrix} v_x \ v_y \end{pmatrix} = {\bf z} + \Delta t \, {\bf v}.
\end{equation}
The Agent should also have a method that changes its velocity to maximize its "CSA utility": given a list of other agents that are the "local flockmates", it should choose its heading to maximize cohesion, separation and alignment. The CSA utility is a function of the locations and velocities that returns a single number, and is to be maximized by varying the direction of the velocity of this Agent.
As an example, define the following utility function:
\begin{equation}
  f \left( {\bf v} ; {\bf z}, {\bf z}i, {\bf v}_i \right) = C \frac{{\bf v} \cdot \Delta {\bf z}}{ \| {\bf v} \| \| \Delta {\bf z} \| } + A \frac{{\bf v} \cdot {\bf V}}{ \| {\bf v} \| \| {\bf V} \| } - S \frac{{\bf v} \cdot \Delta {\bf z}{\text{min}}}{ \| {\bf v} \| \| \Delta {\bf z}_{\text{min}} \|^3 }.
\end{equation}
$f$ is the utility function itself. It depends on the velocity ${\bf v}$ of the current Agent, which can be varied. It also depends on the location ${\bf z}$ of the current Agent, as well as the locations ${\bf z}_i$ and velocities ${\bf v}_i$ of the local flockmates, which cannot be varied.
The coefficients $C, S, A$ give the relative importance of improving the cohesion, separation, and alignment respectively.
$\Delta {\bf z}$ is the average location difference between the current Agent position ${\bf z}$ and the position of its local flockmates ${\bf z}_i$, given by
\begin{equation}
  \Delta {\bf z} = \frac{1}{N} \sum_{i=1}^{N} \begin{pmatrix} x_i - x \ y_i - y \end{pmatrix}
\end{equation}
${\bf V}$ is the average velocity of the local flockmates, given by
\begin{equation}
  {\bf V} =...
Can you write Python code for it?

class Agent(object):

    def __init__(self, location, velocity, C=1, A=5, S=0.1):
        # Complete this

    def step(self, dt):
        # And this

    def steer(self, neighbours):
        # And this


I am working on the file "FEEG6016 Simulation and Modelling/06-Agents-Lab-2.ipynb".
The first task is:
Flocking behaviour
Agents should both interact with their environment and each other, and change their behaviour to reach some goal (often by maximizing a utility function or similar). One example is a simple model of the behaviour of groups of birds (flocking) or fish (schooling). This CSA model says that all individuals only change direction in order to improve

Cohesion: they steer towards the local average location of their flockmates;
Separation: they steer to avoid crowding local flockmates;
Alignment: they steer towards the average heading of their local flockmates.

Here we investigate how this model looks, depending on the initial velocity of the individuals.
Can you write Python code for it?

%matplotlib inline
import math
import numpy
from matplotlib import pyplot, animation

from matplotlib import rcParams
rcParams['font.family'] = 'serif'
rcParams['font.size'] = 16
rcParams['figure.figsize'] = (12,6)

import scipy
from scipy.optimize import minimize

from __future__ import division


I am working on the file "FEEG6016 Simulation and Modelling/07-Finite-Elements-Lab-1.ipynb".
The first task is:
1d problem
The temperature $T$ of a bar of length $1$ at equilibrium satisfies
$$
  \partial_{xx} T + f(x) = 0.
$$
The heat source is given by $f(x)$. We'll hold the temperature fixed at the right edge by setting $T(1) = 0$. We'll allow heat to conduct through the left edge by setting $\partial_x T(0) = 0$.
Dealing with the second derivative is a problem, so we want to remove some derivatives from the formulation. Introducing a weighting function $w(x)$. Multiply the equation by $w$ and integrate by parts to get
$$
  \left[ \partial_x T(x) w(x) \right]_0^1 - \int_0^1 \text{d}x \, \partial_x T(x) \partial_x w(x) = - \int_0^1 \text{d}x \, f(x) w(x).
$$
Putting in the boundary conditions gives
$$
  \int_0^1 \text{d}x \, \partial_x T(x) \partial_x w(x) = \int_0^1 \text{d}x \, w(x) f(x).
$$
This is the weak form. It can be proved that solutions of the strong form (with two derivatives) are also solutions of the weak form, and vice versa.
Function representation
To get an approximate solution we split the domain into intervals, or elements. To start we'll choose $[0, 1/2]$ and $[1/2, 1]$, which means there are nodes at ${ 0, 1/2, 1 }$. We want to represent all unknown functions ($T, w$) on these elements. We'll introduce some shape or basis functions $N_A(x)$ and write, for example,
$$
  T(x) = \sum_A T_A N_A(x).
$$
Here $A$ is a counter associated with the nodes, so here $A = { 0, 1, 2 }$: the nodes are labelled $x_A$. The shape function $N_A(x)$ is $1$ at the associated node, and $0$ at all other nodes. If we choose the shape functions to be piecewise linear then the three shape functions therefore look like
$$
\begin{align}
  N_0(x) &= \begin{cases} 1 - 2 x, & 0 \le x \le 1/2 \ 0, & 1/2 \le x \le 1, \end{cases}, \
  N_1(x) &= \begin{cases} 2 x, & 0 \le x \le 1/2 \ 2 - 2 x, & 1/2 \le x \le 1, \end{cases}, \
  N_2(x) &= \begin{cases} 0, & 0 \le x \le 1/2 \ 2 x - 1, & 1/2 \le x \le 1. \end{cases}
\end{align}
$$
Can you write Python code for it?

%matplotlib inline
import numpy
from matplotlib import pyplot
from matplotlib import rcParams
rcParams['font.family'] = 'serif'
rcParams['font.size'] = 16
rcParams['figure.figsize'] = (12,6)

x = numpy.linspace(0, 1, 1000)
N0 = numpy.where(x <= 0.5, 1-2*x, numpy.zeros_like(x))
N1 = numpy.where(x <= 0.5, 2*x, 2-2*x)
N2 = numpy.where(x <= 0.5, numpy.zeros_like(x), 2*x-1)
pyplot.figure()
pyplot.plot(x, N0, label=r"$N_0$")
pyplot.plot(x, N1, label=r"$N_1$")
pyplot.plot(x, N2, label=r"$N_2$")
pyplot.xlabel(r"$x$")
pyplot.ylim(-0.1, 1.1)
pyplot.legend();


I am working on the file "FEEG6016 Simulation and Modelling/07-Finite-Elements-Lab-1.ipynb".
The first task is:
With this definition of the shape functions, the coefficients are given exactly as (e.g.) $T_A = T(x_A)$. NOTE: because we have imposed the boundary condition $T(1) = 0$ we must have that $T_2 \equiv 0$, so we do not need to calculate $T_2$.
We can now write out the weak form of the problem as
$$
  \sum_A \sum_B T_A w_B \int_0^1 \text{d}x \, \partial_x N_A(x) \partial_x N_B(x) = \sum_B w_B \int_0^1 \text{d}x \, N_A(x) f(x).
$$
This has to be true for any choice of weight function, so for any coefficients $w_B$. This is true only if
$$
  \sum_A T_A \int_0^1 \text{d}x \, \partial_x N_A(x) \partial_x N_B(x) = \int_0^1 \text{d}x \, N_A(x) f(x).
$$
This can be written as a matrix problem:
$$
  K {\bf T} = {\bf F}.
$$
The coefficients of the stiffness matrix $K$ are given by
$$
  K_{AB} = \int_0^1 \text{d}x \, \partial_x N_A(x) \partial_x N_B(x).
$$
The coefficients of the force vector $F$ are given by
$$
  F_{A} = \int_0^1 \text{d}x \, N_A(x) f(x).
$$
Tasks

Compute (analytically) the coefficients of the stiffness matrix given the shape functions above.
Compute (analytically) the coefficients of the force vector when $f(x) = x$.
Solve (numerically) for $T_A$ and plot the resulting solution for $T(x)$. Compare against the exact solution $T(x) = (1 - x^3) / 6)$.

The element viewpoint
This works surprisingly well, but it's clearly going to get very complex when the number of elements (or nodes) gets large as we try to improve accuracy. To work systematically with large numbers of elements, we need to consider one at a time.
First, note that in any element there's only two shape functions that are not zero: those that correspond to the nodes at the edge of the element. We'll call these shape functions $N_1$ and $N_2$.
Second, note that by doing a coordinate transformation, we can take any element from the interval $[ x_A, x_{A+1} ]$ to the interval $[ \xi_1, \xi_2 ]$ where we fix (by convention) $\xi_1 = -1$ and $\xi_2 = 1$. We have
$$
\begin{align}
  \xi(x) &= \frac{2 x - x_A - x_{A+1}}{x_{A+1} - x_{A}}, \
  x(\xi)...
Can you write Python code for it?

Nelements = 4

LM = numpy.zeros((2, Nelements))
for e in range(Nelements):
    LM[0, e] = e
    LM[1, e] = e+1
LM[1, -1] = -1 # Right hand node of final element is not considered thanks to BC.


I am working on the file "FEEG6016 Simulation and Modelling/08-Finite-Elements-Lab-2.ipynb".
The first task is:
Function representation and shape functions
We're going to want to write our unknown functions $T, w$ in terms of shape functions. These are easiest to write down for a single reference element, in the same way as we did for the one dimensional case where our reference element used the coordinates $\xi$. In two dimensions we'll use the reference coordinates $\xi_0, \xi_1$, and the standard "unit" triangle:
Can you write Python code for it?

corners = numpy.array([[0.0, 0.0], [1.0, 0.0], [0.0, 1.0], [0.0, 0.0]])
pyplot.plot(corners[:,0],corners[:,1],linewidth=2)
pyplot.xlabel(r"$\xi_0$")
pyplot.ylabel(r"$\xi_1$")
pyplot.axis('equal')
pyplot.ylim(-0.1,1.1);


I am working on the file "FEEG6016 Simulation and Modelling/08-Finite-Elements-Lab-2.ipynb".
The first task is:
In the one dimensional case we used the location matrix or $LM$ array to link local node numbers in elements to equations. With the $IED$ and $ID$ arrays the $LM$ matrix is strictly redundant, as $LM(a, e) = ID(IEN(e, a))$. However, it's still standard to construct it:
Can you write Python code for it?

LM = numpy.zeros_like(IEN.T)
for e in range(IEN.shape[0]):
    for a in range(IEN.shape[1]):
        LM[a,e] = ID[IEN[e,a]]
LM


I am working on the file "FEEG6016 Simulation and Modelling/08-Finite-Elements-Lab-2.ipynb".
The first task is:
The shape functions on this triangle are
\begin{align}
  N_0(\xi_0, \xi_1) &= 1 - \xi_0 - \xi_1, \
  N_1(\xi_0, \xi_1) &= \xi_0, \
  N_2(\xi_0, \xi_1) &= \xi_1.
\end{align}
The derivatives are all either $0$ or $\pm 1$.
As soon as we have the shape functions, our weak form becomes
$$
  \sum_A T_A \int_{\Omega} \text{d}\Omega \, \left( \partial_{x} N_A (x, y) \partial_{x} N_B(x, y) + \partial_{y} N_A(x, y) \partial_{y} N_B(x, y) \right) = \int_{\Omega} \text{d}\Omega \, N_B(x, y) f(x, y).
$$
If we restrict to a single element the weak form becomes
$$
  \sum_A T_A \int_{\triangle} \text{d}\triangle \, \left( \partial_{x} N_A (x, y) \partial_{x} N_B(x, y) + \partial_{y} N_A(x, y) \partial_{y} N_B(x, y) \right) = \int_{\triangle} \text{d}\triangle \, N_B(x, y) f(x, y).
$$
We need to map the triangle and its $(x, y) = {\bf x}$ coordinates to the reference triangle and its $(\xi_0, \xi_1) = {\bf \xi}$ coordinates. We also need to work out the integrals that appear in the weak form. We need the transformation formula
$$
  \int_{\triangle} \text{d}\triangle \, \phi(x, y) = \int_0^1 \text{d}\xi_1 \, \int_0^{1-\xi_1} \text{d}\xi_0 \, \phi \left( x(\xi_0, \xi_1), y(\xi_0, \xi_1) \right) j(\xi_0, \xi_1),
$$
where the Jacobian matrix $J$ is
$$
  J = \left[ \frac{\partial {\bf x}}{\partial {\bf \xi}} \right] = \begin{pmatrix} \partial_{\xi_0} x & \partial_{\xi_1} x \ \partial_{\xi_0} y & \partial_{\xi_1} y \end{pmatrix}
$$
and hence the Jacobian determinant $j$ is
$$
  j = \det{J} = \det \left[ \frac{\partial {\bf x}}{\partial {\bf \xi}} \right] = \det \begin{pmatrix} \partial_{\xi_0} x & \partial_{\xi_1} x \ \partial_{\xi_0} y & \partial_{\xi_1} y \end{pmatrix}.
$$
We will also need the Jacobian matrix when writing the derivatives of the shape functions in terms of the coordinates on the reference triangle, i.e.
$$
  \begin{pmatrix} \partial_x N_A & \partial_y N_A \end{pmatrix} = \begin{pmatrix} \partial_{\xi_0} N_A & \partial_{\xi_1} N_A \end{pmatrix} J^{-1} .
$$
The integral over the reference triangle can be directly...
Can you write Python code for it?

def generate_2d_grid(Nx):
    """
    Generate a triangular grid covering the plate math:`[0,1]^2` with Nx (pairs of) triangles in each dimension.

    Parameters
    ----------

    Nx : int
        Number of triangles in any one dimension (so the total number on the plate is math:`2 Nx^2`)

    Returns
    -------

    nodes : array of float
        Array of (x,y) coordinates of nodes
    IEN : array of int
        Array linking elements to nodes
    ID : array of int
        Array linking nodes to equations
    """
    Nnodes = Nx+1
    x = numpy.linspace(0, 1, Nnodes)
    y = numpy.linspace(0, 1, Nnodes)
    X, Y = numpy.meshgrid(x,y)
    nodes = numpy.zeros((Nnodes**2,2))
    nodes[:,0] = X.ravel()
    nodes[:,1] = Y.ravel()
    ID = numpy.zeros(len(nodes), dtype=numpy.int)
    n_eq = 0
    for nID in range(len(nodes)):
        if nID % Nnodes == Nx:
            ID[nID] = -1
        else:
            ID[nID] = n_eq
            n_eq += 1
    IEN = numpy.zeros((2*Nx**2,3), dtype=numpy.int)
    for i in range(Nx):
        for j in range(Nx):
            IEN[2*i+2*j*Nx  , :] = i+j*Nnodes, i+1+j*Nnodes, i+(j+1)*Nnodes
            IEN[2*i+1+2*j*Nx, :] = i+1+j*Nnodes, i+1+(j+1)*Nnodes, i+(j+1)*Nnodes
    return nodes, IEN, ID


I am working on the file "FEEG6016 Simulation and Modelling/08-Finite-Elements-Lab-2.ipynb".
The first task is:
What we're doing here is

Providing a list of nodes by their global coordinates.
Providing the element node array IEN which says how the elements are linked to the nodes.

We have that for element $e$ and local node number $a = 0, 1, 2$ the global node number is $A = IEN(e, a)$. This notation is sufficiently conventional that matplotlib recognizes it with its triplot/tripcolor/trisurf functions.
It is convention that the nodes are ordered in the anti-clockwise direction as the local number goes from $0$ to $2$.
The plot shows the

element numbers in the red boxes
the global node numbers in the green boxes
the local element numbers in the blue boxes (the subscript shows the element number).

We will need one final array, which is the $ID$ or destination array. This links the global node number to the global equation number in the final linear system. As the order of the equations in a linear system doesn't matter, this essentially encodes whether a node should have any equation in the linear system. Any node on $\Gamma_D$, where the value of the temperature is given, should not have an equation. In the example above the right edge is fixed, so nodes $1$ and $3$ lie on $\Gamma_D$ and should not have an equation. Thus in our case we have
Can you write Python code for it?


I am working on the file "FEEG6016 Simulation and Modelling/09-Stochastic-DEs-Lab-1.ipynb".
The first task is:
Evaluate the function $u(W(t)) = \sin^2(t + W(t))$, where $W(t)$ is a Brownian process, on $M$ Brownian paths for $M = 500, 1000, 2000$. Compare the average path for each $M$.
The average path at time $t$ should be given by
$$
\begin{equation}
  \int_{-\infty}^{\infty} \frac{\sin(t+s)^2 \exp(-s^2 / 2t)}{\sqrt{2 \pi t}} \,\text{d}s.
\end{equation}
$$
Can you write Python code for it?

# This computes the exact solution!

t_int = numpy.linspace(0.005, numpy.pi, 1000)

def integrand(x,t):
    return numpy.sin(t+x)**2*numpy.exp(-x**2/(2.0*t))/numpy.sqrt(2.0*numpy.pi*t)

int_exact = numpy.zeros_like(t_int)
for i, t in enumerate(t_int):
    int_exact[i], err = quad(integrand, -numpy.inf, numpy.inf, args=(t,))


I am working on the file "FEEG6016 Simulation and Modelling/09-Stochastic-DEs-Lab-1.ipynb".
The first task is:
This background for these exercises is article of D Higham, An Algorithmic Introduction to Numerical Simulation of Stochastic Differential Equations, SIAM Review 43:525-546 (2001).
Higham provides Matlab codes illustrating the basic ideas at http://personal.strath.ac.uk/d.j.higham/algfiles.html, which are also given in the paper.
For random processes in python you should look at the numpy.random module. To set the initial seed (which you should not do in a real simulation, but allows for reproducible testing), see numpy.random.seed.
Brownian processes
A random walk or Brownian process or Wiener process is a way of modelling error introduced by uncertainty into a differential equation. The random variable representing the walk is denoted $W$. A single realization of the walk is written $W(t)$. We will assume that

The walk (value of $W(t)$) is initially (at $t=0$) $0$, so $W(0)=0$, to represent "perfect knowledge" there;
The walk is on average zero, so $\mathbb{E}[W(t+h) - W(t)] = 0$, where the expectation value is
$$
  \mathbb{E}[W] = \int_{-\infty}^{\infty} t W(t) \, \text{d}t
$$
Any step in the walk is independent of any other step, so $W(t_2) - W(t_1)$ is independent of $W(s_2) - W(s_1)$ for any $s_{1,2} \ne t_{1,2}$.

These requirements lead to a definition of a discrete random walk: given the points ${ t_i }$ with $i = 0, \dots, N$ separated by a uniform timestep $\delta t$, we have - for a single realization of the walk - the definition
$$
\begin{align}
  \text{d}W_i &= \sqrt{\delta t} {\cal N}(0, 1), \
  W_i &= \left( \sum_{j=0}^{i-1} \text{d}W_j \right), \
  W_0 &= 0
\end{align}
$$
Here ${\cal N}(0, 1)$ means a realization of a normally distributed random variable with mean $0$ and standard deviation $1$: programmatically, the output of numpy.random.randn.
When working with discrete Brownian processes, there are two things we can do.

We can think about a single realization at different timescales, by averaging over more points. E.g.
$$
  W_i = \left( \sum_{j=0}^{i_1} \sum_{k=0}^{p} \text{d}W_{(p j +...
Can you write Python code for it?

%matplotlib inline
import numpy
from matplotlib import pyplot
from matplotlib import rcParams
rcParams['font.family'] = 'serif'
rcParams['font.size'] = 16
rcParams['figure.figsize'] = (12,6)
from scipy.integrate import quad


I am working on the file "FEEG6016 Simulation and Modelling/10-Stochastic-DEs-Lab-2.ipynb".
The first task is:
This background for these exercises is article of D Higham, An Algorithmic Introduction to Numerical Simulation of Stochastic Differential Equations, SIAM Review 43:525-546 (2001).
Higham provides Matlab codes illustrating the basic ideas at http://personal.strath.ac.uk/d.j.higham/algfiles.html, which are also given in the paper.
Can you write Python code for it?


I am working on the file "FFT Scaling Notes.ipynb".
The first task is:
Note, this also holds for interference between waves of different amplitudes. I've verified this with beam simulations.
FFT Frequency Scaling
Off-axis component of the wave is $k_x = k \sin\theta = \frac{2\pi}{780 nm}\sin\theta$
Can you write Python code for it?

K_x = 2*pi/780e-9 * numpy.sin(0.005) # in radians per meter


I am working on the file "FFT Scaling Notes.ipynb".
The first task is:
So the horizontal axis scale should reflect these values. Note, they will not change with the setup so we can hard-code them into our graphing routines.
For example, if a peak is found with index 166 in the FFT output, that corresponds to:
Can you write Python code for it?


I am working on the file "FFT Scaling Notes.ipynb".
The first task is:
The step size $\Delta k$ is given by $\frac{2\pi}{(20\mu m)(1340)}$ based on pixel size and pixel count
Can you write Python code for it?

delta_K = 2*pi/(20e-6*1340) # deltaK in radians per meter


I am working on the file "FFT Test.ipynb".
The first task is:
The sinusoidal signal can be expressed as a linear sum of exponential values.
$$ sin(2 \pi f t + \phi) = \frac{(e^{j 2 \pi  f t}-e^{- j 2 \pi f t})}{(2j)} $$
In the Fourier Series we know as de synthezis equation
$$ x(t) = \sum^{\infty}_{k=-\infty} a_k e^{j k 2 \pi f t}$$
and the analytical equation to find the coefficients
$$ a_k = \frac{1}{T} \int_T x(t) e^{j k 2 \pi f t}$$
Using the linear combination of the xomplex exponentials of the sine, we can compare to the analytical equation and find the coefficients.
$$ a_1 = \frac{1}{2j} = \frac{-j}{2} $$
$$ a_{-1} = \frac{-1}{2j} = \frac{j}{2}$$
$$ a_0 = 0$$
And there are no other components. What we can see from this result for the sine wave is that bith components are in the imaginary plane, so first of all we can verify this by plotting the FFT of the sine wave. We observe that there are only 2 spikes in the $\Im$ plane, the coefficient $a_1$ is negative and $a_{-1}$ is positive. The values of the $\Re$ plane are very small compared to the $\Im$ plane.
We expect from a Sinewave a power of $$P = A^2 /2$$
In our case with $A=1$ the power should be $0.5$
We can observe that the spikes are at $0.25$ this means that if we add both we obtain the complete power. Therefore it is needed in the calculation of the FFT to apply the normalization of the number of values of the signal.
Can you write Python code for it?

freq_array = np.fft.fftfreq(n, 1.0/Fs)
freq_array_shift = np.fft.fftshift(freq_array)

X = np.fft.fftshift(np.fft.fft(x))/n # fft computing and normalization

fig, ax = plt.subplots(3, 1)
ax[0].plot(freq_array_shift,abs(X),'r') # plotting the spectrum
ax[0].set_xlabel('Freq (Hz)')
ax[0].set_ylabel('|X($\omega$)|')
ax[1].plot(freq_array_shift,np.real(X),'r') # plotting the spectrum
ax[1].set_xlabel('Freq (Hz)')
ax[1].set_ylabel('$\Re$(X)')
ax[2].plot(freq_array_shift,np.imag(X),'r') # plotting the spectrum
ax[2].set_xlabel('Freq (Hz)')
ax[2].set_ylabel('$\Im$(X)')


I am working on the file "FII-year3sem2-CN/Exam.ipynb".
The first task is:
Linear spline functions are calculated with the following:
$$i \in [1,\ \left\vert{X}\right\vert - 1],\ i \in \mathbb{N}: $$
$$P_i = \frac{x-x_i}{x_{i+1}-x_i} * y_{i+1} + \frac{x_{i+1}-x}{x_{i+1}-x_i} * y_i$$
By simplification, we can reduce to the following:
$$P_i = \frac{y_{i+1} (x-x_i) + y_i (x_{i+1}-x)}{x_{i+1}-x_i} = \frac{(y_{i+1}x - y_ix) - y_{i+1}x_i + y_ix_{i+1}}{x_{i+1}-x_i}$$
The final form used will be:
$$P_i = \frac{(y_{i+1}x - y_ix) + (y_ix_{i+1} - y_{i+1}x_i)}{(x_{i+1}-x_i)}$$
As it can be seen, the only gist would be to emulate the x in the first term (num1s below), the other terms being numbers (num2, den). Parantheses used to isolate the formula for each of the 3 variables.
As such, we can write the parantheses as a string, while the others will be simply calculated. After this, the final string is evaluated as a lambda function.
Can you write Python code for it?

print('x1 = %i' % data[0][0])
print('y1 = %i' % data[1][0])
print('---')

# linear spline function aproximation
print('no values: %i' % len(columns))

spline = {}

for i in range(len(columns)-1):
    print('\nP[' + str(i+1) + ']')

    # we calculate the numerator
    num_1s = str(data[1][i+1]) + ' * x - ' + str(data[1][i]) + ' * x'
    print('num_1s: %s' % num_1s)

    num_2  = data[1][i] * data[0][i+1] - data[1][i+1] * data[0][i]
    print('num_2: %i' % num_2)

    # we calculate the denominator
    den = data[0][i+1] - data[0][i]
    print('den: %i' % den)

    # constructing the function
    func = 'lambda x: (' + num_1s + str(num_2) + ') / ' + str(den)
    print('func: %s' % func)
    spline[i] = eval(func)

print('---')

# sanity checks
# P1(x) = -x - 5
assert (spline[0](-5) == 0),"For this example, the value should be 0, but the value returned is " + str(spline[0](-5))
# P2(x) = 4x + 1
# TODO: this is failing (checked my solution, probably my assertion is wrong) !
#assert (spline[1](0) == 1),"For this example, the value should be 1, but the value returned is " + str(spline[1](0))
# P3(x) = 13x - 17
assert (spline[2](1) == -4),"For this example, the value should be -4, but the value returned is " + str(spline[2](1))

print('Approximating values of S\n---')
aproximation_queue = [-1, 1]
results = {}

def approximate(spline, val):
    for i in range(len(spline)-1):
        if data[0][i] <= val <= data[0][i+1]:
            print('Approximation using P[%i] is: %i' % (i, spline[i](val)))
            results[val] = spline[i](val)

for i in range(len(aproximation_queue)):
    approximate(spline, aproximation_queue[i])

# sanity checks
# S(-1) = P1(-1) = -4
assert (spline[0](-1) == -4),"For this example, the value should be -4, but the value returned is " + str(spline[0](-5))
# S(1) = P2(1) = 5
# TODO: same as above !
#assert (spline[1](1) == 5),"For this example, the value should be 5, but the value returned is " + str(spline[1](0))

#x.extend(results.keys())
#y.extend(results.values())
x2 =...


I am working on the file "FITS-images.ipynb".
The first task is:
At this point, we can just close the FITS file.  We have stored everything we wanted to a variable.
Can you write Python code for it?


I am working on the file "FITS-images.ipynb".
The first task is:
Displaying the image with a logarithmic scale
Want to use a logarithmic color scale? To do so we need to load the LogNorm object from matplotlib.
Can you write Python code for it?

from matplotlib.colors import LogNorm

plt.imshow(image_data, cmap='gray', norm=LogNorm())

# I chose the tick marks based on the histogram above
cbar = plt.colorbar(ticks=[5.e3,1.e4,2.e4])
cbar.ax.set_yticklabels(['5,000','10,000','20,000'])


I am working on the file "FITS-images.ipynb".
The first task is:
Generally the image information is located in the PRIMARY block. The blocks are numbered and can be accessed by indexing hdu_list.
Can you write Python code for it?


I am working on the file "FITS-images.ipynb".
The first task is:
I'll use the keywords vmin and vmax to set limits on the color scaling for imshow.
Can you write Python code for it?

plt.imshow(final_image, cmap='gray', vmin=2.e3, vmax=3.e3)
plt.colorbar()


I am working on the file "FITS-images.ipynb".
The first task is:
I'm going to show the image, but I want to decide on the best stretch. To do so I'll plot a histogram of the data.
Can you write Python code for it?

image_hist = plt.hist(final_image.flat, 1000)


I am working on the file "FITS-images.ipynb".
The first task is:
Let's get some basic statistics about our image
Can you write Python code for it?

print('Min:', np.min(image_data))
print('Max:', np.max(image_data))
print('Mean:', np.mean(image_data))
print('Stdev:', np.std(image_data))


I am working on the file "FITS-images.ipynb".
The first task is:
Now I'll stack the images by summing my concatenated list.
Can you write Python code for it?

# The long way
final_image = np.zeros(shape=image_concat[0].shape)

for image in image_concat:
    final_image += image

# The short way
#final_image = np.sum(image_concat, axis=0)


I am working on the file "FITS-images.ipynb".
The first task is:
Opening FITS files and loading the image data
I will open the FITS file and find out what it contains.
Can you write Python code for it?

hdu_list = fits.open(image_file)
hdu_list.info()


I am working on the file "FITS-images.ipynb".
The first task is:
SHORTCUT
If you don't need to examine the FITS header, you can call fits.getdata to bypass the previous steps.
Can you write Python code for it?

image_data = fits.getdata(image_file)
print(type(image_data))
print(image_data.shape)


I am working on the file "FITS-images.ipynb".
The first task is:
Viewing and manipulating FITS images
Can you write Python code for it?

from astropy.io import fits

image_file = download_file('http://data.astropy.org/tutorials/FITS-images/HorseHead.fits', cache=True )


I am working on the file "FITS-images.ipynb".
The first task is:
Viewing the image data and getting basic statistics
Can you write Python code for it?

plt.imshow(image_data, cmap='gray')
plt.colorbar()

# To see more color maps
# http://wiki.scipy.org/Cookbook/Matplotlib/Show_colormaps


I am working on the file "FITS-images.ipynb".
The first task is:
Writing image data to a FITS file
This is easy to do with the writeto() method.
You will receive an error if the file you are trying to write already exists.  That's why I've set clobber=True.
Can you write Python code for it?

outfile = 'stacked_M13_blue.fits'

hdu = fits.PrimaryHDU(final_image)
hdu.writeto(outfile, clobber=True)


I am working on the file "FITS-images.ipynb".
The first task is:
You data is now stored as a 2-D numpy array.  Want to know the dimensions of the image?  Just look at the shape of the array.
Can you write Python code for it?


I am working on the file "FInal Script - Local.ipynb".
The first task is:
Debating what will happen if I remove all entries in the dataset with which a SOC_NAME that only occures once. How many entries will this remove? Can we ignore entries like this?
What about entries with an occurence count of 2? 3? Do these entries matter?
What do we do if they matter?
At this point, I will have to do as much as I can in python, and then switch over to rapidminer.
I have an idea on how I could go through the rows faster.
Can you write Python code for it?

%%timeit
num=3000000
i=1
while i < num:
    try:
        name = cleandata2.iloc[i]['SOC_NAME']
        newname = reducedf.loc[(reducedf['SOC_NAME']==name)].iloc[0]['Name2']
        #cleandata2.set_value(i,['SOC_NAME'], newname)
        i=i+1
    except:
        pass


I am working on the file "FInal Script - Local.ipynb".
The first task is:
Here, we transform some strings to lowercase. This is because there are duplicate entries in the dataset which in both upper and lower.
This increases redundancy
Can you write Python code for it?


I am working on the file "FInal Script - Local.ipynb".
The first task is:
Here, we try to apply what we have reduced to the dataframe. We will see if we have made any progress, and what we can do next.
However, I have a very VERY large problem.
The logic to use the reduced names, takes 1 s.
With about 3000000 rows, that would be about 35 days.
I have no idea how to get around this right now.
I think I might be able to do this, by converting the pandas dataframe into a spark rdd.
From there I could user map and map reduce functions.
But I would need a lot more time in order to do this.
At this point. I have spent many hours on this.
I feel very defeated.
I am now going to see if there is some way I can streamline the algoritm.
Can you write Python code for it?

%%timeit
for index, row in cleandata2.iterrows():
    try:
        cleandata2.set_value([index],['SOC_NAME'],reducedf.loc[(reducedf['SOC_NAME']==row['SOC_NAME'])].iloc[0]['Name2'])
    except:
        pass
    break


I am working on the file "FInal Script - Local.ipynb".
The first task is:
Striping charactors did not help much.
At this point investigated a spellchecker in python. Was not able to get something to work within enviroment.
I am now going to consider removing entries that are unique, with a count of one.
Can you write Python code for it?

dfName2Check = pd.DataFrame({'Name2': reducedf['Name2'].value_counts().index, 'Count':reducedf['Name2'].value_counts().values})

dfName2Check


I am working on the file "FInal Script - Local.ipynb".
The first task is:
Test code to test algorithm
At this point of development, I have switched to running the python notebook locally. I am having trouble with how slow the cloud service is, and I am not using spark yet.
Can you write Python code for it?

%%timeit
errormap = defaultdict(list)

for index, row in cleandata2.iterrows():
    name = row['SOC_NAME']
    newname = reducedf.loc[(reducedf['SOC_NAME']==name)]
    try:
        newname1 = newname.iloc[0]['Name2']
        cleandata2.set_value([index],['SOC_NAME'],newname1)
    except:
        quicklist = ["Something went wrong", name, newname]
        errormap[index].append(quicklist)
    break


I am working on the file "FInal Script - Local.ipynb".
The first task is:
Theres a problem, I am not getting a real representation of the occurence of names in the data. I now need to do something where I can get the actual number of occurences.
Can you write Python code for it?


I am working on the file "FInal Script - Local.ipynb".
The first task is:
We have now cut the number of names in half from the original number.
Can you write Python code for it?

reducedf.sort_values(['Name1'])

reducedf['Name2'] = ""

%%timeit
regex = re.compile('[^a-z\s]')

for index, row in reducedf.iterrows():
    reducedf.set_value([index],['Name2'],(regex.sub('', row['Name1'])))

reducedf.sort_values(['Name1'])

reducedf['Name2'].value_counts().count()


I am working on the file "FInal Script - Local.ipynb".
The first task is:
We have reduced the number of names down to 1585
Can you write Python code for it?

reducedf = pd.DataFrame({'SOC_NAME': cleandata1['SOC_NAME'].value_counts().index, 'Count':cleandata1['SOC_NAME'].value_counts().values})
#df['Counts'] = df.groupby(['SOC_NAME'])['Count'].transform('count') #I don't remember what I was trying to do here.
#df = df.set_index(['SOC_NAME'])
reducedf

reducedf['Name1'] = ''

reducedf

reducedf.iloc[3]['Count'] #example of accessing a location

%%timeit
for index, row in reducedf.iterrows():
    names = row['SOC_NAME'].split(",")
    if(names[0].endswith('*')):
        reducedf.set_value([index],['Name1'],(names[0][:-1]))
    if not (names[0].endswith('s')):
        reducedf.set_value([index],['Name1'],(names[0]+'s'))
    else:
        reducedf.set_value([index],['Name1'],names[0])

reducedf

cleandata1['SOC_NAME'].value_counts().count()

(cleandata1.loc[(cleandata1['SOC_NAME']=='software developers, appllications')]) #an example of a query


I am working on the file "FInal Script - Local.ipynb".
The first task is:
its the setting of the values that takes the majority of the time.
<h1>Following code is for generating heat mape of lat and lon data
Can you write Python code for it?

df_data_2

lon = []
lon = df_data_2['lon'].values,df_data_2['lat'].values
lon

#for long, lat in df_data_2['lon'].values for in df_data_2['lat'].values
geo = [list(a) if a!=['nan', 'nan'] for a in zip(df_data_2['lon'].values,df_data_2['lat'].values)]

geo

numpy.random.seed(0)
seaborn.set()
uniform_data = lon
ax = seaborn.heatmap(uniform_data)

uniform_data


I am working on the file "FInal_Project/.ipynb_checkpoints/Quantifying_Influence_Analysis_maxrose_DSFinal-checkpoint.ipynb".
The first task is:
Beginning to try out time series analysis below, however this exposed problems in the release dates either from my original queries or in the aggregation in Notebook 1. THis is on the TODO list.
A quick measure of songs covered by release year for both artists.
<br> While The Beatle disbanded by 1970, The Stones continue to this day. However, their early work appears far more influential, with the greatest body of influence more or less paralelling that of The more covered Beatles. Let's put a number on that, shall we?
Can you write Python code for it?

### First plot is The Beatles, second The Rolling Stones. Sum of times_covered by year of oriiginal relase of the song.
yticks = np.arange(100, 1000, 100)
data[data.times_covered > 0].groupby('year').times_covered.sum().plot(kind='bar',x='year',y='times_covered',figsize=(16,9))
plt.title('Total songs covered by year of original',size =28)
plt.ylabel('Times Covered(Sum)', size = 24)
plt.yticks(yticks)

bar = data.sort_values(by='year').groupby(['year', 'artist'])['times_covered'].sum().unstack('artist')
yticks = np.arange(25, 1000, 50)
bar.plot(kind='bar', stacked=True,figsize=(16,12),subplots='True')
plt.yticks(yticks)
plt.title('Total songs covered per Artist, by year of original',size =24)
plt.ylabel('Times Covered', size = 24)


I am working on the file "FInal_Project/.ipynb_checkpoints/Quantifying_Influence_Analysis_maxrose_DSFinal-checkpoint.ipynb".
The first task is:
Check coverage of lyrics for the original songs.
96.6% have lyrics (and lyric sentiment polarity score) in the dataset.
Can you write Python code for it?

data[(data.is_cover == 0) & (data.lyrics.isnull())].workid.count().astype(float)/data[(data.is_cover == 0)].workid.count()


I am working on the file "FInal_Project/.ipynb_checkpoints/Quantifying_Influence_Analysis_maxrose_DSFinal-checkpoint.ipynb".
The first task is:
Compute ROC curve and AUC score.
Can you write Python code for it?

# plot ROC curve
fpr, tpr, thresholds = metrics.roc_curve(y_test, y_pred_prob)
plt.plot(fpr, tpr)
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.0])
plt.xlabel('False Positive Rate (1 - Specificity)')
plt.ylabel('True Positive Rate (Sensitivity)')

# calculate AUC
print metrics.roc_auc_score(y_test, y_pred_prob)
print cross_val_score(logreg, X, y, cv=10, scoring='roc_auc').mean()

# TODO: Work on improving accuracy with Random Forest model, TIme Series Analysis
# and at least one Linear or Logistic Regression Model. Also create some basic bar charts to illustrate some basic assumptions about the data.


I am working on the file "FInal_Project/.ipynb_checkpoints/Quantifying_Influence_Analysis_maxrose_DSFinal-checkpoint.ipynb".
The first task is:
Create base set of features for fitting to models
Can you write Python code for it?

feature_cols = [ 'year','num_releases','lyric_sent','title_sent', 'countries', 'avg_rating']
X= data[data.is_cover == 0][feature_cols]
y = data[data.is_cover == 0].is_covered
print X.shape
print y.shape

### TODO - get null accuracy score. ????
means = []
for col in X.columns:
    newmean = np.sqrt(metrics.mean_squared_error(X[col], y))
    means.append(newmean.astype(float))
means = pd.Series(means)
print means.mean()
print zip(X.columns,means)


I am working on the file "FInal_Project/.ipynb_checkpoints/Quantifying_Influence_Analysis_maxrose_DSFinal-checkpoint.ipynb".
The first task is:
It appears that a pretty large percentage of the artist's catalogs have been covered at least once.
Can you write Python code for it?

# Throw out covers recorded by each band and see what percentage of their catalogs have been covered.
data[(data.is_cover == 0) & (data.is_covered == 0)].groupby('artist').count()/data[(data.is_cover == 0)].groupby('artist').count()

data[data.is_covered > 0].groupby(['artist', 'year']).songname.count().unstack('artist').plot(kind='bar',subplots='True',figsize=(16,9))
plt.title("Original Songs Released Per Year", size=20)


I am working on the file "FInal_Project/.ipynb_checkpoints/Quantifying_Influence_Analysis_maxrose_DSFinal-checkpoint.ipynb".
The first task is:
Test Logistic Regression Model
Can you write Python code for it?

X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=101)

# Compute Null accuracy
y_null = np.zeros_like(y_test, dtype=float)
# fill the array with the mean value of y_test
y_null.fill(y_test.mean())
y_null
np.sqrt(metrics.mean_squared_error(y_test, y_null))


I am working on the file "FInal_Project/.ipynb_checkpoints/Quantifying_Influence_Analysis_maxrose_DSFinal-checkpoint.ipynb".
The first task is:
Test with DecisionTreeClassifier
While seeking a less opaque model than the Random Forest, I tried the DecisionTreeClassifier, which has a very nice method to rank the importance of the features.
Can you write Python code for it?

treeclf = DecisionTreeClassifier(max_depth = 10, random_state=1)
treeclf.fit(X, y)

### As shown in the bar chart below, year and lyric sentiment are better predictors of whether or not a song is covered.

pd.DataFrame({'feature':feature_cols, 'importance':treeclf.feature_importances_}).sort_values('importance').plot(kind='bar',x='feature',figsize=(16,5),fontsize='14',title="Feature Importance")


I am working on the file "FInal_Project/.ipynb_checkpoints/Quantifying_Influence_Analysis_maxrose_DSFinal-checkpoint.ipynb".
The first task is:
To this point, my somewhat thin collection of features have not shown a very useful measure of predicting the number of covers of a song (my basic premise for measuring influence. However the Random Forest appears to prefer a whopping 4 features for the meager measure of accuracy per the graph below.
Can you write Python code for it?

# plot max_features (x-axis) versus Accuracy score (y-axis)
plt.plot(feature_range, Acc_scores)
plt.xlabel('max_features')
plt.ylabel('Accuracy (higher is better)')


I am working on the file "FInal_Project/Quantifying_Influence_Analysis_maxrose_DSFinal.ipynb".
The first task is:
"Top of the Pops"
Can you write Python code for it?

top10['dateint'] = top10['minreleasedate'].apply(lambda x: x.year)
colors = np.where(top10.artist == 'The Beatles', 'b', 'g')
top10.groupby('artist').plot(kind='bar', x='songname', y='times_covered',color=colors,rot=30,fontsize=14,legend=True,figsize=(16,5))
#top10.plot(kind='scatter', x='dateint', y='times_covered', s=220,c=colors, legend='artist',figsize=(12,8))
plt.title('Top 10 Covered Per Artist',size=20)
plt.xlabel('Times Covered')
plt.ylabel('Song')

print top10.groupby('artist').times_covered.mean()

data[data.is_covered > 0].groupby(['artist', 'year']).songname.count().unstack('artist').plot(kind='bar',subplots='True',figsize=(16,9))
plt.title("Original Songs Released Per Year", size=20)


I am working on the file "FInal_Project/Quantifying_Influence_Analysis_maxrose_DSFinal.ipynb".
The first task is:
As shown in the bar chart below, year and lyric sentiment are better predictors of whether or not a song is covered.
Can you write Python code for it?


I am working on the file "FInal_Project/Quantifying_Influence_Analysis_maxrose_DSFinal.ipynb".
The first task is:
At 92% and 90% respectively, both the out of bag and cross-validation scores are quite positive for the Random Forest Classifier.
Testing Random Forest Regressor to predict "times_covered"
Can you write Python code for it?

from sklearn.feature_selection import SelectFromModel
rfreg = RandomForestRegressor(n_estimators=100, max_features=6, random_state=111)
rfreg.fit(X,y_regress)
sfm = SelectFromModel(rfreg, threshold='mean', prefit=True)
X_important = sfm.transform(X)
print(X_important.shape[0],X_important.shape[1])

rfreg = RandomForestRegressor(n_estimators=100, max_features=3, random_state=111)
scores = cross_val_score(rfreg, X_important, y_regress, cv=10, scoring='mean_squared_error')
np.mean(np.sqrt(-scores))


I am working on the file "FInal_Project/Quantifying_Influence_Analysis_maxrose_DSFinal.ipynb".
The first task is:
Build model with Random Forest Classifier (RFC)
Can you write Python code for it?

feature_range = range(1, len(feature_cols)+1)

# list to store the accuracy score for each value of max_features
Acc_scores = []

# use 10-fold cross-validation with each value of max_features (WARNING: SLOW!)
for feature in feature_range:
    rfclass = RandomForestClassifier(n_estimators=500, max_features=feature, random_state=50)
    acc_val_scores = cross_val_score(rfclass, X, y, cv=10, scoring='accuracy')
    Acc_scores.append(acc_val_scores.mean())


I am working on the file "FInal_Project/Quantifying_Influence_Analysis_maxrose_DSFinal.ipynb".
The first task is:
Check coverage of lyrics for the original songs.
96.6% have lyrics (and lyric sentiment polarity score) in the dataset. Lyric sentiment is a valuable predictor.
Can you write Python code for it?


I am working on the file "FInal_Project/Quantifying_Influence_Analysis_maxrose_DSFinal.ipynb".
The first task is:
Create base set of features for fitting to models
Can you write Python code for it?

feature_cols = [ 'year','num_releases','lyric_sent','title_sent', 'countries', 'avg_rating']
X= data[data.is_cover == 0][feature_cols]
y = data[data.is_cover == 0].is_covered
y_regress = data[data.is_cover == 0].times_covered
print X.shape
print y.shape
print y_regress.shape


I am working on the file "FInal_Project/Quantifying_Influence_Analysis_maxrose_DSFinal.ipynb".
The first task is:
I am able to predict with 80% probability that a song will be covered.

Compute ROC curve and AUC score.
Can you write Python code for it?

# plot ROC curve
fpr, tpr, thresholds = metrics.roc_curve(y_test, y_pred_prob)
plt.plot(fpr, tpr)
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.0])
plt.xlabel('False Positive Rate (1 - Specificity)')
plt.ylabel('True Positive Rate (Sensitivity)')

# calculate AUC
print metrics.roc_auc_score(y_test, y_pred_prob)
print cross_val_score(logreg, X, y, cv=10, scoring='roc_auc').mean()


I am working on the file "FInal_Project/Quantifying_Influence_Analysis_maxrose_DSFinal.ipynb".
The first task is:
I have chosen 6 features for the RFC model after running a looped evaluation of the maximum features for the model using cross validation.
Can you write Python code for it?

# plot max_features (x-axis) versus Accuracy score (y-axis)
plt.plot(feature_range, Acc_scores)
plt.xlabel('max_features')
plt.ylabel('Accuracy (higher is better)')

rfclass = RandomForestClassifier(n_estimators=175,  max_features=6,oob_score=True, random_state=50)
rfclass.fit(X, y)
print rfclass.oob_score_
print cross_val_score(rfclass, X, y, cv=10, scoring='accuracy').mean()


I am working on the file "FInal_Project/Quantifying_Influence_Analysis_maxrose_DSFinal.ipynb".
The first task is:
It appears that a pretty large percentage of the artist's catalogs have been covered at least once.<br>
90% of Beatles songs have been covered, 66% of Rolling Stones. By this measure, The Liverpuddlians may be deemed "more influential".
Can you write Python code for it?

# Throw out covers recorded by each band and see what percentage of their catalogs have been covered.
inf = data[(data.is_cover == 0) & (data.is_covered == 1)].groupby('artist').workid.count()/data[(data.is_cover == 0)].groupby('artist').workid.count()
inf.plot(kind='bar', figsize=(16,5),fontsize=18,rot=40)

# Create new DataFrame with Top 10 most covered songs for each band.
top10 = data[(data.artistid == 1) & (data.is_cover ==0)][['artist','songname','minreleasedate', 'is_cover','times_covered']].sort_values(by='times_covered',ascending=False)[:10]
top10 = top10.append(data[(data.artistid == 0) & (data.is_cover ==0)][['artist','songname','minreleasedate', 'is_cover','times_covered']].sort_values(by='times_covered',ascending=False)[:10])


I am working on the file "FInal_Project/Quantifying_Influence_Analysis_maxrose_DSFinal.ipynb".
The first task is:
The Beatles catalog essentialy ends in 1970 when they disbanded (the outliers are most likely bad release dates in my data). The Rolling Stones continue into this century. However, their years of greatest influence are similar, spanning the first 8 years of releases.

Test Logistic Regression Model
Can you write Python code for it?


I am working on the file "FMonSpark_demo_a9a.ipynb".
The first task is:
Load one of the saved model
Can you write Python code for it?

model = loadModel('a9a_iteration_5')

evaluate(test, model)


I am working on the file "FMonSpark_demo_a9a.ipynb".
The first task is:
Predictions
To calculate the probabilities according to the model for a test set, call predict(data, model). This return a RDD with probability scores.
Can you write Python code for it?

prediction = predictFM(test, bestModel)
prediction.take(5)


I am working on the file "FMonSpark_demo_a9a.ipynb".
The first task is:
Set some matplotlib parameters for plotting figures directly in the notebook:
Can you write Python code for it?

%matplotlib inline
import matplotlib.pylab as pylab
pylab.rcParams['figure.figsize']=(16.0, 12.0)


I am working on the file "FMonSpark_demo_a9a.ipynb".
The first task is:
Training
trainFM_parallel_sgd
Train a Factorization Machine model using parallel stochastic gradient descent.
Can you write Python code for it?

?trainFM_parallel_sgd

temp = time.time()
model = trainFM_parallel_sgd (sc, trainAll, iterations=1, iter_sgd= 1, alpha=0.01, regParam=0.01, factorLength=4,\
                      verbose=False, savingFilename = None, evalTraining=None)
print 'time :'; print  time.time()-temp;


I am working on the file "FMonSpark_demo_a9a.ipynb".
The first task is:
evalTraining
Used to plot the evaluation (train+validation) during the training
You need to create an instance of the class evaluation first.
You can set evalTraining.modulo = 5 to evaluate the model after each 5 iterations for example (default is 1)
Can you write Python code for it?

temp = time.time()

evalTraining = evaluation(trainAll)
evalTraining.modulo = 1
trainFM_parallel_sgd(sc, trainAll, iterations=10, iter_sgd=1, verbose = True, evalTraining=evalTraining)
print 'total time'
print  time.time()-temp


I am working on the file "FMonSpark_demo_a9a.ipynb".
The first task is:
evaluate(test, model)
Evaluate your model on a test set.
Can you write Python code for it?


I am working on the file "FMonSpark_demo_a9a.ipynb".
The first task is:
plotRegParam(sc, data, regParam_list, iterations, iter_sgd, alpha, factorLength)
Specify the list of factor length you want to plot in the factorLength_list
Can you write Python code for it?

model = plotRegParam(sc, trainSample, regParam_list = [0, 0.0001, 0.001, 0.01], iterations=5, iter_sgd=1, alpha=0.01, factorLength=4)


I am working on the file "FMonSpark_demo_a9a.ipynb".
The first task is:
verbose=True
Split the rdd into a training set and a validation set.
Print the evaluation after each iteration.
savingFilename (string)
Saves the model in a pickle file after each iteration. The files are saved in the current directory.
The files are named 'savingFilename_iteration_#'
Can you write Python code for it?

temp = time.time()
trainFM_parallel_sgd (sc, trainAll, iterations=5, iter_sgd= 3, alpha=0.01, regParam=0.01, factorLength=4,\
                      verbose=True, savingFilename = 'a9a', evalTraining=None)
print 'total time :'; print  time.time()-temp;


I am working on the file "FOSS4G_Hokkaido.ipynb".
The first task is:
3.3 Smoothing data
reference: https://code.earthengine.google.com/a675608eb96f135024b0b2185f3889ee
Can you write Python code for it?

## This field contains UNIX time in milliseconds.
timeField = 'system:time_start'
join = ee.Join.saveAll('images')

interval = 72 ##72 days

##ee.Filter.maxDifference:
##Creates a unary or binary filter that passes if the left and right operands, both numbers, are within a given maximum difference. If used as a join condition, this numeric difference is used as a join measure.
diffFilter =  ee.Filter.maxDifference(difference = (1000 * 60 * 60 * 24) * interval,
                                      leftField = timeField,
                                      rightField = timeField)

NeighborJoin = join.apply(primary = filteredMODIS,
                               secondary = filteredMODIS,
                               condition = diffFilter)

def smooth_func(image):
    collection = ee.ImageCollection.fromImages(image.get('images'))
    return ee.Image(image).addBands(collection.mean().rename(['smooth']))

smoothed = ee.ImageCollection(NeighborJoin.map(smooth_func))


I am working on the file "FOSS4G_Hokkaido.ipynb".
The first task is:
4.2 Calculate annual NDVI average
Can you write Python code for it?

y = 0
MODcoll = smoothed \
        .filterDate(starts[y], ends[y]) \
        .sort('system:time_start') \
        .select('smooth')
start = starts[y]
end = ends[y]
avg = MODcoll.mean().rename([years[y]])

for y in range(1, 16):
    MODcoll = smoothed \
        .filterDate(starts[y], ends[y]) \
        .sort('system:time_start') \
        .select('smooth')
    start = starts[y]
    end = ends[y]
    average = MODcoll.mean()
    avg = avg.addBands(average.rename([years[y]]))

info_bands = avg.getInfo()['bands']
#print('Dimensions:', info_bands[0]['dimensions'])
print('Number of bands:', len(info_bands))
##see band names
for ids in range(0,len(info_bands),1):
    print(info_bands[ids]['id'])


I am working on the file "FOSS4G_Hokkaido.ipynb".
The first task is:
4.5 Export Geotiff
Export the outputs to Google Drive.
It takes some time.
see: https://stackoverflow.com/questions/39219705/how-to-download-images-using-google-earth-engines-python-api
Can you write Python code for it?

globalfilter = ee.Geometry.Polygon([-170, 80, 0, 80, 170, 80, 170, -80, 10, -80, -170, -80])
globalfilter = globalfilter['coordinates'][0]

task_config = {
    'description': 'imageToDriveExample',
    'scale': 231.65635826395828,
    'region': globalfilter,
    'maxPixels': 5e10
    }

task = ee.batch.Export.image(mk_tau, 'kenn_0116', task_config)
task.start()

time.sleep(10)
while task.status()['state'] == 'RUNNING':
    print 'Running...'
    time.sleep(100)
print 'Done.', task.status()


I am working on the file "FOSS4G_Hokkaido.ipynb".
The first task is:
See a map of whole mean NDVI in 2001-2016.
Note that all data is in GEE server, not in this running environment.
Can you write Python code for it?

regionfilter = ee.Geometry.Polygon([-170, 80, 0, 80, 170, 80, 170, -80, 10, -80, -170, -80]).toGeoJSON()

ndvi_palette = 'FFFFFF, CE7E45, DF923D, F1B555, FCD163, 99B718, 74A901, 66A000, 529400, 3E8601, 207401, 056201, 004C00, 023B01, 012E01, 011D01, 011301'
vizParams = {'min': -2000,
             'max': 10000,
             'region':regionfilter,
             'palette': ndvi_palette}

img = smoothed.select('smooth').mean()
%config InlineBackend.figure_format = 'retina'
print(img.getThumbUrl(vizParams))
Image(url=img.getThumbUrl(vizParams), width=900, unconfined=True)