I am working on the file "2017-12-04-DIANA-HEP/notebooks/bqplot.ipynb".
The first task is:
Moving points around
Can you write Python code for it?

from bqplot import *

size = 100
np.random.seed(0)
x_data = range(size)
y_data = np.cumsum(np.random.randn(size) * 100.0)

## Enabling moving of points in scatter. Try to click and drag any of the points in the scatter and
## notice the line representing the mean of the data update

sc_x = LinearScale()
sc_y = LinearScale()

scat = Scatter(x=x_data[:10], y=y_data[:10], scales={'x': sc_x, 'y': sc_y}, default_colors=['blue'],
               enable_move=True)
lin = Lines(scales={'x': sc_x, 'y': sc_y}, stroke_width=4, line_style='dashed', colors=['orange'])
m = Label(value='Mean is %s'%np.mean(scat.y))

def update_line(change):
    with lin.hold_sync():
        lin.x = [np.min(scat.x), np.max(scat.x)]
        lin.y = [np.mean(scat.y), np.mean(scat.y)]
        m.value='Mean is %s'%np.mean(scat.y)

update_line(None)

# update line on change of x or y of scatter
scat.observe(update_line, names='x')
scat.observe(update_line, names='y')

ax_x = Axis(scale=sc_x)
ax_y = Axis(scale=sc_y, tick_format='0.2f', orientation='vertical')

fig = Figure(marks=[scat, lin], axes=[ax_x, ax_y])

## In this case on drag, the line updates as you move the points.
with scat.hold_sync():
    scat.enable_move = True
    scat.update_on_move = True
    scat.enable_add = False

display(m, fig)


I am working on the file "2017-12-04-DIANA-HEP/notebooks/bqplot.ipynb".
The first task is:
Scatter Plot
Can you write Python code for it?

plt.figure(title='Scatter Plot with colors')
plt.scatter(y_data_2, y_data_3, color=y_data)
plt.show()


I am working on the file "2017-12-04-DIANA-HEP/notebooks/bqplot.ipynb".
The first task is:
Selections
Can you write Python code for it?

def interval_change_callback(change):
    db.value = str(change['new'])

intsel = bq.interacts.FastIntervalSelector(scale=xs, marks=[line])
intsel.observe(interval_change_callback, names=['selected'] )

db = widgets.Label()
db.value = str(intsel.selected)
display(db)

fig = bq.Figure(marks=[line], axes=[xax, yax], animation_duration=1000, interaction=intsel)
display(fig)

line.selected


I am working on the file "2017-12-04-DIANA-HEP/notebooks/bqplot.ipynb".
The first task is:
The same holds for the attributes of scales, axes
Can you write Python code for it?

xs.min = 4

xs.min = None

xax.label = 'Some label for the x axis'


I am working on the file "2017-12-04-DIANA-HEP/notebooks/bqplot.ipynb".
The first task is:
Use bqplot figures as input widgets
Can you write Python code for it?

xs = bq.LinearScale()
ys = bq.LinearScale()
x = np.arange(100)
y = np.cumsum(np.random.randn(2, 100), axis=1) #two random walks

line = bq.Lines(x=x, y=y, scales={'x': xs, 'y': ys}, colors=['red', 'green'])
xax = bq.Axis(scale=xs, label='x', grid_lines='solid')
yax = bq.Axis(scale=ys, orientation='vertical', tick_format='0.2f', label='y', grid_lines='solid')


I am working on the file "2017.08_scirep_rev1/2_benchmark_method/kaiju/kaiju_greedy/kaiju_greedy_summarize_virus_hits.ipynb".
The first task is:
Combine similar annotations into a single 'clade'
E.g Enterobacteria phage phi92 and phi91 should be grouped
Can you write Python code for it?

editdist_sp = [ (sp1,sp2,editdistance.eval(sp1,sp2)) for sp1,sp2 in itertools.combinations(read_annot["species_fillna"].unique(),2) ]
editdist_df = pd.DataFrame.from_records(editdist_sp,columns=["sp1","sp2","edit_distance"])
editdist_df["similarity"] = editdist_df.apply(lambda r: (max(len(r["sp1"]),len(r["sp2"]))-r["edit_distance"])/ max(len(r["sp1"]),len(r["sp2"])),axis=1)
editdist_df.head()

cluster_membership = {}
clusters = []

#Single linkage clustering
for _,row in editdist_df[editdist_df.similarity > 0.65 ].sort_values("similarity",ascending=False).iterrows():
    if row["sp1"] not in cluster_membership and row["sp2"] not in cluster_membership:
        #Create new cluster and add them both
        clusters.append([row["sp1"],row["sp2"]])
        cluster_membership[row["sp1"]] = len(clusters)-1
        cluster_membership[row["sp2"]] = len(clusters)-1
    elif row["sp1"] in cluster_membership:
        if row["sp2"] not in cluster_membership:
            #Add sp2 to sp1 cluster
            clusters[cluster_membership[row["sp1"]]].append(row["sp2"])
            cluster_membership[row["sp2"]] = cluster_membership[row["sp1"]]
        else:
            if cluster_membership[row["sp1"]] != cluster_membership[row["sp2"]]:
                #Combine clusters!
                c1_pos = cluster_membership[row["sp1"]]
                c2_pos = cluster_membership[row["sp2"]]
                clusters[c1_pos] += clusters[c2_pos]
                for c in clusters[c2_pos]:
                    cluster_membership[c] = c1_pos
                clusters[c2_pos] = None
    else:
        #Row 2 is in cluster and row1 is not
        clusters[cluster_membership[row["sp2"]]].append(row["sp1"])
        cluster_membership[row["sp1"]] = cluster_membership[row["sp2"]]

cluster_names = [";".join(c) if c else None for c in clusters]


I am working on the file "2017.08_scirep_rev1/2_benchmark_method/kaiju/kaiju_greedy/kaiju_greedy_summarize_virus_hits.ipynb".
The first task is:
Use new clade groups
Can you write Python code for it?

read_annot["sp_group"] = read_annot["species_fillna"].apply(lambda x: cluster_names[cluster_membership[x]] if x in cluster_membership and "unclassified" not in x else x)
df_counts = read_annot.groupby("sp_group").size().reset_index()
df_counts.columns = ["species","read_count"]
print(df_counts[df_counts.read_count >= 2].shape)
df_counts[df_counts.read_count >= 2].sort_values("read_count",ascending=False)

virus_count_histogram = df_counts[df_counts.read_count >= 3].groupby("read_count").size().reset_index()
sns.barplot(x=virus_count_histogram["read_count"],y=virus_count_histogram[0])

df_counts[df_counts.read_count >= 2].sort_values("read_count",ascending=False).to_csv("454_seqs_kaiju_greedy.filt_species.tsv",sep="\t",index=False)

!ls


I am working on the file "2017.08_scirep_rev1/2_benchmark_method/kaiju/kaiju_mem/kaiju_mem_summarize_virus_hits.ipynb".
The first task is:
Use new clade groups
Can you write Python code for it?

read_annot["sp_group"] = read_annot["species_fillna"].apply(lambda x: cluster_names[cluster_membership[x]] if x in cluster_membership and "unclassified" not in x else x)
df_counts = read_annot.groupby("sp_group").size().reset_index()
df_counts.columns = ["species","read_count"]
df_counts[df_counts.read_count >= 3].sort_values("read_count",ascending=False)

virus_count_histogram = df_counts[df_counts.read_count >= 3].groupby("read_count").size().reset_index()
sns.barplot(x=virus_count_histogram["read_count"],y=virus_count_histogram[0])

df_counts[df_counts.read_count >= 3].sort_values("read_count",ascending=False).to_csv("454_seqs_kaiju.filt_species.tsv",sep="\t",index=False)

!ls


I am working on the file "2017/05-naive-bayes/Naive_Bayes_Francisco_Carlos.ipynb".
The first task is:
Questão 2
Can you write Python code for it?

from sklearn.naive_bayes import MultinomialNB, GaussianNB
from sklearn.preprocessing import LabelEncoder

#Transforma os valores string em númericos
#Na minha versão foi feito o tratamento de todos os valores sem precisar converter, porém
#a função do scikit não é adaptado para valores strings
for i in range(0, df_train_sci.shape[1]):
    df_train_sci.iloc[:,i] = LabelEncoder().fit_transform(df_train_sci.iloc[:,i])

for i in range(0, df_test_sci.shape[1]):
    df_test_sci.iloc[:,i] = LabelEncoder().fit_transform(df_test_sci.iloc[:,i])

#Separa em features e classes
train_X_sci =  df_train_sci.iloc[:,:-1]
test_X_sci =  df_test_sci.iloc[:,:-1]
train_y_sci = df_train_sci.iloc[:,-1]
test_y_sci = df_test_sci.iloc[:,-1]

#Faz o fit e o predict com a implementação do scikit
nb = GaussianNB()
nb.fit(train_X_sci, train_y_sci)
result_y_sci = nb.predict(test_X_sci)
correct_pct_sci = accuracy_score(test_y_sci, result_y_sci)

print('Porcentagem de acerto = {0}'.format(correct_pct_sci))

print("\nClassification Report:")
print(classification_report(y_true=test_y_sci, y_pred=result_y_sci, target_names=["unacc", "acc", "good", "vgood"]))


I am working on the file "2017/05-naive-bayes/Naive_Bayes_Francisco_Carlos.ipynb".
The first task is:
Questão 3
Can you write Python code for it?

print('Resultado no meu algoritmo: {0}'.format(correct_pct))
print('Resultado no algoritmo do sklearn: {0}'.format(correct_pct_sci))


I am working on the file "2017/05-naive-bayes/Naive_Bayes_Tutorial_01.ipynb".
The first task is:
1.2 Dividir Arquivo
Em seguida, precisamos dividir os dados em um conjunto de dados de treinamento, o qual possa ser usado pelo Naive Bayes para fazer previsões e um conjunto de dados de teste para que possamos usar para avaliar a precisão do modelo. Precisamos dividir o conjunto de dados aleatoriamente em treino e teste, em conjuntos de dados com uma proporção de 67% de treinamento e 33% de teste (esta é uma razão comum para testar um algoritmo em um conjunto de dados).
Abaixo está a função splitDataset () que dividirá um determinado conjunto de dados em uma proporção de divisão determinada.
Can you write Python code for it?

import random
def splitDataset(dataset, splitRatio):
    trainSize = int(len(dataset) * splitRatio)
    trainSet = []
    copy = list(dataset)
    while len(trainSet) < trainSize:
        index = random.randrange(len(copy))
        trainSet.append(copy.pop(index))
    return [trainSet, copy]


I am working on the file "2017/05-naive-bayes/Naive_Bayes_Tutorial_01.ipynb".
The first task is:
2.3 Sumarizar Atributos por classes
Podemos juntar tudo ao separar nosso conjunto de dados de treinamento em instâncias agrupadas por classe, usando a função summarizeByClass()
Can you write Python code for it?

def summarizeByClass(dataset):
    separated = separateByClass(dataset)
    summaries = {}
    for classValue, instances in separated.items():
        summaries[classValue] = summarize(instances)
    return summaries


I am working on the file "2017/05-naive-bayes/Naive_Bayes_Tutorial_01.ipynb".
The first task is:
3.2 Calcular probabilidades das classes
Neste momento, podemos calcular a probabilidade de um atributo pertencente a uma classe, podemos combinar as probabilidades de todos os valores dos atributos para uma instância de dados e apresentar uma probabilidade de toda a instância de dados pertencente à classe.
Combinamos as probabilidades juntas, multiplicando-as. Na função calculateClassProbabilities () abaixo, a probabilidade de uma determinada instância de dados é calculada multiplicando as probabilidades de atributo para cada classe. O resultado é um mapa de valores de classe para probabilidades.
Can you write Python code for it?

def calculateClassProbabilities(summaries, inputVector):
    probabilities = {}
    for classValue, classSummaries in summaries.items():
        probabilities[classValue] = 1
        for i in range(len(classSummaries)):
            mean, stdev = classSummaries[i]
            x = inputVector[i]
            probabilities[classValue] *= calculateProbability(x, mean, stdev)
    return probabilities


I am working on the file "2017/05-naive-bayes/Naive_Bayes_Tutorial_01.ipynb".
The first task is:
3.3 Fazer a Predição
Agora podemos calcular a probabilidade de uma instância de dados pertencente a cada valor de classe, podemos procurar a maior probabilidade e retornar a classe associada.
A função predict() realiza esta tarefa.
Can you write Python code for it?

def predict(summaries, inputVector):
    probabilities = calculateClassProbabilities(summaries, inputVector)
    bestLabel, bestProb = None, -1
    for classValue, probability in probabilities.items():
        if bestLabel is None or probability > bestProb:
            bestProb = probability
            bestLabel = classValue
    return bestLabel


I am working on the file "2017/05-naive-bayes/Naive_Bayes_Tutorial_01.ipynb".
The first task is:
3.4 Fazer várias predições
Finalmente, podemos estimar a precisão do modelo fazendo previsões para cada instância de dados em nosso conjunto de dados de teste. A função getPredictions () realizará esta tarefa e retornará uma lista de previsões para cada instância de teste.
Can you write Python code for it?

def getPredictions(summaries, testSet):
    predictions = []
    for i in range(len(testSet)):
        result = predict(summaries, testSet[i])
        predictions.append(result)
    return predictions


I am working on the file "2017/05-naive-bayes/Naive_Bayes_Tutorial_01.ipynb".
The first task is:
Exercicio 12
Junte todo o código acima, crie uma função main e execute a predição para o dataset "pima-indians-diabetes.data", observando a acurácia obtida. Execute várias vezes e analise a variação da acurária ao longo dessas execuções.
Can you write Python code for it?


I am working on the file "2017/05-naive-bayes/Naive_Bayes_Tutorial_01.ipynb".
The first task is:
Exercicio 2
Teste esta função definindo um dataset mockado com 5 instancias, divida este arquivo em trainamento e teste. Imprima os conjuntos de treinamento e teste gerados, por exemplo, imprimindo:
"Dividiu arquivo com 5 linhas em arquivo de treino com [[2], [5], [4]] e de teste com [[1], [3]]"
Can you write Python code for it?


I am working on the file "2017/05-naive-bayes/Naive_Bayes_Tutorial_01.ipynb".
The first task is:
Exercicio 3
Teste este função com alguns exemplos de dados sintéticos e imprima as classes separadas com seus respectivas instancias. Perceba no exemplo acima que a classe se refere ao último elemento do vetor. Segue um exemplo de saída:
"Instancias separadas por classes: {1: [[1, 20, 1], [3, 22, 1]], 0: [[2, 21, 0]]}"
Can you write Python code for it?


I am working on the file "2017/05-naive-bayes/Naive_Bayes_Tutorial_01.ipynb".
The first task is:
Exercicio 4
Crie alguns dados fictícios e teste as funções criadas.Exemplo:
"Cálculo de [1, 2, 3, 4, 5]: média=3.0, stdev=1.5811388300841898"
Can you write Python code for it?


I am working on the file "2017/05-naive-bayes/Naive_Bayes_Tutorial_01.ipynb".
The first task is:
Exercicio 5
Crie alguns dados fictícios e teste as funções criadas.Exemplo de saída:
"Sumário dos atributos: [(2.0, 1.0), (21.0, 1.0)]"
Can you write Python code for it?


I am working on the file "2017/05-naive-bayes/Naive_Bayes_Tutorial_01.ipynb".
The first task is:
Exercicio 6
Teste a função acima, usando um pequeno conjunto de dados. Exemplo de saída:
Resumo por classe: {1: [(2.0, 1.4142135623730951), (21.0, 1.4142135623730951)], 0: [(3.0, 1.4142135623730951), (21.5, 0.7071067811865476)]}
Can you write Python code for it?


I am working on the file "2017/05-naive-bayes/Naive_Bayes_Tutorial_01.ipynb".
The first task is:
Exercicio 7
Teste a função acima, usando um pequeno conjunto de dados. Exemplo de saída:
"Probabilidade para pertencer a esta classe: 0.06248965759370005"
Can you write Python code for it?


I am working on the file "2017/05-naive-bayes/Naive_Bayes_Tutorial_01.ipynb".
The first task is:
Exercicio 9
Teste a função acima, usando um pequeno conjunto de dados. Exemplo de saída:
"Entrada {'A': [(1, 0.5)], 'B': [(20, 5.0)]} Consulta [1.1, '?'] Predição: Classe A"
Can you write Python code for it?


I am working on the file "2017/05-naive-bayes/naiveBayes_otacilio_bezerra.ipynb".
The first task is:
Questão 2
Can you write Python code for it?

# Carregando o Dataset
data = pd.read_csv("car.data", header=None)

# Transformando as variáveis categóricas em valores discretos contáveis
# Apesar de diminuir a interpretação dos atributos, essa medida facilita bastante a vida dos métodos de contagem
for i in range(0, data.shape[1]):
    data.iloc[:,i] = LabelEncoder().fit_transform(data.iloc[:,i])

# Separação do Conjunto de Treino e Conjunto de Teste (80%/20%)
X_train, X_test, y_train, y_test = train_test_split(data.iloc[:,:-1], data.iloc[:,-1], test_size=0.2)

# Implementação do Naive Bayes Multinomial do SKLearn
# O Naive Bayes Multinomial é a implementação do SKLearn que funciona para variáveis discretas,
# ao invés de utilizar o modelo da função gaussiana (a classe GaussianNB faz dessa forma)
clf = MultinomialNB()
clf.fit(X_train.values, y_train.values)

y_pred = clf.predict(X_test.values)

# Impressão dos Resultados
print("Multinomial Naive Bayes (SKlearn Version)")
print("Total Accuracy: {}%".format(accuracy_score(y_true=y_test, y_pred=y_pred)))

print("\nClassification Report:")
print(classification_report(y_true=y_test, y_pred=y_pred, target_names=["unacc", "acc", "good", "vgood"]))


I am working on the file "2017/05-naive-bayes/naiveBayes_otacilio_bezerra.ipynb".
The first task is:
Questão 3
Can you write Python code for it?

# Utilização da minha função própria de Naive Bayes para o mesmo conjunto de dados
nBayes = NaiveBayes()
nBayes.fit(X_train.values, y_train.values)
y_pred = nBayes.predict(X_test.values)

# Impressão dos Resultados
print("Naive Bayes (My Version :D)")
print("Total Accuracy: {}%".format(accuracy_score(y_true=y_test, y_pred=y_pred)))

print("\nClassification Report:")
print(classification_report(y_true=y_test, y_pred=y_pred, target_names=["unacc", "acc", "good", "vgood"]))


I am working on the file "2017/05-naive-bayes/nb_abelardo_mota.ipynb".
The first task is:
Analisando um caso em que ambos classificaram como muito bom quando era muito ruim
Can you write Python code for it?

predictions[(predictions.model_2 == 'vgood') & (predictions.model_1 == 'vgood') & (predictions.correct == "unacc")]

# observar que a frequência de exemplos da classe vgood é bem menor que da classe unacc
df_train.groupby(6).size()


I am working on the file "2017/05-naive-bayes/nb_abelardo_mota.ipynb".
The first task is:
Analisando um caso em que ambos classificaram como muito ruim quando era muito bom
Can you write Python code for it?

predictions[(predictions.model_2 == 'unacc') & (predictions.model_1 == 'unacc') & (predictions.correct == "vgood")]


I am working on the file "2017/05-naive-bayes/nb_abelardo_mota.ipynb".
The first task is:
Analisando um caso em que model_1 classificou como muito bom e model_2 classificou como muito ruim
Can you write Python code for it?

predictions[(predictions.model_1 == 'vgood') & (predictions.model_2 == 'unacc')].sample()

probs_1.loc[1662]

probs_2.loc[1662]

# ter a feature lug_boot = big parece contribuir bastante para a probabilidade do modelo 1
#    dado que a frequência de lug_boot = big na classe vgood é bastante alta(68%)
df_train.groupby([6, 4]).size().unstack().div(df_train.groupby(6).size(), axis=0).sort_values('big', ascending=False)


I am working on the file "2017/05-naive-bayes/nb_abelardo_mota.ipynb".
The first task is:
Analisando um caso em que model_2 classificou como muito bom e model_1 classificou como muito ruim
Can you write Python code for it?

predictions[(predictions.model_2 == 'vgood') & (predictions.model_1 == 'unacc')]


I am working on the file "2017/05-naive-bayes/nb_abelardo_mota.ipynb".
The first task is:
Modelo utilizando frequência dos valores das features
Can you write Python code for it?

# leitura do arquivo
df = pd.read_csv("carData.csv", header=None)

# índice das colunas de features
f_cols = list(range(0, 6))

# frequência de registros por classe
freq_classes = df.groupby(6).size()

# classes
classes = df[6].unique()

# probabilidade atribuída às combinações de feature/classe
#    que não possuem exemplos no dataset
DEFAULT_NO_FREQ_PROB = 0.01

def fit(df):
    # probabilidades estimadas como #(feature, classe)/#classe
    base_probs = {}

    # para cada feature
    for f_col in f_cols:

        # matriz de probabilidades da feature x classes
        #              agrupa pela classe e pela feature
        #              manda índice da feature para as colunas
        #              divide pela frequência das classes
        #              preenche os missing values(pares não presentes no dataset)
        base_probs[f_col] = df.groupby([6, f_col]).size()\
                         .unstack() \
                         .div(freq_classes, axis=0)\
                         .fillna(DEFAULT_NO_FREQ_PROB)

    # função que realiza a predição sobre um dataset
    #    a partir das probabilidades estimadas no passo anterior
    def predict(ds):

        predictions = []
        probs = []
        # para cada exemplo
        for ix, row in ds.iterrows():
            prob = {}
            # para cada classe
            for classe in classes:
                # calcula a probabilidade do exemplo estar associado à classe
                prob[classe] = 1.
                # como o produto das probabilidades de cada uma de suas features pertencer à classe
                for f_col in f_cols:
                    prob[classe] = prob[classe] * base_probs[f_col].loc[classe, str(row[f_col])]

            predictions.append(max(prob.keys(), key=lambda key: prob[key]))
            probs.append(prob)

        return probs, predictions

    return predict

from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score

accs = []

for x in range(100):

    #...


I am working on the file "2017/05-naive-bayes/nb_abelardo_mota.ipynb".
The first task is:
O que deu de diferente?
Can you write Python code for it?


I am working on the file "2017/05-naive-bayes/nb_abelardo_mota.ipynb".
The first task is:
Questão 2
Crie uma versão de sua implementação usando as funções disponíveis na biblioteca SciKitLearn para o Naive Bayes (veja aqui)
Can you write Python code for it?

from sklearn.naive_bayes import GaussianNB
import numpy as np
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import confusion_matrix

ds = loadCsv("carData.csv")
df = pd.DataFrame(ds, columns=range(7))

X = df[list(range(6))]
y = df[6]

accs = []

for x in range(100):

    # particiona dataset em treino(77%) e teste(33%)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.33)

    nb = GaussianNB()
    # prediz
    nb.fit(X_train, y_train)

    # calcula acurácia
    acc = accuracy_score(y_test, nb.predict(X_test))
    accs.append(acc)

pd.Series(accs).describe().to_frame().T


I am working on the file "2017/05-naive-bayes/resp_naive_bayes_isaclira.ipynb".
The first task is:
Questão 2
Crie uma versão de sua implementação usando as funções disponíveis na biblioteca SciKitLearn para o Naive Bayes (veja aqui)
Can you write Python code for it?

# Transform as features categóricas em numéricas

from sklearn.naive_bayes import GaussianNB
from sklearn.preprocessing import LabelEncoder

le = LabelEncoder()
le.fit(xtrain['class'])
ytrain = le.transform(xtrain['class'])
ytest = le.transform(xtest['class'])

xtrain =pd.get_dummies(xtrain.drop(['class'],axis=1))
xtest = pd.get_dummies(xtest.drop(['class'],axis=1))

xtrain.head()

# Aplica o classifcador Naive Bayes (sklearn)

nb = GaussianNB()
nb.fit(xtrain,ytrain)
pred = nb.predict(xtest)
acc_sk_nb = np.mean(pred == ytest)*100


I am working on the file "2017/05-naive-bayes/resp_naive_bayes_isaclira.ipynb".
The first task is:
Questão 3
Analise a acurácia dos dois algoritmos e discuta a sua solução.
Can you write Python code for it?

print('Acurária do classificador(from scratch):',acc_my_nb)
print('Acurária do classificador(sklearn version):',acc_sk_nb)


I am working on the file "2017/05-naive-bayes/resp_naive_bayes_moesio.ipynb".
The first task is:
Questão 2
Can you write Python code for it?


I am working on the file "2017/05-naive-bayes/resp_naive_bayes_moesio.ipynb".
The first task is:
Questão 3
Can you write Python code for it?


I am working on the file "2017/06-linear-regression/Linear_Regression_Tutorial.ipynb".
The first task is:
2.2 Calcular Covariância
A covariância de dois grupos de números descreve como esses números mudam juntos. A co-variância é uma generalização da correlação. A correlação descreve a relação entre dois grupos de números, enquanto a covariância pode descrever a relação entre dois ou mais grupos de números. Além disso, a covariância pode ser normalizada para produzir um valor de correlação. No entanto, podemos calcular a covariância entre duas variáveis da seguinte forma:

Abaixo está uma função chamada covariance() que implementa esta estatística. Esta função baseia-se no passo anterior e leva as listas de valores x e y, bem como a média desses valores como argumentos.
Can you write Python code for it?

# Calculate covariance between x and y
def covariance(x, mean_x, y, mean_y):
  covar = 0.0
  for i in range(len(x)):
    covar += (x[i] - mean_x) * (y[i] - mean_y)
  return covar


I am working on the file "2017/06-linear-regression/Linear_Regression_Tutorial.ipynb".
The first task is:
2.3 Estimativa dos Coeficientes
Agora, devemos estimar os valores dos dois coeficientes em regressão linear simples. O primeiro é B1 que pode ser estimado como:

Podemos simplificar esta fórmula usando as funcões covariance e variance apresentadas acima, conforme a fórmula abaixo.

Em seguida, precisamos estimar um valor para B0, também chamado de interceptação, pois controla o ponto inicial da linha onde ele intersecta o eixo y.

Mais uma vez, sabemos como estimar B1 e temos uma função para estimar a média (). Podemos juntar tudo isso em uma função denominada coefficients () que leva o conjunto de dados como um argumento e retorna os coeficientes.
Can you write Python code for it?

# Calculate coefficients
def coefficients(dataset):
      x = [row[0] for row in dataset]
      y = [row[1] for row in dataset]
      x_mean, y_mean = mean(x), mean(y)
      b1 = covariance(x, x_mean, y, y_mean) / variance(x, x_mean)
      b0 = y_mean - b1 * x_mean
      return [b0, b1]


I am working on the file "2017/06-linear-regression/Linear_Regression_Tutorial.ipynb".
The first task is:
Exercicio 1
(a) Junte as duas funções acima e teste-as em um conjunto de dados pequeno dado. Utiloze como exemplo, o pequeno conjunto de dados de valores x e y.
x |  y
--| -
1 | 1
2 | 3
4 | 3
3 | 2
5 | 5
(b) Em seguida crie um gráfico onde voce plot esses pontos
Can you write Python code for it?

## COLOQUE SEU CODIGO AQUI

## FAÇA O PLOT DOS DADOS AQUI


I am working on the file "2017/06-linear-regression/Linear_Regression_Tutorial.ipynb".
The first task is:
Exercicio 2
Teste o cálculo da covariância no mesmo pequeno conjunto de dados apresentado na
seção anterior.
Can you write Python code for it?


I am working on the file "2017/06-linear-regression/Linear_Regression_Tutorial.ipynb".
The first task is:
Exercicio 3
Estenda o exercício anterior incluindo o cáculo dos coeficientes para os dados sintetizados.
Can you write Python code for it?


I am working on the file "2017/06-linear-regression/Linear_Regression_Tutorial.ipynb".
The first task is:
Exercicio 4
Agora junte tudo que foi criado para fazer previsões para o nosso conjunto de dados de teste.
Can you write Python code for it?


I am working on the file "2017/06-linear-regression/Linear_Regression_Tutorial.ipynb".
The first task is:
Para avaliar o modelo
 adicionaremos uma função para gerenciar a avaliação das previsões denominadas evaluate_algorithm () e outra função para estimar o erro quadrático médio da raiz das previsões denominadas métrica rmse_metric (). Veja as funções abaixo:
Can you write Python code for it?

from math import sqrt

# Calculate root mean squared error
def rmse_metric(actual, predicted):
  sum_error = 0.0
  for i in range(len(actual)):
    prediction_error = predicted[i] - actual[i]
    sum_error += (prediction_error ** 2)
  mean_error = sum_error / float(len(actual))
  return sqrt(mean_error)

# Evaluate regression algorithm on training dataset
def evaluate_algorithm(dataset, algorithm):
  test_set = list()
  for row in dataset:
    row_copy = list(row)
    row_copy[-1] = None
    test_set.append(row_copy)
  predicted = algorithm(dataset, test_set)
  print(predicted)
  actual = [row[-1] for row in dataset]
  rmse = rmse_metric(actual, predicted)
  return rmse


I am working on the file "2017/06-linear-regression/resp_abelardo_mota.ipynb".
The first task is:
Baseline model
Can you write Python code for it?

from sklearn.dummy import DummyRegressor
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import ShuffleSplit

dm = DummyRegressor()
param_grid = {"strategy": ["mean", "median"]}
ss = ShuffleSplit(n_splits=1, test_size=.4, random_state=100)

cv = GridSearchCV(dm, cv=ss, param_grid=param_grid, scoring="neg_mean_squared_error")
cv.fit(df[['r']], df['p'])

cv.best_score_ * -1

cv.best_params_


I am working on the file "2017/06-linear-regression/resp_abelardo_mota.ipynb".
The first task is:
Linear Regression Model
Can you write Python code for it?

from sklearn.linear_model import LinearRegression
import math

ln = LinearRegression()

cv = GridSearchCV(ln, param_grid = {}, cv=ss, scoring="neg_mean_squared_error")
cv.fit(df[['r']], df['p'])

math.sqrt(cv.best_score_ * -1)


I am working on the file "2017/06-linear-regression/resp_linear_regression_isaclira.ipynb".
The first task is:
Como esperado, o modelo de regressão linear explica bem os dados, tendo um valor RSME menor que a predição pelo valor médio de pagamento.
Can you write Python code for it?

# plota as predições vs dados reais e mostra o erro de predição
x = insdf['num_reinv'].values
y = insdf['pag_total'].values

y_hat = lm.intercept + x.dot(lm.coef)

fig,axs = plt.subplots(1,2,figsize=[15,5])
axs[0].plot(x,y_hat,'g-',label='Regression Line',alpha=0.6)
axs[0].scatter(x,y,alpha=0.5,label='Actual Values')
axs[0].set_xlabel('Nº Reinvidicações')
axs[0].set_ylabel('Pagamento Total')
axs[0].legend(loc='best')

error = (y_hat-y)
axs[1].scatter(x,error,alpha=0.5,color='b')
axs[1].axhline(0,color='k')
axs[1].set_xlabel('Nº Reinvidicações')
axs[1].set_ylabel('Erro de predição')


I am working on the file "2017/06-linear-regression/resp_linear_regression_moesio.ipynb".
The first task is:
Algoritmo Baseline
Can you write Python code for it?

df = pd.read_csv("insurance.csv", header=None, names=['n_rei', 'pgtTot'])

dm = DummyRegressor()
param_grid = {"strategy": ["mean", "median"]}
ss = ShuffleSplit(n_splits=1, test_size=.4, random_state=100)

cv = GridSearchCV(dm, cv=ss, param_grid=param_grid, scoring="neg_mean_squared_error")

cv.fit(df[['n_rei']], df['pgtTot'])

cv.best_score_ * -1


I am working on the file "2017/06-linear-regression/resp_linear_regression_moesio.ipynb".
The first task is:
Regressao Linear
Can you write Python code for it?

def mean(values):
    return sum(values) / float(len(values))

def coefficients(dataset):
    x = [row[0] for row in dataset]
    y = [row[1] for row in dataset]
    x_mean, y_mean = mean(x), mean(y)
    b2 = covariance(x, x_mean, y, y_mean) / variance(x, x_mean)
    b1 = y_mean - b2 * x_mean
    return [b1, b2]

def variance(values, mean):
    return sum([(x-mean)**2 for x in values])

def covariance(x, mean_x, y, mean_y):
    covar = 0.0
    for i in range(len(x)):
        covar += (x[i] - mean_x) * (y[i] - mean_y)
    return covar

def str_column_to_float(dataset, column):
    for row in dataset:
        row[column] = float(row[column].strip())

def load_csv(filename):
    dataset = list()
    with open(filename, 'r') as file:
        csv_reader = reader(file)
        for row in csv_reader:
            if not row:
                continue
            dataset.append(row)
    return dataset

def train_test_split(dataset, split):
    train = list()
    train_size = split * len(dataset)
    dataset_copy = list(dataset)
    while len(train) < train_size:
        index = randrange(len(dataset_copy))
        train.append(dataset_copy.pop(index))
    return train, dataset_copy

def rmse_metric(actual, predicted):
    sum_error = 0.0
    for i in range(len(actual)):
        prediction_error = predicted[i] - actual[i]
        sum_error += (prediction_error ** 2)
    mean_error = sum_error / float(len(actual))
    return sqrt(mean_error)

def simple_linear_regression(train, test):
    predictions = list()
    b1, b2 = coefficients(train)
    for row in test:
        yhat = b1 + b2 * row[0]
        predictions.append(yhat)
    return predictions

def evaluate_algorithm(dataset, algorithm, split, *args):
    train, test = train_test_split(dataset, split)
    test_set = list()
    for row in test:
        row_copy = list(row)
        row_copy[-1] = None
        test_set.append(row_copy)
    predicted = algorithm(train, test_set, *args)
    actual = [row[-1] for row in test]
    rmse = rmse_metric(actual, predicted)
 ...


I am working on the file "2017/06-linear-regression/resp_otacilio_bezerra.ipynb".
The first task is:
Utilização da Regressão Linear e Avaliação do Algoritmo
Can you write Python code for it?

data = load_csv("insurance")
[trainData, testData] = split_train_test(data, test_size=0.4)

coef = coefficients(trainData)
predRegre = simple_linear_regression(trainData, testData)
predBase = baseline(trainData, testData)

avalRegre = evaluate_algorithm(trainData, testData, simple_linear_regression)
avalBase = evaluate_algorithm(trainData, testData, baseline)

print("A Regressão Linear Simples obteve os coeficientes: alfa = {:.2f} e beta = {:.2f}".format(coef[0], coef[1]))
print("O erro RMSE desse algoritmo é {:.2f}".format(avalRegre))
print("O erro RMSE para o baseline (média dos pagamentos) é {:.2f}".format(avalBase))

print("\nPrimeiras 10 predições:")
for i in range(10):
    print("Y real: {:.2f} \t| Y previsto: {:.2f} \t | Y baseline: {:.2f}".format(testData[i][-1], predRegre[i], predBase[i]))


I am working on the file "2017/06-linear-regression/resp_rlm_otacilio_bezerra.ipynb".
The first task is:
Carregando o conjunto de dados e utilizando o Gradiente Descendente Estocástico
Can you write Python code for it?

data = pd.read_csv("winequality-white.csv", delimiter=";")

X = MinMaxScaler().fit_transform(data.values[:,:-1])
y = data.values[:,-1]

[coef, errorHist] = stochasticGD(X, y)

print("Gradiente Descendente Estocástico\nRMSE: {}".format(RMSE(y - predict(X, coef, True))))
print("Coeficientes:\n", coef.T)


I am working on the file "2017/06-linear-regression/resp_rlm_otacilio_bezerra.ipynb".
The first task is:
Plotagem do Custo por Época
Can you write Python code for it?


I am working on the file "2017/06-linear-regression/resp_slr_sayonara_lailson.ipynb".
The first task is:
Avaliação do algoritmo de Regressão Linear
Can you write Python code for it?

error_rl, prediction_rl = p.evaluate_algorithm(X_train, y_train, X_test, y_test, p.linear_regression)

red_dot, = plt.plot(X_test, y_test, 'ro')
blue_dot, = plt.plot(X_test, prediction_rl, 'b-')
plt.legend([red_dot, blue_dot], ["Dados de teste reais", "Dados de teste preditados"])
plt.title("Regressão Linear")
print('Erro quadrático: {0:.2f}'.format(error_rl))


I am working on the file "2017/07-decision-tree/decision_tree.ipynb".
The first task is:
Faremos a separação dos dados em conjunto de treino e teste
Can you write Python code for it?

size = len(data)
train_size = int(math.floor(size * 0.7))
train_data = data[:train_size]
test_data = data[train_size:]


I am working on the file "2017/07-decision-tree/decision_tree.ipynb".
The first task is:
No código acima, fizemos a leitura do arquivo informando que não há cabeçalho (obrigatório) e embaralhamos os dados.
A coluna 6 (0-6) representa a classe.
Can you write Python code for it?


I am working on the file "2017/07-decision-tree/decision_tree.ipynb".
The first task is:
Um problema é que nossos atributos categóricos são strings, e a implementção de Decision Tree do scikit-learn só aceita atributos numéricos. Precisamos converter os atributos.
O Pandas possui um tipo de dados categórico ("category") que simplifica essa conversão.
Can you write Python code for it?

for h in headers:
    data[h] = data[h].astype('category')
    data[h] = data[h].cat.codes

data.set_index("class", inplace=True)
data.head()


I am working on the file "2017/07-decision-tree/resp_decision_tree_isaclira.ipynb".
The first task is:
2- Faça o balanceamento dos dados contidos em "train.csv", aplique o algoritmo de Decision Tree e faça a submissão no kaggle. Tente melhorar o resultado obtido em sala de aula (posição 3100 no leaderboard).
Can you write Python code for it?

# Carrega os dados
portodf = pd.read_csv('train/train.csv')
testsdf = pd.read_csv('test/test.csv')

# Visualizando os dados
portodf.head()

portodf.target.value_counts()


I am working on the file "2017/07-decision-tree/resp_decision_tree_isaclira.ipynb".
The first task is:
Dados desbalanceados.
Can you write Python code for it?

# Balanceando os dados
samples = 25000

unbalanced_idx = portodf[portodf.target==0].index
balanced_idx = portodf[portodf.target==1].index

selected_idx1 = set(np.random.choice(unbalanced_idx,samples))
selected_idx = selected_idx1.union(set(balanced_idx))
sampled_portodf = portodf.loc[selected_idx]
#sampled_portodf = pd.DataFrame(sampled_portodf.values,columns=sampled_portodf.columns)

# Dividindo em treino e teste
def split_train(data):
    msk = np.random.rand(len(data))<0.7
    xtrain = data.drop(['target'],axis=1).iloc[msk]
    ytrain = data.target[msk]

    xtest = data.drop(['target'],axis=1).iloc[~msk]
    ytest = sampled_portodf.target[~msk]

    return xtrain,ytrain,xtest,ytest

# Instanciando uma árvore de decisão
from sklearn.metrics import classification_report

xtrain,ytrain,xtest,ytest = split_train(sampled_portodf)
dtree = DecisionTreeClassifier()
dtree.fit(xtrain,ytrain)
predictions = dtree.predict(xtest)
print(classification_report(ytest,predictions))

xtrain,ytrain,xtest,ytest = split_train(sampled_portodf)
dtree = DecisionTreeClassifier()
dtree.fit(xtrain,ytrain)
predictions = dtree.predict(xtest)
print(classification_report(ytest,predictions))


I am working on the file "2017/07-decision-tree/resp_decision_tree_isaclira.ipynb".
The first task is:
Prediction Model
Can you write Python code for it?

# Classifica usando árvode de decisão com criério baseado em entropy/gini

dtree_entropy = DecisionTreeClassifier(criterion = 'entropy')
dtree_entropy.fit(xtrain,ytrain)
entropy_score = dtree_entropy.score(xtest,ytest)
print('Score using entropy criterion:',entropy_score)

dtree_gini = DecisionTreeClassifier(criterion = 'gini')
dtree_gini.fit(xtrain,ytrain)
gini_score = dtree.score(xtest,ytest)
print('Score using gini criterion:',gini_score)

plt.barh(range(len(columns)),dtree_gini.feature_importances_,align='center',alpha=0.4,color='b',label='Gini')
plt.barh(range(len(columns)),dtree_entropy.feature_importances_,align='center',alpha=0.4,color='g',label='Entropy')
plt.yticks(range(len(columns)),cat_features)
plt.xlabel('Feature importance')
plt.ylabel('Feature')
plt.legend(loc='best')
plt.show()


I am working on the file "2017/09-clustering/cl_francisco_carlos.ipynb".
The first task is:
Parte 2
Can you write Python code for it?

# Elbows method
clusters_result = []
for i in range(9):
    km = KMeans(n_clusters=i+1)
    km.fit(df_pca)
    clusters_result.append(km.inertia_)

# Plotagens
plt.figure()
plt.plot(np.arange(1,10),clusters_result,'ro-')
plt.show()

# KMeans clustering
km = KMeans(n_clusters=2)
km.fit(df_pca)
clusters = km.predict(df_pca)

#Plota os dados em duas dimensões
fig,ax = plt.subplots()
ax.scatter(df_pca[:,0], df_pca[:,1], c=clusters)
plt.show()


I am working on the file "2017/09-clustering/cl_francisco_carlos.ipynb".
The first task is:
Parte 3
Can you write Python code for it?

#Utilizar métricas de avaliação de clusteres (completeness e homogeneity)
score_homo = metrics.homogeneity_score(y,clusters)
score_comp = metrics.completeness_score(y,clusters)

print('homogeneity = {0}'.format(score_homo))
print('completeness = {0}'.format(score_comp))


I am working on the file "2017/09-clustering/cl_otacilio_bezerra.ipynb".
The first task is:
2ª Questão
Implementando o Método do Cotovelo
Can you write Python code for it?

# Método do Cotevelo baseado na Inertia (Soma Quadrática da Distância Intra-Cluster de cada Ponto)
numK = np.arange(1,10); inertias = []
for i in numK:
    print(".", end="")
    kmeans.n_clusters = i
    kmeans.fit(X)

    inertias.append(kmeans.inertia_)

# Plotagens
plt.figure()
plt.title("Elbow Method")
plt.xlabel("Num of Clusters"); plt.ylabel("Inertia")
plt.plot(numK, inertias, 'bo-')
plt.show()


I am working on the file "2017/09-clustering/cl_otacilio_bezerra.ipynb".
The first task is:
Aplicação do K-Means
Can you write Python code for it?

# Criamos o objeto da classe KMeans
kmeans = KMeans(n_clusters=2, random_state=0)

# Realizamos a Clusterização
kmeans.fit(X)
clts = kmeans.predict(X)

# Plotando uma visualização 3-Dimensional dos Dados, agora com os clusteres designados pelo K-Means
# Compare a visualização com o gráfico da celula de cima
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
ax.scatter(pcaData[:,0], pcaData[:,1], pcaData[:,2], c=clts, cmap=plt.cm.Dark2)
plt.show()


I am working on the file "2017/09-clustering/cl_otacilio_bezerra.ipynb".
The first task is:
Métricas de Avaliação
Can you write Python code for it?

# Utilizamos três métricas de avaliação dos Clusteres, com base nos dados já classificados:
# -> Homogeneity: porcentagem relativa ao objetivo de ter, em cada cluster, apenas membros de uma mesma classe
# -> Completeness: porcentagem relativa ao objetivo de ter todos os membros de uma classe no mesmo cluster
# -> V-Measure: medida que relaciona Homogeneity com Completeness, e é equivalente à uma métrica conhecida como NMI (Normalized Mutual Information).
homoScore = metrics.homogeneity_score(y, clts)
complScore = metrics.completeness_score(y, clts)
vMeasureScore = metrics.v_measure_score(y, clts)

print("### Avaliação ({0} Clusters) ###".format(kmeans.n_clusters))
print("Homogeneity: \t{0:.3}".format(homoScore))
print("Completeness: \t{0:.3}".format(complScore))
print("V-Measure: \t{0:.3}".format(vMeasureScore))


I am working on the file "2017/09-clustering/cl_otacilio_bezerra.ipynb".
The first task is:
Visualização dos Dados
Can you write Python code for it?

# Plotando uma visualização 3-Dimensional dos Dados
# Podemos observar que os dados (em 3-Dimensões) são extremamente superpostos
pcaData = PCA(n_components=3).fit_transform(X)

fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
ax.scatter(pcaData[:,0], pcaData[:,1], pcaData[:,2], c=y, cmap=plt.cm.Dark2)
plt.show()


I am working on the file "2017/09-clustering/resp_clustering_isaclira.ipynb".
The first task is:
2 - Qual o valor de K (número de clusteres) você escolheu para a questão anterior? Desenvolva o Método do Cotovelo (não utilizar lib!) e descubra o K mais adequado. Após descobrir, aplique novamente o K-means com o K adequado.
Can you write Python code for it?

ks = np.arange(1,10)
homog_metrics = []
comp_metrics = []
for k in ks:
    kmeans = KMeans(n_clusters=k,random_state=0).fit(X)
    ypred = kmeans.predict(X)
    homog_metrics.append(metrics.homogeneity_score(y,ypred))
    comp_metrics.append(metrics.completeness_score(y,ypred))

plt.plot(ks,homog_metrics)
plt.plot(ks,comp_metrics)
plt.xlabel('K')
plt.ylabel('Homogeneity Metric')


I am working on the file "2017/09-clustering/resp_clustering_isaclira.ipynb".
The first task is:
Pelo gráfico, o valor de k ideal é 3, pois balanceia as métricas de completude e homogeneidade. Apartir deste valor, também não há ganhos significativos na métrica de homogeneidade. No mais, o valor é coerente com o número de clusters real do dataset.
Can you write Python code for it?

best_k = 3
predict = KMeans(n_clusters=best_k).fit_predict(X)

hg_kmeans = metrics.homogeneity_score(y,predict)
cs_kmeans = metrics.completeness_score(y,predict)
print('Métrica de homogeneida(KMeans): ',hg_kmeans)
print('Métrica de completude(KMeans): ',cs_kmeans)


I am working on the file "2017/09-clustering/resp_clustering_moesio.ipynb".
The first task is:
Algoritmo Baseline
Can you write Python code for it?

df = pd.read_csv("insurance.csv", header=None, names=['n_rei', 'pgtTot'])

dm = DummyRegressor()
param_grid = {"strategy": ["mean", "median"]}
ss = ShuffleSplit(n_splits=1, test_size=.4, random_state=100)

cv = GridSearchCV(dm, cv=ss, param_grid=param_grid, scoring="neg_mean_squared_error")

cv.fit(df[['n_rei']], df['pgtTot'])

cv.best_score_ * -1


I am working on the file "2017/09-clustering/resp_clustering_moesio.ipynb".
The first task is:
Regressao Linear
Can you write Python code for it?

def mean(values):
    return sum(values) / float(len(values))

def covariance(x, mean_x, y, mean_y):
    covar = 0.0
    for i in range(len(x)):
        covar += (x[i] - mean_x) * (y[i] - mean_y)
    return covar

def variance(values, mean):
    return sum([(x-mean)**2 for x in values])

def coefficients(dataset):
    x = [row[0] for row in dataset]
    y = [row[1] for row in dataset]
    x_mean, y_mean = mean(x), mean(y)
    b1 = covariance(x, x_mean, y, y_mean) / variance(x, x_mean)
    b0 = y_mean - b1 * x_mean
    return [b0, b1]

# funcao para ler o arquivo csv
def load_csv(filename):
    dataset = list()
    with open(filename, 'r') as file:
        csv_reader = reader(file)
        for row in csv_reader:
            if not row:
                continue
            dataset.append(row)
    return dataset

# funcao para tranformar coluna de string em float
def str_column_to_float(dataset, column):
    for row in dataset:
        row[column] = float(row[column].strip())

# funcao para dividir o conjunto de dados em treino e teste
def train_test_split(dataset, split):
    train = list()
    train_size = split * len(dataset)
    dataset_copy = list(dataset)
    while len(train) < train_size:
        index = randrange(len(dataset_copy))
        train.append(dataset_copy.pop(index))
    return train, dataset_copy

def simple_linear_regression(train, test):
    predictions = list()
    b0, b1 = coefficients(train)
    for row in test:
        yhat = b0 + b1 * row[0]
        predictions.append(yhat)
    return predictions

# funcao que calcula o RMSE
def rmse_metric(actual, predicted):
    sum_error = 0.0
    for i in range(len(actual)):
        prediction_error = predicted[i] - actual[i]
        sum_error += (prediction_error ** 2)
    mean_error = sum_error / float(len(actual))
    return sqrt(mean_error)

# funcao para avaliar o algoritmo
def evaluate_algorithm(dataset, algorithm, split, *args):
    train, test = train_test_split(dataset, split)
    test_set = list()
    for row in test:
        row_copy =...


I am working on the file "2017/2017_07_17_High-Performance-Python/high_performance_python.ipynb".
The first task is:
And here's how we use it with Numba:
Can you write Python code for it?

import numpy as np
import numba as nb

import numba.cffi_support
numba.cffi_support.register_module(_gslffilib)
gsl_sf_bessel_j1 = _gslffilib.lib.gsl_sf_bessel_j1

import scipy.special

@nb.jit(nopython=True)
def nb_sum2d_cffi(arr):
    M, N = arr.shape
    result = 0.
    for i in range(M):
        for j in range(N):
            result += gsl_sf_bessel_j1(arr[i,j])
    return result

nb_sum2d_cffi(np.random.random((128,128)))


I am working on the file "2017/2017_07_17_High-Performance-Python/high_performance_python.ipynb".
The first task is:
As we expected, the race condition caused the parallel version to give the wrong answer.  In fact, it doesn't even give a consistent answer from run to run:
Can you write Python code for it?

BAD_parallel_answer2 = BAD_nb_parallel_radial_hist(values, box, bin_edges)
print np.allclose(BAD_parallel_answer, BAD_parallel_answer2)


I am working on the file "2017/2017_07_17_High-Performance-Python/high_performance_python.ipynb".
The first task is:
CFFI
https://bitbucket.org/cffi/cffi
CFFI (C Foreign Function Interface) is a way to call C code from Python.  It's designed as a replacement to tools like SWIG, ctypes, or SciPy.weave (now deprecated).
SWIG is probably still appropriate if you want to call your C code from many languages, not just Python.  But ctypes or CFFI are the preferred Python solutions, with a preference for CFFI.  We'll see why below.
Example: calling a GSL function from Numba
Let's go back to the Numba example where we tried to call scipy.special.j1 but Numba wouldn't allow it in nopython mode.  We'll use the recommended "API-level, out-of-line" mode.  This means the C compiler is invoked, and we have to run a build script beforehand.
We're going to write the script to disk and run it, because this would probably be its own script in a real application.
Can you write Python code for it?

%%writefile build_ffilib.py
if __name__ == '__main__':
    # Compile the FFI lib
    import cffi
    ffibuilder = cffi.FFI()
    ffibuilder.set_source('_gslffilib',
        r"""
            #include <gsl/gsl_sf_bessel.h>  // This gets compiled
         """, libraries=['gsl', 'gslcblas'])
    ffibuilder.cdef("""
            double gsl_sf_bessel_j1 (double x);  // This can be copied straight from the man page and is parsed by CFFI
        """)

    ffibuilder.compile(verbose=True)

!python build_ffilib.py

%ls *ffi*


I am working on the file "2017/2017_07_17_High-Performance-Python/high_performance_python.ipynb".
The first task is:
NumExpr is actually slower than NumPy in this simple case; it's possible that the NumExpr overhead isn't worth it for this simple operation.  But if we were to switch to multi-threaded we would see a speed-up:
Can you write Python code for it?

ne.set_num_threads(10)
rho = rho_double.copy()
print 'NumExpr inplace, multi-threaded:\n\t',
ne_inplace_rho2 = rho
%timeit ne.evaluate('rho**2', out=ne_inplace_rho2)


I am working on the file "2017/2017_07_17_High-Performance-Python/high_performance_python.ipynb".
The first task is:
Numba does let you call ctypes and CFFI functions in nopython mode, as we'll see later.
One of Numba's main drawbacks is lack of broad support for parallelism.  jit has a nogil option to release the global interpreter lock, which is useful if you have a higher-level parallelism scheme in place (e.g. Python threading), but it gives no speedup on its own.
However, by using the less-flexible vectorize or guvectorize decorators, we can get excellent parallelization in some specialized cases:
Can you write Python code for it?

@nb.vectorize([nb.float64(nb.float64, nb.float64)], nopython=True, target='cpu')
def vec_op(a, b):
    return np.sin(a**b)**2.

@nb.vectorize([nb.float64(nb.float64, nb.float64)], nopython=True, target='parallel')
def parallel_vec_op(a, b):
    return np.sin(a**b)**2.

%timeit vec_op(arr, 2*arr)
%timeit parallel_vec_op(arr, 2*arr)


I am working on the file "2017/2017_07_17_High-Performance-Python/high_performance_python.ipynb".
The first task is:
Numba has one main function: nb.jit.  It's used as a decorator:
Can you write Python code for it?

def py_sum2d(arr):
    M, N = arr.shape
    result = 0.0
    for i in range(M):
        for j in range(N):
            result += arr[i,j]
    return result

@nb.jit
def nb_sum2d(arr):
    M, N = arr.shape
    result = 0.0
    for i in range(M):
        for j in range(N):
            result += arr[i,j]
    return result

%timeit py_sum2d(arr)
%timeit nb_sum2d(arr)


I am working on the file "2017/2017_07_17_High-Performance-Python/high_performance_python.ipynb".
The first task is:
See below for a non-trivial example where prange fails to live up to its promise, however.
Numba also offers gpu and cuda targets for transparent offloading to the GPU, but I haven't tried it myself.
Example: radial histogramming
The above examples were pretty trivial.  Numba really shines when you can use explicit looping to calculate intermediate values that you would otherwise have to store all at once in a big Numpy array.  In this example, we'll bin values on a rectangular lattice into spherical shells.  Thus, we need to know the radial distance to each lattice site.  This is what we'll calculate on-the-fly with Numba.
Can you write Python code for it?

def py_radial_hist(values, box, bin_edges):
    nx,ny,nz = values.shape
    X,Y,Z = np.ogrid[0:box:box/nx, 0:box:box/ny, 0:box:box/nz]
    radii = X**2 + Y**2 + Z**2

    return np.histogram(radii, bins=bin_edges**2, weights=values)

@nb.jit(nopython=True)
def nb_radial_hist(values, boxsize, bin_edges):
    nx,ny,nz = values.shape

    histogram = np.zeros(len(bin_edges)-1)

    # Do binning with squared distances
    bin_edges = bin_edges**2
    nbp1 = len(bin_edges)

    for i in range(nx):
        dx2 = (boxsize/nx*i)**2
        for j in range(ny):
            dy2 = (boxsize/ny*j)**2
            for k in range(nz):
                dz2 = (boxsize/nz*k)**2
                dist = dx2 + dy2 + dz2
                if dist < bin_edges[0] or dist > bin_edges[-1]:
                    continue
                for b in range(1,nbp1):
                    if dist < bin_edges[b]:
                        histogram[b-1] += values[i,j,k]
                        break
                else:  # last bin is closed
                    histogram[-1] += values[i,j,k]

    return histogram

box = 1.
bin_edges = np.linspace(0,box,100)
values = np.random.random((512,512,512))

%timeit py_radial_hist(values, box, bin_edges)
%timeit nb_radial_hist(values, box, bin_edges)


I am working on the file "2017/2017_07_17_High-Performance-Python/high_performance_python.ipynb".
The first task is:
So we cut the runtime of the Numpy implementation by 1/3, and we didn't even consider the memory usage.  The Numpy implementation doubles the memory usage of the input array, since it has to construct the radii.  This can be a big problem when dealing with $2048^3$ FFT meshes, for example.
In practice, I use a parallel_bcast-based parallel implementation, which gives a huge additional speedup.  In theory, we could achieve this parallelism with nb.prange.  Let's explore this below.
Experimental: Using nb.prange for histogramming
nb.prange has a lot of promise, but as an "experimental feature" it's often clunky to use in practice.  Let's try to apply it to this histogramming example and see what happens.
First, let's write a naive implementation with a race condition.  Note that to complile, the prange must be in the innermost loop.
Can you write Python code for it?

@nb.jit(nopython=True, parallel=True)
def BAD_nb_parallel_radial_hist(values, boxsize, bin_edges):
    nx,ny,nz = values.shape

    histogram = np.zeros(len(bin_edges)-1)

    # parallel=True quirk: some versions wouldn't compile without squaring in-place
    bin_edges = bin_edges**2
    nbp1 = len(bin_edges)

    for i in range(nx):
        for j in range(ny):
            # another quirk: prange must be in inner loop
            for k in nb.prange(nz):
                dx2 = (boxsize/nx*i)**2
                dy2 = (boxsize/ny*j)**2
                dz2 = (boxsize/nz*k)**2
                dist = dx2 + dy2 + dz2
                if dist < bin_edges[0] or dist > bin_edges[-1]:
                    continue
                for b in range(1,nbp1):
                    if dist < bin_edges[b]:
                        histogram[b-1] += values[i,j,k]
                        break
                else:  # last bin is closed
                    histogram[-1] += values[i,j,k]

    # also some versions wouldn't compile without a return!
    return histogram

py_answer = py_radial_hist(values, box, bin_edges)[0]
nb_answer = nb_radial_hist(values, box, bin_edges)
BAD_parallel_answer = BAD_nb_parallel_radial_hist(values, box, bin_edges)

print np.allclose(py_answer, nb_answer)
print np.allclose(py_answer, BAD_parallel_answer)


I am working on the file "2017/2017_07_17_High-Performance-Python/high_performance_python.ipynb".
The first task is:
That was about as fast as we expected, but it still gives the wrong answer.  Maybe when prange supports nested loops it will be possible to write a correct version of this.
For fun, let's time the implementation with my custom parallelization scheme ("parallel_bcast").  It's also doing a periodic wrap, unlike the above, so it's doing more work.
Can you write Python code for it?

import Abacus.Analysis.PowerSpectrum.Histogram as AbacusHistogram
#reload(AbacusHistogram.Histogram.)
# Pre-compute the mean radius of the bin and the counts per bin
bin_info = AbacusHistogram.RadialBinGrid(box, values, bin_edges)
%timeit AbacusHistogram.RadialBinGrid(box, values, bin_edges, bin_info=bin_info)


I am working on the file "2017/2017_07_17_High-Performance-Python/high_performance_python.ipynb".
The first task is:
The NumPy way:
Can you write Python code for it?

%%timeit
delta = np.exp((rho/rho_mean - 1.)**2.)


I am working on the file "2017/2017_07_17_High-Performance-Python/high_performance_python.ipynb".
The first task is:
The Numexpr way:
Can you write Python code for it?

%%timeit
delta = ne.evaluate('exp((rho/rho_mean - 1.)**2.)')


I am working on the file "2017/2017_07_17_High-Performance-Python/high_performance_python.ipynb".
The first task is:
The first time we run the timing test, we get a message that one of the Numba timing tests took way longer than the others.  That's because the first time we call the jitted function, it has to be compiled for the dtype of the arguments.  The compiled function is cached, though, making subsequent calls faster.
It's a good idea to always pass the nopython=True kwarg.  This prevents Numba from falling back into object mode if it finds something it can't compile to machine code, e.g. a Python function call:
Can you write Python code for it?

import scipy.special

@nb.jit(nopython=True)
def nb_sum2d_j1(arr):
    M, N = arr.shape
    result = 0.
    for i in range(M):
        for j in range(N):
            result += scipy.special.j1(arr[i,j])
    return result

nb_sum2d_j1(arr)


I am working on the file "2017/2017_07_17_High-Performance-Python/high_performance_python.ipynb".
The first task is:
There are other ways to use CFFI (ABI vs API, in-line vs out-of-line), but the above is the preferred approach for most applications.
CFFI vs ctypes
Here's how we would call gsl_sf_bessel_j1 with ctypes.
Can you write Python code for it?

import ctypes as ct
libgsl = ct.cdll.LoadLibrary("libgsl.so")

# Set the argument and return types
libgsl.gsl_sf_bessel_j1.restype = ct.c_double
libgsl.gsl_sf_bessel_j1.argtypes = [ct.c_double]

libgsl.gsl_sf_bessel_j1(1.)


I am working on the file "2017/2017_07_17_High-Performance-Python/high_performance_python.ipynb".
The first task is:
Wait, what happened?  Why is single-threaded NumExpr slower than NumPy?
Can you write Python code for it?

np_delta = np.exp((rho/rho_mean - 1.)**2.)
ne_delta = ne.evaluate('exp((rho/rho_mean - 1.)**2.)')
print np_delta.dtype
print ne_delta.dtype


I am working on the file "2017/2017_07_17_High-Performance-Python/high_performance_python.ipynb".
The first task is:
We can see that we just compiled _gslffilib.so.  Here's how we use it:
Can you write Python code for it?


I am working on the file "2017/2017_07_17_High-Performance-Python/high_performance_python.ipynb".
The first task is:
We can write a parallel-safe version by giving each k its own temporary histogram space to write into:
Can you write Python code for it?

@nb.jit(nopython=True, parallel=True)
def nb_parallel_radial_hist(values, boxsize, bin_edges):
    nx,ny,nz = values.shape

    histogram = np.zeros((nz, len(bin_edges)-1))

    bin_edges = bin_edges**2
    nbp1 = len(bin_edges)

    for i in range(nx):
        for j in range(ny):
            # prange only works on inner loop
            for k in nb.prange(nz):
                dx2 = (boxsize/nx*i)**2
                dy2 = (boxsize/ny*j)**2
                dz2 = (boxsize/nz*k)**2
                dist = dx2 + dy2 + dz2
                if dist < bin_edges[0] or dist > bin_edges[-1]:
                    continue
                for b in range(1,nbp1):
                    if dist < bin_edges[b]:
                        histogram[k, b-1] += values[i,j,k]
                        break
                else:  # last bin is closed
                    histogram[k, -1] += values[i,j,k]

    # Silly! This could be written as
    # reduced_hist = histogram.sum(axis=0)
    # but Numba auto-parallelization doesn't support axis reductions
    reduced_hist = np.zeros(len(bin_edges)-1)
    for b in range(len(reduced_hist)):
        for k in range(nz):
            reduced_hist[b] += histogram[k, b]

    return reduced_hist

parallel_answer = nb_parallel_radial_hist(values, box, bin_edges)
print np.allclose(py_answer, parallel_answer)


I am working on the file "2017/2017_07_17_High-Performance-Python/high_performance_python.ipynb".
The first task is:
We were using 10 cores.  Did our speedup come from multi-threading or loop-blocking/vectorization?
Can you write Python code for it?

ne.set_num_threads(10)
%timeit ne.evaluate('exp((rho/rho_mean - 1.)**2.)')
ne.set_num_threads(1)
%timeit ne.evaluate('exp((rho/rho_mean - 1.)**2.)')


I am working on the file "2017/CS_20SI/code/1. Overview of Tensorflow & 2. Operations.ipynb".
The first task is:
2. TF-slim(tf.contrib.slim)

Contrib 중 하나의 library로, 상위 수준의 개념(argument scoping, layer, variable)으로 모델을 짧고 쉽게 정의할 수 있게 만듦
많이 사용되는 regularizer를 사용하여 모델을 단순하게 함. VGG, AlexNet과 같이 많이 쓰이는 모델을 개발 해놓음

without TF-Slim
Can you write Python code for it?

input = ...
with tf.name_scope('conv1_1') as scope:
    kernel = tf.Variable(tf.truncated_normal([3, 3, 64, 128], dtype=tf.float32,
                                           stddev=1e-1), name='weights')
    conv = tf.nn.conv2d(input, kernel, [1, 1, 1, 1], padding='SAME')
    biases = tf.Variable(tf.constant(0.0, shape=[128], dtype=tf.float32),
                       trainable=True, name='biases')
    bias = tf.nn.bias_add(conv, biases)
    conv1 = tf.nn.relu(bias, name=scope)


I am working on the file "2017/CS_20SI/code/1. Overview of Tensorflow & 2. Operations.ipynb".
The first task is:
3. Math Operations
Can you write Python code for it?

a = tf.constant([3, 6])
b = tf.constant([2, 2])

with tf.Session() as sess:
    print(sess.run(tf.add(a, b))) # >> [5 8], 2개의 input을 받아 덧셈

with tf.Session() as sess:
    print(sess.run(tf.add_n([a, b, b]))) # >> [7 10]. 모든 input을 덧셈

with tf.Session() as sess:
    print(sess.run(tf.multiply(a, b))) # >> [6 12] because mul is element wise

# matmul: 2차원이상의 텐서간의 곱
with tf.Session() as sess:
    print(sess.run(tf.matmul(tf.reshape(a, shape=[1, 2]),
          tf.reshape(b, shape=[2, 1]))))

with tf.Session() as sess:
    print(sess.run(tf.div(a, b))) # >> [1 3], 나눗셈 실행

with tf.Session() as sess:
    print(sess.run(tf.mod(a, b))) # >> [1 0], 나머지 반환


I am working on the file "2017/CS_20SI/code/1. Overview of Tensorflow & 2. Operations.ipynb".
The first task is:
4. Data Types
### Python Native Types : 불린, 숫자, 스트링
  - 단일 값은 0차원 텐서 (스칼라)
  - 값의 리스트는 1차원 텐서 (벡터)
  - 리스트의 리스트는 2차원 텐서 (매트릭스)
Can you write Python code for it?

# 0차원 상수텐서 - 스칼라
t_0 = 19

with tf.Session() as sess:
    print(sess.run(tf.zeros_like(t_0))) # ==> 0
    print(sess.run(tf.ones_like(t_0))) # ==> 1

# 1차원 텐서 - 벡터
t_1 = [b"apple", b"peach", b"grape"]
with tf.Session() as sess:
    print(sess.run(tf.zeros_like(t_1))) # ==> ['' '' '']
    print(sess.run(tf.ones_like(t_1))) # ==> TypeError: Expected string, got 1 of type 'int' instead.

# 2차원 텐서 - 메트릭스
t_2 = [[True, False, False],
       [False, False, True],
       [False, True, False]]

with tf.Session() as sess:
    print(sess.run(tf.zeros_like(t_2))) # ==> 2x2 tensor, 모든 원소값 False
    print(sess.run(tf.ones_like(t_2))) # ==> 2x2 tensor, 모든 원소값 True


I am working on the file "2017/CS_20SI/code/1. Overview of Tensorflow & 2. Operations.ipynb".
The first task is:
6. InteractiveSession

인터렉티브 세션은 그것 자체로 디폴트세션으로 작동함
따로 run()을 선언하지 않아도 실행됨
Can you write Python code for it?

sess = tf.InteractiveSession()
a = tf.constant(5.0)
b = tf.constant(6.0)
c = a * b

print(c.eval())
sess.close()


I am working on the file "2017/CS_20SI/code/1. Overview of Tensorflow & 2. Operations.ipynb".
The first task is:
7. Control Dependencies

두개 이상의 오퍼레이션이 존재할 때, 오퍼레이션의 실행 순서 지정
tf.Graph.control_dependencies(control_inputs) 함수사용
Can you write Python code for it?

# your graph g have 5 ops: a, b, c, d, e
with g.control_dependencies([a, b, c]):
    d = ....
    e = ....


I am working on the file "2017/CS_20SI/code/1. Overview of Tensorflow & 2. Operations.ipynb".
The first task is:
9. The trap of lazy loading

속도를 조금 더 빠르게 해주는 팁
연산을 최대한 뒤로 미루는 것
Can you write Python code for it?

# Normal loading
x = tf.Variable(10, name='x')
y = tf.Variable(20, name='y')
z = tf.add(x, y)

with tf.Session() as sess:
    sess.run(tf.global_variables_initializer())
    writer = tf.summary.FileWriter('./my_graph/l2', sess.graph)
    for _ in range(10):
        sess.run(z)

writer.close()

# Lazy loading
x = tf.Variable(10, name='x')
y = tf.Variable(20, name='y')

with tf.Session() as sess:
    sess.run(tf.global_variables_initializer())
    writer = tf.summary.FileWriter('./my_graph/l2', sess.graph)
    for _ in range(10):
        sess.run(tf.add(x, y)) # someone decides to be clever to save one line of code
writer.close()


I am working on the file "2017/CS_20SI/code/1. Overview of Tensorflow & 2. Operations.ipynb".
The first task is:
Assign values to variables

tf.Variable.assign()함수를 사용함
Can you write Python code for it?

W = tf.Variable(10)
W.assign(100) # 100이 W에 할당되지 않음

with tf.Session() as sess:
    sess.run(W.initializer)
    print (W.eval()) # >> 10

W = tf.Variable(10)
assign_op = W.assign(100) # assign이 W를 initialize시킴

with tf.Session() as sess:
    sess.run(assign_op)
    print (W.eval()) # >> 100

# 값이 2인 변수 a 생성
a = tf.Variable(2, name="scalar")

# a_times_two에 a * 2 할당
a_times_two = a.assign(a * 2)

init = tf.global_variables_initializer()

with tf.Session() as sess:
    sess.run(init) # a_times_two가 a에 따라 바뀌기 때문에 반드시 a를 initialize시켜줘야 함
    sess.run(a_times_two) # >> 4
#     sess.run(a_times_two) # >> 8
#     sess.run(a_times_two) # >> 16
    print (a_times_two.eval())

W = tf.Variable(10)

with tf.Session() as sess:
    sess.run(W.initializer) # assign_add와 assign_sub는 assign과는 다르게 variable을 initialize시켜주지 않음
    print(sess.run(W.assign_add(10)))
    print(sess.run(W.assign_sub(2)))


I am working on the file "2017/CS_20SI/code/1. Overview of Tensorflow & 2. Operations.ipynb".
The first task is:
Data Flow Graph
Can you write Python code for it?

import tensorflow as tf

a = tf.add(2, 3)

a = tf.add(3, 5)

print (a)


I am working on the file "2017/CS_20SI/code/1. Overview of Tensorflow & 2. Operations.ipynb".
The first task is:
Declare variables

tf.Variable 함수로 생성
Can you write Python code for it?

# a를 스칼라 값으로 생성
a = tf.Variable(2, name="scalar")

# b를 벡터로 생성
b = tf.Variable([2, 3], name="vector")

# c를 2x2 matrix로 생성
c = tf.Variable([[0, 1], [2, 3]], name="matrix")

# W를 0으로 채워진 784x10 tensor로 생성
W = tf.Variable(tf.zeros([784,10]))


I am working on the file "2017/CS_20SI/code/1. Overview of Tensorflow & 2. Operations.ipynb".
The first task is:
Evaluate values of variables

그냥 프린트하면 텐서와 유형, 쉐입만 볼수 있음
Can you write Python code for it?

W = tf.Variable(tf.truncated_normal([700, 10]))

with tf.Session() as sess:
    sess.run(W.initializer)
    print (W)


I am working on the file "2017/CS_20SI/code/1. Overview of Tensorflow & 2. Operations.ipynb".
The first task is:
How to get the value of a?

Session을 시작해줘야 operation이 작동함
Can you write Python code for it?

sess = tf.Session()
sess.run(a)

a = tf.add(3, 5)

with tf.Session() as sess:
    print (sess.run(a))


I am working on the file "2017/CS_20SI/code/1. Overview of Tensorflow & 2. Operations.ipynb".
The first task is:
More graphs
Can you write Python code for it?

x = 2
y = 3

op1 = tf.add(x, y)
op2 = tf.multiply(x, y)
useless = tf.multiply(x, op1)
op3 = tf.pow(op2, op1)

with tf.Session() as sess:
    op3 = sess.run(op3)

x = 2
y = 3

op1 = tf.add(x, y)
op2 = tf.multiply(x, y)
useless = tf.multiply(x, op1)
op3 = tf.pow(op2, op1)

with tf.Session() as sess:
    op3, not_useless = sess.run([op3, useless])

# Creates a graph.

with tf.device("/cpu:0"): # 연산장치 선택 가능
    a = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape = [2,3], name='a')
    b = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], shape = [3,2], name='b')
    c = tf.matmul(a, b)

# Creates a session with log_device_placement set to True.
sess = tf.Session(config=tf.ConfigProto(log_device_placement=True)) # tf.ConfigProto(log_device_placement=True) cpu와 gpu연산이 모두 가능할 때 gpu선택

# Runs the op.
print (sess.run(a))
print (sess.run(b))
print (sess.run(c))


I am working on the file "2017/CS_20SI/code/1. Overview of Tensorflow & 2. Operations.ipynb".
The first task is:
TensorFlow Native Types

텐서플로우는 Numpy 처럼 tf.int32, tf.float32 독자적인 데이터 타입을 사용함
https://www.tensorflow.org/versions/r0.11/resources/dims_types

5. Variables

변수는 할당될수 있고 변경 될수 있음
상수는 그래프에 값이 저장되어 있어 그래프를 로딩할때 함께 로딩됨
변수는 그래프와 별도로 저장됨(파라미터 서버에 살고 있음)
Can you write Python code for it?

my_const = tf.constant([1.0, 2.0], name="my_const")
print (tf.get_default_graph().as_graph_def())


I am working on the file "2017/CS_20SI/code/1. Overview of Tensorflow & 2. Operations.ipynb".
The first task is:
Tensorflow session은 각각 유지됨
각각의 session은 그래프에서 정의된 변수들의 현재값을 각각 가질 수 있다
Can you write Python code for it?

W = tf.Variable(10)

sess1 = tf.Session()
sess2 = tf.Session()

sess1.run(W.initializer)
sess2.run(W.initializer)

print(sess1.run(W.assign_add(10))) # ==> 20
print(sess2.run(W.assign_sub(2))) # ==> 8

print(sess1.run(W.assign_add(100))) # ==> 120
print(sess2.run(W.assign_sub(50))) # ==> -42

sess1.close()
sess2.close()


I am working on the file "2017/CS_20SI/code/1. Overview of Tensorflow & 2. Operations.ipynb".
The first task is:
To initialize only as subset of varuables
tf.variables_initializer()
Can you write Python code for it?

init_ab = tf.variables_initializer([a, b], name = "init_ab")
with tf.Session() as sess:
    sess.run(init)


I am working on the file "2017/CS_20SI/code/1. Overview of Tensorflow & 2. Operations.ipynb".
The first task is:
eval()함수를 사용하면 값까지 볼수 있음
Can you write Python code for it?

with tf.Session() as sess:
    sess.run(W.initializer)
    print (W.eval())


I am working on the file "2017/CS_20SI/code/1. Overview of Tensorflow & 2. Operations.ipynb".
The first task is:
tf.fill(dims, value, name=None)

텐서를 한가지 스칼라값으로 채움
Can you write Python code for it?

with tf.Session() as sess:
    print(sess.run(tf.fill([2, 3], 8))) # [[8, 8, 8], [8, 8, 8]]


I am working on the file "2017/CS_20SI/code/1. Overview of Tensorflow & 2. Operations.ipynb".
The first task is:
tf.zeros_like(input, dtype=None, name=None, opitmize=True)

모든 원소를 0으로 만드는 함수
Can you write Python code for it?

input_tensor = [[0, 1], [2, 3], [4, 5]]

with tf.Session() as sess:
    print (sess.run(tf.zeros_like(input_tensor)))  # [[0, 0], [0, 0], [0, 0]]

np.zeros_like(input_tensor)


I am working on the file "2017/CS_20SI/code/1. Overview of Tensorflow & 2. Operations.ipynb".
The first task is:
with TF-Slim
Can you write Python code for it?

input = ...
net = slim.conv2d(input, 128, [3, 3], scope='conv1_1')


I am working on the file "2017/CS_20SI/code/1. Overview of Tensorflow & 2. Operations.ipynb".
The first task is:
꼭 placeholder가 아니여도 feed 가능

tf.Graph.is_feedable(tensor)

위의 함수를 사용하여 feed가능 여부를 확인할 수 있음
Can you write Python code for it?

# create Operations, Tensors, etc (using the default graph)
a = tf.add(2, 5)
b = tf.multiply(a, 3)

# start up a `Session` using the default graph
sess = tf.Session()

# define a dictionary that says to replace the value of `a` with 15
replace_dict = {a: 15}

# Run the session, passing in `replace_dict` as the value to `feed_dict`
sess.run(b, feed_dict=replace_dict) # returns 45


I am working on the file "2017/CS_20SI/code/1. Overview of Tensorflow & 2. Operations.ipynb".
The first task is:
다른 변수를 사용해서 변수를 만들수 있음
Can you write Python code for it?

W = tf.Variable(tf.truncated_normal([700, 10]))
U = tf.Variable(W * 2)


I am working on the file "2017/CS_20SI/code/1. Overview of Tensorflow & 2. Operations.ipynb".
The first task is:
텐서는 반복문에 사용할 수 없음
Can you write Python code for it?

for _ in range(4):# OK
    a

for _ in tf.range(4): # TypeError("'Tensor' object is not iterable.")
    a


I am working on the file "2017/CS_20SI/code/1. Overview of Tensorflow & 2. Operations.ipynb".
The first task is:
텐서보드 실행법

터미널에서
python [yourprogram.py]
tensorboard --logdir="./graphs"

http://localhost:6006/ 로 이동

jupyter 터미널에서도 실행 가능

윈도우버전 jupyter 실행 불가

2. Constant types

상수유형 공식문서 https://www.tensorflow.org/api_docs/python/constant_op/
constant 함수로 상수, 스칼라, 텐서값 등을 생성할 수 있음

tf.constant(value, dtype=None, shape=None, name='const', verify=False)
Can you write Python code for it?

# constant of 1d tensor (vector)
a = tf.constant([2, 2], name="vector")

# constant of 2x2 tensor (matrix)
b = tf.constant([[0, 1], [2, 3]], name="b")

with tf.Session() as sess:
    print(sess.run(a))
    print(sess.run(b))


I am working on the file "2017/CS_20SI/code/1. Overview of Tensorflow & 2. Operations.ipynb".
The first task is:
텐서의 원소로 특정한 값을 생성할 수 있음
유사함수: numpy.zeros, numpy.zeros_like, numpy.ones, numpy.ones_like

tf.zero(shape, dtype=tf.float32, name=None)

모든 원소가 0인 텐서쉐입을 만드는 함수
Can you write Python code for it?

with tf.Session() as sess:
    print (sess.run(tf.zeros([2, 3], tf.int32))) # [[0, 0, 0], [0, 0, 0]]

import numpy as np

np.zeros((2,3), dtype=np.int32)


I am working on the file "2017/CS_20SI/code/1. Overview of Tensorflow & 2. Operations.ipynb".
The first task is:
특정 분포에서 난수를 생성할 수 있음

tf.random_normal(shape, mean=0.0, stddev=1.0, dtype=tf.float32, seed=None, name=None)
정규분포로부터의 난수 생성

tf.truncated_normal(shape, mean=0.0, stddev=1.0, dtype=tf.float32, seed=None,name=None)
절단정규분포로부터의 난수 생성

tf.random_uniform(shape, minval=0, maxval=None, dtype=tf.float32, seed=None,name=None)
균등분포로부터의 난수 생성

tf.multinomial(logits, num_samples, seed=None, name=None)
다항분포로부터의 난수 생성

tf.random_gamma(shape, alpha, beta=None, dtype=tf.float32, seed=None, name=None)
감마분포로부터의 난수 생성

tf.random_shuffle(value, seed=None, name=None)
값의 첫번째 차원을 기준으로 랜덤하게 섞어줌

tf.random_crop(value, size, seed=None, name=None)
텐서를 주어진 value만큼 랜덤하게 잘라냄
Can you write Python code for it?

with tf.Session() as sess:
    print(sess.run(tf.random_normal(shape = [2,3])))

with tf.Session() as sess:
    print(sess.run(tf.truncated_normal(shape = [2,3])))

with tf.Session() as sess:
    print(sess.run(tf.multinomial(tf.random_normal(shape = [2,3]),5)))

with tf.Session() as sess:
    print(sess.run(tf.random_gamma(shape = [2,3], alpha = 1)))

a = tf.constant([[2,1], [3,2], [7,3]])

with tf.Session() as sess:
    print(sess.run(tf.random_shuffle(a)))

with tf.Session() as sess:
    print(sess.run(tf.random_crop(a, [2,1])))


I am working on the file "2017/Com597I/visualization/VisualizationNotebook.ipynb".
The first task is:
Bar charts
The line charts above are great for cardinal data, but python can do many other kinds of plots. Here, we'll make a bar chart. There are a few differences:

First of all, we still need an x and a y variable. I made a fake set of data for the x axis using range and len. What are the values in fake_x_data?
Play with the width variable to see what it does to the plot.
Note that we use xticks to label our groupings.
Can you write Python code for it?


I am working on the file "2017/Com597I/visualization/VisualizationNotebook.ipynb".
The first task is:
Challenge
Last week, we processed the congressional district data to produce a data set with the percent of the working population in each industry compared to the mean and median salary in that district. You opened that data in Excel to produce a plot.
Recreate a plot of percent of the district in engineering and the mean salary for that district. Make sure to label your axes.
(Hint) remember: all of the python you have learned will work in the notebook. You can still use open for instance.
Can you write Python code for it?


I am working on the file "2017/Com597I/visualization/VisualizationNotebook.ipynb".
The first task is:
Do it!
Read in the mean salary per congressional district and construct a histogram from that data.
* Investigate the bins argument. What happens if you set this too high? What if you set it too low? Generally, sqrt(n) is a good size for n data points.
Challenge
Get the salary data from here (it's the tsv file).  Import this file into python and build a histogram of salary.

How many bins are a good fit?
There are clearly outliers in this data. Investigate the range argument to hist to help truncate the plot.
What does the normalize argument do?

Hint: here is the documentation
Can you write Python code for it?


I am working on the file "2017/Com597I/visualization/VisualizationNotebook.ipynb".
The first task is:
Improving the plot
When I made the plot above, it was a very jumbled line chart, which didn't look very good. Using the tutorial figure out how to plot the points as dots rather than as a line chart.
Can you write Python code for it?


I am working on the file "2017/Com597I/visualization/VisualizationNotebook.ipynb".
The first task is:
Try This
Using the documentation for matplotlib.pyplot found here make a plot for the following data set. In this case, you want to plot both x and y values.
Can you write Python code for it?


I am working on the file "2017/Com597I/visualization/VisualizationNotebook.ipynb".
The first task is:
Try it
Add another bar to the same chart for the monthly low temperature. Make sure you label your axes!
Styles
Now that we've created some simple charts in python, let's see how to make them beautiful!
Matplotlib comes with many pre-built style packages that can be used to make your plots automatically look professional. Try the samples below for a taste.
The full set can be found by running the command below:
Can you write Python code for it?


I am working on the file "2017/Com597I/visualization/VisualizationNotebook.ipynb".
The first task is:
Try it!
Set a context like above and rerun any of the plots in your document.
Histograms
Pyplot includes excellent support for histograms. A histogram uses the range of your data on the x-axis. It divides that range in to buckets (usually with equal widths) and counts the number of values that fall into each bucket. Look at the example below:
Can you write Python code for it?


I am working on the file "2017/Com597I/visualization/VisualizationNotebook_workedinclass.ipynb".
The first task is:
A super simple chart
The simplest possible plot is a line chart. See the example below. Answer the following questions:
* What variable holds the data?
* We only defined one data series. Is it on the y or x axis?
* What do the functions ylabel, xlabel, and title do? Remove them to test your idea.
Can you write Python code for it?

my_data = [1,2.4,-1.3,4]

plt.plot(my_data)
plt.ylabel("yi")
plt.xlabel("hi")
plt.title("A plot!")
plt.show()


I am working on the file "2017/Com597I/visualization/VisualizationNotebook_workedinclass.ipynb".
The first task is:
Improving the plot
When I made the plot above, it was a very jumbled line chart, which didn't look very good. Using the tutorial figure out how to plot the points as dots rather than as a line chart.
Can you write Python code for it?


I am working on the file "2017/Com597I/visualization/VisualizationNotebook_workedinclass.ipynb".
The first task is:
Try This
Using the documentation for matplotlib.pyplot found here make a plot for the following data set. In this case, you want to plot both x and y values.
Can you write Python code for it?

height = [68, 66, 75, 71, 67, 65, 67, 75, 72, 74, 72, 75, 69, 70, 65, 64, 68, 73, 70, 76, 64, 64, 66, 63, 68, 62, 72, 74, 76, 69, 65, 65, 64, 66, 64, 72, 74]
shoe_size = [12 ,9 ,12 ,11 ,12 ,8.5 ,9 ,13 ,11 ,12 ,12 ,12 ,10 ,11 ,10 ,8 ,9 ,11 ,8 ,12 ,8 ,9 ,11 ,10 ,9 ,9 ,11 ,11 ,12 ,10 ,8 ,7 ,9 ,10 ,13 ,12 ,11]

# put your code here
plt.scatter(shoe_size, height, s=range(len(height)), color="red", marker="o", linestyle="None", alpha=.7)
plt.show()

# want:
hist = {}
for i in range(len(height)):
    combo = str(height[i]) + "," + str(shoe_size[i])
    if combo not in hist:
        hist[combo] = 0
    hist[combo] = hist[combo] + 1

new_height = []
new_shoe_size = []
counts = []

for key in hist:
    key_split = key.split(",")
    new_height.append(int(key_split[0]))
    new_shoe_size.append(float(key_split[1]))
    counts.append(hist[key] * 50)

plt.scatter(new_height, new_shoe_size, counts)
plt.show()


I am working on the file "2017/Com597I/visualization/VisualizationNotebook_workedinclass.ipynb".
The first task is:
Try it!
Set a context like above and rerun any of the plots in your document.
Histograms
Pyplot includes excellent support for histograms. A histogram uses the range of your data on the x-axis. It divides that range in to buckets (usually with equal widths) and counts the number of values that fall into each bucket. Look at the example below:
Can you write Python code for it?


I am working on the file "2017/Com597I/visualization/VisualizationNotebook_workedinclass.ipynb".
The first task is:
Use a style like this:
Can you write Python code for it?


I am working on the file "2017/coursera/code/0110/family_tree_implementation.ipynb".
The first task is:
Code
Can you write Python code for it?

import tensorflow as tf
import numpy as np

x_data_1 = np.array(person_1_input).astype(np.float32)
x_data_2 = np.array(relationship_input).astype(np.float32)
y_data = np.transpose(triple_relationship)[2].astype(np.float32)

# Weights and bias
W11 = tf.Variable(tf.zeros([24, 6]))
W12 = tf.Variable(tf.zeros([12, 6]))
W21 = tf.Variable(tf.zeros([24, 12]))
W22 = tf.Variable(tf.zeros([12, 12]))
W3 = tf.Variable(tf.zeros([6, 6]))

b11 = tf.Variable(tf.zeros([6]))
b12 = tf.Variable(tf.zeros([6]))
b2 = tf.Variable(tf.zeros([12]))
b3 = tf.Variable(tf.zeros([6]))

# Hypothesis
L11 = tf.sigmoid(tf.matmul(x_data_1, W11) + b11) # 24 by 6 mat
L12 = tf.sigmoid(tf.matmul(x_data_2, W12) + b12) # 12 by 6 mat
L2 = tf.sigmoid(tf.matmul(tf.transpose(L11), W21) + tf.matmul(tf.transpose(L12), W22) + b2) # 6 by 12
hypothesis = tf.nn.softmax(tf.matmul(tf.transpose(L2), W3) + b3)
print(hypothesis)
# hypothesis = tf.nn.tanh(tf.matmul(L2, W3) + b3)
# hypothesis = tf.nn.relu(tf.matmul(L2, W3) + b3)

# Cost function
cost = -tf.reduce_mean( y_data*tf.log(hypothesis)+(1-y_data)* tf.log(1.-hypothesis) )
# cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(Y_pred, Y))

# # Minimize cost.
# # a = tf.Variable(0.1)
# # optimizer = tf.train.GradientDescentOptimizer(a)
# # train = optimizer.minimize(cost)

# # # Initializa all variables.
# # init = tf.initialize_all_variables()

# # # Loop
# # with tf.Session() as sess:
# #     sess.run(init)

# #     for step in range(8001):
# #         sess.run(train)

# #         if step % 1000 == 0:
# #             print(
# #                 step,
# #                 sess.run(cost, feed_dict={X:x_data, Y:y_data}),
# #                 sess.run(W1),
# #                 sess.run(W2)
# #             )

# #     # Test model
# #     correct_prediction = tf.equal( tf.floor(hypothesis+0.5), Y)
# #     accuracy = tf.reduce_mean(tf.cast( correct_prediction, "float" ) )

# #     # Check accuracy
# #     print( sess.run( [hypothesis, tf.floor(hypothesis+0.5), correct_prediction,...


I am working on the file "2017/coursera/code/0110/family_tree_implementation.ipynb".
The first task is:
Relationship Representation
Can you write Python code for it?


I am working on the file "2017/coursera/code/0110/family_tree_implementation_adam.ipynb".
The first task is:
Code
Can you write Python code for it?

import tensorflow as tf
import numpy as np

x1_data = np.array([person_1_input[data[0]] for  data in triple_relationship],dtype=np.float32)
x2_data = np.array([relationship_input[data[1]] for  data in triple_relationship],dtype=np.float32)
y_data = np.array([person_2_output[data[2]] for  data in triple_relationship],dtype=np.float32)

X1 = tf.placeholder(tf.float32, [None, 24])
X2 = tf.placeholder(tf.float32, [None, 12])
Y = tf.placeholder(tf.float32, [None, 24])

# Weights and bias
W11 = tf.Variable(tf.zeros([24, 6]))
W12 = tf.Variable(tf.zeros([12, 6]))
W21 = tf.Variable(tf.zeros([6, 12]))
W22 = tf.Variable(tf.zeros([6, 12]))
W3 = tf.Variable(tf.zeros([12, 24]))

b11 = tf.Variable(tf.zeros([6]))
b12 = tf.Variable(tf.zeros([6]))
b2 = tf.Variable(tf.zeros([12]))
b3 = tf.Variable(tf.zeros([24]))

# Hypothesis
L11 = tf.sigmoid(tf.matmul(X1, W11) + b11) # 24 by 6 mat
L12 = tf.sigmoid(tf.matmul(X2, W12) + b12) # 12 by 6 mat

# L2 = tf.sigmoid(tf.matmul(L11, W21) + tf.matmul(L12, W22) + b2) # Dimensions must be equal, but are 24 and 12 for 'add_22' (op: 'Add') with input shapes: [24,12], [12,12].
L2 = tf.sigmoid(tf.matmul(L11, W21) + tf.matmul(L12, W22) + b2)
hypothesis = tf.nn.softmax(tf.matmul(L2, W3) + b3)

# Minimize cost.
a = tf.Variable(0.01)
# cost = tf.reduce_mean(hypothesis, Y)
cost = tf.reduce_mean(-tf.reduce_sum(Y*tf.log(hypothesis), reduction_indices=1))

train_step = tf.train.AdamOptimizer(a).minimize(cost)

# Initializa all variables.
init = tf.global_variables_initializer()
sess = tf.Session()
sess.run(init)

# Loop
for i in range(1000):
    sess.run(train_step, feed_dict={
            X1: x1_data,
            X2: x2_data,
            Y:y_data}
            )
    if i % 100 == 0:
        print(
                i,
                sess.run(cost, feed_dict={X1:x1_data, X2:x2_data, Y:y_data})
        )

correct_prediction = tf.equal(tf.argmax(hypothesis,1), tf.argmax(Y,1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

print(sess.run(accuracy, feed_dict={X1: x1_data, X2: x2_data,...


I am working on the file "2017/coursera/code/0110/family_tree_implementation_adam.ipynb".
The first task is:
Relationship Representation
Can you write Python code for it?


I am working on the file "2017/ferran/day16/day16.ipynb".
The first task is:
Part 2
It may not be necessary to carry out the insanely big $10^9$ rounds of shuffling. We will search for the order of the shuffling process, i.e., the minimum number of rounds for which the programs are arranged as they were at the start. Then we will mod $n$.
Can you write Python code for it?

from itertools import count

def least_fixed_point(s, moves):
    a = s
    b = shuffle(list(s), moves)
    visited = [s]
    for i in count():
        if b not in visited:
            visited.append(b)
            a = b
            b = shuffle(list(b), moves)
        else:
            return a, i

def iterated_dances(s, moves, N):
    for i in range(N):
        s = shuffle(list(s), moves)
    return s


I am working on the file "2017/ferran/day16/day16.ipynb".
The first task is:
Solution
Can you write Python code for it?


s = 'abcdefghijklmnop'
least_fixed_point(list(s), moves)

iterated_dances(list(s), moves, (10 ** 9) % 60)


I am working on the file "2017/ferran/day23/day23.ipynb".
The first task is:
Part 1
Can you write Python code for it?

from collections import defaultdict
import re

def init(input_path):
    globals()['prog'] = defaultdict(int)
    globals()['mul_count'] = 0
    globals()['registers'] = parse_registers(input_path)

def retrieve(string, prog):
    if re.search('\d+', string):
        return int(string)
    else:
        return prog[string]

def r_set(x, y):
    y = retrieve(y, prog)
    prog[x] = y

def r_mul(x, y):
    y = retrieve(y, prog)
    prog[x] *= y
    globals()['mul_count'] += 1

def r_sub(x, y):
    y = retrieve(y, prog)
    prog[x] -= y

def exec_register(reg):
    comm = reg[0]
    argx = reg[1][0]
    argy = reg[1][1]
    globals()['r_' + comm](argx, argy)

def run_update(registers, initial_head):
    head = initial_head
    while head < len(registers):
        reg = registers[head]
        if reg[0] != 'jnz':
            exec_register(reg)
            head += 1
        else:
            req = retrieve(reg[1][0], prog)
            offset = retrieve(reg[1][1], prog)
            if req != 0:
                head += offset
            else:
                head += 1


I am working on the file "2017/ferran/day23/day23.ipynb".
The first task is:
Part 2
If we try the natural stretch of the method devised in Part 1, we got too much running time.
So we will try to figure a straightforward code optimization of the following lines of code:
This is the Python translation of our chunk of assembly code...
Can you write Python code for it?

def run_coprocessor_rough(alpha):
    loop0, loop1, loop2 = False, False, False
    a = alpha
    b = 79
    c = b
    d = 0
    e = 0
    f = 0
    g = 0
    h = 0
    if a != 0:
        b *= 100
        b += 100000
        c = b
        c += 17000
    while (g != 0) or not loop0:
        loop0 = True
        f = 1
        d = 2
        while (g != 0) or not loop1:
            loop1 = True
            e = 2
            while (g != 0) or not loop2:
                loop2 = True
                g = d
                g *= e
                g = g - b
                if g == 0:
                    f = 0
                e += 1
                g = e
                g = g - b
            d += 1
            g = d
            g = g - b
        if f == 0:
            h += 1
        g = b
        g = g - c
        if g == 0:
            return a, b, c, d, e, f, g, h
        else:
            b += 17


I am working on the file "2017/ferran/day23/day23.ipynb".
The first task is:
Solution
Can you write Python code for it?

init('input.txt')
run_update(registers, 0)
mul_count


I am working on the file "2017/ferran/day23/day23.ipynb".
The first task is:
Test
Can you write Python code for it?


I am working on the file "2017/iker/Ferran AoC 2017.ipynb".
The first task is:
Day 15
Generators are an esaier way to make iterators. In the end the result is similar
Can you write Python code for it?

class FancyGen(object):

    def __init__(self, start, factor):
        self.start = start
        self.factor = factor
        self.q = 2147483647

    def __iter__(self):
        self.a = self.start
        return self

    def __next__(self):
        n = (self.a * self.factor) % self.q
        self.a = n
        return n

def compare_lowest_bits(n, m):
    n = n % (2 ** 16)
    m = m % (2 ** 16)
    return n == m

def duel(starta, startb):
    N = 40 * 10 ** 6
    count = 0
    gena = iter(FancyGen(starta, 16807))
    genb = iter(FancyGen(startb, 48271))
    for _ in range(N):
        if compare_lowest_bits(next(gena), next(genb)):
            count += 1
    return count

%%timeit
duel(65, 8921)

def generator(start_value, factor):
    val = start_value
    while True:
        val = val * factor % 2147483647
        yield val

def compare(start_A, start_B, rounds):
    matches = 0
    for i, values in enumerate(zip(generator(start_A, 16807), generator(start_B, 48271))):
        if i >= rounds:
            return matches
        else:
            vA, vB = values
            if vA.to_bytes(100, 'big')[-2:] == vB.to_bytes(100, 'big')[-2:]:
                matches += 1

%%timeit
compare(65, 8921, 40*10**6)


I am working on the file "2017/iker/Ferran AoC 2017.ipynb".
The first task is:
Day 16
Regex are expensive, even if you do not "precompile" them
Can you write Python code for it?

import re
import numpy as np
import copy

def shuffle(p, moves):
    s = copy.copy(p)
    for move in moves:
        spin = re.search('s(\d+)', move)
        swapx = re.search('x(\d+)\/(\d+)', move)
        swapp = re.search('p(\w)\/(\w)', move)
        if spin:
            s = np.roll(s, int(spin.group(1)))
        if swapx:
            a = int(swapx.group(1))
            b = int(swapx.group(2))
            s[a], s[b] = s[b], s[a]
        if swapp:
            a = swapp.group(1)
            b = swapp.group(2)
            a = ''.join(s).index(a)
            b = ''.join(s).index(b)
            s[a], s[b] = s[b], s[a]
    return ''.join(s)

%%timeit
shuffle(list('abcde'), ['s1', 'x3/4', 'pe/b'])

def parse(instruction):
    name = instruction[0]
    params = instruction[1:]
    if name == 's':
        params = [int(params)]
    else:
        params = params.split('/')
        if name == 'x':
            params = list(map(int, params))
    return name, params

class Programs:

    def __init__(self, progs):
        self.progs = progs
        self.length = len(self.progs)
        self.instructions_dict = {'s': self.spin, 'x': self.exchange, 'p': self.partner}

    def spin(self, pos):
        pos = pos % self.length
        if pos > 0:
            tmp = self.progs[-pos:]
            progs = tmp + self.progs
            self.progs = progs[:self.length]

    def exchange(self, pos1, pos2):
        v1 = self.progs[pos1]
        v2 = self.progs[pos2]
        self.progs = self.progs[:pos1] + v2 + self.progs[pos1+1:]
        self.progs = self.progs[:pos2] + v1 + self.progs[pos2+1:]

    def partner(self, prog1, prog2):
        self.exchange(self.progs.index(prog1), self.progs.index(prog2))

    def dance(self, instructions):
        for inst, params in instructions:
            self.instructions_dict[inst](*params)
        return p.progs

%%timeit
p = Programs('abcde')
p.dance([parse(inst) for inst in ['s1', 'x3/4', 'pe/b']])

import re
import numpy as np
import copy

regex1 = re.compile('s(\d+)')
regex2 =...


I am working on the file "2017/iker/Ferran AoC 2017.ipynb".
The first task is:
Day 22
Useless day just takes memory and time
Can you write Python code for it?

test_input = """..#
#..
...""".splitlines()

import numpy as np
from collections import defaultdict

def parse_grid(f_input):
    grid = defaultdict(lambda: '.')
    size = 0
    for l in f_input:
        hash_row = {hash(np.array([size, i], dtype=np.int16).tostring()): v for i, v in enumerate(list(l.rstrip()))}
        grid.update(hash_row)
        size += 1
    return grid, size

class Virus(object):
    def __init__(self, grid, size):
        self.grid = grid  # enclosing the hashes and states of infected positions
        self.pos = np.array([(size - 1) // 2, (size - 1) // 2], dtype=np.int16)  # initially in the center of a positive grid
        self.facing = np.array([-1, 0], dtype=np.int16)  # initially facing up in our coords
        self.count_infect = 0
    def burst(self):
        hash_pos = hash(self.pos.tostring())
        rotation = np.array([[0, -1], [1, 0]], dtype=np.int16)
        self.facing = np.dot(rotation, self.facing)
        if self.grid[hash_pos] == '#':
            self.grid[hash_pos] = '.'
            self.facing *= -1
        else:
            self.grid[hash_pos] = '#'
            self.count_infect += 1
        self.pos += self.facing

def count_infect(grid, size, n):
    test_virus = Virus(grid, size)
    for _ in range(n):
        test_virus.burst()
    return test_virus.count_infect

grid, size = parse_grid(test_input)

%%timeit
count_infect(grid, size, 10000)

directions = 'nesw'
directions2move = {'n': (0, 1), 's': (0, -1), 'e': (1, 0), 'w': (-1, 0)}

def parse(lines):
    nodes = []
    size = len(lines[0].strip())
    v = size // 2
    for i, line in enumerate(lines):
        for j, c in enumerate(line.strip()):
            if c == '#':
                nodes.append((j-v, (i-v)*(-1)))
    return set(nodes)

def burst(infected_nodes, pos, direction):
    x, y = pos

    # next direction
    if pos in infected_nodes:
        i = (directions.index(direction) + 1) % 4

        infected_nodes.remove(pos)

    else:
        i = (directions.index(direction) - 1) % 4

       ...


I am working on the file "2017/iker/Ferran AoC 2017.ipynb".
The first task is:
Day 23
Just different approaches
Can you write Python code for it?

def is_prime(x):
    if x >= 2:
        for y in range(2,x):
            if not ( x % y ):
                return False
    else:
        return False
    return True

def run_coprocessor(alpha):
    loop = False
    a, b = alpha, 79
    c = b
    d, e, f, g, h = 0, 0, 0, 0, 0
    if a != 0:
        b *= 100
        b += 100000
        c = b
        c += 17000
    while (g != 0) or not loop:
        loop = True
        f = 1
        d = 2
        e = 2
        if not is_prime(b):
            f = 0
        e = b
        d = b
        if f == 0:
            h += 1
        g = b
        g = g - c
        if g == 0:
            return a, b, c, d, e, f, g, h
        else:
            b += 17

run_coprocessor(1)

def isprime(value):
    for i in range(2, value):
        if (value % i) == 0:
            return False
    return True

def count_primes(init, end, step):
    count = 0
    for i in range(init, end+1, step):
        if isprime(i):
            count += 1
    return count

def part2():
   b = 106500
   c = 123500
   h = (c-b)/17  # each loop b increases 17 until it matches c
   h += 1  # there is an extra loop when b == c ??
   h -= count_primes(b, c, 17)  # on primes, f is set to 0 and h not increased
   return int(h)

part2()


I am working on the file "2017/iker/Ferran AoC 2017.ipynb".
The first task is:
Day 3
Well done
Can you write Python code for it?

import numpy as np

def number_to_coordinates(n):

    q = int(np.sqrt(n))
    r = n - q ** 2
    if q % 2 != 0:
        x = (q - 1) // 2 + min(1, r) + min(q - r + 1, 0)
        y = - (q - 1) // 2 + min(max(r - 1, 0), q)
    else:
        x = 1 - (q // 2) - min(1, r) - min(q - r + 1, 0)
        y = q // 2 - min(max(r - 1, 0), q)
    return x, y

def spiral_manhattan(n):
    x, y = number_to_coordinates(n)
    return abs(x) + abs(y)

spiral_manhattan(1024)

import math

def get_side_size(point):
    side_size = math.ceil(math.sqrt(point))
    if side_size % 2 == 0:
        side_size += 1
    return side_size

def get_displacement(point, ring):
    distances = []
    for i in [1,3,5,7]:
        distances.append(abs(point-i*ring))
    return min(distances)

def distance(point):
    if point == 1:
        return 0
    else:
        side_size = get_side_size(point)
        radius = (side_size - 1) // 2
        rescaled = point - (side_size-2)**2
        displacement = get_displacement(rescaled, radius)
        return displacement + radius

distance(1024)


I am working on the file "2017/iker/Ferran AoC 2017.ipynb".
The first task is:
Day 6
It can be difficult to decide when classes are useful
Can you write Python code for it?

banks = [0, 2, 7, 0]

def reallocate(val, pos, n):
        l = [val // n] * n
        r = val % n
        for i in range(r):
            l[(pos + i + 1) % n] += 1
        return l

def update(b):
    blocks = sorted(list(enumerate(b)), key=lambda v: (v[1], -v[0]), reverse=True)
    pos = blocks[0][0]
    val = blocks[0][1]
    c = [b[i] if i != pos else 0 for i in range(len(b))]
    l = reallocate(val, pos, len(b))
    for i, v in enumerate(c):
        c[i] += l[i]
    return c

def count_until_loop(b):
    count = 0
    previous = set()
    h = hash(tuple(b))
    while h not in previous:
        previous.add(h)
        count += 1
        b = update(b)
        h = hash(tuple(b))
    return count

count_until_loop(banks)

class Memory:

    def __init__(self, banks):
        self.banks = banks
        self.states = []

    def _find_fullest(self):
        blocks = max(self.banks)
        return self.banks.index(blocks), blocks

    def _redistribue(self):
        pos, blocks = self._find_fullest()
        self.banks[pos] = 0
        while blocks > 0:
            pos += 1
            if pos >= len(self.banks):
                pos = 0
            self.banks[pos] += 1
            blocks -= 1

    def realloate_till_loop(self):
        redistributions = 0
        self.states.append(self.banks.copy())
        while True:
            self._redistribue()
            redistributions += 1
            configuration = self.banks.copy()
            if configuration in self.states:
                break
            else:
                self.states.append(configuration)
        return redistributions

Memory(banks).realloate_till_loop()

%%timeit
count_until_loop(banks)

%%timeit
Memory(banks).realloate_till_loop()