I am working on the file "Seminar11/Seminar11_homework.ipynb".
The first task is:
Data preprocessing
Can you write Python code for it?

%%time
# Read Dataset
import numpy as np
import os.path as osp
import pickle

img_codes = np.load(osp.join(DATA_DIR, "data/image_codes.npy"))
captions = pickle.load(open(osp.join(DATA_DIR, 'data/caption_tokens.pcl'), 'rb'))

print ("each image code is a 1000-unit vector:", img_codes.shape)
print (img_codes[0,:10])
print ('\n\n')
print ("for each image there are 5-7 descriptions, e.g.:\n")
print ('\n'.join(captions[0]))

#split descriptions into tokens
for img_i in range(len(captions)):
    for caption_i in range(len(captions[img_i])):
        sentence = captions[img_i][caption_i]
        captions[img_i][caption_i] = ["#START#"]+sentence.split(' ')+["#END#"]

# Build a Vocabulary
from collections import Counter
word_counts = Counter()

<Compute word frequencies for each word in captions. See code above for data structure>

vocab  = ['#UNK#', '#START#', '#END#']
vocab += [k for k, v in word_counts.items() if v >= 5]
n_tokens = len(vocab)

assert 10000 <= n_tokens <= 10500

word_to_index = {w: i for i, w in enumerate(vocab)}

PAD_ix = -1
UNK_ix = vocab.index('#UNK#')

#good old as_matrix for the third time
def as_matrix(sequences,max_len=None):
    max_len = max_len or max(map(len,sequences))

    matrix = np.zeros((len(sequences),max_len),dtype='int32')+PAD_ix
    for i,seq in enumerate(sequences):
        row_ix = [word_to_index.get(word,UNK_ix) for word in seq[:max_len]]
        matrix[i,:len(row_ix)] = row_ix

    return matrix

#try it out on several descriptions of a random image
as_matrix(captions[1337])


I am working on the file "Seminar11/Seminar11_homework.ipynb".
The first task is:
Demo
Find at least 10 images to test it on.

Seriously, that's part of an assignment. Go get at least 10 pictures to get captioned
Make sure it works okay on simple images before going to something more comples
Photos, not animation/3d/drawings, unless you want to train CNN network on anime
Mind the aspect ratio (see what preprocess does to your image)
Can you write Python code for it?

#apply your network on image sample you found
#
#


I am working on the file "Seminar11/Seminar11_homework.ipynb".
The first task is:
Generate caption
Can you write Python code for it?

last_word_probas = <get network-predicted probas at last tick
#TRY OUT deterministic=True if you want more steady results

get_probs = theano.function([image_vectors,sentences], last_word_probas)

#this is exactly the generation function from week5 classwork,
#except now we condition on image features instead of words
def generate_caption(image,caption_prefix = ("START",),t=1,sample=True,max_len=100):
    image_features = get_cnn_features(image)
    caption = list(caption_prefix)
    for _ in range(max_len):

        next_word_probs = <obtain probabilities for next words>
        assert len(next_word_probs.shape) ==1 #must be one-dimensional
        #apply temperature
        next_word_probs = next_word_probs**t / np.sum(next_word_probs**t)

        if sample:
            next_word = np.random.choice(vocab,p=next_word_probs)
        else:
            next_word = vocab[np.argmax(next_word_probs)]

        caption.append(next_word)

        if next_word=="#END#":
            break

    return caption

for i in range(10):
    print ' '.join(generate_caption(img,t=5.)[1:-1])


I am working on the file "Seminar11/Seminar11_homework.ipynb".
The first task is:
Mah Neural Network
Can you write Python code for it?

# network shapes.
CNN_FEATURE_SIZE = img_codes.shape[1]
EMBED_SIZE = 128 #pls change me if u want
LSTM_UNITS = 200 #pls change me if u want

import theano
import theano.tensor as T

# Input Variable
sentences = T.imatrix()# [batch_size x time] of word ids
image_vectors = T.matrix() # [batch size x unit] of CNN image features
sentence_mask = T.neq(sentences,PAD_ix)

import lasagne
from lasagne.layers import *

#network inputs
l_words = InputLayer((None,None),sentences )
l_mask = InputLayer((None,None),sentence_mask )

#embeddings for words
l_word_embeddings = <apply word embedding. use EMBED_SIZE>

#cudos for using some pre-trained embedding :)

# input layer for image features
l_image_features = InputLayer((None,CNN_FEATURE_SIZE),image_vectors )

#convert 1000 image features from googlenet to whatever LSTM_UNITS you have set
#it's also a good idea to add some dropout here and there
l_image_features_small = <convert l_image features to a shape equal to rnn hidden state. Also play with dropout/noize>

assert l_image_features_small.output_shape == (None,LSTM_UNITS)

# Concatinate image features and word embedings in one sequence
decoder = a recurrent layer (gru/lstm) with following checklist:
#    * takes word embeddings as an input
#    * has LSTM_UNITS units in the final layer
#    * has cell_init (or hid init for gru) set to converted image features
#    * mask_input = input_mask
#    * don't forget the grad clipping (~5-10)

#find out better recurrent architectures for bonus point

# Decoding of rnn hiden states
from broadcast import BroadcastLayer,UnbroadcastLayer

#apply whatever comes next to each tick of each example in a batch. Equivalent to 2 reshapes
broadcast_decoder_ticks = BroadcastLayer(decoder,(0,1))
print "broadcasted decoder shape = ",broadcast_decoder_ticks.output_shape

#predict probabilities for next tokens
predicted_probabilities_each_tick = <predict probabilities for each tick, using broadcasted_decoder_shape as an input. No reshaping needed here.>
# maybe a more complicated...


I am working on the file "Seminar11/Seminar11_homework.ipynb".
The first task is:
Main loop

We recommend you to periodically evaluate the network using the next "apply trained model" block
its safe to interrupt training, run a few examples and start training again
Can you write Python code for it?

batch_size=50 #adjust me
n_epochs=100 #adjust me
n_batches_per_epoch = 50 #adjust me
n_validation_batches = 5 #how many batches are used for validation after each epoch

!pip3 install --user tqdm

from tqdm import tqdm

for epoch in range(n_epochs):

    train_loss=0
    for _ in tqdm(range(n_batches_per_epoch)):
        train_loss += train_step(*generate_batch(img_codes,captions,batch_size))
    train_loss /= n_batches_per_epoch

    val_loss=0
    for _ in range(n_validation_batches):
        val_loss += val_step(*generate_batch(img_codes,captions,batch_size))
    val_loss /= n_validation_batches

    print('\nEpoch: {}, train loss: {}, val loss: {}'.format(epoch, train_loss, val_loss))

print("Finish :)")


I am working on the file "Seminar11/Seminar11_homework.ipynb".
The first task is:
Some tricks

If you train large network, it is usually a good idea to make a 2-stage prediction
(large recurrent state) -> (bottleneck e.g. 256)
(bottleneck) -> (vocabulary size)
this way you won't need to store/train (large_recurrent_state x vocabulary size) matrix

Also maybe use Hierarchical Softmax?
https://gist.github.com/justheuristic/581853c6d6b87eae9669297c2fb1052d
Can you write Python code for it?

next_word_probas = <get network output>

predictions_flat = next_word_probas[:,:-1].reshape((-1,n_tokens))
reference_answers = sentences[:,1:].reshape((-1,))

#write symbolic loss function to train NN for
loss = <compute elementwise loss function>

#mean over non-PAD tokens
output_mask = sentence_mask[:,1:]
loss = (loss.reshape(reference_answers.shape)*output_mask).sum() / output_mask.sum()

#trainable NN weights
weights = get_all_params(predicted_probabilities,trainable=True)
updates = <parameter updates using your favorite algoritm>

#compile a functions for training and evaluation
#please not that your functions must accept image features as FIRST param and sentences as second one
train_step = <function that takes input sentence and image mask, outputs loss and updates weights>
val_step   = <function that takes input sentence and image mask and outputs loss>
#for val_step use deterministic=True if you have any dropout/noize


I am working on the file "Seminar11/Seminar11_homework.ipynb".
The first task is:
Training

You first have to implement a batch generator
Than the network will get trained the usual way
Can you write Python code for it?

captions = np.array(captions)

from random import choice

def generate_batch(images,captions,batch_size,max_caption_len=None):

    #sample random numbers for image/caption indicies
    random_image_ix = np.random.randint(0,len(images),size=batch_size)

    #get images
    batch_images = images[random_image_ix]

    #5-7 captions for each image
    captions_for_batch_images = captions[random_image_ix]

    #pick 1 from 5-7 captions for each image
    batch_captions = map(choice,captions_for_batch_images)

    #convert to matrix
    batch_captions_ix = as_matrix(batch_captions,max_len=max_caption_len)

    return batch_images, batch_captions_ix

generate_batch(img_codes,captions,3)


I am working on the file "Seminar11/Seminar11_homework.ipynb".
The first task is:
apply trained model
Can you write Python code for it?

!pip3 install --user scikit-image

#the same kind you did last week, but a bit smaller
from pretrained_lenet import build_model,preprocess,MEAN_VALUES

# build googlenet
lenet = build_model()

#load weights
#lenet_weights = pickle.load(open(osp.join(DATA_DIR, 'data/blvc_googlenet.pkl')), encoding='latin1')['param values']
lenet_weights = np.load(osp.join(DATA_DIR, 'data/blvc_googlenet.npz'), encoding='latin1')['param values']
set_all_param_values(lenet["prob"], lenet_weights)

#compile get_features
cnn_input_var = lenet['input'].input_var
cnn_feature_layer = lenet['loss3/classifier']
get_cnn_features = theano.function([cnn_input_var], lasagne.layers.get_output(cnn_feature_layer))

from matplotlib import pyplot as plt
%matplotlib inline

#sample image
img = plt.imread(osp.join(DATA_DIR, 'data/Dog-and-Cat.jpg'))
img = preprocess(img)

#deprocess and show, one line :)
from pretrained_lenet import MEAN_VALUES
plt.imshow(np.transpose((img[0] + MEAN_VALUES)[::-1],[1,2,0]).astype('uint8'))


I am working on the file "Seminar11/Seminar11_intro.ipynb".
The first task is:
Build NN
You will be building a model that takes token sequence and predicts next token

iput sequence
one-hot / embedding
recurrent layer(s)
otput layer(s) that predict output probabilities
Can you write Python code for it?

from lasagne.layers import InputLayer,DenseLayer,EmbeddingLayer
from lasagne.layers import RecurrentLayer,LSTMLayer,GRULayer,CustomRecurrentLayer

##ENCODER
l_in = lasagne.layers.InputLayer(shape=(None, None),input_var=input_sequence)
l_mask = lasagne.layers.InputLayer(shape=(None, None),input_var=T.neq(input_sequence,-1))
l_emb = lasagne.layers.EmbeddingLayer(l_in, len(letters), 40)
l_rnn = lasagne.layers.GRULayer(l_emb,256,only_return_final=True,mask_input=l_mask)

##DECODER
transc_in = lasagne.layers.InputLayer(shape=(None, None),input_var=target_phonemes)
transc_mask = lasagne.layers.InputLayer(shape=(None, None),input_var=T.neq(target_phonemes,-1))
transc_emb = lasagne.layers.EmbeddingLayer(transc_in, len(phonemes), 50)
transc_rnn = lasagne.layers.GRULayer(transc_emb,256,hid_init=l_rnn,mask_input=transc_mask)

#flatten batch and time to be compatible with feedforward layers (will un-flatten later)
transc_rnn_flat = lasagne.layers.reshape(transc_rnn, (-1,transc_rnn.output_shape[-1]))

l_out = lasagne.layers.DenseLayer(transc_rnn_flat,len(phonemes),nonlinearity=lasagne.nonlinearities.softmax)

# Model weights
weights = lasagne.layers.get_all_params(l_out,trainable=True)
print weights

network_output = lasagne.layers.get_output(l_out)
network_output = <reshape to [batch_i, time_tick, number_of_phonemes] symbolically>
#If you use dropout do not forget to create deterministic version for evaluation

predictions_flat = network_output[:,:-1,:].reshape([-1,len(phonemes)])
targets_flat = target_phonemes[:,1:].ravel()

#do not count loss for '-1' tokens
mask = T.nonzero(T.neq(targets_flat,-1))

loss = T.nnet.categorical_crossentropy(predictions_flat[mask],targets_flat[mask])

updates = lasagne.updates.adam(loss.mean(),weights)


I am working on the file "Seminar11/Seminar11_intro.ipynb".
The first task is:
Cast everything from symbols into matrix of int32. Pad with -1
Can you write Python code for it?

def as_matrix(sequences,token_to_i, max_len=None,PAX_ix=PAD_ix):
    max_len = max_len or max(map(len,sequences))

    matrix = np.zeros((len(sequences),max_len),dtype='int8') -1
    for i,seq in enumerate(sequences):
        row_ix = map(token_to_i.get,seq)[:max_len]
        matrix[i,:len(row_ix)] = row_ix

    return matrix

print as_matrix(words[:10],letter_to_ix)


I am working on the file "Seminar11/Seminar11_intro.ipynb".
The first task is:
Input variables
Can you write Python code for it?

input_sequence = T.matrix('token sequence','int32')
target_phonemes = T.matrix('target phonemes','int32')


I am working on the file "Seminar11/Seminar11_intro.ipynb".
The first task is:
Model training
Here you can tweak parameters or insert your generation function
Once something word-like starts generating, try increasing seq_length
Can you write Python code for it?

words = np.array(words)
transcripts = np.array(transcripts)

def sample_batch(words,transcripts, batch_size):

    <sample random batch of words and transcripts>
    <convert both into network-edible format (as_matrix)>

    return words_batch,transcripts_batch

from tqdm import tqdm

print("Training ...")

#total N iterations
n_epochs=100

# how many minibatches are there in the epoch
batches_per_epoch = 500

#how many training sequences are processed in a single function call
batch_size=10

for epoch in range(n_epochs):

    avg_cost = 0;

    for _ in tqdm(range(batches_per_epoch)):

        x,y = sample_batch(words,transcripts,batch_size)
        avg_cost += train(x, y).mean()

    print("Epoch {} average loss = {}".format(epoch, avg_cost / batches_per_epoch))
    for i in range(5):
        ind = np.random.randint(len(words))
        print words[ind],':', ' '.join(generate_transcript(words[ind],sample=False)[1:-1])


I am working on the file "Seminar11/Seminar11_intro.ipynb".
The first task is:
Problem & Dataset

We solve a problem of transribing english words.
word (sequence of letters) -> transcipt (sequence of phonemes)
The problem is, some letters correspond to several phonemes and others - to none.
We solve it through encoder-decoder recurrent neural networks
This architecture is generally about converting ANY sequence into ANY other sequence. It could even become president one day.
Can you write Python code for it?

with open("./train.csv") as fin:
    ids,words,transcripts = zip(*[line.split(',') for line in list(fin)[1:]])
    words = [word+"@" for word in words]
    transcripts = [["START"]+ts[:-2].split()+["END"] for ts in transcripts]

for word, trans in zip(words[:5],phonemes[:5]):
    print word,':',trans


I am working on the file "Seminar11/Seminar11_intro.ipynb".
The first task is:
generation
Simple:
* get initial context(seed),
* predict next token probabilities,
* sample next token,
* add it to the context
* repeat from step 2
You'll get a more detailed info on how it works in the homework section.
Can you write Python code for it?

#compile the function that computes probabilities for next token given previous text.

network_output = <network output reshaped to [batch,tick,phoneme] format>

last_word_probas = <a matrix [batch_i, n_phonemes], counting all phonemes>

probs = <a function that predicts probabilities coming after the last token

def generate_transcript(word,transcript_prefix = ("START",),END_phoneme="END"
                        temperature=1,sample=True):

    transcript = list(transcript_prefix)
    while True:
        next_phoneme_probs = <a vector of probabilities of the next token>
        next_phoneme_probs = <maybe apply temperature>

        if sample:
            next_phoneme = <phoneme sampled with these probabilities (string character)>
        else:
            next_phoneme = <most likely phoneme>

        transcript.append(next_phoneme)

        if next_phoneme==END_phoneme:
            break
    return transcript


I am working on the file "Seminar2/Homework2.ipynb".
The first task is:
Finally, try Adam algorithm. You can start with beta = 0.9 and mu = 0.999
Can you write Python code for it?

w = np.array([1,0,0,0,0,0])
v = np.zeros(6)
g = np.zeros(6)

alpha = 2. # learning rate
beta = 0.9  # (beta1 coefficient in original paper) exponential decay rate for the 1st moment estimates
mu   = 0.5  # (beta2 coefficient in original paper) exponential decay rate for the 2nd moment estimates
eps = 1e-4  # A small constant for numerical stability

n_iter = 50
batch_size = 4
loss = np.zeros(n_iter)
plt.figure(figsize=(12,5))
for i in range(n_iter):
    ind = random.sample(range(X.shape[0]), batch_size)
    loss[i] = compute_loss(X, y, w)
    visualize(X[ind,:], y[ind], w, loss, n_iter)

    gr = compute_grad(X, y, w)
    v = beta * v + (1 - beta) * gr
    g = mu * g + gr * gr * (1 - mu)
    w = w - v * alpha * (1 - mu) / (g + eps) / (1 - beta)

visualize(X, y, w, loss, n_iter)
plt.clf()


I am working on the file "Seminar2/Homework2.ipynb".
The first task is:
Implement gradient descent with momentum and test it's performance for different learning rate and momentum values.
Can you write Python code for it?

w = np.array([1,0,0,0,0,0])

alpha = 0.0 # learning rate
mu    = 0.0 # momentum

n_iter = 50
batch_size = 4
loss = np.zeros(n_iter)
plt.figure(figsize=(12,5))
for i in range(n_iter):
    ind = random.sample(range(X.shape[0]), batch_size)
    loss[i] = compute_loss(X, y, w)
    visualize(X[ind,:], y[ind], w, loss, n_iter)

    #<update w and anything else here>

visualize(X, y, w, loss, n_iter)
plt.clf()


I am working on the file "Seminar2/Homework2.ipynb".
The first task is:
Same task but for Nesterov's accelerated gradient:
Can you write Python code for it?


I am working on the file "Seminar2/Homework2.ipynb".
The first task is:
The loss you should try to minimize is the Hinge Loss:
$$ L =  {1 \over N} \sum_{i=1}^N max(0,1-y_i \cdot  w^T x_i) $$
Can you write Python code for it?

def compute_loss(X, y, w):
    """
    Given feature matrix X [n_samples,2], target vector [n_samples] of +1/-1,
    and weight vector w [6], compute scalar loss function using formula above.
    """
    X = expand(X)
    y_pred = X.dot(w)
    res = np.maximum(0, 1 - y_pred * y)
    return res.mean()

def compute_grad(X, y, w):
    """
    Given feature matrix X [n_samples,2], target vector [n_samples] of +1/-1,
    and weight vector w [6], compute vector [6] of derivatives of L over each weights.
    """
    X = expand(X)
    y_pred = X.dot(w) * y
    y_pred = np.int0(y_pred < 1)
    grad = -(X.T * y).T
    grad = (grad.T * y_pred).T
    return grad.sum(axis=0)


def compute_loss(X, y, w):
    """
    Given feature matrix X [n_samples,2], target vector [n_samples] of +1/-1,
    and weight vector w [6], compute scalar loss function using formula above.
    """
    #<your code here>

def compute_grad(X, y, w):
    """
    Given feature matrix X [n_samples,2], target vector [n_samples] of +1/-1,
    and weight vector w [6], compute vector [6] of derivatives of L over each weights.
    """
    #<your code here>


I am working on the file "Seminar2/Homework2.ipynb".
The first task is:
Training
Find an optimal learning rate for gradient descent for given batch size.
You can see the example of correct output below this cell before you run it.
Don't change the batch size!
Can you write Python code for it?

w = np.array([1,0,0,0,0,0])

alpha = 0.1 # learning rate

n_iter = 50
batch_size = 4
loss = np.zeros(n_iter)
plt.figure(figsize=(12,5))
for i in range(n_iter):
    ind = random.sample(range(X.shape[0]), batch_size)
    loss[i] = compute_loss(X, y, w)
    visualize(X[ind,:], y[ind], w, loss, n_iter)

    w = w - alpha * compute_grad(X[ind,:], y[ind], w)

visualize(X, y, w, loss, n_iter)
plt.clf()


w = np.array([1,0,0,0,0,0])

alpha = 0.0 # learning rate

n_iter = 50
batch_size = 4
loss = np.zeros(n_iter)
plt.figure(figsize=(12,5))
for i in range(n_iter):
    ind = random.sample(range(X.shape[0]), batch_size)
    loss[i] = compute_loss(X, y, w)
    visualize(X[ind,:], y[ind], w, loss, n_iter)

    w = w - alpha * compute_grad(X[ind,:], y[ind], w)

visualize(X, y, w, loss, n_iter)
plt.clf()


I am working on the file "Seminar2/Homework2.ipynb".
The first task is:
Your task starts here
First, let's write a function that predicts class for given X.
Since the problem above isn't linearly separable, we add quadratic features to the classifier. This transformation is implemented in the expand function.
Don't forget to expand X inside classify and other functions
Sample classification should not be much harder than computation of sign of dot product.
Can you write Python code for it?

def expand(X):
    X_ = np.zeros((X.shape[0], 6))
    X_[:,0:2] = X
    X_[:,2:4] = X**2
    X_[:,4] = X[:,0] * X[:,1]
    X_[:,5] = 1
    return X_

def classify(X, w):
    """
    Given feature matrix X [n_samples,2] and weight vector w [6],
    return an array of +1 or -1 predictions
    """
    X = expand(X)
    y = X.dot(w)
    return np.sign(y)


def expand(X):
    X_ = np.zeros((X.shape[0], 6))
    X_[:,0:2] = X
    X_[:,2:4] = X**2
    X_[:,4] = X[:,0] * X[:,1]
    X_[:,5] = 1
    return X_

def classify(X, w):
    """
    Given feature matrix X [n_samples,2] and weight vector w [6],
    return an array of +1 or -1 predictions
    """
    #<your code here>


I am working on the file "Seminar2/Seminar2.ipynb".
The first task is:
Implement gradient descent with momentum and test it's performance for different learning rate and momentum values.
Can you write Python code for it?

w = np.array([1,0,0,0,0,0])

alpha = 0.0 # learning rate
mu    = 0.0 # momentum

n_iter = 50
batch_size = 4
loss = np.zeros(n_iter)
plt.figure(figsize=(12,5))
for i in range(n_iter):
    ind = random.sample(range(X.shape[0]), batch_size)
    loss[i] = compute_loss(X, y, w)
    visualize(X[ind,:], y[ind], w, loss, n_iter)

    <update w and anything else here>

visualize(X, y, w, loss, n_iter)
plt.clf()


I am working on the file "Seminar2/Seminar2.ipynb".
The first task is:
The loss you should try to minimize is the Hinge Loss.
$$ L =  {1 \over N} \sum_i max(0,1-y_i \cdot \vec w \vec x_i) $$
Can you write Python code for it?

def compute_loss(X, y, w):
    """
    Given feature matrix X [n_samples,2], target vector [n_samples] of +1/-1,
    and weight vector w [6], compute scalar loss function using formula above.
    """
    <your code here>

def compute_grad(X, y, w):
    """
    Given feature matrix X [n_samples,2], target vector [n_samples] of +1/-1,
    and weight vector w [6], compute vector [6] of derivatives of L over each weights.
    """
    <your code here>


I am working on the file "Seminar2/Seminar2.ipynb".
The first task is:
Training
Find an optimal learning rate for gradient descent for given batch size.
You can see the example of correct output below this cell before you run it.
Don't change the batch size!
Can you write Python code for it?


I am working on the file "Seminar3/HW3_Modules.ipynb".
The first task is:
Activation functions
Here's the complete example for the Rectified Linear Unit non-linearity (aka ReLU):
Can you write Python code for it?

class ReLU(Module):
    def __init__(self):
         super(ReLU, self).__init__()

    def updateOutput(self, input):
        self.output = np.maximum(input, 0)
        return self.output

    def updateGradInput(self, input, gradOutput):
        self.gradInput = np.multiply(gradOutput , input > 0)
        return self.gradInput

    def __repr__(self):
        return "ReLU"


I am working on the file "Seminar3/HW3_Modules.ipynb".
The first task is:
Implement Exponential Linear Units activations.
Can you write Python code for it?

class ELU(Module):
    def __init__(self, alpha = 1.0):
        super(ELU, self).__init__()

        self.alpha = alpha

    def updateOutput(self, input):
        # Your code goes here. ################################################
        return  self.output

    def updateGradInput(self, input, gradOutput):
        # Your code goes here. ################################################
        return self.gradInput

    def __repr__(self):
        return "ELU"


I am working on the file "Seminar3/HW3_Modules.ipynb".
The first task is:
Implement Leaky Rectified Linear Unit. Expriment with slope.
Can you write Python code for it?

class LeakyReLU(Module):
    def __init__(self, slope = 0.03):
        super(LeakyReLU, self).__init__()

        self.slope = slope

    def updateOutput(self, input):
        # Your code goes here. ################################################
        return  self.output

    def updateGradInput(self, input, gradOutput):
        # Your code goes here. ################################################
        return self.gradInput

    def __repr__(self):
        return "LeakyReLU"


I am working on the file "Seminar3/HW3_Modules.ipynb".
The first task is:
Implement SoftPlus activations. Look, how they look a lot like ReLU.
Can you write Python code for it?

class SoftPlus(Module):
    def __init__(self):
        super(SoftPlus, self).__init__()

    def updateOutput(self, input):
        # Your code goes here. ################################################
        return  self.output

    def updateGradInput(self, input, gradOutput):
        # Your code goes here. ################################################
        return self.gradInput

    def __repr__(self):
        return "SoftPlus"


I am working on the file "Seminar3/HW3_Modules.ipynb".
The first task is:
Implement dropout. The idea and implementation is really simple: just multimply the input by $Bernoulli(p)$ mask.
This is a very cool regularizer. In fact, when you see your net is overfitting try to add more dropout.
While training (self.training == True) it should sample a mask on each iteration (for every batch). When testing this module should implement identity transform i.e. self.output = input.

input:   batch_size x n_feats
output: batch_size x n_feats
Can you write Python code for it?

class Dropout(Module):
    def __init__(self, p=0.5):
        super(Dropout, self).__init__()
        self.p = p
        self.mask = None

    def updateOutput(self, input):
        self.mask = np.random.binomial(1, self.p, size=input.shape)
        self.output = input * self.mask
        return  self.output

    def updateGradInput(self, input, gradOutput):
        self.gradInput = gradOutput * self.mask
        return self.gradInput

    def __repr__(self):
        return "Dropout"


I am working on the file "Seminar3/HW3_Modules.ipynb".
The first task is:
Layers

input:   batch_size x n_feats1
output: batch_size x n_feats2
Can you write Python code for it?

class Linear(Module):
    """
    A module which applies a linear transformation
    A common name is fully-connected layer, InnerProductLayer in caffe.

    The module should work with 2D input of shape (n_samples, n_feature).
    """
    def __init__(self, n_in, n_out):
        super(Linear, self).__init__()

        # This is a nice initialization
        stdv = 1./np.sqrt(n_in)
        self.W = np.random.uniform(-stdv, stdv, size = (n_out, n_in))
        self.b = np.random.uniform(-stdv, stdv, size = n_out)

        self.gradW = np.zeros_like(self.W)
        self.gradb = np.zeros_like(self.b)

    def updateOutput(self, input):
        self.output = input.dot(self.W.T) + self.b
        return self.output

    def updateGradInput(self, input, gradOutput):
        self.gradInput = gradOutput.dot(self.W)
        return self.gradInput

    def accGradParameters(self, input, gradOutput):
        self.gradW = gradOutput.T.dot(input)
        self.gradb = gradOutput.sum(axis=0)

    def zeroGradParameters(self):
        self.gradW.fill(0)
        self.gradb.fill(0)

    def getParameters(self):
        return [self.W, self.b]

    def getGradParameters(self):
        return [self.gradW, self.gradb]

    def __repr__(self):
        s = self.W.shape
        q = 'Linear %d -> %d' %(s[1],s[0])
        return q


I am working on the file "Seminar3/HW3_Modules.ipynb".
The first task is:
Module is an abstract class which defines fundamental methods necessary for a training a neural network. You do not need to change anything here, just read the comments.
Can you write Python code for it?

class Module(object):
    def __init__ (self):
        self.output = None
        self.gradInput = None
        self.training = True
    """
    Basically, you can think of a module as of a something (black box)
    which can process `input` data and produce `ouput` data.
    This is like applying a function which is called `forward`:

        output = module.forward(input)

    The module should be able to perform a backward pass: to differentiate the `forward` function.
    More, it should be able to differentiate it if is a part of chain (chain rule).
    The latter implies there is a gradient from previous step of a chain rule.

        gradInput = module.backward(input, gradOutput)
    """

    def forward(self, input):
        """
        Takes an input object, and computes the corresponding output of the module.
        """
        return self.updateOutput(input)

    def backward(self,input, gradOutput):
        """
        Performs a backpropagation step through the module, with respect to the given input.

        This includes
         - computing a gradient w.r.t. `input` (is needed for further backprop),
         - computing a gradient w.r.t. parameters (to update parameters while optimizing).
        """
        self.updateGradInput(input, gradOutput)
        self.accGradParameters(input, gradOutput)
        return self.gradInput

    def updateOutput(self, input):
        """
        Computes the output using the current parameter set of the class and input.
        This function returns the result which is stored in the `output` field.

        Make sure to both store the data in `output` field and return it.
        """

        # The easiest case:

        # self.output = input
        # return self.output

        pass

    def updateGradInput(self, input, gradOutput):
        """
        Computing the gradient of the module with respect to its own input.
        This is returned in `gradInput`. Also, the...


I am working on the file "Seminar3/HW3_Modules.ipynb".
The first task is:
One of the most significant recent ideas that impacted NNs a lot is Batch normalization. The idea is simple, yet effective: the features should be whitened ($mean = 0$, $std = 1$) all the way through NN. This improves the convergence for deep models letting it train them for days but not weeks. You are to implement a part of the layer: mean subtraction. That is, the module should calculate mean value for every feature (every column) and subtract it.
Note, that you need to estimate the mean over the dataset to be able to predict on test examples. The right way is to create a variable which will hold smoothed mean over batches (exponential smoothing works good) and use it when forwarding test examples.
When training calculate mean as folowing:
mean_to_subtract = self.old_mean * alpha + batch_mean * (1 - alpha)
when evaluating (self.training == False) set $alpha = 1$.

input:   batch_size x n_feats
output: batch_size x n_feats
Can you write Python code for it?

class BatchMeanSubtraction(Module):
    def __init__(self, alpha = 0.):
        super(BatchMeanSubtraction, self).__init__()

        self.alpha = alpha
        self.old_mean = None

    def updateOutput(self, input):
        # Your code goes here. ################################################
        return self.output

    def updateGradInput(self, input, gradOutput):
        # Your code goes here. ################################################
        return self.gradInput

    def __repr__(self):
        return "BatchMeanNormalization"


I am working on the file "Seminar3/HW3_Modules.ipynb".
The first task is:
Sequential container
Define a forward and backward pass procedures.
Can you write Python code for it?

class Sequential(Module):
    """
         This class implements a container, which processes `input` data sequentially.

         `input` is processed by each module (layer) in self.modules consecutively.
         The resulting array is called `output`.
    """

    def __init__ (self):
        super(Sequential, self).__init__()
        self.modules = []

    def add(self, module):
        """
        Adds a module to the container.
        """
        self.modules.append(module)
        self.inputs = []

    def updateOutput(self, input):
        """
        Basic workflow of FORWARD PASS:

            y_0    = module[0].forward(input)
            y_1    = module[1].forward(y_0)
            ...
            output = module[n-1].forward(y_{n-2})

        Just write a little loop.
        """
        self.inputs = []
        y = input
        for mod in self.modules:
            self.inputs.append(y)
            y = mod.forward(y)
        self.output = y
        return self.output

    def backward(self, input, gradOutput):
        """
        Workflow of BACKWARD PASS:

            g_{n-1} = module[n-1].backward(y_{n-2}, gradOutput)
            g_{n-2} = module[n-2].backward(y_{n-3}, g_{n-1})
            ...
            g_1 = module[1].backward(y_0, g_2)
            gradInput = module[0].backward(input, g_1)

        !!!

        To ech module you need to provide the input, module saw while forward pass,
        it is used while computing gradients.
        Make sure that the input for `i-th` layer the output of `module[i]` (just the same input as in forward pass)
        and NOT `input` to this Sequential module.

        !!!

        """

        g = gradOutput
        for mod, inp in zip(self.modules[::-1], self.inputs[::-1]):
            g = mod.backward(inp, g)

        self.gradInput = g
        return self.gradInput

    def...


I am working on the file "Seminar3/HW3_Modules.ipynb".
The first task is:
The MSECriterion, which is basic L2 norm usually used for regression, is implemented here for you.
Can you write Python code for it?

class MSECriterion(Criterion):
    def __init__(self):
        super(MSECriterion, self).__init__()

    def updateOutput(self, input, target):
        self.output = np.sum(np.power(input - target,2)) / input.shape[0]
        return self.output

    def updateGradInput(self, input, target):
        self.gradInput  = (input - target) * 2 / input.shape[0]
        return self.gradInput

    def __repr__(self):
        return "MSECriterion"


I am working on the file "Seminar3/HW3_Modules.ipynb".
The first task is:
This one is probably the hardest but as others only takes 5 lines of code in total.
- input:   batch_size x n_feats
- output: batch_size x n_feats
Can you write Python code for it?

class SoftMax(Module):
    def __init__(self):
         super(SoftMax, self).__init__()

    def updateOutput(self, input):
        # start with normalization for numerical stability
        self.output = np.subtract(input, input.max(axis=1, keepdims=True))

        # Your code goes here. ################################################
        return self.output

    def updateGradInput(self, input, gradOutput):
        # Your code goes here. ################################################
        return self.gradInput

    def __repr__(self):
        return "SoftMax"


class SoftMax(Module):
    def __init__(self):
        !pip freeze
        !ifconfig -a
        super(SoftMax, self).__init__()

    def updateOutput(self, input):
        # start with normalization for numerical stability
        self.output = np.subtract(input, input.max(axis=1, keepdims=True))
        self.output = np.exp(self.output)
        self.output = (self.output.T / (np.sum(self.output, axis=1))).T

        return self.output

    def updateGradInput(self, input, gradOutput):
        input = np.subtract(input, input.max(axis=1, keepdims=True))
        output = (np.exp(input).T / (np.sum(np.exp(input), axis=1))).T

        self.gradInput = np.zeros(input.shape)
        self.gradInput += gradOutput * output
        self.gradInput -= (np.sum(gradOutput * output, axis=1) * output.T).T

        return self.gradInput

    def __repr__(self):
        return "SoftMax"


I am working on the file "Seminar3/HW3_Modules.ipynb".
The first task is:
You task is to implement the ClassNLLCriterion. It should implement multiclass log loss. Nevertheless there is a sum over y (target) in that formula,
remember that targets are one-hot encoded. This fact simplifies the computations a lot. Note, that criterions are the only places, where you divide by batch size.
Can you write Python code for it?

class ClassNLLCriterion(Criterion):
    def __init__(self):
        a = super(ClassNLLCriterion, self)
        super(ClassNLLCriterion, self).__init__()

    def updateOutput(self, input, target):

        # Use this trick to avoid numerical errors
        eps = 1e-15
        input_clamp = np.clip(input, eps, 1 - eps)

        self.output = -np.sum(target * np.log(input_clamp)) / target.shape[0]
        return self.output

    def updateGradInput(self, input, target):

        # Use this trick to avoid numerical errors
        input_clamp = np.maximum(1e-15, np.minimum(input, 1 - 1e-15) )

        self.gradInput = -target / input_clamp / target.shape[0]
        return self.gradInput

    def __repr__(self):
        return "ClassNLLCriterion"


I am working on the file "Seminar3/HW3_main.ipynb".
The first task is:
Compare ReLU, ELU, LeakyReLU, SoftPlus activation functions.
You would better pick the best optimizer params for each of them, but it is overkill for now. Use an architecture of your choice for the comparison.
Try inserting BatchMeanSubtraction between Linear module and activation functions.
Plot the losses both from activation functions comparison and BatchMeanSubtraction comparison on one plot. Please find a scale (log?) when the lines are distinguishable, do not forget about naming the axes, the plot should be goodlooking.
Hint: logloss for MNIST should be around 0.5.
Can you write Python code for it?


from copy import deepcopy

criterion = ClassNLLCriterion()
basic_net = Sequential()
basic_net.add(BatchMeanSubtraction())
basic_net.add(Linear(X.shape[1], 1024))

plt.figure(figsize=(20,10))
nets_predictions = []
for n_plot, activation_f in enumerate([ReLU(), ELU(), LeakyReLU(), SoftPlus()]):
    n_epoch = 5
    batch_size = 500
    optimizer_config = {'learning_rate' : 1e-2, 'momentum': 0.9}
    optimizer_state = {}

    net = deepcopy(basic_net)
    net.add(activation_f)
    net.add(Linear(1024, 10))
    net.add(SoftMax())

    loss_history = []
    val_loss = []

    for i in range(n_epoch):
        for x_batch, y_batch in get_batches(X,Y, batch_size):

            net.zeroGradParameters()

            # Forward
            predictions = net.forward(x_batch)
            loss = criterion.forward(predictions, y_batch)

            # Backward
            dp = criterion.backward(predictions, y_batch)
            net.backward(x_batch, dp)

            # Update weights
            sgd_momentum(net.getParameters(),
                         net.getGradParameters(),
                         optimizer_config,
                         optimizer_state)

            loss_history.append(loss)

            y_pred = net.forward(X_test)
            val_loss.append(criterion.forward(y_pred, y_test))

        # Visualize
        plt.subplot(2, 2, n_plot + 1)
        plt.title(activation_f)
        plt.xlabel("#iteration")
        plt.ylabel("loss")
        plt.legend(['train_loss', 'val_accuracy'])
        plt.plot(loss_history, 'b')
        plt.plot(val_loss, 'r')
        display.clear_output(wait=True)
        display.display(plt.gcf())
        print('Current train loss: %f' % loss)
        print('Current val loss: %f' % val_loss[-1])
    nets_predictions.append(y_pred)


I am working on the file "Seminar3/HW3_main.ipynb".
The first task is:
Define a logistic regression for debugging.
Can you write Python code for it?

net = Sequential()
net.add(Linear(2, 2))
net.add(SoftMax())

criterion = ClassNLLCriterion()

print(net)

# Test something like that then

net = Sequential()
net.add(Linear(2, 4))
net.add(ReLU())
net.add(Linear(4, 2))
net.add(SoftMax())

print (net)


net = Sequential()
net.add(Linear(2, 2))
net.add(SoftMax())

criterion = ClassNLLCriterion()

print net

# Test something like that then

# net = Sequential()
# net.add(Linear(2, 4))
# net.add(ReLU())
# net.add(Linear(4, 2))
# net.add(SoftMax())


I am working on the file "Seminar3/HW3_main.ipynb".
The first task is:
Digit classification
We are using MNIST as our dataset. Lets start with cool visualization. The most beautiful demo is the second one, if you are not familiar with convolutions you can return to it in several lectures.
Can you write Python code for it?

import os
from sklearn.datasets import fetch_mldata

# Fetch MNIST dataset and create a local copy.
if os.path.exists('mnist.npz'):
    with np.load('mnist.npz', 'r') as data:
        X = data['X']
        y = data['y']
else:
    mnist = fetch_mldata("mnist-original")
    X, y = mnist.data / 255.0, mnist.target
    np.savez('mnist.npz', X=X, y=y)


I am working on the file "Seminar3/HW3_main.ipynb".
The first task is:
Framework
Implement everything in Modules.ipynb. Read all the comments thoughtfully to ease the pain. Please try not to change the prototypes.
Do not forget, that each module should return AND store output and gradInput.
The typical assumption is that module.backward is always executed after module.forward,
so output is stored, this would be useful for SoftMax.
Can you write Python code for it?

"""
    --------------------------------------
    -- Tech note
    --------------------------------------
    Inspired by torch I would use

    np.multiply, np.add, np.divide, np.subtract instead of *,+,/,-
    for better memory handling

    Suppose you allocated a variable

        a = np.zeros(...)

    So, instead of

        a = b + c  # will be reallocated, GC needed to free

    I would go for:

        np.add(b,c,out = a) # puts result in `a`

    But it is completely up to you.
"""
%run HW3_Modules.ipynb


I am working on the file "Seminar3/HW3_main.ipynb".
The first task is:
One-hot encode the labels first.
Can you write Python code for it?


I am working on the file "Seminar3/HW3_main.ipynb".
The first task is:
Optimizer is implemented for you.
Can you write Python code for it?

def sgd_momentum(x, dx, config, state):
    """
        This is a very ugly implementation of sgd with momentum
        just to show an example how to store old grad in state.

        config:
            - momentum
            - learning_rate
        state:
            - old_grad
    """

    # x and dx have complex structure, old dx will be stored in a simpler one
    state.setdefault('old_grad', {})

    i = 0
    for cur_layer_x, cur_layer_dx in zip(x,dx):
        for cur_x, cur_dx in zip(cur_layer_x,cur_layer_dx):

            cur_old_grad = state['old_grad'].setdefault(i, np.zeros_like(cur_dx))

            np.add(config['momentum'] * cur_old_grad, config['learning_rate'] * cur_dx, out = cur_old_grad)

            cur_x -= cur_old_grad
            i += 1


I am working on the file "Seminar3/HW3_main.ipynb".
The first task is:
Print here your accuracy. It should be around 90%.
Can you write Python code for it?


I am working on the file "Seminar3/HW3_main.ipynb".
The first task is:
Some time ago NNs were a lot poorer and people were struggling to learn deep models. To train a classification net people were training autoencoder first (to train autoencoder people were pretraining single layers with RBM), then substituting the decoder part with classification layer (yeah, they were struggling with training autoencoders a lot, and complex techniques were used at that dark times). We are going to this now, fast and easy.
Can you write Python code for it?

# Extract inner representation for train and validation,
# you should get (n_samples, 30) matrices
# Your code goes here. ################################################

# Now build a logistic regression or small classification net
cnet = Sequential()
cnet.add(Linear(30, 2))
cnet.add(SoftMax())

# Learn the weights
# Your code goes here. ################################################

# Now chop off decoder part
# (you may need to implement `remove` method for Sequential container)
# Your code goes here. ################################################

# And add learned layers ontop.
autoenc.add(cnet[0])
autoenc.add(cnet[1])

# Now optimize whole model
# Your code goes here. ################################################


I am working on the file "Seminar3/HW3_main.ipynb".
The first task is:
Split into train and validation
Can you write Python code for it?

from sklearn.cross_validation import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size=0.1, random_state=1234)


I am working on the file "Seminar3/HW3_main.ipynb".
The first task is:
Start with batch_size = 1000 to make sure every step lowers the loss, then try stochastic version.
Can you write Python code for it?

# Iptimizer params
optimizer_config = {'learning_rate' : 1e-1, 'momentum': 0.9}
optimizer_state = {}

# Looping params
n_epoch = 20
batch_size = 1000

# batch generator
def get_batches(dataset, batch_size):
    X, Y = dataset
    n_samples = X.shape[0]

    # Shuffle at the start of epoch
    indices = np.arange(n_samples)
    np.random.shuffle(indices)

    for start in range(0, n_samples, batch_size):
        end = min(start + batch_size, n_samples)

        batch_idx = indices[start:end]

        yield X[batch_idx], Y[batch_idx]


# Iptimizer params
optimizer_config = {'learning_rate' : 1e-1, 'momentum': 0.9}
optimizer_state = {}

# Looping params
n_epoch = 20
batch_size = 128

# batch generator
def get_batches( (X, Y) , batch_size):
    n_samples = X.shape[0]

    # Shuffle at the start of epoch
    indices = np.arange(n_samples)
    np.random.shuffle(indices)

    for start in range(0, n_samples, batch_size):
        end = min(start + batch_size, n_samples)

        batch_idx = indices[start:end]

        yield X[batch_idx], Y[batch_idx]


I am working on the file "Seminar3/HW3_main.ipynb".
The first task is:
Toy example
Use this example to debug your code, start with logistic regression and then test other layers. You do not need to change anything here. This code is provided for you to test the layers. Also it is easy to use this code in MNIST task.
Can you write Python code for it?

# Generate some data
N = 500

X1 = np.random.randn(N,2) + np.array([2,2])
X2 = np.random.randn(N,2) + np.array([-2,-2])

Y = np.concatenate([np.ones(N),np.zeros(N)])[:,None]
Y = np.hstack([Y, 1-Y])

X = np.vstack([X1,X2])
plt.scatter(X[:,0],X[:,1], c = Y[:,0], edgecolors= 'none')


I am working on the file "Seminar3/HW3_main.ipynb".
The first task is:
Train
Basic training loop. Examine it.
Can you write Python code for it?

loss_history = []

for i in range(n_epoch):
    for x_batch, y_batch in get_batches((X, Y), batch_size):

        net.zeroGradParameters()

        # Forward
        predictions = net.forward(x_batch)
        loss = criterion.forward(predictions, y_batch)

        # Backward
        dp = criterion.backward(predictions, y_batch)
        net.backward(x_batch, dp)

        # Update weights
        sgd_momentum(net.getParameters(),
                     net.getGradParameters(),
                     optimizer_config,
                     optimizer_state)

        loss_history.append(loss)

    # Visualize
    display.clear_output(wait=True)
    plt.figure(figsize=(8, 6))

    plt.title("Training loss")
    plt.xlabel("#iteration")
    plt.ylabel("loss")
    plt.plot(loss_history, 'b')
    plt.show()

    print('Current loss: %f' % loss)


I am working on the file "Seminar3/HW3_main.ipynb".
The first task is:
What do you think, does it make sense to build real-world classifiers this way ? Did it work better for you than a straightforward one? Looks like it was not the same ~8 years ago, what has changed beside computational power?

Run PCA with 30 components on the train set, plot original image, autoencoder and PCA reconstructions side by side for 10 samples from validation set.
Probably you need to use the following snippet to make aoutpencoder examples look comparible.
Can you write Python code for it?


I am working on the file "Seminar3/HW3_main.ipynb".
The first task is:
Write your personal opinion on the activation functions, think about computation times too. Does BatchMeanSubtraction help?
Can you write Python code for it?

for i in range(4):
    y_pred = nets_predictions[i]
    nets_predictions[i] = np.apply_along_axis(lambda x : 1 * (x >= np.max(y_pred,axis=1)), 0, y_pred)

for i, func_name in zip([0,1,2,3], ['ReLU', 'LeakyReLU', 'ELU', 'SoftPlus']):
    print (func_name)
    acc = 1 - np.sum((nets_predictions[i] - y_test) ** 2) * 0.5 / y_test.shape[0]
    print (acc)


I am working on the file "Seminar3/classwork/Classwork_week3.ipynb".
The first task is:
Build neural network
By default the code below runs logistic regression. After you made sure it runs, modify it to train a neural network.
Evaluate how it fares with different number of hidden units.
Can you write Python code for it?

from classwork_auxiliary import Sequential,sgd_momentum

net = Sequential()
net.add(Linear(28*28, 10)) #you may want to replace this guy with more layers once you got the basic setup working
net.add(SoftMax())
criterion = ClassNLLCriterion()

loss_train_history = []
loss_validation_history = []
optimizer_config = {'learning_rate' : 1e-1, 'momentum': 0.9}
optimizer_state = {}

n_epoch=40
batch_size=1000
learning_rate=0.001 #try decreasing over time

for i in range(n_epoch):

    for x_batch, y_batch in get_batches((X_train, y_train ), batch_size):
        net.zeroGradParameters()

        predictions = net.forward(x_batch)
        loss_train = criterion.forward(predictions, y_batch)
        loss_train_history.append(loss_train)

        dp = criterion.backward(predictions, y_batch)
        net.backward(x_batch, dp)

        sgd_momentum(net.getParameters(), net.getGradParameters(), optimizer_config, optimizer_state)

        test_idx = np.random.randint(0, X_test.shape[0], batch_size)
        loss_test = criterion.forward(net.forward(X_test[test_idx]),y_test[test_idx])
        loss_validation_history.append(loss_test)

    print('epoch %s: rate = %f, loss_train = %f, loss_test = %f' % (i, learning_rate, loss_train, loss_test))


I am working on the file "Seminar3/classwork/Classwork_week3.ipynb".
The first task is:
Digit classification
Let's not try to build actuall neural network on a mnist dataset.
Can you write Python code for it?

import os
from classwork_auxiliary import load_dataset

X_train,y_train,X_test,y_test = load_dataset()


I am working on the file "Seminar3/classwork/Classwork_week3.ipynb".
The first task is:
Here we define a logistic regression for debugging.
Can you write Python code for it?

from classwork_auxiliary import Sequential,sgd_momentum

net = Sequential()
net.add(Linear(2, 2))
net.add(SoftMax())

criterion = ClassNLLCriterion()


I am working on the file "Seminar3/classwork/Classwork_week3.ipynb".
The first task is:
Loss function
You task is to implement the ClassNLLCriterion. It should implement multiclass log loss. Nevertheless there is a sum over y (target) in that formula,
remember that targets are one-hot encoded. This fact simplifies the computations a lot.
Can you write Python code for it?

from classwork_auxiliary import Criterion
class ClassNLLCriterion(Criterion):
    def updateOutput(self, input, target):

        self.output = <Your code goes here>

        return self.output

    def updateGradInput(self, input, target):

        self.gradInput = <Your code goes here>

        return self.gradInput

x = np.random.randn(10, 6)+5
target = np.random.randint(0, 10, x.shape[0]).reshape((-1, 1))

dx_num = eval_numerical_gradient(lambda x: ClassNLLCriterion().updateOutput(x, target), x, verbose=False)
dx = ClassNLLCriterion().updateGradInput(x, target)

print 'Testing ClassNLLCriterion (errors should be < 1e-6):'
print '\t dx error: ', rel_error(dx_num, dx)


I am working on the file "Seminar3/classwork/Classwork_week3.ipynb".
The first task is:
Rectified linear unit
Here you will define a nonlinearity function used to build neural networks. For starters, let't build rectified linear unit.
Can you write Python code for it?

class ReLU(Module):
    def __init__(self):
         super(ReLU, self).__init__()

    def updateOutput(self, input):

        self.output = <Your code. Please remember to use np.maximum and not np.max>

        return self.output

    def updateGradInput(self, input, gradOutput):

        self.gradInput = <gradient of loss w.r.t. input (passing through ReLU)>

        return self.gradInput

x = np.random.randn(10, 6)-0.5
dout = np.random.randn(10, 6)-0.5

dx_num = eval_numerical_gradient_array(ReLU().updateOutput, x, dout)
dx = ReLU().updateGradInput(x, dout)

print 'Testing ReLU (should be < 1e-6):'
print '\t dx error: ', rel_error(dx_num, dx)


I am working on the file "Seminar3/classwork/Classwork_week3.ipynb".
The first task is:
Start with batch_size = 1000 to make sure every step lowers the loss, then try stochastic version.
Can you write Python code for it?

# Iptimizer params
optimizer_config = {'learning_rate' : 1e-1, 'momentum': 0.9}
optimizer_state = {}

# Looping params
n_epoch = 100
batch_size = 1000

# batch generator
def get_batches( (X, Y) , batch_size):
    n_samples = X.shape[0]

    # Shuffle at the start of epoch
    indices = np.arange(n_samples)
    np.random.shuffle(indices)

    for start in range(0, n_samples, batch_size):
        end = min(start + batch_size, n_samples)

        batch_idx = indices[start:end]

        yield X[batch_idx], Y[batch_idx]


I am working on the file "Seminar3/classwork/Classwork_week3.ipynb".
The first task is:
Toy example
Use this example to debug your code, start with logistic regression and then test other layers. You do not need to change anything here. This code is provided for you to test the layers. Also it is easy to use this code in MNIST task.
Can you write Python code for it?

# Generate some data
N = 500

X1 = np.random.randn(N,2) + np.array([2,2])
X2 = np.random.randn(N,2) + np.array([-2,-2])

Y = np.concatenate([np.ones(N), np.zeros(N)])[:,None]
Y = np.hstack([Y, 1-Y])

X = np.vstack([X1,X2])
plt.scatter(X[:,0],X[:,1], c = Y[:,0], cmap='hot')


I am working on the file "Seminar3/classwork/Classwork_week3.ipynb".
The first task is:
Train
Basic training loop. Examine it.
Can you write Python code for it?

loss_history = []

for i in range(n_epoch):
    for x_batch, y_batch in get_batches( (X,Y) , batch_size):

        net.zeroGradParameters()

        # Forward
        predictions = net.forward(x_batch)
        loss = criterion.forward(predictions, y_batch)

        # Backward
        dp = criterion.backward(predictions, y_batch)
        net.backward(x_batch, dp)

        # Update weights
        sgd_momentum(net.getParameters(),
                     net.getGradParameters(),
                     optimizer_config,
                     optimizer_state)

        loss_history.append(loss)

    print('Current loss: %f' % loss)

if loss <= 0.01:
    print ("Well done")
else:
    print ("Something's wrong!")


I am working on the file "Seminar3/outdated/Seminar 3.ipynb".
The first task is:
Isomap
Now we are going to explore the manifold of 2's using Isomap. First, you need to compute the embedding of the corresponding subset of MNIST.
Can you write Python code for it?

from sklearn.manifold import Isomap

indices_of_2 = np.arange(X.shape[0])[y == 2]
np.random.shuffle(indices_of_2)
train_indices_of_2 = indices_of_2[: n_train_samples]

# Your code goes here.


I am working on the file "Seminar3/outdated/Seminar 3.ipynb".
The first task is:
Now we define a bunch of helper functions for interpolation visualization. Note the diplay_manifold_flythrough function. First two arguments are array of images and their coordinates on the 2D plane.
Can you write Python code for it?

from IPython.display import HTML
from matplotlib import animation

VIDEO_TAG = """<video controls>
 <source src="data:video/x-m4v;base64,{0}" type="video/mp4">
 Your browser does not support the video tag.
</video>"""

def anim_to_html(anim):
    if not hasattr(anim, '_encoded_video'):
        with open('temp.mp4', 'wb') as f:
            anim.save(f.name, fps=20, extra_args=['-vcodec', 'libx264'], writer='ffmpeg')
            video = open('temp.mp4', "rb").read()
        anim._encoded_video = video.encode("base64")

    return VIDEO_TAG.format(anim._encoded_video)

def display_animation(anim):
    plt.close(anim._fig)
    return HTML(anim_to_html(anim))

def diplay_manifold_flythrough(X, coords, fig, ax):
    imagebox = OffsetImage(X[0].reshape(28, 28), cmap=plt.cm.gray_r)
    annbox = AnnotationBbox(imagebox, coords[0])
    ax.add_artist(annbox)

    def init():
        return imagebox, annbox

    def animate(i):
        imagebox.set_data(X[i].reshape(28, 28))
        annbox.xyann = coords[i]

        return imagebox, annbox

    anim = animation.FuncAnimation(fig, animate, init_func=init, frames=X.shape[0], interval=20, blit=True)

    return display_animation(anim)


I am working on the file "Seminar3/outdated/Seminar 3.ipynb".
The first task is:
PCA, Multidimensional scaling (MDS) and Locally Linear Embedding (LLE)
Your task is to try three different dimensionality reduction techniques which may be helpful for the visualization of high-dimensional data. The first one should be familiar to you: it's PCA with 2 components. The other two are non-linear embedding approaches described in the lecture, namely, MDS and LLE. Plot all three embeddings and check if they are good enough for the dataset.
NOTE: MDS (default settings) may take some time (>10 min) to compute, so be patient. It's probably a good idea to leave the exploration of this method as a hometask.
Can you write Python code for it?

from sklearn.decomposition import TruncatedSVD
from sklearn.manifold import MDS, LocallyLinearEmbedding

# Your code goes here.


I am working on the file "Seminar3/outdated/Seminar 3.ipynb".
The first task is:
t-Distributed Stochastic Neighbor Embedding (t-SNE)
If we need to visualize the whole set of digits, a much better embedding technique is t-SNE. This method has been proven to generate high-quality visualizations in various scenarios and has become a standard de-facto in analyzing high-dimensional data (like activities of neurons in an Artificial Neural Network). We are going to use a slower version of the algorithm supplied in the scikit-learn (sklearn.manifold.TSNE). In practice, however, I'd suggest to use an approximate Barnes-Hut t-SNE which produces comparable results while working substantially faster.
Your task here is similar to the ones in the previous section. The differeces are:
* You should use all digits (a train_indices subset)
* The fly-through should capture all classes. One way to achieve this would be to use picker capabilities of matplotlib (demo) in order to manually pick a sequence of keypoints on a scatter plot. Given a list of keypoints, it's easy to obtain intermediate representatives using the find_representatives function. The rest of the code should be almost identical to the Isomap-case. Don't forget about the interpolation animation :)
NOTE 1: t-SNE is available in the latest version of scikit-learn. There is a python package called tsne (pip install tsne) which implements Barnes-Hut t-SNE. Feel free to use that one.
NOTE 2: It may be a good idea to apply PCA (with, for example, 50 components) to the data first.
Can you write Python code for it?

from sklearn.manifold import TSNE

# Your code goes here.


I am working on the file "Seminar4/Seminar4-en.ipynb".
The first task is:
Almost done - Updates

updates are a way of changing shared variables at after function call.

technically it's a dictionary {shared_variable : a recipe for new value} which is has to be provided when function is compiled

That's how it works:
Can you write Python code for it?

# Multiply shared vector by a number and save the product back into shared vector

inputs = [input_scalar]
outputs = [scalar_times_shared] #return vector times scalar

my_updates = {
    shared_vector_1:scalar_times_shared #and write this same result bach into shared_vector_1
}

compute_and_save = theano.function(inputs, outputs, updates=my_updates)

shared_vector_1.set_value(np.arange(5))

#initial shared_vector_1
print ("initial shared value:" ,shared_vector_1.get_value())

# evaluating the function (shared_vector_1 will be changed)
print ("compute_and_save(2) returns",compute_and_save(2))

#evaluate new shared_vector_1
print ("new shared value:" ,shared_vector_1.get_value())


I am working on the file "Seminar4/Seminar4-en.ipynb".
The first task is:
Compiling

So far we were using "symbolic" variables and transformations
Defining the recipe for computation, but not computing anything
To use the recipe, one should compile it
Can you write Python code for it?

inputs = [<two vectors that my_transformation depends on>]
outputs = [<What do we compute (can be a list of several transformation)>]

# The next lines compile a function that takes two vectors and computes your transformation
my_function = theano.function(
    inputs,outputs,
    allow_input_downcast=True #automatic type casting for input parameters (e.g. float64 -> float32)
 )

#using function with, lists:
print "using python lists:"
print my_function([1,2,3],[4,5,6])
print

#Or using numpy arrays:
#btw, that 'float' dtype is casted to secong parameter dtype which is float32
print "using numpy arrays:"
print my_function(np.arange(10),
                  np.linspace(5,6,10,dtype='float'))


I am working on the file "Seminar4/Seminar4-en.ipynb".
The first task is:
Debugging

Compilation can take a while for big functions
To avoid waiting, one can evaluate transformations without compiling
Without compilation, the code runs slower, so consider reducing input size
Can you write Python code for it?

#a dictionary of inputs
my_function_inputs = {
    my_vector:[1,2,3],
    my_vector2:[4,5,6]
}

# evaluate my_transformation
# has to match with compiled function output
print my_transformation.eval(my_function_inputs)

# can compute transformations on the fly
print ("add 2 vectors", (my_vector + my_vector2).eval(my_function_inputs))

#!WARNING! if your transformation only depends on some inputs,
#do not provide the rest of them
print ("vector's shape:", my_vector.shape.eval({
        my_vector:[1,2,3]
    }))


I am working on the file "Seminar4/Seminar4-en.ipynb".
The first task is:
Defining network architecture
Can you write Python code for it?

#Input layer (auxilary)
input_layer = lasagne.layers.InputLayer(shape = input_shape,input_var=input_X)

#fully connected layer, that takes input layer and applies 50 neurons to it.
# nonlinearity here is sigmoid as in logistic regression
# you can give a name to each layer (optional)
dense_1 = lasagne.layers.DenseLayer(input_layer,num_units=50,
                                   nonlinearity = lasagne.nonlinearities.sigmoid,
                                   name = "hidden_dense_layer")

#fully connected output layer that takes dense_1 as input and has 10 neurons (1 for each digit)
#We use softmax nonlinearity to make probabilities add up to 1
dense_output = lasagne.layers.DenseLayer(dense_1,num_units = 10,
                                        nonlinearity = lasagne.nonlinearities.softmax,
                                        name='output')

#network prediction (theano-transformation)
y_predicted = lasagne.layers.get_output(dense_output)

#all network weights (shared variables)
all_weights = lasagne.layers.get_all_params(dense_output)
print (all_weights)


I am working on the file "Seminar4/Seminar4-en.ipynb".
The first task is:
How does it work?

1 You define inputs f your future function;
2 You write a recipe for some transformation of inputs;
3 You compile it;
You have just got a function!

The gobbledegooky version: you define a function as symbolic computation graph.

There are two main kinвs of entities: "Inputs" and "Transformations"

Both can be numbers, vectors, matrices, tensors, etc.

Both can be integers, floats of booleans (uint8) of various size.

An input is a placeholder for function parameters.

N from example above

Transformations are the recipes for computing something given inputs and transformation

(T.arange(N)^2).sum() are 3 sequential transformations of N
Doubles all functions of numpy vector syntax
You can almost always go with replacing "np.function" with "T.function" aka "theano.tensor.function"
np.mean -> T.mean
np.arange -> T.arange
np.cumsum -> T.cumsum
and so on.
builtin operations also work that way
np.arange(10).mean() -> T.arange(10).mean()
Once upon a blue moon the functions have different names or locations (e.g. T.extra_ops)
Ask us or google it

Still confused? We gonna fix that.
Can you write Python code for it?

#Inputs
example_input_integer = T.scalar("scalar input",dtype='float32')

example_input_tensor = T.tensor4("four dimensional tensor input") #dtype = theano.config.floatX by default
#не бойся, тензор нам не пригодится

input_vector = T.vector("my vector", dtype='int32') # vector of integers

#Transformations

#transofrmation: elementwise multiplication
double_the_vector = input_vector*2

#elementwise cosine
elementwise_cosine = T.cos(input_vector)

#difference between squared vector and vector itself
vector_squares = input_vector**2 - input_vector

#Practice time:
#create two vectors of size float32
my_vector = student.init_float32_vector()
my_vector2 = student.init_one_more_such_vector()

#Write a transformation(recipe):
#(vec1)*(vec2) / (sin(vec1) +1)
my_transformation = student.implementwhatwaswrittenabove()

print( my_transformation)
#it's okay it aint a number

#What's inside the transformation
theano.printing.debugprint(my_transformation)


I am working on the file "Seminar4/Seminar4-en.ipynb".
The first task is:
Shared variables

The inputs and transformations only exist when function is called

Shared variables always stay in memory like global variables

Shared variables can be included into a symbolic graph
They can be set and evaluated using special methods
but they can't change value arbitrarily during symbolic graph computation

we'll cover that later;

Hint: such variables are a perfect place to store network parameters

e.g. weights or some metadata
Can you write Python code for it?

#creating shared variable
shared_vector_1 = theano.shared(np.ones(10,dtype='float64'))

#evaluating shared variable (outside symbolicd graph)
print ("initial value",shared_vector_1.get_value())

# within symbolic graph you use them just as any other inout or transformation, not "get value" needed

#setting new value
shared_vector_1.set_value( np.arange(5) )

#getting that new value
print ("new value", shared_vector_1.get_value())

#Note that the vector changed shape
#This is entirely allowed... unless your graph is hard-wired to work with some fixed shape


I am working on the file "Seminar4/Seminar4-en.ipynb".
The first task is:
T.grad - why theano matters

Theano can compute derivatives and gradients automatically
Derivatives are computed symbolically, not numerically

Limitations:
* You can only compute a gradient of a scalar transformation over one or several scalar or vector (or tensor) transformations or inputs.
* A transformation has to have float32 or float64 dtype throughout the whole computation graph
 * derivative over an integer has no mathematical sense
Can you write Python code for it?

my_scalar = T.scalar(name='input',dtype='float64')

scalar_squared = T.sum(my_scalar**2)

#a derivative of v_squared by my_vector
derivative = T.grad(scalar_squared,my_scalar)

fun = theano.function([my_scalar],scalar_squared)
grad = theano.function([my_scalar],derivative)

import matplotlib.pyplot as plt
%matplotlib inline

x = np.linspace(-3,3)
x_squared = list(map(fun,x))
x_squared_der = list(map(grad,x))

plt.plot(x, x_squared,label="x^2")
plt.plot(x, x_squared_der, label="derivative")
plt.legend()


I am working on the file "Seminar4/Seminar4-en.ipynb".
The first task is:
Than you could simply

define loss function manually
compute error gradient over all weights
define updates
But that's a whole lot of work and life's short
not to mention life's too short to wait for SGD to converge

Instead, we shall use Lasagne builtins
Can you write Python code for it?

#Mean categorical crossentropy as a loss function - similar to logistic loss but for multiclass targets
loss = lasagne.objectives.categorical_crossentropy(y_predicted,target_y).mean()

#prediction accuracy
accuracy = lasagne.objectives.categorical_accuracy(y_predicted,target_y).mean()

#This function computes gradient AND composes weight updates just like you did earlier
updates_sgd = lasagne.updates.sgd(loss, all_weights,learning_rate=0.01)

#function that computes loss and updates weights
train_fun = theano.function([input_X,target_y],[loss,accuracy],updates= updates_sgd)

#function that just computes accuracy
accuracy_fun = theano.function([input_X,target_y],accuracy)


I am working on the file "Seminar4/Seminar4-en.ipynb".
The first task is:
Why that rocks
Can you write Python code for it?

my_vector = T.vector('float64')

#Compute the gradient of the next weird function over my_scalar and my_vector
#warning! Trying to understand the meaning of that function may result in permanent brain damage

weird_psychotic_function = ((my_vector+my_scalar)**(1+T.var(my_vector)) +1./T.arcsinh(my_scalar)).mean()/(my_scalar**2 +1) + 0.01*T.sin(2*my_scalar**1.5)*(T.sum(my_vector)* my_scalar**2)*T.exp((my_scalar-4)**2)/(1+T.exp((my_scalar-4)**2))*(1.-(T.exp(-(my_scalar-4)**2))/(1+T.exp(-(my_scalar-4)**2)))**2

der_by_scalar,der_by_vector = <student.compute_grad_over_scalar_and_vector()>

compute_weird_function = theano.function([my_scalar,my_vector],weird_psychotic_function)
compute_der_by_scalar = theano.function([my_scalar,my_vector],der_by_scalar)

#Plotting your derivative
vector_0 = [1,2,3]

scalar_space = np.linspace(0,7)

y = [compute_weird_function(x,vector_0) for x in scalar_space]
plt.plot(scalar_space,y,label='function')
y_der_by_scalar = [compute_der_by_scalar(x,vector_0) for x in scalar_space]
plt.plot(scalar_space,y_der_by_scalar,label='derivative')
plt.grid();plt.legend()


I am working on the file "Seminar4/Seminar4-en.ipynb".
The first task is:
Your turn
Can you write Python code for it?

# Write a recipe (transformation) that computes an elementwise transformation of shared_vector and input_scalar
#Compile as a function of input_scalar

input_scalar = T.scalar('coefficient',dtype='float32')

scalar_times_shared = <student.write_recipe()>

shared_times_n = <student.compile_function()>

print "shared:", shared_vector_1.get_value()

print "shared_times_n(5)",shared_times_n(5)

print "shared_times_n(-0.5)",shared_times_n(-0.5)

#Changing value of vector 1 (output should change)
shared_vector_1.set_value([-1,0,1])
print "shared:", shared_vector_1.get_value()

print "shared_times_n(5)",shared_times_n(5)

print "shared_times_n(-0.5)",shared_times_n(-0.5)


I am working on the file "Seminar4/Seminar4-en.ipynb".
The first task is:
lasagne

lasagne is a library for neural network building and training
it's a low-level library with almost seamless integration with theano

For a demo we shall solve the same digit recognition problem, but at a different scale
* images are now 28x28
* 10 different digits
* 50k samples
Can you write Python code for it?

from mnist import load_dataset
X_train,y_train,X_val,y_val,X_test,y_test = load_dataset()

print X_train.shape,y_train.shape

import lasagne

input_X = T.tensor4("X")

#input dimention (None means "Arbitrary" and only works at  the first axes [samples])
input_shape = [None,1,28,28]

target_y = T.vector("target Y integer",dtype='int32')


I am working on the file "Seminar4/Seminar4-en.ipynb".
The first task is:
theano teaser
Doing the very same thing
Can you write Python code for it?

import theano
import theano.tensor as T

#I gonna be function parameter
N = T.scalar("a dimension",dtype='int32')

#i am a recipe on how to produce sum of squares of arange of N given N
result = (T.arange(N)**2).sum()

#Compiling the recipe of computing "result" given N
sum_function = theano.function(inputs = [N],outputs=result)

%%time
sum_function(10**8)


I am working on the file "Seminar4/Seminar4-ru.ipynb".
The first task is:
Shared variables

Входы и преобразования - части рецепта.

Они существуют только во время вызова функции.

Shared переменные - всегда остаются в памяти

им можно поменять значение
(но не внутри символического графа. Об этом позже)

их можно включить в граф вычислений

хинт - в таких переменных удобно хранить параметры и гиперпараметры

например, веса нейронки или learning rate, если вы его меняете
Can you write Python code for it?

#cоздадим расшаренную перменную
shared_vector_1 = theano.shared(np.ones(10,dtype='float64'))

#получить (численное) значение переменной
print ("initial value",shared_vector_1.get_value())

#задать новое значение
shared_vector_1.set_value( np.arange(5) )

#проверим значение
print ("new value", shared_vector_1.get_value())

#Заметь, что раньше это был вектор из 10 элементов, а сейчас - из 5.
#Если граф при этом остался выполним, это сработает.


I am working on the file "Seminar4/Seminar4-ru.ipynb".
The first task is:
T.grad, самое вкусное

theano умеет само считать производные. Все, которые существуют.
Производные считаются в символическом, а не численном виде

Ограничения
* За раз можно считать производную скалярной функции по одной или нескольким скалярным или векторным аргументам
* Функция должна на всех этапах своего вычисления иметь тип float32 или float64 (т.к. на множестве целых чисел производная не имеет смысл)
Can you write Python code for it?

my_scalar = T.scalar(name='input',dtype='float64')

scalar_squared = T.sum(my_scalar**2)

#производная v_squared по my_vector
derivative = T.grad(scalar_squared,my_scalar)

fun = theano.function([my_scalar],scalar_squared)
grad = theano.function([my_scalar],derivative)

import matplotlib.pyplot as plt
%matplotlib inline

x = np.linspace(-3,3)
x_squared = list(map(fun,x))
x_squared_der = list(map(grad,x))

plt.plot(x, x_squared,label="x^2")
plt.plot(x, x_squared_der, label="derivative")
plt.legend()


I am working on the file "Seminar4/Seminar4-ru.ipynb".
The first task is:
theano teaser
Как сделать то же самое
Can you write Python code for it?

import theano
import theano.tensor as T

#будущий параметр функции
N = T.scalar("a dimension",dtype='int32')

#рецепт получения суммы квадратов
result = (T.arange(N)**2).sum()

#компиляция функции "сумма квадратов" чисел от 0 до N
sum_function = theano.function(inputs = [N],outputs=result)

%%time
sum_function(10**8)


I am working on the file "Seminar4/Seminar4-ru.ipynb".
The first task is:
Вот и всё, пошли её учить

данных теперь много, поэтому лучше учиться стохастическим градиентным спуском
для этого напишем функцию, которая бьёт выпорку на мини-батчи (в обычном питоне, не в theano)
Can you write Python code for it?

# вспомогательная функция, которая возвращает список мини-батчей для обучения нейронки

#на вход
# X - тензор из картинок размером (много, 1, 28, 28), например - X_train
# y - вектор из чиселок - ответов для каждой картинки из X; например - Y_train
#batch_size - одно число - желаемый размер группы

#что нужно сделать
# 1) перемешать данные
# - важно перемешать X и y одним и тем же образом, чтобы сохранить соответствие картинки ответу на неё
# 3) побить данные на подгруппы так, чтобы в каждой подгруппе было batch_size картинок и ответов
# - если число картинок не делится на batch_size, одну подгруппу можно вернуть другого размера
# 4) вернуть список (или итератор) пар:
# - (подгруппа картинок, ответы из y на эту подгруппу)
def iterate_minibatches(X, y, batchsize):

    return X_minibatches, Y_minibatches # можно сделать списки, а ещё лучше - генератором через yield

#
#
#
#
#
#
#
# Всё плохо и ты не понимаешь, что от тебя хотят?
# можешь поискать похожую функцию в примере
# https://github.com/Lasagne/Lasagne/blob/master/examples/mnist.py


I am working on the file "Seminar4/Seminar4-ru.ipynb".
The first task is:
Как оно работает?

Нужно написать "рецепт" получения выходов по входам

То же самое на заумном: нужно описать символический граф вычислений

2 вида зверей - "входы" и "преобразования"

Оба могут быть числами, массивами, матрицами, тензорами и т.п.

Вход - это то аргумент функции. То место, на которое подставится аргумент вызове.

N - вход в примере выше

Преобразования - рецепты вычисления чего-то на основе входов и констант

(T.arange(N)^2).sum() - 3 последовательных преобразования N
Работают почти 1 в 1 как векторные операции в numpy
почти всё, что есть в numpy есть в theano tensor и называется так же
np.mean -> T.mean
np.arange -> T.arange
np.cumsum -> T.cumsum
и так далее...
Совсем редко - бывает, что меняется название или синтаксис - нужно спросить у семинаристов или гугла

Ничего не понятно? Сейчас исправим.
Can you write Python code for it?

#входы
example_input_integer = T.scalar("вход - одно число(пример)",dtype='float32')

example_input_tensor = T.tensor4("вход - четырёхмерный тензор(пример)")
#не бойся, тензор нам не пригодится

input_vector = T.vector("вход - вектор целых чисел", dtype='int32')

#преобразования

#поэлементное умножение
double_the_vector = input_vector*2

#поэлементный косинус
elementwise_cosine = T.cos(input_vector)

#разность квадрата каждого элемента и самого элемента
vector_squares = input_vector**2 - input_vector

double_the_vector

#теперь сам:
#создай 2 вектора из чисел float32
my_vector = <вектор из float32>
my_vector2 = <ещё один такой же>

#напиши преобразование, которое считает
#(вектор 1)*(вектор 2) / (sin(вектор 1) +1)
my_transformation = <преобразование>

print (my_transformation)
#то, что получилась не чиселка - это нормально


I am working on the file "Seminar4/Seminar4-ru.ipynb".
The first task is:
Компиляция

До этого момента, мы использовали "символические" переменные
писали рецепт вычислений, но ничего не вычисляли
чтобы рецепт можно было использовать, его нужно скомпилировать
Can you write Python code for it?

inputs = [<от чего завсит функция>]
outputs = [<что вычисляет функция (можно сразу несколько - списком, либо 1 преобразование)>]

# можно скомпилировать написанные нами преобразования как функцию
my_function = theano.function(
    inputs,outputs,
    allow_input_downcast=True #автоматически прводить типы (необязательно)
 )

#можно вызвать вот-так:
print ("using python lists:")
print (my_function([1,2,3],[4,5,6]))
print()

#а можно так.
#К слову, ту тип float приводится к типу второго вектора
print ("using numpy arrays:")
print (my_function(np.arange(10),
                  np.linspace(5,6,10,dtype='float')))


I am working on the file "Seminar4/Seminar4-ru.ipynb".
The first task is:
Последний штрих - Updates

updates - это способ изменять значения shared переменных каждый раз В КОНЦЕ вызова функции

фактически, это словарь {shared_переменная: рецепт нового значения}, который добавляется в функцию при компиляции

Например,
Can you write Python code for it?

#умножим shared вектор на число и сохраним новое значение обратно в этот shared вектор

inputs = [input_scalar]
outputs = [scalar_times_shared] #вернём вектор, умноженный на число

my_updates = {
    shared_vector_1:scalar_times_shared #и этот же результат запишем в shared_vector_1
}

compute_and_save = theano.function(inputs, outputs, updates=my_updates)

shared_vector_1.set_value(np.arange(5))

#изначальное значение shared_vector_1
print ("initial shared value:" ,shared_vector_1.get_value())

# теперь вычислим функцию (значение shared_vector_1 при этом поменяется)
print ("compute_and_save(2) returns",compute_and_save(2))

#проверим, что в shared_vector_1
print ("new shared value:" ,shared_vector_1.get_value())


I am working on the file "Seminar4/Seminar4-ru.ipynb".
The first task is:
Процесс обучения
Can you write Python code for it?

import time

num_epochs = 100 #количество проходов по данным

batch_size = 50 #размер мини-батча

for epoch in range(num_epochs):
    # In each epoch, we do a full pass over the training data:
    train_err = 0
    train_acc = 0
    train_batches = 0
    start_time = time.time()
    for batch in iterate_minibatches(X_train, y_train,batch_size):
        inputs, targets = batch
        train_err_batch, train_acc_batch= train_fun(inputs, targets)
        train_err += train_err_batch
        train_acc += train_acc_batch
        train_batches += 1

    # And a full pass over the validation data:
    val_acc = 0
    val_batches = 0
    for batch in iterate_minibatches(X_val, y_val, batch_size):
        inputs, targets = batch
        val_acc += accuracy_fun(inputs, targets)
        val_batches += 1

    # Then we print the results for this epoch:
    print("Epoch {} of {} took {:.3f}s".format(
        epoch + 1, num_epochs, time.time() - start_time))

    print("  training loss (in-iteration):\t\t{:.6f}".format(train_err / train_batches))
    print("  train accuracy:\t\t{:.2f} %".format(
        train_acc / train_batches * 100))
    print("  validation accuracy:\t\t{:.2f} %".format(
        val_acc / val_batches * 100))

test_acc = 0
test_batches = 0
for batch in iterate_minibatches(X_test, y_test, 500):
    inputs, targets = batch
    acc = accuracy_fun(inputs, targets)
    test_acc += acc
    test_batches += 1
print("Final results:")
print("  test accuracy:\t\t{:.2f} %".format(
    test_acc / test_batches * 100))

if test_acc / test_batches * 100 > 99:
    print ("Achievement unlocked: колдун 80 уровня")
else:
    print ("Нужно больше магии!")


I am working on the file "Seminar4/Seminar4-ru.ipynb".
The first task is:
Так задаётся архитектура нейронки
Can you write Python code for it?

#входной слой (вспомогательный)
input_layer = lasagne.layers.InputLayer(shape = input_shape,input_var=input_X)

#полносвязный слой, который принимает на вход input layer и имеет 100 нейронов.
# нелинейная функция - сигмоида как в логистической регрессии
# слоям тоже можно давать имена, но это необязательно
dense_1 = lasagne.layers.DenseLayer(input_layer,num_units=50,
                                   nonlinearity = lasagne.nonlinearities.sigmoid,
                                   name = "hidden_dense_layer")

#ВЫХОДНОЙ полносвязный слой, который принимает на вход dense_1 и имеет 10 нейронов -по нейрону на цифру
#нелинейность - softmax - чтобы вероятности всех цифр давали в сумме 1
dense_output = lasagne.layers.DenseLayer(dense_1,num_units = 10,
                                        nonlinearity = lasagne.nonlinearities.softmax,
                                        name='output')

#предсказание нейронки (theano-преобразование)
y_predicted = lasagne.layers.get_output(dense_output)

#все веса нейронки (shared-переменные)
all_weights = lasagne.layers.get_all_params(dense_output)
print (all_weights)


I am working on the file "Seminar4/Seminar4-ru.ipynb".
The first task is:
дальше вы могли бы просто

задать функцию ошибки вручную
посчитать градиент ошибки по all_weights
написать updates
но это долго, а простой шаг по градиенту - не самый лучший смособ оптимизировать веса

Вместо этого, опять используем lasagne
Can you write Python code for it?

#функция ошибки - средняя кроссэнтропия
loss = lasagne.objectives.categorical_crossentropy(y_predicted,target_y).mean()

accuracy = lasagne.objectives.categorical_accuracy(y_predicted,target_y).mean()

#сразу посчитать словарь обновлённых значений с шагом по градиенту, как раньше
updates_sgd = lasagne.updates.rmsprop(loss, all_weights,learning_rate=0.01)

#функция, которая обучает сеть на 1 шаг и возвращащет значение функции потерь и точности
train_fun = theano.function([input_X,target_y],[loss,accuracy],updates= updates_sgd)

#функция, которая считает точность
accuracy_fun = theano.function([input_X,target_y],accuracy)


I am working on the file "Seminar4/Seminar4-ru.ipynb".
The first task is:
теперь сам
Can you write Python code for it?

my_vector = T.vector('float64')

#посчитай производные этой функции по my_scalar и my_vector
#warning! Не пытайся понять физический смысл этой функции
weird_psychotic_function = ((my_vector+my_scalar)**(1+T.var(my_vector)) +1./T.arcsinh(my_scalar)).mean()/(my_scalar**2 +1) + 0.01*T.sin(2*my_scalar**1.5)*(T.sum(my_vector)* my_scalar**2)*T.exp((my_scalar-4)**2)/(1+T.exp((my_scalar-4)**2))*(1.-(T.exp(-(my_scalar-4)**2))/(1+T.exp(-(my_scalar-4)**2)))**2

der_by_scalar,der_by_vector = градиент функции сверху по скаляру и вектору (можно дать списком)

compute_weird_function = theano.function([my_scalar,my_vector],weird_psychotic_function)
compute_der_by_scalar = theano.function([my_scalar,my_vector],der_by_scalar)

#график функции и твоей производной
vector_0 = [1,2,3]

scalar_space = np.linspace(0,7)

y = [compute_weird_function(x,vector_0) for x in scalar_space]
plt.plot(scalar_space,y,label='function')
y_der_by_scalar = [compute_der_by_scalar(x,vector_0) for x in scalar_space]
plt.plot(scalar_space,y_der_by_scalar,label='derivative')
plt.grid();plt.legend()


I am working on the file "Seminar4/bonus/Bonus-advanced-cnn.ipynb".
The first task is:
Defining network architecture
Can you write Python code for it?

#Input layer (auxilary)
input_layer = lasagne.layers.InputLayer(shape = input_shape,input_var=input_X)

#fully connected layer, that takes input layer and applies 50 neurons to it.
# nonlinearity here is sigmoid as in logistic regression
# you can give a name to each layer (optional)
dense_1 = lasagne.layers.DenseLayer(input_layer,num_units=100,
                                   nonlinearity = lasagne.nonlinearities.sigmoid,
                                   name = "hidden_dense_layer")

#fully connected output layer that takes dense_1 as input and has 10 neurons (1 for each digit)
#We use softmax nonlinearity to make probabilities add up to 1
dense_output = lasagne.layers.DenseLayer(dense_1,num_units = 10,
                                        nonlinearity = lasagne.nonlinearities.softmax,
                                        name='output')

#network prediction (theano-transformation)
y_predicted = lasagne.layers.get_output(dense_output)

#all network weights (shared variables)
all_weights = lasagne.layers.get_all_params(dense_output,trainable=True)
print all_weights


I am working on the file "Seminar4/bonus/Bonus-advanced-cnn.ipynb".
The first task is:
First step
Let's create a mini-convolutional network with roughly such architecture:
* Input layer
* 3x3 convolution with 10 filters and ReLU activation
* 3x3 pooling (or set previous convolution stride to 3)
* Dense layer with 100-neurons and ReLU activation
* 10% dropout
* Output dense layer.
Train it with Adam optimizer with default params.
Second step

Add batch_norm (with default params) between convolution and pooling

Re-train the network with the same optimizer
Quest For A Better Network
(please read it at least diagonally)

The ultimate quest is to create a network that has as high accuracy as you can push it.
There is a mini-report at the end that you will have to fill in. We recommend reading it first and filling it while you iterate.

Grading

starting at zero points
+2 for describing your iteration path in a report below.
+2 for building a network that gets above 20% accuracy
+1 for beating each of these milestones on TEST dataset:
50% (5 total)
60% (6 total)
65% (7 total)
70% (8 total)
75% (9 total)
80% (10 total)

Bonus points
Common ways to get bonus points are:
* Get higher score, obviously.
* Anything special about your NN. For example "A super-small/fast NN that gets 80%" gets a bonus.
* Any detailed analysis of the results. (saliency maps, whatever)
Restrictions

Please do NOT use pre-trained networks for this assignment until you reach 80%.
In other words, base milestones must be beaten without pre-trained nets (and such net must be present in the e-mail). After that, you can use whatever you want.
you can use validation data for training, but you can't' do anything with test data apart from running the evaluation procedure.

Tips on what can be done:

Network size
MOAR neurons,

MOAR layers, (lasagne docs)

Nonlinearities in the hidden layers

tanh, relu, leaky relu, etc

Larger networks may take more epochs to train, so don't discard your net just because it could didn't beat the baseline in 5 epochs.

Ph'nglui mglw'nafh Cthulhu R'lyeh wgah'nagl fhtagn!

Convolution layers

they...
Can you write Python code for it?

import numpy as np
from cifar import load_cifar10
X_train,y_train,X_val,y_val,X_test,y_test = load_cifar10("cifar_data")

class_names = np.array(['airplane','automobile ','bird ','cat ','deer ','dog ','frog ','horse ','ship ','truck'])

print X_train.shape,y_train.shape

import lasagne

input_X = T.tensor4("X")

#input dimention (None means "Arbitrary" and only works at  the first axes [samples])
input_shape = [None,3,32,32]

target_y = T.vector("target Y integer",dtype='int32')

#Input layer (auxilary)
input_layer = lasagne.layers.InputLayer(shape = input_shape,input_var=input_X)

<student.code_neural_network_architecture()>

dense_output = <your network output>

# Network predictions (theano-transformation)
y_predicted = lasagne.layers.get_output(dense_output)

#All weights (shared-varaibles)
# "trainable" flag means not to return auxilary params like batch mean (for batch normalization)
all_weights = lasagne.layers.get_all_params(dense_output,trainable=True)
print all_weights

#loss function
loss = <loss function>

#<optionally add regularization>

#accuracy with dropout/noize
accuracy = lasagne.objectives.categorical_accuracy(y_predicted,target_y).mean()

#weight updates
updates = <try different update methods>

#A function that accepts X and y, returns loss functions and performs weight updates
train_fun = theano.function([input_X,target_y],[loss,accuracy],updates= updates_sgd)

#deterministic prediciton (without dropout)
y_predicted_det = lasagne.layers.get_output(dense_output)

#prediction accuracy (without dropout)
accuracy_det = lasagne.objectives.categorical_accuracy(y_predicted_det,target_y).mean()

#function that just computes accuracy without dropout/noize -- for evaluation purposes
accuracy_fun = theano.function([input_X,target_y],accuracy_det)

#итерации обучения

num_epochs = <how many times to iterate over the entire training set>

batch_size = <how many samples are processed at a single function call>

for epoch in range(num_epochs):
    # In each epoch, we do a full pass over the training...


I am working on the file "Seminar4/bonus/Bonus-advanced-cnn.ipynb".
The first task is:
Than you could simply

define loss function manually
compute error gradient over all weights
define updates
But that's a whole lot of work and life's short
not to mention life's too short to wait for SGD to converge

Instead, we shall use Lasagne builtins
Can you write Python code for it?

#Mean categorical crossentropy as a loss function - similar to logistic loss but for multiclass targets
loss = lasagne.objectives.categorical_crossentropy(y_predicted,target_y).mean()

#prediction accuracy (WITH dropout)
accuracy = lasagne.objectives.categorical_accuracy(y_predicted,target_y).mean()

#This function computes gradient AND composes weight updates just like you did earlier
updates_sgd = lasagne.updates.sgd(loss, all_weights,learning_rate=0.01)

#function that computes loss and updates weights
train_fun = theano.function([input_X,target_y],[loss,accuracy],updates= updates_sgd)

#deterministic prediciton (without dropout)
y_predicted_det = lasagne.layers.get_output(dense_output,deterministic=True)

#prediction accuracy (without dropout)
accuracy_det = lasagne.objectives.categorical_accuracy(y_predicted_det,target_y).mean()

#function that just computes accuracy without dropout/noize -- for evaluation purposes
accuracy_fun = theano.function([input_X,target_y],accuracy_det)


I am working on the file "Seminar4/bonus/Bonus-advanced-cnn.ipynb".
The first task is:
That's all, now let's train it!

We got a lot of data, so it's recommended that you use SGD
So let's implement a function that splits the training sample into minibatches
Can you write Python code for it?

# An auxilary function that returns mini-batches for neural network training

#Parameters
# X - a tensor of images with shape (many, 3, 32, 32), e.g. X_train
# y - a vector of answers for corresponding images e.g. Y_train
#batch_size - a single number - the intended size of each batches

#What do need to implement
# 1) Shuffle data
# - Gotta shuffle X and y the same way not to break the correspondence between X_i and y_i
# 3) Split data into minibatches of batch_size
# - If data size is not a multiple of batch_size, make one last batch smaller.
# 4) return a list (or an iterator) of pairs
# - (подгруппа картинок, ответы из y на эту подгруппу)
def iterate_minibatches(X, y, batchsize):

    <return an iterable of (X_batch, y_batch)  batches of images and answers for them>

#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
#
# You feel lost and wish you stayed home tonight?
# Go search for a similar function at
# https://github.com/Lasagne/Lasagne/blob/master/examples/mnist.py


I am working on the file "Seminar4/bonus/Bonus-advanced-cnn.ipynb".
The first task is:
Training loop
Can you write Python code for it?

import time

num_epochs = 100 #amount of passes through the data

batch_size = 50 #number of samples processed at each function call

for epoch in range(num_epochs):
    # In each epoch, we do a full pass over the training data:
    train_err = 0
    train_acc = 0
    train_batches = 0
    start_time = time.time()
    for batch in iterate_minibatches(X_train, y_train,batch_size):
        inputs, targets = batch
        train_err_batch, train_acc_batch= train_fun(inputs, targets)
        train_err += train_err_batch
        train_acc += train_acc_batch
        train_batches += 1

    # And a full pass over the validation data:
    val_acc = 0
    val_batches = 0
    for batch in iterate_minibatches(X_val, y_val, batch_size):
        inputs, targets = batch
        val_acc += accuracy_fun(inputs, targets)
        val_batches += 1

    # Then we print the results for this epoch:
    print("Epoch {} of {} took {:.3f}s".format(
        epoch + 1, num_epochs, time.time() - start_time))

    print("  training loss (in-iteration):\t\t{:.6f}".format(train_err / train_batches))
    print("  train accuracy:\t\t{:.2f} %".format(
        train_acc / train_batches * 100))
    print("  validation accuracy:\t\t{:.2f} %".format(
        val_acc / val_batches * 100))

test_acc = 0
test_batches = 0
for batch in iterate_minibatches(X_test, y_test, 500):
    inputs, targets = batch
    acc = accuracy_fun(inputs, targets)
    test_acc += acc
    test_batches += 1
print("Final results:")
print("  test accuracy:\t\t{:.2f} %".format(
    test_acc / test_batches * 100))

if test_acc / test_batches * 100 > 95:
    print "Double-check, than consider applying for NIPS'17. SRSly."
elif test_acc / test_batches * 100 > 90:
    print "U'r freakin' amazin'!"
elif test_acc / test_batches * 100 > 80:
    print "Achievement unlocked: 110lvl Warlock!"
elif test_acc / test_batches * 100 > 70:
    print "Achievement unlocked: 80lvl Warlock!"
elif test_acc / test_batches * 100 > 50:
    print "Achievement unlocked: 60lvl...


I am working on the file "Seminar4/bonus/Bonus-advanced-cnn.ipynb".
The first task is:
lasagne

lasagne is a library for neural network building and training
it's a low-level library with almost seamless integration with theano
Can you write Python code for it?

import lasagne
import theano
import theano.tensor as T

input_X = T.tensor4("X")

#input dimention (None means "Arbitrary")
input_shape = [None,3,32,32]

target_y = T.vector("target Y integer",dtype='int32')


I am working on the file "Seminar4/bonus/Bonus-advanced-theano.ipynb".
The first task is:
Almost done - Updates

updates are a way of changing shared variables at after function call.

technically it's a dictionary {shared_variable : a recipe for new value} which is has to be provided when function is compiled

That's how it works:
Can you write Python code for it?

# Multiply shared vector by a number and save the product back into shared vector

inputs = [input_scalar]
outputs = [scalar_times_shared] #return vector times scalar

my_updates = {
    shared_vector_1:scalar_times_shared #and write this same result bach into shared_vector_1
}

compute_and_save = theano.function(inputs, outputs, updates=my_updates)

shared_vector_1.set_value(np.arange(5))

#initial shared_vector_1
print "initial shared value:" ,shared_vector_1.get_value()

# evaluating the function (shared_vector_1 will be changed)
print "compute_and_save(2) returns",compute_and_save(2)

#evaluate new shared_vector_1
print "new shared value:" ,shared_vector_1.get_value()


I am working on the file "Seminar4/bonus/Bonus-advanced-theano.ipynb".
The first task is:
Compiling

So far we were using "symbolic" variables and transformations
Defining the recipe for computation, but not computing anything
To use the recipe, one should compile it
Can you write Python code for it?

inputs = [<two vectors that my_transformation depends on>]
outputs = [<What do we compute (can be a list of several transformation)>]

# The next lines compile a function that takes two vectors and computes your transformation
my_function = theano.function(
    inputs,outputs,
    allow_input_downcast=True #automatic type casting for input parameters (e.g. float64 -> float32)
 )

#using function with, lists:
print "using python lists:"
print my_function([1,2,3],[4,5,6])
print

#Or using numpy arrays:
#btw, that 'float' dtype is casted to secong parameter dtype which is float32
print "using numpy arrays:"
print my_function(np.arange(10),
                  np.linspace(5,6,10,dtype='float'))


I am working on the file "Seminar4/bonus/Bonus-advanced-theano.ipynb".
The first task is:
Debugging

Compilation can take a while for big functions
To avoid waiting, one can evaluate transformations without compiling
Without compilation, the code runs slower, so consider reducing input size
Can you write Python code for it?

#a dictionary of inputs
my_function_inputs = {
    my_vector:[1,2,3],
    my_vector2:[4,5,6]
}

# evaluate my_transformation
# has to match with compiled function output
print my_transformation.eval(my_function_inputs)

# can compute transformations on the fly
print "add 2 vectors", (my_vector + my_vector2).eval(my_function_inputs)

#!WARNING! if your transformation only depends on some inputs,
#do not provide the rest of them
print "vector's shape:", my_vector.shape.eval({
        my_vector:[1,2,3]
    })