I am working on the file "DeepLearningA-Z/02-supervised-deep-learning/01-Artificial-Neural-Networks-ANN/.ipynb_checkpoints/02-Building-an-ANN-checkpoint.ipynb".
The first task is:
y (actual value): exited, this is the value we are trying to predict, which means if the customer stays or exit the bank.
Can you write Python code for it?

print (X.shape)
X

print (y.shape)
y

# Encoding categorical data
from sklearn.preprocessing import LabelEncoder, OneHotEncoder
labelencoder_X_1 = LabelEncoder()
X[:, 1] = labelencoder_X_1.fit_transform(X[:, 1])
labelencoder_X_2 = LabelEncoder()
X[:, 2] = labelencoder_X_2.fit_transform(X[:, 2])
onehotencoder = OneHotEncoder(categorical_features = [1])
X = onehotencoder.fit_transform(X).toarray()
X = X[:, 1:]

print (X.shape)
X

print (y.shape)
y

# Splitting the dataset into the Training set and Test set
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 0)

# Feature Scaling
from sklearn.preprocessing import StandardScaler
sc = StandardScaler()
X_train = sc.fit_transform(X_train)
X_test = sc.transform(X_test)


I am working on the file "DeepLearningA-Z/02-supervised-deep-learning/01-Artificial-Neural-Networks-ANN/02-Building-an-ANN.ipynb".
The first task is:
Evaluating, Improving and Tuning the ANN

Using K-Fold Cross validation with Keras
Can you write Python code for it?

# Evaluating the ANN
from keras.wrappers.scikit_learn import KerasClassifier
from sklearn.model_selection import cross_val_score
from keras.models import Sequential
from keras.layers import Dense
def build_classifier():
    classifier = Sequential()
    classifier.add(Dense(units = 6, kernel_initializer = 'uniform', activation = 'relu', input_dim = 11))
    classifier.add(Dense(units = 6, kernel_initializer = 'uniform', activation = 'relu'))
    classifier.add(Dense(units = 1, kernel_initializer = 'uniform', activation = 'sigmoid'))
    classifier.compile(optimizer = 'adam', loss = 'binary_crossentropy', metrics = ['accuracy'])
    return classifier
classifier = KerasClassifier(build_fn = build_classifier, batch_size = 10, epochs = 100)
accuracies = cross_val_score(estimator = classifier, X = X_train, y = y_train, cv = 10, n_jobs = -1)
mean = accuracies.mean()
variance = accuracies.std()

# Improving the ANN
# Dropout Regularization to reduce overfitting if needed

# Tuning the ANN
from keras.wrappers.scikit_learn import KerasClassifier
from sklearn.model_selection import GridSearchCV
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import Dropout

def build_classifier(optimizer):
    classifier = Sequential()
    classifier.add(Dense(units = 6, kernel_initializer = 'uniform', activation = 'relu', input_dim = 11))
    # classifier.add(Dropout(p = 0.1))
    classifier.add(Dense(units = 6, kernel_initializer = 'uniform', activation = 'relu'))
    # classifier.add(Dropout(p = 0.1))
    classifier.add(Dense(units = 1, kernel_initializer = 'uniform', activation = 'sigmoid'))
    classifier.compile(optimizer = optimizer, loss = 'binary_crossentropy', metrics = ['accuracy'])
    return classifier
classifier = KerasClassifier(build_fn = build_classifier)
parameters = {'batch_size': [25, 32],
              'epochs': [100, 500],
              'optimizer': ['adam', 'rmsprop']}
grid_search = GridSearchCV(estimator = classifier,
                           param_grid = parameters,
      ...


I am working on the file "DeepLearningA-Z/02-supervised-deep-learning/01-Artificial-Neural-Networks-ANN/02-Building-an-ANN.ipynb".
The first task is:
Making predictions and evaluating the model
Can you write Python code for it?

y_pred = classifier.predict(X_test)
y_pred = (y_pred > 0.5)

# Making the Confusion Matrix
from sklearn.metrics import confusion_matrix
cm = confusion_matrix(y_test, y_pred)

cm


I am working on the file "DeepLearningA-Z/02-supervised-deep-learning/01-Artificial-Neural-Networks-ANN/02-Building-an-ANN.ipynb".
The first task is:
Tips:
number of nodes in the hidden layer = average of number of nodes in the input layer and number of nodes in the output layer.
(11 + 1)/2 = 6
Can you write Python code for it?

# Adding the input layer and the first hidden layer
classifier.add(Dense(units = 6, kernel_initializer = 'uniform', activation = 'relu', input_dim = 11))

# Adding the second hidden layer
classifier.add(Dense(units = 6, kernel_initializer = 'uniform', activation = 'relu'))

# Adding the output layer
classifier.add(Dense(units = 1, kernel_initializer = 'uniform', activation = 'sigmoid'))

# Compiling the ANN
classifier.compile(optimizer = 'adam', loss = 'binary_crossentropy', metrics = ['accuracy'])

# Fitting the ANN to the Training set
classifier.fit(X_train, y_train, batch_size = 10, epochs = 100)


I am working on the file "DeepLearningA-Z/02-supervised-deep-learning/02-Convolutional-Neural-Networks-CNN/02-Building-a-CNN.ipynb".
The first task is:
Building the CNN
Can you write Python code for it?

# Importing the Keras libraries and packages
from keras.models import Sequential
from keras.layers import Conv2D
from keras.layers import MaxPooling2D
from keras.layers import Flatten
from keras.layers import Dense

# Initialising the CNN
classifier = Sequential()

# Step 1 - Convolution
classifier.add(Conv2D(32, (3, 3), input_shape = (64, 64, 3), activation = 'relu'))

# Step 2 - Pooling
classifier.add(MaxPooling2D(pool_size = (2, 2)))

# Adding a second convolutional layer
classifier.add(Conv2D(32, (3, 3), activation = 'relu'))
classifier.add(MaxPooling2D(pool_size = (2, 2)))

# Step 3 - Flattening
classifier.add(Flatten())

# Step 4 - Full connection
classifier.add(Dense(units = 128, activation = 'relu'))
classifier.add(Dense(units = 1, activation = 'sigmoid'))

# Compiling the CNN
classifier.compile(optimizer = 'adam', loss = 'binary_crossentropy', metrics = ['accuracy'])


I am working on the file "DeepLearningA-Z/02-supervised-deep-learning/02-Convolutional-Neural-Networks-CNN/02-Building-a-CNN.ipynb".
The first task is:
Challenge
Evaluation was already made during the training with the validation set, therefore k-Fold Cross Validation is not needed.
Then the techniques to improve and tune a CNN model are the same as for ANNs. So here is the challenge:
Apply the techniques you've learnt and use your architect power to make a CNN that will give you the gold medal:
- Bronze medal: Test set accuracy between 80% and 85%
- Silver medal: Test set accuracy between 85% and 90%
- Gold medal: Test set accuracy over 90%!!
Rules:
- Use the same training set
- Dropout allowed
- Customized SGD allowed
- Specific seeds not allowed
Better solution

Prediction Challenge Solution - Result: 90%
Can you write Python code for it?

from tensorflow.contrib.keras.api.keras.layers import Dropout
from tensorflow.contrib.keras.api.keras.models import Sequential
from tensorflow.contrib.keras.api.keras.layers import Conv2D
from tensorflow.contrib.keras.api.keras.layers import MaxPooling2D
from tensorflow.contrib.keras.api.keras.layers import Flatten
from tensorflow.contrib.keras.api.keras.layers import Dense
from tensorflow.contrib.keras.api.keras.callbacks import Callback
from tensorflow.contrib.keras.api.keras.preprocessing.image import ImageDataGenerator
from tensorflow.contrib.keras import backend
import os

class LossHistory(Callback):
    def __init__(self):
        super().__init__()
        self.epoch_id = 0
        self.losses = ''

    def on_epoch_end(self, epoch, logs={}):
        self.losses += "Epoch {}: accuracy -> {:.4f}, val_accuracy -> {:.4f}\n"\
            .format(str(self.epoch_id), logs.get('acc'), logs.get('val_acc'))
        self.epoch_id += 1

    def on_train_begin(self, logs={}):
        self.losses += 'Training begins...\n'

script_dir = os.path.dirname(__file__)
training_set_path = os.path.join(script_dir, '../dataset/training_set')
test_set_path = os.path.join(script_dir, '../dataset/test_set')

# Initialising the CNN
classifier = Sequential()

# Step 1 - Convolution
input_size = (128, 128)
classifier.add(Conv2D(32, (3, 3), input_shape=(*input_size, 3), activation='relu'))

# Step 2 - Pooling
classifier.add(MaxPooling2D(pool_size=(2, 2)))  # 2x2 is optimal

# Adding a second convolutional layer
classifier.add(Conv2D(32, (3, 3), activation='relu'))
classifier.add(MaxPooling2D(pool_size=(2, 2)))

# Adding a third convolutional layer
classifier.add(Conv2D(64, (3, 3), activation='relu'))
classifier.add(MaxPooling2D(pool_size=(2, 2)))

# Step 3 - Flattening
classifier.add(Flatten())

# Step 4 - Full connection
classifier.add(Dense(units=64, activation='relu'))
classifier.add(Dropout(0.5))
classifier.add(Dense(units=1, activation='sigmoid'))

# Compiling the CNN
classifier.compile(optimizer='adam',...


I am working on the file "DeepLearningA-Z/02-supervised-deep-learning/02-Convolutional-Neural-Networks-CNN/02-Building-a-CNN.ipynb".
The first task is:
Making new predictions
Can you write Python code for it?

import numpy as np
from keras.preprocessing import image
test_image = image.load_img('Convolutional_Neural_Networks/dataset/single_prediction/cat_or_dog_1.jpg',
                            target_size = (64, 64))
test_image = image.img_to_array(test_image)
test_image = np.expand_dims(test_image, axis = 0)
result = classifier.predict(test_image)
training_set.class_indices
if result[0][0] == 1:
    prediction = 'dog'
else:
    prediction = 'cat'

print (prediction)


I am working on the file "DeepNetwork/Keras/BankChurn.ipynb".
The first task is:
Create matrix of features and matrix of target variable. In this case we are excluding column 1, 2 & 3 as those are ‘row_number’, ‘customerid’ & 'Surname' which are not useful in our analysis. Column 14, ‘Exited’ is our Target Variable
Can you write Python code for it?

from sklearn.preprocessing import LabelEncoder, OneHotEncoder
## Read this for categorical Encoding : http://pbpython.com/categorical-encoding.html
##pd.get_dummies(dataset, columns=["Geography", "Gender"], prefix=["Geography", "Gender"]).head()

def getXy_1(dataset,target):
    df = pd.get_dummies(dataset, columns=["Geography", "Gender"], prefix=["Geography", "Gender"])

    y = df[target]
    X = df.loc[:, df.columns != target]
    return X,y

def getXy_2(dataset,target):
    lb = LabelEncoder()
    dataset['Gender'] = lb.fit_transform(dataset['Gender'])
    dataset['Geography'] = lb.fit_transform(dataset['Geography'])
    ## One-Hot Coding
    dataset = pd.get_dummies(dataset, columns = ['Geography','Gender'])
    y = dataset[target]
    X = dataset.loc[:, dataset.columns != target]
    return X,y

X,y = getXy_2(dataset,target='Exited')
print("X.columns:",X.columns)
X = X.iloc[:, 3:].values
y = y.values
y

# Splitting the dataset into the Training set and Test set
from sklearn.model_selection import train_test_split
X_train,X_test, y_train, y_test = train_test_split(X,y,test_size=0.2)


I am working on the file "DeepNetwork/Keras/BankChurn.ipynb".
The first task is:
I know you are tired of data preprocessing but I promise this is the last step. If you carefully observe data, you will find that data is not scaled properly. Some variable has value in thousands while some have value is tens or ones. We don’t want any of our variable to dominate on other so let’s go and scale data.
‘StandardScaler’ is available in ScikitLearn. In the following code we are fitting and transforming StandardScaler method on train data. We have to standardize our scaling so we will use the same fitted method to transform/scale test data.
Can you write Python code for it?

from sklearn.preprocessing import StandardScaler
sc = StandardScaler()
X_train = sc.fit_transform(X_train)
X_test = sc.fit_transform(X_test)

X_train

X_train.shape

import keras
from keras.models import Sequential

from keras.layers.core import Dense,Dropout, Dropout,Activation

model = Sequential()

model.add(Dense(16,input_dim=13))
model.add(Activation('relu'))
model.add(Dropout(0.2))

model.add(Dense(16))
model.add(Activation('relu'))
model.add(Dropout(0.2))

model.add(Dense(8))
model.add(Activation('relu'))
model.add(Dropout(0.2))

model.add(Dense(1))
model.add(Activation('sigmoid'))

model.summary()

# Compiling Neural Network
model.compile(optimizer = 'adam', loss = 'binary_crossentropy', metrics = ['accuracy'])

# Fitting our model
model.fit(X_train, y_train, batch_size = 10, nb_epoch = 100)

##Predicting the test set results
y_pred = model.predict(X_test)
y_pred = (y_pred>0.4)
y_pred

# Creating the Confusion Matrix
from sklearn.metrics import confusion_matrix
cm = confusion_matrix(y_test, y_pred)

cm


I am working on the file "DeepNetwork/Keras/MnistKerasModelsGood.ipynb".
The first task is:
An excellent feature of Keras, that sets it apart from frameworks such as TensorFlow, is automatic inference of shapes; we only need to specify the shape of the input layer, and afterwards Keras will take care of initialising the weight variables with proper shapes. Once all the layers have been defined, we simply need to identify the input(s) and the output(s) in order to define our model, as illustrated below.
Can you write Python code for it?

inp = Input(shape=(height * width,)) # Our input is a 1D vector of size 784
hidden_1 = Dense(hidden_size, activation='relu')(inp) # First hidden ReLU layer
hidden_2 = Dense(hidden_size, activation='relu')(hidden_1) # Second hidden ReLU layer
out = Dense(num_classes, activation='softmax')(hidden_2) # Output softmax layer

model = Model(inputs=inp, outputs=out) # To define a model, just specify its input and output layers

model.compile(loss='categorical_crossentropy', # using the cross-entropy loss function
              optimizer='adam', # using the Adam optimiser
              metrics=['accuracy']) # reporting the accuracy


I am working on the file "DeepNetwork/Keras/MnistKerasModelsGood.ipynb".
The first task is:
Now lets predict one single sample
Can you write Python code for it?

import matplotlib.pyplot as plt
%matplotlib inline
X_test_0 = X_test[0,:].reshape(1,height * width)
Y_test_0 = Y_test[0,:]
print(Y_test_0)
plt.imshow(X_test_0.reshape(28,28))

import numpy as np
pred = model.predict(X_test_0)
print('Label of testing sample:', np.argmax(Y_test_0))
print('\nOutput of the softmax layer:',pred[0])
print('\nNeural Network prediction:', np.argmax(pred[0]))


I am working on the file "DeepNetwork/deeplearningai_AndrewNG/Python+Basics+With+Numpy+v3.ipynb".
The first task is:
As you may have noticed, the vectorized implementation is much cleaner and more efficient. For bigger vectors/matrices, the differences in running time become even bigger.
Note that np.dot() performs a matrix-matrix or matrix-vector multiplication. This is different from np.multiply() and the * operator (which is equivalent to  .* in Matlab/Octave), which performs an element-wise multiplication.
2.1 Implement the L1 and L2 loss functions
Exercise: Implement the numpy vectorized version of the L1 loss. You may find the function abs(x) (absolute value of x) useful.
Reminder:
- The loss is used to evaluate the performance of your model. The bigger your loss is, the more different your predictions ($ \hat{y} $) are from the true values ($y$). In deep learning, you use optimization algorithms like Gradient Descent to train your model and to minimize the cost.
- L1 loss is defined as:
$$\begin{align} & L_1(\hat{y}, y) = \sum_{i=0}^m|y^{(i)} - \hat{y}^{(i)}| \end{align}\tag{6}$$
Can you write Python code for it?

# GRADED FUNCTION: L1

def L1(yhat, y):
    """
    Arguments:
    yhat -- vector of size m (predicted labels)
    y -- vector of size m (true labels)

    Returns:
    loss -- the value of the L1 loss function defined above
    """

    ### START CODE HERE ### (≈ 1 line of code)
    loss = np.sum(np.abs(y-yhat))
    ### END CODE HERE ###

    return loss

yhat = np.array([.9, 0.2, 0.1, .4, .9])
y = np.array([1, 0, 0, 1, 1])
print("L1 = " + str(L1(yhat,y)))


I am working on the file "DeepNetwork/deeplearningai_AndrewNG/Python+Basics+With+Numpy+v3.ipynb".
The first task is:
Expected Output:
<table style="width:20%">

     <tr>
       <td> **L1** </td>
       <td> 1.1 </td>
     </tr>
</table>

Exercise: Implement the numpy vectorized version of the L2 loss. There are several way of implementing the L2 loss but you may find the function np.dot() useful. As a reminder, if $x = [x_1, x_2, ..., x_n]$, then np.dot(x,x) = $\sum_{j=0}^n x_j^{2}$.

L2 loss is defined as $$\begin{align} & L_2(\hat{y},y) = \sum_{i=0}^m(y^{(i)} - \hat{y}^{(i)})^2 \end{align}\tag{7}$$
Can you write Python code for it?

# GRADED FUNCTION: L2

def L2(yhat, y):
    """
    Arguments:
    yhat -- vector of size m (predicted labels)
    y -- vector of size m (true labels)

    Returns:
    loss -- the value of the L2 loss function defined above
    """

    ### START CODE HERE ### (≈ 1 line of code)
    #loss = np.sum(np.square(y-yhat))  ##  working
    loss = np.dot((y-yhat),(y-yhat))
    ### END CODE HERE ###

    return loss

yhat = np.array([.9, 0.2, 0.1, .4, .9])
y = np.array([1, 0, 0, 1, 1])
print("L2 = " + str(L2(yhat,y)))


I am working on the file "DeepNetwork/deeplearningai_AndrewNG/Python+Basics+With+Numpy+v3.ipynb".
The first task is:
Expected Output:

    ** basic_sigmoid(3) **
        0.9525741268224334

Actually, we rarely use the "math" library in deep learning because the inputs of the functions are real numbers. In deep learning we mostly use matrices and vectors. This is why numpy is more useful.
Can you write Python code for it?

### One reason why we use "numpy" instead of "math" in Deep Learning ###
x = [1, 2, 3]
basic_sigmoid(x) # you will see this give an error when you run it, because x is a vector.


I am working on the file "DeepNetwork/deeplearningai_AndrewNG/Python+Basics+With+Numpy+v3.ipynb".
The first task is:
Expected Output:

        **image2vector(image)**
        [[ 0.67826139]
 [ 0.29380381]
 [ 0.90714982]
 [ 0.52835647]
 [ 0.4215251 ]
 [ 0.45017551]
 [ 0.92814219]
 [ 0.96677647]
 [ 0.85304703]
 [ 0.52351845]
 [ 0.19981397]
 [ 0.27417313]
 [ 0.60659855]
 [ 0.00533165]
 [ 0.10820313]
 [ 0.49978937]
 [ 0.34144279]
 [ 0.94630077]]

1.4 - Normalizing rows
Another common technique we use in Machine Learning and Deep Learning is to normalize our data. It often leads to a better performance because gradient descent converges faster after normalization. Here, by normalization we mean changing x to $ \frac{x}{\| x\|} $ (dividing each row vector of x by its norm).
For example, if $$x =
\begin{bmatrix}
    0 & 3 & 4 \
    2 & 6 & 4 \
\end{bmatrix}\tag{3}$$ then $$\| x\| = np.linalg.norm(x, axis = 1, keepdims = True) = \begin{bmatrix}
    5 \
    \sqrt{56} \
\end{bmatrix}\tag{4} $$and        $$ x_normalized = \frac{x}{\| x\|} = \begin{bmatrix}
    0 & \frac{3}{5} & \frac{4}{5} \
    \frac{2}{\sqrt{56}} & \frac{6}{\sqrt{56}} & \frac{4}{\sqrt{56}} \
\end{bmatrix}\tag{5}$$ Note that you can divide matrices of different sizes and it works fine: this is called broadcasting and you're going to learn about it in part 5.
Exercise: Implement normalizeRows() to normalize the rows of a matrix. After applying this function to an input matrix x, each row of x should be a vector of unit length (meaning length 1).
Can you write Python code for it?

# GRADED FUNCTION: normalizeRows

def normalizeRows(x):
    """
    Implement a function that normalizes each row of the matrix x (to have unit length).

    Argument:
    x -- A numpy matrix of shape (n, m)

    Returns:
    x -- The normalized (by row) numpy matrix. You are allowed to modify x.
    """

    ### START CODE HERE ### (≈ 2 lines of code)
    # Compute x_norm as the norm 2 of x. Use np.linalg.norm(..., ord = 2, axis = ..., keepdims = True)
    x_norm = np.linalg.norm(x, ord = 2, axis = 1, keepdims = True)

    # Divide x by its norm.
    x = x/x_norm
    ### END CODE HERE ###

    return x

x = np.array([
    [0, 3, 4],
    [1, 6, 4]])
print("normalizeRows(x) = " + str(normalizeRows(x)))


I am working on the file "DeepNetwork/deeplearningai_AndrewNG/Python+Basics+With+Numpy+v3.ipynb".
The first task is:
Expected output:
test: Hello World

What you need to remember:
- Run your cells using SHIFT+ENTER (or "Run cell")
- Write code in the designated areas using Python 3 only
- Do not modify the code outside of the designated areas
1 - Building basic functions with numpy
Numpy is the main package for scientific computing in Python. It is maintained by a large community (www.numpy.org). In this exercise you will learn several key numpy functions such as np.exp, np.log, and np.reshape. You will need to know how to use these functions for future assignments.
1.1 - sigmoid function, np.exp()
Before using np.exp(), you will use math.exp() to implement the sigmoid function. You will then see why np.exp() is preferable to math.exp().
Exercise: Build a function that returns the sigmoid of a real number x. Use math.exp(x) for the exponential function.
Reminder:
$sigmoid(x) = \frac{1}{1+e^{-x}}$ is sometimes also known as the logistic function. It is a non-linear function used not only in Machine Learning (Logistic Regression), but also in Deep Learning.

To refer to a function belonging to a specific package you could call it using package_name.function(). Run the code below to see an example with math.exp().
Can you write Python code for it?

# GRADED FUNCTION: basic_sigmoid

import math

def basic_sigmoid(x):
    """
    Compute sigmoid of x.

    Arguments:
    x -- A scalar

    Return:
    s -- sigmoid(x)
    """

    ### START CODE HERE ### (≈ 1 line of code)
    s = 1/(1+ math.exp(-x))
    ### END CODE HERE ###

    return s

basic_sigmoid(3)


I am working on the file "DeepNetwork/deeplearningai_AndrewNG/Python+Basics+With+Numpy+v3.ipynb".
The first task is:
In fact, if $ x = (x_1, x_2, ..., x_n)$ is a row vector then $np.exp(x)$ will apply the exponential function to every element of x. The output will thus be: $np.exp(x) = (e^{x_1}, e^{x_2}, ..., e^{x_n})$
Can you write Python code for it?

import numpy as np

# example of np.exp
x = np.array([1, 2, 3])
print(np.exp(x)) # result is (exp(1), exp(2), exp(3))


I am working on the file "DeepSpeech.ipynb".
The first task is:
As previously mentioned, the BRNN is not simply fed the MFCC features of a given time-slice. It is fed, in addition, a context of $C \in {5, 7, 9}$ frames on either side of the frame in question. The number of frames in this context is captured in the variable n_context
Can you write Python code for it?

n_context = 9 # TODO: Determine the optimal value using a validation data set


I am working on the file "DeepSpeech.ipynb".
The first task is:
Geometric Constants
Now we will introduce several constants related to the geometry of the network.
The network views each speech sample as a sequence of time-slices $x^{(i)}_t$ of length $T^{(i)}$. As the speech samples vary in length, we know that $T^{(i)}$ need not equal $T^{(j)}$ for $i \ne j$. For each batch, BRNN in TensorFlow needs to know n_steps which is the maximum $T^{(i)}$ for the batch.
Each of the at maximum n_steps vectors is a vector of MFCC features of a time-slice of the speech sample. We will make the number of MFCC features dependent upon the sample rate of the data set. Generically, if the sample rate is 8kHz we use 13 features. If the sample rate is 16kHz we use 26 features... We capture the dimension of these vectors, equivalently the number of MFCC features, in the variable n_input
Can you write Python code for it?

n_input = 26 # TODO: Determine this programatically from the sample rate


I am working on the file "DeepSpeech.ipynb".
The first task is:
Graph Creation
Next we concern ourselves with graph creation.
However, before we do so we must introduce a utility function variable_on_cpu() used to create a variable in CPU memory.
Can you write Python code for it?

def variable_on_cpu(name, shape, initializer):
    # Use the /cpu:0 device for scoped operations
    with tf.device('/cpu:0'):
        # Create or get apropos variable
        var = tf.get_variable(name=name, shape=shape, initializer=initializer)
    return var


I am working on the file "DeepSpeech.ipynb".
The first task is:
Let's also introduce a helper function for logging collections of gradient/variable tuples.
Can you write Python code for it?

def log_grads_and_vars(grads_and_vars):
    for gradient, variable in grads_and_vars:
        log_variable(variable, gradient=gradient)


I am working on the file "DeepSpeech.ipynb".
The first task is:
Let's also re-populate a central JS file, that contains all the dumps at once.
Can you write Python code for it?


I am working on the file "DeepSpeech.ipynb".
The first task is:
Let's introduce a routine to print a WER report under a given caption. It prints the given mean WER plus summaries of the top ten lowest loss items of the given array of WER result tuples (only items with WER=0 and ordered by their WER).
Can you write Python code for it?

def print_wer_report(caption, mean, items=[]):
    print
    print "#" * 80
    print "%s WER: %f" % (caption, mean)
    if len(items) > 0:
        # Filter out all items with WER=0
        items = [a for a in items if a[2] > 0]
        # Order the remaining items by their loss (lowest loss on top)
        items.sort(key=lambda a: a[3])
        # Take only the first 10 items
        items = items[:10]
        # Order this top ten items by their WER (lowest WER on top)
        items.sort(key=lambda a: a[2])
        for a in items:
            print "-" * 80
            print " - WER:    %f" % a[2]
            print " - loss:   %f" % a[3]
            print " - source: \"%s\"" % a[0]
            print " - result: \"%s\"" % a[1]
    print "#" * 80
    print


I am working on the file "DeepSpeech.ipynb".
The first task is:
Logging
We introduce a function for logging a tensor variable's current state.
It logs scalar values for the mean, standard deviation, minimum and maximum.
Furthermore it logs a histogram of its state and (if given) of an optimization gradient.
Can you write Python code for it?

def log_variable(variable, gradient=None):
    name = variable.name
    mean = tf.reduce_mean(variable)
    tf.scalar_summary(name + '/mean', mean)
    tf.scalar_summary(name + '/sttdev', tf.sqrt(tf.reduce_mean(tf.square(variable - mean))))
    tf.scalar_summary(name + '/max', tf.reduce_max(variable))
    tf.scalar_summary(name + '/min', tf.reduce_min(variable))
    tf.histogram_summary(name, variable)
    if gradient is not None:
        if isinstance(gradient, tf.IndexedSlices):
            grad_values = gradient.values
        else:
            grad_values = gradient
        if grad_values is not None:
            tf.histogram_summary(name + "/gradients", grad_values)


I am working on the file "DeepSpeech.ipynb".
The first task is:
Logging Hyper Parameters and Results
Now, as training and test are done, we persist the results alongside with the involved hyper parameters for further reporting.
Can you write Python code for it?

with open('%s/%s' % (log_dir, 'hyper.json'), 'w') as dump_file:
    json.dump({ \
        'context': { \
            'time_started': time_started.isoformat(), \
            'time_finished': time_finished.isoformat(), \
            'git_hash': get_git_revision_hash(), \
            'git_branch': get_git_branch() \
        }, \
        'parameters': { \
            'learning_rate': learning_rate, \
            'beta1': beta1, \
            'beta2': beta2, \
            'epsilon': epsilon, \
            'training_iters': training_iters, \
            'batch_size': batch_size, \
            'validation_step': validation_step, \
            'dropout_rate': dropout_rate, \
            'relu_clip': relu_clip, \
            'n_input': n_input, \
            'n_context': n_context, \
            'n_hidden_1': n_hidden_1, \
            'n_hidden_2': n_hidden_2, \
            'n_hidden_3': n_hidden_3, \
            'n_hidden_5': n_hidden_5, \
            'n_hidden_6': n_hidden_6, \
            'n_cell_dim': n_cell_dim, \
            'n_character': n_character, \
            'total_batches_train': data_sets.train.total_batches, \
            'total_batches_validation': data_sets.dev.total_batches, \
            'total_batches_test': data_sets.test.total_batches, \
            'data_set': { \
                'name': ds_importer \
            }, \
        }, \
        'results': { \
            'duration': duration, \
            'last_train_wer': last_train_wer, \
            'last_validation_wer': last_validation_wer, \
            'test_wer': test_wer \
        } \
    }, dump_file, sort_keys=True, indent = 4)


I am working on the file "DeepSpeech.ipynb".
The first task is:
Next we will introduce constants that specify the geometry of some of the non-recurrent layers of the network. We do this by simply specifying the number of units in each of the layers
Can you write Python code for it?

n_hidden_1 = n_input + 2*n_input*n_context # Note: This value was not specified in the original paper
n_hidden_2 = n_input + 2*n_input*n_context # Note: This value was not specified in the original paper
n_hidden_5 = n_input + 2*n_input*n_context # Note: This value was not specified in the original paper


I am working on the file "DeepSpeech.ipynb".
The first task is:
Next, we introduce an additional variable n_character which holds the number of characters in the target language plus one, for the $blamk$. For English it is the cardinality of the set ${a,b,c, . . . , z, space, apostrophe, blank}$ we referred to earlier.
Can you write Python code for it?

n_character = 29 # TODO: Determine if this should be extended with other punctuation


I am working on the file "DeepSpeech.ipynb".
The first task is:
Note also that this code acts as a syncronization point as it requires all GPU's to be finished with their mini-batch before it can run to completion.
Now next we introduce a function to apply the averaged gradients to update the model's paramaters on the CPU
Can you write Python code for it?

def apply_gradients(optimizer, average_grads):
    apply_gradient_op = optimizer.apply_gradients(average_grads)
    return apply_gradient_op


I am working on the file "DeepSpeech.ipynb".
The first task is:
Note that we use the Adam optimizer[3] instead of Nesterov’s Accelerated Gradient [4] used in the original DeepSpeech paper, as, at the time of writing, TensorFlow does not have an implementation of Nesterov’s Accelerated Gradient [4].
As we will also employ dropout on the feedforward layers of the network, we need to define a parameter dropout_rate that keeps track of the dropout rate for these layers
Can you write Python code for it?

dropout_rate = float(os.environ.get('ds_dropout_rate', 0.05))  # TODO: Validate this is a reasonable value

# This global placeholder will be used for all dropout definitions
dropout_rate_placeholder = tf.placeholder(tf.float32)

# The feed_dict used for training employs the given dropout_rate
feed_dict_train = { dropout_rate_placeholder: dropout_rate }

# While the feed_dict used for validation, test and train progress reporting employs zero dropout
feed_dict = { dropout_rate_placeholder: 0.0 }


I am working on the file "DeepSpeech.ipynb".
The first task is:
Now the trained model is tested using an unbiased test set.
Can you write Python code for it?

# Define CPU as device on which the muti-gpu testing is orchestrated
with tf.device('/cpu:0'):
    # Test network
    test_decodings, test_labels, test_total_losses = decode_batch(data_sets.test)
    _, test_wer = calculate_and_print_wer_report(session, "Test", test_decodings, test_labels, test_total_losses)


I am working on the file "DeepSpeech.ipynb".
The first task is:
Plus a convenience method to calculate and print the WER report all at once.
Can you write Python code for it?

def calculate_and_print_wer_report(session, caption, tower_decodings, tower_labels, tower_total_losses, show_ranked=True):
    items, mean = calculate_wer(session, tower_decodings, tower_labels, tower_total_losses)
    if show_ranked:
        print_wer_report(caption, mean, items=items)
    else:
        print_wer_report(caption, mean)
    return items, mean


I am working on the file "DeepSpeech.ipynb".
The first task is:
Test and Validation
First we need a helper method to create a normal forward calculation without optimization, dropouts and special reporting.
Can you write Python code for it?

def decode_batch(data_set):
    # Get gradients for each tower (Runs across all GPU's)
    tower_decodings, tower_labels, _, tower_total_losses, _, _ = get_tower_results(data_set)
    return tower_decodings, tower_labels, tower_total_losses


I am working on the file "DeepSpeech.ipynb".
The first task is:
The first few lines of the function BiRNN
python
def BiRNN(batch_x, seq_length):
    # Input shape: [batch_size, n_steps, n_input + 2*n_input*n_context]
    batch_x_shape = tf.shape(batch_x)
    # Permute n_steps and batch_size
    batch_x = tf.transpose(batch_x, [1, 0, 2])
    # Reshape to prepare input for first layer
    batch_x = tf.reshape(batch_x, [-1, n_input + 2*n_input*n_context])
    ...
reshape batch_x which has shape [batch_size, n_steps, n_input + 2*n_input*n_context] initially, to a tensor with shape [n_steps*batch_size, n_input + 2*n_input*n_context]. This is done to prepare the batch for input into the first layer which expects a tensor of rank 2.
The next few lines of  BiRNN
python
    #Hidden layer with clipped RELU activation and dropout
    b1 = variable_on_cpu('b1', [n_hidden_1], tf.random_normal_initializer())
    h1 = variable_on_cpu('h1', [n_input + 2*n_input*n_context, n_hidden_1], tf.random_normal_initializer())
    layer_1 = tf.minimum(tf.nn.relu(tf.add(tf.matmul(batch_x, h1), b1)), relu_clip)
    layer_1 = tf.nn.dropout(layer_1, (1.0 - dropout_rate_placeholder))
    ...
pass batch_x through the first layer of the non-recurrent neural network, then applies dropout to the result.
The next few lines do the same thing, but for the second and third layers
python
    #Hidden layer with clipped RELU activation and dropout
    b2 = variable_on_cpu('b2', [n_hidden_2], tf.random_normal_initializer())
    h2 = variable_on_cpu('h2', [n_hidden_1, n_hidden_2], tf.random_normal_initializer())
    layer_2 = tf.minimum(tf.nn.relu(tf.add(tf.matmul(layer_1, h2), b2)), relu_clip)
    layer_2 = tf.nn.dropout(layer_2, (1.0 - dropout_rate_placeholder))
    #Hidden layer with clipped RELU activation and dropout
    b3 = variable_on_cpu('b3', [n_hidden_3], tf.random_normal_initializer())
    h3 = variable_on_cpu('h3', [n_hidden_2, n_hidden_3], tf.random_normal_initializer())
    layer_3 = tf.minimum(tf.nn.relu(tf.add(tf.matmul(layer_2, h3), b3)), relu_clip)
    layer_3 = tf.nn.dropout(layer_3, (1.0 -...
Can you write Python code for it?

def calculate_accuracy_and_loss(batch_set):
    # Obtain the next batch of data
    batch_x, batch_seq_len, batch_y = batch_set.next_batch()

    # Calculate the logits of the batch using BiRNN
    logits = BiRNN(batch_x, tf.to_int64(batch_seq_len))

    # Compute the CTC loss
    if use_warpctc:
        total_loss = tf.contrib.warpctc.warp_ctc_loss(logits, batch_y, batch_seq_len)
    else:
        total_loss = ctc_ops.ctc_loss(logits, batch_y, batch_seq_len)

    # Calculate the average loss across the batch
    avg_loss = tf.reduce_mean(total_loss)

    # Beam search decode the batch
    decoded, _ = ctc_ops.ctc_beam_search_decoder(logits, batch_seq_len)

    # Compute the edit (Levenshtein) distance
    distance = tf.edit_distance(tf.cast(decoded[0], tf.int32), batch_y)

    # Compute the accuracy
    accuracy = tf.reduce_mean(distance)

    # Return results to the caller
    return total_loss, avg_loss, accuracy, decoded, batch_y


I am working on the file "DeepSpeech.ipynb".
The first task is:
The number of units in the sixth layer is determined by n_character as follows
Can you write Python code for it?


I am working on the file "DeepSpeech.ipynb".
The first task is:
The number of units in the third layer, which feeds in to the LSTM, is determined by n_cell_dim as follows
Can you write Python code for it?


I am working on the file "DeepSpeech.ipynb".
The first task is:
To report progress and to get an idea of the current state of the model, we create a method that calculates the word error rate (WER) out of (tower) decodings and their respective (original) labels. It will return an array of WER result tuples, each consisting of

the original text (text version of X),
the resulting text (Y),
the calculated WER and
the total loss for that item,

plus the mean WER across all tuples.
Can you write Python code for it?

def calculate_wer(session, tower_decodings, tower_labels, tower_total_losses):
    originals = []
    results = []
    losses = []

    # Normalization
    tower_decodings = [j for i in tower_decodings for j in i]

    # Iterating over the towers
    for i in range(len(tower_decodings)):
        decoded, labels, loss = session.run([tower_decodings[i], tower_labels[i], tower_total_losses[i]], feed_dict)
        originals.extend(sparse_tensor_value_to_texts(labels))
        results.extend(sparse_tensor_value_to_texts(decoded))
        losses.extend(loss)

    # Pairwise calculation of all rates
    rates, mean = wers(originals, results)
    return zip(originals, results, rates, losses), mean


I am working on the file "DeepSpeech.ipynb".
The first task is:
Towers
In order to properly make use of multiple GPU's, one must introduce new abstractions, not present when using a single GPU, that facilitate the multi-GPU use case.
In particular, one must introduce a means to isolate the inference and gradient calculations on the various GPU's. The abstraction we intoduce for this purpose is called a 'tower'. A tower is specified by two properties:
* Scope - A scope, as provided by tf.name_scope(), is a means to isolate the operations within a tower. For example, all operations within "tower 0" could have their name prefixed with tower_0/.
*  Device - A hardware device, as provided by tf.device(), on which all operations within the tower execute. For example, all operations of "tower 0" could execute on the first GPU tf.device('/gpu:0').
As we are introducing one tower for each GPU, first we must determine how many GPU's are available
Can you write Python code for it?

# Get a list of the available gpu's ['/gpu:0', '/gpu:1'...]
available_devices = get_available_gpus()

# If there are no GPU's use the CPU
if 0 == len(available_devices):
    available_devices = ['/cpu:0']


I am working on the file "DeepSpeech.ipynb".
The first task is:
Training
Now, as we have prepared all the apropos operators and methods, we can create the method which trains the network.
Can you write Python code for it?

def train(session, data_sets):
    # Calculate the total number of batches
    total_batches = data_sets.train.total_batches
    batches_per_device = float(total_batches) / len(available_devices)

    # Create optimizer
    optimizer = create_optimizer()

    # Get gradients for each tower (Runs across all GPU's)
    tower_decodings, \
    tower_labels, \
    tower_gradients, \
    tower_total_losses, \
    tower_avg_losses, \
    avg_accuracy \
    = get_tower_results(data_sets.train, optimizer)

    # Validation step preparation
    validation_tower_decodings, \
    validation_tower_labels, \
    validation_tower_total_losses \
    = decode_batch(data_sets.dev)

    # Average tower gradients
    avg_tower_gradients = average_gradients(tower_gradients)

    # Add logging of averaged gradients
    log_grads_and_vars(avg_tower_gradients)

    # Apply gradients to modify the model
    apply_gradient_op = apply_gradients(optimizer, avg_tower_gradients)

    # Create a saver to checkpoint the model
    saver = tf.train.Saver(tf.all_variables())

    # Prepare tensor board logging
    merged = tf.merge_all_summaries()
    writer = tf.train.SummaryWriter(log_dir, session.graph)

    # Init all variables in session
    session.run(tf.initialize_all_variables())

    # Start queue runner threads
    tf.train.start_queue_runners(sess=session)

    # Start importer's queue threads
    data_sets.start_queue_threads(session)

    # Init recent word error rate levels
    last_train_wer = 0.0
    last_validation_wer = 0.0

    # Loop over the data set for training_epochs epochs
    for epoch in range(training_iters):
        # Define total accuracy for the epoch
        total_accuracy = 0.0

        # Loop over the batches
        for batch in range(int(ceil(batches_per_device))):
            extra_params = { }
            if do_fulltrace:
                loss_run_metadata            = tf.RunMetadata()
                extra_params['options']      =...


I am working on the file "Deep_Learning_Project-Pytorch.ipynb".
The first task is:
A small comparison of IMDB and TMDB
Now that we have both systems running, let's do a very short comparison for the same movie?
Can you write Python code for it?

print "The genres for The Matrix pulled from IMDB are -",movie['genres']
print "The genres for The Matrix pulled from TMDB are -",get_movie_genres_tmdb("The Matrix")


I am working on the file "Deep_Learning_Project-Pytorch.ipynb".
The first task is:
Also, let's remove movies for which we have no posters!
Can you write Python code for it?

poster_movies=[]
counter=0
movies_no_poster=[]
print("Total movies : ",len(movies))
print("Started downloading posters...")
for movie in movies:
    if counter%10==0:
        print(counter)
    id=movie['id']
    title=movie['title']
    if counter==1:
        print('Downloaded first. Code is working fine. Please wait, this will take quite some time...')
    if counter%300==0 and counter!=0:
        print "Done with ",counter," movies!"
        print "Trying to get poster for ",title
    try:
        #grab_poster_tmdb(title)
        poster_movies.append(movie)
    except:
        try:
            time.sleep(7)
            grab_poster_tmdb(title)
            poster_movies.append(movie)
        except:
            movies_no_poster.append(movie)
    counter+=1
print("Done with all the posters!")

print len(movies_no_poster)
print len(poster_movies)

f=open('poster_movies.pckl','w')
pickle.dump(poster_movies,f)
f.close()

f=open('poster_movies.pckl','r')
poster_movies=pickle.load(f)
f.close()

f=open('no_poster_movies.pckl','w')
pickle.dump(movies_no_poster,f)
f.close()

f=open('no_poster_movies.pckl','r')
movies_no_poster=pickle.load(f)
f.close()


I am working on the file "Deep_Learning_Project-Pytorch.ipynb".
The first task is:
Are all words equally important?
At the cost of sounding "Animal Farm" inspired, I would say not all words are equally important.
For example, let's consider the overview for the Matrix -
Can you write Python code for it?


I am working on the file "Deep_Learning_Project-Pytorch.ipynb".
The first task is:
Congratulations, we are done scraping!
Building a dataset out of the scraped information!
This task is simple, but extremely important. It's basically what will set the stage for the whole project. Given that you have the freedom to cast their own project within the framework I am providing, there are many decisions that you must make to finalize your own version of the project.
As we are working on a classification problem, we need to make two decisions given the data at hand -
* What do we want to predict, i.e. what's our Y?
* What features to use for predicting this Y, i.e. what X should we use?
There are many different options possible, and it comes down to you to decide what's most exciting. I will be picking my own version for the example, but it is imperative that you think this through, and come up with a version which excites you!
As an example, here are some possible ways to frame Y, while still sticking to the problem of genre prediction -

Assume every movie can have multiple genres, and then it becomes a multi-label classification problem. For example, a movie can be Action, Horror and Adventure simultaneously. Thus, every movie can be more than one genre.

Make clusters of genres as we did in Milestone 1 using biclustering, and then every movie can have only 1 genre. This way, the problem becomes a simpler, multi-class problem. For example, a movie could have the class - Uplifting (refer Milestone 1), or Horror or History. No movie get's more than one class.

For the purposes of this implementation, I'm going with the first case explained above - i.e. a multi-label classification problem.
Similarly, for designing our input features i.e. X, you may pick any features you think make sense, for example, the Director of a movie may be a good predictor for genre. OR, they may choose any features they design using algorithms like PCA. Given the richness of IMDB, TMDB and alternate sources like Wikipedia, there is a plethora of options available. Be creative here!
Another important thing to note is...
Can you write Python code for it?

movies_with_overviews=[]
for i in range(len(no_duplicate_movies)):
    movie=no_duplicate_movies[i]
    id=movie['id']
    overview=movie['overview']

    if len(overview)==0:
        continue
    else:
        movies_with_overviews.append(movie)

len(movies_with_overviews)


I am working on the file "Deep_Learning_Project-Pytorch.ipynb".
The first task is:
For "The Matrix" a word like "computer" is a stronger indicators of it being a Sci-Fi movie, than words like "who" or "powerful" or "vast". One way computer scientists working with natural language tackled this problem in the past (and it is still used very popularly) is what we call TF-IDF i.e. Term Frequence, Inverse Document Frequency. The basic idea here is that words that are strongly indicative of the content of a single document (every movie overview is a document in our case) are words that occur very frequently in that document, and very infrequently in all other documents. For example, "Computer" occurs twice here but probably will not in most other movie overviews. Hence, it is indicative. On the other hand, generic words like "a","and","the" will occur very often in all documents. Hence, they are not indicative.
So, can we use this information to reduce our insanely high 30,000 dimensional vector representation to a smaller, more handle-able number? But first up, why should we even care? The answer is probably one of the most used phrases in ML - "The Curse of Dimensionality".
The Curse of Dimensionality
This section is strongly borrowing from one of the greatest ML papers I've ever read.
This expression was coined by Bellman in 1961 to refer to the fact that many algorithms that work fine in low dimensions become intractable when the input is high-dimensional. The reason for them not working in high dimensions is very strongly linked to what we discussed earlier - having a representative dataset. Consider this, you have a function $f$ dependent only one dependent variable $x$, and $x$ can only integer values from 1 to 100. Since it's one dimensional, it can be plotted on a line. To get a representative sample, you'd need to sample something like - $f(1),f(20),f(40),f(60),f(80),f(100)$
Now, let's increase the dimensionality i.e. number of dependent variables and see what happens. Say, we have 2 variables $x_1$ and $x_2$, same...
Can you write Python code for it?

# The min_df paramter makes sure we exclude words that only occur very rarely
# The default also is to exclude any words that occur in every movie description
vectorize=CountVectorizer(max_df=0.95, min_df=0.005)
X=vectorize.fit_transform(content)


I am working on the file "Deep_Learning_Project-Pytorch.ipynb".
The first task is:
Getting movie information from IMDB
Now that we know how to get information from TMDB, here's how we can get information about the same movie from IMDB. This makes it possible for us to combine more information, and get a richer dataset. I urge you to try and see what dataset you can make, and go above and beyond the basic things I've done in this tutorial. Due to the differences between the two datasets, you will have to do some cleaning, however both of these datasets are extremely clean and it will be minimal.
Can you write Python code for it?

import imdb
# Create the IMDB object that will be used to access the IMDb's database.
imbd_object = imdb.IMDb() # by default access the web.

# Search for a movie (get a list of Movie objects).
results = imbd_object.search_movie('The Matrix')
#print(len(results))
# As this returns a list of all movies containing the word "The Matrix", we pick the first element
movie = results[0]

imbd_object.update(movie)

print "All the information we can get about this movie from IMDB-"
movie.keys()

print "The genres associated with the movie are - ",movie['genres']


I am working on the file "Deep_Learning_Project-Pytorch.ipynb".
The first task is:
Here is a broad outline of technical steps to be done for data collection

Sign up for TMDB (themoviedatabase.org), and set up API to scrape movie posters for above movies.
Set up and work with TMDb to get movie information from their database
Do the same for IMDb
Compare the entries of IMDb and TMDb for a movie
Get a listing and information of a few movies
Think and ponder over the potential challenges that may come our way, and think about interesting questions we can answer given the API's we have in our hands.
Get data from the TMDb

Let's go over each one of these one by one.
Signing up for TMDB and getting set up for getting movie metadata.

Step 1. Head over to tmdb.org and create a new account there by signing up.
Step 2. Click on your account icon on the top right, then from drop down menu select "Settings".
Step 3. On the settings page, you will see the option "API" on the left pane. Click on that.
Step 4. Apply for a new developer key. Fill out the form as required. The fields "Application Name" and "Application URL" are not important. Fill anything there.
Step 5. It should generate a new API key for you and you should also receive a mail.

Now that you have the API key for TMDB, you can query using TMDB. Remember, it allows only 40 queries per 10 seconds.
An easy way to respect this is to just have a call to time.sleep(1) after each iteration. This is also being very nice to the server.
If you want to try and maximize your throughput you can embed every TMDB request in a nested try except block. If the first try fails, the second try first uses python's sleep function to give it a little rest, and then try again to make a request. Something like this -
~~~~
try:
    search.movie(query=movie) #An API request
except:
    try:
        time.sleep(10) #sleep for a bit, to give API requests a rest.
        search.movie(query=movie_name) #Make second API request
    except:
        print "Failed second attempt too, check if there's any error in request"
~~~~
Using TMDB using the obtained...
Can you write Python code for it?

import os
# set here the path where you want the scraped folders to be saved!
poster_folder='posters_final/'
if poster_folder.split('/')[0] in os.listdir('./'):
    print('Folder already exists')
else:
    os.mkdir('./'+poster_folder)

poster_folder

# For the purpose of this example, i will be working with the 1999 Sci-Fi movie - "The Matrix"!
import tmdbsimple as tmdb
api_key = '' #Enter your own API key here to run the code below.
# Generate your own API key as explained above :)

tmdb.API_KEY = api_key #This sets the API key setting for the tmdb object
search = tmdb.Search() #this instantiates a tmdb "search" object which allows your to search for the movie
import os.path
# These functions take in a string movie name i.e. like "The Matrix" or "Interstellar"
# What they return is pretty much clear in the name - Poster, ID , Info or genre of the Movie!
def grab_poster_tmdb(movie):
    response = search.movie(query=movie)
    id=response['results'][0]['id']
    movie = tmdb.Movies(id)
    posterp=movie.info()['poster_path']
    title=movie.info()['original_title']
    if os.path.isfile(poster_folder+title+'.jpg '):
        return
    url='image.tmdb.org/t/p/original'+posterp
    title='_'.join(title.split(' '))
    strcmd='wget -O '+poster_folder+title+'.jpg '+url
    os.system(strcmd)

def get_movie_id_tmdb(movie):
    response = search.movie(query=movie)
    movie_id=response['results'][0]['id']
    return movie_id

def get_movie_info_tmdb(movie):
    response = search.movie(query=movie)
    id=response['results'][0]['id']
    movie = tmdb.Movies(id)
    info=movie.info()
    return info

def get_movie_genres_tmdb(movie):
    response = search.movie(query=movie)
    id=response['results'][0]['id']
    movie = tmdb.Movies(id)
    genres=movie.info()['genres']
    return genres


I am working on the file "Deep_Learning_Project-Pytorch.ipynb".
The first task is:
Let's convert this list into a nice dictionary to look up genre names from genre IDs!
Can you write Python code for it?

Genre_ID_to_name={}
for i in range(len(list_of_genres)):
    genre_id=list_of_genres[i]['id']
    genre_name=list_of_genres[i]['name']
    Genre_ID_to_name[genre_id]=genre_name


I am working on the file "Deep_Learning_Project-Pytorch.ipynb".
The first task is:
Let's first get the labels on our 1342 samples first! As image download fails on a few instances, the best way to work with the right model is to read the poster names downloaded, and working from there. These posters cannot be uploaded to Github as they are too large, and so are being downloaded and read from my local computer. If you do re-do it, you might have to check and edit the paths in the code to make sure it runs.
Can you write Python code for it?


I am working on the file "Deep_Learning_Project-Pytorch.ipynb".
The first task is:
Let's look at one of these movies. It's the same format as above, as we had information on the movie "The Matrix", as you can see below. It's a dictionary which can be queried  for specific information on that movie
Can you write Python code for it?

first_movie=top20_movs[0]
print "Here is all the information you can get on this movie - "
print first_movie
print "\n\nThe title of the first movie is - ", first_movie['title']


I am working on the file "Deep_Learning_Project-Pytorch.ipynb".
The first task is:
Let's print out top 5 movie's titles!
Can you write Python code for it?

for i in range(len(top20_movs)):
    mov=top20_movs[i]
    title=mov['title']
    print title
    if i==4:
        break


I am working on the file "Deep_Learning_Project-Pytorch.ipynb".
The first task is:
Let's take a look at the structure we just made. It is a 19X19 structure, as shown below. Also, see that we had 19 Genres. Needless to say, this structure counts the number of simultaneous occurrences of genres in same movie.
Can you write Python code for it?

print visGrid.shape
print len(Genre_ID_to_name.keys())

annot_lookup = []
for i in xrange(len(nr_ids)):
    annot_lookup.append(Genre_ID_to_name[nr_ids[i]])

sns.heatmap(visGrid, xticklabels=annot_lookup, yticklabels=annot_lookup)


I am working on the file "Deep_Learning_Project-Pytorch.ipynb".
The first task is:
Looking at the above figure, "boxes" or groups of movie genres automatically emerge!
Intuitively - Crime, Sci-Fi, Mystery, Action, Horror, Drama, Thriller, etc co-occur.
AND, Romance, Fantasy, Family, Music, Adventure, etc co-occur.
That makes a lot of intuitive sense, right?
One challenge is the broad range of the drama genre. It makes the two clusters highly overlapping. If we merge it together with action thriller, etc. We will end up with nearly all movies just having that label.
Based on playing around with the stuff above, we can sort the data into the following genre categories - "Drama, Action, ScienceFiction, exciting(thriller, crime, mystery), uplifting(adventure, fantasy, animation, comedy, romance, family), Horror, History"
Note: that this categorization is subjective and by no means the only right solution. One could also just stay with the original labels and only exclude the ones with not enough data. Such tricks are important to balance the dataset, it allows us to increase or decrease the strength of certain signals, making it possible to improve our inferences :)
Interesting Questions
This really should be a place for you to get creative and hopefully come up with better questions than me.
Here are some of my thoughts:
- Which actors are bound to a genre, and which can easily hop genres?
- Is there a trend in genre popularity over the years?
- Can you use sound tracks to identify the genre of a movie?
- Are top romance actors higher paid than top action actors?
- If you look at release date vs popularity score, which movie genres have a longer shelf life?
Ideas to explore specifically for feature correlations:
- Are title length correlated with movie genre?
- Are movie posters darker for horror than for romance end comedy?
- Are some genres specifically released more often at a certain time of year?
- Is the RPG rating correlated with the genre?
Based on this new category set, we will now pull posters from TMDB as our training data!
Can you write Python code for it?

# Done before, reading from pickle file now to maintain consistency of data!
# We now sample 100 movies per genre. Problem is that the sorting is by popular movies, so they will overlap.
# Need to exclude movies that were already sampled.
movies = []
baseyear = 2017

print('Starting pulling movies from TMDB. If you want to debug, uncomment the print command. This will take a while, please wait...')
done_ids=[]
for g_id in nr_ids:
    #print('Pulling movies for genre ID '+g_id)
    baseyear -= 1
    for page in xrange(1,6,1):
        time.sleep(0.5)

        url = 'https://api.themoviedb.org/3/discover/movie?api_key=' + api_key
        url += '&language=en-US&sort_by=popularity.desc&year=' + str(baseyear)
        url += '&with_genres=' + str(g_id) + '&page=' + str(page)

        data = urllib2.urlopen(url).read()

        dataDict = json.loads(data)
        movies.extend(dataDict["results"])
    done_ids.append(str(g_id))
print("Pulled movies for genres - "+','.join(done_ids))

f6=open("movies_for_posters",'wb')
pickle.dump(movies,f6)
f6.close()

f6=open("movies_for_posters",'rb')
movies=pickle.load(f6)
f6.close()


I am working on the file "Deep_Learning_Project-Pytorch.ipynb".
The first task is:
Now let's store the genre's for these movies in a list that we will later transform into a binarized vector.
Binarized vector representation is a very common and important way data is stored/represented in ML. Essentially, it's a way to reduce a categorical variable with n possible values to n binary indicator variables. What does that mean? For example, let [(1,3),(4)] be the list saying that sample A has two labels 1 and 3, and sample B has one label 4. For every sample, for every possible label, the representation is simply 1 if it has that label, and 0 if it doesn't have that label. So the binarized version of the above list will be -
~~~~~
[(1,0,1,0]),
(0,0,0,1])]
~~~~~
Can you write Python code for it?

# genres=np.zeros((len(top1000_movies),3))
genres=[]
all_ids=[]
for i in range(len(movies_with_overviews)):
    movie=movies_with_overviews[i]
    id=movie['id']
    genre_ids=movie['genre_ids']
    genres.append(genre_ids)
    all_ids.extend(genre_ids)

from sklearn.preprocessing import MultiLabelBinarizer
mlb=MultiLabelBinarizer()
Y=mlb.fit_transform(genres)

genres[1]

print Y.shape
print np.sum(Y, axis=0)

len(list_of_genres)


I am working on the file "Deep_Learning_Project-Pytorch.ipynb".
The first task is:
Now, we can simply look up for a word in the above loaded model. For example, to get the Word2Vec representation of the word "King" we just do - model2['king']
Can you write Python code for it?

print model2['king'].shape
print model2['dog'].shape


I am working on the file "Deep_Learning_Project-Pytorch.ipynb".
The first task is:
Now, we create our own keras neural network to use the VGG features and then classify movie genres. Keras makes this super easy.
Neural network architectures have gotten complex over the years. But the simplest ones contain very standard computations organized in layers, as described above. Given the popularity of some of these, Keras makes it as easy as writing out the names of these operations in a sequential order. This way you can make a network while completely avoiding the Mathematics (HIGHLY RECOMMENDED SPENDING MORE TIME ON THE MATH THOUGH)
Sequential() allows us to make models the follow this sequential order of layers. Different kinds of layers like Dense, Conv2D etc can be used, and many activation functions like RELU, Linear etc are also available.
Important Question : Why do we need activation functions?
Copy pasting the answer I wrote for this question on Quora Feel free to leave comments there.
""Sometimes, we tend to get lost in the jargon and confuse things easily, so the best way to go about this is getting back to our basics.
Don’t forget what the original premise of machine learning (and thus deep learning) is - IF the input and output are related by a function y=f(x), then if we have x, there is no way to exactly know f unless we know the process itself. However, machine learning gives you the ability to approximate f with a function g, and the process of trying out multiple candidates to identify the function g best approximating f is called machine learning.
Ok, that was machine learning, and how is deep learning different? Deep learning simply tries to expand the possible kind of functions that can be approximated using the above mentioned machine learning paradigm. Roughly speaking, if the previous model could learn say 10,000 kinds of functions, now it will be able to learn say 100,000 kinds (in actuality both are infinite spaces but one is larger than the other,...
Can you write Python code for it?

from __future__ import division
import torch
import time
import copy

class FeatureDataset():
    def __init__(self, X_train, Y_train, transform=None):

        self.X_train = X_train
        self.Y_train = Y_train
        self.transform = transform

    def __len__(self):
        return len(self.X_train)

    def __getitem__(self, idx):

        #sample = {'image': self.X_train[idx], 'landmarks': self.Y_train[idx]}

#         if self.transform:
#             sample = self.transform(sample)

        return X_train[idx].astype(float), Y_train[idx].astype(float)

def train_model(model, criterion, optimizer, num_epochs):

    since = time.time()
    USE_TENSORBOARD = 0
    best_model = model
    best_acc = 0.0
    use_gpu = 0

    for epoch in range(num_epochs):
        print('Epoch {}/{}'.format(epoch, num_epochs - 1))
        print('-' * 10)

        # Each epoch has a training and validation phase
        for phase in ['train', 'test']:
            if phase == 'train':
                mode='train'
                #optimizer = lr_scheduler(optimizer, epoch)
                model.train()  # Set model to training mode
                print "TRAINING STARTED"
            else:
                model.eval()
                mode='val'
                print "TESTING STARTED"

            running_loss = 0.0
            running_corrects = 0

            counter=0
            # Iterate over data.
            answer = []
            for data in dset_loaders[phase]:
                inputs, labels = data

                # wrap them in Variable
                if use_gpu:
                    try:
                        inputs, labels = Variable(inputs.float().cuda()),
                        Variable(labels.long().cuda())
                    except:
                        print(inputs,labels)
                else:
                    inputs, labels = Variable(Variable(inputs).float()), Variable(Variable(labels).float())

                # Set gradient to...


I am working on the file "Deep_Learning_Project-Pytorch.ipynb".
The first task is:
Our binarized Y numpy array contains the binarized labels corresponding to the genre IDs of the 1277 movies
Can you write Python code for it?

visual_problem_data=(X,Y)
f8=open('visual_problem_data_clean.pckl','wb')
pickle.dump(visual_problem_data,f8)
f8.close()

f8=open('visual_problem_data_clean.pckl','rb')
visual_features=pickle.load(f8)
f8.close()

(X,Y)=visual_features

X.shape

mask = np.random.rand(len(X)) < 0.8

X_train=X[mask]
X_test=X[~mask]
Y_train=Y[mask]
Y_test=Y[~mask]
X_train

Y_test.shape


I am working on the file "Deep_Learning_Project-Pytorch.ipynb".
The first task is:
Section 4 - Building a dataset to work with : Let's take a look at the top 1000 movies from the database
Making use of the same api as before, we will just pull results from the top 50 pages. As mentioned earlier, the "page" attribute of the command top_movies=all_movies.popular() can be used for this purpose.
Please note: Some of the code below will store the data into python "pickle" files so that it can be ready directly from memory, as opposed to being downloaded every time. Once done, you should comment out any code which generated an object that was pickled and is no longer needed.
Can you write Python code for it?

all_movies=tmdb.Movies()
top_movies=all_movies.popular()

# This is a dictionary, and to access results we use the key 'results' which returns info on 20 movies
len(top_movies['results'])
top20_movs=top_movies['results']

# Comment out this cell once the data is saved into pickle file.
all_movies=tmdb.Movies()
top1000_movies=[]
print('Pulling movie list, Please wait...')
for i in range(1,51):
    if i%15==0:
        time.sleep(7)
    movies_on_this_page=all_movies.popular(page=i)['results']
    top1000_movies.extend(movies_on_this_page)
len(top1000_movies)
f3=open('movie_list.pckl','wb')
pickle.dump(top1000_movies,f3)
f3.close()
print('Done!')

f3=open('movie_list.pckl','rb')
top1000_movies=pickle.load(f3)
f3.close()


I am working on the file "Deep_Learning_Project-Pytorch.ipynb".
The first task is:
So, TMDB doesn't want to make your job as easy as you thought. Why these random numbers? Want to see their genre names? Well, there's the Genre() class for it. Let's get this done!
Can you write Python code for it?

# Create a tmdb genre object!
genres=tmdb.Genres()
# the list() method of the Genres() class returns a listing of all genres in the form of a dictionary.
list_of_genres=genres.list()['genres']


I am working on the file "Deep_Learning_Project-Pytorch.ipynb".
The first task is:
So, each movie's overview gets represented by a 1x1365 dimensional vector.
Now, we are ready for the kill. Our data is cleaned, hypothesis is set (Overview can predict movie genre), and the feature/output vectors are prepped. Let's train some models!
Can you write Python code for it?

import pickle
f4=open('X.pckl','wb')
f5=open('Y.pckl','wb')
pickle.dump(X,f4)
pickle.dump(Y,f5)
f6=open('Genredict.pckl','wb')
pickle.dump(Genre_ID_to_name,f6)
f4.close()
f5.close()
f6.close()


I am working on the file "Deep_Learning_Project-Pytorch.ipynb".
The first task is:
So, even with just the poster i.e. visual features we are able to make great predictions! Sure, text outperforms the visual features, but the important thing is that it still works. In more complicated models, we can combine the two to make even better predictions. That is precisely what I work on in my research.
These models were trained on CPU's, and a simple 1 layer model was used to show that there is a lot of information in this data that the models can extract. With a larger dataset, and more training I was able to bring these numbers to as high as 70%, which is the similar to textual features. Some teams in my class outperformed this even more. More data is the first thing you should try if you want better results. Then, you can start playing with training on GPUs, learning rate schedules and other hyperparameters. Finally, you can consider using ResNet, a much more powerful neural network model than VGG. All of these can be tried once you have a working knowledge of machine learning.
Section 8 - Deep Learning to get Textual Features
Let's do the same thing as above with text now?
We will use an off the shelf representation for words - Word2Vec model. Just like VGGnet before, this is a model made available to get a meaningful representation. As the total number of words is small, we don't even need to forward propagate our sample through a network. Even that has been done for us, and the result is stored in the form of a dictionary. We can simply look up the word in the dictionary and get the Word2Vec features for the word.
You can download the dictionary from here - https://drive.google.com/file/d/0B7XkCwpI5KDYNlNUTTlSS21pQmM/edit <br>
Download it to the directory of this tutorial i.e. in the same folder as this ipython notebook.
Can you write Python code for it?

from gensim import models
# model2 = models.Word2Vec.load_word2vec_format('GoogleNews-vectors-negative300.bin', binary=True)
model2 = models.KeyedVectors.load_word2vec_format('GoogleNews-vectors-negative300.bin', binary=True)


I am working on the file "Deep_Learning_Project-Pytorch.ipynb".
The first task is:
So, how do we store this movie overview in a matrix?
Do we just store the whole string? We know that we need to work with numbers, but this is all text. What do we do?!
The way we will be storing the X matrix is called a "Bag of words" representation. The basic idea of this representation in our context is that we can think of all the distinct words that are possible in the movies' reviews as a distinct object. And then every movie overview can be thought as a "Bag" containing a bunch of these possible objects.
For example, in the case of Zootopia the movie above - The "Bag" contains the words ("Determined", "to", "prove", "herself"......"the", "mystery"). We make such lists for all movie overviews. Finally, we binarize again like we did above for Y. scikit-learn makes our job easy here by simply using a function CountVectorizer() because this representation is so often used in Machine Learning.
What this means is that, for all the movies that we have the data on, we will first count all the unique words. Say, there's 30,000 unique words. Then we can represent every movie overview as a 30000x1 vector, where each position in the vector corresponds to the presence or absence of a particular word. If the word corresponding to that position is present in the overview, that position will have 1, otherwise it will be 0.
Ex - if our vocabular was 4 words - "I","am","a","good","boy", then the representation for the sentence "I am a boy" would be [1 1 1 0 1], and for the sentence "I am good" would be [1 1 0 1 0].
Can you write Python code for it?

from sklearn.feature_extraction.text import CountVectorizer
import re

content=[]
for i in range(len(movies_with_overviews)):
    movie=movies_with_overviews[i]
    id=movie['id']
    overview=movie['overview']
    overview=overview.replace(',','')
    overview=overview.replace('.','')
    content.append(overview)

print len(movies_with_overviews)
print content[0]
print len(content)


I am working on the file "Deep_Learning_Project-Pytorch.ipynb".
The first task is:
So, to get the tagline of the movie we can use the above dictionary key -
Can you write Python code for it?

info=get_movie_info_tmdb("The Matrix")
print info['tagline']


I am working on the file "Deep_Learning_Project-Pytorch.ipynb".
The first task is:
Text needs some pre-processing before we can train the model. The only preprocessing we do here is - we delete commonly occurring words which we know are not informative about the genre. Think of it as the clutter in some sense. These words are often removed and are referred to as "stop words". You can look them up online. These include simple words like "a", "and", "but", "how", "or" and so on. They can be easily removed using the python package NLTK.
From the above dataset, movies with overviews which contain only stop words, or movies with overviews containing no words with word2vec representation are neglected. Others are used to build our Mean word2vec representation. Simply, put for every movie overview -

Take movie overview
Throw out stop words
For non stop words:
If in word2vec - take it's word2vec representation which is 300 dimensional
If not - throw word

For each movie, calculate the arithmetic mean of the 300 dimensional vector representations for all words in the overview which weren't thrown out

This mean becomes the 300 dimensional representation for the movie. For all movies, these are stored in a numpy array. So the X matrix becomes (1263,300). And, Y is (1263,20) i.e. binarized 20 genres, as before
Why do we take the arithmetic mean?
If you feel that we should have kept all the words separately - Then you're thinking correct, but sadly we're limited by the way current day neural networks work. I will not mull over this for the fear of stressing too much on an otherwise irrelevant detail. But if you're interested, read this awesome paper -
https://jiajunwu.com/papers/dmil_cvpr.pdf
Can you write Python code for it?

genres=[]
rows_to_delete=[]
for i in range(len(final_movies_set)):
    mov=final_movies_set[i]
    movie_genres=mov['genre_ids']
    genres.append(movie_genres)
    overview=mov['overview']
    tokens = tokenizer.tokenize(overview)
    stopped_tokens = [k for k in tokens if not k in en_stop]
    count_in_vocab=0
    s=0
    if len(stopped_tokens)==0:
        rows_to_delete.append(i)
        genres.pop(-1)
#         print overview
#         print "sample ",i,"had no nonstops"
    else:
        for tok in stopped_tokens:
            if tok.lower() in model2.vocab:
                count_in_vocab+=1
                s+=model2[tok.lower()]
        if count_in_vocab!=0:
            movie_mean_wordvec[i]=s/float(count_in_vocab)
        else:
            rows_to_delete.append(i)
            genres.pop(-1)
#             print overview
#             print "sample ",i,"had no word2vec"

len(genres)

mask2=[]
for row in range(len(movie_mean_wordvec)):
    if row in rows_to_delete:
        mask2.append(False)
    else:
        mask2.append(True)

X=movie_mean_wordvec[mask2]

X.shape

Y=mlb.fit_transform(genres)

Y.shape

textual_features=(X,Y)
f9=open('textual_features.pckl','wb')
pickle.dump(textual_features,f9)
f9.close()

# textual_features=(X,Y)
f9=open('textual_features.pckl','rb')
textual_features=pickle.load(f9)
f9.close()

(X,Y)=textual_features

X.shape

Y.shape

mask_text=np.random.rand(len(X))<0.8

X_train=X[mask_text]
Y_train=Y[mask_text]
X_test=X[~mask_text]
Y_test=Y[~mask_text]


I am working on the file "Deep_Learning_Project-Pytorch.ipynb".
The first task is:
The above image shows how often the genres occur together, as a heatmap
Important thing to notice in the above plot is the diagonal. The diagonal corresponds to self-pairs, i.e. number of times a genre, say Drama occurred with Drama. Which is basically just a count of the total times that genre occurred!
As we can see there are a lot of dramas in the data set, it is also a very unspecific label. There are nearly no documentaries or TV Movies. Horror is a very distinct label, and romance is also not too widely spread.
To account for this unbalanced data, there are multiple things we can try to explore what interesting relationships can be found.
Delving Deeper into co-occurrence of genres
What we want to do now is to look for nice groups of genres that co-occur, and see if it makes sense to us logically? Intuitively speaking, wouldn't it be fun if we saw nice boxes on the above plot - boxes of high intensity i.e. genres that occur together and don't occur much with other genres. In some ways, that would isolate the co-occurrence of some genres, and heighten the co-occurrence of others.
While the data may not show that directly, we can play with the numbers to see if that's possible. The technique used for that is called biclustering.
Can you write Python code for it?

from sklearn.cluster import SpectralCoclustering

model = SpectralCoclustering(n_clusters=5)
model.fit(visGrid)

fit_data = visGrid[np.argsort(model.row_labels_)]
fit_data = fit_data[:, np.argsort(model.column_labels_)]

annot_lookup_sorted = []
for i in np.argsort(model.row_labels_):
    annot_lookup_sorted.append(Genre_ID_to_name[nr_ids[i]])

sns.heatmap(fit_data, xticklabels=annot_lookup_sorted, yticklabels=annot_lookup_sorted, annot=False)
plt.title("After biclustering; rearranged to show biclusters")

plt.show()


I am working on the file "Deep_Learning_Project-Pytorch.ipynb".
The first task is:
This looks odd, why are we re-running the loop we ran above again below? The reason is simple, the most important thing to know about numpy is that using vstack() and hstack() are highly sub-optimal. Numpy arrays when created, a fixed size is allocated in the memory and when we stack, a new one is copied and created in a new location. This makes the code really, really slow. The best way to do it (and this remains the same with MATLAB matrices if you work with them), is to create a numpy array of zeros, and over-write it row by row. The above code was just to see what size numpy array we will need!
The final movie poster set for which we have all the information we need, is 1265 movies. In the above code we are making an X numpy array containing the visual features of one image per row. So, the VGG features are reshaped to be in the shape (1,25088) and we finally obtain a matrix of shape (1265,25088)
Can you write Python code for it?

np_features=np.zeros((len(feature_list),feature_size))
print len(feature_list),feature_size
for i in range(len(feature_list)):
    feat=feature_list[i]
    reshaped_feat=feat.reshape(1,-1)
    #print reshaped_feat.shape
    np_features[i]=reshaped_feat

# np_features[-1]

X=np_features

from sklearn.preprocessing import MultiLabelBinarizer
mlb=MultiLabelBinarizer()
Y=mlb.fit_transform(genre_list)

Y.shape


I am working on the file "Deep_Learning_Project-Pytorch.ipynb".
The first task is:
This way, we can represent the words in our overviews using this word2vec model. And then, we can use that as our X representations. So, instead of count of words, we are using a representation which is based on the semantic representation of the word. Mathematically, each word went from 3-4 dimensional (the length) to 300 dimensions!
For the same set of movies above, let's try and predict the genres from the deep representation of their overviews!
Can you write Python code for it?

final_movies_set = movies_with_overviews
len(final_movies_set)

from nltk.tokenize import RegexpTokenizer
from stop_words import get_stop_words
tokenizer = RegexpTokenizer(r'\w+')

# create English stop words list
en_stop = get_stop_words('en')

movie_mean_wordvec=np.zeros((len(final_movies_set),300))
movie_mean_wordvec.shape


I am working on the file "Deep_Learning_Project-Pytorch.ipynb".
The first task is:
Training a simple neural network model using these VGG features.
Can you write Python code for it?


I am working on the file "Deep_Learning_Project-Pytorch.ipynb".
The first task is:
We are excluding all words that occur in too many or too few documents, as these are very unlikely to be discriminative. Words that only occur in one document most probably are names, and words that occur in nearly all documents are probably stop words. Note that the values here were not tuned using a validation set. They are just guesses. It is ok to do, because we didn't evaluate the performance of these parameters. In a strict case, for example for a publication, it would be better to tune these as well.
Can you write Python code for it?


I am working on the file "Deep_Learning_Project-Pytorch.ipynb".
The first task is:
We train the model using the fit() function. The parameters it takes are - training features and training labels, epochs, batch_size and verbose.
Simplest one - verbose. 0="dont print anything as you work", 1="Inform me as you go".
Often the data set is too large to be loaded into the RAM. So, we load data in batches. For batch_size=32 and epochs=10, the model starts loading rows from X in batches of 32 everytime it calculates the loss and updates the model. It keeps on going till it has covered all the samples 10 times.
So, the no. of times model is updated = (Total Samples/Batch Size) * (Epochs)
Can you write Python code for it?


I am working on the file "Deep_Learning_Project-Pytorch.ipynb".
The first task is:
Well, this genre ID wasn't given to us by TMDB when we asked it for all possible genres. How do we go about this now? We can either neglect all samples that have this genre. But if you look up you'll see there's too many of these samples. So, I googled more and went into their documentation and found that this ID corresponds to the genre "Foreign". So, we add it to the dictionary of genre names ourselves. Such problems are ubiquitous in machine learning, and it is up to us to diagnose and correct them. We must always make a decision about what to keep, how to store data and so on.
Can you write Python code for it?

Genre_ID_to_name[10769]="Foreign" #Adding it to the dictionary

len(Genre_ID_to_name.keys())


I am working on the file "Deep_Learning_Project-Pytorch.ipynb".
The first task is:
While the above functions have been made to make it easy to get genres, posters and ID, all the information that can be accessed can be seen by calling the function get_movie_info() as shown below
Can you write Python code for it?

print get_movie_genres_tmdb("The Matrix")

info=get_movie_info_tmdb("The Matrix")
print "All the Movie information from TMDB gets stored in a dictionary with the following keys for easy access -"
info.keys()


I am working on the file "Deep_Learning_Project-Pytorch.ipynb".
The first task is:
Yes, I know. I'm a little upset too seeing The Nun above Avengers: Infinity War!
Moving on, we can get their genres the same way.
Can you write Python code for it?

for i in range(len(top20_movs)):
    mov=top20_movs[i]
    genres=mov['genre_ids']
    print genres
    if i==4:
        break


I am working on the file "Deep_Learning_Project.ipynb".
The first task is:
Are all words equally important?
At the cost of sounding "Animal Farm" inspired, I would say not all words are equally important.
For example, let's consider the overview for the Matrix -
Can you write Python code for it?


I am working on the file "Deep_Learning_Project.ipynb".
The first task is:
As can be seen above, the results seem promising, but how do we really compare the two models? We need to quantify our performance so that we can say which one's better. Takes us back to what we discussed right in the beginning - we're learning a function $g$ which can approximate the original unknown function $f$. For some values of $x_i$, the predictions will be wrong for sure, and we want to minimize it.
For multi label systems, we often keep track of performance using "Precision" and "Recall". These are standard metrics, and you can google to read up more about them if you're new to these terms.
Evaluation Metrics
We will use the standard precision recall metrics for evaluating our system.
Can you write Python code for it?

def precision_recall(gt,preds):
    TP=0
    FP=0
    FN=0
    for t in gt:
        if t in preds:
            TP+=1
        else:
            FN+=1
    for p in preds:
        if p not in gt:
            FP+=1
    if TP+FP==0:
        precision=0
    else:
        precision=TP/float(TP+FP)
    if TP+FN==0:
        recall=0
    else:
        recall=TP/float(TP+FN)
    return precision,recall

precs=[]
recs=[]
for i in range(len(test_movies)):
    if i%1==0:
        pos=test_movies[i]
        test_movie=movies_with_overviews[pos]
        gtids=test_movie['genre_ids']
        gt=[]
        for g in gtids:
            g_name=Genre_ID_to_name[g]
            gt.append(g_name)
#         print predictions[i],movies_with_overviews[i]['title'],gt
        a,b=precision_recall(gt,predictions[i])
        precs.append(a)
        recs.append(b)

print np.mean(np.asarray(precs)),np.mean(np.asarray(recs))

precs=[]
recs=[]
for i in range(len(test_movies)):
    if i%1==0:
        pos=test_movies[i]
        test_movie=movies_with_overviews[pos]
        gtids=test_movie['genre_ids']
        gt=[]
        for g in gtids:
            g_name=Genre_ID_to_name[g]
            gt.append(g_name)
#         print predictions[i],movies_with_overviews[i]['title'],gt
        a,b=precision_recall(gt,predictionsnb[i])
        precs.append(a)
        recs.append(b)

print np.mean(np.asarray(precs)),np.mean(np.asarray(recs))


I am working on the file "Deep_Learning_Project.ipynb".
The first task is:
As we can see, both the systems are correct, but the way they package information is different. TMDB calls it "Science Fiction" and has an ID for every genre. While IMDB calls it "Sci-Fi". Thus, it is important to keep track of these things when making use of both the datasets simultaneously.
Now that we know how to scrape information for one movie, let's take a bigger step towards scraping multiple movies?
Working with multiple movies : Obtaining Top 20 movies from TMDB
We first instantiate an object that inherits from class Movies from TMDB. Then We use the popular() class method (i.e. function) to get top movies. To get more than one page of results, the optional page argument lets us see movies from any specified page number.
Can you write Python code for it?

all_movies=tmdb.Movies()
top_movies=all_movies.popular()

# This is a dictionary, and to access results we use the key 'results' which returns info on 20 movies
print(len(top_movies['results']))
top20_movs=top_movies['results']


I am working on the file "Deep_Learning_Project.ipynb".
The first task is:
Congratulations, we are done scraping!
Building a dataset out of the scraped information!
This task is simple, but extremely important. It's basically what will set the stage for the whole project. Given that you have the freedom to cast their own project within the framework I am providing, there are many decisions that you must make to finalize your own version of the project.
As we are working on a classification problem, we need to make two decisions given the data at hand -
* What do we want to predict, i.e. what's our Y?
* What features to use for predicting this Y, i.e. what X should we use?
There are many different options possible, and it comes down to you to decide what's most exciting. I will be picking my own version for the example, but it is imperative that you think this through, and come up with a version which excites you!
As an example, here are some possible ways to frame Y, while still sticking to the problem of genre prediction -

Assume every movie can have multiple genres, and then it becomes a multi-label classification problem. For example, a movie can be Action, Horror and Adventure simultaneously. Thus, every movie can be more than one genre.

Make clusters of genres as we did in Milestone 1 using biclustering, and then every movie can have only 1 genre. This way, the problem becomes a simpler, multi-class problem. For example, a movie could have the class - Uplifting (refer Milestone 1), or Horror or History. No movie get's more than one class.

For the purposes of this implementation, I'm going with the first case explained above - i.e. a multi-label classification problem.
Similarly, for designing our input features i.e. X, you may pick any features you think make sense, for example, the Director of a movie may be a good predictor for genre. OR, they may choose any features they design using algorithms like PCA. Given the richness of IMDB, TMDB and alternate sources like Wikipedia, there is a plethora of options available. Be creative here!
Another important thing to note is...
Can you write Python code for it?

movies_with_overviews=[]
for i in range(len(no_duplicate_movies)):
    movie=no_duplicate_movies[i]
    id=movie['id']
    overview=movie['overview']

    if len(overview)==0:
        continue
    else:
        movies_with_overviews.append(movie)

len(movies_with_overviews)


I am working on the file "Deep_Learning_Project.ipynb".
The first task is:
Congratulations, we have our data set ready!
A note : As we are building our own dataset, and I didn't want you to spend all your time waiting for poster image downloads to finish, I am working with an EXTREMELY small dataset. That is why, the results we will see for the deep learning portion will not be spectacular as compared to conventional machine learning methods. If you want to see the real power, you should spend some more time scraping something of the order of 100,000 images, as opposed to 1000 odd like I am doing here. Quoting the paper I mentioned above -  MORE DATA BEATS A CLEVERER ALGORITHM.
As the TA, I saw that most teams working on the project had data of the order of 100,000 movies. So, if you want to extract the power of these models, consider scraping a larger dataset than me.
Section 5 - Non-deep, Conventional ML models with above data
Here is a layout of what we will be doing -

We will implement two different models
We will decide a performance metric i.e. a quantitative method to be sure about how well difference models are doing.
Discussion of the differences between the models, their strengths, weaknesses, etc.

As discussed earlier, there are a LOT of implementation decisions to be made. Between feature engineering, hyper-parameter tuning, model selection and how interpretable do you want your model to be (Read : Bayesian vs Non-Bayesian approaches) a lot is to be decided. For example, some of these models could be:

Generalized Linear Models
SVM
Shallow (1 Layer, i.e. not deep) Neural Network
Random Forest
Boosting
Decision Tree

Or go more bayesian:
- Naive Bayes
- Linear or Quadratic Discriminant Analysis
- Bayesian Hierarchical models
The list is endless, and not all models will make sense for the kind of problem you have framed for yourself.  Think about which model best fits for your purpose.
For our purposes here, I will be showing the example of 2 very simple models, one picked from each category above -

SVM
Multinomial Naive Bayes

A quick overview of the whole...
Can you write Python code for it?

from sklearn.feature_extraction.text import TfidfTransformer

tfidf_transformer = TfidfTransformer()
X_tfidf = tfidf_transformer.fit_transform(X)
X_tfidf.shape


I am working on the file "Deep_Learning_Project.ipynb".
The first task is:
For the first 10 epochs I trained the model in a verbose fashion to show you what's happening. After that, in the below cell you can see I turned off the verbosity to keep the code cleaner.
Can you write Python code for it?


I am working on the file "Deep_Learning_Project.ipynb".
The first task is:
Let's convert this list into a nice dictionary to look up genre names from genre IDs!
Can you write Python code for it?

Genre_ID_to_name={}
for i in range(len(list_of_genres)):
    genre_id=list_of_genres[i]['id']
    genre_name=list_of_genres[i]['name']
    Genre_ID_to_name[genre_id]=genre_name


I am working on the file "Deep_Learning_Project.ipynb".
The first task is:
Let's look at one of these movies. It's the same format as above, as we had information on the movie "The Matrix", as you can see below. It's a dictionary which can be queried  for specific information on that movie
Can you write Python code for it?

first_movie=top20_movs[0]
print "Here is all the information you can get on this movie - "
print first_movie
print "\n\nThe title of the first movie is - ", first_movie['title']


I am working on the file "Deep_Learning_Project.ipynb".
The first task is:
Let's look at some of our predictions?
Can you write Python code for it?

f6=open('Genredict.pckl','rb')
Genre_ID_to_name=pickle.load(f6)
f6.close()

sum(Y_preds[1])

sum(Y_preds[2])

genre_list=sorted(list(Genre_ID_to_name.keys()))

precs=[]
recs=[]
for i in range(len(Y_preds)):
    row=Y_preds[i]
    gt_genres=Y_test[i]
    gt_genre_names=[]
    for j in range(19):
        if gt_genres[j]==1:
            gt_genre_names.append(Genre_ID_to_name[genre_list[j]])
    top_3=np.argsort(row)[-3:]
    predicted_genres=[]
    for genre in top_3:
        predicted_genres.append(Genre_ID_to_name[genre_list[genre]])
    (precision,recall)=precision_recall(gt_genre_names,predicted_genres)
    precs.append(precision)
    recs.append(recall)
    if i%50==0:
        print "Predicted: ",','.join(predicted_genres)," Actual: ",','.join(gt_genre_names)

print np.mean(np.asarray(precs)),np.mean(np.asarray(recs))


I am working on the file "Deep_Learning_Project.ipynb".
The first task is:
Let's print out top 5 movie's titles!
Can you write Python code for it?

for i in range(len(top20_movs)):
    mov=top20_movs[i]
    title=mov['title']
    print title
    if i==4:
        break


I am working on the file "Deep_Learning_Project.ipynb".
The first task is:
Let's remove any duplicates that we have in the list of movies
Can you write Python code for it?

movie_ids = [m['id'] for m in movies]
print "originally we had ",len(movie_ids)," movies"
movie_ids=np.unique(movie_ids)
print len(movie_ids)
seen_before=[]
no_duplicate_movies=[]
for i in range(len(movies)):
    movie=movies[i]
    id=movie['id']
    if id in seen_before:
        continue
#         print "Seen before"
    else:
        seen_before.append(id)
        no_duplicate_movies.append(movie)
print "After removing duplicates we have ",len(no_duplicate_movies), " movies"


I am working on the file "Deep_Learning_Project.ipynb".
The first task is:
Let's try our second model? The naive bayes model.
Can you write Python code for it?

from sklearn.naive_bayes import MultinomialNB
classifnb = OneVsRestClassifier(MultinomialNB())
classifnb.fit(X[msk].toarray(), Y_train)
predsnb=classifnb.predict(X[~msk].toarray())

import pickle
f2=open('classifer_nb','wb')
pickle.dump(classifnb,f2)
f2.close()

predictionsnb=[]
for i in range(X_test_tfidf.shape[0]):
    pred_genres=[]
    movie_label_scores=predsnb[i]
    for j in range(19):
        #print j
        if movie_label_scores[j]!=0:
            genre=Genre_ID_to_name[genre_list[j]]
            pred_genres.append(genre)
    predictionsnb.append(pred_genres)

for i in range(X_test_tfidf.shape[0]):
    if i%50==0 and i!=0:
        print 'MOVIE: ',movies_with_overviews[i]['title'],'\tPREDICTION: ',','.join(predictionsnb[i])


I am working on the file "Deep_Learning_Project.ipynb".
The first task is:
Looking at the above figure, "boxes" or groups of movie genres automatically emerge!
Intuitively - Crime, Sci-Fi, Mystery, Action, Horror, Drama, Thriller, etc co-occur.
AND, Romance, Fantasy, Family, Music, Adventure, etc co-occur.
That makes a lot of intuitive sense, right?
One challenge is the broad range of the drama genre. It makes the two clusters highly overlapping. If we merge it together with action thriller, etc. We will end up with nearly all movies just having that label.
Based on playing around with the stuff above, we can sort the data into the following genre categories - "Drama, Action, ScienceFiction, exciting(thriller, crime, mystery), uplifting(adventure, fantasy, animation, comedy, romance, family), Horror, History"
Note: that this categorization is subjective and by no means the only right solution. One could also just stay with the original labels and only exclude the ones with not enough data. Such tricks are important to balance the dataset, it allows us to increase or decrease the strength of certain signals, making it possible to improve our inferences :)
Interesting Questions
This really should be a place for you to get creative and hopefully come up with better questions than me.
Here are some of my thoughts:
- Which actors are bound to a genre, and which can easily hop genres?
- Is there a trend in genre popularity over the years?
- Can you use sound tracks to identify the genre of a movie?
- Are top romance actors higher paid than top action actors?
- If you look at release date vs popularity score, which movie genres have a longer shelf life?
Ideas to explore specifically for feature correlations:
- Are title length correlated with movie genre?
- Are movie posters darker for horror than for romance end comedy?
- Are some genres specifically released more often at a certain time of year?
- Is the RPG rating correlated with the genre?
Based on this new category set, we will now pull posters from TMDB as our training data!
Can you write Python code for it?

# Done before, reading from pickle file now to maintain consistency of data!
# We now sample 100 movies per genre. Problem is that the sorting is by popular movies, so they will overlap.
# Need to exclude movies that were already sampled.
movies = []
baseyear = 2017

print('Starting pulling movies from TMDB. If you want to debug, uncomment the print command. This will take a while, please wait...')
done_ids=[]
for g_id in nr_ids:
    #print('Pulling movies for genre ID '+g_id)
    baseyear -= 1
    for page in xrange(1,6,1):
        time.sleep(0.5)

        url = 'https://api.themoviedb.org/3/discover/movie?api_key=' + api_key
        url += '&language=en-US&sort_by=popularity.desc&year=' + str(baseyear)
        url += '&with_genres=' + str(g_id) + '&page=' + str(page)

        data = urllib2.urlopen(url).read()

        dataDict = json.loads(data)
        movies.extend(dataDict["results"])
    done_ids.append(str(g_id))
print("Pulled movies for genres - "+','.join(done_ids))

# f6=open("movies_for_posters",'wb')
# pickle.dump(movies,f6)
# f6.close()

f6=open("movies_for_posters",'rb')
movies=pickle.load(f6)
f6.close()


I am working on the file "Deep_Learning_Project.ipynb".
The first task is:
Now, let's re-print the genres of top 20 movies?
Can you write Python code for it?

for i in range(len(top20_movs)):
    mov=top20_movs[i]
    title=mov['title']
    genre_ids=mov['genre_ids']
    genre_names=[]
    for id in genre_ids:
        genre_name=Genre_ID_to_name[id]
        genre_names.append(genre_name)
    print title,genre_names
    if i==4:
        break


I am working on the file "Deep_Learning_Project.ipynb".
The first task is:
Now, we can simply look up for a word in the above loaded model. For example, to get the Word2Vec representation of the word "King" we just do - model2['king']
Can you write Python code for it?

print model2['king'].shape
print model2['dog'].shape