'''
Date: 29 October 2017
@authors: Apurba Sengupta

'''

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import time, re
from sklearn.feature_extraction.text import CountVectorizer
from scipy import sparse, interpolate

start_time1 = time.time()

# **********************************************************************************************
#
#   function definitions
#
# **********************************************************************************************

def loadTrainData(filename):

    '''
    Question 1.1

    For the first part you are required to download the Sentiment140 dataset found
    here (http://cs.stanford.edu/people/alecmgo/trainingandtestdata.zip). Unzip the
    file and read in the CSV with training data (it has 1.6 million entries). The
    file contains a table of Sentiment, UserID, Date, no-query, user id, the actual
    tweet. Read in only the sentiment and the tweet. You are welcome to use other
    values as additional features. Sentiment in the data file is classified as
    either 0 (negative emotion) or 4 (positive emotion). Convert these to -1 and 1
    respectively.

    '''

    print "\n Loading the training data ... "

    # read the data into a pandas DataFrsme object and convert it into a NumPy matrix
    train_data_matrix = np.array(pd.read_csv(filename, header = None))

    # the last column of the matrix is the tweet, taken only upto the first comma in the tweet
    X_train = np.array([tweet.split(',')[0] for tweet in train_data_matrix[:,-1]])

    # the first column of the matrix is the sentiment (either 0 or 4)
    Y_train = train_data_matrix[:,0]

    # label the sentimnents with value 4 as 1
    Y_train[Y_train==4] = 1

    # label the sentiments with value 0 as -1
    Y_train[Y_train==0] = -1

    end_time1 = time.time() - start_time1

    print "\n Training data Loaded ..."

    print "\n Time taken to load the training data (in seconds) = ", end_time1

    start_time2 = time.time()

    '''
    Question 1.2 : cleaning up the data

    Perform the following operations on each of the tweets.
    1. Convert all letters into lowercase.
    2. Convert all occurences of 'www.' or 'https://' or 'http://' to 'URL'.
    3. Remove additional white spaces.
    4. Remove all punctuation.
    5. Replace @username with AT-USER.
    6. Replace duplicate words - e.g. 'very very' should be replaced by 'very'.
    7. After Steps 1-6 have been performed, check if the tweet has any of the words
    from stopwords.txt (provided on Canvas). If there are any other words that are
    stop words, ignore them. Whatever now remains of the original tweet is the
    feature vector for that tweet. Create a table of feature vectors and sentiment.
    For example for the first tweet in the dataset, which looks like "@switchfoot
    http://twitpic.com/2y1zl - Awww, that's a bummer. You shoulda got David Carr
    of Third Day to do it. ;D" the extracted feature is "aww". The second tweet
    "is upset that he can't update his Facebook by texting it... and might cry as a
    result School today also. Blah!" after doing steps 1-5 yields the features
    ['upset', 'update', 'facebook', 'texting', 'cry', 'result', 'school']

    '''

    print "\n\n Cleaning the training data ... \n\n"

    # convert the tweets into String objects
    X_train = X_train.astype(str)

    print "\n Converting the tweets to lowercase ..."

    # 1. convert all the tweets into lowercase
    X_train = np.core.defchararray.lower(X_train)

    print "\n Converting all occurences of 'www.', 'https://' and 'http://' to 'URL' ..."

    # 2. convert all occurences of 'www.','https://','http://' to 'URL'
    X_train = np.array([re.sub(r'((http://)|(www.)|(https://))(?:[a-zA-Z]|[0-9]|[$-_~@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', 'URL', tweet) for tweet in X_train])

    print "\n Removing additional white spaces at beginning of text and in between text ..."

    # 3. remove additional white spaces at beginning of text and in between text
    X_train = np.core.defchararray.strip(X_train)
    X_train = np.core.defchararray.replace(X_train, '  ', ' ')

    print "\n Removing all punctuation ..."

    # 4. remove all punctuation
    X_train = np.array([re.sub(r'[!"#$%&\'\(\)*+,-./:;<=>?\[\\\]^_`\{|\}~]', '', tweet) for tweet in X_train])

    print "\n Converting all occurances of '@<username>' to 'AT-USER' ..."

    # 5. convert all occurances of @username to 'AT-USER'
    X_train = np.array([re.sub(r'(@)(?:[a-zA-Z]|[0-9])+', 'AT-USER', tweet) for tweet in X_train])

    print "\n Replacing consecutive duplicate words with a single occurance of the word ..."

    # 6. replace consecutive duplicate words with a single occurance
    X_train = np.array([re.sub(r'\b(\w+)(\s\1\b)+', r'\1', tweet) for tweet in X_train])

    print "\n Removing stop words ..."

    # 7. remove stop words
    with open(r"stopwords.txt", 'r') as stopWordsFile:
        wordList = stopWordsFile.read().split('\n')
        regex = re.compile(r'\b(%s)\b' % '|'.join(wordList))
        X_train = np.array([re.sub(regex, ' ', tweet) for tweet in X_train])

    print "\n\n Cleaned the training data ...\n\n"

    '''
    Question 1.3

    Extract unigram features from the bag of words. The bag of words here is the
    set of all words collected after performing steps 1-6. You are free to use any
    library to create unigram features from the bag of words. To give an example
    (from wikipedia) : Consider we have the tweets "John likes to watch movies.
    Mary likes movies too." and the tweet "John also likes to watch football
    games". The bag of words (Steps 1-6) gives us (John,likes,to,watch,movies,Mary,
    too,also,football,games.) To now extract unigram features, in the tweet "John
    likes to watch movies. Mary likes movies too." count the number of times each
    word appears in the bag of words. Thus, the feature vector would look like
    [1, 2, 1, 1, 2, 1, 1, 0, 0, 0]. John occurs once in the tweet, likes occurs
    twice and so on. For more detail see the wikipedia article on bag of words.
    You are free to extract bi-gram/n-gram features. Thus, at the end of this you
    should have a list of features and the sentiment attached to these features.

    '''

    print "\n Creating bags-of-words (BoW) of the features ..."

    # generate bags-of-words (BoW) of the features
    X_train = np.core.defchararray.split(X_train)
    train_bags_of_words = np.array([list(set(tweetWords)) for tweetWords in X_train])

    print "\n BoW of the features created ..."

    print "\n\n Creating table of BoW of features and corresponding sentiments ...\n\n"

    # generate table of BoW of features and corresponding sentiments
    data_table = pd.DataFrame(train_bags_of_words, columns = ['Features'])
    data_table['Sentiments'] = Y_train
    print "\n", data_table

    end_time2 = time.time() - start_time2

    print "\n\n Table of BoW of features and corresponding sentiments created ..."

    print "\n\n Time taken to clean the training data to form table of BoW of features and corresponding sentiments (in seconds) = ", end_time2

    return X_train, Y_train

def loadTestData(filename):

    start_time6 = time.time()

    '''
    Question 1.6

    Read in the test CSV dataset. Perform all steps in Question 1.1 and 1.2. An
    additional step required is that the sentiments in the test set are 0, 2 and 4.
    Convert the tweets with sentiment 2 to 4. Report test accuracy and a plot of
    test error vs number of iterations for classfiers trained in Part 1.4 and Part 1.5.

    '''

    print "\n\n Loading the test data ... "

    # read the data into a pandas DataFrsme object and convert it into a NumPy matrix
    test_data_matrix = np.array(pd.read_csv(filename, header = None))

    # the last column of the matrix is the tweet, taken only upto the first comma in the tweet
    X_test = np.array([tweet.split(',')[0] for tweet in test_data_matrix[:,-1]])

    # the first column of the matrix is the sentiment (either 0 or 4)
    Y_test = test_data_matrix[:,0]

    # label the sentimnents with value 2 as 4
    Y_test[Y_test==2] = 4

    # label the sentimnents with value 4 as 1
    Y_test[Y_test==4] = 1

    # label the sentiments with value 0 as -1
    Y_test[Y_test==0] = -1

    end_time6 = time.time() - start_time6

    print "\n Test data Loaded ..."

    print "\n\n Time taken to load the test data (in seconds) = ", end_time6

    start_time7 = time.time()

    print "\n\n Cleaning the test data ... \n\n"

    # convert the tweets into String objects
    X_test = X_test.astype(str)

    print "\n\n Converting the tweets to lowercase ..."

    # 1. convert all the tweets into lowercase
    X_test = np.core.defchararray.lower(X_test)

    print "\n Converting all occurences of 'www.', 'https://' and 'http://' to 'URL' ..."

    # 2. convert all occurences of 'www.','https://','http://' to 'URL'
    X_test = np.array([re.sub(r'((http://)|(www.)|(https://))(?:[a-zA-Z]|[0-9]|[$-_~@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', 'URL', tweet) for tweet in X_test])

    print "\n Removing additional white spaces at beginning of text and in between text ..."

    # 3. remove additional white spaces at beginning of text and in between text
    X_test = np.core.defchararray.strip(X_test)
    X_test = np.core.defchararray.replace(X_test, '  ', ' ')

    print "\n Removing all punctuation ..."

    # 4. remove all punctuation
    X_test = np.array([re.sub(r'[!"#$%&\'\(\)*+,-./:;<=>?\[\\\]^_`\{|\}~]', '', tweet) for tweet in X_test])

    print "\n Converting all occurances of '@<username>' to 'AT-USER' ..."

    # 5. convert all occurances of @username to 'AT-USER'
    X_test = np.array([re.sub(r'(@)(?:[a-zA-Z]|[0-9])+', 'AT-USER', tweet) for tweet in X_test])

    print "\n Replacing consecutive duplicate words with a single occurance of the word ..."

    # 6. replace consecutive duplicate words with a single occurance
    X_test = np.array([re.sub(r'\b(\w+)(\s\1\b)+', r'\1', tweet) for tweet in X_test])

    print "\n Removing stop words ..."

    # 7. remove stop words
    with open(r"stopwords.txt", 'r') as stopWordsFile:
        wordList = stopWordsFile.read().split('\n')
        regex = re.compile(r'\b(%s)\b' % '|'.join(wordList))
        X_test = np.array([re.sub(regex, ' ', tweet) for tweet in X_test])

    print "\n\n Cleaned the test data ...\n\n"

    # get words of each tweet
    X_test = np.core.defchararray.split(X_test)

    end_time7 = time.time() - start_time7

    print "\n\n Time taken to clean the test data (in seconds) = ", end_time7

    return X_test, Y_test

def createUnigramFeatureMatrixFromTweets(X_train, X_test):

    start_time3 = time.time()

    '''
    Question 1.3

    Extract unigram features from the bag of words. The bag of words here is the
    set of all words collected after performing steps 1-6. You are free to use any
    library to create unigram features from the bag of words. To give an example
    (from wikipedia) : Consider we have the tweets "John likes to watch movies.
    Mary likes movies too." and the tweet "John also likes to watch football
    games". The bag of words (Steps 1-6) gives us (John,likes,to,watch,movies,Mary,
    too,also,football,games.) To now extract unigram features, in the tweet "John
    likes to watch movies. Mary likes movies too." count the number of times each
    word appears in the bag of words. Thus, the feature vector would look like
    [1, 2, 1, 1, 2, 1, 1, 0, 0, 0]. John occurs once in the tweet, likes occurs
    twice and so on. For more detail see the wikipedia article on bag of words.
    You are free to extract bi-gram/n-gram features. Thus, at the end of this you
    should have a list of features and the sentiment attached to these features.

    Question 1.5

    Use AdaGrad to train a classifier on the features extracted above. Report a plot
    of training error vs. number of iterations for every 1000 iterations. Merge this
    plot with the one from the previous question. No libraries are allowed here.

    '''

    # encode the string entries as 'latin-1' and get back the preprocessed tweets
    train_tweets_latin = np.array([[word.decode('latin-1') for word in tweet] for tweet in X_train])
    preprocessed_tweet_corpus_train = [' '.join(tweet) for tweet in train_tweets_latin]

    # encode the string entries as 'latin-1' and get back the preprocessed tweets
    test_tweets_latin = np.array([[word.decode('latin-1') for word in tweet] for tweet in X_test])
    preprocessed_tweet_corpus_test = [' '.join(tweet) for tweet in test_tweets_latin]

    # corpus of preprocessed tweets
    preprocessed_tweet_corpus = preprocessed_tweet_corpus_train + preprocessed_tweet_corpus_test

    print "\n\n Creating unigram features and converting the data into a matrix of feature occurances ..."

    # convert the preprocessed tweets to a matrix of unigram feature counts
    vectorizer = CountVectorizer(ngram_range = (1,1))

    # training set matrix
    X_train_sparse = vectorizer.fit_transform(preprocessed_tweet_corpus)[:X_train.shape[0],:]

    # test set amtrix
    X_test_sparse = vectorizer.fit_transform(preprocessed_tweet_corpus)[X_train.shape[0]:,:]

    print "\n\n Converted the data into training and test matrices of feature occurances ..."

    end_time3 = time.time() - start_time3

    print "\n\n Time taken to construct the training and test set matrices (in seconds) = ", end_time3

    return X_train_sparse, X_test_sparse

def runPegasosAndAdaGrad(X_train_sparse, Y_train):

    start_time4 = time.time()

    '''
    Question 1.4

    Use PEGASOS to train an SVM on the features extracted above. Make a plot of
    training error v/s number of iterations.

    '''

    # **************************************************************************************************
    #
    #                       PEGASOS - Primal Estimated sub-GrAdient SOlver for SVM
    #
    #   Input : Training Set S = {(x1, y1), (x2, y2), ....., (xn, yn)}, Regularization parameter lambda,
    #           Number of iterations = T
    #
    #   Initialize : w such that norm(w) <= 1/sqrt(lambda)
    #
    #   For t = 1, 2, ..., T:
    #
    #       1. Choose A (a randomly chosen subset of S) with |A| = B
    #
    #       2. A+ = {(x,y) (present in A) such that y<w,x> < 1}
    #
    #       3. learning rate = eta = 1/(count * lambda)
    #
    #       4. gradient = lambda * w - (eta / B) * <y,x> for (x,y) in A+
    #
    #       5. new w = w - eta * gradient
    #
    #       6. projection of new w = new w * min(1, 1/(sqrt(lambda) * norm(new w))
    #
    # **************************************************************************************************

    print "\n\n\n\n Running PEGASOS on the training data ...\n"

    # set number of iterations
    T_P = 5000

    # set lambda (regularizer)
    regularizer_P = 0.00001

    # set number of random rows to be selected at every iteration
    B_P = 1000

    # initialize count
    count_P = 1

    # initialize decision boundary
    w_P = np.zeros(X_train_sparse.shape[1])

    # initial prediction
    Y_pred_P = X_train_sparse.dot(w_P)

    # initial training error
    training_error_P = float(np.logical_xor((Y_pred_P > 0).astype(int), (Y_train.astype(int) > 0).astype(int)).sum())/float(X_train_sparse.shape[0])

    # lsit to hold training error2
    training_error_list_P = [training_error_P]

    # list to hold number of iterations
    count_list_P = [count_P]

    # list to hold the decision values for each iteration
    w_P_list = [w_P]

    for i in range(T_P):

        # randomly generate B_P row numbers
        j = np.random.randint(0, X_train_sparse.shape[0] , B_P)

        # generate the corresponding random subsets of X_train_sparse and Y_train
        x_P = X_train_sparse[j]
        y_P = Y_train[j].astype(int)

        # compute the learning rate
        eta_P = 1/float(count_P * regularizer_P)

        # see the sign of y<w,x> and generate a binary decision vector after checking if y<w,x> < 1
        decision_vector_P = np.multiply(x_P.dot(w_P), y_P)
        decision_vector_P = (decision_vector_P < 1).astype(int)

        # convert decision vector to a sparse CSR vector
        decision_vector_sparse_P = sparse.csr_matrix(decision_vector_P).T

        # filter training examples for which y<w,x> >= 1
        x_new_P = x_P.multiply(decision_vector_sparse_P)
        y_new_P = np.multiply(y_P, decision_vector_P)

        # compute the gradient
        grad_P = np.subtract(np.multiply(w_P, regularizer_P), np.multiply(x_new_P.T.dot(y_new_P), float(eta_P)/float(B_P)))

        # compute new decision boundary
        w_new_P = np.subtract(w_P, np.multiply(grad_P, eta_P))

        # take projection of decision boundary so that it does not exit the set 1/srqt(regularizer_P)
        w_new_proj_P = np.multiply(w_new_P, min(1, 1/(float(np.sqrt(regularizer_P)) *  float(np.linalg.norm(w_new_P)))))

        # increment the counter and put it in count_list
        count_P = count_P + 1
        count_list_P.append(count_P)

        # decision boundary is new decision boundary, put it in w_P_list
        w_P = w_new_proj_P
        w_P_list.append(w_P)

        # calculate training error and put it in training_error_list
        y_pred_P = x_P.dot(w_P)

        training_error_P = float(np.logical_xor((y_pred_P > 0).astype(int), (y_P > 0).astype(int)).sum())/float(B_P)
        training_error_list_P.append(training_error_P)

    print "\n\n PEGASOS run successfully on the training data ...\n"

    end_time4 = time.time() - start_time4

    print "\n\n Time taken to train data using PEGASOS (in seconds) = ", end_time4

    print "\n\n\n Plotting Training Error v/s Number of Iterations for the trained model for PEGASOS ...\n"

    # plot training error v/s number of iterations
    plt.title('\n Training Error v/s Number of Iterations\n')
    plt.xlabel('Number of Iterations')
    plt.ylabel('Training Error')
    plt.plot(count_list_P, training_error_list_P, color = '#e0ffff')
    tck1 = interpolate.splrep(count_list_P, training_error_list_P, k = 3, s = 900)
    training_error_list_P_int = interpolate.splev(count_list_P, tck1, der = 0)
    plt.plot(count_list_P, training_error_list_P_int, color = 'blue', label = 'PEGASOS')
    plt.legend()
    plt.show()

    start_time5 = time.time()

    '''
    Question 1.5

    Use AdaGrad to train a classifier on the features extracted above. Report a plot
    of training error vs. number of iterations for every 1000 iterations. Merge this
    plot with the one from the previous question. No libraries are allowed here.

    '''

    # **************************************************************************************************
    #
    #                                   AdaGrad - Adaptive Gradient
    #
    #   Input : Training Set S = {(x1, y1), (x2, y2), ....., (xn, yn)}, Regularization parameter lambda,
    #           Number of iterations = T
    #
    #   Initialize : w such that norm(w) <= 1/sqrt(lambda), S old = (1,1,1.....,1)
    #
    #   For t = 1, 2, ..., T:
    #
    #       1. Choose A (a randomly chosen subset of S)
    #
    #       2. A+ = {(x,y) (present in A) such that y<w,x> < 1}
    #
    #       3. learning rate = eta = 0.01
    #
    #       4. gradient = - <y,x> for (x,y) in A+
    #
    #       5. S new = S old + gradient^2
    #
    #       6. new w = w - [eta / (B * sqrt(S new))] * gradient
    #
    #       7. projection of new w = new w * min(1, 1/(sqrt(lambda) * norm(S new * new w))
    #
    # **************************************************************************************************

    print "\n\n\n\n Running AdaGrad on the training data ...\n"

    # set number of iterations
    T_A = 5000

    # set lambda (regularizer)
    regularizer_A = 0.005

    # set number of random rows to be selected at every iteration
    B_A = 1000

    # set the learning rate
    eta_A = 0.01

    # initialize count
    count_A = 1

    # initialize decision boundary
    w_A = np.zeros(X_train_sparse.shape[1])

    # initialize vector of diagonal elements of tranformation matrix G
    S_A = np.ones(X_train_sparse.shape[1])

    # initial prediction
    Y_pred_A = X_train_sparse.dot(w_A)

    # initial training error
    training_error_A = float(np.logical_xor((Y_pred_A > 0).astype(int), (Y_train.astype(int) > 0).astype(int)).sum())/float(X_train_sparse.shape[0])

    # lsit to hold training error2
    training_error_list_A = [training_error_A]

    # list to hold number of iterations
    count_list_A = [count_A]

    # list to hold the decision values for each iteration
    w_A_list = [w_A]

    for i in range(T_A):

        # randomly generate B_A row numbers
        j = np.random.randint(0, X_train_sparse.shape[0], B_A)

        # generate the corresponding random subsets of X_train_sparse and Y_train
        x_A = X_train_sparse[j]
        y_A = Y_train[j].astype(int)

        # see the sign of y<w,x> and generate a binary decision vector after checking if y<w,x> < 1
        decision_vector_A = np.multiply(x_A.dot(w_A), y_A)
        decision_vector_A = (decision_vector_A < 1).astype(int)

        # convert decision vector to a sparse CSR vector
        decision_vector_sparse_A = sparse.csr_matrix(decision_vector_A).T

        # filter training examples for which y<w,x> >= 1
        x_new_A = x_A.multiply(decision_vector_sparse_A)
        y_new_A = np.multiply(y_A, decision_vector_A)

        # compute the gradient
        grad_A = np.multiply((x_new_A.T.dot(y_new_A)), -1)

        # compute new vector of diagonal elements of tranformation matrix G
        S_new_A = np.add(S_A, np.square(grad_A))

        # compute new decision boundary
        w_new_A = np.subtract(w_A, np.multiply(grad_A, np.multiply(float(eta_A)/float(B_A), np.divide(1, np.sqrt(S_new_A)))))

        # take projection of decision boundary so that it does not exit the set 1/srqt(regularizer_A)
        w_new_proj_A = np.multiply(w_new_A, min(1, 1/(float(np.sqrt(regularizer_A)) *  float(np.linalg.norm(np.multiply(S_new_A, w_new_A))))))

        # increment the counter and put it in count_list
        count_A = count_A + 1
        count_list_A.append(count_A)

        # decision boundary is new decision boundary, put it in w_A_list
        w_A = w_new_proj_A
        w_A_list.append(w_A)

        # vector of diagonal elements of tranformation matrix G is the new vector of diagonal elements of G
        S_A = S_new_A

        # calculate training error and put it in training_error_list
        y_pred_A = x_A.dot(w_A)

        training_error_A = float(np.logical_xor((y_pred_A > 0).astype(int), (y_A > 0).astype(int)).sum())/float(B_A)
        training_error_list_A.append(training_error_A)

    print "\n\n AdaGrad ran successfully on the training data ...\n"

    end_time5 = time.time() - start_time5

    print "\n\n Time taken to train data using AdaGrad (in seconds) = ", end_time5

    print "\n\n\n Plotting Training Error v/s Number of Iterations for the trained model for AdaGrad ...\n"

    # plot training error v/s number of iterations
    plt.title('\n Training Error v/s Number of Iterations\n')
    plt.xlabel('Number of Iterations')
    plt.ylabel('Training Error')
    plt.plot(count_list_A, training_error_list_A, color = '#ffe4e1')
    tck1 = interpolate.splrep(count_list_A, training_error_list_A, k = 3, s = 900)
    training_error_list_A_int = interpolate.splev(count_list_A, tck1, der = 0)
    plt.plot(count_list_A, training_error_list_A_int, color = 'red', label = 'AdaGrad')
    plt.legend()
    plt.show()

    print "\n\n\n Plotting Training Error v/s Number of Iterations for the trained model for PEGASOS and AdaGrad...\n"

    # plot training error v/s number of iterations
    plt.title('\n Training Error v/s Number of Iterations\n')
    plt.xlabel('Number of Iterations')
    plt.ylabel('Training Error')
    plt.plot(count_list_P[::999], training_error_list_P_int[::999], color = 'blue', label = 'PEGASOS')
    plt.plot(count_list_A[::999], training_error_list_A_int[::999], color = 'red', label = 'AdaGrad')
    plt.legend()
    plt.show()

    return w_P_list, w_A_list, count_list_P, count_list_A

def testAccuracies(X_test_sparse, Y_test, w_P_list, w_A_list, count_list_P, count_list_A):

    '''
    Question 1.6

    Read in the test CSV dataset. Perform all steps in Question 1.1 and 1.2. An
    additional step required is that the sentiments in the test set are 0, 2 and 4.
    Convert the tweets with sentiment 2 to 4. Report test accuracy and a plot of
    test error vs number of iterations for classfiers trained in Part 1.4 and Part 1.5.

    '''

    print "\n\n\n\n Making predictions on the test data and calculating the classification accuracies ..."

    test_error_P = [float(np.logical_xor((X_test_sparse.dot(w_P) > 0).astype(int), (Y_test.astype(int) > 0).astype(int)).sum())/float(Y_test.shape[0]) for w_P in w_P_list]

    print "\n\nPercentage Accuracy for PEGASOS: " + str(round(100 - (test_error_P[-1] * 100), 3)) + "%"

    test_error_A = [float(np.logical_xor((X_test_sparse.dot(w_A) > 0).astype(int), (Y_test.astype(int) > 0).astype(int)).sum())/float(Y_test.shape[0]) for w_A in w_A_list]

    print "\n\nPercentage Accuracy for AdaGrad: " + str(round(100 - (test_error_A[-1] * 100), 3)) + "%"

    print "\n\n\n Plotting Test Error v/s Number of Iterations for the trained model for PEGASOS and AdaGrad...\n"

    # plot test error v/s number of iterations
    plt.title('\n Test Error v/s Number of Iterations\n')
    plt.xlabel('Number of Iterations')
    plt.ylabel('Test Error')
    plt.plot(count_list_P, test_error_P, color = '#e0ffff')
    plt.plot(count_list_A, test_error_A, color = '#ffe4e1')
    tck1 = interpolate.splrep(count_list_P, test_error_P, k = 3, s = 900)
    test_error_P_int = interpolate.splev(count_list_P, tck1, der = 0)
    tck2 = interpolate.splrep(count_list_A, test_error_A, k = 3, s = 900)
    test_error_A_int = interpolate.splev(count_list_A, tck2, der = 0)
    plt.plot(count_list_P, test_error_P_int, color = 'blue', label = 'PEGASOS')
    plt.plot(count_list_A, test_error_A_int, color = 'red', label = 'AdaGrad')
    plt.legend()
    plt.show()

    return test_error_P, test_error_A

# **********************************************************************************************
#
#   function calls
#
# **********************************************************************************************

# load training data
X_train, Y_train = loadTrainData('training.1600000.processed.noemoticon.csv')

# load test data
X_test, Y_test = loadTestData('testdata.manual.2009.06.14.csv')

# generate feature matrices of training and test data
X_train_sparse, X_test_sparse = createUnigramFeatureMatrixFromTweets(X_train, X_test)

# get decision
w_P_list, w_A_list, count_list_P, count_list_A = runPegasosAndAdaGrad(X_train_sparse, Y_train)

# print test accuracy results
test_error_P, test_error_A = testAccuracies(X_test_sparse, Y_test, w_P_list, w_A_list, count_list_P, count_list_A)

end_time = time.time() - start_time1

print "\n\n\n\n Total time taken by program to run (in seconds) = ", end_time
print "\n\n"


![Callysto.ca Banner](https://github.com/callysto/curriculum-notebooks/blob/master/callysto-notebook-banner-top.jpg?raw=true)

%%html

<form action="javascript:code_toggle()"><input type="submit" id="toggleButton" value="Show Code"></form>

from ipywidgets import Output, IntSlider, VBox, Layout
from IPython.display import Javascript,clear_output, display, HTML
import ipywidgets as widgets
import random
from plotly.offline import init_notebook_mode, iplot
from ipywidgets import HBox
import plotly.graph_objs as go
import numpy as np
import random

init_notebook_mode(connected=True)

#This function produces a multiple choice form with four options
def multiple_choice(option_1, option_2, option_3, option_4):
    option_list = [option_1, option_2, option_3, option_4]
    answer = option_list[0]
    letters = ["(A) ", "(B) ", "(C) ", "(D) "]

    #Boldface letters at the beginning of each option
    start_bold = "\033[1m"; end_bold = "\033[0;0m"

    #Randomly shuffle the options
    random.shuffle(option_list)

    #Prints the letters (A) to (D) in sequence with randomly chosen options
    for i in range(4):
        option_text = option_list.pop()
        print(start_bold + letters[i] + end_bold + option_text)

        #Stores the correct answer
        if option_text == answer:
            letter_answer = letters[i]

    button1 = widgets.Button(description="(A)"); button2 = widgets.Button(description="(B)")
    button3 = widgets.Button(description="(C)"); button4 = widgets.Button(description="(D)")

    button1.style.button_color = 'Whitesmoke'; button2.style.button_color = 'Whitesmoke'
    button3.style.button_color = 'Whitesmoke'; button4.style.button_color = 'Whitesmoke'

    container = widgets.HBox(children=[button1,button2,button3,button4])
    display(container)
    print(" ", end='\r')

    def on_button1_clicked(b):
        if "(A) " == letter_answer:
            print("Correct! 👏", end='\r')
            button1.style.button_color = '#abffa8'; button2.style.button_color = 'Whitesmoke'
            button3.style.button_color = 'Whitesmoke'; button4.style.button_color = 'Whitesmoke'
        else:
            print("Try again! ", end='\r')
            button1.style.button_color = '#ffbbb8'; button2.style.button_color = 'Whitesmoke'
            button3.style.button_color = 'Whitesmoke'; button4.style.button_color = 'Whitesmoke'

    def on_button2_clicked(b):
        if "(B) " == letter_answer:
            print("Correct! 👏", end='\r')
            button1.style.button_color = 'Whitesmoke'; button2.style.button_color = '#abffa8'
            button3.style.button_color = 'Whitesmoke'; button4.style.button_color = 'Whitesmoke'
        else:
            print("Try again! ", end='\r')
            button1.style.button_color = 'Whitesmoke'; button2.style.button_color = '#ffbbb8'
            button3.style.button_color = 'Whitesmoke'; button4.style.button_color = 'Whitesmoke'

    def on_button3_clicked(b):
        if "(C) " == letter_answer:
            print("Correct! 👏", end='\r')
            button1.style.button_color = 'Whitesmoke'; button2.style.button_color = 'Whitesmoke'
            button3.style.button_color = '#abffa8'; button4.style.button_color = 'Whitesmoke'
        else:
            print("Try again! ", end='\r')
            button1.style.button_color = 'Whitesmoke'; button2.style.button_color = 'Whitesmoke'
            button3.style.button_color = '#ffbbb8'; button4.style.button_color = 'Whitesmoke'

    def on_button4_clicked(b):
        if "(D) " == letter_answer:
            print("Correct! 👏", end='\r')
            button1.style.button_color = 'Whitesmoke'; button2.style.button_color = 'Whitesmoke'
            button3.style.button_color = 'Whitesmoke'; button4.style.button_color = '#abffa8'
        else:
            print("Try again! ", end='\r')
            button1.style.button_color = 'Whitesmoke'; button2.style.button_color = 'Whitesmoke'
            button3.style.button_color = 'Whitesmoke'; button4.style.button_color = '#ffbbb8'

    button1.on_click(on_button1_clicked); button2.on_click(on_button2_clicked)
    button3.on_click(on_button3_clicked); button4.on_click(on_button4_clicked)

def run_all(ev):
    display(Javascript('IPython.notebook.execute_cell()'))

# Thermal Linear Expansion

## Introduction

The goal of this notebook is to discuss the following:
- Review **Kinetic Molecular Theory** and how it relates to thermal expansion.
- Derive a **formula** to compute thermal expansion.
- **Apply** this formula to real-world scenarios.

## Kinetic Molecular Theory

**Summary:**

- Matter is made up of particles that are constantly moving.
- The temperature of a substance is a measure of the average kinetic energy of the particles.
- There are spaces between particles of matter. The average amount of empty space between molecules gets progressively larger as a sample of matter moves from solid to the liquid and gas phases.

start_bold = "\033[1m"; end_bold = "\033[0;0m"
question = start_bold + "Key Question: " + end_bold + \
"Most solids " + start_bold + "expand" + end_bold + " when heated. Based on the points above, " + start_bold + "why is this the case?" + end_bold
print(question)
print("")
option_1 = "When heat is added to a substance, the molecules vibrate faster causing the space between molecules to increase."
option_2 = "An increase in temperature causes the molecules to loosen up and move freely but slowly around."
option_3 = "As objects become hotter, they contract and the molecules increase their speed."
option_4 = "An increase in temperature causes the molecules to tighten up and move in a constricted manner."

multiple_choice(option_1, option_2, option_3, option_4)

## Thermal Linear Expansion

**Definition:** The increase in length of a solid in one direction (due to an increase in temperature) is called *thermal linear expansion*.

**Goal:** Given any solid, we wish to **predict** how much the solid will expand with a change in temperature; i.e. how do we predict the *change in length* ($\Delta L$) of a solid due to an increase in temperature?

start_bold = "\033[1m"; end_bold = "\033[0;0m"
question = start_bold + "Key Question: " + end_bold + "What " + start_bold + "variable(s)" + end_bold + " would you need know to predict how much a substance will expand when heat is added?"
print(question)
print("")
option_1 = "Type of material, Initial length/size, Amount of heat added"
option_2 = "Type of material, Amount of heat added"
option_3 = "Pressure acting material, Amount of heat added"
option_4 = "Roughness of the material, Type of material"

multiple_choice(option_1, option_2, option_3, option_4)

**Fact:** The change of length $(\Delta L)$ of a solid depends on **three** factors:
1. *Initial length* of the solid $(L)$
2. *Temperature change* $(\Delta T)$
3. Type of material, distinguished by a constant value named the *coefficient of linear expansion* $(\alpha)$

Some values for the coefficient of linear expansion are shown in the table below:

 Materials                 | Coefficient of linear expansion, $\alpha$ $\:$ ($1/°\rm{C}$)
 ---                       | ---
 Aluminum                  | 25 $\times$ 10$^{-6}$
 Concrete, iron, steel     | 12 $\times$ 10$^{-6}$
 Copper                    | 16 $\times$ 10$^{-6}$
 Glass (soft), Platinum    | 9 $\times$ 10$^{-6}$
 Glass (Pyrex)             | 3 $\times$ 10$^{-6}$

The following formula relates these **four quantities variables** together:

$$\Delta L = \alpha L \Delta T$$

We will derive this formula later on in the notebook. Before that, let's get accustomed to using this formula in practical situations.

### Example

The longest continuous bridge in Saskatchewan is a 380 m long steel bridge in North Battleford. The temperature in the area varies from -40.0 $^\circ$C to 30.0 $^\circ$C. What is the *change in length of the bridge*? Round to two significant figures.

out1 = Output()
Step1B = widgets.Button(description="STEP 1", layout=Layout(width='20%', height='100%'))
count1 = 1

info1 = widgets.HTMLMath(value="Identify the variables that we know and what we're looking for:")
math_text1 = widgets.HTMLMath(value="Initial length: $L=380\:m$")
math_text2 = widgets.HTMLMath(value="Change in temperature: $\Delta T = T_{final} - T_{initial}$ = 30°C $-$ ($-$40°C) = 70°C")
math_text3 = widgets.HTMLMath(value="Coefficient of linear expansion (specific to steel): " + chr(0x3B1) + " = $12 × 10^{-6}$ °C$^{-1}$ (reference from table)")
math_text4 = widgets.HTMLMath(value="Change in length of the bridge:  $\Delta L$ = ?")

def on_Step1B_clicked(b):
    global count1
    count1 += 1
    with out1:
        clear_output()
        if count1 % 2 == 0:
            display(info1, math_text1, math_text2, math_text3, math_text4)

display(VBox([Step1B, out1]))
Step1B.on_click(on_Step1B_clicked)

out2 = Output()
Step2B = widgets.Button(description="STEP 2", layout=Layout(width='20%', height='100%'))
count2 = 1

info2 = widgets.HTMLMath(value="Substitute each known value into the formula and solve for the missing variable:")
math_text5 = widgets.HTMLMath(value="$\Delta L$ = " + chr(0x3B1) + " L $\Delta T$ = ($12 × 10^{-6}$ °C$^{-1}$)(380 m)(70 °C) = 0.3192 m")

def on_Step2B_clicked(b):
    global count2
    count2 += 1
    with out2:
        clear_output()
        if count2 % 2 == 0:
            display(info2, math_text5)

display(VBox([Step2B, out2]))
Step2B.on_click(on_Step2B_clicked)

out3 = Output()
Step3B = widgets.Button(description="STEP 3", layout=Layout(width='20%', height='100%'))
count3 = 1

info3 = widgets.HTMLMath(value="Round to the correct number of significant figures and convert to the correct units (if needed):")
math_text6 = widgets.HTMLMath(value="0.3192 m = 0.32 m or 32 cm")

def on_Step3B_clicked(b):
    global count3
    count3 += 1
    with out3:
        clear_output()
        if count3 % 2 == 0:
            display(info3, math_text6)

display(VBox([Step3B, out3]))
Step3B.on_click(on_Step3B_clicked)

### Practice
Try multiple practice problems by clicking the 'Generate New Question' button upon completing a problem.

button = widgets.Button(description="Generate New Question", layout=Layout(width='20%', height='100%'))
button.on_click(run_all)
display(button)

#Variables to randomize
initial_length = round(random.uniform(2.0, 6.0), 2)
initial_temp = round(random.uniform(-50.0, -20.0), 1)
final_temp = round(random.uniform(30.0, 45.0), 1)

#Dictionary of different materials
materials = {"aluminum": 0.000025, "iron": 0.000012, "steel": 0.000012, "glass (soft)": 0.000009, "glass (pyrex)": 0.000003, "concrete": 0.000012, "platinum": 0.000009, "copper": 0.000016}
chosen_material = random.choice(list(materials.keys()))

#Print question
question = "A piece of {} is {} m on a cold winter day ({} °C). How much longer is it on a very hot summer day ({} °C)? Round to two decimal places.".format(chosen_material, initial_length, initial_temp, final_temp)
print(question)

#Answer calculation
answer = round((initial_length * materials[chosen_material] * (final_temp - initial_temp))*100, 2)

#Define range of values for random multiple choices
mini = 1
maxa = 100

#Create three choices that are unique (and not equal to the answer)
choice_list = random.sample(range(mini,maxa),3)
while choice_list.count(int(answer*100)) >= 1:
    choice_list = random.sample(range(mini,maxa),3)

#Assign each multiple choice to these four variables
#Option_1 contains the answer
option_1 = '{:.2f}'.format(answer) + " cm"
option_2 = '{:.2f}'.format(choice_list[0]/100) + " cm"
option_3 = '{:.2f}'.format(choice_list[1]/100) + " cm"
option_4 = '{:.2f}'.format(choice_list[2]/100) + " cm"

multiple_choice(option_1, option_2, option_3, option_4)

## Deriving the Formula

Now that you are comfortable using the thermal expansion formula, we now discuss how it is derived experimentally. Of the four quantities in the equation, $\Delta L = \alpha L \Delta T$, it is the coefficient of linear expansion ($\alpha$) that may be still be mysterious in how it is measured. Previously, we have referenced a table of values to determine $\alpha$, but *where do these values come from?*

We demonstrate how to calculate $\alpha$ experimentally below. We will require two different materials so we can compare each expansion (which you can select in the dropdown menus below). We then apply *the exact same amount of heat to both materials* (which you can adjust using the slider). When you are ready, click the "Calculate Linear Expansion" button below.

coefficients = {"Aluminum": 0.000025, "Iron": 0.000012, "Steel": 0.000012, "Glass (soft)": 0.000009, "Glass (Pyrex)": 0.000003, "Concrete": 0.000012, "Platinum": 0.000009, "Copper": 0.000016}

graph_out = Output()

selection1 = widgets.Dropdown(
    options={'Aluminum', 'Steel', 'Copper'},
    value='Aluminum',
    description='Material 1:',
)

selection2 = widgets.Dropdown(
    options={'Iron', 'Platinum', 'Glass (Pyrex)'},
    value='Iron',
    description='Material 2:',
)

temp_slider = IntSlider(continuous_update=False, wait=True, value=500, min=50, max=1000, step=50, description=chr(0x0394) + 'T (°C)')
initial_length_slider = IntSlider(continuous_update=False, wait=True, value=5, min=1, max=10, step=1, description='L (m)')

submit_button = widgets.Button(description="Calculate Linear Expansion", layout=Layout(width='20%', height='88%'),button_style='success')

################ Functions ##################

def initial_graph(material1, material2):
################# Parametric Equations #################
    s3 = np.linspace(0, 2 * np.pi, 30)
    t3 = np.linspace(0, 5, 30)
    tGrid3, sGrid3 = np.meshgrid(s3, t3)

    x3 = np.cos(tGrid3) + 1.5
    y3 = np.sin(tGrid3) + 1.5
    z3 = sGrid3

    s4 = np.linspace(0, 2 * np.pi, 30)
    t4 = np.linspace(0, 5, 30)
    tGrid4, sGrid4 = np.meshgrid(s4, t4)

    x4 = np.cos(tGrid4) + 3.5
    y4 = np.sin(tGrid4) + 3.5
    z4 = sGrid4

    ######################################################

    trace1 = go.Surface(x=x3, y=y3, z=z3, colorscale='Greens', showscale=False, text = material1, hoverinfo='text')
    trace2 = go.Surface(x=x4, y=y4, z=z4, colorscale='Blues', showscale=False, text = material2, hoverinfo='text')
    data = [trace1, trace2]
    layout = go.Layout(title = material1 + " and " + material2 + " (before heat is applied)",
    scene = dict(xaxis = dict(title='', range = [0,5],
                     backgroundcolor="rgb(200, 200, 230)",
                     gridcolor="rgb(255, 255, 255)",
                     showbackground=True,
                     zerolinecolor="rgb(255, 255, 255)",
                     showticklabels=False,),
                yaxis = dict(title='', range = [0,5],
                    backgroundcolor="rgb(230, 200,230)",
                    gridcolor="rgb(255, 255, 255)",
                    showbackground=True,
                    zerolinecolor="rgb(255, 255, 255)",
                    showticklabels=False),
                zaxis = dict(title='', range = [0,4],
                    backgroundcolor="rgb(230, 230,200)",
                    gridcolor="rgb(255, 255, 255)",
                    showbackground=True,
                    zerolinecolor="rgb(255, 255, 255)",
                    showticklabels=False,),),
              )
    config={'showLink': False, 'editable': False}
    fig = go.Figure(data=data, layout=layout)
    iplot(fig, config=config)

def graph(h, h2, material1, material2, initial_length):
    with graph_out:
        clear_output(wait=True)

        ################# Parametric Equations #################
        s = np.linspace(0, 2 * np.pi, 30)
        t = np.linspace(0, h, 30)
        tGrid, sGrid = np.meshgrid(s, t)

        x = np.cos(tGrid) + 1.5
        y = np.sin(tGrid) + 1.5
        z = sGrid + initial_length

        s2 = np.linspace(0, 2 * np.pi, 30)
        t2 = np.linspace(0, h2, 30)
        tGrid2, sGrid2 = np.meshgrid(s2, t2)

        x2 = np.cos(tGrid2) + 3.5
        y2 = np.sin(tGrid2) + 3.5
        z2 = sGrid2 + initial_length

        s3 = np.linspace(0, 2 * np.pi, 30)
        t3 = np.linspace(0, initial_length, 30)
        tGrid3, sGrid3 = np.meshgrid(s3, t3)

        x3 = np.cos(tGrid3) + 1.5
        y3 = np.sin(tGrid3) + 1.5
        z3 = sGrid3

        s4 = np.linspace(0, 2 * np.pi, 30)
        t4 = np.linspace(0, initial_length, 30)
        tGrid4, sGrid4 = np.meshgrid(s4, t4)

        x4 = np.cos(tGrid4) + 3.5
        y4 = np.sin(tGrid4) + 3.5
        z4 = sGrid4
        ######################################################

        trace1 = go.Surface(x=x, y=y, z=z, colorscale='Reds', showscale=False, text = chr(0x0394) + 'L (' + material1 + ') = ' + str(round(h/5 * 100 ,5)) + ' cm', hoverinfo='text')
        trace2 = go.Surface(x=x2, y=y2, z=z2, colorscale='Reds', showscale=False, text = chr(0x0394) + 'L (' + material2 + ') = ' + str(round(h2/5 * 100,5)) + ' cm', hoverinfo='text')
        trace3 = go.Surface(x=x3, y=y3, z=z3, colorscale='Greens', showscale=False, text = material1, hoverinfo='text')
        trace4 = go.Surface(x=x4, y=y4, z=z4, colorscale='Blues', showscale=False, text = material2, hoverinfo='text')

        data = [trace1, trace2, trace3, trace4]
        layout = go.Layout(title = chr(0x0394) + 'L (' + material1 + ') = ' + str(round(h/5 * 100 ,5)) + ' cm;  ' + chr(0x0394) + 'L (' + material2 + ') = ' + str(round(h2/5 * 100,5)) + ' cm',
                    scene = dict(
                    xaxis = dict(title='', range = [0,5],
                         backgroundcolor="rgb(200, 200, 230)",
                         gridcolor="rgb(255, 255, 255)",
                         showbackground=True,
                         zerolinecolor="rgb(255, 255, 255)",
                         showticklabels=False,),
                    yaxis = dict(title='', range = [0,5],
                        backgroundcolor="rgb(230, 200,230)",
                        gridcolor="rgb(255, 255, 255)",
                        showbackground=True,
                        zerolinecolor="rgb(255, 255, 255)",
                        showticklabels=False),
                    zaxis = dict(title='', range = [0,initial_length + max(h,h2)],
                        backgroundcolor="rgb(230, 230,200)",
                        gridcolor="rgb(255, 255, 255)",
                        showbackground=True,
                        zerolinecolor="rgb(255, 255, 255)",
                        showticklabels=False,),),
                  )
        config={'showLink': False, 'editable': False}
        fig = go.Figure(data=data, layout=layout)
        note = widgets.HTMLMath(value=r"$\textbf{Note: } \text{The thermal expansion shown below (coloured red) is magnified (5x) for easier viewing.}$")
        display(note)
        iplot(fig, config=config)

def on_submit_button_clicked(b):
    height = (initial_length_slider.value*temp_slider.value*coefficients[selection1.value]) * 5
    height2 = (initial_length_slider.value*temp_slider.value*coefficients[selection2.value]) *  5
    graph(height, height2, selection1.value, selection2.value, initial_length_slider.value)

display(HBox([VBox([selection1, selection2]), VBox([temp_slider, initial_length_slider])]))
display(submit_button)
display(graph_out)

with graph_out:
    initial_graph(selection1.value, selection2.value)

submit_button.on_click(on_submit_button_clicked)

*Use the interactive simulation above to answer the following two questions.*

start_bold = "\033[1m"; end_bold = "\033[0;0m"
question = "Keep ΔT fixed and only adjust L. Run the simulation many times, each with a different value for L (but the same value for ΔT). What happens to ΔL?"
print(question)
print("")
option_1 = "ΔL increases as L increases and ΔL decreases as L decreases."
option_2 = "ΔL increases as L decreases and ΔL decreases as L increases."
option_3 = "There is no relationship between ΔL and L."
option_4 = "When L decreases or increases, ΔL remains constant."

multiple_choice(option_1, option_2, option_3, option_4)

start_bold = "\033[1m"; end_bold = "\033[0;0m"
question = "Now, keep L fixed and only adjust ΔT. Again, run the simulation many times, each with a different value for ΔT (but the same value for L). What happens to ΔL?"
print(question)
print("")
option_1 = "ΔL increases as ΔT increases and ΔL decreases as ΔT decreases."
option_2 = "ΔL increases as ΔT decreases and ΔL decreases as ΔT increases."
option_3 = "There is no relationship between ΔL and ΔT."
option_4 = "When ΔT decreases or increases, ΔL remains constant."

multiple_choice(option_1, option_2, option_3, option_4)

**Goal:** To establish a thermal expansion formula capable of accurately predicting how much a material will expand linearly when heat is applied.

Observe how $\Delta L$ is different for some materials (aluminum versus platinum), but the same for others (steel versus iron). This is due to similarities and differences between the chemical composition of each material. For example, steel is comprised of around 90-95% iron, which accounts for both materials' identical rate of thermal expansion.

Simply knowing the initial length ($L$) and change in temperature ($\Delta T$) is clearly not enough to predict change in length ($\Delta L$). We can see this above: when both materials share the same values of $L$ and $\Delta T$, their $\Delta L$ is often different. We somehow must assign a *constant* value (which we call $\alpha$) for each material that we can use alongside $L$ and $\Delta T$ to compute $\Delta L$. Since we experimentally measured $\Delta L$ above, we in fact have all the information we need to establish such a constant.

The above simulation and questions indicates that the relationship between $L$, $\Delta T$, and $\Delta L$ is direct rather than inverse ($1/\Delta T$ or $1/L$). Since $\Delta L$ becomes larger as both $L$ and $\Delta T$ become larger, we guess that their relationship is multiplicative. Hence, we begin by writing down the following formula that we want:

$$\Delta L = \alpha L \Delta T$$

The $\alpha$ is the constant that we need, but do not know. However, we measured $\Delta L$ and we know both $L$ and $\Delta T$, so computing $\alpha$ is easy:

$$\alpha  = \dfrac{\Delta L}{L \Delta T}$$

Using the sliders below, input the change in length ($\Delta L$) for both materials 1 and 2 from the above experimental simulation to compute $\alpha$ for each material. **Hint:** Use your *arrow keys* on your keyboard to use the sliders with precision.

out_text = Output()

################ Widgets #################
style = {'description_width': 'initial'}
temp1_slider = IntSlider(continuous_update=False, wait=True, value=500, min=10, max=1000, step=1, description=chr(0x0394) + 'T (°C)')
change1_slider = widgets.FloatSlider(continuous_update=False, wait=True, value=0, min=0, max=10, step=0.001, readout_format='.3f', description=chr(0x0394) + 'L'+ chr(0x2081) + ' (cm)', style=style)
change2_slider = widgets.FloatSlider(continuous_update=False, wait=True, value=0, min=0, max=10, step=0.001, readout_format='.3f', description=chr(0x0394) + 'L'+ chr(0x2082) + ' (cm)', style=style)
coeff_calc = widgets.Button(description="Calculate " + chr(0x03b1) + chr(0x2081) + " and " + chr(0x03b1) + chr(0x2082), layout=Layout(width='20%', height='88%'),button_style='success')
###########################################

display(VBox([temp1_slider, change1_slider, change2_slider, coeff_calc]))

def on_submit_button_clicked(b):
    ans1 = round(change1_slider.value / (300.0 * temp1_slider.value), 8)
    ans2 = round(change2_slider.value / (300.0 * temp1_slider.value), 8)
    with out_text:
        clear_output()
        text_answer = widgets.HTMLMath(value=r"$\alpha_1 = $ " + str(ans1) + " °C$^{-1}$" + "$\qquad$" + r"$\alpha_2 = $ " + str(ans2) + " °C$^{-1}$")
        text_answer2 = widgets.HTMLMath(value=r"Compare each $\alpha_1$ and $\alpha_2$ to those found in the table of coefficients of linear expansion above. See if they match.")
        display(text_answer)
        display(text_answer2)

### Widget Interaction Function Calls ###
coeff_calc.on_click(on_submit_button_clicked)

display(out_text)

## Variations on the Thermal Expansion Formula

As we've just seen, the formula $\Delta L = \alpha L \Delta T$ is not always used to find $\Delta L$. Often, we can experimentally measure $\Delta L$. Depending on the application, we can then theoretically compute one of the other quantities (assuming we know the value of two of the other quantities). It is worth noting the three variations of the thermal expansion formula:

$$\alpha = \dfrac{\Delta L}{L \Delta T} \qquad L = \dfrac{\Delta L}{\alpha \Delta T} \qquad \Delta T = \dfrac{\Delta L}{\alpha L}$$

**Note:** This is **one** formula being displayed in **three** different ways. It is convenient to make use of the variation that corresponds to the unknown value that you wish to determine.

### Practice

Try the three different types of questions using the above variations of the thermal expansion formula. Use the 'Generate New Question' button to complete additional practice problems.

button1 = widgets.Button(description="Generate New Question", layout=Layout(width='20%', height='100%'))
button1.on_click(run_all)
display(button1)

#Variables to randomize
change_length = round(random.uniform(0.2, 0.9), 2)
initial_length = round(random.uniform(1.0, 3.0), 2)
initial_temp = round(random.uniform(10.0, 25.0), 1)
final_temp = round(random.uniform(100.0, 150.0), 1)

#Print question
question = "An newly made synthetic material {} m long expands {} mm when heated from {}°C to {}°C. What is the coefficient of linear expansion of this new material?".format(initial_length, change_length, initial_temp, final_temp)
print(question)

#Answer calculation
answer = round((change_length/1000) / (initial_length * (final_temp - initial_temp)), 8)

#Define range of values for random multiple choices
mini = 100
maxa = 900

#Create three choices that are unique (and not equal to the answer)
choice_list = random.sample(range(mini,maxa),3)
while choice_list.count(int(answer*100000000)) >= 1:
    choice_list = random.sample(range(mini,maxa),3)

#Assign each multiple choice to these four variables
#Option_1 contains the answer
option_1 = str(answer) + " °C" + chr(0x207B) + chr(0x00B9)
option_2 = str(choice_list[0]/100000000) + " °C" + chr(0x207B) + chr(0x00B9)
option_3 = str(choice_list[1]/100000000) + " °C" + chr(0x207B) + chr(0x00B9)
option_4 = str(choice_list[2]/100000000) + " °C" + chr(0x207B) + chr(0x00B9)

multiple_choice(option_1, option_2, option_3, option_4)

button2 = widgets.Button(description="Generate New Question", layout=Layout(width='20%', height='100%'))
button2.on_click(run_all)
display(button2)

#Variables to randomize
change_length = round(random.uniform(0.5, 2.0), 2)
initial_temp = round(random.uniform(20.0, 30.0), 1)
final_temp = round(random.uniform(100.0, 250.0), 1)

#Dictionary of different materials
materials = {"aluminum": 0.000025, "iron": 0.000012, "steel": 0.000012, "glass (soft)": 0.000009, "glass (pyrex)": 0.000003, "concrete": 0.000012, "platinum": 0.000009, "copper": 0.000016}
chosen_material = random.choice(list(materials.keys()))

#Print question
question = "A piece of {} changes in length by {} m. The initial temperature was {}°C and the final temperature was {}°C. Determine the original length of the material. Leave your answer unrounded.".format(chosen_material, change_length, initial_temp, final_temp)
print(question)

#Answer calculation
answer = round(change_length / (materials[chosen_material] * (final_temp - initial_temp)), 2)

#Define range of values for random multiple choices
mini = 500
maxa = 1500

#Create three choices that are unique (and not equal to the answer)
choice_list = random.sample(range(mini,maxa),3)
while choice_list.count(int(answer)) >= 1:
    choice_list = random.sample(range(mini,maxa),3)

#Assign each multiple choice to these four variables
#Option_1 contains the answer
option_1 = '{:.2f}'.format(answer) + " m"
option_2 = '{:.2f}'.format(choice_list[0]) + " m"
option_3 = '{:.2f}'.format(choice_list[1]) + " m"
option_4 = '{:.2f}'.format(choice_list[2]) + " m"

multiple_choice(option_1, option_2, option_3, option_4)

button3 = widgets.Button(description="Generate New Question", layout=Layout(width='20%', height='100%'))
button3.on_click(run_all)
display(button3)

#Variables to randomize
change_length = round(random.uniform(1.0, 4.0), 2)
initial_length = round(random.uniform(100.0, 150.0), 1)

#Dictionary of different materials
materials = {"aluminum": 0.000025, "iron": 0.000012, "steel": 0.000012, "glass (soft)": 0.000009, "glass (pyrex)": 0.000003, "concrete": 0.000012, "platinum": 0.000009, "copper": 0.000016}
chosen_material = random.choice(list(materials.keys()))

#Print question
question = "By how much would you need to heat a {} inch {} sample to make it expand by {} inches? Round to the nearest degree Celsius.".format(initial_length, chosen_material, change_length)
print(question)

#Answer calculation
answer = int(change_length / (materials[chosen_material] * initial_length))

#Define range of values for random multiple choices
mini = 600
maxa = 3000

#Create three choices that are unique (and not equal to the answer)
choice_list = random.sample(range(mini,maxa),3)
while choice_list.count(answer) >= 1:
    choice_list = random.sample(range(mini,maxa),3)

#Assign each multiple choice to these four variables
#Option_1 contains the answer
option_1 = str(answer) + " °C"
option_2 = str(choice_list[0]) + " °C"
option_3 = str(choice_list[1]) + " °C"
option_4 = str(choice_list[2]) + " °C"

multiple_choice(option_1, option_2, option_3, option_4)

## Application: Aircraft Components

Aircraft materials often require *specialized properties* in order to operate in the most desirable manner. The specialized property of the material may be the most important consideration in materials selection. Listed below are several key specialized properties considered in aircraft materials selection:

- Electrical conductivity (important for the outer skin of the aircraft)
- Stealth (materials that can absorb radar waves and/or reduce the infrared visibility are used in the external surface of covert military aircraft)
- Thermal conductivity (used in high-temperature applications, such as heat shields and engine components)
- Thermal expansion (used in high-temperature range applications, such as wing and engine components)

### Problem

An unknown metal alloy is being tested to discover its thermal properties to see if it suitable for use as a *spar*, which is a component of an airplane wing (shown in red below). The alloy is formed into a bar measuring 1.00 metre in length and it is then heated from its starting temperature of 30 °C to a final temperature of 100.0 °C. The length of the heated bar is measured to be exactly 1.002 metres in length. **What is the coefficient of thermal expansion of the alloy? Round your answer to two significant figures.**

option_1 = str(0.000029) + " °C" + chr(0x207B) + chr(0x00B9)
option_2 = str(0.014) + " °C" + chr(0x207B) + chr(0x00B9)
option_3 = str(0.000032) + " °C" + chr(0x207B) + chr(0x00B9)
option_4 = str(0.0140) + " °C" + chr(0x207B) + chr(0x00B9)

multiple_choice(option_1, option_2, option_3, option_4)

The aircraft wing (from above) experiences temperature extremes that span 210 ℃. The spar for the wing will have a length of 18.0 metres. Testing indicates that the aircraft wing will remain stable only if the spar never expands to a length larger than 18.103 metres. **If the component is made from the metal alloy in question, will it meet this requirement?**

**Hint:** Use the sliders below to calculate the linear expansion of the spar.

################ Widgets #################
app_out = Output()
style = {'description_width': 'initial'}
temp_span = IntSlider(continuous_update=False, wait=True, value=0, min=0, max=500, step=1, description=chr(0x0394) + 'T (°C)')
initialL_alloy = widgets.FloatSlider(continuous_update=False, wait=True, value=0, min=0, max=30, step=1, readout_format='.1f', description= 'L (m)', style=style)
linear_constant = widgets.FloatSlider(continuous_update=False, wait=True, value=0, min=0, max=10, step=0.1, readout_format='.1f', description= chr(0x03b1) + " (1" + chr(0x2A09) + "10" + chr(0x207B) + chr(0x2075) + " , " + chr(0x2103) + chr(0x207B) + chr(0x00B9) + ")", style=style)
calc_change = widgets.Button(description="Calculate Linear Expansion", layout=Layout(width='20%', height='88%'),button_style='success')

############## Functions ################
def initial_spar():
    layout = go.Layout(title = "Unknown Alloy (before heat is applied)", scene = dict(
                    xaxis = dict(title='', range = [0,5],
                         backgroundcolor="rgb(200, 200, 230)",
                         gridcolor="rgb(255, 255, 255)",
                         showbackground=True,
                         zerolinecolor="rgb(255, 255, 255)",
                         showticklabels=False,),
                    yaxis = dict(title='', range = [0,5],
                        backgroundcolor="rgb(230, 200,230)",
                        gridcolor="rgb(255, 255, 255)",
                        showbackground=True,
                        zerolinecolor="rgb(255, 255, 255)",
                        showticklabels=False),
                    zaxis = dict(title='', range = [0,18],
                        backgroundcolor="rgb(230, 230,200)",
                        gridcolor="rgb(255, 255, 255)",
                        showbackground=True,
                        zerolinecolor="rgb(255, 255, 255)",
                        showticklabels=False,),),
                  )
    data = [go.Mesh3d(
            x = [2, 2, 4, 4, 2, 2, 4, 4],
            y = [2, 3, 3, 2, 2, 3, 3, 2],
            z = [0, 0, 0, 0, 18, 18, 18, 18],
            i = [7, 0, 0, 0, 4, 4, 6, 6, 4, 0, 3, 2],
            j = [3, 4, 1, 2, 5, 6, 5, 2, 0, 1, 6, 3],
            k = [0, 7, 2, 3, 6, 7, 1, 1, 5, 5, 7, 6],
            name='Unknown Alloy',
            color='#000066',
            text = "Spar (wing component; unknown alloy)", hoverinfo='text',
        )]
    config={'showLink': False, 'editable': False}
    fig = go.Figure(data=data, layout=layout)
    iplot(fig, config=config)

def draw_spar(new, i_height):
    clear_output(wait=True)
    layout = go.Layout(title = chr(0x0394) + 'L (Unknown Alloy) = ' + str(round(new * 100 ,5)) + ' cm', scene = dict(
                    xaxis = dict(title='', range = [0,5],
                         backgroundcolor="rgb(200, 200, 230)",
                         gridcolor="rgb(255, 255, 255)",
                         showbackground=True,
                         zerolinecolor="rgb(255, 255, 255)",
                         showticklabels=False,),
                    yaxis = dict(title='', range = [0,5],
                        backgroundcolor="rgb(230, 200,230)",
                        gridcolor="rgb(255, 255, 255)",
                        showbackground=True,
                        zerolinecolor="rgb(255, 255, 255)",
                        showticklabels=False),
                    zaxis = dict(title='', range = [0,18],
                        backgroundcolor="rgb(230, 230,200)",
                        gridcolor="rgb(255, 255, 255)",
                        showbackground=True,
                        zerolinecolor="rgb(255, 255, 255)",
                        showticklabels=False,),),
                  )
    data = [go.Mesh3d(
            x = [2, 2, 4, 4, 2, 2, 4, 4],
            y = [2, 3, 3, 2, 2, 3, 3, 2],
            z = [0, 0, 0, 0, 18, 18, 18, 18],
            i = [7, 0, 0, 0, 4, 4, 6, 6, 4, 0, 3, 2],
            j = [3, 4, 1, 2, 5, 6, 5, 2, 0, 1, 6, 3],
            k = [0, 7, 2, 3, 6, 7, 1, 1, 5, 5, 7, 6],
            name='Unknown Alloy',
            color='#000066',
            text = "Initial length (L) = " + str(i_height) + ' m', hoverinfo='text',
        ), go.Mesh3d(
            x = [2, 2, 4, 4, 2, 2, 4, 4],
            y = [2, 3, 3, 2, 2, 3, 3, 2],
            z = [18, 18, 18, 18, 18+new, 18+new, 18+new, 18+new],
            i = [7, 0, 0, 0, 4, 4, 6, 6, 4, 0, 3, 2],
            j = [3, 4, 1, 2, 5, 6, 5, 2, 0, 1, 6, 3],
            k = [0, 7, 2, 3, 6, 7, 1, 1, 5, 5, 7, 6],
            name='Thermal Expansion',
            color = "#CC0000",
            text = "Thermal Expansion ("+chr(0x0394)+"L) = " + str(round(new,5)) + ' m', hoverinfo='text',
        )]
    config={'showLink': False, 'editable': False}
    fig = go.Figure(data=data, layout=layout)
    iplot(fig, config=config)

def on_submit_button_clicked(b):
    alloy_change = temp_span.value * initialL_alloy.value * (linear_constant.value * 0.00001)
    with app_out:
        draw_spar(alloy_change, initialL_alloy.value)

######### Display #########
display(VBox([temp_span, initialL_alloy, linear_constant]))
display(calc_change)
display(app_out)

with app_out:
    initial_spar()

calc_change.on_click(on_submit_button_clicked)

option_1 = "The final length of the spar is 18.10962 m; therefore, it fails the requirement."
option_2 = "The final length of the spar is 10.962 cm; therefore, it passes the requirement."
option_3 = "The final length of the spar is 18.103 m; therefore, it passes the requirement."
option_4 = "The final length of the spar is 28.962 m; therefore, it fails the requirement."

multiple_choice(option_1, option_2, option_3, option_4)

## Conclusion

- Thermal Linear Expansion is calculated using the formula: $\Delta L = \alpha L \Delta T$.
- The coefficient of linear expansion ($\alpha$) is experimentally calculated. The experiment involves measuring $\Delta L$ and solving for $\alpha$ using the above formula.
- If the material is known (such as steel, concrete, iron, etc.) you can reference the material's coefficient of linear expansion using a Table of Coefficient of Linear Expansion.
- There are many applications to being able to predict thermal expansion, such as the building of bridges, skyscrapers, airplanes, cars, and piping (to name a few).
- This notebook covered *linear* thermal expansion. If you are interested in applying what you have learned or gaining further knowledge of material science, you're next step is to study *area expansion* and *volume expansion*, which will explore the 2-dimensional and 3-dimensional expansion of materials.

[![Callysto.ca License](https://github.com/callysto/curriculum-notebooks/blob/master/callysto-notebook-banner-bottom.jpg?raw=true)](https://github.com/callysto/curriculum-notebooks/blob/master/LICENSE.md)


# -*- coding: utf-8 -*-
# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.11.3
#   kernelspec:
#     display_name: 'Python 3.6.7 64-bit (''base'': conda)'
#     name: python367jvsc74a57bd050da0f6fa72fb86d21724871d314354b884db45bd357078f1680189ca335f685
# ---

# + [markdown] id="view-in-github" colab_type="text"
#

# + [markdown] id="Yn51eYujm5S1"
# # Text preprocessing
#
# We discuss how to convert a sequence of words or characters into numeric form, which can then be fed into an ML model.
#
#
#

# + id="ysx0t0REm4r0"
import numpy as np
import matplotlib.pyplot as plt
np.random.seed(seed=1)
import math

import torch
from torch import nn
from torch.nn import functional as F
from torch.utils import data

# !mkdir figures # for saving plots

# + id="V6Jbluorndzr"
import collections
import re
import random
import os
import requests
import zipfile
import hashlib

# +
# Required functions for downloading data

def download(name, cache_dir=os.path.join('..', 'data')):
    """Download a file inserted into DATA_HUB, return the local filename."""
    assert name in DATA_HUB, f"{name} does not exist in {DATA_HUB}."
    url, sha1_hash = DATA_HUB[name]
    os.makedirs(cache_dir, exist_ok=True)
    fname = os.path.join(cache_dir, url.split('/')[-1])
    if os.path.exists(fname):
        sha1 = hashlib.sha1()
        with open(fname, 'rb') as f:
            while True:
                data = f.read(1048576)
                if not data:
                    break
                sha1.update(data)
        if sha1.hexdigest() == sha1_hash:
            return fname  # Hit cache
    print(f'Downloading {fname} from {url}...')
    r = requests.get(url, stream=True, verify=True)
    with open(fname, 'wb') as f:
        f.write(r.content)
    return fname

def download_extract(name, folder=None):
    """Download and extract a zip/tar file."""
    fname = download(name)
    base_dir = os.path.dirname(fname)
    data_dir, ext = os.path.splitext(fname)
    if ext == '.zip':
        fp = zipfile.ZipFile(fname, 'r')
    elif ext in ('.tar', '.gz'):
        fp = tarfile.open(fname, 'r')
    else:
        assert False, 'Only zip/tar files can be extracted.'
    fp.extractall(base_dir)
    return os.path.join(base_dir, folder) if folder else data_dir

# + [markdown] id="e9vbpUMwTRY1"
# # Basics
#
# This section is based on sec 8.2 of http://d2l.ai/chapter_recurrent-neural-networks/text-preprocessing.html
#

# + [markdown] id="RMrGxkRNnOx_"
# ## Data
#
# As a simple example, we use the book "The Time Machine" by H G Wells, since it is short (30k words) and public domain.

# + colab={"base_uri": "https://localhost:8080/"} id="D7OJT7o8nDQN" outputId="2dd7e687-6b9a-48d2-ab49-e1bb5b64d41b"
DATA_HUB = dict()
DATA_URL = 'http://d2l-data.s3-accelerate.amazonaws.com/'

DATA_HUB['time_machine'] = (DATA_URL + 'timemachine.txt',
                                '090b5e7e70c295757f55df93cb0a180b9691891a')

def read_time_machine():
    """Load the time machine dataset into a list of text lines."""
    with open(download('time_machine'), 'r') as f:
        lines = f.readlines()
    return [re.sub('[^A-Za-z]+', ' ', line).strip().lower() for line in lines]

lines = read_time_machine()
print(f'number of lines: {len(lines)}')

# + colab={"base_uri": "https://localhost:8080/"} id="uCsuaurvnlK8" outputId="8dcf484e-8f45-4748-cc93-c2a2ada427d2"
for i in range(11):
  print(i, lines[i])

# + colab={"base_uri": "https://localhost:8080/"} id="btVyl4dItGVT" outputId="6b67aec4-4c26-43f7-ea6e-0c7fd1440f55"
nchars = 0
nwords = 0
for i in range(len(lines)):
  nchars += len(lines[i])
  words = lines[i].split()
  nwords += len(words)
print('total num characters ', nchars)
print('total num words ', nwords)

# + [markdown] id="KKBbwDcKnwsA"
# ## Tokenization

# + colab={"base_uri": "https://localhost:8080/"} id="X32lM-XvnxhC" outputId="4783ff38-b282-4b0e-fc59-996f6ec0d6a6"
def tokenize(lines, token='word'):
    """Split text lines into word or character tokens."""
    if token == 'word':
        return [line.split() for line in lines]
    elif token == 'char':
        return [list(line) for line in lines]
    else:
        print('ERROR: unknown token type: ' + token)

tokens = tokenize(lines)
for i in range(11):
    print(tokens[i])

# + [markdown] id="X-Tbg9jjn8XN"
# ## Vocabulary
#
# We map each word to a unique integer id, sorted by decreasing frequency.
# We reserve the special id of 0 for the "unknown word".
# We also allow for a list of reserved tokens, such as “pad" for padding, "bos" to present the beginning for a sequence, and “eos” for the end of a sequence.
#

# + id="8ZOLrVNon9dk"
class Vocab:
    """Vocabulary for text."""
    def __init__(self, tokens=None, min_freq=0, reserved_tokens=None):
        if tokens is None:
            tokens = []
        if reserved_tokens is None:
            reserved_tokens = []
        # Sort according to frequencies
        counter = count_corpus(tokens)
        self.token_freqs = sorted(counter.items(), key=lambda x: x[1],
                                  reverse=True)
        # The index for the unknown token is 0
        self.unk, uniq_tokens = 0, ['<unk>'] + reserved_tokens
        uniq_tokens += [
            token for token, freq in self.token_freqs
            if freq >= min_freq and token not in uniq_tokens]
        self.idx_to_token, self.token_to_idx = [], dict()
        for token in uniq_tokens:
            self.idx_to_token.append(token)
            self.token_to_idx[token] = len(self.idx_to_token) - 1

    def __len__(self):
        return len(self.idx_to_token)

    def __getitem__(self, tokens):
        if not isinstance(tokens, (list, tuple)):
            return self.token_to_idx.get(tokens, self.unk)
        return [self.__getitem__(token) for token in tokens]

    def to_tokens(self, indices):
        if not isinstance(indices, (list, tuple)):
            return self.idx_to_token[indices]
        return [self.idx_to_token[index] for index in indices]

def count_corpus(tokens):
    """Count token frequencies."""
    # Here `tokens` is a 1D list or 2D list
    if len(tokens) == 0 or isinstance(tokens[0], list):
        # Flatten a list of token lists into a list of tokens
        tokens = [token for line in tokens for token in line]
    return collections.Counter(tokens)

# + [markdown] id="CV0rTlaqoSNE"
# Here are the top 10 words (and their codes) in our corpus.

# + colab={"base_uri": "https://localhost:8080/"} id="tYmbCwY6oUFB" outputId="31a05a85-5113-4db8-aacf-f944f2c576f8"
vocab = Vocab(tokens)
print(list(vocab.token_to_idx.items())[:10])

# + [markdown] id="sKXJQdbXoiqT"
# Here is a tokenization of a few sentences.

# + colab={"base_uri": "https://localhost:8080/"} id="jd73-1zzoUWo" outputId="f2e7dbda-4053-4773-d385-686f6c549144"
for i in [0, 10]:
    print('words:', tokens[i])
    print('indices:', vocab[tokens[i]])

# + [markdown] id="-6LsXchMop3u"
# ## Putting it altogether
#
# We tokenize the corpus at the character level, and return the sequence of integers, as well as the corresponding Vocab object.

# + id="1BywQ9iUoq_D"
def load_corpus_time_machine(max_tokens=-1):
    """Return token indices and the vocabulary of the time machine dataset."""
    lines = read_time_machine()
    tokens = tokenize(lines, 'char')
    vocab = Vocab(tokens)
    # Since each text line in the time machine dataset is not necessarily a
    # sentence or a paragraph, flatten all the text lines into a single list
    corpus = [vocab[token] for line in tokens for token in line]
    if max_tokens > 0:
        corpus = corpus[:max_tokens]
    return corpus, vocab

# + colab={"base_uri": "https://localhost:8080/"} id="oQzX4Am8osdh" outputId="53318718-5dba-4574-8f7d-3de538584c0d"
corpus, vocab = load_corpus_time_machine()
len(corpus), len(vocab)

# + colab={"base_uri": "https://localhost:8080/"} id="IgDxt_PRovAb" outputId="7aaffeaa-0b08-4796-bce7-e3ef6fe9cc58"
print(corpus[:20])

# + colab={"base_uri": "https://localhost:8080/"} id="egONc6CRowLa" outputId="030047c5-7199-4000-9e0c-39ba75275fd6"
print(list(vocab.token_to_idx.items())[:10])

# + colab={"base_uri": "https://localhost:8080/"} id="9xKwPjAAozaX" outputId="68c7959c-7bfe-42a1-f324-b9b4a1a90113"
print([vocab.idx_to_token[i] for i in corpus[:20]])

# + [markdown] id="X3fLUodCZebY"
# ## One-hot encodings
#
# We can convert a sequence of N integers into a N*V one-hot matrix, where V is the vocabulary size.

# + colab={"base_uri": "https://localhost:8080/"} id="Qk21iCFhZj89" outputId="b5323706-52f8-459b-b382-70acbf54ba26"
x = torch.tensor(corpus[:3])
print(x)
X = F.one_hot(x, len(vocab))
print(X.shape)
print(X)

# + [markdown] id="8vO99OOSuYhX"
# # Language modeling
#
# When fitting language models, we often need to chop up a long sequence into a set of short sequences, which may be overlapping, as shown below, where we extract subsequences of length $n=5$.
#
#
#
# Below we show how to do this.
#
# This section is based on sec 8.3.4 of
# http://d2l.ai/chapter_recurrent-neural-networks/language-models-and-dataset.html#reading-long-sequence-data
#

# + [markdown] id="Vert2-4qw5K7"
# ## Random ordering

# + [markdown] id="_rARqDyZuvlu"
# To increase variety of the data, we can start the extraction at a random offset. We can thus create a random sequence data iterator, as follows.
#

# + id="meuw3vkjpL22"
def seq_data_iter_random(corpus, batch_size, num_steps):
    """Generate a minibatch of subsequences using random sampling."""
    # Start with a random offset (inclusive of `num_steps - 1`) to partition a
    # sequence
    corpus = corpus[random.randint(0, num_steps - 1):]
    # Subtract 1 since we need to account for labels
    num_subseqs = (len(corpus) - 1) // num_steps
    # The starting indices for subsequences of length `num_steps`
    initial_indices = list(range(0, num_subseqs * num_steps, num_steps))
    # In random sampling, the subsequences from two adjacent random
    # minibatches during iteration are not necessarily adjacent on the
    # original sequence
    random.shuffle(initial_indices)

    def data(pos):
        # Return a sequence of length `num_steps` starting from `pos`
        return corpus[pos:pos + num_steps]

    num_batches = num_subseqs // batch_size
    for i in range(0, batch_size * num_batches, batch_size):
        # Here, `initial_indices` contains randomized starting indices for
        # subsequences
        initial_indices_per_batch = initial_indices[i:i + batch_size]
        X = [data(j) for j in initial_indices_per_batch]
        Y = [data(j + 1) for j in initial_indices_per_batch]
        yield torch.tensor(X), torch.tensor(Y)

# + [markdown] id="71kdus7mvMFQ"
# For example, let us generate a sequence 0,1,..,34, and then extract subsequences of length 5. Each minibatch will have 2 such subsequences, starting at random offsets. There is no ordering between the subsequences, either within or across minibatches. There are $\lfloor (35-1)/5 \rfloor = 6$ such subsequences, so the iterator will generate 3 minibatches, each of size 2.
#
# For language modeling tasks, we define $X$ to be the first $n-1$ tokens, and $Y$ to be the $n$'th token, which is the one to be predicted.

# + colab={"base_uri": "https://localhost:8080/"} id="x8GXyqOgvOI7" outputId="efc1667a-624e-461c-a72c-c9d8ac244f81"
my_seq = list(range(35))
b = 0
for X, Y in seq_data_iter_random(my_seq, batch_size=2, num_steps=5):
    print('batch: ', b)
    print('X: ', X, '\nY:', Y)
    b += 1

# + [markdown] id="wdg490Gow7la"
# ## Sequential ordering

# + [markdown] id="55ECVkQLwL8K"
# We can also require that the $i$'th subsequence in minibatch $b$ follows the $i$'th subsequence in minibatch $b-1$. This is useful when training RNNs, since when the model encounters batch $b$, the hidden state of the model will already be initialized by the last token in sequence $i$ of batch $b-1$.

# + id="r3uVV7lYwCdv"
def seq_data_iter_sequential(corpus, batch_size, num_steps):
    """Generate a minibatch of subsequences using sequential partitioning."""
    # Start with a random offset to partition a sequence
    offset = random.randint(0, num_steps)
    num_tokens = ((len(corpus) - offset - 1) // batch_size) * batch_size
    Xs = torch.tensor(corpus[offset:offset + num_tokens])
    Ys = torch.tensor(corpus[offset + 1:offset + 1 + num_tokens])
    Xs, Ys = Xs.reshape(batch_size, -1), Ys.reshape(batch_size, -1)
    num_batches = Xs.shape[1] // num_steps
    for i in range(0, num_steps * num_batches, num_steps):
        X = Xs[:, i:i + num_steps]
        Y = Ys[:, i:i + num_steps]
        yield X, Y

# + [markdown] id="KGRIkFvXwZX6"
# Below we give an example. We see that the first subsequence in batch 1
# is [0,1,2,3,4], and the first subsequence in batch 2 is [5,6,7,8,9], as desired.

# + colab={"base_uri": "https://localhost:8080/"} id="aLQzm2qrwY0m" outputId="27a0f0e7-46db-469c-dd1f-906b599be01d"
for X, Y in seq_data_iter_sequential(my_seq, batch_size=2, num_steps=5):
    print('X: ', X, '\nY:', Y)

# + [markdown] id="SP96EBA-w9MF"
# ## Data iterator
# -

def load_corpus_time_machine(max_tokens=-1):
    """Return token indices and the vocabulary of the time machine dataset."""
    lines = read_time_machine()
    tokens = tokenize(lines, 'char')
    vocab = Vocab(tokens)
    # Since each text line in the time machine dataset is not necessarily a
    # sentence or a paragraph, flatten all the text lines into a single list
    corpus = [vocab[token] for line in tokens for token in line]
    if max_tokens > 0:
        corpus = corpus[:max_tokens]
    return corpus, vocab

# + id="IpjIv8tMw-QD"
class SeqDataLoader:  #@save
    """An iterator to load sequence data."""
    def __init__(self, batch_size, num_steps, use_random_iter, max_tokens):
        if use_random_iter:
            self.data_iter_fn = seq_data_iter_random
        else:
            self.data_iter_fn = seq_data_iter_sequential
        self.corpus, self.vocab = load_corpus_time_machine(max_tokens)
        self.batch_size, self.num_steps = batch_size, num_steps

    def __iter__(self):
        return self.data_iter_fn(self.corpus, self.batch_size, self.num_steps)

# + id="pIy_YUk9w-0A"
def load_data_time_machine(batch_size, num_steps,  #@save
                           use_random_iter=False, max_tokens=10000):
    """Return the iterator and the vocabulary of the time machine dataset."""
    data_iter = SeqDataLoader(batch_size, num_steps, use_random_iter,
                              max_tokens)
    return data_iter, data_iter.vocab

# + id="Y44o8-MkxA3t"
data_iter, vocab = load_data_time_machine(2, 5)

# + colab={"base_uri": "https://localhost:8080/"} id="sf-py0roxmAC" outputId="66691d42-4905-4fe3-b906-8856e525665b"
print(list(vocab.token_to_idx.items())[:10])

# + colab={"base_uri": "https://localhost:8080/"} id="6XhwWfMHxXTA" outputId="a29066f2-d2e2-447c-d290-ea473ecc0ee1"
b = 0
for X, Y in data_iter:
    print('batch: ', b)
    print('X: ', X, '\nY:', Y)
    b += 1
    if b > 2:
      break

# + [markdown] id="yDmK1xQ9T4IY"
# # Machine translation
#
# When dealing with sequence-to-sequence tasks, such as NMT, we need to create a vocabulary for the source and target language. In addition, the input and output sequences may have different lengths, so we need to use padding to ensure that we can create fixed-size minibatches. We show how to do this below.
#
# This is based on sec 9.5 of
# http://d2l.ai/chapter_recurrent-modern/machine-translation-and-dataset.html
#
#
#

# + [markdown] id="gBUgcAcmUdCJ"
# ## Data
#
# We use an English-French dataset that consists of bilingual sentence pairs from the [Tatoeba Project](http://www.manythings.org/anki/). Each line in the dataset is a tab-delimited pair of an English text sequence (source) and the translated French text sequence (target).
#

# + colab={"base_uri": "https://localhost:8080/"} id="UnjXAtdYUUW8" outputId="66c19d66-217b-43ef-9877-854ea32725d0"
DATA_HUB['fra-eng'] = (DATA_URL + 'fra-eng.zip',
                           '94646ad1522d915e7b0f9296181140edcf86a4f5')

def read_data_nmt():
    """Load the English-French dataset."""
    data_dir = download_extract('fra-eng')
    with open(os.path.join(data_dir, 'fra.txt'), 'r') as f:
        return f.read()

raw_text = read_data_nmt()
print(raw_text[:100])

# + [markdown] id="sqZImjzDVMHa"
# ## Preprocessing
#
# We apply several preprocessing steps: we replace non-breaking space with space, convert uppercase letters to lowercase ones, and insert space between words and punctuation marks.
#

# + colab={"base_uri": "https://localhost:8080/"} id="r5ZUH4ZaUquY" outputId="316a660f-1d3a-45e4-ddb2-f97ed3128733"
def preprocess_nmt(text):
    """Preprocess the English-French dataset."""
    def no_space(char, prev_char):
        return char in set(',.!?') and prev_char != ' '

    # Replace non-breaking space with space, and convert uppercase letters to
    # lowercase ones
    text = text.replace('\u202f', ' ').replace('\xa0', ' ').lower()
    # Insert space between words and punctuation marks
    out = [
        ' ' + char if i > 0 and no_space(char, text[i - 1]) else char
        for i, char in enumerate(text)]
    return ''.join(out)

text = preprocess_nmt(raw_text)
print(text[:110])

# + [markdown] id="yGChAGPjVgUn"
# We tokenize at the word level.  The following tokenize_nmt function tokenizes the the first `num_examples` text sequence pairs, where each token is either a word or a punctuation mark.

# + id="PZ-iR79zVKM_"
def tokenize_nmt(text, num_examples=None):
    """Tokenize the English-French dataset."""
    source, target = [], []
    for i, line in enumerate(text.split('\n')):
        if num_examples and i > num_examples:
            break
        parts = line.split('\t')
        if len(parts) == 2:
            source.append(parts[0].split(' '))
            target.append(parts[1].split(' '))
    return source, target

# + colab={"base_uri": "https://localhost:8080/"} id="v282HIgRVfza" outputId="12cafea1-eee6-43e9-ba8d-79c6ef5affc2"
source, target = tokenize_nmt(text)
source[:10], target[:10]

# + [markdown] id="LC8u2YndV6-P"
# ## Vocabulary
#
# We can make a source and target vocabulary. To avoid having too many unique tokens, we specify a minimum frequency of 2 - all others will get replaced by "unk". We also add special tags for padding, begin of sentence, and end of sentence.

# + colab={"base_uri": "https://localhost:8080/"} id="v9SrJQc3VtSn" outputId="8b0e6cd5-890d-46c5-ce79-14f256ba63b3"
src_vocab = Vocab(source, min_freq=2,
                      reserved_tokens=['<pad>', '<bos>', '<eos>'])
len(src_vocab)

# + colab={"base_uri": "https://localhost:8080/"} id="KPF2FthfV840" outputId="f08bca8e-8564-411d-e58f-6e5b2e19f671"
# French has more high frequency words than English
target_vocab = Vocab(target, min_freq=2,
                      reserved_tokens=['<pad>', '<bos>', '<eos>'])
len(target_vocab)

# + [markdown] id="d0DyArAtWcob"
# ## Truncation and padding
#
# To create minibatches of sequences, all of the same length, we truncate sentences that are too long, and pad ones that are too short.

# + colab={"base_uri": "https://localhost:8080/"} id="x2D62tczWP-2" outputId="25ca1de2-97a6-4273-abbf-644a41562bb8"
def truncate_pad(line, num_steps, padding_token):
    """Truncate or pad sequences."""
    if len(line) > num_steps:
        return line[:num_steps]  # Truncate
    return line + [padding_token] * (num_steps - len(line))  # Pad

print(truncate_pad(source[0], 10, 'pad'))
print(truncate_pad(src_vocab[source[0]], 10, src_vocab['<pad>']))

# + id="RgyPxL6tWvJC"
def build_array_nmt(lines, vocab, num_steps):
    """Transform text sequences of machine translation into minibatches."""
    lines = [vocab[l] for l in lines]
    lines = [l + [vocab['<eos>']] for l in lines]
    array = torch.tensor([
        truncate_pad(l, num_steps, vocab['<pad>']) for l in lines])
    valid_len = (array != vocab['<pad>']).type(torch.int32).sum(1)
    return array, valid_len

# + colab={"base_uri": "https://localhost:8080/"} id="SlwULfBwW9ma" outputId="51eae96b-bfd0-4325-93ef-c98315abb6b0"
num_steps = 10
src_array, src_valid_len = build_array_nmt(source, src_vocab, num_steps)
print(src_array.shape)
print(src_valid_len.shape)

# + colab={"base_uri": "https://localhost:8080/"} id="br6L3nDbXFHY" outputId="e70924a1-6e71-49dc-b393-364feac92296"
print(src_array[0,:]) # go, ., eos, pad, ..., pad
print(src_valid_len[0])

# + [markdown] id="UyXmgFUvXVnA"
# ## Data iterator
#
# Below we combine all of the above pieces into a handy function.

# + id="AkD1QMiJXKAP"
def load_array(data_arrays, batch_size, is_train=True):
    """Construct a PyTorch data iterator."""
    dataset = data.TensorDataset(*data_arrays)
    return data.DataLoader(dataset, batch_size, shuffle=is_train)

def load_data_nmt(batch_size, num_steps, num_examples=600):
    """Return the iterator and the vocabularies of the translation dataset."""
    text = preprocess_nmt(read_data_nmt())
    source, target = tokenize_nmt(text, num_examples)
    src_vocab = Vocab(source, min_freq=2,
                          reserved_tokens=['<pad>', '<bos>', '<eos>'])
    tgt_vocab = Vocab(target, min_freq=2,
                          reserved_tokens=['<pad>', '<bos>', '<eos>'])
    src_array, src_valid_len = build_array_nmt(source, src_vocab, num_steps)
    tgt_array, tgt_valid_len = build_array_nmt(target, tgt_vocab, num_steps)
    data_arrays = (src_array, src_valid_len, tgt_array, tgt_valid_len)
    data_iter = load_array(data_arrays, batch_size)
    return data_iter, src_vocab, tgt_vocab

# + [markdown] id="ARsiX21oXdOd"
# Show the first minibatch.

# + colab={"base_uri": "https://localhost:8080/"} id="vl00eydyXeYF" outputId="a48df679-efd7-4e87-b3cd-fb66689e56e0"
train_iter, src_vocab, tgt_vocab = load_data_nmt(batch_size=2, num_steps=8)
for X, X_valid_len, Y, Y_valid_len in train_iter:
    print('X:', X.type(torch.int32))
    print('valid lengths for X:', X_valid_len)
    print('Y:', Y.type(torch.int32))
    print('valid lengths for Y:', Y_valid_len)
    break

# + id="thnQxtaIXenj"