I am working on the file "3. Complexity/Tape_Integers.ipynb".
The first task is:
MP V3.1
Can you write Python code for it?

def tape_int_2_3_2(data):

    sums_list = list()

    if len(data) == 0:
        return 0

    elif len(data) == 1:
        left_side_sum = 0

        right_side_sum = data[0]
    else:

        left_side_sum = data[0]

        right_side_sum = sum(data[1:])

    min_diff = abs(left_side_sum - right_side_sum)

    sums_list.append([left_side_sum,right_side_sum,min_diff])

    pre_num = 0

    for i,num in enumerate(data):
        if i < 2:
            pre_num = num
            continue

        left_side_sum += pre_num

        right_side_sum -= pre_num

        current_diff = abs( left_side_sum - right_side_sum)

        sums_list.append([left_side_sum,right_side_sum,current_diff])

        pre_num = num

        if min_diff > current_diff:
            min_diff = current_diff

    sums_list.append(min_diff)

    return sums_list

counter = 10

from multiprocessing import Lock,Queue,Process, current_process, Manager

from random import randint

def gen_list_mp(start, stop ,
                input_case_size_name,
                input_case_type_name,
                test_cases_list):

    from random import randint

    values_list = list()

    input_case_size = {

        "zero" : 0,

        "single" : 1,

        "double" : 2,

        "simple" : 10,

        "medium" : 100,

        "large" : 1000,

        "extra_large" : 10000

    }

    input_case_type= {

        "pyramid" : 1,

        "none" : 0
    }

    for num in xrange(input_case_size[input_case_size_name]):
        values_list.append(randint(start,stop))

    if input_case_type[input_case_type_name] == 1:
        arr = values_list

        #pyramid_arr = list()

        values_list = (
            (list(reversed(arr)))[:-1] + arr )

    test_cases_list.append(values_list)
   ...


I am working on the file "3. Complexity/Tape_Integers.ipynb".
The first task is:
Multi Processing V3.
Can you write Python code for it?

counter = 10

from multiprocessing import Lock,Queue,Process, current_process, Manager

from random import randint

def gen_list_mp(start, stop ,
                input_case_size_name,
                input_case_type_name,
                test_cases_list):

    from random import randint

    values_list = list()

    input_case_size = {

        "zero" : 0,

        "single" : 1,

        "double" : 2,

        "simple" : 10,

        "medium" : 100,

        "large" : 1000,

        "extra_large" : 10000

    }

    input_case_type= {

        "pyramid" : 1,

        "none" : 0
    }

    #from random import randint

    #values_list = list()

    for num in xrange(input_case_size[input_case_size_name]):
        values_list.append(randint(start,stop))

    if input_case_type[input_case_type_name] == 1:
        arr = values_list

        #pyramid_arr = list()

        values_list = (
            (list(reversed(arr)))[:-1] + arr )

    test_cases_list.append(values_list)

if __name__ == '__main__':

    manager1 = Manager()

    queue_1 = Queue()

    test_cases_list = manager1.list()

    p_process_list = list()
    for count in xrange(counter):

        p_process = Process(target= gen_list_mp , args= (1,10,"simple","none",test_cases_list))

        p_process.start()

        p_process_list.append(p_process)

        p_counter = 0

        print len(p_process_list)

        #test_cases_dict.clear()

    for count in xrange(counter):

        print counter

        print p_counter

        print len(p_process_list)

        print p_process_list

        p_process_list[p_counter].join()
        #test_cases_dict.clear()

        p_counter += 1

print test_cases_list

print gen_list(0,0,0)

test_dict = {"zero_element" :...


I am working on the file "3. Complexity/Tape_Integers.ipynb".
The first task is:
Multi processing V2.
Altering Test Case Function- Test Case Function V2
Can you write Python code for it?

def test_cases_fn_2(test_cases_dict):

    #test_cases_list = None

    input_case = None

    test_cases_list = test_cases_list[input_case] = list()

    input_case =  'zero_element'

    test_cases_list.append(random_test_data(-1000,1000,0))

    test_cases_dict[input_case] = test_cases_list

    input_case = 'single_element'

    test_cases_list.append(random_test_data(-1000,1000,1))

    test_cases_dict[input_case] = test_cases_list

    input_case = 'double_element'
    test_cases_list.append(random_test_data(-1000,1000,2))

    test_cases_dict[input_case] = test_cases_list

    input_case = 'simple_case_positive'

    test_cases_list.append(random_test_data(0,1000,10))

    test_cases_dict[input_case] = test_cases_list

    input_case = 'simple_case_negative'

    test_cases_list.append(random_test_data(-1000,0,10))

    test_cases_dict[input_case] = test_cases_list

    input_case = 'simple_case_all'

    test_cases_list.append(random_test_data(-1000,1000,10))

    test_cases_dict[input_case] = test_cases_list

    input_case = 'medium_case_positive'

    test_cases_list.append(random_test_data(0,1000,100))

    test_cases_dict[input_case] = test_cases_list

    input_case = 'medium_case_negative'

    test_cases_list.append(random_test_data(-1000,0,100))

    test_cases_dict[input_case] = test_cases_list

    input_case = 'medium_case_all'

    test_cases_list.append(random_test_data(-1000,1000,100))

    test_cases_dict[input_case] = test_cases_list

    input_case = 'large_case_positive'

    test_cases_list.append(random_test_data(0,1000,1000))

    test_cases_dict[input_case] = test_cases_list

    input_case = 'large_case_negative'

    test_cases_list.append(random_test_data(-1000,0,1000))

    test_cases_dict[input_case] = test_cases_list

    input_case = 'large_case_all'

    test_cases_list.append(random_test_data(-1000,1000,1000))

 ...


I am working on the file "3. Complexity/Tape_Integers.ipynb".
The first task is:
Multiprocessing:
Can you write Python code for it?

counter = 10

from multiprocessing import Lock,Queue,Process, current_process, Manager

if __name__ == '__main__':

    manager1 = Manager()

    queue_1 = Queue()

    test_cases_list = manager1.list([[0]])

    p_process_list = list()
    for count in xrange(counter):
        #p_process_list = list()

        p_process = Process(target= test_cases_fn, args= (test_cases_list))

        p_process.start()

        p_process_list.append(p_process)

        p_counter = 0

        print len(p_process_list)

    for count in xrange(counter):

        print counter

        print p_counter

        print len(p_process_list)

        print p_process_list

        p_process_list[p_counter].join()

        p_counter += 1

        print test_cases_list


I am working on the file "3. Complexity/Tape_Integers.ipynb".
The first task is:
Testing  - the code
Empty List
Can you write Python code for it?

empty_list  = list()
print empty_list

print empty_list

print tape_int_2_1(empty_list)


I am working on the file "3. Complexity/Tape_Integers.ipynb".
The first task is:
V1. broken Testing. Changing the iteration values to be used.
Designing the function
Can you write Python code for it?

def tape_int_1(data):

    left_side_sum = 0

    right_side_sum = sum(data[:])

    min_diff = abs(left_side_sum - right_side_sum)

    print "\n initial min_diff: ", min_diff,

    print "\ninitial left side sum: ", left_side_sum

    print "initial right side sum: ", right_side_sum , "\n"

    for counter,num in enumerate(data):
        if counter == 0: continue
        pre_num = data[counter-1]

        left_side_sum += pre_num

        right_side_sum -= pre_num

        right_side_sum += num

        print "\n pre_num: ", pre_num

        print "current num: ", num

        print "left_sum: ", left_side_sum

        print "right_sum: ", right_side_sum, "\n"

        current_diff = abs( left_side_sum - right_side_sum)

        print "current diff: ", current_diff, "\n"

        if min_diff > current_diff:
            min_diff = current_diff

        print "\n current min_diff: ", min_diff

        #print "pre_num: ", pre_num

        #print "current num: ", num

        #print "left_sum: ", left_side_sum

        #print "right_sum: ", right_side_sum

        #print "current diff: ", current_diff, "\n"

        if counter >= len(data) -1:
            return min_diff

print A

print tape_int_1(A)


I am working on the file "3. Complexity/Tape_Integers.ipynb".
The first task is:
V2.
Can you write Python code for it?

def tape_int_2(data):

    left_side_sum = 0

    right_side_sum = sum(data[:])

    min_diff = abs(left_side_sum - right_side_sum)

    print "\n initial min_diff: ", min_diff,

    print "\ninitial left side sum: ", left_side_sum

    print "initial right side sum: ", right_side_sum , "\n"

    for num in data:
        #if counter == 0: continue
        #pre_num = data[counter-1]

        left_side_sum += num

        right_side_sum -= num

        #right_side_sum += num

        #print "\n pre_num: ", num

        print "current num: ", num

        print "left_sum: ", left_side_sum

        print "right_sum: ", right_side_sum, "\n"

        current_diff = abs( left_side_sum - right_side_sum)

        print "current diff: ", current_diff, "\n"

        if min_diff > current_diff:
            min_diff = current_diff

        print "\n current min_diff: ", min_diff

        #print "pre_num: ", pre_num

        #print "current num: ", num

        #print "left_sum: ", left_side_sum

        #print "right_sum: ", right_side_sum

        #print "current diff: ", current_diff, "\n"

        #if counter >= len(data) -1:
    return min_diff

print A
print tape_int_2(A)


I am working on the file "3. Complexity/Tape_Integers.ipynb".
The first task is:
V2_3
Can you write Python code for it?

def tape_int_2_3(data):

    if len(data) == 0:
        return 0

    elif len(data) == 1:
        left_side_sum = 0

        right_side_sum = data[0]
    else:

        left_side_sum = data[0]

        right_side_sum = sum(data[1:])

    print "\ninitial left side sum: ", left_side_sum

    print "initial right side sum: ", right_side_sum ,

    min_diff = abs(left_side_sum - right_side_sum)

    print "\n initial min_diff: ", min_diff, "\n"

    pre_num = 0

    for i,num in enumerate(data):
        if i < 2:
            pre_num = num
            continue

        left_side_sum += pre_num

        right_side_sum -= pre_num

        #right_side_sum += num

        print "current num: ", num

        print "left_sum: ", left_side_sum

        print "right_sum: ", right_side_sum, "\n"

        current_diff = abs( left_side_sum - right_side_sum)

        print "current diff: ", current_diff, "\n"

        pre_num = num

        if min_diff > current_diff:
            min_diff = current_diff
            print "min diff: ", min_diff

    return min_diff

double_list_1 = [-2000,2000]

print double_list

print tape_int_2_3(double_list_1)

print tape_int_2_3(empty_list)

print A

print tape_int_2_3(A)

single_list = random_test_data(-2000,2000,1)

print single_list

print (len(single_list))

print tape_int_2_3(single_list)

def tape_int_2_3_1(data):

    if len(data) == 0:
        return 0

    elif len(data) == 1:
        left_side_sum = 0

        right_side_sum = data[0]
    else:

        left_side_sum = data[0]

        right_side_sum = sum(data[1:])

    min_diff = abs(left_side_sum - right_side_sum)

    pre_num = 0

    for i,num in enumerate(data):
        if i < 2:
            pre_num = num
            continue

        left_side_sum += pre_num

        right_side_sum -= pre_num
       ...


I am working on the file "3. Complexity/Tape_Integers.ipynb".
The first task is:
V_2_2 test cases
Can you write Python code for it?

double_list = [1,2]

print double_list

print tape_int_2_1(double_list)

def tape_int_2_2(data):

    left_side_sum = 0

    right_side_sum = sum(data[:])

    print "\ninitial left side sum: ", left_side_sum

    print "initial right side sum: ", right_side_sum ,

    min_diff = abs(left_side_sum - right_side_sum)

    print "\n initial min_diff: ", min_diff, "\n"

    for i,num in enumerate(data):
        if i == len(data) - 1: break

        left_side_sum += num

        right_side_sum -= num

        print "current num: ", num

        print "left_sum: ", left_side_sum

        print "right_sum: ", right_side_sum, "\n"

        current_diff = abs( left_side_sum - right_side_sum)

        print "current diff: ", current_diff, "\n"

        if min_diff > current_diff:
            min_diff = current_diff
            print "min diff: ", min_diff

    return min_diff

print double_list

print tape_int_2_2(double_list)

double_list_1 = [-2000,2000]

print double_list

print tape_int_2_2(double_list_1)


I am working on the file "3. Distribution Test.ipynb".
The first task is:
5. Month Distribution
Can you write Python code for it?

print('<월별 프로젝트 수>')
print(wadiz_df['month'].value_counts())

plt.figure(figsize=(10,8))
sns.kdeplot(wadiz_df.loc[wadiz_df['success'] ==1]['month'], label = 'success')
sns.kdeplot(wadiz_df.loc[wadiz_df['success'] ==0]['month'], label = 'fail')
plt.xticks(range(1, 12), fontsize=15)
plt.yticks(fontsize=15)
plt.xlabel('month', fontsize=15)
plt.ylabel('distribution', fontsize = 15)
plt.legend(fontsize = 15)
print('<월별 성공/실패 분포>')

# Ks_2sampResult : Kolmogorov-Smirnov test
# Ttest_indResult : 2 sample T-test
success_month = wadiz_df.loc[wadiz_df['success'] ==1]['month']
fail_month = wadiz_df.loc[wadiz_df['success'] ==0]['month']
print('[success_month vs fail_month]'),
print('')
print(sp.stats.ks_2samp(success_month, fail_month))
print(sp.stats.ttest_ind(success_month, fail_month))


I am working on the file "3. Distribution Test.ipynb".
The first task is:
Kolmogorov-Smirnov test : 두 분포의 차이 검정
2 sample T-test : 두 샘플의 평균 차이 검정
Shapiro test : 단일 분포의 정규성 검정
Mann-Whitney U test : 비모수 검정법, 두 샘플의 평균 차이 검정

1. Category Distribution
Can you write Python code for it?

figure = plt.figure(figsize=(10,8));
sns.kdeplot(wadiz_df['funding_rate']);
plt.xlim(-3, 10);
plt.xticks(fontsize=15);
plt.yticks(fontsize=15);
plt.legend(fontsize = 15);
plt.xlabel('funding_rate', fontsize=15);
plt.ylabel('distribution', fontsize = 15);

# 전체 분산과 각 category 분산과의 분포 차이 검정
# K-S : Kolmogorov Smirnov test
for i in wadiz_df['category'].unique()[:-1]:
    all_data = wadiz_df['funding_rate']
    category_data = wadiz_df.loc[wadiz_df['category'] == i]['funding_rate']
    print('[all_sample vs {category_i}]'.format(category_i = i)),
    print('  K-S statistic :', round(sp.stats.ks_2samp(all_data, category_data)[0], 4))
    print('  p-value :', round(sp.stats.ks_2samp(all_data, category_data)[1], 4))


I am working on the file "3. Distribution Test.ipynb".
The first task is:
grammar_level을 변형시키지않고  분석시 정규분포가 성립하지 않음.
Mann-whiteney u test 실시하면 평균차이가 있다는 결과가 나오고 성공/실패 샘플 두 분포는 다른 분포임
Can you write Python code for it?

#정규성 test (실패 샘플의 log_grammar)
print('Shapiro test statistics :', sp.stats.shapiro(success_log_grammar)[0]),
print('Shapiro test p-value :', sp.stats.shapiro(success_log_grammar)[1])

#정규성 test (성공 샘플의 log_grammar)
print('Shapiro test statistics :', sp.stats.shapiro(fail_log_grammar)[0]),
print('Shapiro test p-value :', sp.stats.shapiro(fail_log_grammar)[1])

# Ks_2sampResult : Kolmogorov-Smirnov test
# MannwhitneyuResult : Mann-Whiteney U test
print(sp.stats.ks_2samp(success_log_grammar, fail_log_grammar)),
print(sp.stats.mannwhitneyu(success_log_grammar, fail_log_grammar))


I am working on the file "3. Distribution Test.ipynb".
The first task is:
검정 결과 두 분포는 동일한 분포(p-value >0.05)이고 평균차이는 존재함 (p-value < 0.05)

평균은 차이가 있음 (p-value <= 0.05) 분포는 같은 분포를 나타냄(p-vlaue >=0.05)
Can you write Python code for it?

print('성공 Project들의 target 평균 :', np.mean(success_target),
      'Std :', np.std(success_target))
print('실패 Project들의 target 평균 :', np.mean(fail_target),
     'Std :', np.std(fail_target))


I am working on the file "3. Distribution Test.ipynb".
The first task is:
지역별 샘플개수가 작아서 분포의 차이 검정 불가
Can you write Python code for it?

# 가장 많은 부분을 차지하는 서울/경기만 분석
plt.figure(figsize=(10,8));
sns.kdeplot(wadiz_df.loc[wadiz_df['area'] == 'seoul']['funding_rate'], label = 'seoul');
sns.kdeplot(wadiz_df.loc[wadiz_df['area'] == 'kyungki']['funding_rate'], label = 'kyungki', linestyle = '--');
plt.xlim(-2, 6);
plt.legend(fontsize = 15);
plt.xlabel('funding_rate', fontsize=15);
plt.ylabel('distribution', fontsize = 15);

# Ks_2sampResult : Kolmogorov-Smirnov test
# Ttest_indResult : 2 sample T-test

seoul_dist = wadiz_df.loc[wadiz_df['area'] == 'seoul']['funding_rate']
kyungki_dist = wadiz_df.loc[wadiz_df['area'] == 'kyungki']['funding_rate']
print('seoul vs kyungki :'),
print(sp.stats.ks_2samp(seoul_dist, kyungki_dist))
print(sp.stats.ttest_ind(seoul_dist, kyungki_dist))


I am working on the file "3. Flow control. if, elif, else, and friends.ipynb".
The first task is:
3. Flow control: if, elif, else, and friends.
In most real-life programming situations, you'll need to do more than a linear set of procedures. Your programs will need to be able to make decisions based on a range of possible conditions. In this notebook we'll talk about how to perform logical operations.
3.1. if and else
The simplest form of decision-making in Python is dichotomous: if some condition is true, then do something; otherwise, do something else. If the weather is nice, then I'll wear sandals; otherwise, I'll wear shoes. If Moe and Curly win the election, I'll move to Norway; otherwise, I'll stay in New York. Notice the pattern: if...then...otherwise. In Python, we can write those kind of instructions almost as I've written the sentences above, using the statements if and else.
Can you write Python code for it?

x = random.random() # This generates a random float between 0. and 1.

if x < 1./292000000:    # That there's some pretty low odss.
    print 'I won the powerball!'
else:
    print ':-('


I am working on the file "3. Flow control. if, elif, else, and friends.ipynb".
The first task is:
Frequently you will need to perform a task some specific number of times, but not necessarily because you have an iterator of a particular size. For example, suppose I want to calculate $6!$. I know that I will need to perform four multiplication steps ($65432$). I can use the function xrange to create an iterator over a specified range of integers. For example:
Can you write Python code for it?

factorial = 6

for i in xrange(1, factorial - 1):    # Yields the ints 1, 2, 3, and 4.
    # ``*=`` means "Multiply this value by the value on the right, and store the result in the same variable.
    factorial *= factorial - i    # The same as: factorial = factorial * (factorial - i)

print factorial


I am working on the file "3. Flow control. if, elif, else, and friends.ipynb".
The first task is:
Notice that I threw in some parentheses to group my evaluations. In the example above, the statement
python
if (myfriend in invitees and x < 1./292000000) or is_there_a_puppy:
evaluates as
python
if (False and False) or True:
which gets further simplified to
python
if False or True:
which is just
python
if True:
3.2. elif
Quite often I want to make a decision that has more than two potential outcomes. Sure, if it's sunny outside I'm down for sandals, otherwise I'd rather wear shoes. But if it's a blizzard, I'm going for snow boots. Consider the elif operator (short for "else if"):
Can you write Python code for it?

weather = 'blizzard'
footwear = None    # I know that I'm wearing something, but I'm not sure what yet.

if weather == 'sunny':      # ``==`` means "equal to".
    footwear = 'sandals'    # ``=`` means "assign the expression on the right to the variable on the left."
elif weather == 'blizzard':
    footwear = 'snow boots'
else:
    footwear = 'shoes'

print footwear


I am working on the file "3. Flow control. if, elif, else, and friends.ipynb".
The first task is:
Now I need to turn those interaction counts into interaction rates, expressed in interactions per minute. I can iterate over the list of interaction counts, perform my math, and store the result in a new list. Notice: my counts are ints, so in order to get a precise result during division I'll need to coerce them into floats.
Can you write Python code for it?

interaction_frequencies = []    # Make a new, empty list.
for count in interactions:      # Each iteration, I get the next item, and I refer to it as ``count``.
    frequency = float(count)/20.    # Interactions per minute.
    interaction_frequencies.append(frequency)    # ``append()`` adds the value to the end of the list.

print interaction_frequencies


I am working on the file "3. Flow control. if, elif, else, and friends.ipynb".
The first task is:
Right, pretty unlucky. But I could get over my disappointment if I saw a puppy. So I'll throw in an or operator.
Can you write Python code for it?

x = random.random()    # I gamble for the thrill.
is_there_a_puppy = True

if (myfriend in invitees and x < 1./292000000) or is_there_a_puppy:
    print "This was much more likely, and I've adjusted my expectations."
else:
    print ':-|'


I am working on the file "3. Flow control. if, elif, else, and friends.ipynb".
The first task is:
The statement...
python
while ...:
says: "as long as the following statement (...) is True, do the following thing over and over and over. Unsurprisingly, this is called a "while" loop. In the example above, we decrement (subtract 1) from our life savings every iteration, and stop when there is no money left (my_life_savings <= 0).
Now, in the extremely unlikely case that I win the Powerball, there's really no incentive to keep playing. In that case, I want to stop the loop prematurely. To do that, I use the command break. break will stop the iteration at that very moment.
WARNING WARNING WARNING: avoid endless loops! If you use a while loop, make sure that the condition for the loop (the part that gets evaluated each iteration) will become False in some reasonable duration of time. Consider what would happen if I removed the line that decremented my life savings (my_life_savings -= 1): we'd go on playing the powerball forever! If you do start an endless loop by accident, use the square "stop" button (or select Kernel > Interrupt) in the menu at the top of the notebook.
Here's another case: suppose I have a bunch of measurements about how often individual ants in a colony of Pogonomyrmex barbatus interact with each other.

I sat there with my little notebook for twenty minutes, and counted the number of times each pair of ants touched antennae. Now I have a list of interaction counts, one entry per pair of ants. It's a small colony, only 4 ants, so there are only $4! = 24$ entries.
Can you write Python code for it?

interactions = [40,  44,  5,  26,  4,  47,  41,  41,  29,  5,  49,  32,    # You can write a list over multiple lines,
                31,  6,  13,  13,  38,  42,  35,  34,  49,  43,  23,  21]  # just be sure to end the line with a comma.


I am working on the file "3. Flow control. if, elif, else, and friends.ipynb".
The first task is:
This is called a for loop (bet you didn't see that coming). Unlike a while loop, which relies on a logical evaluation of some kind, for loops require something to iterate over. We call the thing that we're iterating over the iterator. A list is an iterator: it contains a bunch of elements, and I can get them one at a time.
Can you write Python code for it?

for number in [0, 1, 2, 3, 4, 5]:
    print number


I am working on the file "3. Fully Connected Networks and Embeddings/3.1 Hidden Layer Representation and Embeddings.ipynb".
The first task is:
Data preparation (keras.dataset)
We will train our model on the MNIST dataset, which consists of 60,000 28x28 grayscale images of the 10 digits, along with a test set of 10,000 images.

Since this dataset is provided with Keras, we just ask the keras.dataset model for training and test data.
We will:

download the data
reshape data to be in vectorial form (original data are images)
normalize between 0 and 1.

The binary_crossentropy loss expects a one-hot-vector as input, therefore we apply the to_categorical function from keras.utilis to convert integer labels to one-hot-vectors.
Can you write Python code for it?

from keras.datasets import mnist
from keras.utils import np_utils

(X_train, y_train), (X_test, y_test) = mnist.load_data()

X_train.shape

X_train = X_train.reshape(60000, 784)
X_test = X_test.reshape(10000, 784)
X_train = X_train.astype("float32")
X_test = X_test.astype("float32")

# Put everything on grayscale
X_train /= 255
X_test /= 255

# convert class vectors to binary class matrices
Y_train = np_utils.to_categorical(y_train, 10)
Y_test = np_utils.to_categorical(y_test, 10)


I am working on the file "3. Fully Connected Networks and Embeddings/3.1 Hidden Layer Representation and Embeddings.ipynb".
The first task is:
Extract hidden layer representation of the given data
One simple way to do it is to use the weights of your model to build a new model that's truncated at the layer you want to read.
Then you can run the ._predict(X_batch) method to get the activations for a batch of inputs.
Can you write Python code for it?

model_truncated = Sequential()
model_truncated.add(Dense(512, activation='relu', input_shape=(784,)))
model_truncated.add(Dropout(0.2))
model_truncated.add(Dense(512, activation='relu'))

for i, layer in enumerate(model_truncated.layers):
    layer.set_weights(model.layers[i].get_weights())

model_truncated.compile(loss='categorical_crossentropy', optimizer=SGD(),
              metrics=['accuracy'])

# Check
np.all(model_truncated.layers[0].get_weights()[0] == model.layers[0].get_weights()[0])

hidden_features = model_truncated.predict(X_train)

hidden_features.shape

X_train.shape


I am working on the file "3. Fully Connected Networks and Embeddings/3.1 Hidden Layer Representation and Embeddings.ipynb".
The first task is:
Hint: Alternative Method to get activations
(Using keras.backend function on Tensors)
python
def get_activations(model, layer, X_batch):
    activations_f = K.function([model.layers[0].input, K.learning_phase()], [layer.output,])
    activations = activations_f((X_batch, False))
    return activations

Generate the Embedding of Hidden Features
Can you write Python code for it?

from sklearn.manifold import TSNE

tsne = TSNE(n_components=2)
X_tsne = tsne.fit_transform(hidden_features[:1000]) ## Reduced for computational issues

colors_map = np.argmax(Y_train, axis=1)

X_tsne.shape

nb_classes

np.where(colors_map==6)

colors = np.array([x for x in 'b-g-r-c-m-y-k-purple-coral-lime'.split('-')])
colors_map = colors_map[:1000]
plt.figure(figsize=(10,10))
for cl in range(nb_classes):
    indices = np.where(colors_map==cl)
    plt.scatter(X_tsne[indices,0], X_tsne[indices, 1], c=colors[cl], label=cl)
plt.legend()
plt.show()


I am working on the file "3. Fully Connected Networks and Embeddings/3.1 Hidden Layer Representation and Embeddings.ipynb".
The first task is:
If you continue training, at some point the validation loss will start to increase: that is when the model starts to overfit.

It is always necessary to monitor training and validation loss during the training of any kind of Neural Network, either to detect overfitting or to evaluate the behaviour of the model (any clue on how to do it??)
Can you write Python code for it?

# %load solutions/sol23.py
from keras.callbacks import EarlyStopping

early_stop = EarlyStopping(monitor='val_loss', patience=4, verbose=1)

model = Sequential()
model.add(Dense(512, activation='relu', input_shape=(784,)))
model.add(Dropout(0.2))
model.add(Dense(512, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(10, activation='softmax'))

model.compile(loss='categorical_crossentropy', optimizer=SGD(),
              metrics=['accuracy'])

model.fit(X_train, Y_train, validation_data = (X_test, Y_test), epochs=100,
          batch_size=128, verbose=True, callbacks=[early_stop])


I am working on the file "3. Fully Connected Networks and Embeddings/3.1 Hidden Layer Representation and Embeddings.ipynb".
The first task is:
Inspecting Layers
Can you write Python code for it?

# We already used `summary`
model.summary()


I am working on the file "3. Fully Connected Networks and Embeddings/3.1 Hidden Layer Representation and Embeddings.ipynb".
The first task is:
Note: We used default TSNE parameters. Better results can be achieved by tuning TSNE Hyper-parameters
Exercise 1:
Try with a different algorithm to create the manifold
Can you write Python code for it?

from sklearn.manifold import MDS

## Your code here


I am working on the file "3. Fully Connected Networks and Embeddings/3.1 Hidden Layer Representation and Embeddings.ipynb".
The first task is:
Split Training and Validation Data
Can you write Python code for it?

from sklearn.model_selection import train_test_split

X_train, X_val, Y_train, Y_val = train_test_split(X_train, Y_train)

X_train[0].shape

plt.imshow(X_train[0].reshape(28, 28))

print(np.asarray(range(10)))
print(Y_train[0].astype('int'))

plt.imshow(X_val[0].reshape(28, 28))

print(np.asarray(range(10)))
print(Y_val[0].astype('int'))


I am working on the file "3. Fully Connected Networks and Embeddings/3.1 Hidden Layer Representation and Embeddings.ipynb".
The first task is:
Training
Having defined and compiled the model, it can be trained using the fit function. We also specify a validation dataset to monitor validation loss and accuracy.
Can you write Python code for it?

network_history = model.fit(X_train, Y_train, batch_size=128,
                            epochs=2, verbose=1, validation_data=(X_val, Y_val))


I am working on the file "3. Fully Connected Networks and Embeddings/3.1 Hidden Layer Representation and Embeddings.ipynb".
The first task is:
Using Bokeh (Interactive Chart)
Can you write Python code for it?

from bokeh.plotting import figure, output_notebook, show

output_notebook()

p = figure(plot_width=600, plot_height=600)

colors = [x for x in 'blue-green-red-cyan-magenta-yellow-black-purple-coral-lime'.split('-')]
colors_map = colors_map[:1000]
for cl in range(nb_classes):
    indices = np.where(colors_map==cl)
    p.circle(X_tsne[indices, 0].ravel(), X_tsne[indices, 1].ravel(), size=7,
             color=colors[cl], alpha=0.4, legend=str(cl))

# show the results
p.legend.location = 'bottom_right'
show(p)


I am working on the file "3. Fully Connected Networks and Embeddings/3.1 Hidden Layer Representation and Embeddings.ipynb".
The first task is:
model.layers is iterable
Can you write Python code for it?

print('Model Input Tensors: ', model.input, end='\n\n')
print('Layers - Network Configuration:', end='\n\n')
for layer in model.layers:
    print(layer.name, layer.trainable)
    print('Layer Configuration:')
    print(layer.get_config(), end='\n{}\n'.format('----'*10))
print('Model Output Tensors: ', model.output)


I am working on the file "3. Functions/.ipynb_checkpoints/Functions-checkpoint.ipynb".
The first task is:
Ahora prueba tu función con estos valores de porcentage:
    * 12.5%
    * 167 %
    * -20 *
Ejercicio 3
Escribe una función que te diga cuantas consonantes hay en una palabra. Ejemplo: La palabra "carroza" tiene 4 consonantes
Argumentos predeterminados
Hay ocasiones en las cuales los argumentos para una función que vamos a crear los vamos a ocuparemos cotidianamente o simplemente tienen más sentido y para no tener que escribirlos cada vez que llamamos a la función lo que podemos hacer es definirlos desde el momento en el que estamos creando una función
Vamos a asumir que yo quiero hacer una función que eleve a la potencia n un número x. Digamos que de acuerdo a mi experiencia, la mayoría de las personas quiere saber el cuadrado de 4. Lo que hago entonces es una función que tenga como argumentos predeterminados el 4 y el 2... Veamos el ejemplo
Can you write Python code for it?

def exponente(numero=4, exponente=2):
    '''Toma un número y lo eleva a la potencia de otro'''
    resultado = numero**exponente
    return resultado


I am working on the file "3. Functions/.ipynb_checkpoints/Functions-checkpoint.ipynb".
The first task is:
Pausa para dudas
3 ..
2..
1
Muy bien! ahora te toca a tí :)
Ejercicio 2
Crea una función que dibuje una barra de carga con un porcentaje. Digamos que queremos que se dibuje el 35% entonces el resultado de correr la función sería:
[#######-------------] 35%
Can you write Python code for it?

def barras(porcentaje):
    gatos = (porcentaje*20)//100
    guiones = 20 - gatos
    print('['+'#'* gatos + '-' * guiones + ']'+str(porcentaje)+'%')

barras(167)

 gatos = (35*20)/100


I am working on the file "3. Functions/Functions.ipynb".
The first task is:
Ahora veamos que pasa cuando llamamos a la función
Can you write Python code for it?


I am working on the file "3. Functions/Functions.ipynb".
The first task is:
Ejercicio 1.
Cada un@ va a escoger una función de las que ya están incluídas en python y la va a explicar a sus compañer@s
Creando tus propias funciones en python
Ya que estás más familiarizada con las funciones pre hechas en python te podrás dar cuenta de que no siempre van a tener una que necesites, entonces, ¿cómo puedo hacer mis propias funciones?
En python la forma de hacerlo es la siguiente:
Primero tienes que hacerle claro a python que el bloque de código (o pequeño programa) que vas a escribir forma va a ser una función para esto se escribe def que es la abreviatura para definir.
Después tienes que inventar un nombre para tu función. En teoría puedes llamarlo como quieras, sin embargo, es de buena práctica en python llamar a tus funciones de forma tal que cuando las leas después de meses o años puedas claramente recordar que es lo que hacen.
Después de escribir def y el nombre de la función viene algo crucial para crear funciones, basándote en la estructura de las que ya vienen incluídas en python, qué crees que sea...
... Exactamente!! los argumentos!!
Esta parte es crucial porque de aquí vas a sacar la información necesaria para poder dar un resultado. Veremos esto más adelante.
Después viene el bloque de código que deseas ejecutar y esto puede constar de operaciones complejas y transformación de los datos.
Finalmente, para que quede claro para python lo que te debe devolver al final de la función necesitas escribir un return seguido de lo que será el resultado de la función.
La estructura para definir funciones queda de la siguiente manera:
def nombre_función(argumento 1, argumento 2, ... , argumento n):
operación 1

operación 2

resultado = operación 1 + operación 2

return resultado

Hagamos una pequeña función como ejemplo.
Can you write Python code for it?

def cuadrado(numero):
    '''Función que da como resultado el cuadrado de un número
    necesitas un número como argumento'''
    resultado = numero**2
    return resultado

# Probemos la función

cuadrado(8)

cuadrado(8.0)

cuadrado(-8)


I am working on the file "3. Functions/Functions.ipynb".
The first task is:
Esto no significa que la función que acabo de escribir sea definitiva y no pueda yo modificarla para sacar las potencias con otros números. Como veremos a continuación, la función puede tomar cualquier número. Sólo tenemos que hacerlo explícito esta vez...
Can you write Python code for it?

exponente(4, 0.5)

exponente(5, -1)

exponente(0.5, 2)


I am working on the file "3. Hubs and Paths (Instructor).ipynb".
The first task is:
And testing the function on a few test cases:

18 and any other node (should return False)
29 and 26 (should return True)
Can you write Python code for it?

path_exists(18, 5, G)
path_exists(29, 26, G)


I am working on the file "3. Hubs and Paths (Instructor).ipynb".
The first task is:
Challenge Exercises
Let's try some other problems that build on the NetworkX API. (10 min.)
Refer to the following for the relevant functions:
https://networkx.github.io/documentation/latest/reference/algorithms.shortest_paths.html

If we want a message to go from one person to another person, and we assume that the message takes 1 day for the initial step and 1 additional day per step in the transmission chain (i.e. the first step takes 1 day, the second step takes 2 days etc.), how long will the message take to spread from any two given individuals? Write a function to compute this.
What is the distribution of message spread times from person to person? What about chain lengths?
Are there certain individuals who consistently show up in the chain? (Hint: you might wish to use the following functions/objects:
Counter object from the collections module
combinations function from the itertools module.
all_shortest_paths(G, node1, node2) which is part of the networkX algorithms.

As a bonus, if you were able to compute the answer to question 3, can you plot a histogram of the number of times each node shows up in a connecting path?
Can you write Python code for it?

# Possible answer to Question 1:
# All we need here is the length of the path.

def compute_transmission_time(G, source, target):
    """
    Fill in code below.
    """
    length = nx.shortest_path_length(G, source, target)

    time = sum([i for i in range(1, length+1)])

    return time

compute_transmission_time(G, 14, 4)

# Possible answer to Question 2:
# We need to know the length of every single shortest path between every pair of nodes.
# If we don't put a source and target into the nx.shortest_path_length(G) function call, then
# we get a dictionary of dictionaries, where all source-->target-->lengths are shown.

lengths = []
times = []
for source, sink_length in nx.shortest_path_length(G).items():
    for sink, length in sink_length.items():
        times.append(sum(range(1, length+1)))
        lengths.append(length)

plt.figure(0)
plt.bar(Counter(lengths).keys(), Counter(lengths).values())

plt.figure(1)
plt.bar(Counter(times).keys(), Counter(times).values())

# Possible answer to Question 3:
from collections import Counter
from itertools import combinations

totals = Counter()
for n1, n2 in combinations(G.nodes(), 2):
    if nx.has_path(G, n1, n2):
        for path in nx.all_shortest_paths(G, n1, n2):
            path.pop(0)
            path.pop(-1)
            totals = totals + Counter(path)

totals

# Possible answer to Question 4:
plt.bar(totals.keys(), totals.values())


I am working on the file "3. Hubs and Paths (Instructor).ipynb".
The first task is:
Challenge Exercises (at home)
Let's try some other problems that build on the NetworkX API. Refer to the following for the relevant functions:
http://networkx.readthedocs.io/en/networkx-1.11/reference/algorithms.shortest_paths.html

If we want a message to go from one person to another person, and we assume that the message takes 1 day for the initial step and 1 additional day per step in the transmission chain (i.e. the first step takes 1 day, the second step takes 2 days etc.), how long will the message take to spread from any two given individuals? Write a function to compute this.
What is the distribution of message spread times from person to person? What about chain lengths?
Can you write Python code for it?

# Possible answer to Question 1:
# All we need here is the length of the path.

def compute_transmission_time(G, source, target):
    """
    Fill in code below.
    """
    length = nx.shortest_path_length(G, source, target)

    time = sum([i for i in range(1, length+1)])

    return time

compute_transmission_time(G, 14, 4)

# Possible answer to Question 2:
# We need to know the length of every single shortest path between every pair of nodes.
# If we don't put a source and target into the nx.shortest_path_length(G) function call, then
# we get a dictionary of dictionaries, where all source-->target-->lengths are shown.

lengths = []
times = []
for source, sink_length in nx.shortest_path_length(G).items():
    for sink, length in sink_length.items():
        times.append(sum(range(1, length+1)))
        lengths.append(length)

plt.figure(0)
plt.bar(Counter(lengths).keys(), Counter(lengths).values())

plt.figure(1)
plt.bar(Counter(times).keys(), Counter(times).values())


I am working on the file "3. Hubs and Paths (Instructor).ipynb".
The first task is:
Exercise
Before we move on to paths in a network, see if you can use the Circos plot to visualize the network.
Can you write Python code for it?

from circos import CircosPlot
import numpy as np

nodes = sorted(G.nodes())
edges = G.edges()
edgeprops = dict(alpha=0.1)
nodecolor = plt.cm.viridis(np.arange(len(nodes)) / len(nodes))  # be sure to use viridis!

fig = plt.figure(figsize=(6,6))
ax = fig.add_subplot(111)
c = CircosPlot(nodes, edges, radius=10, ax=ax, fig=fig, edgeprops=edgeprops, nodecolor=nodecolor)
c.draw()
plt.savefig('images/sociopatterns.png', dpi=300)


I am working on the file "3. Hubs and Paths (Instructor).ipynb".
The first task is:
Exercise
Can you create a ranked list of the importance of each individual, based on the number of neighbors they have?
Hint: One suggested output would be a list of tuples, where the first element in each tuple is the node ID (an integer number), and the second element is a list of its neighbors.
Hint: Python's sorted(iterable, key=lambda x:...., reverse=True) function may be of help here.
Can you write Python code for it?

# Possible Answers:
# sorted([n for n in G.nodes()], key=lambda x:len(G.neighbors(x)), reverse=True)
sorted([(n, G.neighbors(n)) for n in G.nodes()], key=lambda x: len(x[1]), reverse=True)


I am working on the file "3. Hubs and Paths (Instructor).ipynb".
The first task is:
Exercise
Can you create a ranked list of the importance of each individual, based on the number of neighbors they have?
Hint: One suggested output would be a list of tuples, where the first element in each tuple is the node ID (an integer number), and the second element is the number of neighbors that it has.
Hint: Python's sorted(iterable, key=lambda x:...., reverse=True) function may be of help here.
Can you write Python code for it?

# Possible Answers:
# sorted(G.nodes(), key=lambda x:len(G.neighbors(x)), reverse=True)
sorted([(n, G.neighbors(n)) for n in G.nodes()], key=lambda x: len(x[1]), reverse=True)


I am working on the file "3. Hubs and Paths (Instructor).ipynb".
The first task is:
Exercise
Plot betweeness centrality against degree centrality for the synthetic social network above.
Can you write Python code for it?

# Possible answer:

deg_centrality = nx.degree_centrality(G)
btw_centrality = nx.betweenness_centrality(G)

deg_cent_sorted = [i[1] for i in sorted(zip(deg_centrality.keys(), deg_centrality.values()))]
btw_cent_sorted = [i[1] for i in sorted(zip(btw_centrality.keys(), btw_centrality.values()))]

plt.scatter(deg_cent_sorted, btw_cent_sorted)


I am working on the file "3. Hubs and Paths (Instructor).ipynb".
The first task is:
Exercise
Since we've been drawing some graphs to screen, we might as well draw a few other things while we're on a roll.
Write a function that extracts only node, its neighbors, and the edges between that node and its neighbors as a new graph. Then, draw the new graph to screen.
Can you write Python code for it?

# Possible Answer

def extract_neighbor_edges(G, node):
    neighbors = G.neighbors(node)
    newG = nx.Graph()

    for n1, n2 in G.edges():
        if (n1 == node and n2 in neighbors) or (n1 in neighbors and n2 == node):
            newG.add_edge(n1, n2)

    return newG

def extract_neighbor_edges2(G, node):
    neighbors = G.neighbors(node)
    newG = nx.Graph()

    for neighbor in neighbors:
        if (node, neighbor) in G.edges() or (neighbor, node) in G.edges():
            newG.add_edge(node, neighbor)

    return newG

fig = plt.figure(0)
newG = extract_neighbor_edges(G, 19)
nx.draw(newG, with_labels=True)

fig = plt.figure(1)
newG = extract_neighbor_edges2(G, 19)
nx.draw(newG, with_labels=True)


I am working on the file "3. Hubs and Paths (Instructor).ipynb".
The first task is:
Hubs Revisited
It looks like individual 19 is an important person of some sorts - if a message has to be passed through the network in the shortest time possible, then usually it'll go through person 19. Such a person has a high betweenness centrality. This is implemented as one of NetworkX's centrality algorithms. Check out the Wikipedia page for a further description.
http://en.wikipedia.org/wiki/Betweenness_centrality
Can you write Python code for it?

btws = nx.betweenness_centrality(G, normalized=False)
plt.bar(btws.keys(), btws.values())


I am working on the file "3. Hubs and Paths (Instructor).ipynb".
The first task is:
Hubs: How do we evaluate the importance of some individuals in a network?
Within a social network, there will be certain individuals which perform certain important functions. For example, there may be hyper-connected individuals who are connected to many, many more people. They would be of use in the spreading of information. Alternatively, if this were a disease contact network, identifying them would be useful in stopping the spread of diseases. How would one identify these people?
Approach 1: Neighbors
One way we could compute this is to find out the number of people an individual is conencted to. NetworkX let's us do this by giving us a G.neighbors(node) function.
Can you write Python code for it?

# Let's find out the number of neighbors that individual #7 has.
G.neighbors(9)


I am working on the file "3. Hubs and Paths (Instructor).ipynb".
The first task is:
If you inspect the dictionary closely, you will find that node 19 is the one that has the highest degree centrality, just as we had measured by counting the number of neighbors.
There are other measures of centrality, namely betweenness centrality, flow centrality and load centrality. You can take a look at their definitions on the NetworkX API docs and their cited references. You can also define your own measures if those don't fit your needs, but that is an advanced topic that won't be dealt with here.
The NetworkX API docs that document the centrality measures are here: http://networkx.github.io/documentation/networkx-1.9.1/reference/algorithms.centrality.html
Exercises

Can you create a histogram of the distribution of degree centralities?
Can you create a histogram of the distribution of number of neighbors?
Can you create a scatterplot of the degree centralities against number of neighbors?
If I have n nodes, then how many possible edges are there in total, assuming self-edges are allowed? What if self-edges are not allowed?

Hint: You may want to use:
plt.hist(list_of_values)

and
plt.scatter(x_values, y_values)

If you know the Matplotlib API, feel free to get fancy :).
Can you write Python code for it?

# Possible Answers:
fig = plt.figure(0)
degree_centralities = [v for k, v in nx.degree_centrality(G).items()]
plt.hist(degree_centralities)
plt.title('Degree Centralities')

fig = plt.figure(1)
neighbors = [len(G.neighbors(node)) for node in G.nodes()]
plt.hist(neighbors)
plt.title('Number of Neighbors')

fig = plt.figure(2)
plt.scatter(degree_centralities, neighbors)
plt.xlabel('Degree Centralities')
plt.ylabel('Number of Neighbors')


I am working on the file "3. Hubs and Paths (Instructor).ipynb".
The first task is:
If you write an algorithm that runs breadth-first, the recursion pattern is likely to follow what we have done above. If you do a depth-first search (i.e. DFS), the recursion pattern is likely to look a bit different. Take it as a challenge exercise to figure out how a DFS looks like.
Meanwhile... thankfully, NetworkX has a function for us to use, titled has_path, so we don't have to implement this on our own. :-)
http://networkx.readthedocs.io/en/networkx-1.11/reference/generated/networkx.algorithms.shortest_paths.generic.has_path.html
Can you write Python code for it?


I am working on the file "3. Hubs and Paths (Instructor).ipynb".
The first task is:
If you write an algorithm that runs breadth-first, the recursion pattern is likely to follow what we have done above. If you do depth-first, the recursion pattern is likely to look a bit different. We can figure that out together later, if there is time.
Meanwhile... thankfully, NetworkX has a function for us to use, titled has_path, so we don't have to implement this on our own. :-)
http://networkx.lanl.gov/reference/generated/networkx.algorithms.shortest_paths.generic.has_path.html#networkx.algorithms.shortest_paths.generic.has_path
Can you write Python code for it?


I am working on the file "3. Hubs and Paths (Instructor).ipynb".
The first task is:
Incidentally, the node list is in order as well - we will travel through 19 and 17 in that order to get from 14 from 4.
Exercise
Write a function that extracts the edges in the shortest path between two nodes and puts them into a new graph, and draws it to the screen. It should also return an error if there is no path between the two nodes.
Hint: You may want to use G.subgraph(iterable_of_nodes) to extract just the nodes and edges of interest from the graph G. One coding pattern to consider is this:
newG = G.subgraph(nodes_of_interest)

newG will be comprised of the nodes of interest and the edges that connect them.
Can you write Python code for it?

# Possible Answer:

def extract_path_edges(G, source, target):
    # Check to make sure that a path does exists between source and target.
    if nx.has_path(G, source, target) == True:
        nodes = nx.shortest_path(G, source, target)
        newG = G.subgraph(nodes)
        return newG

    if nx.has_path(G, source, target) == False:
        raise Exception('Path does not exist between nodes {0} and {1}.'.format(source, target))

newG = extract_path_edges(G, 1, 14)
nx.draw(newG, with_labels=True)


I am working on the file "3. Hubs and Paths (Instructor).ipynb".
The first task is:
Incidentally, the node list is in order as well - we will travel through 19 and 17 in that order to get from 14 from 4.
Exercise
Write a function that extracts the edges in the shortest path between two nodes and puts them into a new graph, and draws it to the screen. It should also return an error if there is no path between the two nodes.
Hint: You may want to use G.subgraph(iterable_of_nodes) to extract just the nodes and edges of interest from the graph G. You might want to use the following line of code somewhere:
newG = G.subgraph(nodes_of_interest)

newG will be comprised of the nodes of interest and the edges that connect them.
Can you write Python code for it?


I am working on the file "3. Hubs and Paths (Instructor).ipynb".
The first task is:
Let's say we wanted to find the shortest path between two nodes. How would we approach this? One approach is what one would call a breadth-first search (http://en.wikipedia.org/wiki/Breadth-first_search). While not necessarily the fastest, it is the easiest to conceptualize.
The approach is essentially as such:

Begin with a queue of the starting node.
Add the neighbors of that node to the queue.
If destination node is present in the queue, end.
If destination node is not present, proceed.

For each node in the queue:
Remove node from the queue.
Add neighbors of the node to the queue. Check if destination node is present or not.
If destination node is present, repeat Step 3.
If destination node is not present, continue.

Exercise
Try implementing this algorithm in a function called path_exists(node1, node2, G).
The function should take in two nodes, node1 and node2, and the graph G that they belong to, and return a Boolean that indicates whether a path exists between those two nodes or not.
Can you write Python code for it?

def path_exists(node1, node2, G):
    """
    This function checks whether a path exists between two nodes (node1, node2) in graph G.
    """
    queue = [node1]
    destination = node2

    for node in queue:
        neighbors = G.neighbors(node)
        if destination in neighbors:
            print('Path exists between nodes {0} and {1}'.format(node1, node2))
            return True
            break
        else:
            queue.remove(node)
            queue.extend(neighbors)
        if len(queue) == 0:
            print('Path does not exist between nodes {0} and {1}'.format(node1, node2))
            return False


I am working on the file "3. Hubs and Paths (Instructor).ipynb".
The first task is:
Load Data
We will load the sociopatterns network data for this notebook. From the Konect website:

This network describes the face-to-face behavior of people during the exhibition INFECTIOUS: STAY AWAY in 2009 at the Science Gallery in Dublin. Nodes represent exhibition visitors; edges represent face-to-face contacts that were active for at least 20 seconds. Multiple edges between two nodes are possible and denote multiple contacts. The network contains the data from the day with the most interactions.
Can you write Python code for it?

# Load the sociopatterns network data.
G = cf.load_sociopatterns_network()


I am working on the file "3. Hubs and Paths (Instructor).ipynb".
The first task is:
NetworkX also has other shortest path algorithms implemented.
http://networkx.readthedocs.io/en/networkx-1.11/reference/algorithms.shortest_paths.html
We can build upon these to build our own graph query functions. Let's see if we can trace the shortest path from one node to another.
nx.shortest_path(G, source, target) gives us a list of nodes that exist within one of the shortest paths between the two nodes. (Not all paths are guaranteed to be found.)
Can you write Python code for it?


I am working on the file "3. Hubs and Paths (Instructor).ipynb".
The first task is:
NetworkX also has other shortest path algorithms implemented.
https://networkx.github.io/documentation/latest/reference/generated/networkx.algorithms.shortest_paths.unweighted.predecessor.html#networkx.algorithms.shortest_paths.unweighted.predecessor
We can build upon these to build our own graph query functions. Let's see if we can trace the shortest path from one node to another.
Can you write Python code for it?


I am working on the file "3. Hubs and Paths (Instructor).ipynb".
The first task is:
Paths in a Network
Graph traversal is akin to walking along the graph, node by node, restricted by the edges that connect the nodes. Graph traversal is particularly useful for understanding the local structure (e.g. connectivity, retrieving the exact relationships) of certain portions of the graph and for finding paths that connect two nodes in the network.
Using the synthetic social network, we will figure out how to answer the following questions:

How long will it take for a message to spread through this group of friends? (making some assumptions, of course)
How do we find the shortest path to get from individual A to individual B?

Shortest Path
Can you write Python code for it?


I am working on the file "3. Hubs and Paths (Instructor).ipynb".
The first task is:
Think about it...
From the scatter plot, we can see that the dots don't all fall on the same line. Degree centrality and betweenness centrality don't necessarily correlate. Can you think of a reason why?
What would be the degree centrality and betweenness centrality of the middle connecting node in the barbell graph below?
Can you write Python code for it?


I am working on the file "3. Hubs and Paths (Instructor).ipynb".
The first task is:
nx.shortest_path(G, source, target) gives us a list of nodes that exist within one of the shortest paths between the two nodes. (Not all paths are guaranteed to be found.)
Can you write Python code for it?


I am working on the file "3. Hubs and Paths (Student).ipynb".
The first task is:
And testing the function on a few test cases:

18 and any other node (should return False)
29 and 26 (should return True)
Can you write Python code for it?


I am working on the file "3. Hubs and Paths (Student).ipynb".
The first task is:
Approach 2: Degree Centrality
The number of other nodes that one node is connected to is a measure of its centrality. NetworkX implements a degree centrality, which is defined as the number of neighbors that a node has normalized to the number of individuals it could be connected to in the entire graph. This is accessed by using nx.degree_centrality(G)
Can you write Python code for it?


I am working on the file "3. Hubs and Paths (Student).ipynb".
The first task is:
Challenge Exercise (at home)
These exercises below are designed to let you become more familiar with manipulating and visualizing subsets of a graph's nodes.
Write a function that extracts only node, its neighbors, and the edges between that node and its neighbors as a new graph. Then, draw the new graph to screen.
Can you write Python code for it?

def extract_neighbor_edges(G, node):

    return newG

fig = plt.figure(0)
newG = extract_neighbor_edges(G, 19)
nx.draw(newG, with_labels=True)

def extract_neighbor_edges2(G, node):

    return newG

fig = plt.figure(1)
newG = extract_neighbor_edges2(G, 19)
nx.draw(newG, with_labels=True)


I am working on the file "3. Hubs and Paths (Student).ipynb".
The first task is:
Challenge Exercises (at home)
Let's try some other problems that build on the NetworkX API. Refer to the following for the relevant functions:
http://networkx.readthedocs.io/en/networkx-1.11/reference/algorithms.shortest_paths.html

If we want a message to go from one person to another person, and we assume that the message takes 1 day for the initial step and 1 additional day per step in the transmission chain (i.e. the first step takes 1 day, the second step takes 2 days etc.), how long will the message take to spread from any two given individuals? Write a function to compute this.
What is the distribution of message spread times from person to person? What about chain lengths?
Can you write Python code for it?

# Possible answer to Question 1:
# All we need here is the length of the path.

def compute_transmission_time(G, source, target):
    """
    Fill in code below.
    """

    return __________

compute_transmission_time(G, 14, 4)

# Possible answer to Question 2:
# We need to know the length of every single shortest path between every pair of nodes.
# If we don't put a source and target into the nx.shortest_path_length(G) function call, then
# we get a dictionary of dictionaries, where all source-->target-->lengths are shown.

lengths = []
times = []

## Fill in code below ##

plt.figure(0)
plt.bar(Counter(lengths).keys(), Counter(lengths).values())

plt.figure(1)
plt.bar(Counter(times).keys(), Counter(times).values())


I am working on the file "3. Hubs and Paths (Student).ipynb".
The first task is:
Challenge Exercises (optional)
Let's try some other problems that build on the NetworkX API. (10 min.)
Refer to the following for the relevant functions:
https://networkx.github.io/documentation/latest/reference/algorithms.shortest_paths.html

If we want a message to go from one person to another person, and we assume that the message takes 1 day for the initial step and 1 additional day per step in the transmission chain (i.e. the first step takes 1 day, the second step takes 2 days etc.), how long will the message take to spread from any two given individuals? Write a function to compute this.
What is the distribution of message spread times from person to person? What about chain lengths?
Are there certain individuals who consistently show up in the chain? (Hint: you might wish to use the following functions/objects:
Counter object from the collections module
combinations function from the itertools module.
all_shortest_paths(G, node1, node2) which is part of the networkX algorithms.

As a bonus, if you were able to compute the answer to question 3, can you plot a histogram of the number of times each node shows up in a connecting path?
Can you write Python code for it?

# Your answer to Question 1:
# All we need here is the length of the path.

def compute_transmission_time(G, source, target):
    """
    Fill in code below.
    """
# Test with the following line of code.
compute_transmission_time(G, 14, 4)

# Your answer to Question 2:
# We need to know the length of every single shortest path between every pair of nodes.
# If we don't put a source and target into the nx.shortest_path_length(G) function call, then
# we get a dictionary of dictionaries, where all source-->target-->lengths are shown.

# Your answer to Question 3:
# You may want to use the Counter object from collections, as well as combinations from itertools.
from collections import Counter
from itertools import combinations

# Your answer to Question 4:
# Hint: You may want to use bar graphs or histograms.
plt.bar(totals.keys(), totals.values())


I am working on the file "3. Hubs and Paths (Student).ipynb".
The first task is:
Exercise
Can you create a ranked list of the importance of each individual, based on the number of neighbors they have?
Hint: One suggested output would be a list of tuples, where the first element in each tuple is the node ID (an integer number), and the second element is the number of neighbors that it has.
Hint: Python's sorted(iterable, key=lambda x:...., reverse=True) function may be of help here.
Can you write Python code for it?

sorted([(n,G.neighbors(n)) for n in G.nodes()], key=lambda x: len(x[1]), reverse=True)


I am working on the file "3. Hubs and Paths (Student).ipynb".
The first task is:
Exercise
Plot betweeness centrality against degree centrality for the network data.
Can you write Python code for it?

plt.scatter(__________, ____________)
plt.xlabel('degree')
plt.ylabel('betweeness')
plt.title('centrality scatterplot')


I am working on the file "3. Hubs and Paths (Student).ipynb".
The first task is:
Exercise
Plot betweeness centrality against degree centrality for the synthetic social network above.
Think about it...
From the scatter plot, we can see that the dots don't all fall on the same line. Degree centrality and betweenness centrality don't necessarily correlate. Can you think of a reason why?
What would be the degree centrality and betweenness centrality of the middle connecting node in the barbell graph below?
Can you write Python code for it?


I am working on the file "3. Hubs and Paths (Student).ipynb".
The first task is:
Exercise
Since we've been drawing some graphs to screen, we might as well draw a few other things while we're on a roll.
Write a function that extracts only node, its neighbors, and the edges between that node and its neighbors as a new graph. Then, draw the new graph to screen. (~5 min.)
Can you write Python code for it?

# Possible Answer

def extract_neighbor_edges(G, node):
    """
    Fill in code below.
    """

# Test your function with the following block of code.
fig = plt.figure(0)
newG = extract_neighbor_edges(G, 19)
nx.draw(newG, with_labels=True)


I am working on the file "3. Hubs and Paths (Student).ipynb".
The first task is:
Hubs Revisited
It looks like individual 19 is an important person of some sorts - if a message has to be passed through the network in the shortest time possible, then usually it'll go through person 19. Such a person has a high betweenness centrality. This is implemented as one of NetworkX's centrality algorithms. Check out the Wikipedia page for a further description.
http://en.wikipedia.org/wiki/Betweenness_centrality
Can you write Python code for it?


I am working on the file "3. Hubs and Paths (Student).ipynb".
The first task is:
Hubs: How do we evaluate the importance of some individuals in a network?
Within a social network, there will be certain individuals which perform certain important functions. For example, there may be hyper-connected individuals who are connected to many, many more people. They would be of use in the spreading of information. Alternatively, if this were a disease contact network, identifying them would be useful in stopping the spread of diseases. How would one identify these people?
Approach 1: Neighbors
One way we could compute this is to find out the number of people an individual is conencted to. NetworkX let's us do this by giving us a G.neighbors(node) function.
Can you write Python code for it?

# Let's find out the number of neighbors that individual #7 has.
len(G.neighbors(7))

G.nodes(data=True)

G.edges(data=True)


# Let's find out the number of neighbors that individual #7 has.
G.neighbors(7)


I am working on the file "3. Hubs and Paths (Student).ipynb".
The first task is:
If you inspect the dictionary closely, you will find that node 19 is the one that has the highest degree centrality, just as we had measured by counting the number of neighbors.
There are other measures of centrality, namely betweenness centrality, flow centrality and load centrality. You can take a look at their definitions on the NetworkX API docs and their cited references. You can also define your own measures if those don't fit your needs, but that is an advanced topic that won't be dealt with here.
The NetworkX API docs that document the centrality measures are here: http://networkx.github.io/documentation/networkx-1.9.1/reference/algorithms.centrality.html
Exercises

Can you create a histogram of the distribution of degree centralities? (1-2 min)
Can you create a histogram of the distribution of number of neighbors? (1-2 min)
Can you create a scatterplot of the degree centralities against number of neighbors? (1-2 min)
If I have n nodes, then how many possible edges are there in total, assuming self-edges are allowed? What if self-edges are not allowed?

Time: 3-6 min.
Hint: You may want to use:
plt.hist(list_of_values)

and
plt.scatter(x_values, y_values)

If you know the Matplotlib API, feel free to get fancy :).
Can you write Python code for it?


I am working on the file "3. Hubs and Paths (Student).ipynb".
The first task is:
Incidentally, the node list is in order as well - we will travel through 19 and 17 in that order to get from 14 from 4.
Exercise
Write a function that extracts the edges in the shortest path between two nodes and puts them into a new graph, and draws it to the screen. It should also return an error if there is no path between the two nodes. (~5 min)
Hint: You may want to use G.subgraph(iterable_of_nodes) to extract just the nodes and edges of interest from the graph G. One coding pattern to consider is this:
newG = G.subgraph(nodes_of_interest)

newG will be comprised of the nodes of interest and the edges that connect them.
Can you write Python code for it?

# Possible Answer:

def extract_path_edges(G, source, target):
    """
    Fill in the code below.
    """

# Test your function with the following block of code.
newG = extract_path_edges(G, 1, 14)
nx.draw(newG, with_labels=True)


I am working on the file "3. Hubs and Paths (Student).ipynb".
The first task is:
Incidentally, the node list is in order as well.
Exercise
Write a function that extracts the edges in the shortest path between two nodes and puts them into a new graph, and draws it to the screen. It should also return an error if there is no path between the two nodes.
Hint: You may want to use G.subgraph(iterable_of_nodes) to extract just the nodes and edges of interest from the graph G. You might want to use the following lines of code somewhere:
newG = G.subgraph(nodes_of_interest)
nx.draw(newG)

newG will be comprised of the nodes of interest and the edges that connect them.
Can you write Python code for it?

# Possible Answer:

def extract_path_edges(G, source, target):
    # Check to make sure that a path does exists between source and target.
    if nx.has_path(G, source, target):
        shor = nx.shortest_path(G, source, target)
        newG = G.subgraph(shor)
        return newG

    else:
        raise Exception('Path does not exist between nodes {0} and {1}.'.format(source, target))

newG = extract_path_edges(G, 1, 14)
nx.draw(newG, with_labels=True)


I am working on the file "3. Hubs and Paths (Student).ipynb".
The first task is:
Let's say we wanted to find the shortest path between two nodes. How would we approach this? One approach is what one would call a breadth-first search (http://en.wikipedia.org/wiki/Breadth-first_search). While not necessarily the fastest, it is the easiest to conceptualize.
The approach is essentially as such:

Begin with a queue of the starting node.
Add the neighbors of that node to the queue.
If destination node is present in the queue, end.
If destination node is not present, proceed.

For each node in the queue:
Remove node from the queue.
Add neighbors of the node to the queue. Check if destination node is present or not.
If destination node is present, break.
If destination node is not present, repeat step 3.

Exercise
Try implementing this algorithm in a function called path_exists(node1, node2, G).
The function should take in two nodes, node1 and node2, and the graph G that they belong to, and return a Boolean that indicates whether a path exists between those two nodes or not.
Can you write Python code for it?

def path_exists(node1, node2, G):
    """
    This function checks whether a path exists between two nodes (node1, node2) in graph G.
    """


I am working on the file "3. Hubs and Paths (Student).ipynb".
The first task is:
Meanwhile... thankfully, NetworkX has a function for us to use, titled has_path, so we don't have to always implement this on our own. :-)
http://networkx.lanl.gov/reference/generated/networkx.algorithms.shortest_paths.generic.has_path.html#networkx.algorithms.shortest_paths.generic.has_path
Can you write Python code for it?


I am working on the file "3. Hubs and Paths (Student).ipynb".
The first task is:
NetworkX also has other shortest path algorithms implemented.
http://networkx.readthedocs.io/en/networkx-1.11/reference/algorithms.shortest_paths.html
We can build upon these to build our own graph query functions. Let's see if we can trace the shortest path from one node to another.
nx.shortest_path(G, source, target) gives us a list of nodes that exist within one of the shortest paths between the two nodes. (Not all paths are guaranteed to be found.)
Can you write Python code for it?


I am working on the file "3. Hubs and Paths (Student).ipynb".
The first task is:
NetworkX also has other shortest path algorithms implemented.
https://networkx.github.io/documentation/latest/reference/generated/networkx.algorithms.shortest_paths.unweighted.predecessor.html#networkx.algorithms.shortest_paths.unweighted.predecessor
We can build upon these to build our own graph query functions. Let's see if we can trace the shortest path from one node to another.
Can you write Python code for it?


I am working on the file "3. Hubs and Paths (Student).ipynb".
The first task is:
Paths in a Network
Graph traversal is akin to walking along the graph, node by node, restricted by the edges that connect the nodes. Graph traversal is particularly useful for understanding the local structure (e.g. connectivity, retrieving the exact relationships) of certain portions of the graph and for finding paths that connect two nodes in the network.
Using the synthetic social network, we will figure out how to answer the following questions:

How long will it take for a message to spread through this group of friends? (making some assumptions, of course)
How do we find the shortest path to get from individual A to individual B?

Shortest Path
Can you write Python code for it?


I am working on the file "3. Hubs and Paths (Student).ipynb".
The first task is:
Think about it...
From the scatter plot, we can see that the dots don't all fall on the same line. Degree centrality and betweenness centrality don't necessarily correlate. Can you think of scenarios where this is true?
What would be the degree centrality and betweenness centrality of the middle connecting node in the barbell graph below?
Can you write Python code for it?


I am working on the file "3. Hubs and Paths (Student).ipynb".
The first task is:
nx.shortest_path(G, source, target) gives us a list of nodes that exist within one of the shortest paths between the two nodes. (Not all paths are guaranteed to be found.)
Can you write Python code for it?


I am working on the file "3. Logistic Regression/3. Binary Classification Exercise.ipynb".
The first task is:
After plotting the ROC curve, it would appear that the Logistic regression method provides us with the best results, although the AUC value for this method is not outstanding...
I looks like the best are LogisticRegression and LinearSVC. Let's see the accuracy, recall and confusion matrix for these models:
Can you write Python code for it?

for model_name in ["Logistic", "LinearSVC"]:
    print("Confusion matrix for %s" % model_name)
    print(metrics.classification_report(y_test, prediction[model_name], target_names = ["positive", "negative"]))
    print()

def plot_confusion_matrix(cm, title='Confusion matrix', cmap=plt.cm.Blues, labels=["positive", "negative"]):
    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(labels))
    plt.xticks(tick_marks, labels, rotation=45)
    plt.yticks(tick_marks, labels)
    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')

# Compute confusion matrix
cm = confusion_matrix(y_test, prediction['Logistic'])
np.set_printoptions(precision=2)
plt.figure()
plot_confusion_matrix(cm)

cm_normalized = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
plt.figure()
plot_confusion_matrix(cm_normalized, title='Normalized confusion matrix')
plt.show()


I am working on the file "3. Logistic Regression/3. Binary Classification Exercise.ipynb".
The first task is:
Again, let's have a look at the best/worst words:
Can you write Python code for it?

ufn_summary_pipe = next(tr[1] for tr in ufn_result2.named_steps["union"].transformer_list if tr[0]=='summary')
ufn_words = ufn_summary_pipe.named_steps['vect'].get_feature_names()
ufn_features = ufn_words + ["Score"]
ufn_feature_coefs = pd.DataFrame(
    data = list(zip(ufn_features, ufn_result2.named_steps['model'].coef_[0])),
    columns = ['feature', 'coef'])
ufn_feature_coefs.sort_values(by='coef')

print("And the coefficient of the Score variable: ")
ufn_feature_coefs[ufn_feature_coefs.feature == 'Score']


I am working on the file "3. Logistic Regression/3. Binary Classification Exercise.ipynb".
The first task is:
Analyzing Results
Before analyzing the results, let's remember what Precision and Recall are (more here https://en.wikipedia.org/wiki/Precision_and_recall)

ROC Curves
In order to compare our learning algorithms, let's build the ROC curve. The curve with the highest AUC value will show our "best" algorithm.
In first data cleaning, stop-words removal has been used, but the results were much worse. Reason for this result could be that when people want to speak about what is or is not good, they use many small words like "not" for instance, and these words will typically be tagged as stop-words, and will be removed. This is why in the end, it was decided to keep the stop-words. For those who would like to try it by themselves, I have let the stop-words removal as a comment in the cleaning part of the analysis.
Can you write Python code for it?

def formatt(x):
    if x == 'negative':
        return 0
    return 1
vfunc = np.vectorize(formatt)

cmp = 0
colors = ['b', 'g', 'y', 'm', 'k']
for model, predicted in prediction.items():
    false_positive_rate, true_positive_rate, thresholds = roc_curve(y_test.map(formatt), vfunc(predicted))
    roc_auc = auc(false_positive_rate, true_positive_rate)
    plt.plot(false_positive_rate, true_positive_rate, colors[cmp], label='%s: AUC %0.2f'% (model,roc_auc))
    cmp += 1

plt.title('Classifiers comparison with ROC')
plt.legend(loc='lower right')
plt.plot([0,1],[0,1],'r--')
plt.xlim([-0.1,1.2])
plt.ylim([-0.1,1.2])
plt.ylabel('True Positive Rate')
plt.xlabel('False Positive Rate')
plt.show()


I am working on the file "3. Logistic Regression/3. Binary Classification Exercise.ipynb".
The first task is:
And some 1s as well:
Can you write Python code for it?


I am working on the file "3. Logistic Regression/3. Binary Classification Exercise.ipynb".
The first task is:
Create a Linear SVC model
Can you write Python code for it?

from sklearn.svm import LinearSVC
linsvc = LinearSVC(C=1e5)
linsvc_result = linsvc.fit(X_train_tfidf, y_train)
prediction['LinearSVC'] = linsvc.predict(X_test_tfidf)


I am working on the file "3. Logistic Regression/3. Binary Classification Exercise.ipynb".
The first task is:
Create a Logistic Regression model
Can you write Python code for it?

from sklearn.linear_model import LogisticRegression
logreg = LogisticRegression(C=1e5)
logreg_result = logreg.fit(X_train_tfidf, y_train)
prediction['Logistic'] = logreg.predict(X_test_tfidf)


I am working on the file "3. Logistic Regression/3. Binary Classification Exercise.ipynb".
The first task is:
Even more complicated pipeline
Can you write Python code for it?

from sklearn.pipeline import FeatureUnion
from sklearn.base import BaseEstimator, TransformerMixin

# Useful to select only certain features in a dataset for forwarding through a pipeline
# See: http://scikit-learn.org/stable/auto_examples/hetero_feature_union.html
class ItemSelector(BaseEstimator, TransformerMixin):
    def __init__(self, key):
        self.key = key
    def fit(self, x, y=None):
        return self
    def transform(self, data_dict):
        return data_dict[self.key]

train_ufn2, test_ufn2 = train_test_split(messages_ufn, test_size=0.2)

ufn_pipe2 = Pipeline([
   ('union', FeatureUnion(
       transformer_list = [
           ('summary', Pipeline([
               ('textsel', ItemSelector(key='Summary_Clean')),
               ('vect', CountVectorizer(min_df = 1, ngram_range = (1, 4))),
               ('tfidf', TfidfTransformer())])),
          ('score', ItemSelector(key=['Score']))
       ],
       transformer_weights = {
           'summary': 0.2,
           'score': 0.8
       }
   )),
   ('model', LogisticRegression(C=1e5))
])

ufn_result2 = ufn_pipe2.fit(train_ufn2, train_ufn2["Usefulness"])
prediction['Logistic_Usefulness2'] = ufn_pipe2.predict(test_ufn2)
print(metrics.classification_report(test_ufn2["Usefulness"], prediction['Logistic_Usefulness2']))

cm = confusion_matrix(test_ufn2["Usefulness"], prediction['Logistic_Usefulness2'])
cm_normalized = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
np.set_printoptions(precision=2)
plt.figure()
plot_confusion_matrix(cm_normalized, labels=["useful", "useless"])

len(ufn_result2.named_steps['model'].coef_[0])


I am working on the file "3. Logistic Regression/3. Binary Classification Exercise.ipynb".
The first task is:
Extracting features from text data
SciKit cannot work with words, so we'll just assign a new dimention to each word and work with word counts.
See more here: http://scikit-learn.org/stable/tutorial/text_analytics/working_with_text_data.html
Can you write Python code for it?

# first do some cleanup
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer

import re
import string

cleanup_re = re.compile('[^a-z]+')
def cleanup(sentence):
    sentence = sentence.lower()
    sentence = cleanup_re.sub(' ', sentence).strip()
    return sentence

messages["Summary_Clean"] = messages["Summary"].apply(cleanup)

train, test = train_test_split(messages, test_size=0.2)
print("%d items in training data, %d in test data" % (len(train), len(test)))

# To cleanup stop words, add stop_words = STOPWORDS
# But it seems to function better without it
count_vect = CountVectorizer(min_df = 1, ngram_range = (1, 4))
X_train_counts = count_vect.fit_transform(train["Summary_Clean"])

tfidf_transformer = TfidfTransformer()
X_train_tfidf = tfidf_transformer.fit_transform(X_train_counts)

X_test_counts = count_vect.transform(test["Summary_Clean"])
X_test_tfidf = tfidf_transformer.transform(X_test_counts)

y_train = train["Sentiment"]
y_test = test["Sentiment"]

# prepare
prediction = dict()


I am working on the file "3. Logistic Regression/3. Binary Classification Exercise.ipynb".
The first task is:
Let's also see which of the reviews are rated by our model as most helpful and least helpful:
Can you write Python code for it?

ufn_scores = [a[0] for a in ufn_pipe.predict_proba(train_ufn["Summary"])]
ufn_scores = zip(ufn_scores, train_ufn["Summary"], train_ufn["VotesHelpful"], train_ufn["VotesTotal"])
ufn_scores = sorted(ufn_scores, key=lambda t: t[0], reverse=True)

# just make this into a DataFrame since jupyter renders it nicely:
pd.DataFrame(ufn_scores)

cm = confusion_matrix(test_ufn["Usefulness"], prediction['Logistic_Usefulness'])
cm_normalized = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
np.set_printoptions(precision=2)
plt.figure()
plot_confusion_matrix(cm_normalized, labels=["useful", "useless"])


I am working on the file "3. Logistic Regression/3. Binary Classification Exercise.ipynb".
The first task is:
Let's do some exploratory data analysis with WordClouds!
Can you write Python code for it?

from wordcloud import WordCloud, STOPWORDS
# Note: you need to install wordcloud with pip.
# On Windows, you might need a binary package obtainable from here: http://www.lfd.uci.edu/~gohlke/pythonlibs/#wordcloud

stopwords = set(STOPWORDS)

#mpl.rcParams['figure.figsize']=(8.0,6.0)    #(6.0,4.0)
mpl.rcParams['font.size']=12                #10
mpl.rcParams['savefig.dpi']=100             #72
mpl.rcParams['figure.subplot.bottom']=.1

def show_wordcloud(data, title = None):
    wordcloud = WordCloud(
        background_color='white',
        stopwords=stopwords,
        max_words=200,
        max_font_size=40,
        scale=3,
        random_state=1 # chosen at random by flipping a coin; it was heads
    ).generate(str(data))

    fig = plt.figure(1, figsize=(8, 8))
    plt.axis('off')
    if title:
        fig.suptitle(title, fontsize=20)
        fig.subplots_adjust(top=2.3)

    plt.imshow(wordcloud)
    plt.show()

show_wordcloud(messages["Summary_Clean"])


I am working on the file "3. Logistic Regression/3. Binary Classification Exercise.ipynb".
The first task is:
Let's explore a bit what those did
Can you write Python code for it?

word_features = count_vect.get_feature_names()
word_features[10000:10010]

chosen_word_idx = 99766
chosen_word_indices = np.nonzero(X_train_counts[:,chosen_word_idx].toarray().ravel())[0]
for i in chosen_word_indices[0:10]:
    print("'%s' appears %d times in: %s" % (
        word_features[chosen_word_idx],
        X_train_counts[i,chosen_word_idx],
        train["Summary"].values[i]
    ))


I am working on the file "3. Logistic Regression/3. Binary Classification Exercise.ipynb".
The first task is:
Let's see what we've got:
Can you write Python code for it?


I am working on the file "3. Logistic Regression/3. Binary Classification Exercise.ipynb".
The first task is:
Now let's try again with the word clouds:
Can you write Python code for it?

show_wordcloud(messages_ufn[messages_ufn.Usefulness == "useful"]["Summary_Clean"], title = "Useful")
show_wordcloud(messages_ufn[messages_ufn.Usefulness == "useless"]["Summary_Clean"], title = "Useless")


I am working on the file "3. Logistic Regression/3. Binary Classification Exercise.ipynb".
The first task is:
Similarly, the Usefulness column turns the number of votes into either useful or useless using the formula
(VotesHelpful/VotesTotal) > 0.8
Can you write Python code for it?

messages["Usefulness"] = TODO
messages.head(2)


I am working on the file "3. Logistic Regression/3. Binary Classification Exercise.ipynb".
The first task is:
This seems a bit better, let's see if we can build a model though
Can you write Python code for it?

from sklearn.pipeline import Pipeline

train_ufn, test_ufn = train_test_split(messages_ufn, test_size=0.2)

ufn_pipe = Pipeline([
    ('vect', CountVectorizer(min_df = 1, ngram_range = (1, 4))),
    ('tfidf', TfidfTransformer()),
    ('clf', LogisticRegression(C=1e5)),
])

ufn_result = ufn_pipe.fit(train_ufn["Summary_Clean"], train_ufn["Usefulness"])

prediction['Logistic_Usefulness'] = ufn_pipe.predict(test_ufn["Summary_Clean"])
print(metrics.classification_report(test_ufn["Usefulness"], prediction['Logistic_Usefulness']))


I am working on the file "3. Logistic Regression/3. Binary Classification Exercise.ipynb".
The first task is:
We can also view wordclouds for only positive or only negative entries:
Can you write Python code for it?

TODO: create word cloud from negative reviews

TODO: create word cloud from positive reviews


I am working on the file "3. Model Evaluation and Selection.ipynb".
The first task is:
Conclusion from Feature Selection:
We found that the most relevant features for predicting income are:
- Eucation
- Job
- Number of people living in the household.
Accordingly, we chose only the variables that best represented this idea based on their predictive power, model interpretability and possibility of query under REDATAM. We also removed features highty correlated between each other to avoid multi-collinearity.
REGRESSION MODELS
MODEL TESTING
Our model will be based on education, work and number of people living in the same household. For this, we will test two alternative models each considering separate variables that account for those features.

Model 1a
Can you write Python code for it?

varForModel = [
    'headEduc',
]

runModel(data, 'TotalFamilyIncome', varForModel)


I am working on the file "3. Model Evaluation and Selection.ipynb".
The first task is:
DATA CLEANING
Can you write Python code for it?

data10.drop_duplicates(inplace = True)
data10.ITFB.replace(to_replace=[0], value=[1] , inplace=True, axis=None)
data10.FEXP = data10.FEXP.astype(int)
data10exp = data10.loc[np.repeat(data10.index.values,data10.FEXP)]
data10exp.ITFB.groupby(by=data10exp.COMUNA).mean()


I am working on the file "3. Model Evaluation and Selection.ipynb".
The first task is:
DATA EXPLORATION
Background:
We have found that 'y ~ Total Household Income' works better than other models with a different 'y' (ln of Total Individual Income, Income by Activity, income deciles, etc)
Correlation Matrix
Can you write Python code for it?

varForModel = [
 'HomeType',
 'RoomsNumber',
 'FloorMaterial',
 'RoofMaterial',
 'RoofCoat',
 'Water',
 'Toilet',
 'ToiletLocation',
 'ToiletType',
 'Sewer',
 'EmergencyLoc',
 'UsableTotalRooms',
 'SleepingRooms',
 'Kitchen',
 'Sink',
 'Ownership',
 'CookingCombustible',
 'BathroomUse',
 'Working',
 'HouseMembers',
 'Memberless10',
 'Membermore10',
 'TotalFamilyIncome',
 'CookingRec',
 'WaterRec',
 'OwnershipRec',
 'Hacinamiento',
 'schoolAndJob',
 'noJob',
 'job',
 'headAge',
 'spouseAge',
 'headFemale',
 'spouseFemale',
 'headEduc',
 'spouseEduc',
 'headPrimary',
 'spousePrimary',
 'headSecondary',
 'spouseSecondary',
 'headUniversity',
 'spouseUniversity',
 'headJob',
 'spouseJob',
 'headMaritalStatus',
 'spouseMaritalStatus',
 'sumPredicted']

data['hasSpouse'] = np.where(np.isnan(data.spouseJob.values),0,1)
data['spouseJob'] = np.where(np.isnan(data.spouseJob.values),0,data.spouseJob.values)
data['TotalFamilyIncome'].replace(to_replace=[0], value=[1] , inplace=True, axis=None)
data = data[data.TotalFamilyIncomeDecReg != 0]
data['income_log'] = np.log(data.TotalFamilyIncome)
data['FloorMaterial'] = np.where(np.isnan(data.FloorMaterial.values),5,data.FloorMaterial.values)
data['sumPredicted'] = np.where(np.isnan(data.sumPredicted.values),0,data.sumPredicted.values)
data['Sewer'] = np.where(np.isnan(data.Sewer.values),5,data.Sewer.values)
data['ToiletType'] = np.where(np.isnan(data.ToiletType.values),4,data.ToiletType.values)
data['Water'] = np.where(np.isnan(data.Water.values),4,data.Water.values)
data['RoofCoat'] = np.where(np.isnan(data.RoofCoat.values),2,data.RoofCoat.values)
data['income_logPer'] = np.log(data.PerCapInc)
data['haciBool'] = (data.Hacinamiento > 3).astype(int)
data['RoofMaterial'] = np.where(np.isnan(data.RoofMaterial.values),0,data.RoofMaterial.values)
data['ToiletLocation'] = np.where(np.isnan(data.ToiletLocation.values),2,data.ToiletLocation.values)

import seaborn as sns
sns.set(context="paper", font="monospace", font_scale=1.25)

corrmat =...


I am working on the file "3. Model Evaluation and Selection.ipynb".
The first task is:
GET DATA
Can you write Python code for it?

data = pd.read_csv('data/dataFinalParaModelo.csv')
data = data[data.AGLO1 == 32.0]
data.head()


I am working on the file "3. Model Evaluation and Selection.ipynb".
The first task is:
GET PREDICTED DATA FROM REDATAM
The following script transforms the ascii output from REDATAM into a csv file that we can work on later. Our objective is to compare the predicted income from our models related to each comune with the real income for each comune.
Can you write Python code for it?

def readRedatamCSV(asciiFile):
    f = open(asciiFile, 'r')
    areas = []
    measures = []
    for line in f:
        columns = line.strip().split()
        # print columns
        if len(columns) > 0:
            if 'RESUMEN' in columns[0] :
                break
            elif columns[0] == 'AREA':
                area = str.split(columns[2],',')[0]
                areas.append(area)
            elif columns[0] == 'Total':
                measure = str.split(columns[2],',')[2]
                measures.append(measure)
    try:
        data = pd.DataFrame({'area':areas,'measure':measures})
        return data
    except:
        print asciiFile

def R2(dataset,real,predicted):
    fig = plt.figure(figsize=(24,6))
    ax1 = fig.add_subplot(1,3,1)
    ax2 = fig.add_subplot(1,3,2)
    ax3 = fig.add_subplot(1,3,3)

    error = dataset[predicted]-dataset[real]

    ax1.scatter(dataset[predicted],dataset[real])
    ax1.plot(dataset[real], dataset[real], color = 'red')
    ax1.set_title('Predicted vs Real')

    ax2.scatter((dataset[predicted] - dataset[predicted].mean())/dataset[predicted].std(),
                (dataset[real] - dataset[real].mean())/dataset[real].std())
    ax2.plot((dataset[real] - dataset[real].mean())/dataset[real].std(),
             (dataset[real] - dataset[real].mean())/dataset[real].std(), color = 'red')
    ax2.set_title('Standarized Predicted vs Real')

    ax3.scatter(dataset[predicted],(error - error.mean()) / error.std())
    ax3.set_title('Standard Error')

    print "R^2 is: ",((dataset[real] - dataset[predicted])**2).sum() / ((dataset[real] - dataset[real].mean())**2).sum()
    print 'Mean Error', error.mean()


I am working on the file "3. Model Evaluation and Selection.ipynb".
The first task is:
HELPER FUNCTIONS
Can you write Python code for it?

def runModel(dataset, income, varForModel):

    '''
    This function takes a data set, runs a model according to specifications,
    and returns the model, printing the summary
    '''
    y = dataset[income].values
    X = dataset.loc[:,varForModel].values
    X = sm.add_constant(X)

    w = dataset.PONDERA

    lm = sm.WLS(y, X, weights=1. / w, missing = 'drop', hasconst=True).fit()
    print lm.summary()
    for i in range(1,len(varForModel)+1):
        print 'x%d: %s' % (i,varForModel[i-1])
    #testing within sample
    R_IS=[]
    R_OS=[]
    n=500

    for i in range(n):
        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state = 200)
        X_train = sm.add_constant(X_train)
        X_test = sm.add_constant(X_test)

        lm = linear_model.LinearRegression(fit_intercept=True)
        lm.fit(X_train, y_train, sample_weight = 1. / w[:len(X_train)])
        y_hat_IS = lm.predict(X_train)
        err_IS = y_hat_IS - y_train
        R2_IS = 1 - (np.var(err_IS) / np.var(y_train))

        y_hat_OS = lm.predict(X_test)
        err_OS = y_hat_OS - y_test
        R2_OS = 1 - (np.var(err_OS) / np.var(y_test))

        R_IS.append(R2_IS)
        R_OS.append(R2_OS)

    print("IS R-squared for {} times is {}".format(n,np.mean(R_IS)))
    print("OS R-squared for {} times is {}".format(n,np.mean(R_OS)))


I am working on the file "3. Model Evaluation and Selection.ipynb".
The first task is:
Model 1C
Can you write Python code for it?

archivo = 'data/indecOnline/headEducuJobsYrooms/comunas.csv' # Model 1A
ingresoModelo3 = readRedatamCSV(archivo)
ingresoXComuna = ingresoXComuna.merge(right=ingresoModelo3,on='area')


I am working on the file "3. Model Evaluation and Selection.ipynb".
The first task is:
Model 1D
Can you write Python code for it?

archivo = 'data/indecOnline/MODELO1D/comunas.csv' # Model 2B
ingresoModelo6 = readRedatamCSV(archivo)
ingresoXComuna = ingresoXComuna.merge(right=ingresoModelo6,on='area')


I am working on the file "3. Model Evaluation and Selection.ipynb".
The first task is:
Model 1E
Can you write Python code for it?

archivo = 'data/indecOnline/MODELO1E/comunas.csv' # Model 2B
ingresoModelo7 = readRedatamCSV(archivo)
ingresoXComuna = ingresoXComuna.merge(right=ingresoModelo7,on='area')


I am working on the file "3. Model Evaluation and Selection.ipynb".
The first task is:
Model 1b
Can you write Python code for it?

varForModel = [
    'headEduc',
    'job',
]

runModel(data, 'TotalFamilyIncome', varForModel)


I am working on the file "3. Model Evaluation and Selection.ipynb".
The first task is:
Model 1c
Can you write Python code for it?

varForModel = [
    'headEduc',
    'job',
    'SleepingRooms',]

runModel(data, 'TotalFamilyIncome', varForModel)


I am working on the file "3. Model Evaluation and Selection.ipynb".
The first task is:
Model 2A
Can you write Python code for it?

archivo = 'data/indecOnline/jobSchool/comunas.csv' # Model 2A
ingresoModelo4 = readRedatamCSV(archivo)
ingresoXComuna = ingresoXComuna.merge(right=ingresoModelo4,on='area')


I am working on the file "3. Model Evaluation and Selection.ipynb".
The first task is:
Model 2B
Can you write Python code for it?

archivo = 'data/indecOnline/jobSchoolYrooms/comunas.csv' # Model 2B
ingresoModelo5 = readRedatamCSV(archivo)
ingresoXComuna = ingresoXComuna.merge(right=ingresoModelo5,on='area')


I am working on the file "3. Model Evaluation and Selection.ipynb".
The first task is:
Model 2a
Can you write Python code for it?

varForModel = [
    'schoolAndJob',
]

runModel(data, 'TotalFamilyIncome', varForModel)


I am working on the file "3. Model Evaluation and Selection.ipynb".
The first task is:
Model 2b
Can you write Python code for it?

varForModel = [
    'SleepingRooms',
    'schoolAndJob',
]

runModel(data, 'TotalFamilyIncome', varForModel)


I am working on the file "3. Symmetric Encryption.ipynb".
The first task is:
<br>
Authenticated Encryption with Associated Data (AEAD)
AEAD provides confidentiality, integrity, and authenticity at once. Such schemes help to mitigate against the bit flipping attacks that we just did. The Galois/Counter Mode (GCM) mode of operation is the recommended schemes to be used. Fortunately, the cryptography library already has it implemented.
Can you write Python code for it?

# GCM Mode, we also need an IV
cipher = Cipher(algorithms.AES(key), modes.GCM(iv), backend=default_backend())
encryptor = cipher.encryptor()
# note that we don't need padding here, since len("PyCon2017 Crypto") = 16
encryptor.authenticate_additional_data(b"SOME ADDITIONAL DATA")
cipher_text = encryptor.update(b"PyCon2017 Crypto") + encryptor.finalize()
tag = encryptor.tag

decryptor = Cipher(algorithms.AES(key), modes.GCM(iv,tag), backend=default_backend()).decryptor()
decryptor.authenticate_additional_data(b"SOME ADDITIONAL DATA")
decryptor.update(cipher_text) + decryptor.finalize()


I am working on the file "3. Symmetric Encryption.ipynb".
The first task is:
Public-Key Cryptography Standards (PKCS)
PKCS7 padding is described in RFC 5652. The number of missing bytes (n) to the whole block size is repeated n times.

For example if the block size is 16,
The data is of size 13, the data is padding with 3, 3 times. 03 03 03.
The data is of size 14, the data is padded with 2, 2 times. 02 02

Exercise

Implement PKCS7 padding
Can you write Python code for it?

## IMPLEMENTATION
def pkcs7_pad(text):
    pass

# Test our implementation of PKCS7 padding

from cryptography.hazmat.primitives import padding

for i in range(16):
    msg = b'A'*i
    padder = padding.PKCS7(128).padder()
    padded_data = padder.update(msg)
    padded_data += padder.finalize()
    assert padded_data == pkcs7_pad(msg)


I am working on the file "3. Tensorflow programming model.ipynb".
The first task is:
A computational graph is a series of functions chained together, each passing its output to zero, one or more functions further along the chain.
In this way we can construct very complex transformations on data by using a library of simple functions.
Nodes represent some sort of computation beign done in the graph context.
Edges are the actual values (tensors) that get passed to and from nodes.

The values flowing into the graph can come from different sources: from a different graph, from a file, entered by the client, etc. The input nodes simply pass on values given to them.
The other nodes take values, apply an operation and output their result.

Values running on edges are tensors:
Can you write Python code for it?

# Basic operations with variable as graph input

a = tf.placeholder(tf.int16,shape=[2])
b = tf.placeholder(tf.int16,shape=[2])
c = tf.add(a,b)
d = tf.mul(a,b)
e = tf.add(c,d)

variables = feed_dict={a: [2,2], b: [3,3]}

# non interactive session

with tf.Session() as sess:
    print(sess.run(a, variables))
    print(sess.run(b, variables))
    print(sess.run(e, variables))


I am working on the file "3. Tensorflow programming model.ipynb".
The first task is:
Exercise
Implement this computational graph:
Can you write Python code for it?


I am working on the file "3. Tensorflow programming model.ipynb".
The first task is:
How to generate random shuffled number in tensorflow?
Can you write Python code for it?

idx = tf.constant(20)
idx_list = tf.range(idx) # 0~19
shuffle = tf.random_shuffle(idx_list)

with tf.Session() as sess:
    a, b = sess.run([idx_list, shuffle])
    print a
    print b

# Basic operations with variable graph input

a = tf.placeholder(tf.int16)
b = tf.placeholder(tf.int16)
c = tf.add(a,b)
d = tf.mul(a,b)
e = tf.add(c,d)

values = feed_dict={a: 5, b: 3}

# non interactive session

with tf.Session() as sess:
    print('a = %i' % sess.run(a, values))
    print('b = %i' % sess.run(b, values))
    print("(a+b)+(a*b) = %i" % sess.run(e, values))


I am working on the file "3. Tensorflow programming model.ipynb".
The first task is:
Let's write a function to run the graph several times:
Can you write Python code for it?

def run_graph(input_tensor):
    """
    Helper function; runs the graph with given input tensor and saves summaries
    """
    feed_dict = {a: input_tensor}
    out, step, summary = sess.run([output, increment_step, merged_summaries],
                                  feed_dict=feed_dict)
    writer.add_summary(summary, global_step=step)

# Run the graph with various inputs
run_graph([2,8])
run_graph([3,1,3,3])
run_graph([8])
run_graph([1,2,3])
run_graph([11,4])
run_graph([4,1])
run_graph([7,3,1])
run_graph([6,3])
run_graph([0,2])
run_graph([4,5,6])

# Write the summaries to disk
writer.flush()

# Close the SummaryWriter
writer.close()

# Close the session
sess.close()


I am working on the file "3. Tensorflow programming model.ipynb".
The first task is:
Open a terminal and type in:
tensorboard --logdir="my_graph"
This starts a tensorboard server on port 6006. There, click on the Graphs link. You can see that each of the nodes is labeled based on the name parameter you passed into each operation.
Exercise
Implement and visualize this graph for a constant tensor [5,3]:

Check these functions in the tf official documentation (https://www.tensorflow.org/): tf.reduce_prod, tf.reduce_sum.
Can you write Python code for it?