I am working on the file "Computacion/Verano-2019/clase_de_notebooks/Analisis de artistas.ipynb".
The first task is:
Tambien podemos combinar varias canciones de varios artistas para hacer algo aun mas loco.
Can you write Python code for it?

import csv

data = {}

authors =  ["ABBA","Stevie Wonder","Hank Williams"]

with open("songdata.csv") as file:

    prev = ("","")
    for author,_,_,lyric in csv.reader(file):

        if author not in authors:
            continue

        for word in lyric.lower().split():
            new_word = word

            if not word[0].isalpha():
                data[prev] = data.get(prev,[])
                data[prev].append(word[0])

                prev = (prev[-1],word[0])

            while new_word and not new_word[0].isalpha():
                new_word = new_word[1:]

            while new_word and not new_word[-1].isalpha():
                new_word = new_word[:-1]

            data[prev] = data.get(prev,[])
            data[prev].append(new_word)

            prev = (prev[-1],new_word)

            if not word[-1].isalpha():
                data[prev] = data.get(prev,[])
                data[prev].append(word[-1])

                prev = (prev[-1],word[-1])

        data[prev] = data.get(prev,[])
        data[prev].append("")
        prev = (prev[-1],"")

cancion = []
prev = ("","")
for _ in range(100):
    next_word = random.choice(data[prev])
    cancion.append(next_word)
    prev = (prev[-1],next_word)

display(Markdown("_" + " ".join(cancion) + "_") )


I am working on the file "Computacion/Verano-2019/clase_de_notebooks/Analisis de compuestos.ipynb".
The first task is:
Checkeo si los valores se transformaron correctamente
Can you write Python code for it?

random.sample(tabla2,10)

mediciones = {}
for cantidad,temperatura in tabla2:
    mediciones[cantidad] = mediciones.get(cantidad,[])
    mediciones[cantidad].append(temperatura)


I am working on the file "Compute distance to roads.ipynb".
The first task is:
Function declarations
connect to an attribute table
Can you write Python code for it?

def connectToAttributeTable(map):
    vector = VectorTopo(map)
    vector.open(mode='r')
    dblinks = DBlinks(vector.c_mapinfo)
    link = dblinks[0]
    return link.table()


I am working on the file "Compute distance to roads.ipynb".
The first task is:
GRASS import statements
Can you write Python code for it?

import grass.script as gscript

from grass.pygrass.vector import VectorTopo
from grass.pygrass.vector.table import DBlinks


I am working on the file "Compute distance to roads.ipynb".
The first task is:
Get 'points' attribute table
Can you write Python code for it?

point_table = connectToAttributeTable(map=points)
point_table.filters.select()
columns = point_table.columns.names()
cursor = point_table.execute()
result = np.array(cursor.fetchall())
cursor.close()
point_data = pandas.DataFrame(result, columns=columns).set_index('cat')


I am working on the file "Compute distance to roads.ipynb".
The first task is:
Get unique 'roads' types
Can you write Python code for it?

roads_table = connectToAttributeTable(map=roads)
roads_table.filters.select(road_type_field)
cursor = roads_table.execute()
result = np.array(cursor.fetchall())
cursor.close()
road_types = np.unique(result)

print(road_types)


I am working on the file "Compute distance to roads.ipynb".
The first task is:
Loop through 'roads' types and compute the distances from all 'points'
Can you write Python code for it?

distances = pandas.DataFrame(columns=road_types, index=point_data.index)

progress_bar = pyprind.ProgBar(road_types.size, bar_char='█', title='Progress', monitor=True, stream=1, width=50)

for type_ in road_types:

    # update progress bar
    progress_bar.update(item_id=type_)

    # extract road data based on type query
    extractFeatures(input_=roads, type_=type_, output='roads_tmp')

    # compute distance from points to road type
    results = computeDistance(points, 'roads_tmp')

    # save results to data frame
    distances[type_] = [ d.split(',')[1] for d in results[1:len(results)-1] ]

# match index with SiteID
distances['SiteID'] = point_data['ID']
distances.set_index('SiteID', inplace=True)


I am working on the file "Compute distance to roads.ipynb".
The first task is:
finds the nearest element in a vector map (to) for elements in another vector map (from) <br />
calls the GRASS v.distance command
Can you write Python code for it?

def computeDistance(from_map, to_map):

    upload = 'dist'
    result = gscript.read_command('v.distance',
                         from_=from_map,
                         to=to_map,
                         upload=upload,
                         separator='comma',
                         flags='p')
    return result.split('\n')


I am working on the file "Compute distance to roads.ipynb".
The first task is:
selects vector features from an existing vector map and creates a new vector map containing only the selected features <br />
calls the GRASS v.extract command
Can you write Python code for it?

def extractFeatures(input_, type_, output):

    where = "{0} = '{1}'".format(road_type_field, type_)
    gscript.read_command('v.extract',
                         input_=input_,
                         where=where,
                         output=output,
                         overwrite=True)


I am working on the file "Compute landcover areas.ipynb".
The first task is:
Compute landcover areas
connect to 'sample_points' attribute table
Can you write Python code for it?

point_table = connectToAttributeTable(map=sample_points)
point_table.filters.select()
columns = point_table.columns.names()
cursor = point_table.execute()
result = np.array(cursor.fetchall())
cursor.close()
point_data = pandas.DataFrame(result, columns=columns).set_index('cat')


I am working on the file "Compute landcover areas.ipynb".
The first task is:
Export landcover area table
create table with landcover areas by type for each point
Can you write Python code for it?

#create table
index_start = 41
''' set the first index number,
    allowing easier insertion into a database table that already contains
    area calculations with other parameters
    (i.e radius and use_viewshed)'''
columns = ['ID', 'SiteID', 'IncludedArea']
columns = columns + [ str(n) for n in range(1,n_types+1) ]
area_table = pandas.DataFrame(columns=columns)

#set naming variables
included_area = '{0}m{1}'.format(radius, getViewshedSuffix())  #0=radius, 1=viewshed_suffix
map_pattern = 'vect_{0}_landcover_{1}m{2}'  #0=ID, 1=radius, 2=viewshed_suffix

#iterate through points
for index, point in point_data.iterrows():
    ID = point['ID']
    map = map_pattern.format(ID, radius, getViewshedSuffix())
    #initiate row
    row = {'ID':"{0:.3g}".format(int(index) + index_start),
           'SiteID': str(ID),
           'IncludedArea': included_area}
    #get landcover areas
    areas = getLandcoverAreaByType(map)

    #iterate through area types
    for index, area in areas.iterrows():
        #add area to row
        row["{0:.3g}".format(int(index))] = area['area_square_meters']
    #append row to table
    area_table = area_table.append(row, ignore_index=True)

area_table.set_index('ID', inplace=True)

#export table to file
area_table.to_csv(landcover_area_table_filename, header=False)


I am working on the file "Compute landcover areas.ipynb".
The first task is:
Function declarations
connect to attribute table
Can you write Python code for it?


I am working on the file "Compute landcover areas.ipynb".
The first task is:
GRASS import statements
Can you write Python code for it?

import grass.script as gscript

from grass.pygrass.vector.geometry import Point
from grass.pygrass.vector import Vector

from grass.pygrass.vector import VectorTopo
from grass.pygrass.vector.table import DBlinks


I am working on the file "Compute landcover areas.ipynb".
The first task is:
Import statements
Can you write Python code for it?

import numpy as np
import pandas
import pyprind


I am working on the file "Compute landcover areas.ipynb".
The first task is:
Utility functions
export vector map to a shapefile
Can you write Python code for it?

def exportVectorToShapefile(map, output):
    gscript.read_command('v.out.ogr',
                         input=map,
                         format='ESRI_Shapefile',
                         output=output,
                         flags='e',
                         overwrite=True)


I am working on the file "Compute landcover areas.ipynb".
The first task is:
add area column to vector map
Can you write Python code for it?

def calculateAreas(map):

    #add new area column
    gscript.read_command('v.db.addcolumn',
                         map=map,
                         columns="area_square_meters DOUBLE PRECISION")

    #compute area and insert into area column
    gscript.read_command('v.to.db',
                         map=map,
                         type='centroid',
                         option='area',
                         columns='area_square_meters',
                         unit='meters')


I am working on the file "Compute landcover areas.ipynb".
The first task is:
convert viewshed from raster to vector map
Can you write Python code for it?

def vectorizeViewshed(input, ID, output):
    type = 'area'
    column = 'visible'

    gscript.read_command('r.to.vect',
                         input=input,
                         output=output,
                         type=type,
                         column=column,
                         overwrite=True)


I am working on the file "Compute landcover areas.ipynb".
The first task is:
create table showing total area by landcover type
Can you write Python code for it?

def getLandcoverAreaByType(map):
    #get area data
    table = connectToAttributeTable(map=map)
    table.filters.select()
    columns = table.columns.names()
    cursor = table.execute()
    result = np.array(cursor.fetchall())
    cursor.close()
    data = pandas.DataFrame(result, columns=columns).set_index('cat')

    #sum areas by landcover type
    data['area_square_meters'] = pandas.to_numeric(data['area_square_meters'])
    areas = data[['b_' + landcover_type_field, 'area_square_meters']].groupby(by='b_' + landcover_type_field).sum()
    return areas


I am working on the file "Compute landcover areas.ipynb".
The first task is:
overlay a vector map on an underlying vector map using 'and' selection operator
Can you write Python code for it?

def overlay(overlay, underlay, output):
    operator='and'
    gscript.read_command('v.overlay',
                         ainput=overlay,
                         binput=underlay,
                         operator=operator,
                         output=output,
                         overwrite=True)


I am working on the file "Compute location distance error.ipynb".
The first task is:
compute the distance mean and standard deviation
Can you write Python code for it?


I am working on the file "Compute location distance error.ipynb".
The first task is:
load locations from database
Can you write Python code for it?

generated = get_geodataframe(SampleLocation.objects.all())
actual = get_geodataframe(Site.objects.filter(id__lte=30))


I am working on the file "Compute location distance error.ipynb".
The first task is:
loop through locations and compute distance
Can you write Python code for it?

distance_array = numpy.zeros(30)
distances = pandas.DataFrame({'id': generated.index, 'name': actual.sort_index().name, 'distance': distance_array}).set_index('id')
for i in range(1, 31):
    x1 = generated[generated.index == i].geometry.as_matrix()[0].coords.xy[0][0]
    x2 = actual[actual.index == i].geometry.as_matrix()[0].coords.xy[0][0]
    y1 = generated[generated.index == i].geometry.as_matrix()[0].coords.xy[1][0]
    y2 = actual[actual.index == i].geometry.as_matrix()[0].coords.xy[1][0]
    distance_array[i - 1] = numpy.sqrt((x2 - x1)**2 + (y2 - y1)**2)
distances['distance'] = distance_array

distances


I am working on the file "Compute naturalness areas.ipynb".
The first task is:
GRASS import statements
Can you write Python code for it?


I am working on the file "Compute naturalness areas.ipynb".
The first task is:
Import statements
Can you write Python code for it?


I am working on the file "Compute naturalness areas.ipynb".
The first task is:
extract point from vector map
Can you write Python code for it?

def extractPoint(input, ID, output):

    where = 'ID = {0}'.format(ID)
    type = 'point'

    gscript.read_command('v.extract',
                         input=input,
                         where=where,
                         output=output,
                         type=type,
                         overwrite=True)


I am working on the file "Compute naturalness areas.ipynb".
The first task is:
loop through sample points
Can you write Python code for it?

with Vector(sample_points, mode='r') as points:

    #setup progress bar
    progress_bar = pyprind.ProgBar(points.n_lines, bar_char='█', title='Naturalness analysis progress', monitor=True, stream=1, width=50)

    #iterate through points
    for point in points:

        #get point ID (SiteID)
        ID = point_data['ID'][point.cat-1]

        #update progress bar
        progress_bar.update(item_id=ID)

        #buffer current point
        extractPoint(input='sample_points_field', ID=ID, output='tmp_buffer_point')
        bufferPoint(input='tmp_buffer_point', output='tmp_point_buffer', radius=radius)

        #set buffer as overlay
        overlay_input = 'tmp_point_buffer'
        #consider only visible naturalness if 'use_viewshed' = True
        if use_viewshed:
            viewshed = 'vect_{0}_viewshed'.format(ID)
            visible_viewshed = 'vect_{0}_viewshed_{1}m'.format(ID, radius)
            #vectorize viewshed
            vectorizeViewshed(input='{0}_viewshed'.format(ID), ID=ID, output=viewshed)
            #overlay buffer on viewshed
            overlay(overlay=overlay_input,
                    underlay=viewshed,
                    output=visible_viewshed)
            #set overlay to the visible viewshed
            overlay_input = visible_viewshed
        overlay_output = 'vect_{0}_naturalness_{1}m{2}'.format(ID, radius, getViewshedSuffix())
        #overlay naturalness
        overlay(overlay=overlay_input,
                         underlay=naturalness,
                         output=overlay_output)

        #calculate naturalness area
        calculateAreas(map=overlay_output)


I am working on the file "Compute naturalness areas.ipynb".
The first task is:
overlay a vector map on an underlying vector map using 'and' selection operator
Can you write Python code for it?


I am working on the file "Compute viewsheds.ipynb".
The first task is:
Compute Viewshed
Can you write Python code for it?

radius = -1  #default = -1 (infinity)

tmp_viewshed_name = 'tmp_viewshed'
tmp_point = 'tmp_current_point'

# connect to attributes table
sample_points = VectorTopo(observer_points)
sample_points.open(mode='r')
dblinks = DBlinks(sample_points.c_mapinfo)
link = dblinks[0]
sample_points_table = link.table()
sample_points_table.filters.select('cat', 'ID')

with Vector(observer_points, mode='r') as points:

    progress_bar = pyprind.ProgBar(points.n_lines, bar_char='█', title='Viewshed progress', monitor=True, stream=1, width=50)

    for point in points:
        progress_bar.update(item_id = str(point.cat))

        # get ID of point
        sample_points_table.filters.where('cat = {0}'.format(point.cat))
        cursor = sample_points_table.execute()
        result = np.array(cursor.fetchall())
        cursor.close()
        data = pandas.DataFrame(result, columns=['cat', 'ID']).set_index('cat')
        ID = data['ID'].get_value(point.cat)

        # compute viewshed
        viewshed_id = str(ID)
        viewshed_name = viewshed_id + '_viewshed'
        gscript.read_command('r.viewshed',
                             input=elevation,
                             observer_elevation=1.5,
                             output=tmp_viewshed_name,
                             coordinates=point.coords(),
                             overwrite=True)
        gscript.mapcalc(exp="{viewshed} = if({tmp}, {vid}, null())".format(viewshed=viewshed_name,
                                                                                tmp=tmp_viewshed_name,
                                                                                vid=viewshed_id),
                        overwrite=True)


I am working on the file "Compute viewsheds.ipynb".
The first task is:
Import packages
Can you write Python code for it?


I am working on the file "Compute viewsheds.ipynb".
The first task is:
Utility functions
delete viewshed
Can you write Python code for it?

def delete_viewshed(name):
    gscript.read_command('g.remove',
                         type='raster',
                         name=name,
                         flags='f')


I am working on the file "ComputeCincinnatiNeighborhoodCentroids.ipynb".
The first task is:
Compute Cincinnati neighborhood centroids

Read shapefile
Can you write Python code for it?

def init_neighborhoods(readerobj):
    """ Initializes a dictionary that stores a description of
    City of Cincinnati neighborhoods

    Args:
        readerobj: shapfilemodule Reader class object handle

    Returns:
        neighborhood: Dictionary that stores a description of
                      City of Cincinnati neighborhoods"""
    shapes = readerobj.shapes()

    fieldnames = [re.sub('_', '', elem[0].lower())
                  for elem in readerobj.fields[1:]]

    neighborhood = {}

    for idx in range(0, sf.numRecords):
        row_dict = dict(zip(fieldnames, readerobj.record(idx)))

        row_dict['boundingbox'] = np.array(shapes[idx].bbox)

        row_dict['centroid'] = [np.mean(row_dict['boundingbox'][0:4:2]),
                               np.mean(row_dict['boundingbox'][1:4:2])]

        cur_neighborhood = row_dict.pop('neigh').lower()

        cur_neighborhood = re.sub('[-\s+]','', cur_neighborhood)

        neighborhood[cur_neighborhood] = row_dict

    return neighborhood

readerobj = shapefile.Reader(output_shapefile)

neighborhood = init_neighborhoods(readerobj)

from pyspark.mllib.feature import Vectors
from pyspark.mllib.linalg import DenseVector

neighborhood_centroid = []

for key in neighborhood.keys():
    neighborhood_centroid.append(Vectors.dense(neighborhood[key]['centroid']))


I am working on the file "ComputeCincinnatiNeighborhoodCentroids.ipynb".
The first task is:
Reproject Cincinnati Area Geographic Information System (CAGIS) cc neighbndy.shp

Cagis Homepage
Reproject shapefile with ogr2ogr
Run system command with subprocess library
Can you write Python code for it?

output_shapefile = "../CagisOpenDataQuarterly/neighborhood.shp"

if not os.path.exists(output_shapefile):
    sys_command = 'ogr2ogr ' + output_shapefile + ' ' +\
              '"../CagisOpenDataQuarterly/cc neighbndy.shp" -t_srs EPSG:4326'

    process = subprocess.Popen(sys_command,
                               shell=True,
                               stdout=subprocess.PIPE)
    process.wait()
    print process.returncode


I am working on the file "ComputerVisionPatterns-Copy1.ipynb".
The first task is:
Again, this is a lousy model like the previous one, as it just predicts 9.
Can you write Python code for it?

# let's try logistic regression.
# Stochastic Logistic Regression
model = Sequential()

# validation loss
model.add(Flatten(input_shape=(3, 50, 50)))
model.add(Dense(output_dim=17, activation='sigmoid', W_regularizer=l1(0.01)))

# Compile model
sgd = SGD(lr=0.1)
model.compile(loss='mean_squared_error', optimizer=sgd, metrics=['accuracy'])

model.summary()

# Fit the model
model = model.fit(X_train, y_train, batch_size = 256,
          nb_epoch = 100, verbose=0, validation_data=(X_test,y_test))

predicted = model.model.predict_classes(X_test)

# put the y_test back into a format of non-one-hot encoded for comparison
y_test_orig = pd.DataFrame(y_test).idxmax(1)
print "Logistic Regression Results\nConfusion Matrix: "
print confusion_matrix(y_test_orig,predicted)
print "\n Classifcation Report"
print classification_report(y_test_orig,predicted)


I am working on the file "ComputerVisionPatterns-Copy1.ipynb".
The first task is:
The random forest is actually working much better than the neural networks and logistic regression models, and better precision than even the convolutional neural network.  The convolutional neural network still has the best recall, however.  It's still not a great model, but it has precision and recall better then 50%, however, most test records are being predicted as 9 as in the other models.  Apparently, our data is unbalanced among the classes - there are tons of category 'plain' compared to others.
Can you write Python code for it?

import matplotlib.pyplot as plt

g=imgdf.groupby(['category']).count()['_unit_id']

objects = g.index
y_pos = np.arange(len(objects))
performance = g

plt.bar(y_pos, performance, align='center', alpha=0.5)
#plt.xticks(y_pos, objects)
plt.xticks(y_pos, objects, rotation='vertical')
plt.ylabel('Class Counts')
plt.title('Dress Classes in Data')

plt.show()


I am working on the file "ComputerVisionPatterns-Copy1.ipynb".
The first task is:
These neural network based models are giving the same results, except for the convolutional network.  We'll try it again for a logistic regression, but pre-flattening the data.
Can you write Python code for it?

# reshape the training and test data
X_train_new = np.empty(shape=[X_train.shape[0]] + [7500], dtype='float32')
for i in range(X_train.shape[0]):
    X_train_new[i,:] = X_train[i,:,:,:].flatten()

X_test_new = np.empty(shape=[X_test.shape[0]] + [7500], dtype='float32')
for i in range(X_test.shape[0]):
    X_test_new[i,:] = X_test[i,:,:,:].flatten()

print X_test_new.shape
print X_train_new.shape

# Stochastic Logistic Regression
model = Sequential()

# validation loss
model.add(Dense(output_dim=17, input_shape=[7500],
                activation='sigmoid', W_regularizer=l2(0)))

# Compile model
sgd = SGD(lr=0.1)
model.compile(loss='categorical_crossentropy', optimizer=sgd)

model.summary()

# Fit the model
model = model.fit(X_train_new, y_train, batch_size = 256,
          nb_epoch = 100, verbose=0, validation_data=(X_test_new,y_test))

predicted = model.model.predict_classes(X_test_new)

# put the y_test back into a format of non-one-hot encoded for comparison
y_test_orig = pd.DataFrame(y_test).idxmax(1)
print "Logistic Regression Results\nConfusion Matrix: "
print confusion_matrix(y_test_orig,predicted)
print "\n Classifcation Report"
print classification_report(y_test_orig,predicted)


I am working on the file "ComputerVisionPatterns-Copy1.ipynb".
The first task is:
This is a much worse model, and it is always predicting a value of 7.
Can you write Python code for it?

# Let's try something totally different, a neural network based on the homework from week 5:
model = Sequential()
model.add(Flatten(input_shape=(3, 50, 50)))
model.add(Dense(output_dim=100,
                activation='sigmoid', W_regularizer=l2(0.01)))
model.add(Dense(output_dim=500, activation='sigmoid', W_regularizer=l2(0.01)))
model.add(Dense(output_dim=17, activation='sigmoid', W_regularizer=l2(0.01)))

# Compile model
sgd = SGD(lr=0.1)
model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy'])

model.summary()

# Fit the model
model = model.fit(X_train, y_train, batch_size = 256,
          nb_epoch = 20, verbose=2, validation_data=(X_test,y_test))

predicted = model.model.predict_classes(X_test)

# put the y_test back into a format of non-one-hot encoded for comparison
y_test_orig = pd.DataFrame(y_test).idxmax(1)
print "Neural Network Results\nConfusion Matrix: "
print confusion_matrix(y_test_orig,predicted)
print "\n Classifcation Report"
print classification_report(y_test_orig,predicted)


I am working on the file "ComputerVisionPatterns-Copy1.ipynb".
The first task is:
This model is by no means great, but it does predict with .63 recall and .54 precision.
Can you write Python code for it?

# Let's try with the Keras documentation example
# apply a 5x5 convolution with 32 output filters on a 50x50 image:
model = Sequential()
model.add(Convolution2D(16, 3, 3, border_mode='valid', input_shape=(3, 50, 50), activation='relu'))

# add a 3x3 convolution on top, with 16 output filters:
#model.add(Convolution2D(16, 3, 3, border_mode='same', activation='relu'))

# and flatten it and add a dense layer to get it to match the 17 categories.
model.add(Flatten())
model.add(Dense(17, activation='relu'))
# Compile model
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])

model.summary()

model = model.fit(X_train, y_train,  batch_size = 256,
          nb_epoch = 5, verbose=2, validation_data=(X_test,y_test))

predicted = model.model.predict_classes(X_test)

# put the y_test back into a format of non-one-hot encoded for comparison
y_test_orig = pd.DataFrame(y_test).idxmax(1)
print "Convolution Network Results\nConfusion Matrix: "
print confusion_matrix(y_test_orig,predicted)
print "\n Classifcation Report"
print classification_report(y_test_orig,predicted)


I am working on the file "ComputerVisionPatterns-Copy1.ipynb".
The first task is:
With SciKitLearn's logistic regression algorithm, we get much better (but not good) precision than with the
previous methods (excluding convolutional neural networks), but worse recall.
Can you write Python code for it?

#Finally, we'll try the random forest, which is not known for being great with image data.
from sklearn.ensemble import RandomForestClassifier
from sklearn.grid_search import GridSearchCV

rfc = RandomForestClassifier(random_state=47, n_estimators=100)
rfc.fit(X_train_new, y_train_orig)

predicted = rfc.predict(X_test_new)

print "Decision Tree Results\nConfusion Matrix: "
print confusion_matrix(y_test_orig,predicted)
print "\n Classifcation Report"
print classification_report(y_test_orig,predicted)


I am working on the file "Computing Heterogeneity and clustering with KMeans++.ipynb".
The first task is:
Plotting graph K vs. Heterogeneity
Plot a graph between the K values (no. of clusters) and the heterogeneity of each model, <b>choose the elbow of the graph</b> to be the best value for k.
Can you write Python code for it?

k_list = [3, 10, 25, 50, 100]
heterogeneity_values = []
for k in k_list:
    model_kmeans= KMeans(n_clusters=k, init='k-means++', max_iter=100, n_init=1)
    model_kmeans.fit(tfidf_matrix)
    hetero= compute_heterogeneity(model_kmeans,tfidf_matrix)
    print hetero
    heterogeneity_values.append(hetero)
plot_k_vs_heterogeneity(k_list, heterogeneity_values)


I am working on the file "Computing Volumes.ipynb".
The first task is:

The above tetrahedron has a=2, b=2, c=5, for a volume of 20.  The remaining three lengths have not been computed as it's sufficient to know only a, b, c if the angles between them are those of the regular tetrahedron.
That's how IVM volume is computed:  multiply a * b * c from a regular tetrahedron corner, then "close the lid" to see the volume.
Can you write Python code for it?


I am working on the file "Computing Volumes.ipynb".
The first task is:
Allowing for floating point error, this space-filling right tetrahedron has a volume of 0.125 or 1/8. Since 24 of them form a cube, said cube has a volume of 3.  The XYZ volume, on the other hand, is what we'd expect from a regular tetrahedron of edges 0.5 in the current calibration system.
Can you write Python code for it?

regular = Tetrahedron(0.5, 0.5, 0.5, 0.5, 0.5, 0.5)
print("MITE volume in XYZ units:", round(regular.xyz_volume(),5))
print("XYZ volume of 24-Mite Cube:", round(24 * regular.xyz_volume(),5))


regular = Tetrahedron(0.5, 0.5, 0.5, 0.5, 0.5, 0.5)
print("MITE volume in XYZ units:", regular.xyz_volume())
print("XYZ volume of 24-Mite Cube:", 24 * regular.xyz_volume())


I am working on the file "Computing Volumes.ipynb".
The first task is:
Lets define a MITE, one of these 24 identical space-filling tetrahedrons, with reference to D=1, R=0.5, as this is how our Tetrahedron class is calibrated.  The cubes 12 edges will all be √2/2.
Edges 'a' 'b' 'c' fan out from the cube center, with 'b' going up to a face center, with 'a' and 'c' to adjacent ends of the face's edge.
From the cube's center to mid-face is √2/4 (half an edge), our 'b'. 'a' and 'c' are both half the cube's body diagonal of √(3/2)/2 or √(3/8).
Edges 'd', 'e' and 'f' define the facet opposite the cube's center.
'd' and 'e' are both half face diagonals or 0.5, whereas 'f' is a cube edge, √2/2.  This gives us our tetrahedron:
Can you write Python code for it?

b = sqrt(2)/4
a = c = sqrt(3/8)
d = e = 0.5
f = sqrt(2)/2
mite = Tetrahedron(a, b, c, d, e, f)
print("IVM volume of Mite:", mite.ivm_volume())
print("XYZ volume of Mite:", mite.xyz_volume())


I am working on the file "Computing Volumes.ipynb".
The first task is:
The MITE (minimum tetrahedron) further dissects into component modules, a left and right A module, then either a left or right B module.  Outwardly, the positive and negative MITEs look the same.  Here are some drawings from R. Buckminster Fuller's research, the chief popularizer of the A and B modules.

In a different Jupyter Notebook, we could run these tetrahedra through our volume computer to discover both As and Bs have a volume of 1/24 in IVM units.
Instead, lets take a look at the E-module and compute its volume.

The black hub is at the center of the RT, as shown here...

RT center is the black hub (Koski with vZome)

Can you write Python code for it?

from math import sqrt as rt2
from tetravolume import make_tet, Vector

ø = (rt2(5)+1)/2
e0 = Black_Yellow = rt2(3)*ø**-1
e1 = Black_Blue = 1
e3 = Yellow_Blue = (3 - rt2(5))/2
e6 = Black_Red = rt2((5 - rt2(5))/2)
e7 = Blue_Red = 1/ø

# E-mod is a right tetrahedron, so xyz is easy
v0 = Vector((Black_Blue, 0, 0))
v1 = Vector((Black_Blue, Yellow_Blue, 0))
v2 = Vector((Black_Blue, 0, Blue_Red))

# assumes R=0.5 so computed result is 8x needed
# volume, ergo divide by 8.
ivm, xyz = make_tet(v0,v1,v2)

print("IVM volume:", round(ivm/8, 5))
print("XYZ volume:", round(xyz/8, 5))


I am working on the file "Computing Volumes.ipynb".
The first task is:
The Python code below encodes a Tetrahedron type based solely on its six edge lengths.  The code makes no attempt to determine the consequent angles.
A complicated volume formula, mined from the history books and streamlined by mathematician Gerald de Jong, outputs the volume of said tetrahedron in both IVM and XYZ units.

The unittests that follow assure it's producing the expected results.  The formula bears great resemblance to the one by Piero della Francesca.
Can you write Python code for it?

from math import sqrt as rt2
from qrays import Qvector, Vector

R =0.5
D =1.0

S3    = pow(9/8, 0.5)
root2 = rt2(2)
root3 = rt2(3)
root5 = rt2(5)
root6 = rt2(6)
PHI = (1 + root5)/2.0

class Tetrahedron:
    """
    Takes six edges of tetrahedron with faces
    (a,b,d)(b,c,e)(c,a,f)(d,e,f) -- returns volume
    in ivm and xyz units
    """

    def __init__(self, a,b,c,d,e,f):
        self.a, self.a2 = a, a**2
        self.b, self.b2 = b, b**2
        self.c, self.c2 = c, c**2
        self.d, self.d2 = d, d**2
        self.e, self.e2 = e, e**2
        self.f, self.f2 = f, f**2

    def ivm_volume(self):
        ivmvol = ((self._addopen() - self._addclosed() - self._addopposite())/2) ** 0.5
        return ivmvol

    def xyz_volume(self):
        xyzvol = rt2(8/9) * self.ivm_volume()
        return xyzvol

    def _addopen(self):
        a2,b2,c2,d2,e2,f2 = self.a2, self.b2, self.c2, self.d2, self.e2, self.f2
        sumval = f2*a2*b2
        sumval +=  d2 * a2 * c2
        sumval +=  a2 * b2 * e2
        sumval +=  c2 * b2 * d2
        sumval +=  e2 * c2 * a2
        sumval +=  f2 * c2 * b2
        sumval +=  e2 * d2 * a2
        sumval +=  b2 * d2 * f2
        sumval +=  b2 * e2 * f2
        sumval +=  d2 * e2 * c2
        sumval +=  a2 * f2 * e2
        sumval +=  d2 * f2 * c2
        return sumval

    def _addclosed(self):
        a2,b2,c2,d2,e2,f2 = self.a2, self.b2, self.c2, self.d2, self.e2, self.f2
        sumval =   a2 * b2 * d2
        sumval +=  d2 * e2 * f2
        sumval +=  b2 * c2 * e2
        sumval +=  a2 * c2 * f2
        return sumval

    def _addopposite(self):
        a2,b2,c2,d2,e2,f2 = self.a2, self.b2, self.c2, self.d2, self.e2, self.f2
        sumval =  a2 * e2 * (a2 + e2)
        sumval += b2 * f2 * (b2 + f2)
        sumval += c2 * d2 * (c2 + d2)
        return sumval

def make_tet(v0,v1,v2):
    """
    three edges from any corner, remaining three edges computed
    """
    tet = Tetrahedron(v0.length(), v1.length(), v2.length(),
                      (v0-v1).length(),...


I am working on the file "Computing Volumes.ipynb".
The first task is:
The Python code below encodes a Tetrahedron type based solely on its six edge lengths.  The code makes no attempt to determine the consequent angles.
A complicated volume formula, mined from the history books and streamlined by mathematician Gerald de Jong, outputs the volume of said tetrahedron in both IVM and XYZ units.
The unittests that follow assure it's producing the expected results.
Can you write Python code for it?

from math import sqrt, hypot

class Tetrahedron:
    """
    Takes six edges of tetrahedron with faces
    (a,b,d)(b,c,e)(c,a,f)(d,e,f) -- returns volume
    in ivm and xyz units
    """

    def __init__(self, a,b,c,d,e,f):
        self.a, self.a2 = a, a**2
        self.b, self.b2 = b, b**2
        self.c, self.c2 = c, c**2
        self.d, self.d2 = d, d**2
        self.e, self.e2 = e, e**2
        self.f, self.f2 = f, f**2

    def ivm_volume(self):
        ivmvol = ((self._addopen() - self._addclosed() - self._addopposite())/2) ** 0.5
        return ivmvol

    def xyz_volume(self):
        xyzvol = sqrt(8/9) * self.ivm_volume()
        return xyzvol

    def _addopen(self):
        a2,b2,c2,d2,e2,f2 = self.a2, self.b2, self.c2, self.d2, self.e2, self.f2
        sumval = f2*a2*b2
        sumval +=  d2 * a2 * c2
        sumval +=  a2 * b2 * e2
        sumval +=  c2 * b2 * d2
        sumval +=  e2 * c2 * a2
        sumval +=  f2 * c2 * b2
        sumval +=  e2 * d2 * a2
        sumval +=  b2 * d2 * f2
        sumval +=  b2 * e2 * f2
        sumval +=  d2 * e2 * c2
        sumval +=  a2 * f2 * e2
        sumval +=  d2 * f2 * c2
        return sumval

    def _addclosed(self):
        a2,b2,c2,d2,e2,f2 = self.a2, self.b2, self.c2, self.d2, self.e2, self.f2
        sumval =   a2 * b2 * d2
        sumval +=  d2 * e2 * f2
        sumval +=  b2 * c2 * e2
        sumval +=  a2 * c2 * f2
        return sumval

    def _addopposite(self):
        a2,b2,c2,d2,e2,f2 = self.a2, self.b2, self.c2, self.d2, self.e2, self.f2
        sumval =  a2 * e2 * (a2 + e2)
        sumval += b2 * f2 * (b2 + f2)
        sumval += c2 * d2 * (c2 + d2)
        return sumval

PHI = sqrt(5)/2 + 0.5

R =0.5
D =1.0

tet = Tetrahedron(D, D, D, D, D, D)
print(tet.ivm_volume())

import unittest
class Test_Tetrahedron(unittest.TestCase):

    def test_unit_volume(self):
        tet = Tetrahedron(D, D, D, D, D, D)
        print(tet.ivm_volume())
        self.assertAlmostEqual(tet.ivm_volume(), 1.0, places=5)

    def test_unit_volume(self):
      ...


I am working on the file "Concatenating Basis Functions with Heterogeneous Data.ipynb".
The first task is:
(To no one's surprise, our model performs quite poorly, since zero effort was made to identify and incorporate the most informative features or appropriate basis functions. Rather, they were chosen solely to maximize clarity of exposition.)
Recap
So let's recap what we've done.

We started out with a dataset with 442 samples and 10 features, represented by 442x10 matrix X
For one reason or another, we wanted to use different basis functions for different subsets of features. Apparently, we wanted features 2 and 5 for one basis function and features 6, 8 and 9 for another. Therefore, we
sliced the matrix X to obtain 442 by 2 matrix X1 and
sliced the matrix X to obtain 442 by 3 matrix X2.
We
applied a polynomial basis function of degree 2 to X1 with 2 features and 442 samples. This returns a dataset X1_poly with $\begin{pmatrix} 4 \ 2 \end{pmatrix} = 6$ features and 442 samples. (NB: In general, the number of output features for a polynomial basis function of degree $d$ on $n$ features is the number of multisets of cardinality $d$, with elements taken from a finite set of cardinality $n+1$, which is given by the multiset coefficient $\begin{pmatrix} \begin{pmatrix} n + 1 \ d \end{pmatrix} \end{pmatrix} = \begin{pmatrix} n + d \ d \end{pmatrix}$.) So from 442 by 2 matrix X1 we obtain 442 by 6 matrix X1_poly
applied a radial basis function with 2 mean vectors $\mu_1 = \begin{pmatrix} 0 & 0.2 & 0.4 \end{pmatrix}^T$ and $\mu_2 = \begin{pmatrix} 0.6 & 0.8 & 1.0 \end{pmatrix}^T$, which is represented by the 2 by 3 matrix mu. From the 442 by 3 matrix X2, we obtain 442 by 2 matrix X2_radial
Next, we horizontally concatenated 442 by 6 matrix X1_poly with 442 by 2 matrix X2_radial to obtain the final 442 by 8 matrix X_concat
Finally, we fitted a linear model on X_concat.

So this is how we went from a 442x10 matrix X to the 442x8 matrix X_concat.
With Pipeline and Feature Union
First we define a transformer that slices up the input data. Note instead of working with (tuples of) slice objects, it is usually more...
Can you write Python code for it?

class ArraySlicer(BaseEstimator, TransformerMixin):

    def __init__(self, index_exp):
        self.index_exp = index_exp

    def fit(self, X, y=None):
        return self

    def transform(self, X, y=None):
        return X[self.index_exp]

model = \
make_pipeline(
    make_union(
        make_pipeline(
            ArraySlicer(np.index_exp[:, np.array([2, 5])]),
            PolynomialFeatures()
        ),
        make_pipeline(
            ArraySlicer(np.index_exp[:, np.array([6, 8, 9])]),
            RadialFeatures(mu)
        )
    )
)

model.fit(X)

model.transform(X).shape

# sanity check
model.transform(X)[:5, ::2].round(2)


I am working on the file "Concatenating Basis Functions with Heterogeneous Data.ipynb".
The first task is:
Assume for some reason we are interested in training a model using, say, features 2 and 5 with a polynomial basis, and features 6, 8 and 9 with a radial basis. We first slice up our original dataset.
Can you write Python code for it?

X1 = X[:, np.array([2, 5])]
X1.shape

# sanity check
X1[:5]

X2 = X[:, np.array([6, 8, 9])]
X2.shape

# sanity check
X2[:5]


I am working on the file "Concatenating Basis Functions with Heterogeneous Data.ipynb".
The first task is:
Feature Union with Heterogeneous Data Sources
Polynomial basis function
The polynomial basis function is provided by scikit-learn in the sklearn.preprocessing module.
Can you write Python code for it?

X = np.arange(1, 9).reshape(4, 2)
X

PolynomialFeatures(degree=2).fit_transform(X)


I am working on the file "Concatenating Basis Functions with Heterogeneous Data.ipynb".
The first task is:
If we run this on the original 442x10 dataset, we expect to get a dataset with the same number of samples and $\begin{pmatrix} 12 \ 2 \end{pmatrix} + 3 = 66 + 3 = 69$ features.
Can you write Python code for it?


I am working on the file "Concatenating Basis Functions with Heterogeneous Data.ipynb".
The first task is:
Now we apply the respective basis functions.
Polynomial
Can you write Python code for it?

X1_poly = PolynomialFeatures().fit_transform(X1)
X1_poly.shape

# sanity check
X1_poly[:5].round(2)


I am working on the file "Concatenating Basis Functions with Heterogeneous Data.ipynb".
The first task is:
Now we're ready to concatenate these augmented datasets.
Can you write Python code for it?

X_concat = np.hstack((X1_poly, X2_radial))
X_concat.shape

# sanity check
X_concat[:5, ::2].round(2)


I am working on the file "Concatenating Basis Functions with Heterogeneous Data.ipynb".
The first task is:
Putting it all together
The above union applies the basis functions on the entire dataset, but we're interested in applying different basis functions to different features. To do this, we can simply define a rather frivolous transformer that simply slices the input data, and that's exactly what ArraySlicer was for.
Can you write Python code for it?

model = \
make_pipeline(
    ArraySlicer(np.index_exp[:, np.array([2, 5])]),
    PolynomialFeatures()
)

model.fit(X)

model.transform(X).shape

# sanity check
model.transform(X)[:5].round(2)


I am working on the file "Concatenating Basis Functions with Heterogeneous Data.ipynb".
The first task is:
This effectively composes each of the steps we had to manually perform and amalgamated it into a single transformer. We can even append a regressor at the end to make it a complete estimator/predictor.
Can you write Python code for it?

model = \
make_pipeline(
    make_union(
        make_pipeline(
            ArraySlicer(np.index_exp[:, np.array([2, 5])]),
            PolynomialFeatures()
        ),
        make_pipeline(
            ArraySlicer(np.index_exp[:, np.array([6, 8, 9])]),
            RadialFeatures(mu)
        )
    ),
    LinearRegression()
)

model.fit(X, y)

model.score(X, y)


I am working on the file "Concatenating Basis Functions with Heterogeneous Data.ipynb".
The first task is:
Union
A union is a transformer that is initialized with an arbitrary number of transformers. When fit_transform is called on a dataset, it simply calls fit_transform of the transformers it was given and horizontally concatenates its results.
Can you write Python code for it?

mu_ = np.linspace(0, 10, 30).reshape(3, 10)

model = \
make_union(
    PolynomialFeatures(),
    RadialFeatures(mu_)
)


I am working on the file "Concordances.ipynb".
The first task is:
For this demonstration we will focus only on one file, the 2013 section of the corpus. As evidenced in the last exercise, <i>Adding Context to Word Frequency Counts</i>, there was a significant increase in the usage of the word privacy between 2012 and 2013, which amounted to an increase of about 40%. Here we will take a closer look at 2013 in an attempt to identify any patterns of word use
This is a case where cleaning the text may also destroy some of the context. While it is nice to have the numbers line up (in terms of word frequencies vs. number of concordance lines), removing the punctuation and capitalization makes the text harder to read and understand.
Can you write Python code for it?

#this is the path to the file we want to read
file = '../Counting Word Frequencies/data/2013.txt'

#this calls on a definition from above: it stores the filename as a variable to use later
name = get_filename(file)

#reads the file
text = read_file(file)
#splits the text into a list of individual words
words = text.split()
#assigns NLTK functionality to the text
text = nltk.Text(words)


I am working on the file "Concordances.ipynb".
The first task is:
The NLTK module is limited in the amount of processing it can conduct on concordances. It is more useful to output the entire concordance to a text file, which can then be sorted and manipulated in many ways. The following code prints the entire concordance to file. The '79' on line 8 refers to the total number of characters contained in each span, including all letters, punctuation and spaces.
Can you write Python code for it?

#creates a new file that can be written by the print queue
fileconcord = codecs.open(name + '_collocates.txt', 'w', 'utf-8')
#makes a copy of the empty print queue, so that we can return to it at the end of the function
tmpout = sys.stdout
#stores the text in the print queue
sys.stdout = fileconcord
#generates and prints the concordance, the number pertains to the total number of bytes per line
text.concordance("privacy", 79, sys.maxsize)
#closes the file
fileconcord.close()
#returns the print queue to an empty state
sys.stdout = tmpout


I am working on the file "Conda-tools.ipynb".
The first task is:
Neat stuff
Convenient access to the package cache and environment metadata allows you to do some neat stuff relatively easily.
Below are a few examples of some quick ideas that can be implemented with little effort.
Can you write Python code for it?

# Calculate potential collisions in environments by packages claiming the same file paths
# Very quick and naive way of detecting file path collisions.
for i, p1 in enumerate(pkg_cache):
    for p2 in pkg_cache[i+1:]:
        if p1.name == p2.name:
            continue
        x = p1.files.intersection(p2.files)
        if x:
            print("{} collides with {}".format(p1, p2))
            print("\tCollisions: ", x)

# Cache Utils has some higher order, convenience functions

# See what environments a package is linked into
# Note that this is a O(n) operation where n is the sum of the installed packages in each environment you're checking.
# If you're running this for the first time, it has to read all the metadata for each environment.
# Also note, that this creates new package info objects and environment objects each run, so each run
# prompts a full scan of both the package cache and all environments.
cu.linked_environments((pkg_cache[0],), envs)

# Find which environments the latest packages are linked to.
# This example uses Versio to parse and compare PEP440 compliant version numbers
# This will exclude packages like packages like jpeg and openssl

# This loop simple creates Version objects so we can compare them later.
Versions = {}
for x in pkg_cache:
    try:
        if x.name in Versions:
            Versions[x.name].append(Version(x.version))
        else:
            Versions[x.name] = [Version(x.version)]
    except:
        print("Skipping ", x.name, x.version)

# sort the value lists and pick the latest versions
#pversions = {k: str(list(sorted(v))[-1]) for k, v in Versions.items()}

# sort the value lists and pick the older versions
pversions = {k: list(map(str, list(sorted(v))[:-1])) for k, v in Versions.items()}

# The most up-to-date packages are linked to which environments?
#latest_pkgs = [x for x in pkg_cache if x.name in pversions and x.version == pversions[x.name]]

# Find the environments that older packages are linked to
latest_pkgs = [x for x in pkg_cache if x.name in...


I am working on the file "Conda-tools.ipynb".
The first task is:
Packages
Conda packages all have an info/ subdirectory for storing metadata about the package.  PackageInfo provide convenient access to this metadata.
Can you write Python code for it?

pi = pkg_cache[0]
pi.index  # info/index.json

# We can access fields of index.json directly from the object.
pi.name, pi.version, pi.build

# Access to info/files
pi.files

# The full spec of the package.  This is always "name-version-build"
pi.full_spec

# We can queries against the information we have on packages
# For example, I want to find all MIT licensed packages in the cache
{pi.full_spec: pi.license for pi in pkg_cache if pi.license == 'MIT'}


I am working on the file "Conda-tools.ipynb".
The first task is:
The two core components of the conda ecosystem are the package cache and the environment subfolders.  These are abstracted with PackageInfo and Environment objects respectively.
Here we create "pools" of PackageInfo and Environment objects.  These objects permit easy, read-only access to various bits of metadata stored in the package cache and conda-meta/ subfolders in the environment.  We want to reuse the objects as much as we can to minimize disk I/O.  All the disk reads are currently cached with the objects, so the more objects you work with, the more RAM will be required.
Can you write Python code for it?

# Create pkg_cache and environments
pkg_cache = cache.packages(root_pkgs)
envs = environment.environments(root_envs)
print(pkg_cache[:5])
print()
print(envs[:5])


I am working on the file "Condition&Loop.ipynb".
The first task is:
for
이터레이터가 내부적으로 생김
for [loop_var] in [sequence]: 로 쓰임
sequence는 리스트, 문자열, 튜플, 딕셔너리, 셋 등 순회 가능한 객체
Can you write Python code for it?

a = [1, 2, 3, 4, 5]

for i in a:
    print i

nums = [1, 3, 2, 5, 4, 6, 7, 8, 9]
for val in nums:
    print val,
print
for i in nums:
    print i,
print
for num in nums:
    print num,

str = 'hello world'
for char in str:
    print char

capitals = {'korea' : 'seoul', 'japan' : 'tokyo', 'usa' : 'Washington D.C'}

for country in capitals:
    print country, capitals[country]
print '-' * 20
for key in capitals.keys():
    print key
print '-' * 20
for val in capitals.values():
    print val
print '-' * 20
for k, v in capitals.items():
    print k, v

nums = [1, 3, 2, 5, 4, 6, 7, 9, 8]

for val in nums:
    print val,

for i, _ in enumerate(nums):
    print 'value at index', i

a = [1, 2, 3, 4]
b = [2, 3, 4]

for i in a:
    for j in b:
        print i, j


I am working on the file "Condition&Loop.ipynb".
The first task is:
if & for 연습문제

1 - 100 까지 정수 중 2의 배수 또는 11의 배수를 모두 출력
a = [22, 1, 3, 4, 7, 98, 21, 55, 87, 99, 19, 20, 45]의 최대값과 최소값을 구하라
a = [22, 1, 3, 4, 7, 98, 21, 55, 87, 99, 19, 20, 45]의 평균을 구하라
Can you write Python code for it?


I am working on the file "Condition&Loop.ipynb".
The first task is:
practice

Celsius = [39.2, 36.5, 37.3, 37.8] F = (9/5) * C + 32 일 때, 리스트의 각 값을 화씨로 바꾼 리스트를 만드시오.
주어진 문자열에서 띄어쓰기의 개수는? 'Today is very nice and I want to go out for dinner'
주어진 문자열에서 모음을 모두 제거하시오. 'Today is very nice and I want to go out for dinner'
Can you write Python code for it?

Celsius = [39.2, 36.5, 37.3, 37.8]

F = [1.8 * c + 32 for c in Celsius]

print F

str1 = 'Today is very nice and I want to go out for dinner'

temp = [ch for ch in str1 if ch == ' ']
print len(temp)

str1 = 'Today is very nice and I want to go out for dinner'

vowels = 'aeiou'

temp = [ch for ch in str1 if not ch in vowels]
print ''.join(temp)


I am working on the file "Conditional+Statements.ipynb".
The first task is:
Break Statement
The break statement terminates the loop containing it. The control of the program flows to the statement immediately below the loop.
Can you write Python code for it?

for val in "string":
    if val == "i":
        break
    print(val)

print("The End")


I am working on the file "Conditional+Statements.ipynb".
The first task is:
Continue Statement
The continue statement is used to skip the rest of the code inside a loop for the current iteration only. Loop does not terminate but continues on with next iteration.
Can you write Python code for it?

for val in "string":
    if val == "i":
        continue
    print(val)

print("End Loop")


I am working on the file "Conditional+Statements.ipynb".
The first task is:
Nested if
Can you write Python code for it?

x = 8
if x == 0:
    print(x,"is zero")
elif x > 0:
    if x == 10:
        print(x,"is equal to 10")
    elif x > 10:
        print(x,"is greater than 10")
    else:
        print(x,"is lesser than 10")
else:
    print(x,"is negative")

print('End of if statement')


I am working on the file "Conditional+Statements.ipynb".
The first task is:
Pass Statement
Pass statement is generally used as a placeholder for loop or function or classes as they cannot have empty body. So "Pass" is used to construct the body that does nothing.
The difference between comment (#) and "pass" is that python interpreter ignores the comments while it treat "pass" as no operation.
Can you write Python code for it?

def dummy():
    pass

dummy()

for i in range(3):
    pass


I am working on the file "Conditional+Statements.ipynb".
The first task is:
Pls note the indentation is important in Python. All the statements that is part of if statement must be indented.
Can you write Python code for it?

x = 1
if x > 0:
    print(x,"is positive number")
print('Outside of if condition')


I am working on the file "Conditional+Statements.ipynb".
The first task is:
While
While loop is used to iterate over a block of code as long as the test expression is true. Please note in python any non-zero value is True and zero is interpreted as False.
Mainly used when we do not know how many times to iterate
python
while test_expression:
    # body of while
Can you write Python code for it?

sum = 0
i = 1
while i <= 10:
    sum += c
    i += 1

print(sum)

n = 5
sum = 0
i = 1
while i <= n:
    sum += i
    i += 1

print(sum)


I am working on the file "Conditional+Statements.ipynb".
The first task is:
for loop with else
Can you write Python code for it?

digits = [0,1,5]

for i in digits:
    print(i)
else:
    print('No more digits to print')


I am working on the file "Conditional+Statements.ipynb".
The first task is:
if..elif..else statement
python
if test expression:
    # body of if
elif test expression:
    # body of elif
else
    # body of else
Can you write Python code for it?

x = -5
if x == 0:
    print(x,"is zero")
elif x > 0:
    print(x,"is positive")
else:
    print(x,"is negative")

print('End of if statement')


I am working on the file "Conditional+Statements.ipynb".
The first task is:
if..else.. statement
python
if test expression:
    # body of if
else:
    # body of else
Can you write Python code for it?

x = 1
if x >= 0:
    print(x,"is positive number")
else:
    print(x,"is negative number")

print('End of if..else..')


I am working on the file "ConditionalProbabilities.ipynb".
The first task is:
Notice that we have generated samples of whether the song has lyrics or not. Above I have also printed the associated genre label. In many probabilistic modeling problems some information is not available to the observer. For example we could be provided only the yes/no outcomes and the genres could be "hidden".
Now let's use these generated samples to estimate probabilities of the model. Basically we pretend that we don't know the parameters and estimate them directly by frequency counting through the samples we generated.
Can you write Python code for it?

# First only consider jazz samples
jazz_samples = [x for x in random_lyrics_samples if x[1] == 'jazz']
for s in jazz_samples[0:20]:
    print(s)


I am working on the file "ConditionalProbabilities.ipynb".
The first task is:
Now that we have selected the samples that are jazz we can simply count the lyrics yes and lyrics no entries and divide them by the total number of jazz samples to get estimates of the conditional probabilities. Think about the relationships: we can use the data to estimate the parameters of a model (learning), we can use the model to generate samples (generation), and we can use the model to calculate probabilities for various events (inference).
Can you write Python code for it?

est_no_if_jazz = len([x for x in jazz_samples if x[0] == 'no']) / len(jazz_samples)
est_yes_if_jazz = len([x for x in jazz_samples if x[0] == 'yes']) / len(jazz_samples)
print(est_no_if_jazz, est_yes_if_jazz)


I am working on the file "ConditionalProbabilities.ipynb".
The first task is:
We have seen in the slides that the probability of a song being jazz if we know that it is instrumental is 0.66.
\begin{equation}
P(genre = jazz | hasLyrics = no) = \frac{0.3 * 0.9}{0.3 * 0.9 + 0.7 * 0.2} = 0.66
\end{equation}
This is based on our knowledge of probabilities. If we have some data we can also estimate this probability directly. This is called approximate inference in contrast to the exact inference of $0.66$. When problems become complicated exact inference can become too costly to compute while approximate inference can provide reasonable answers much faster. We will see that later when examining probabilistic graphical models. As you can see in this case both the exact and approximate inference probability estimates are relatively close.
Can you write Python code for it?

no_samples = [x for x in random_lyrics_samples if x[0] == 'no']
est_jazz_if_no_lyrics = len([x for x in no_samples if x[1] == 'jazz']) / len(no_samples)
print(est_jazz_if_no_lyrics)


I am working on the file "Conference Talks Rankings.ipynb".
The first task is:

Community Voting Results
Absolute Ranking
Can you write Python code for it?

rankings = pd.read_csv('./ranking_pyconx.txt', sep='~', skiprows=1, header=0)
rankings.drop('TID', axis=1, inplace=True)
rankings.drop('Duration', axis=1, inplace=True)

union_jack = '\U0001F1EC\U0001F1E7'
tricolore = '\U0001F1EE\U0001F1F9'
female = '\U0001f469\u200d\U0001f3a4'
male = '\U0001f468\u200d\U0001f3a4'

rankings.Gender = rankings.Gender.apply(lambda g: '+'.join(female if l=='female' else male
                                         for l in g.split('+')))
rankings.Lang = rankings.Lang.apply(lambda l: union_jack if l=='en' else tricolore)
pd.set_option('display.max_rows', rankings.index.size)
pd.set_option('display.max_colwidth', 300)

rankings.index = np.arange(1, rankings.index.size+1)

HTML(rankings.to_html())

trainings = rankings[rankings['Type'].values == 'Training']
talks = rankings[rankings['Type'].values == 'Talk']

# Drop cols
trainings = trainings.drop('Type', axis=1)
talks = talks.drop('Type', axis=1)


I am working on the file "Conference Talks Rankings.ipynb".
The first task is:

Track PyDatabase
Can you write Python code for it?

pydb = talks[talks['Track'].values == 'PyDatabase']
pydb.index = np.arange(1, pydb.index.size+1)
HTML(pydb.to_html())

print('Number of Talks Proposed: ', pydb.index.size, end='\n\n')

print('Language of Talks Proposed: ')
print(pydb.Lang.value_counts(), end='\n\n')

print('Levels of Talks Proposed: ')
print(pydb.Level.value_counts())


I am working on the file "Conference Talks Rankings.ipynb".
The first task is:

Track Python & Friends
Can you write Python code for it?

pylang = talks[talks['Track'].values == 'Python & Friends']
pylang.index = np.arange(1, pylang.index.size+1)
HTML(pylang.to_html())

print('Number of Talks Proposed: ', pylang.index.size, end='\n\n')

print('Language of Talks Proposed: ')
print(pylang.Lang.value_counts(), end='\n\n')

print('Levels of Talks Proposed: ')
print(pylang.Level.value_counts())


I am working on the file "Conference Talks Rankings.ipynb".
The first task is:

Track PyWeb & DevOps
Can you write Python code for it?

pyweb = talks[talks['Track'].values == 'PyWeb & DevOps']
pyweb.index = np.arange(1, pyweb.index.size+1)
HTML(pyweb.to_html())

print('Number of Talks Proposed: ', pyweb.index.size, end='\n\n')

print('Language of Talks Proposed: ')
print(pyweb.Lang.value_counts(), end='\n\n')

print('Levels of Talks Proposed: ')
print(pyweb.Level.value_counts())


I am working on the file "Conference Talks Rankings.ipynb".
The first task is:

Stats
Speakers Stats
Can you write Python code for it?

from collections import defaultdict
speakers = defaultdict(set)

for i, speaker in enumerate(rankings.Speakers.values):
    for j, name in enumerate(speaker.split(',')):
        gender = rankings.Gender[i+1].split('+')[j]
        speakers[gender].add(name)

count_speakers = sum(len(s) for s in speakers.values())
print(count_speakers)

for k in speakers:
    print('{} --> {}'.format(k, len(speakers[k])))


I am working on the file "Conference Talks Rankings.ipynb".
The first task is:

Talks Ranking (per single Track)

Can you write Python code for it?

pydataers = talks[talks['Track'].values == 'PyData']
pydataers.index = np.arange(1, pydataers.index.size+1)
HTML(pydataers.to_html())

print('Number of Talks Proposed: ', pydataers.index.size, end='\n\n')

print('Language of Talks Proposed: ')
print(pydataers.Lang.value_counts(), end='\n\n')

print('Levels of Talks Proposed: ')
print(pydataers.Level.value_counts())


I am working on the file "Conference Talks Rankings.ipynb".
The first task is:

Trainings Ranking
Can you write Python code for it?

trainings.index = np.arange(1, trainings.index.size+1)
HTML(trainings.to_html())


I am working on the file "Conference Talks Rankings.ipynb".
The first task is:
Talks Stats
Can you write Python code for it?

print('Talks --> {}'.format(talks.index.size))
print('Trainings --> {}'.format(trainings.index.size))


I am working on the file "Confidence Intervals - bootstrap.ipynb".
The first task is:
Another variant is ABC, or Approximate Bootstrap Confidence.  This attempts to approximate what bootstrapping would do without actually bootstrapping.  I have no idea how it works and what assumptions are being made, but it can be handy for getting quick estimates.  Again, I'm using the implementation from https://github.com/cgevans/scikits-bootstrap which doesn't quite seem to be as fast as it could be, so I'm not seeing much speedup over standard boostrap, but that might be able to be improved.
Can you write Python code for it?

import scikits.bootstrap

def test_inside_bounds(n_samples):
    data = rng.normal(size=n_samples)
    ci = scikits.bootstrap.ci(data, np.average, method='abc', n_samples=3000)
    if ci[0] > 0 or ci[1] < 0:
        return 1, ci
    else:
        return 0, ci

Ss = [5, 10, 15, 20, 25, 30, 35, 40]
rs = []
mins = []
maxs = []
for S in Ss:
    results = [test_inside_bounds(n_samples=S) for i in range(1000)]
    rs.append(np.mean([r[0] for r in results]))
    mins.append(np.mean([r[1][0] for r in results]))
    maxs.append(np.mean([r[1][1] for r in results]))
    print S, rs[-1]

pylab.figure()
pylab.title('Normal distribution')
pylab.plot(Ss, rs)
pylab.xlabel('number of samples')
pylab.ylabel('probability of error')
pylab.figure()
pylab.fill_between(Ss, mins, maxs, color='#888888')
pylab.xlabel('number of samples')
pylab.ylabel('confidence interval')
pylab.show()


I am working on the file "Confidence Intervals - bootstrap.ipynb".
The first task is:
Doesn't seem to depend on the distribution, which is good (because in a real situation, we don't know what the underlying distribution is!
Why doesn't it rely on the distribution?  And how does it change with the number of samples?
Can you write Python code for it?

def test_inside_bounds(n_samples):
    data = rng.normal(size=n_samples)
    if np.min(data) > 0 or np.max(data) < 0:
        return 1
    else:
        return 0

Ss = [2, 3, 4, 5, 6, 7, 8, 9, 10]
rs = []
for S in Ss:
    results = [test_inside_bounds(n_samples=S) for i in range(10000)]
    rs.append(np.mean(results))
    print S, rs[-1]

pylab.plot(Ss, rs)
pylab.xlabel('number of samples')
pylab.ylabel('probability of error')
pylab.show()


I am working on the file "Confidence Intervals - bootstrap.ipynb".
The first task is:
For this simple example, where do you think the actual mean is on this graph?  Note: this is just meant to be a typical example and I'm not doing anything tricky here like searching for a random number seed that gives horrible results (seed=0).
What do you think the chances are that the mean is above the largest value or below the smallest value?
Can you write Python code for it?

rng = np.random.RandomState(seed=0)

data = rng.normal(size=3)
pylab.scatter(np.zeros_like(data), data, marker='x', s=100, c='k')
pylab.xticks([])
pylab.show()


I am working on the file "Confidence Intervals - bootstrap.ipynb".
The first task is:
General rule of thumb: Need about 20 samples to do a 95% confidence interval
Why?  What would the rule of thumb be for an 80% confidence interval?
Can you write Python code for it?

def needed_samples(p=0.95):
    return 1.0 / (1-p)

print 0.95, needed_samples(p=0.95)
print 0.8, needed_samples(p=0.8)


I am working on the file "Confidence Intervals - bootstrap.ipynb".
The first task is:
It's not too bad for other distributions, unless they're really pathological.  Still, it doesn't get the expected 5% error unless all the assumptions hold: normally distributed data and enough samples (~50 to be safe).
Other statistics
Oh, and one other assumption: it only works for computing the mean.
What if I want a confidence interval on my standard deviation?  Or on the median?  Or on the regression coefficient?  Or something more complicated than that?
There are analytical results for a few of these, but we'd really like something that works for whatever statistic I want to do.  This is especially the case for situations where we're looking at human data and it's the distribution of performance that matters, not just the mean.
Bootstrap Confidence Intervals
The term "bootstrap" refers to a whole family of algorithms that involve re-sampling from the samples you already have to approximate the underlying distribution.
The core idea for a bootstrap confidence interval is to simulate the process of running your experiment an infinite number of times.  However, we can't actually re-run the experiment.  Instead, what we do is to take your sample data and pretend that is the exact distribution of results.  So we can simulate re-running the experiment by sampling from the results we already have.
In other words, if I have 20 data points, I can estimate what would happen if I re-ran that full experiment once by just sampling 20 values from those 20 data points.  This, of course, has to be sampling with replacement, otherwise I'd just get the original data back again.  So I can repeat the experiment and do that same measure on it (say, compute the mean).
Just doing it once doesn't tell me what would happen if I did it an infinite number of times.  So let's do it over and over again.  The general number is 3000 (which is pretty close to infinity).  Now you have 3000 computations of the mean.  To determine your range, sort them, throw out the bottom 2.5% and the top 2.5%, and you're left with the 95% confidence interval.
Can you write Python code for it?

def bootstrap_ci(data, func, n=3000, p=0.95):
    index = int(n*(1-p)/2)

    samples = np.random.choice(data, size=(n, len(data)))
    try:
        r = func(samples, axis=1)  # if the function supports axis
    except TypeError:
        r = [func(s) for s in samples]  # otherwise do it the slow way
    r.sort()
    return r[index], r[-index]

def test_inside_bounds(n_samples):
    data = rng.normal(size=n_samples)
    ci = bootstrap_ci(data, np.mean)
    if ci[0] > 0 or ci[1] < 0:
        return 1, ci
    else:
        return 0, ci

Ss = [5, 10, 15, 20, 25, 30, 35, 40]
rs = []
mins = []
maxs = []
for S in Ss:
    results = [test_inside_bounds(n_samples=S) for i in range(1000)]
    rs.append(np.mean([r[0] for r in results]))
    mins.append(np.mean([r[1][0] for r in results]))
    maxs.append(np.mean([r[1][1] for r in results]))
    print S, rs[-1]

pylab.figure()
pylab.title('Normal distribution')
pylab.plot(Ss, rs)
pylab.xlabel('number of samples')
pylab.ylabel('probability of error')
pylab.figure()
pylab.fill_between(Ss, mins, maxs, color='#888888')
pylab.xlabel('number of samples')
pylab.ylabel('confidence interval')
pylab.show()

def test_inside_bounds(n_samples):
    data = rng.binomial(1, p=0.5, size=n_samples)-0.5
    ci = bootstrap_ci(data, np.mean)
    if ci[0] > 0 or ci[1] < 0:
        return 1, ci
    else:
        return 0, ci

Ss = [5, 10, 15, 20, 25, 30, 35, 40]
rs = []
mins = []
maxs = []
for S in Ss:
    results = [test_inside_bounds(n_samples=S) for i in range(10000)]
    rs.append(np.mean([r[0] for r in results]))
    mins.append(np.mean([r[1][0] for r in results]))
    maxs.append(np.mean([r[1][1] for r in results]))
    print S, rs[-1]

pylab.figure()
pylab.title('1-sample binomial distribution')
pylab.plot(Ss, rs)
pylab.xlabel('number of samples')
pylab.ylabel('probability of error')
pylab.figure()
pylab.fill_between(Ss, mins, maxs, color='#888888')
pylab.xlabel('number of samples')
pylab.ylabel('confidence interval')
pylab.show()

def...


I am working on the file "Confidence Intervals - bootstrap.ipynb".
The first task is:
Let's say we have some model of some task and we want to know how well it's doing.  Maybe we're about to change some parameter and want to to know whether changing that parameter makes the model works better.  But every time we run our model we get a slightly different result, due to random variation.
Instead of running the model once, we might run the model 3 times and take the average.  But what does that actually tell us?  It tells us the average of those 3 runs.  We don't care about those particular 3 runs: we care about what the actual average is: i.e. if we ran it an infinite number of times and took the average.  But that might take too long.
The goal of this notebook is to deal with exactly this problem: what can you conclude about the underlying distribution, given only samples of that distribution.
An example sampling
Here are 3 samples from a normal distribution.  Since it's the normal distribution, we know that the actual mean if we did an infinite number of samples is 0.
Can you write Python code for it?

rng = np.random.RandomState(seed=0)

data = rng.normal(size=3)

pylab.scatter(np.zeros_like(data), data, marker='x', s=100, c='k')
pylab.xticks([])
pylab.yticks([])
pylab.show()


I am working on the file "Confidence Intervals - bootstrap.ipynb".
The first task is:
Okay, that's better than before, but it's still a bit problematic.
Why is it wrong more than 5% of the time?
We're fine on the assumption that it's normally distributed data...
But there's another assumption: that you have enough samples.  What is enough?
Can you write Python code for it?

def test_inside_bounds(n_samples):
    data = rng.normal(size=n_samples)
    ci = normal_ci(data)
    if ci[0] > 0 or ci[1] < 0:
        return 1, ci
    else:
        return 0, ci

Ss = [5, 10, 15, 20, 25, 30, 35, 40]
rs = []
mins = []
maxs = []
for S in Ss:
    results = [test_inside_bounds(n_samples=S) for i in range(10000)]
    rs.append(np.mean([r[0] for r in results]))
    mins.append(np.mean([r[1][0] for r in results]))
    maxs.append(np.mean([r[1][1] for r in results]))
    print S, rs[-1]

pylab.figure()
pylab.title('Normal distribution')
pylab.plot(Ss, rs)
pylab.xlabel('number of samples')
pylab.ylabel('probability of error')
pylab.figure()
pylab.fill_between(Ss, mins, maxs, color='#888888')
pylab.xlabel('number of samples')
pylab.ylabel('confidence interval')
pylab.show()


I am working on the file "Confidence Intervals - bootstrap.ipynb".
The first task is:
Ouch.  Even without trying to find an example where the sampling is horribly off, we have one.  If I was working with a model and I made a change and the average used to be zero, and now it's this, then I would have convinced myself that I made an improvement, when I actually haven't.
Let's see how common this is.
Can you write Python code for it?

def test():
    data = rng.normal(size=3)
    if np.min(data) > 0 or np.max(data) < 0:
        return 1
    else:
        return 0

results = [test() for i in range(100000)]
print np.mean(results)


I am working on the file "Confidence Intervals - bootstrap.ipynb".
The first task is:
So this says that the mean is somewhere between -0.34 and 2.44, which is correct, as the mean is actually 0!
What happens with more samples?
Can you write Python code for it?

def test_inside_bounds(n_samples):
    data = rng.normal(size=n_samples)
    ci = normal_ci(data)
    if ci[0] > 0 or ci[1] < 0:
        return 1, ci
    else:
        return 0, ci

Ss = [2, 3, 4, 5, 6, 7, 8, 9, 10]
rs = []
mins = []
maxs = []
for S in Ss:
    results = [test_inside_bounds(n_samples=S) for i in range(10000)]
    rs.append(np.mean([r[0] for r in results]))
    mins.append(np.mean([r[1][0] for r in results]))
    maxs.append(np.mean([r[1][1] for r in results]))
    print S, rs[-1]

pylab.figure()
pylab.plot(Ss, rs)
pylab.xlabel('number of samples')
pylab.ylabel('probability of error')
pylab.figure()
pylab.fill_between(Ss, mins, maxs, color='#888888')
pylab.xlabel('number of samples')
pylab.ylabel('confidence interval')
pylab.show()


I am working on the file "Confidence Intervals - bootstrap.ipynb".
The first task is:
The idea here is that you really should have enough samples that you should expect one to be outside the range.  If you don't have any, then it's very hard to figure out where the range should be.
Other distributions
But the algorithm described only works if the data is normally distributed.  What happens if it isn't?
Uniform distribution
Can you write Python code for it?

def test_inside_bounds(n_samples):
    data = rng.uniform(-1,1, size=n_samples)
    ci = normal_ci(data)
    if ci[0] > 0 or ci[1] < 0:
        return 1, ci
    else:
        return 0, ci

Ss = [5, 10, 15, 20, 25, 30, 35, 40]
rs = []
mins = []
maxs = []
for S in Ss:
    results = [test_inside_bounds(n_samples=S) for i in range(10000)]
    rs.append(np.mean([r[0] for r in results]))
    mins.append(np.mean([r[1][0] for r in results]))
    maxs.append(np.mean([r[1][1] for r in results]))
    print S, rs[-1]

pylab.figure()
pylab.title('Uniform distribution')
pylab.plot(Ss, rs)
pylab.xlabel('number of samples')
pylab.ylabel('probability of error')
pylab.figure()
pylab.fill_between(Ss, mins, maxs, color='#888888')
pylab.xlabel('number of samples')
pylab.ylabel('confidence interval')
pylab.show()

def test_inside_bounds(n_samples):
    data = rng.binomial(1, p=0.5, size=n_samples)-0.5
    ci = normal_ci(data)
    if ci[0] > 0 or ci[1] < 0:
        return 1, ci
    else:
        return 0, ci

Ss = [5, 10, 15, 20, 25, 30, 35, 40]
rs = []
mins = []
maxs = []
for S in Ss:
    results = [test_inside_bounds(n_samples=S) for i in range(10000)]
    rs.append(np.mean([r[0] for r in results]))
    mins.append(np.mean([r[1][0] for r in results]))
    maxs.append(np.mean([r[1][1] for r in results]))
    print S, rs[-1]

pylab.figure()
pylab.title('1-sample binomial distribution')
pylab.plot(Ss, rs)
pylab.xlabel('number of samples')
pylab.ylabel('probability of error')
pylab.figure()
pylab.fill_between(Ss, mins, maxs, color='#888888')
pylab.xlabel('number of samples')
pylab.ylabel('confidence interval')
pylab.show()

def test_inside_bounds(n_samples):
    data = rng.gamma(0.1, size=n_samples)-0.1
    ci = normal_ci(data)
    if ci[0] > 0 or ci[1] < 0:
        return 1, ci
    else:
        return 0, ci

Ss = [5, 10, 15, 20, 25, 30, 35, 40]
rs = []
mins = []
maxs = []
for S in Ss:
    results = [test_inside_bounds(n_samples=S) for i in range(10000)]
    rs.append(np.mean([r[0] for r in...


I am working on the file "Confidence Intervals - bootstrap.ipynb".
The first task is:
What could we do instead?
Confidence Intervals
In your stats class, they dealt with this problem by computing a "confidence interval".
A 95% confidence interval says that "If you assume the actual value for whatever distribution you are sampling from is inside this range, then you will only be wrong 5% of the time".
Note that this does not tell you whether you're wrong in this particular case.  There's no way to know that.  But on average, this should give you a range that has the right answer in it 95% of the time.
If the assumptions hold.   This is a big IF.
The main one they covered in your stats class was IF the data is normally distributed, then the confidence interval is $\bar x \pm {s \over \sqrt N}t_{N-1, 95\%}$ where:
- $\bar x$ is the sample mean
- $s$ is the sample standard deviation
- $N$ is the number of samples
- $t_{N-1, 95\%}$ is the magic scaling factor that we look up on the table for Student's t-distribution.
Here it is computed for the samples we started with
Can you write Python code for it?

import scipy.stats

def normal_ci(data, p=0.95):
    t = np.array(scipy.stats.t.interval(0.95, len(data)-1))
    ci = np.mean(data)+np.std(data)/np.sqrt(len(data))*t
    return ci
print data
print normal_ci(data)


I am working on the file "Confidence_Interval.ipynb".
The first task is:
And we have solved our run time problems; yeah!
Back to Contents
Poisson Distribution
Now, that expression, $(1-p)^{n-m}$, has a huge exponent. Maybe, there's even more run time to be saved? Let's find an approximation for it and see if that helps.
Consider, for $p$ very small:
$$(1 - p)^{n-m} = \frac{(1 - p)^n}{(1 - p)^m} \approx (1 - p)^n \approx e^{-np}$$
(Recall that:
$$e^x = \sum_{k=0}^\infty\frac{x^k}{k!}$$
$$e^{-x} = 1 - x + \frac{x^2}{2} - \frac{x^3}{6} + \cdots \approx 1 - x \quad \text{, for x very small.)}$$
Let's work this further approximation into our probability expression:
$$P(m) = \frac{(np)^m}{m!}e^{-np}$$
We have arrived at the well known Poisson Distribution, which is used almost exclusively in the serial communication industry to estimate link BER.
Let's code this up and see if it works...
Can you write Python code for it?

from numpy import exp

def prob3(m):
    'Uses a further approximation to eliminate large exponential term.'

    return pow(kNumBits * kBER, m) * exp(-kNumBits * kBER) / factorial(m)

for m in range(3):
    print "P(%d):" % m, prob3(m)


I am working on the file "Confidence_Interval.ipynb".
The first task is:
Another interpretation of this is that longer observation periods yield more precise measures of system BER.
Back to Contents
Expectation
Now, let's go back a few steps and consider what our expectation is, regarding the number of errors we observe in any given observation period. This has a very simple and precise mathematical defintion:
$$
E[x] = np_x
$$
where $E[x]$ is the expected number of observed errors, $n$ is the number of bits observed, and $p_x$ is the probability that any bit is erroneous. If we tabulate this expected number of observed errors against the actual number of observed errors, for the 5 different cases above, we get:
N |E[x]|N / E[x]
:-:|:--:|:------:
 0| 5|0
 1| 7|0.14
 2| 9|0.22
 5|14|0.36
10|21|0.48
What we find is that, if we are willing to observe the link for longer periods of time, then the actual number of errors we observe can come closer to our expected number of errors while still providing the same level of confidence in our BER estimate.
Back to Contents
An alternative library function for calculating the Poisson distribution.
Now, what about an extreme case? The following data  were taken from an actual system running in a lab for 18 hours at a line rate of 7.8 Gbps:

Bits observed: 527.1e12
Errors observed: 373

What can we infer about the BER of this system? Let's find out...
Can you write Python code for it?

cis = []
for ber in bers:
    cis.append(conf_int(373, 527.1e12, ber))
plt.plot(bers, cis)
plt.title("Confidence Interval vs. BER Estimate")
plt.xlabel("BER Estimate")
plt.ylabel("Confidence Interval")


I am working on the file "Confidence_Interval.ipynb".
The first task is:
At first glance, we see what we expected: If we observe more errors, then we must observe the link for a longer time, before claiming a BER better than $10^{-12}$ with confidence greater than 99%. But, notice something else, too: If we increase the number of observed errors by a factor of 10 (i.e. - going from the "1 errors" to the "10 errors" case in the plot, above), we only need to increase our observation time by, roughly, a factor of 3 (i.e. - from 6 normalized time units to approximately 19), in order to quote our BER estimate with the same confidence!
This is rather remarkable and merrits further investigation. It seems to suggest that observations involving more errors are somehow more precise than those involving fewer errors. Can we develop a stronger, and more quantitative, sense of this, by approaching the problem from a slightly different direction? Let's try...
Consider what our options are, after observing our link run error free for $5 * 10^{12}$ bits. We know we can claim a BER better than $10^{-12}$ with confidence greater than 99%. However, there are, actually, an infinite number of other claims we can make, all of which are mathematically valid. Here are just a few:
Can you write Python code for it?

bers = [1e-11, 1e-12, 1e-13]
for ber in bers:
    print "BER better than {} with confidence greater than {}".format(ber, conf_int(0, 5 * kNumBits, ber))


I am working on the file "Confidence_Interval.ipynb".
The first task is:
Contents

Binomial Distribution
The Problem with Factorials
Approximating the Binomial Distribution
Poisson Distribution
Confidence in Estimation
Measurement Precision
Expectation
Alternative Poisson Distribution Calculator
Conclusion

Binomial Distribution
If we have a set of events, each having the same two possible outcomes (say, heads and tails, or error and no error), then the probability of having m events with outcome A after n trials is given by the Binomial Distribution:
$$
P(m=A \mid n) = p_A^m * (1 - p_A)^{n-m} \frac{n!}{{(n-m)}!m!}
$$
where $p_A$ is the probability of any single trial having outcome A.
Now, let's try and use this to calculate the probability of observing a certain number of errors in a stream of bits transmitted through a serial link with a certain BER...
Can you write Python code for it?

from math  import factorial

kNumBits = 1.e12   # 'n' in the equation, above.
kBER     = 1.e-12  # 'p-sub-A'.

def prob(m):
    'Probabillity of observing m errors.'

    return pow(kBER, m) * pow((1 - kBER), (kNumBits - m)) * factorial(kNumBits) / factorial(kNumBits - m) / factorial(m)

for m in range(3):
    print "P(%d):" % m, prob(m)


I am working on the file "Confidence_Interval.ipynb".
The first task is:
From the first plot, above, we can see that we've done a good job of normalizing the observation times, as all 5 curves fall within, approximately, a +/- 0.1% range, regarding the confidence of a 1e-12 BER estimate.
The second plot, above, shows us something interesting: the steepness of the active parts of the plots appears to be increasing, as we increase the number of observed errors. This is important, because it limits, quantitatively, the range of BER estimates that are "interesting". For instance, if we decide that the usefulness of any one of the curves, above, is limited to that range of BER estimates for which it provides a confidence interval in the range (0.70, 0.99), then we can plot this "usefulness range" as a function of observation time:
Can you write Python code for it?

from numpy import where

ranges = []
for cis in map(array, ciss):
    inrange = map(lambda x: (x > 0.70) and (x < 0.99), cis)
    idxs = where(inrange)[0]
    ranges.append(bers[idxs[-1]] - bers[idxs[0]])
plt.plot(times, ranges)
plt.title("Useful BER Range vs. Observation Time")
plt.xlabel("Observation Time (Normalized to Tbit/BER)")
plt.ylabel("Useful BER Range")


I am working on the file "Confidence_Interval.ipynb".
The first task is:
I had to give up, after 5 hours, and interrupt the code, above. What happened?!
The problem is those two factorial functions with very large arguments.
Watch what happens to execution time, as the argument to the factorial function increases:
Back to Contents
The Problem with Factorials
Can you write Python code for it?

%matplotlib inline

from matplotlib import pyplot as plt
from time       import clock
from numpy      import array

test_vals = array([1, 2, 5, 10, 20, 50, 100]) * 1e3
run_times = []
results   = []
for i in test_vals:
    start = clock()
    results.append(factorial(i))
    run_times.append(clock() - start)
plt.plot(test_vals, run_times)
plt.title("Factorial function run time vs. input argument")
plt.xlabel("Input Argument")
plt.ylabel("Run Time (s)")


I am working on the file "Confidence_Interval.ipynb".
The first task is:
Let's see what this trend looks like when plotted, using several different values for the number of observed errors. For each value chosen, we'll adjust the observation time, so as to just provide 99% confidence in a BER estimate of 10e-12. And, in that way, we hope to be doing an "apples to apples" comparison. We'll produce two plots:
- One zoomed in, to check our 99% alignment.
- One full scale, to see the nature of the whole curves.
Can you write Python code for it?

from numpy import arange

errors = [0, 1, 2,  5, 10]
times  = [5, 7, 9, 14, 21]
bers   = arange(1e-15, 2e-12, 1e-14)
ciss   = []
for (error, time) in zip(errors, times):
    cis = []
    for ber in bers:
        cis.append(conf_int(error, time * kNumBits, ber))
    plt.figure(1)
    plt.plot(bers, cis, label="{} errors".format(error))
    plt.figure(2)
    plt.plot(bers, cis, label="{} errors".format(error))
    ciss.append(cis)

plt.figure(1)
plt.title("Confidence Interval vs. BER Estimate")
plt.xlabel("BER Estimate")
plt.ylabel("Confidence Interval")
plt.legend(loc="lower right")
plt.axis(ymin=0.98, xmin=0.9e-12, xmax=1.1e-12)

plt.figure(2)
plt.title("Confidence Interval vs. BER Estimate")
plt.xlabel("BER Estimate")
plt.ylabel("Confidence Interval")
plt.legend(loc="lower right")
plt.axis(xmax=1e-12)


I am working on the file "Confidence_Interval.ipynb".
The first task is:
Since the partial approximation runs approximately twice as fast and represents a smaller deviation from the true answer, we'll use it going forward.
Back to Contents
Confidence in Estimation
Now, let's try to answer the following question:
If I go into the lab and observe my link operate error free for $10^{12}$ bits, what can I say about my link?
Many younger engineers use the following faulty reasoning when answering this question:
"Since I observed my link operate error free for $10^{12}$ bits, I know that the worst case possibility is that I had observed it for one additional bit and observed an error."
This leads them to believe that the worst possible BER their link could ever exhibit is:
$$BER \leq \frac{1}{10^{12}+1} \approx 10^{-12}$$
There are two faults in this line of reasoning:

That is not the worst case possibility; one that is worse is having observed the link for two extra bits and having them both be erroneous. In that case, we'd claim our BER was $\leq 2\cdot10^{-12}$, a $100\%$ error, relative to our previous claim!
We can't be certain that, if we made another $10^{12}$ bit observation, we wouldn't observe an error.

So, as well as giving someone our estimate of our link BER, we also need to quanitfy our confidence in that estimate. How do we do that?
Hint: "I'm pretty sure" is not the answer!
Well, what if we were to invent and precisely define a new term: confidence interval?
(We use the new, two word phrase, in order to clearly delineate our meaning from that of the English word, confidence.)
$$CI = Pr(m > \hat{m} \mid BER > \widehat{BER})$$
or, in English: My confidence in my estimate is simply the probability that I would have observed more errors than I did, if the BER were worse than I'm telling you it is.
If we start down the path of evaluating the expression, above, we quickly arrive at a dead end, when we have to evaluate an infinite sum (from $\hat{m} + 1$ to infinity). While some of these infinite sums often converge to nice, closed form...
Can you write Python code for it?

# Redefine our probability calculation function to take in 'n' and 'p' as arguments.
def prob2(m, n=kNumBits, p=kBER): # By giving n and p the proper defaults, we avoid breaking older client code.
    """
    Calculate the probability of observing m errors in a stream of n bits, given link BER p.

    Uses approximation to eliminate large factorials.
    """

    return pow(n * p, m) * pow((1 - p), (n - m)) / factorial(m)

# Define our confidence interval calculation function.
def conf_int(m_hat, n, ber):
    """
    Calculate the confidence interval of a BER estimation.

    Inputs:
      - m_hat  Number of errors observed.
      - n      Number of bits received.
      - ber    Estimate of link bit error ratio.

    """

    return 1 - sum(prob2(m, n, ber) for m in range(m_hat + 1))

# Calculate the actual confidence interval of our junior engineer's claim.
print "Confidence in claim, 'BER <= 1e-12', after observing no errors in 1e12 received bits:", conf_int(0, 1e12, 1.e-12)


I am working on the file "Confidence_Interval.ipynb".
The first task is:
We see that we need to extend our observation time to 6 normalized time units, in order to achieve the same level of confidence in our estimate of the BER. This is intuitively satisfying, as we observed an error afterall.
What if we ran a really long test? Say, overnight, or maybe even a few days. If our link is really running somewhere near a BER of $10^{-12}$, we'd expect to observe a significant number of errors. What does this increase in observation time and observed number of errors do to our Confidence Interval curve? Let's find out:
Can you write Python code for it?

# Plot confidence interval vs. observation time, for several different numbers of observed errors.
errors    = [0, 1, 2, 5, 10]
intervals = range(20)             # Numbers are normalized to the quantity '1/BER' bits.
ciss      = []
for error in errors:
    cis = []
    for interval in intervals:
        cis.append(conf_int(error, interval * kNumBits, kBER))
    ciss.append(cis)
for (error, cis) in zip(errors, ciss):
    plt.plot(intervals, cis, label="{} errors".format(error))
    plt.title("Confidence Interval vs. Observation Time, by Number of Observed Errors")
    plt.xlabel("Observation Time (Normalized to Tbit/BER)")
    plt.ylabel("Confidence Interval")
    plt.legend(loc="lower right")


I am working on the file "ConnectedThresholdAndOtherFilterPerformance.ipynb".
The first task is:
Breifly show that there is a performance difference. These images are only 2D an small to it may not be very accurate, but the filters use different algoritms and the GrayscaleDilate changes which algorithm based on the structuring element.
Can you write Python code for it?

%timeit -n 100 sitk.BinaryDilate(bimg, 10, sitk.sitkBall)
%timeit -n 100 sitk.GrayscaleDilate(bimg, 10, sitk.sitkBall)

%timeit -n 100 sitk.BinaryDilate(bimg, 10, sitk.sitkBox)
%timeit -n 100 sitk.GrayscaleDilate(bimg, 10, sitk.sitkBox)


I am working on the file "ConnectedThresholdAndOtherFilterPerformance.ipynb".
The first task is:
Demonstrate reporting of Progress
Can you write Python code for it?


I am working on the file "ConnectedThresholdAndOtherFilterPerformance.ipynb".
The first task is:
Performance comparison of Median and Rank image fitlers
Can you write Python code for it?

img = sitk.Image(100,100,100,sitk.sitkFloat32)
img = sitk.AdditiveGaussianNoise(img)
myshow(img)

radius = [3,3,3]

%timeit -n 1 sitk.Median(img,radius=radius)

%timeit -n 1 sitk.FastApproximateRank(img, rank=0.5, radius=radius)

%timeit -n 1 sitk.Rank(img, rank=0.5, radius=radius)

%timeit -n 1 sitk.Mean(img,radius=radius)


I am working on the file "Constante_de_Planck/Constante_Plank.ipynb".
The first task is:
aquí se puede ver que apezar de lo simple de este acercamiento, se obtuvo un error de tan solo $5 %$ respecto al valor real de $h$ extraido de la librería scipy.
Segunda Aproximación
en este intento se sigue el algoritmo prouesto por el articulo, graficar los voltajes contra las corrientes de los leds, luego, extraer su incercepto con el eje y como la corriente de activación y con la ley de ohm, el voltaje.
en este caso se usaron leds de color: infrarojo, rojo,naranja y verde, se deseaba usar un azul tambien pero los datos quedaron corruptos y no pudieron recuperarse
Can you write Python code for it?

plt.plot(VIR,IIR,'ro')

pendiente, intercepto, r_value, p_value, std_err = stats.linregress(VIR,IIR)
yir=[]
VActivacion_IR=intercepto*330
for i in VIR:
    yir.append((pendiente*i)+intercepto)
plt.plot(VIR,yir,'k-')

print ('medido ',led[0],'ajustado ',VActivacion_IR)
print ('r: ',r_value)

plt.plot(Vred,Ired,'ro')

pendiente, intercepto, r_value, p_value, std_err = stats.linregress(Vred,Ired)
yred=[]
VActivacion_rojo=intercepto*330
for i in Vred:
    yred.append((pendiente*i)+intercepto)
plt.plot(Vred,yred,'r-')

print ('medido ',led[1],'ajustado ',VActivacion_rojo)
print ('r: ',r_value)

plt.plot(Vorange,Iorange,'yo')

pendiente, intercepto, r_value, p_value, std_err = stats.linregress(Vorange,Iorange)
yorange=[]
VActivacion_naranja=intercepto*330
for i in Vorange:
    yorange.append((pendiente*i)+intercepto)
plt.plot(Vorange,yorange,'k-')

print ('medido ',led[-2],'ajustado',VActivacion_naranja)
print ('r: ',r_value)

plt.plot(Vgreen,Igreen,'go')

pendiente, intercepto, r_value, p_value, std_err = stats.linregress(Vgreen,Igreen)
ygreen=[]
VActivacion_verde=intercepto*330
for i in Vgreen:
    ygreen.append((pendiente*i)+intercepto)
plt.plot(Vgreen,ygreen,'k-')

print ('ajustado',VActivacion_verde)
print ('r: ',r_value)


I am working on the file "Constante_de_Planck/Constante_Plank.ipynb".
The first task is:
el motivo de este comportamiento se debe a la forma en que se toman los datos, ya que variar la resistencia en los potenciometros azules usados en clase no era facil, si embargo, se realizó desde un valor de voltaje lo suficientemente bajo para que no pasara corriente por el led, hasta los 3.3 voltios que ofrece la tarjeta Chipkit.
Primera Aproximación
Inicialmente para hallar los valores de voltaje de activacion tomamos un camino más empirico, variamos el voltaje hasta notar una pequeña chispa en el led y tomamos este valor de voltaje.
Este procedimiento se realizó para un led infrarojo (el cual se miró a travez de una camara de celular para poder distinguir la emisión de luz), uno rojo, uno naranja y uno azul. Sus respectivos voltajes de activacion fueron graficados contra $1/\lambda$ provisto por el articulo y finalmente se ajustó usando las funciones de scipy obteniendose lo siguiente:
Can you write Python code for it?

led=np.array(led)
led*=(3.3/5.)
#lamb=[1.10e6,1.60514e6,1.70648e6,1.76367e6,2.14133e6]
slope, intercept, r_value, p_value, std_err = stats.linregress(lamb,led)
x=np.linspace(lamb[0],lamb[-1],100)
y=slope*x+intercept
plt.plot(lamb,led,'o')
plt.plot(x,y,'-')
plt.show()
h_planck=slope*cons.e/cons.c
h=cons.h
error=(h_planck-h)/h
print ('r: ',r_value)
print ('pendiente: ',slope)
print ('error: ',std_err)
print ('h_planck: ',h_planck)
print ('h_real: ',h)
print ('error_h: ',error*100,'%')


I am working on the file "Constante_de_Planck/Constante_Plank.ipynb".
The first task is:
la explicación posible de este mayor error puede ser el hecho de que no se siguió un procedimiento riguroso y formarl para encontrar las corrientes con comportamiento lineal en cada caso. solo se buscó la zona con mejor linealidad.
finalmente, se quizó estudiar el posible resultado al eliminar el led infrarojo, el cual como se observa en la grafica de su voltaje, no dió valores muy estables, además de tener una r de tan solo $0.5$ lo cual es bastante bajo en comparacion de los demás leds.
Can you write Python code for it?

V_ajuste2=[VActivacion_rojo,VActivacion_naranja,VActivacion_verde] #ir,rojo,naranja,verde,azul
#print (len(V_ajuste2))
#print (len(lamb_ajuste2))
slope, intercept, r_value, p_value, std_err = stats.linregress(lamb_ajuste2,V_ajuste2)
x=np.linspace(lamb_ajuste[0],lamb_ajuste[-1],100)
y=slope*x+intercept
plt.plot(lamb_ajuste2,V_ajuste2,'o')
plt.plot(x,y,'-')
plt.show()
h_planck=slope*cons.e/cons.c
h=cons.h
error=abs(h_planck-h)/h
print ('r: ',r_value)
print ('pendiente: ',slope)
print ('error: ',std_err)
print ('h_planck: ',h_planck)
print ('h_real: ',h)
print ('error_h: ',error*100)


I am working on the file "Constante_de_Planck/Constante_Plank.ipynb".
The first task is:
una vez se obtienen los datos, se procede arealizar el mismo ajuste de la primera aproximacion, pero con un error mayor:
Can you write Python code for it?

V_ajuste=[VActivacion_IR,VActivacion_rojo,VActivacion_naranja,VActivacion_verde] #ir,rojo,naranja,verde,azul
#print (len(V_ajuste))
#print (len(lamb_ajuste))
slope, intercept, r_value, p_value, std_err = stats.linregress(lamb_ajuste,V_ajuste)
x=np.linspace(lamb_ajuste[0],lamb_ajuste[-1],100)
y=slope*x+intercept
plt.plot(lamb_ajuste,V_ajuste,'o')
plt.plot(x,y,'-')
plt.show()
h_planck=slope*cons.e/cons.c
h=cons.h
error=abs(h_planck-h)/h
print ('r: ',r_value)
print ('pendiente: ',slope)
print ('error: ',std_err)
print ('h_planck: ',h_planck)
print ('h_real: ',h)
print ('error_h: ',error*100, '%')


I am working on the file "Construct Dataset.ipynb".
The first task is:
先处理questions，数据清洗
Can you write Python code for it?

# extract all the code part
temp_code = questions['Body'].str.extractall(r'([^<]+)')

temp_code.head()

# unstack and convert into a single column for cleaning
test = temp_code.unstack('match')

test.columns = test.columns.droplevel()
# put all columns together
code = pd.DataFrame(test.apply(lambda x: x.str.cat(), axis=1,reduce=True))
# rename
code.columns = ['CodeBody']
# remove the html tags finally
code['CodeBody'] = code['CodeBody'].str.replace(r'<[^>]+>|\n|\r',' ')

# remove the code part from questions
body = questions['Body'].str.replace(r'[^<]+',' ')
# build up the question part from questions
questions['QuestionBody'] = body.str.replace(r"<[^>]+>|\n|\r", " ")

# Join the codebody by index
questions = questions.join(code)
# final cleaned dataset
questions_final = questions.drop('Body',axis=1)

questions_final.head()

questions_final.info()


I am working on the file "Construct Dataset.ipynb".
The first task is:
数据整合完毕，开始构造实验数据集
start from here
Can you write Python code for it?

import numpy as np
import pandas as pd
experts_count=pd.read_pickle('experts_count.pkl')
experts_count=experts_count.fillna('none')

experts_count.columns

train1 = experts_count[:80000][experts_count.Count>10]
test1 = experts_count[80000:]

train1_unique_expert = train1.ExpertId.unique()
print ("number of experts in train set: %r " % train1_unique_expert.shape)

test1_unique_expert = test1.ExpertId.unique()
print ("number of experts in test set: %r" % test1_unique_expert.shape)

print ("type : %r" % type(test1_unique_expert))

l = np.intersect1d(train1_unique_expert,test1_unique_expert)
print ("the number of experts both in train set and test set: %r" % l.shape)

title_train1 = train1.drop(['Id','QuestionBody','CodeBody','Tag', \
                           'ExpertId','Count',],axis=1)
title_test1 = test1.drop(['Id','QuestionBody','CodeBody','Tag', \
                         'ExpertId','Count',],axis=1)

title_train1 = title_train1.set_index('Label')
title_test1 = title_test1.set_index('Label')

title_train1.info()

title_test1.info()

# 通过fastText的测试类别我们得出，测试集中7404个问题中有5176个属于训练集中存在的专家
5176/7404.0

title_train1.to_csv('title_train1', encoding='utf-8')
title_test1.to_csv('title_test1', encoding='utf-8')


I am working on the file "Construct Dataset.ipynb".
The first task is:
整合数据
Can you write Python code for it?

questions_answers = questions_tags.merge(accepted_answers,on='Id',how='right')

questions_answers.head()

questions_answers.info()

experts_count = questions_answers.merge(count_df, on='ExpertId', how='left')

experts_count.head()

experts_count.info()

experts_count.columns

experts_count = experts_count.reindex(columns=[u'Id', u'Title', u'QuestionBody', u'CodeBody', u'Tag', u'ExpertId',
       u'Count', u'Label'])

from sklearn.preprocessing import LabelEncoder
label=LabelEncoder()
experts_count['Label']=label.fit_transform(experts_count['ExpertId'])

experts_count.head()

max_lable = np.max(experts_count.Label)
min_lable = np.min(experts_count.Label)
print (max_lable)
print (min_lable)

experts_count.info()

import pickle
pickle.dump(experts_count,open('experts_count.pkl','wb'))