I am working on the file "Get fin data for LSTM.ipynb".
The first task is:
We want:
- stock prices for several companies from various sectors, daily data over many years
- main index, daily data over many years
- sector information for every one of those companies
- google domestic trends data hardest to get - either all, or the most important, of indicators, daily, for the entire period. Query 4 months at a time to obtain daily data (otherwise it's monthly - according to pytrends documentation)

make a google account for the purpose of getting google d trends.
Can you write Python code for it?

from pytrends.request import TrendReq

google_username = "mm.trends.api@gmail.com"
google_password = ""
path = ""

# connect to Google
pytrend = TrendReq(google_username, google_password, custom_useragent='Pytrends')

trend_payload = {'q': 'Pizza, Italian, Spaghetti, Breadsticks, Sausage', 'cat': '0-71'}

# trend
trend = pytrend.trend(trend_payload)
print(trend)
df = pytrend.trend(trend_payload, return_type='dataframe')

df

# toprelated
toprelated = pytrend.related(trend_payload, related_type='top')
print(toprelated)
risingrelated = pytrend.related(trend_payload, related_type='rising')
print(risingrelated)

# top30in30
top30in30 = pytrend.top30in30()
print(top30in30)

country_payload = {'geo': 'US'}
# hottrends
hottrends = pytrend.hottrends(country_payload)
print(hottrends)

# hottrendsdetail
# returns XML data
hottrendsdetail = pytrend.hottrendsdetail(country_payload)
print(hottrendsdetail)

payload = {'date': '201601', 'geo': 'US'}
# alltopcharts
topcharts = pytrend.topcharts(payload)
print(topcharts)

keyword = 'pizza'
# suggestions
suggestions = pytrend.suggestions(keyword)
print(suggestions)


I am working on the file "GetChecksums Tests.ipynb".
The first task is:
Request the checksum of a file in the collection.
Can you write Python code for it?

%bitmag get-checksums -c integrationtest1 -i $TESTFILE1


I am working on the file "GetChecksums Tests.ipynb".
The first task is:
The correct checksum should be returned by all the pillars.
getChecksum (salted)
Request the salted checksum of a file in the collection
Can you write Python code for it?

import hmac
import hashlib
import base64

def getSaltedChecksum(file,saltHex,algorithm):
    #Note, the salt is a hex string
    salt = bytearray.fromhex(saltHex)
    digester = hmac.new(salt,None,algorithm)
    with open(file,'rb') as from_fh:
        while True:
            chunk = from_fh.read()
            if not chunk:
                break
            digester.update(chunk)
    return digester.hexdigest().lower()

file = os.environ.get('TESTFILE1')
print(getSaltedChecksum(file,saltHex='abcd',algorithm=hashlib.md5))

%bitmag get-checksums -c integrationtest1 -i $TESTFILE1 -R HMAC_MD5 -S 'abcd'


I am working on the file "GetChecksums Tests.ipynb".
The first task is:
The correct salted checksum should be returned by all the pillars (except the ChecksumPillar).
Missing file
Request the checksum of a file that does not exist in the collection.
Can you write Python code for it?

%bitmag get-checksums -c integrationtest1 -i ThisFileDoesNotExist


I am working on the file "GetRadLexTermsExpanded.ipynb".
The first task is:
Get Data
Pull all the reports from the database and convert to individual sentences.
Can you write Python code for it?

if GRABREPORTS:
    conn = sq.connect(os.path.join(RADDIR,"DBs",
                                   "criticalFindingsAll.sqlite"))
    cursor = conn.cursor()

    r_date  = re.compile(r"""([0-9]{1,2}(/[0-9]{1,2})?/[0-9]{2,4})""")
    r_time = re.compile(r"""([0-9]{1,2}:\d\d(pm|am)?)""")
    cursor.execute("""SELECT rowid,impression from reports""")
    data = [(d[0],d[1].lower()) for d in cursor.fetchall()]
    data =[(d[0],r_date.sub("",r_time.sub("",d[1]))) for d in data]

    reports = [d[1] for d in data]

    def report2sentences(report):
        return TextBlob(report).raw_sentences

    sentences = list(itertools.chain.from_iterable(dview.map_async(report2sentences,reports).get()))


I am working on the file "GetRadLexTermsExpanded.ipynb".
The first task is:
count = 0
for r in query_results["pathophysiology_metaclass"]:
    print(r[1],r[2])
    count += 1
    if count >=50:
        break
Do some printing for verifying things look as expected
Can you write Python code for it?

for i in itemData['pathophysiology_metaclass'][0:50]:
    print(i)
    #print("LITERAL: %s; RE: %s"%(i.literal, i.re))


I am working on the file "GetRadLexTermsExpanded.ipynb".
The first task is:
g=rdflib.Graph()
g.load(os.path.join(RADDIR,"Radlex_3.12.owl"))
Example RadLex queries
qres = g.query("""SELECT DISTINCT ?type WHERE {?s a ?type.}""")
In creating the regular expressions from the RadLex terms, I've learned from sad (and slow) experience the need to have the word boundaries around the phrases.
Can you write Python code for it?

def create_regex(r1,r2):
    if r2:
        return r"\b%s\b|\b%s\b"%(r1,r2)
    else:
        return r"\b%s\b"%r1

print("Grabradlex: ", GRABRADLEX)
if GRABRADLEX:
    g = rdflib.Graph()
    #result = g.parse("http://data.bioontology.org/ontologies/RADLEX/download?apikey=8b5b7825-538d-40e0-9e9e-5ab9274a9aeb&download_format=rdf",
    #                 format="xml")

    result = g.parse(os.path.join(RADDIR, 'Radlex3.13.1.owl'), format="xml")
    print("result: ", result)

import pyConTextNLP.functional.itemData as ID
import pyConTextNLP.functional.conTextItem as CI
import pyConTextNLP.functional.ConTextMarkup as CM

if GRABRADLEX:
    g = rdflib.Graph()
    #result = g.parse("http://data.bioontology.org/ontologies/RADLEX/download?apikey=8b5b7825-538d-40e0-9e9e-5ab9274a9aeb&download_format=rdf",
    #                 format="xml")

    result = g.parse(os.path.join(RADDIR, 'Radlex3.13.1.owl'), format="xml")

    query_results = {}
    types = ["anatomy_metaclass","pathophysiology_metaclass","imaging_observation_metaclass"]
    q1 = """

    PREFIX Ontology1447432460: <http://www.owl-ontologies.com/Ontology1447432460.owl#>

    SELECT DISTINCT ?s ?preferred_name ?synonym ?type

    WHERE {

        ?s Ontology1447432460:Preferred_name ?preferred_name.
        ?s rdf:type ?type
        OPTIONAL {?s Ontology1447432460:Synonym ?synonym.}

        FILTER  regex(str(?type), "%s")}"""
    #print(q1)

    for t in types:
        q = q1%t
        print(q)
        query_results[t] = g.query(q)

    for t,r in query_results.items():
        print(t,len(r))
        print("*"*42)

    rslts = g.query(q)

    itemData = {}
    for t in types:
        itemData[t] = [CI.create_ConTextItem([r[1],
                                              r[3].split("#")[1],
                                              create_regex(r[1],r[2]),
                                              ""]) \
                    for r in query_results[t]]
    with open(os.path.join(RADDIR,"DBs","radlex.pickle"),"wb") as f0:
       ...


I am working on the file "Get_data.ipynb".
The first task is:
Data cleansing (according to description)
Quickly setting all values to those we are used to work with.
Can you write Python code for it?

# Set dates to a date format
data_dict['account']['date'] = pd.to_datetime(data_dict['account']['date'], format='%y%m%d')
data_dict['card']['issued'] = pd.to_datetime(data_dict['card']['issued'],
                                             format='%y%m%d %H:%M:%S')
data_dict['trans']['date'] = pd.to_datetime(data_dict['trans']['date'], format='%y%m%d')
data_dict['loan']['date'] = pd.to_datetime(data_dict['loan']['date'], format='%y%m%d')

# Client needs modification (seperate the sexes and adjust the dates)
data_dict['client']['sex'] = "M"
tmp = (data_dict['client']['birth_number']/100).round().astype(int)
tmp = tmp % 100 > 50
data_dict['client'].loc[tmp, "sex"] = "F"
data_dict['client'].loc[tmp, "birth_number"] = data_dict['client'].loc[tmp, "birth_number"] - 5000

# Assumption: Everyone borne in the 1900's, (min birth_number -> 110820, max -> 870927)
# 113 is rather old.
data_dict['client']['birth_number'] = data_dict['client']['birth_number'] + 19000000
data_dict['client']['birth_number'] = pd.to_datetime(data_dict['client']['birth_number'],
                                                format='%Y%m%d')

if check:
    print data_dict['account']['date'].describe()
    print data_dict['card']['issued'].describe()
    print data_dict['trans']['date'].describe()
    print data_dict['loan']['date'].describe()
    print data_dict['client']['birth_number'].describe()


I am working on the file "Get_data.ipynb".
The first task is:
District headers are not directly usefull.
Print the explanation file.
Can you write Python code for it?

if check:
    from IPython.core.display import HTML
    html_file = open('Downloads/berka.htm').read()
    HTML(html_file)


I am working on the file "Get_data.ipynb".
The first task is:
Extract the zipfile
Can you write Python code for it?

import zipfile

if not os.path.exists('Downloads/data_berka'):
    os.makedirs('Downloads/data_berka')

zip_ref = zipfile.ZipFile('Downloads/data_berka.zip', 'r')
zip_ref.extractall('Downloads/data_berka')
zip_ref.close()


I am working on the file "Get_data.ipynb".
The first task is:
Merge data into three data.frames, client info, demographic info and transactions:
We can now start to merge the data in to less dataframes. All merges are left-joins such that none of the information in the left data frames is lost.
Can you write Python code for it?

# Add disposition
client_info = pd.merge(left = data_dict['client'],
                       right = data_dict['disp'],
                       how ='left',
                       on = 'client_id')
# Add account info
client_info = pd.merge(left = client_info,
                       right = data_dict['account'],
                       how ='left',
                       on = 'account_id',
                       suffixes=('_client', '_branch'))

# Add Loans
client_info = pd.merge(left = client_info,
                       right = data_dict['loan'],
                       how ='left',
                       on = 'account_id',
                       suffixes=('_client', '_loan'))
# Add credit cards
client_info = pd.merge(left = client_info,
                       right = data_dict['card'],
                       how ='left',
                       on = 'disp_id',
                       suffixes=('_client', '_card'))


I am working on the file "Get_data.ipynb".
The first task is:
Print the first two lines for every file
Inspect filetype, seperator, quote and column types. Print it for the first and last two lines in the file.
Can you write Python code for it?

files = os.listdir('Downloads/data_berka')
if check:
    for fl in files:
        with open('Downloads/data_berka/' + fl) as myfile:
            print fl
            lines = myfile.readlines()
            print '\t' + repr(lines[0])
            print '\t' + repr(lines[1])
            print '\t ...'
            print '\t' + repr(lines[-2])
            print '\t' + repr(lines[-1])


I am working on the file "GetsDrawnDotCom.ipynb".
The first task is:
Filter the non jpeg/png links. Need to perform request or imgur api to get the jpeg/png files from the link. Hey maybe bs4?
Can you write Python code for it?

#from imgurpython import ImgurClient

#opps = open('/home/wcmckee/ps.txt', 'r')
#opzs = open('/home/wcmckee/ps2.txt', 'r')
#oprd = opps.read()
#opzrd = opzs.read()

#client = ImgurClient(oprd, opzrd)

# Example request
#items = client.gallery()
#for item in items:
#    print(item.link)

#itz = client.get_album_images()

#galim = client.get_image('SBaV275')

#galim.size

#gelim = client.get_album_images('LTDJ9')

#gelim

#from urlparse import urlparse

#linklis = []


I am working on the file "GetsDrawnDotCom.ipynb".
The first task is:
Instead of creating these white images, why not download the art replies of the reference photo.
Can you write Python code for it?

#for lisa in lisauth:
#    #print lisa + '-line.png'
#    im = Image.new("RGB", (512, 512), "white")
#    im.save(lisa + '-line.png')
#    im = Image.new("RGB", (512, 512), "white")
#    im.save(lisa + '-bw.png')

    #print lisa + '-bw.png'
#    im = Image.new("RGB", (512, 512), "white")
#    im.save(lisa + '-colour.png')

    #print lisa + '-colour.png'

#lisauth


I am working on the file "GetsDrawnDotCom.ipynb".
The first task is:
Something is wrong with the script and it's no longer creating these dir in the correct folder. How did this break?
Fixed that but problems with it
Getting error:
OSError: [Errno 17] File exists: '/home/wcmckee/getsdrawndotcom/imgs/15/01'
If the file exists it should be skipping over it, thats why it has the os.path.isdir == True:
 print its true
else
 print its false, and make the dir
Can you write Python code for it?

if os.path.isdir(monzpath) == True:
    print ('its true')
else:
    print ('its false')
    #os.mkdir('/home/wcmckee/getsdrawndotcom/' + monzpath)

if os.path.isdir(imgzdays) == True:
    print ('its true')
else:
    print ('its false')
    os.mkdir(imgzdays)

if os.path.isdir(rmgzdays) == True:
    print ('its true')
else:
    print ('its false')
    os.mkdir(rmgzdays)

if os.path.isdir(metzdays) == True:
    print ('its true')
else:
    print ('its false')
    os.mkdir(metzdays)

if os.path.isdir(dayzpath) == True:
    print ('its true')
else:
    print ('its false')
    os.mkdir(dayzpath)


I am working on the file "GetsDrawnDotCom.ipynb".
The first task is:
Tweet each reference img in list, removing the item when it's tweeted so that same item isn't tweeted twice.
Make new list of items to tweet, appending in new items when site is updated
Can you write Python code for it?

for apt in aptype:
    print (apt)

#opad = open('/home/wcmckee/ad.html', 'r')

#opred = opad.read()

#str2 = opred.replace("\n", "")

#str2

doc = dominate.document(title='GetsDrawn')

with doc.head:
    link(rel='stylesheet', href='style.css')
    script(type ='text/javascript', src='script.js')
    #str(str2)

    with div():
        attr(cls='header')
        h1('GetsDrawn')
        p(img('imgs/getsdrawn-bw.png', src='imgs/getsdrawn-bw.png'))
        #p(img('imgs/15/01/02/ReptileLover82-reference.png', src= 'imgs/15/01/02/ReptileLover82-reference.png'))
        h1('Updated ', strftime("%a, %d %b %Y %H:%M:%S +0000", gmtime()))
        #p(panz)
        p(bodycom)

with doc:
    with div(id='body').add(ol()):
        for rdz in reliz:
            #h1(rdz.title)
            #a(rdz.url)
            #p(img(rdz, src='%s' % rdz))
            #print rdz
            p(img(rdz, src = rdz))
            p(rdz)

            #print rdz.url
            #if '.jpg' in rdz.url:
            #    img(rdz.urlz)
            #else:
            #    a(rdz.urlz)
            #h1(str(rdz.author))

            #li(img(i.lower(), src='%s' % i))

    with div():
        attr(cls='body')
        p('GetsDrawn is open source')
        a('https://github.com/getsdrawn/getsdrawndotcom')
        a('https://reddit.com/r/redditgetsdrawn')

#print doc

docre = doc.render()
#s = docre.decode('ascii', 'ignore')
yourstring = docre.encode('ascii', 'ignore').decode('ascii')
indfil = ('/home/wcmckee/getsdrawndotcom/index.html')
mkind = open(indfil, 'w')
mkind.write(yourstring)
mkind.close()

mkind = open(indfil, 'w')
mkind.write(yourstring)
mkind.close()

#os.system('scp -r /home/wcmckee/getsdrawndotcom/ wcmckee@getsdrawn.com:/home/wcmckee/getsdrawndotcom')

#rsync -azP source destination

#updatehtm = raw_input('Update index? Y/n')
#updateref = raw_input('Update reference? Y/n')

#if 'y' or '' in updatehtm:
#    os.system('scp -r /home/wcmckee/getsdrawndotcom/index.html...


I am working on the file "GetsDrawnDotCom.ipynb".
The first task is:
if i save the data to the file how am i going to get it to update as the post is archieved. Such as up and down votes.
Can you write Python code for it?

rgde = len(lisrgc)

rgde

alrgds = dict()

#for lisr in lisrgc:
#    print(lisr.author)
#    print(lisr.title[0:30])

for lisz in lisrgc:
    metadict.update({'up': lisz.ups})
    metadict.update({'down': lisz.downs})
    metadict.update({'title': lisz.title})
    metadict.update({'created': lisz.created})

    #metadict.update({'createdutc': lisz.created_utc})
    #print lisz.ups
    #print lisz.downs
    #print lisz.created
    #print lisz.comments

import random

ranchor = random.choice(lisrgc)

titshort = ranchor.title[0:30]

titsre =titshort.replace(' ', '')

titsre

ranchor.url

ranautr =  (ranchor.author)

hasra = ('#') + str(ranautr)

hasra

hasgd = ('#getsdrawn')

urlfin = ('http://getsdrawn.com/' + iwcpath + '/' + str(ranautr) + '-reference.png')

(urlfin)

twez = (titsre + ' ' + urlfin + ' ' + hasra + ' ' + hasgd)

len(twez)


I am working on the file "Getting Started With Badfish.ipynb".
The first task is:
A MissFrame converts your data to a boolean matrix where a missing cell indicates a True value while a filled cell is given a False value.
What are the different functions that can be used with the MissFrame?
Can you write Python code for it?


I am working on the file "Getting Started With Badfish.ipynb".
The first task is:
A tabular function to show which columns seem to go missing together reports these correlations of missing data-
Can you write Python code for it?

mf.corr(columns = ['Ozone', 'Temp','Wind'])


I am working on the file "Getting Started With Badfish.ipynb".
The first task is:
All MissFrame methods contain the same structure of arguments.

Where - Show me data where '_____' are missing.
How - Show me data where '_____' columns are ALL/ANY missing
Columns - Show me description for only '_____' columns

We can replicate the same functionality of counting the missing values with mf.counts()
Can you write Python code for it?


I am working on the file "Getting Started With Badfish.ipynb".
The first task is:
Lets quickly use Pandas isnull().sum() function to check how many missing values are present in the different columns.
Can you write Python code for it?


I am working on the file "Getting Started With Badfish.ipynb".
The first task is:
Okay, so we've got 8 missing cells of Temp, 2 of Wind and Solar each when Ozone goes missing.
What happens when Ozone OR temp go missing? How does it affect the other three?
Can you write Python code for it?


I am working on the file "Getting Started With Badfish.ipynb".
The first task is:
Or perhaps let's look at only the correlations of missing data of other columns with Ozone:
Can you write Python code for it?


I am working on the file "Getting Started with Images - OpenCV.ipynb".
The first task is:
About cv2.waitKey() function:
Keyboard binding function   Argument in milliseconds
If zero is passed, it waits indefinitely for a key stroke.
About cv2.destroyAllWindows():
Destroy all windows we created
To destroy any specific window,
use cv2.destroyWindow(), pass argument with window's name.
Can you write Python code for it?

cv2.namedWindow('image', cv2.WINDOW_AUTOSIZE)
cv2.imshow('Image', img)
cv2.waitKey(5)
cv2.destroyWindow('Image')

cv2.waitKey?

help(cv2.namedWindow)


I am working on the file "Getting Started with Images - OpenCV.ipynb".
The first task is:
Using matplotlib
Can you write Python code for it?

import numpy as np
import cv2
from matplotlib import pyplot as plt
import seaborn; seaborn.set()
img = cv2.imread('Me1.png',0)
plt.imshow(img, cmap = 'gray', interpolation='bicubic')
# plt.xticks([]), plt.yticks([]) # To hide tick values on x & y -axis
plt.show()
# 255, 255, 0 -  R G B


I am working on the file "Getting Started with Images - OpenCV.ipynb".
The first task is:
Writing an Image
Use the function cv2.imwrite() to save an image.
2 Arguments:

First argument is file name
Image you want to save
Can you write Python code for it?

cv2.imwrite('Me1.jpg', img)

import numpy as np
import cv2

img2 = cv2.imread('Me1.png', -1)
cv2.imshow('IMAGE', img2)
k = cv2.waitKey(0) & 0xFF
if k == 27:
    cv2.destroyAllWindows()
elif k == ord('s'):
    cv2.imwrite('Oh.png', img2)
    cv2.destroyAllWindows()


I am working on the file "Getting Started with PySpark on Windows 10.ipynb".
The first task is:
(2) Compute accuracy against the test data using <b>Evaluator</b>
using https://spark.apache.org/docs/2.1.1/api/python/pyspark.ml.html#pyspark.ml.evaluation.MulticlassClassificationEvaluator
Can you write Python code for it?

evaluator = MulticlassClassificationEvaluator(
                labelCol="indexedLabel", predictionCol="prediction", metricName="accuracy")
accuracy = evaluator.evaluate(predictions)
print("Test set accuracy = " + str(accuracy))


I am working on the file "Getting Started with PySpark on Windows 10.ipynb".
The first task is:
(3) Display trained decision tree using <b>toDebugString</b>
using https://spark.apache.org/docs/2.1.1/api/python/pyspark.ml.html#pyspark.ml.classification.DecisionTreeClassificationModel.toDebugString
Can you write Python code for it?

treeModel = model.stages[-1] # last stage in Pipeline
# summary only
print(treeModel.toDebugString)


I am working on the file "Getting Started with PySpark on Windows 10.ipynb".
The first task is:
2.B. Review Data
using https://spark.apache.org/docs/2.1.0/api/python/pyspark.sql.html#pyspark.sql.DataFrame
Can you write Python code for it?

print("Number of data rows:", df.count())

df.printSchema() # or data.dtypes or df.columns

df.describe("class", "cap-shape", "cap-surface", "cap-color").show()

df.first() # or df.head(1) or df.show(1)


I am working on the file "Getting Started with PySpark on Windows 10.ipynb".
The first task is:
2.C.c. Assemble Features
Using <b>VectorAssembler</b> to combine features in a single vector column
Can you write Python code for it?

inputFeatures = ['stringindexed_' + col for col in categorical_columns]

assembler = VectorAssembler(
                inputCols=inputFeatures,
                outputCol="indexedFeatures")


I am working on the file "Getting Started with PySpark on Windows 10.ipynb".
The first task is:
3. Create Train-Test Split

Split the data into training and test sets (30% held out for testing)
Use seed=1234 for reproducibility

using https://spark.apache.org/docs/2.1.1/api/python/pyspark.sql.html#pyspark.sql.DataFrame.randomSplit
Can you write Python code for it?

(trainingData, testData) = df.randomSplit([0.7, 0.3], 1234)


I am working on the file "Getting Started with PySpark on Windows 10.ipynb".
The first task is:
4. Create Machine Learning Object

Create a <b>Decision Tree</b> model

using https://spark.apache.org/docs/2.1.1/api/python/pyspark.ml.html#pyspark.ml.classification.DecisionTreeClassifier
Can you write Python code for it?

clf = DecisionTreeClassifier(labelCol="indexedLabel", featuresCol="indexedFeatures")


I am working on the file "Getting Started with PySpark on Windows 10.ipynb".
The first task is:
6. Train Machine Learning Object

Execute the stages in the pipeline including featurization and model training on test data using <b>fit</b>

using https://spark.apache.org/docs/2.1.1/api/python/pyspark.ml.html#pyspark.ml.Pipeline.fit
Can you write Python code for it?


I am working on the file "Getting Started with PySpark on Windows 10.ipynb".
The first task is:
8. Stop Spark Session
using https://spark.apache.org/docs/2.1.1/api/python/pyspark.sql.html#pyspark.sql.SparkSession.stop
Can you write Python code for it?


I am working on the file "Getting Started with PySpark on Windows 10.ipynb".
The first task is:
Example 1: Calculate value of Pi
Adapted from https://github.com/apache/spark/blob/master/examples/src/main/python/pi.py
Can you write Python code for it?

from random import random
from operator import add
from pyspark.sql import SparkSession

def f(_):
    x = random() * 2 - 1
    y = random() * 2 - 1
    return 1 if x ** 2 + y ** 2 <= 1 else 0

spark = SparkSession \
        .builder \
        .appName("PythonPi") \
        .getOrCreate()

partitions = 10
num_samples = 10000

count = spark.sparkContext.parallelize(range(1, num_samples + 1), partitions).map(f).reduce(add)

print("Pi is roughly %f" % (4.0 * count / num_samples))

spark.stop()


I am working on the file "Getting Started with PySpark on Windows 10.ipynb".
The first task is:
Example 2A: Perform Binary Classification using Decision Tree
Adapted from https://github.com/apache/spark/blob/master/examples/src/main/python/ml/decision_tree_classification_example.py
Can you write Python code for it?

from pyspark.ml import Pipeline
from pyspark.ml.classification import DecisionTreeClassifier
from pyspark.ml.feature import StringIndexer, VectorIndexer
from pyspark.ml.evaluation import MulticlassClassificationEvaluator

from pyspark.sql import SparkSession

spark = SparkSession\
        .builder\
        .appName("DecisionTreeBinaryClassificationExample")\
        .getOrCreate()

data = spark.read.format("libsvm").load("data/sample_libsvm_data.txt")

# Index labels, adding metadata to the label column.
# Fit on whole dataset to include all labels in index.
labelIndexer = StringIndexer(inputCol="label", outputCol="indexedLabel").fit(data)

# Automatically identify categorical features, and index them.
# We specify maxCategories so features with > 4 distinct values are treated as continuous.
featureIndexer =\
    VectorIndexer(inputCol="features", outputCol="indexedFeatures", maxCategories=4).fit(data)

# Split the data into training and test sets (30% held out for testing)
(trainingData, testData) = data.randomSplit([0.7, 0.3], 1234)

# Create a DecisionTree model.
dt = DecisionTreeClassifier(labelCol="indexedLabel", featuresCol="indexedFeatures")

# Chain indexers and tree in a Pipeline
pipeline = Pipeline(stages=[labelIndexer, featureIndexer, dt])

# Train model.  This also runs the indexers.
model = pipeline.fit(trainingData)

# Make predictions.
predictions = model.transform(testData)

# Select example rows to display.
predictions.select("prediction", "indexedLabel", "features").show(5)

# Select (prediction, true label) and compute accuracy
evaluator = MulticlassClassificationEvaluator(
                labelCol="indexedLabel", predictionCol="prediction", metricName="accuracy")
accuracy = evaluator.evaluate(predictions)
print("Test set accuracy = " + str(accuracy))

treeModel = model.stages[2]

# summary only
print(treeModel.toDebugString)

spark.stop()


I am working on the file "Getting Started with PySpark on Windows 10.ipynb".
The first task is:
Example 2B: Perform Multiclass Classification using Decision Tree
Can you write Python code for it?

from pyspark.ml import Pipeline
from pyspark.ml.classification import DecisionTreeClassifier
from pyspark.ml.feature import StringIndexer, VectorIndexer
from pyspark.ml.evaluation import MulticlassClassificationEvaluator

from pyspark.sql import SparkSession

spark = SparkSession\
        .builder\
        .appName("DecisionTreeMulticlassClassificationExample")\
        .getOrCreate()

data = spark.read.format("libsvm").load("data/sample_multiclass_classification_data.txt")

# Index labels, adding metadata to the label column.
# Fit on whole dataset to include all labels in index.
labelIndexer = StringIndexer(inputCol="label", outputCol="indexedLabel").fit(data)

# Automatically identify categorical features, and index them.
# We specify maxCategories so features with > 4 distinct values are treated as continuous.
featureIndexer =\
    VectorIndexer(inputCol="features", outputCol="indexedFeatures", maxCategories=4).fit(data)

# Split the data into training and test sets (40% held out for testing)
(trainingData, testData) = data.randomSplit([0.6, 0.4], 1234)

# Create a DecisionTree model.
dt = DecisionTreeClassifier(labelCol="indexedLabel", featuresCol="indexedFeatures")

# Chain indexers and tree in a Pipeline
pipeline = Pipeline(stages=[labelIndexer, featureIndexer, dt])

# Train model.  This also runs the indexers.
model = pipeline.fit(trainingData)

# Make predictions.
predictions = model.transform(testData)

# Select example rows to display.
predictions.select("prediction", "indexedLabel", "features").show(5)

# Select (prediction, true label) and compute accuracy
evaluator = MulticlassClassificationEvaluator(
                labelCol="indexedLabel", predictionCol="prediction", metricName="accuracy")
accuracy = evaluator.evaluate(predictions)
print("Test set accuracy = " + str(accuracy))

treeModel = model.stages[2]

# summary only
print(treeModel.toDebugString)

spark.stop()


I am working on the file "Getting Started with PySpark on Windows 10.ipynb".
The first task is:
Example 3: Perform Multiclass Classification using Multilayer Perceptron
Adapted from https://github.com/apache/spark/blob/master/examples/src/main/python/ml/multilayer_perceptron_classification.py
Can you write Python code for it?

from pyspark.ml.classification import MultilayerPerceptronClassifier
from pyspark.ml.evaluation import MulticlassClassificationEvaluator

from pyspark.sql import SparkSession

spark = SparkSession\
        .builder\
        .appName("multilayer_perceptron_classification_example")\
        .getOrCreate()

 # Load training data
data = spark.read.format("libsvm").load("data/sample_multiclass_classification_data.txt")

# Split the data into train and test
splits = data.randomSplit([0.6, 0.4], 1234)
train = splits[0]
test = splits[1]

# specify layers for the neural network:
# input layer of size 4 (features), two intermediate of size 5 and 4
# and output of size 3 (classes)
layers = [4, 5, 4, 3]

# create the trainer and set its parameters
trainer = MultilayerPerceptronClassifier(maxIter=100, layers=layers, blockSize=128, seed=1234)

# train the model
model = trainer.fit(train)

# compute accuracy on the test set
predictions = model.transform(test)

# Select example rows to display.
predictions.select("prediction", "label", "features").show(5)

# Select (prediction, true label) and compute accuracy
evaluator = MulticlassClassificationEvaluator(
                labelCol="label", predictionCol="prediction", metricName="accuracy")
accuracy = evaluator.evaluate(predictions)
print("Test set accuracy = " + str(accuracy))

spark.stop()


I am working on the file "Getting Started with PySpark on Windows 10.ipynb".
The first task is:
Example 4: End-to-End Workflow using Decision Tree
Using mushroom dataset from https://archive.ics.uci.edu/ml/datasets/Mushroom
This example describes the basic workflow consisting of the following phases:

Prepare Spark Environment
Prepare Data
Read Data
Review Data
Transform Data
Transform Label
Transform Features
Assemble Features

Create Train-Test Split
Create Machine Learning Object
Create Pipeline
Train Machine Learning Object
Evaluate Trained Machine Learning Object
Perform Predictions
Compute Accuracy
Display Trained Model

Stop Spark Session

1. Prepare Spark Environment
Start Spark Session
using https://spark.apache.org/docs/2.1.1/api/python/pyspark.sql.html#module-pyspark.sql
Can you write Python code for it?

from pyspark.ml import Pipeline
from pyspark.ml.classification import DecisionTreeClassifier
from pyspark.ml.feature import StringIndexer, VectorIndexer, VectorAssembler
from pyspark.ml.evaluation import MulticlassClassificationEvaluator

from pyspark.sql import SparkSession

spark = SparkSession\
        .builder\
        .appName("DecisionTreeBinaryClassificationExampleOnMushrooms")\
        .getOrCreate()


I am working on the file "Getting Started.ipynb".
The first task is:
I used the as_of parameter here to indicate that this data from January. When as_of is ommited, the dimension will always use the current date.
Updating dimension with new data
Now lets import a new clients table:
Can you write Python code for it?

df = pd.read_csv('clients 2015-02.csv')
df


I am working on the file "Getting Started.ipynb".
The first task is:
Note that Fred has moved from FL to NY.
Before updating the dimension with this new data, lets indicate that this time the data is from February.
Can you write Python code for it?

dim.as_of = '2015-02-01'

dim.update(df)

dim.df


I am working on the file "Getting Started.ipynb".
The first task is:
Now, lets create a dimension from this dataframe:
Can you write Python code for it?

dim = pyscd.SlowlyChangingDimension(
    df, source_keys='ssn', as_of='2015-01-01')
dim.df


I am working on the file "Getting and Cleaning Data/Data_Cleaning_Exercises.ipynb".
The first task is:
Exercise 2: The table contains a column called 'categories' and each entry in this column is populated by a list. We are interested in those businesses that restaurants. Create a new column 'Restaurant_type' that contains a description of the restaurant based on the other elements of 'categories.
That is, if we have '[Sushi Bars, Japanese, Restaurants]' in categories the 'Restaurant_type will be '{'SushiBars': 1, 'Japanese': 1, 'Mexican': 0, ...}'
Can you write Python code for it?

# create a set containing all the possible categories
cat = set([])

for row in business['categories']:
    if 'Restaurants' in row:
        cat = cat.union(set(row))

cat = cat.difference(set(['Restaurants']))

# function that creates a dictionary based on the previous categories and sets the values to 1 if the category is presenti in the argument
def cat_dict(row):
    if 'Restaurants' in row:
        d = dict.fromkeys(cat, 0)
        for e in row:
            if e != 'Restaurants':
                d[e] = 1
        return d
    else:
        return np.NaN

business['Restaurant_type'] = business['categories'].apply(cat_dict)

business[['Restaurant_type', 'categories']].head()


I am working on the file "Getting and Cleaning Data/Data_Cleaning_Exercises.ipynb".
The first task is:
Exercise 3: Lets clean the 'attributes' column. The entries in this column are dictionaries. We need to do two things:
1) Turn all the True or False in the dictionary to 1s and 0s.
2) There are some entries within dictionaries that are dictionaries themselves, lets turn the whole entry into just one dictionary, for example if we have
'{'Accepts Credit Cards': True, 'Alcohol': 'none','Ambience': {'casual': False,'classy': False}}'
then turn it into
'{'Accepts Credit Cards':1, 'Alcohol_none': 1, 'Ambience_casual': 0, 'Ambience_classy': 0}'.
There might be other entries like {'Price Range': 1} where the values are numerical so we might want to change that into {'Price_Range_1': 1}.
The reason we modify categorical variables like this is that machine learning algorithms cannot interpret textual data like "True" and "False". They need numerical inputs such as 1 and 0.
Can you write Python code for it?

# if the value is a boolean replaces the spaces in the key with underscores and returns 0-1 instead of True-False
# else replaces the spaces in the key with underscores, adds the value to the key and returns 1
def replace_entry(key, value):
    if type(value) is bool:
        if value:
            return key.replace(' ', '_'), 1
        else:
            return key.replace(' ', '_'), 0
    else:
        return key.replace(' ', '_') + '_' + str(value).replace(' ', '_'), 1

# returns the dictionary after applying the function above, navigates through the dictionary inside the values
def clean_dict(dct):

    out_dct = {}

    for key in dct:
        value = dct[key]
        if type(value) is dict:
            for subkey in value:
                new_key, new_value = replace_entry(subkey, value[subkey])
                out_dct[key.replace(' ', '_') + '_' + new_key] = new_value
        else:
            new_key, new_value = replace_entry(key, value)
            out_dct[new_key] = new_value

    return out_dct

business['attributes'] = business['attributes'].apply(clean_dict)

business[['attributes']].head(10)


I am working on the file "Getting and Cleaning Data/Data_Cleaning_Exercises.ipynb".
The first task is:
Exercise 4: Create a new column for every day of the week and fill it with the amount of hours the business is open that day.
Your approach should handle businesses that stay open late like bars and nightclubs.
Can you write Python code for it?

# dct = {'Friday': {'close': '21:00', 'open': '11:00'},
#  'Monday': {'close': '21:00', 'open': '11:00'},
#  'Thursday': {'close': '21:00', 'open': '11:00'},
#  'Tuesday': {'close': '21:00', 'open': '11:00'},
#  'Wednesday': {'close': '21:00', 'open': '11:00'}}

dct = {'Friday': {'close': '02:00', 'open': '11:00'},
 'Monday': {'close': '02:00', 'open': '11:00'},
 'Saturday': {'close': '02:00', 'open': '12:00'},
 'Sunday': {'close': '22:00', 'open': '12:00'},
 'Thursday': {'close': '02:00', 'open': '11:00'},
 'Tuesday': {'close': '02:00', 'open': '11:00'},
 'Wednesday': {'close': '02:00', 'open': '11:00'}}

day_map = {'Friday': '2017-08-04',
 'Monday': '2017-07-31',
 'Saturday': '2017-08-05',
 'Sunday': '2017-08-06',
 'Thursday': '2017-08-03',
 'Tuesday': '2017-08-01',
 'Wednesday': '2017-08-02'}

def daily_hours(dct):
    # initialize output
    hours = dict.fromkeys(range(0, 7), 0)
    for day in dct:
        # set start datetime, end datetime and end of day datetime
        start = pd.to_datetime(day_map[day] + ' ' + dct[day]['open'])
        end = pd.to_datetime(day_map[day] + ' ' + dct[day]['close'])
        end_of_day = pd.to_datetime(day_map[day] + ' 00:00:00') + pd.Timedelta(1, unit='D')
        # if closing hours is in the next day add one to the end and add to output the two days
        if dct[day]['close'] < dct[day]['open']:
            end = end + pd.Timedelta(1, unit='D')
            diff = end_of_day - start
            hours[start.dayofweek] += int(np.floor(diff.seconds / 3600))
            diff = end - end_of_day
            hours[end_of_day.dayofweek] += int(np.floor(diff.seconds / 3600))
        # else add only the current day
        else:
            diff = end - start
            hours[start.dayofweek] += int(np.floor(diff.seconds / 3600))
    return hours[0], hours[1], hours[2], hours[3], hours[4], hours[5], hours[6]

business['Monday'], business['Tuesday'], business['Wednesday'], business['Thursday'], business['Friday'], business['Saturday'], business['Sunday'] =...


I am working on the file "Getting and Cleaning Data/Data_Cleaning_Exercises.ipynb".
The first task is:
Exercise 5: Create a table with the average review for a business.
You will need to pull in a new json file and merge DataFrames for the next 2 exercises.
Can you write Python code for it?

review = pd.read_json('Yelp/yelp_academic_dataset_review.json', lines=True)
print(review.shape)
review.head()

average_review = review[['business_id', 'stars']].groupby('business_id').mean()

average_review.head()


I am working on the file "Getting and Cleaning Data/Data_Cleaning_Exercises.ipynb".
The first task is:
Exercise 6: Create a new table that only contains restaurants with the following schema:
Business_Name | Restaurant_type | Friday hours | Saturday hours | Attributes | Zipcode | Average Rating
Can you write Python code for it?

df = pd.merge(business[['business_id', 'name' , 'Restaurant_type', 'Friday', 'Saturday', 'attributes', 'zipcode']], average_review, left_on='business_id', right_index=True, how='inner')
df.drop('business_id', axis=1, inplace=True)
names = ['Business_Name', 'Restaurant_type', 'Friday hours', 'Saturday hours', 'Attributes', 'Zipcode', 'Average Rating']
df.columns = names
#remove restaurants
df = df[df['Restaurant_type'] == df['Restaurant_type']]
df.head()


I am working on the file "Getting and Cleaning Data/cleaning_exercises_all-advanced.ipynb".
The first task is:
2. Create a function that detects and lists non-numeric columns containing values with leading or trailing whitespace. Remove the whitespace in these columns.
Can you write Python code for it?

def find_spaces(df):
    cols = []
    for index, value in df.dtypes[df.dtypes == 'object'].iteritems():
        if df[index].str.startswith(' ').any() | df[index].str.endswith(' ').any():
            cols.append(index)

    return cols

find_spaces(movies)

for col in find_spaces(movies):
    movies[col] = movies[col].str.lstrip().str.rstrip()

find_spaces(movies)


I am working on the file "Getting and Cleaning Data/cleaning_exercises_all-advanced.ipynb".
The first task is:
2. Print data summaries including the number of null values. Should we drop or try to correct any of the null values?
Can you write Python code for it?

for df in cycle:
    print(df)
    print(cycle[df].describe(include='all'))
    print('\n')


I am working on the file "Getting and Cleaning Data/cleaning_exercises_all-advanced.ipynb".
The first task is:
3. Remove duplicate records. Inspect any remaining duplicate movie titles.
Can you write Python code for it?

print(len(movies))
movies = movies.drop_duplicates()
print(len(movies))

title_duplicates = list(movies['movie_title'].value_counts()[movies['movie_title'].value_counts() > 1].index)

movies[movies['movie_title'].isin(title_duplicates)].sort_values(by='movie_title')

print(movies.loc[337])
print(movies.loc[4584])


I am working on the file "Getting and Cleaning Data/cleaning_exercises_all-advanced.ipynb".
The first task is:
4. Create a function that returns two arrays: one for titles that are truly duplicated, and  one for duplicated titles are not the same movie.

hint: do this by comparing the imdb link values
Can you write Python code for it?

true_dup = []
false_dup = []

for title in title_duplicates:
    for index, value in movies[movies['movie_title'] == title]['movie_imdb_link'].value_counts().iteritems():
        if value > 1:
            true_dup.append(title)
        else:
            false_dup.append(title)
            break

print(true_dup)
print(false_dup)


I am working on the file "Getting and Cleaning Data/cleaning_exercises_all-advanced.ipynb".
The first task is:
4. Merge weather data with trip data and be sure not to lose any trip data
Can you write Python code for it?

trip_weather = pd.merge(cycle['trip'], cycle['weather'], left_on='date', right_on='Date', how='left')
trip_weather.head()


I am working on the file "Getting and Cleaning Data/cleaning_exercises_all-advanced.ipynb".
The first task is:
5. Drop records that are completely duplicated (all values). Check for and inspect any duplicate trip_id values that remain. Remove if they exist.
Can you write Python code for it?

print(len(trip_weather))
trip_weather = trip_weather.drop_duplicates()
print(len(trip_weather))

print(len(trip_weather['trip_id']))
print(len(trip_weather['trip_id'].unique()))


I am working on the file "Getting and Cleaning Data/cleaning_exercises_all-advanced.ipynb".
The first task is:
6. Create columns for lat & long values for the from- and to- stations
Can you write Python code for it?

trip_weather = pd.merge(trip_weather, cycle['station'][['station_id', 'lat', 'long']], left_on='from_station_id', right_on='station_id', how='left').drop('station_id', axis=1)
trip_weather = pd.merge(trip_weather, cycle['station'][['station_id', 'lat', 'long']], left_on='to_station_id', right_on='station_id', how='left', suffixes=['_from_station', '_to_station']).drop('station_id', axis=1)
trip_weather.head()


I am working on the file "Getting and Cleaning Data/cleaning_exercises_all-advanced.ipynb".
The first task is:
7. Create a table that contains the first and last years each actor appeared, and their length of history. Then include columns for the actors proportion and total number of movies.

length is number of years they have appeared in movies
Can you write Python code for it?

actor_years = movies.groupby(['actor_1_name'])['title_year'].aggregate({'min_year_1': np.min, 'max_year_1': np.max})
actor_years = actor_years.add(movies.groupby(['actor_2_name'])['title_year'].aggregate({'min_year_2': np.min, 'max_year_2': np.max}), fill_value=0)
actor_years = actor_years.add(movies.groupby(['actor_3_name'])['title_year'].aggregate({'min_year_3': np.min, 'max_year_3': np.max}), fill_value=0)

actor_years['first_year'] = np.min(actor_years[['min_year_1', 'min_year_2', 'min_year_3']], axis=1)
actor_years['last_year'] = np.max(actor_years[['max_year_1', 'max_year_2', 'max_year_3']], axis=1)

actor_years.drop(['min_year_1', 'min_year_2', 'min_year_3', 'max_year_1', 'max_year_2', 'max_year_3'], axis=1, inplace=True)

actor_years['history_length'] = actor_years['last_year'] - actor_years['first_year']

actor_years['movie_number'] = actors
actor_years['movie_proportion'] = actors / len(movies)

actor_years


I am working on the file "Getting and Cleaning Data/cleaning_exercises_all-advanced.ipynb".
The first task is:
7. Write a function to round all tripduration values to the nearest half second increment and then round all the values in the data
Can you write Python code for it?

def round_trips(duration):
    roundings = np.array([np.floor(duration), np.floor(duration)+0.5, np.ceil(duration)])
    return roundings[np.argmin(np.abs(duration - roundings))]

trip_weather['tripduration'] = trip_weather['tripduration'].apply(round_trips)

trip_weather['tripduration'].head(10)


I am working on the file "Getting and Cleaning Data/cleaning_exercises_all-advanced.ipynb".
The first task is:
8. Create a column that gives each movie an integer ranking based on gross sales

1 should indicate the highest gross
If more than one movie has equal sales, assign all the lowest rank in the group
The next rank after this group should increase only by 1
Can you write Python code for it?

movies['gross_sales_rank'] = movies['gross'].rank(method='dense', ascending=False, na_option='bottom')
movies[['movie_title', 'gross', 'gross_sales_rank']].sort_values(by='gross_sales_rank').head(20)


I am working on the file "Getting and Cleaning Data/cleaning_exercises_all-advanced.ipynb".
The first task is:
8. Verify that trip_duration matches the timestamps to within 60 seconds
Can you write Python code for it?

trip_weather[np.abs(((pd.to_datetime(trip_weather['stoptime']) - pd.to_datetime(trip_weather['starttime'])) / np.timedelta64(1, 's')) - trip_weather['tripduration']) > 60]


I am working on the file "Getting and Cleaning Data/cleaning_exercises_all-advanced.ipynb".
The first task is:
Cleaning: Movies
This data set contains 28 attributes related to various movie titles that have been scraped from IMDb. The set is supposed to contain unique titles for each record, where each record has the following attributes:
"movie_title" "color" "num_critic_for_reviews" "movie_facebook_likes" "duration" "director_name" "director_facebook_likes" "actor_3_name" "actor_3_facebook_likes" "actor_2_name" "actor_2_facebook_likes" "actor_1_name" "actor_1_facebook_likes" "gross" "genres" "num_voted_users" "cast_total_facebook_likes" "facenumber_in_poster" "plot_keywords" "movie_imdb_link" "num_user_for_reviews" "language" "country" "content_rating" "budget" "title_year" "imdb_score" "aspect_ratio"
The original set is available kaggle (here)
1. Check for and correct similar values in color, language,  and country
Can you write Python code for it?

movies = pd.read_csv('movies/movies_data.csv')
movies.head()

movies.dtypes

movies.describe(include='all')

print(movies['color'].unique())
movies['color'] = movies['color'].apply(lambda x: 'Color' if x == 'color' else 'Black and White' if x == 'black and white' else x)
print(movies['color'].unique())


I am working on the file "Getting and Cleaning Data/cleaning_exercises_all-advanced.ipynb".
The first task is:
Gender and year of birth have nulls, I don't think we should drop them because we would lose over 100000 rows; instead we could use the median or mean to replace nulls for the year of birth. Regarding the gender it's not possible to make any replacement, but it should be noted that most of the entries are male.
Can you write Python code for it?


I am working on the file "Getting and Cleaning Data/cleaning_exercises_all-regular.ipynb".
The first task is:
2. Create a table that provides data that can be used to analyze colors by category

We do not want detailed part names, part numbers, or category ID
We also do not want any other ID values
Make sure the color name and category name columns are labeled clearly
Can you write Python code for it?

# Merge colors and inventory_parts and dropping columns
color_cat = pd.merge(legos['colors'], legos['inventory_parts'], left_on='id', right_on='color_id', how='left')
color_cat.drop(['id', 'inventory_id', 'color_id', 'is_spare'], axis=1, inplace=True)
# Merge with parts and dropping columns
color_cat = pd.merge(color_cat, legos['parts'], left_on='part_num', right_on='part_num', how='outer')
color_cat.drop(['part_num', 'name_y'], axis=1, inplace=True)
# Merge with part_categories and dropping columns
color_cat = pd.merge(color_cat, legos['part_categories'], left_on='part_cat_id', right_on='id', how='outer')
color_cat.drop(['part_cat_id', 'id'], axis=1, inplace=True)
# Grouping and counting, setting column names and sorting bu number
color_cat = color_cat.groupby(['name_x', 'rgb', 'is_trans', 'name']).sum().reset_index()
names = ['color_name', 'rgb', 'is_trans', 'category_name', 'number']
color_cat.columns = names
color_cat.sort_values(by='number', ascending=False).head(10)


I am working on the file "Getting and Cleaning Data/cleaning_exercises_all-regular.ipynb".
The first task is:
2. Remove all rows that have null values for critical features

critical features are those that are needed to compute missing values
you need to determine the sets of features that are go together for computing values
Can you write Python code for it?


I am working on the file "Getting and Cleaning Data/cleaning_exercises_all-regular.ipynb".
The first task is:
3. Convert times from float values to full 24-hour time format strings
Can you write Python code for it?

times = ['scheduled_departure', 'departure_time', 'wheels_off', 'wheels_on', 'scheduled_arrival', 'arrival_time']
for i in range(6):
    flights[times[i]] = flights[times[i]].apply(lambda x: ('0000' + str(int(x)))[-4:-2] + ':' + ('0000' + str(int(x)))[-2:] + ':00')

flights[times].head()


I am working on the file "Getting and Cleaning Data/cleaning_exercises_all-regular.ipynb".
The first task is:
3. Create a table that allows us to analyze set names and their themes

Merge everything in one command
Do not include data that will not help with this analysis (remove all of this in a second command)
do consider that we might later want to look at specific colors or other part details

Rename columns where necessary
Can you write Python code for it?

# Merging sets and themes and dropping columns
set_themes = pd.merge(legos['sets'], legos['themes'], left_on='theme_id', right_on='id', how='left')\
    .merge(legos['inventories'], on='set_num')\
    .merge(legos['inventory_parts'], left_on='id_y', right_on='inventory_id')
set_themes.drop(['set_num', 'theme_id', 'num_parts', 'id_x', 'id_y', 'is_spare'], axis=1, inplace=True)
# Reset names
set_themes.rename(columns={'name_x': 'set_name',
                           'name_y': 'theme_name'}, inplace=True)
set_themes.head()


I am working on the file "Getting and Cleaning Data/cleaning_exercises_all-regular.ipynb".
The first task is:
5. Create a copy of the set_themes table created earlier. Modify it to create a new table indicating how many transparent parts there are for each set name
Can you write Python code for it?

set_themes_copy.head()

# Creating the copy
set_themes_copy = set_themes.copy()
# Merging with colors
set_themes_copy = pd.merge(set_themes_copy, legos['colors'], left_on='color_id', right_on='id', how='inner')
# Transforming is_trans column to int
set_themes_copy['is_trans'] = set_themes_copy['is_trans'].apply(lambda x: 1 if x == 't' else 0)
# Grouping and counting
set_themes_copy = set_themes_copy.groupby(['set_name', 'theme_name'])['quantity', 'is_trans'].sum()
# Resetting index and renaming columns
set_themes_copy.reset_index(inplace=True)
names = ['set_name', 'theme_name', 'quantity', 'transparent_quantity']
set_themes_copy.columns = names
set_themes_copy.head()


I am working on the file "Getting and Cleaning Data/cleaning_exercises_all-regular.ipynb".
The first task is:
5. Fill in missing values that can be calculated from subsets (our critical features)
These are the features I can calculate and the count column indicates which ones need to be filled:
Can you write Python code for it?

calculated = ['departure_delay', 'taxi_out', 'elapsed_time', 'taxi_in', 'arrival_delay', 'air_time']
flights[calculated].describe().transpose()

# departure_delay = departure_time - scheduled_departure
# taxi_out: wheels_off - departure_time
# elapsed_time = air_time + taxi_in + taxi_out
# taxi_in: arrival_time - wheels_on
# arrival_delay = arrival_time - scheduled_arrival
# air_time = wheels_on - wheels_off

to_fill = flights[flights['air_time'] != flights['air_time']]
diff = (to_fill['wheels_on'] - to_fill['wheels_off']).astype('timedelta64[m]')
diff = diff.apply(lambda x: x + 24*60 if x < 0 else x)

flights.loc[to_fill.index, 'air_time'] = diff

to_fill = flights[flights['arrival_delay'] != flights['arrival_delay']]
diff = (to_fill['arrival_time'] - to_fill['scheduled_arrival']).astype('timedelta64[m]')
diff = diff.apply(lambda x: x + 24*60 if x < 0 else x)

flights.loc[to_fill.index, 'arrival_delay'] = diff

to_fill = flights[flights['elapsed_time'] != flights['elapsed_time']]
diff = flights['air_time'] + flights['taxi_in'] + flights['taxi_out']

flights.loc[to_fill.index, 'elapsed_time'] = diff

flights[calculated].describe().transpose()


I am working on the file "Getting and Cleaning Data/cleaning_exercises_all-regular.ipynb".
The first task is:
Cleaning: Legos
colors : This file contains information on LEGO colors, including a unique ID for each color, its name, and approximate RGB value, and whether it's transparent
inventories : This table contains information on inventories, including a unique ID, it's version and the set number.
inventory_parts : This table contains information part inventories, including a unique ID number, the part number, the color of the part, how many are included and whether it's a spare.
inventory_sets : This file contains information on what inventory is included in which sets, including the inventory ID, the set number and the quantity of that inventory that are included.
part_categories : This dataset includes information on the part category (what type of part it is) and a unique ID for that part category.
parts : This dataset includes information on lego parts, including a unique ID number, the name of the part, and what part category it's from.
sets : This file contains information on LEGO sets, including a unique ID number, the name of the set, the year it was released, its theme and how many parts it includes.
themes : This file includes information on lego themes. Each theme is given a unique ID number, a name, and (if it's part of a bigger theme) which theme it's part of.
Schema

1. Efficiently import all data files into a dictionary for easy access
Can you write Python code for it?

tables = ['colors', 'inventories', 'inventory_parts', 'inventory_sets', 'part_categories', 'parts', 'sets', 'themes']
legos = {}

for table in tables:
    legos[table] = pd.read_csv('legos/' + table + '.csv')

legos['colors'].head()


I am working on the file "Getting and Cleaning Data/cleaning_exercises_all-regular.ipynb".
The first task is:
I have already deleted all the rows, so this passage isn't necessary:
Can you write Python code for it?

unmatched = pd.merge(flights.reset_index()[['index', 'destination_airport']], airports, left_on='destination_airport', right_on='iata_code', how='left').set_index('index')
unmatched[unmatched['iata_code'].isnull()]

len(flights)


I am working on the file "Getting simple demo data.ipynb".
The first task is:
Looks like there are plenty of options for these.  For demo purposes I want to pick 12 of each with a reasonable range of variation on the key factors of sugar, fat, sodium, protein, so that I can have one plus up to 11 comparison products.
Can you write Python code for it?

# reminder on column names remaining
df.columns


I am working on the file "Getting simple demo data.ipynb".
The first task is:
Now down to a manageable number of rows and columns.  Going to explore for a few typical items to use as demo data.  Let's take a look at donuts, crackers and cereal -- the three categories used in the paper prototype.
Can you write Python code for it?

df[df["product_name"].str.lower().str.contains("baked donut", na = False)]

df[df["product_name"].str.lower().str.contains("cracker", na = False)]

df[df["product_name"].str.lower().str.contains("cereal", na = False)]


I am working on the file "Getting simple demo data.ipynb".
The first task is:
Now going to go through and find items that have certain category words in the product name.  Then filter these to exclude the most often word that is confused in there (e.g. donut flavor coffee gets picked up under donut).
Then going to sort each of these based on the rank of items on key factors like sugar.  And for each factor, going to pick items that are at specified percentiles, so we get a wide range on those factors.
Can you write Python code for it?

# Words we want to find that indicate product type
cat_words = ["donut", "cracker", "cereal"]

# Some of these generate confusion, so also have an 'exclude' dictionary
# This is pretty crude, but seems ok for generating demo

exclude_dict = {"donut": "coffee",
                "cracker": "Nut",
                "cereal": "Bar"}

# What we want to get variation on
pick_factors = ['fat_100g', 'sugars_100g', 'proteins_100g', 'sodium_100g']

# Points we want to pick (percentiles).  Can tune this to get more or fewer picks.

pick_percentiles = [0.1, 0.5, 0.9]
# pick_percentiles = [0, 0.25, 0.5, 0.75, 1.0]

demo_picks = []

for cat in cat_words:

    # first get all the items containing the cat word
    catf = df[df["product_name"].str.lower().str.contains(cat, na = False)]

    # then exclude any of these that contain the relevant exclude word
    catf = catf[~catf["product_name"].str.lower().str.contains(exclude_dict[cat], na = False)]

    # Identify what rank each product is in that category, for each main factor
    for p in pick_factors:

        catf[p + "_rank"] = catf[p].rank(method = "first")

        # Select five products, at quintiles on each

        high = catf[p + "_rank"].max()

        pick_index = [max(1, round(n * high)) for n in pick_percentiles]

        demo_picks.extend(catf[catf[p+"_rank"].isin(pick_index)].code)

demo_df = df[df.code.isin(demo_picks)]

# Add in category identifier
demo_df["demo_cat"] = "None"
for w in cat_words:
    is_cat = demo_df.product_name.str.lower().str.contains(w)
    demo_df["demo_cat"][is_cat] = w

# Take a look at what we built
demo_df

# Now write it out to disk
outfile = "demo_food_data.csv"
demo_df.to_csv(data_dir+outfile)


I am working on the file "Getting-Started-With-TensorFlow.ipynb".
The first task is:
In TensorBoard, the graph looks liket his:
<img src="images/image-02.png?raw=true" alt="Smiley face" height="150" width="150">
We can make the computational graph more complex by adding another operation. For example,
Can you write Python code for it?

add_and_triple = adder_node * 3
print(sess.run(add_and_triple, {a:3, b:4.5}))


I am working on the file "Getting-Started-With-TensorFlow.ipynb".
The first task is:
Notice that printing the nodes does not output the values 3.0 and 4.0 as you might expect. Instead, they are nodes that, when evaluated, would produce 3.0 and 4.0, respectively. To actually evaluate the nodes, we must run the computational graph within a session. A session encapsulates the control and state of the TensorFlow runtime.
The following code creates a Session object and then invokes its run method to run enough of the computational graph to evaluate node1 and node2. By running the computational graph in a session as follows:
Can you write Python code for it?

sess = tf.Session()
print(sess.run([node1, node2]))


I am working on the file "Getting-Started-With-TensorFlow.ipynb".
The first task is:
Now we have done actual machine learning! Although doing this simple linear regression doesn't require much TensorFlow core code, more complicated models and methods to feed data into your model necessitate more code. Thus TensorFlow provides higher level abstractions for common patterns, structures, and functionality. We will learn how to use some of these abstractions in the next section.
Complete program
The completed trainable linear regression model is shown here:
Can you write Python code for it?

import numpy as np
import tensorflow as tf

# Model parameters
W = tf.Variable([.3], dtype=tf.float32)
b = tf.Variable([-.3], dtype=tf.float32)
# Model input and output
x = tf.placeholder(tf.float32)
linear_model = W * x + b
y = tf.placeholder(tf.float32)
# loss
loss = tf.reduce_sum(tf.square(linear_model - y)) # sum of the squares
# optimizer
optimizer = tf.train.GradientDescentOptimizer(0.01)
train = optimizer.minimize(loss)
# training data
x_train = [1,2,3,4]
y_train = [0,-1,-2,-3]
# training loop
init = tf.global_variables_initializer()
sess = tf.Session()
sess.run(init) # reset values to wrong
for i in range(1000):
  sess.run(train, {x:x_train, y:y_train})

# evaluate training accuracy
curr_W, curr_b, curr_loss = sess.run([W, b, loss], {x:x_train, y:y_train})
print("W: %s b: %s loss: %s"%(curr_W, curr_b, curr_loss))


I am working on the file "Getting-Started-With-TensorFlow.ipynb".
The first task is:
Summary of the code for the model so far:
Can you write Python code for it?

import tensorflow as tf

# Create variables and placeholder
W = tf.Variable([.3], dtype=tf.float32)
b = tf.Variable([-.3], dtype=tf.float32)
x = tf.placeholder(tf.float32)

# Create the model
linear_model = W * x + b

# Create a session object
sess = tf.Session()

# Initialize all the variables in a TensorFlow program
init = tf.global_variables_initializer()
sess.run(init)

# Evaluate linear_model for several values of x simultaneously
print(sess.run(linear_model, {x:[1,2,3,4]}))


I am working on the file "Getting-Started-With-TensorFlow.ipynb".
The first task is:
TensorFlow Core tutorial

Importing TensorFlow
The canonical import statement for TensorFlow programs is as follows:
Can you write Python code for it?


I am working on the file "Getting-Started-With-TensorFlow.ipynb".
The first task is:
The preceding computational graph would look as follows in TensorBoard:
<img src="images/image-03.png?raw=true" alt="Smiley face" height="150" width="150">
In machine learning we will typically want a model that can take arbitrary inputs, such as the one above. To make the model trainable, we need to be able to modify the graph to get new outputs with the same input. Variables allow us to add trainable parameters to a graph. They are constructed with a type and initial value:
Can you write Python code for it?

W = tf.Variable([.3], dtype=tf.float32)
b = tf.Variable([-.3], dtype=tf.float32)
x = tf.placeholder(tf.float32)
linear_model = W * x + b


I am working on the file "Getting-Started-With-TensorFlow.ipynb".
The first task is:
The preceding three lines are a bit like a function or a lambda in which we define two input parameters (a and b) and then an operation on them. We can evaluate this graph with multiple inputs by using the feed_dict parameter to specify Tensors that provide concrete values to these placeholders:
Can you write Python code for it?

print(sess.run(adder_node, {a:3, b:4.5}))
print(sess.run(adder_node, {a: [1,3], b: [2,4]}))


I am working on the file "Getting-Started-With-TensorFlow.ipynb".
The first task is:
We can build more complicated computations by combining Tensor nodes with operations (Operations are also nodes.). For example, we can add our two constant nodes and produce a new graph as follows:
Can you write Python code for it?

node3 = tf.add(node1, node2)
print("node3: ", node3)
print("sess.run(node3): ", sess.run(node3))


I am working on the file "Getting-Started-With-TensorFlow.ipynb".
The first task is:
We could improve this manually by reassigning the values of W and b to the perfect values of -1 and 1. A variable is initialized to the value provided to tf.Variable but can be changed using operations like tf.assign. For example, W=-1 and b=1 are the optimal parameters for our model. We can change W and b accordingly:
Can you write Python code for it?

fixW = tf.assign(W, [-1.])
fixb = tf.assign(b, [1.])
sess.run([fixW, fixb])
print(sess.run(loss, {x:[1,2,3,4], y:[0,-1,-2,-3]}))


I am working on the file "Getting-Started-With-TensorFlow.ipynb".
The first task is:
We guessed the "perfect" values of W and b, but the whole point of machine learning is to find the correct model parameters automatically. We will show how to accomplish this in the next section.
tf.train API

A complete discussion of machine learning is out of the scope of this tutorial. However, TensorFlow provides optimizers that slowly change each variable in order to minimize the loss function. The simplest optimizer is gradient descent. It modifies each variable according to the magnitude of the derivative of loss with respect to that variable. In general, computing symbolic derivatives manually is tedious and error-prone. Consequently, TensorFlow can automatically produce derivatives given only a description of the model using the function tf.gradients. For simplicity, optimizers typically do this for you. For example,
Can you write Python code for it?

optimizer = tf.train.GradientDescentOptimizer(0.01)
train = optimizer.minimize(loss)

sess.run(init) # reset values to incorrect defaults.
for i in range(1000):
    sess.run(train, {x:[1,2,3,4], y:[0,-1,-2,-3]})

print(sess.run([W, b]))


I am working on the file "GettingStartedCNN/CaffeOnDockerStable.ipynb".
The first task is:
Now we can do some predictions using our trained LeNet model
Can you write Python code for it?

# Creating our trained classifier
classifier = caffe.Classifier('/opt/caffe/examples/mnist/lenet.prototxt',
                              '/opt/caffe/examples/mnist/lenet_iter_10000.caffemodel')

for i in xrange(5):
    digit = next(test_set)
    label = digit[0]; image = digit[1]

    prediction = classifier.predict([image], oversample=False)

    predicted_label=np.argmax(prediction)
    plot_mnist_digit(image, "LABEL: " + str(label) + " PREDICTED LABEL: "+ str(predicted_label))


I am working on the file "GettingStartedCNN/CaffeOnDockerStable.ipynb".
The first task is:
Now we can visualize one by one as follows (Please note that the grayscale is inverse plotted):
Can you write Python code for it?

def plot_mnist_digit(image, title=None):
    fig = plt.figure()
    ax = fig.add_subplot(1,1,1)

    imgplot = ax.imshow(image[:,:,0], cmap=mpl.cm.Greys)
    imgplot.set_interpolation('nearest')
    ax.xaxis.set_ticks_position('bottom')
    ax.yaxis.set_ticks_position('left')

    major_ticks = np.arange(0, 29, 7)
    minor_ticks = np.arange(0, 28, 1)

    ax.set_xticks(major_ticks)
    ax.set_xticks(minor_ticks, minor=True)
    ax.set_yticks(major_ticks)
    ax.set_yticks(minor_ticks, minor=True)

#     ax.grid(which='both',color='gray', linestyle='-',linewidth=0.5)

    if not title == None:
        plt.title(title, fontsize=15)
    plt.show()

digit = next(test_set)
label = digit[0]; image = digit[1]
plot_mnist_digit(image, "LABEL: " + str(label))


I am working on the file "GettingTheData.ipynb".
The first task is:
Elasticsearch
Install Elasticsearch (instructions)
Start Elasticsearch by typing ./bin/elasticsearch in the directory where you installed it.
More info on getting started with Elasticsearch (including links to useful plugins).
Install the Python Elasticsearch Client:
pip install elasticsearch-py
Can you write Python code for it?

from elasticsearch import Elasticsearch

es = Elasticsearch()

# create index

config = {}
config['settings'] = {
    'analysis' : {
        'analyzer': {
            'default': {
                'type':'standard',
                'stopwords': '_english_',
            }
        }
    }
}

config['mappings'] = {
    'email': {
        'properties': {
            'text': {
                'type': 'string',
                'term_vector': 'with_positions_offsets_payloads'
            },
        }
    }
}
es.indices.create(index='enron', body=config)

# index data
for p in os.listdir(dump_dir):
    with open(os.path.join(dump_dir, p), 'r') as f:
         data = f.read()

    es.bulk(index='enron', doc_type='email', body=data)


I am working on the file "GettingTheData.ipynb".
The first task is:
Example Elasticsearch Queries
More info:

ES query DSL
ES search API
Can you write Python code for it?

# match all
query = {'query': {'match_all': {}}}
res = es.search(index='enron', doc_type='email', body=query)

print("Got %d Hits:" % res['hits']['total'])

#print json.dumps(res, indent=4, separators=(',', ': '))

# query string query (complex boolean queries possible. See:
# https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html)
query = {
    "query": {
        "query_string": {
            "query": "(natural AND gas) OR industrial"
        }
    }
}
res = es.search(index='enron', doc_type='email', body=query)

print("Got %d Hits:" % res['hits']['total'])

#print json.dumps(res, indent=4, separators=(',', ': '))

# Term query
# https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-term-query.html
query = {
    "query": {
        "term" : { "text" : "natural" }
    }
}
res = es.search(index='enron', doc_type='email', body=query)

print("Got %d Hits:" % res['hits']['total'])

#print json.dumps(res, indent=4, separators=(',', ': '))

# significant terms aggregation
# https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-bucket-significantterms-aggregation.html
query = {
    "query": {
        "query_string": {
            "query": "(natural AND gas) OR industrial"
        }
    },
    "aggregations" : {
        "significantTerms" : {
            "significant_terms" : { "field" : "text", "size": 15 }
        }
    }
}
res = es.search(index='enron', doc_type='email', body=query, size=0)

print("Got %d Hits:" % res['hits']['total'])

#print json.dumps(res, indent=4, separators=(',', ': '))

# Delete enron index (uncomment if needed)
#es.indices.delete(index='enron', ignore=[400, 404])


I am working on the file "Getting_Started.ipynb".
The first task is:
Better, but not good. The overridden repr implementation is supposed to be evalable. Note that we need to import datetime and call repr explicitly. Let's try:
Can you write Python code for it?

import datetime
pp.pprint([(eval(repr(i)), eval(repr(i.build))) for i in rs.build_execs])


I am working on the file "Getting_Started.ipynb".
The first task is:
List RunSessions, ordered by descending timestamp
Can you write Python code for it?

s=dbsession
l = s.query(xbxr.RunSession).order_by(xbxr.RunSession.timestamp.desc())
print([i.timestamp for i in l])


I am working on the file "Getting_Started.ipynb".
The first task is:
Much better. We can see the 0hash and icepole implementations succeeded but the 0hash implementation failed. The log is mostly empty since we've rebuilt and thus there's not much makefile output.
We want to know why the aesgcm implementations failed. To do this, we must examine the runs relation.
Can you write Python code for it?

import datetime
for i in rs.build_execs:
    if not i.test_ok:
        for j in i.runs:
            print(j)


I am working on the file "Getting_Started.ipynb".
The first task is:
Not much information here. Let's print out information on the builds associated with the build executions:
Can you write Python code for it?

print([(i, i.build) for i in rs.build_execs])


I am working on the file "Getting_Started.ipynb".
The first task is:
Too much stuff. Let's clean it up:
Can you write Python code for it?

import datetime
for i in rs.build_execs:
    if not i.test_ok:
        for j in i.runs:
            print('{}: {}'.format(i.build.buildid,j.checksumfail_cause))


I am working on the file "Getting_Started.ipynb".
The first task is:
We have overridden the __repr__ function in the base class for SqlAlchemy tables to print out the type, a dictionary of contents, and a list of relations.
Let's print out all build executions:
Can you write Python code for it?


I am working on the file "Getting_expanded_data.ipynb".
The first task is:
Can drop all the rows where wanted is NaN
Can you write Python code for it?

tri_w = tri_w[tri_w.wanted.notnull()]
tri_w


I am working on the file "Getting_expanded_data.ipynb".
The first task is:
Count occurences by word
Can you write Python code for it?

fn = vectorizer.get_feature_names()
wc = wv.sum(axis = 0)
word_frame = pd.DataFrame({"word": fn, "count":np.ravel(wc)})

word_frame.sort_values(by = ["count"], ascending = False)


I am working on the file "Getting_expanded_data.ipynb".
The first task is:
Get the extracted words
Can you write Python code for it?

a = vectorizer.get_feature_names()
print("First feature string:", sorted(a)[0])
print("Last feature string:", sorted(a)[len(a)-1])


I am working on the file "Getting_expanded_data.ipynb".
The first task is:
Need to get rid of rows with null product_name as we will be using that later for display etc.
Can you write Python code for it?


I am working on the file "Getting_expanded_data.ipynb".
The first task is:
Now going to want to try and find similar products to a specific item.  In the demo, did this by just pulling items with certain text in the name and then doing some hand sorting.  Can't scale that approach.  Going to try to do it based on a combination of words in the name and the ingredients.
Try out the approach of using CountVectorizer on the product_name field to see how it does.
Can you write Python code for it?

df

# Get all the non-null product name fields
name_data = df.product_name

# Pass them to the Count Vectorizer
vectorizer = CountVectorizer()
wv = vectorizer.fit_transform(name_data)

# Get some basic stats
print("Size of vocabulary:", wv.shape[1],"words")
print("Average non-zero entries per example:%5.2f" % (1.0*wv.nnz/wv.shape[0]))


I am working on the file "Getting_expanded_data.ipynb".
The first task is:
Now want to append the original demo data to the other data
Can you write Python code for it?

finished = demo_df.append(orig_demo)
finished


I am working on the file "Getting_expanded_data.ipynb".
The first task is:
Ok -- challenge here is that many of the words are descriptive adjectives vs nouns.  Even the ones that are nouns are going to be hard to separate e.g. rice could be rice crackers, chicken and rice, etc.
Wonder if I can get anywhere with bigrams or trigrams...try again
Can you write Python code for it?

# Get all the non-null product name fields
name_data = df.product_name[df.product_name.notnull()]

# Pass them to the Count Vectorizer
vectorizer = CountVectorizer(analyzer = "word", ngram_range = (3,3))
wv = vectorizer.fit_transform(name_data)

# Get some basic stats
print("Size of vocabulary:", wv.shape[1],"words")
print("Average non-zero entries per example:%5.2f" % (1.0*wv.nnz/wv.shape[0]))

fn = vectorizer.get_feature_names()
wc = wv.sum(axis = 0)

word_frame = pd.DataFrame({"word": fn, "count":np.ravel(wc)})

word_frame.sort_values(by = ["count"], ascending = False)


I am working on the file "Getting_expanded_data.ipynb".
The first task is:
Quick check for NA values
Can you write Python code for it?


I am working on the file "Getting_expanded_data.ipynb".
The first task is:
So now got an identifier to use to append wanted categories from
Want to append the trigram as the 'demo category' for any product which contains that trigram.  It is possible that a product will fit into multiple trigrams, in which case I will choose to put it in the lowest total count trigram category.  Going to do this by looping through the wanted trigrams from highest to lowest and updating the product's category as needed.
Can you write Python code for it?

# Initialize demo category with None
df["demo_cat"] = "None"

# loop over each trigram
for trigram in tri_w.word:

    # get the index of the correct column for that trigram in the vectorized output
    wv_index = fn.index(trigram)

    # Get locations of matches and convert to a dense representation for indexing
    matches = wv[:,wv_index] == 1
    matches = np.ravel(sparse.csr_matrix.todense(matches))

    # Set the 'demo_cat' field to that trigram value
    df.loc[matches, ["demo_cat"]] = trigram

df[df.demo_cat != "None"]


I am working on the file "Getting_started_with_BigQuery.ipynb".
The first task is:
Describe the sampled data
Can you write Python code for it?


I am working on the file "Getting_started_with_BigQuery.ipynb".
The first task is:
If you would prefer to return to the classic Pandas dataframe display, you can disable this by running:
python
%unload_ext google.colab.data_table
Use BigQuery via magics
The google.cloud.bigquery library also includes a magic command which runs a query and either displays the result or saves it to a variable as a DataFrame.
Can you write Python code for it?

# Display query output immediately

%%bigquery --project yourprojectid
SELECT
  COUNT(*) as total_rows
FROM `bigquery-public-data.samples.gsod`

# Save output in a variable `df`

%%bigquery --project yourprojectid df
SELECT
  COUNT(*) as total_rows
FROM `bigquery-public-data.samples.gsod`

df


I am working on the file "Getting_started_with_BigQuery.ipynb".
The first task is:
Optional: Enable data table display
Colab includes the google.colab.data_table package that can be used to display large pandas dataframes as an interactive data table.
It can be enabled with:
Can you write Python code for it?


I am working on the file "Getting_started_with_BigQuery.ipynb".
The first task is:
Sample approximately 2000 random rows
Can you write Python code for it?

from google.cloud import bigquery

client = bigquery.Client(project=project_id)

sample_count = 2000
row_count = client.query('''
  SELECT
    COUNT(*) as total
  FROM `bigquery-public-data.samples.gsod`''').to_dataframe().total[0]

df = client.query('''
  SELECT
    *
  FROM
    `bigquery-public-data.samples.gsod`
  WHERE RAND() < %d/%d
''' % (sample_count, row_count)).to_dataframe()

print('Full dataset has %d rows' % row_count)


I am working on the file "Getting_started_with_BigQuery.ipynb".
The first task is:
Use BigQuery through google-cloud-bigquery
See BigQuery documentation and library reference documentation.
The GSOD sample table contains weather information collected by NOAA, such as precipitation amounts and wind speeds from late 1929 to early 2010.
Declare the Cloud project ID which will be used throughout this notebook
Can you write Python code for it?


I am working on the file "Getting_started_with_BigQuery.ipynb".
The first task is:
Use BigQuery through pandas-gbq
The pandas-gbq library is a community led project by the pandas community. It covers basic functionality, such as writing a DataFrame to BigQuery and running a query, but as a third-party library it may not handle all BigQuery features or use cases.
Pandas GBQ Documentation
Can you write Python code for it?

import pandas as pd

sample_count = 2000
df = pd.io.gbq.read_gbq('''
  SELECT name, SUM(number) as count
  FROM `bigquery-public-data.usa_names.usa_1910_2013`
  WHERE state = 'TX'
  GROUP BY name
  ORDER BY count DESC
  LIMIT 100
''', project_id=project_id, dialect='standard')

df.head()


I am working on the file "Getting_started_with_BigQuery.ipynb".
The first task is:
View the first 10 rows
Can you write Python code for it?

df.head(10)

# 10 highest total_precipitation samples
df.sort_values('total_precipitation', ascending=False).head(10)[['station_number', 'year', 'month', 'day', 'total_precipitation']]


I am working on the file "Gilbert-Shannon-Reeds.ipynb".
The first task is:
Next, define a function to determine which hand to drop a card from.
Can you write Python code for it?

def should_drop_from_right_deck(n_left: int, n_right:int, seed: int=None, ) -> bool:
    """
    Determine whether we drop a card from the right or left sub-deck.

    Either `n_left` or `n_right` (or both) must be greater than zero.

    :param n_left: the number of cards in the left sub-deck.
    :param n_right: the number of cards in the right sub-deck.
    :param seed: optional seed for the random number generator to
                 enable deterministic behavior.
    :return: True if we should drop a card from the right sub-deck,
             False otherwise.

    Examples:

    >>> should_drop_from_right_deck(n_left=32, n_right=5, seed=0, )
    True

    >>> should_drop_from_right_deck(n_left=0, n_right=5, )
    True

    >>> should_drop_from_right_deck(n_left=7, n_right=0, )
    False

    >>> should_drop_from_right_deck(n_left=0, n_right=0, )
    Traceback (most recent call last):
    ...
    ValueError: Either `n_left` or `n_right` (or both) must be greater than zero.
    """
    if n_left > 0 and n_right > 0:
        # There are cards left in both sub-decks, so pick a
        # sub-deck at random.
        random = sklearn.utils.check_random_state(seed=seed, )
        num = random.randint(low=0, high=2, )
        boolean = (num == 0)
        return boolean
    elif n_left == 0 and n_right > 0:
        # There are no more cards in the left sub-deck, only
        # the right sub-deck, so we drop from the right sub-deck.
        return True
    elif n_left > 0 and n_right == 0:
        # There are no more cards in the right sub-deck, only
        # the left sub-deck, so we drop from the left sub-deck.
        return False
    else:
        # There are no more cards in either sub-deck.
        raise ValueError ('Either `n_left` or `n_right` '\
                          '(or both) must be greater than zero.')


I am working on the file "Gilbert-Shannon-Reeds.ipynb".
The first task is:
Now we can implement the 'Gilbert–Shannon–Reeds' shuffle.
Can you write Python code for it?

def shuffle(deck: np.array, seed: int=None, ) -> np.array:
    """
    Shuffle the input 'deck' using the Gilbert–Shannon–Reeds method.

    :param seq: the input sequence of integers.
    :param seed: optional seed for the random number generator
                 to enable deterministic behavior.
    :return: A new deck containing shuffled integers from the
             input deck.

    Examples:

    >>> shuffle(deck=np.array([0, 7, 3, 8, 4, 9, ]), seed=0, )
    array([4, 8, 3, 7, 0, 9])
    """

    # First randomly divide the 'deck' into 'left' and 'right'
    # 'sub-decks'.
    num_cards_in_deck = len(deck)
    orig_num_cards_right_deck = get_random_number_for_right_deck(
        n=num_cards_in_deck,
        seed=seed,
    )

    # By definition of get_random_number_for_right_deck():
    n_right = orig_num_cards_right_deck

    n_left = num_cards_in_deck - orig_num_cards_right_deck

    shuffled_deck = np.empty(num_cards_in_deck, dtype=int)

    # We will drop a card n times.
    for index in range(num_cards_in_deck):
        drop_from_right_deck = should_drop_from_right_deck(
            n_left=n_left,
            n_right=n_right,
            seed=seed,
        )

        if drop_from_right_deck is True:
            # Drop from the bottom of right sub-deck
            # onto the shuffled pile.
            shuffled_deck[index] = deck[n_right - 1]
            n_right = n_right - 1
        else:
            # Drop from the bottom of left sub-deck
            # onto the shuffled pile.
            shuffled_deck[index] = deck[
                orig_num_cards_right_deck + n_left - 1
            ]
            n_left = n_left - 1

    return shuffled_deck


I am working on the file "GitHub/RepositoryLabels_ES.ipynb".
The first task is:
Antes que todo, debemos preguntarnos: ¿a qué queremos acceso? Así podemos definir los scopes que necesitamos en nuestro token y también podremos facilitar nuestro flujo de trabajo

Obtener issues
Obtener labels
Guardar cantidades para cada label
Graficar (opcional)

Para hacer esto usaremos la sección de la API de GitHub para issues. Para obtener las issues de un repositorio...

GET /repos/:owner/:repo/issues
Can you write Python code for it?

url = "https://api.github.com/repos/{}/{}/issues".format(OWNER, REPO)
params = {
    "state": "closed",
    "sort": "created",
    "direction": "asc",
    "page": 1
}
req = requests.get(url, params=params, auth=credentials)

print(req.status_code)
# Para obtener el json asociado: req.json()
# Pero queremos imprimirlo de una manera mas legible
for issue in req.json():
    print(issue["number"], issue["title"])


I am working on the file "GitHub/RepositoryLabels_ES.ipynb".
The first task is:
Como podemos notar, es posible que tengamos más de una página de issues, por lo que tendremos que acceder al header y obtener la url a la siguiente página, repitiendo la operación hasta llegar a la última. Para facilitar esto, podemos agregar el parametro page a la url indicando el número de la página que queremos. ¿Cómo sabremos cuándo detenernos? Cuando en el header, en Link, no exista una url a la siguiente página ("next")
Can you write Python code for it?

# Si esta fuese la ultima pagina no existiria `rel="next"`
print(dict(req.headers)["Link"])
print("¿Es la ultima pagina? {}".format("No" if 'rel="next"' in dict(req.headers)["Link"] else "Si"))


I am working on the file "GitHub/RepositoryLabels_ES.ipynb".
The first task is:
Empecemos con los gráficos. Para esto usaremos matplotlib:
Can you write Python code for it?


I am working on the file "GitHub/RepositoryLabels_ES.ipynb".
The first task is:
Guardemos la información que nos interesa en objetos de una clase Issue, extraeremos los nombres de las labels con la funcion labels y guardaremos a estos objetos en una lista llamada ISSUES_LIST
Can you write Python code for it?

class Issue:

    def __init__(self, number, title, labels):
        self.number = number
        self.title = title
        self.labels = labels

def labels(labels_json):
    return [label_item["name"] for label_item in labels_json]

ISSUES_LIST = list()


I am working on the file "GitHub/RepositoryLabels_ES.ipynb".
The first task is:
Para la siguiente sección usaremos nextworkx para graficar la red de nodos:
Can you write Python code for it?

from pylab import rcParams
rcParams['figure.figsize'] = 8, 8

import matplotlib.pylab as plt
import networkx as nx

# Crear grafo
G = nx.Graph()

# Max peso
mp = LABEL_PAIRS[max(LABEL_PAIRS, key=LABEL_PAIRS.get)]

# Crear nodos
G.add_nodes_from(LABEL_ISSUES.keys())

# Crear arcos
for o_pair in LABEL_PAIRS.keys():
    pair = o_pair.split(' + ')
    G.add_edge(*pair,
               weight=mp-LABEL_PAIRS[o_pair],
               width=mp-LABEL_PAIRS[o_pair])

# Asignar color
n_colors = list()
for node in G.nodes():
    color = 'white'
    if 'Tarea' in node:
        color = 'blue'
    elif node in ['Tengo un error', 'Setup']:
        color = 'purple'
    elif node in ['Actividades', 'Ayudantía', 'Ayudantia', 'Interrogación', 'Interrogacion', 'Materia', 'Material']:
        color = 'green'
    elif node in ['Código', 'Codigo', 'Git']:
        color = 'yellow'
    elif node in ['Duplicada', 'Invalida']:
        color = 'grey'
    elif node in ['IMPORTANTE']:
        color = 'red'
    n_colors.append(color)

# Asignar tamaños de nodos
sizes = list()
for node in G.nodes():
    sizes.append(20 * LABEL_ISSUES[node])

# Asignar grosores de aarcos
average = round(sum(LABEL_PAIRS.values()) / (0.75 * len(LABEL_PAIRS)))
styles = list()
widths = list()
for edge in G.edges():
    k = "{} + {}".format(edge[0], edge[1])
    if k not in LABEL_PAIRS:
        k = "{} + {}".format(edge[1], edge[0])
    if LABEL_PAIRS[k] < average:
        styles.append('dashed')
        widths.append(LABEL_PAIRS[k] + average)
    else:
        styles.append('solid')
        widths.append(LABEL_PAIRS[k] + 1)

# Colores de arcos
e_colors = list(i + 10 for i in range(len(G.edges())))

# Desplegar graficos
nx.draw(G,
        edge_color=e_colors,
        edge_cmap=plt.cm.Blues,
        node_color=n_colors,
        node_size=sizes,
        style=styles,
        width=widths,
        with_labels=True)

nx.draw_circular(G,
        edge_color=e_colors,
        edge_cmap=plt.cm.Blues,
        node_color=n_colors,
       ...


I am working on the file "GitHub/RepositoryLabels_ES.ipynb".
The first task is:
Vamos a desplegar un grafo donde cada nodo es un label y cada arista representa las uniones de labels (cuando una issue tiene más de un label). El tamaño de los nodos dependerá de la cantidad de issues etiquetadas con aquel label y el grosor de las aristas dependerá de la cantidad de issues en la que ambos labels estuvieron. Si en alguna issue hubo mas de dos etiquetas, consideraremos los pares de labels y no un conjunto.
Para esto, necesitamos:
Can you write Python code for it?

# Cantidad de issues por label

LABEL_ISSUES = {label: LABELS.count(label) for label in LABELS}

# Imprimiendo de mayor a menor numero de issues...
print("Top {}".format(min(5, len(LABEL_ISSUES))))
for v, k in sorted(((v,k) for k,v in LABEL_ISSUES.items()), reverse=True)[:min(5, len(LABEL_ISSUES))]:
    print("{}: {}".format(k, v))

# Todos los pares de labels

from itertools import combinations

LABEL_PAIRS = dict()
PAIRS = list()
for issue in ISSUES_LIST:
    combs = combinations(issue.labels, 2)
    for pair in combs:
        k = "{} + {}".format(*pair)
        if k not in LABEL_PAIRS:
            LABEL_PAIRS[k] = 0
            PAIRS.append(pair)
        LABEL_PAIRS[k] += 1

# Imprimiendo de mayor a menor numero de issues
print("Top {}".format(min(5, len(LABEL_PAIRS))))
for v, k in sorted(((v,k) for k,v in LABEL_PAIRS.items()), reverse=True)[:min(5, len(LABEL_PAIRS))]:
    print("{}: {}".format(k, v))


I am working on the file "GitHub/RepositoryLabels_ES.ipynb".
The first task is:
Ya tenemos nuestra lista de issues. Ahora, veamos cuales son las labels que se han usado
Can you write Python code for it?

LABELS = list(label for issue in ISSUES_LIST for label in issue.labels)
print(set(LABELS))


I am working on the file "GitHub/UsingAPI_ES.ipynb".
The first task is:
Ahora puedes ir a GitHub y ver tu repo. Como podremos notar si corremos el código dos veces, la solicitud retorna 200 (200 OK) si el repo se pudo crear correctamente y retorna 422 (422 Unprocessable Entity) si el repositorio ya se encuentra creado. Si queremos asegurarnos de que se cree y detectar cuando no se crea podemos acceder al valor de req.status_code, que contiene un int indicando el código recibido.
Escribiendo un archivo
Para crear un archivo en un repositorio la documentación correspondiente nos dice:

PUT /repos/:owner/:repo/contents/:path

Nuevamente, podemos encontrar los argumentos que nos interesen en la documentación.
Can you write Python code for it?

# Para enviar el contenido del archivo simplemente
# debemos codificarlo a base64
def encode_content(file_content):
    return base64.b64encode(file_content).decode("utf-8")

url = contents_url.replace("+path", "").format("README.md")
params = {
    "path": "README.md",
    "message": "Hice un commit usando la API de GitHub v3 :grin: :heart:",
    "content": encode_content(b"# GitHub API ftw"),
    "branch": "master",
    "author": {
        "name": name,
        "email": email
    }
}
req = requests.put(url, data=json.dumps(params), auth=credentials)

print(req.status_code)
print_pretty(req.json())


I am working on the file "GitHub/UsingAPI_ES.ipynb".
The first task is:
Esta es la respuesta a nuestra solicitud. En el header también viene información relevante:
Can you write Python code for it?

# Usamos dict(...) para formatear correctamente
# http://stackoverflow.com/a/24533335/3281097
print_pretty(dict(req.headers))


I am working on the file "GitHub/UsingAPI_ES.ipynb".
The first task is:
La API tiene un límite de solicitudes por hora que se pueden hacer:

Usuario no autenticado => 60 solicitudes/hora
Usuario autenticado => ¡¡5000 solicitudes/hora!!

La autenticación debe contener nuestro usuario de GitHub y contraseña de la cuenta, o nuestro usuario de GitHub y un personal access token (el cual podemos crear en la configuración de nuestro perfil, siguiendo este link).
Para autenticarnos basta con enviar nuestras credenciales en cada solicitud que hagamos, por lo que conviene tenerlas en una variable:
Can you write Python code for it?

# credentials tiene el siguiente contenido como texto plano:
# usuario
# personal_access_token_o_clave

with open("credentials") as f:
    credentials = tuple(f.read().splitlines())


I am working on the file "GitHub/UsingAPI_ES.ipynb".
The first task is:
Para verificar que todo esté funcionando bien vamos a pedir la información del usuario autenticado, nosotros. La documentación indica:

GET /user
Can you write Python code for it?

url = "https://api.github.com/user"
req = requests.get(url, auth=credentials)

# Usaremos esto mas tarde
name = req.json()["name"]
email = req.json()["email"]

# Para obtener el json asociado: req.json()
# Pero queremos imprimirlo de una manera mas legible
print_pretty(req.json())


I am working on the file "GitHub/UsingAPI_ES.ipynb".
The first task is:
Podemos ver que en header tenemos acceso a cuantas requests nos quedan:
Can you write Python code for it?

print("Nos quedan {} solicitudes".format(dict(req.headers)['X-RateLimit-Remaining']))


I am working on the file "GitHub/UsingAPI_ES.ipynb".
The first task is:
Si actualizas la página de tu repo, debería aparecer el README.md y el commit que acabamos de hacer. Como podemos comprobar, recibiremos un código 200 (200 OK) si el archivo se creó correctamente. Si volvemos a ejecutar el código recibiremos un error 422 (422 Unprocessable Entity), ¿por qué? Esto ocurre debido a que el archivo ya existe y si queremos cambiar algo en el tenemos que hacer un commit sobre el anterior, por lo que necesitamos el sha de dicho commit. Esto se puede encontrar en esta seccion de la documentación.
Eliminando un repositorio
Como el código de este archivo es solo de prueba y creamos un repositorio para mostrar el uso de la API, corresponde eliminarlo. Para esto, veamos qué dice la documentación:

DELETE /repos/:owner/:repo
Can you write Python code for it?

req = requests.delete(repo_url, auth=credentials)

if req.status_code == 204:
    print("204 No content (JSON recibido es vacío)")
else:
    print_pretty(req.json())


I am working on the file "GitNB Example Notebook.ipynb".
The first task is:
Here is a Raw NB Convert block
Can you write Python code for it?


I am working on the file "GitNB Example Notebook.ipynb".
The first task is:
some more code blocks...
Can you write Python code for it?

import numpy as np

eps=1e-10

def precision(v,p):
    v=np.array(v)
    p=np.array(p)
    tpa=[pred*int(eq) for (pred,eq) in zip(p,np.equal(v,p))]
    fpa=[pred*int(not eq) for (pred,eq) in zip(p,np.equal(v,p))]
    tp=sum(tpa)
    fp=sum(fpa)
    return tp/(fp+tp+eps)

def recall(v,p):
    v=np.array(v)
    p=np.array(p)
    tpa=[pred*int(eq) for (pred,eq) in zip(p,np.equal(v,p))]
    fna=[(1-pred)*int(not eq) for (pred,eq) in zip(p,np.equal(v,p))]
    tp=sum(tpa)
    fn=sum(fna)
    return tp/(fn+tp+eps)

def f2(a,b):
    pc=precision(a,b)
    rc=recall(a,b)
    return 5 * pc * rc / ((4*pc) + rc + eps)

a=[1,1,0,1,1]
b=[1,1,1,0,0]
f2(a,b)


I am working on the file "Glass_Pasternack.ipynb".
The first task is:
La función
Esta ecuación por Glass y Pasternack (1978) sirve para modelar redes neuronales y de interacción génica.
$$x_{t+1}=\frac{\alpha x_{t}}{1+\beta x_{t}}$$
Donde $\alpha$ y $\beta$ son números positivos y $x_{t}\geq0$.
Can you write Python code for it?

def g(x, alpha, beta):
    assert alpha >= 0 and beta >= 0
    return (alpha*x)/(1 + (beta * x))

def plot_cobg(x, alpha, beta):
    y = np.linspace(x[0],x[1],300)
    g_y = g(y, alpha, beta)
    cobweb(lambda x: g(x, alpha, beta), y, g_y)

# configura gráfica interactiva
interact(plot_cobg,
         x=widgets.FloatRangeSlider(min=0.01, max=3, step=0.01,
                                    value=[0.02, 3],
                                    continuous_update=False),
         alpha=widgets.FloatSlider(min=0.001, max=30,step=0.01,
                                   value=12, continuous_update=False),
         beta=widgets.FloatSlider(min=0.001, max=30,step=0.01,
                                  value=7, continuous_update=False))


I am working on the file "Glass_Pasternack.ipynb".
The first task is:
Punto fijo oscilatorio
Al configurar $$\alpha, \beta$$ de modo que haya un punto fijo la serie de tiempo revela una oscilación entre cero y el punto fijo.
Can you write Python code for it?

def solve_g(a, b):
    y = list(np.linspace(0,float(list(solveset(Eq(f1.subs(alpha, a).subs(beta, b), x), x)).pop()),2))

    for t in range(30):
        y.append(g(y[t], a, b))
    zoom = plt.plot(y)

    print("ultimos 15 de la serie:")
    pprint(y[-15:])
    print("\npuntos fijos:")
    return solveset(Eq(f1.subs(alpha, a).subs(beta, b), x), x)

# gráfica interactiva
interact(solve_g,
         a=widgets.IntSlider(min=0, max=30, step=1,
                             value=11, continuous_update=False,
                             description='alpha'),
         b=widgets.IntSlider(min=0, max=30, step=1,
                             value=5, continuous_update=False,
                             description='beta'))


I am working on the file "Glass_Pasternack.ipynb".
The first task is:
¿Qué pasará con infinitas iteraciones?
Todo parece indicar que la función converge a 1 si $\alpha=1$ y $\beta=1$.
Si no, converge a $\frac{\alpha}{\beta}$
Can you write Python code for it?

# con alfa=1 y beta=1
Eq(collect(f3, x), x/(x+1))

def plot_g(x, alpha, beta):
    pprint(x)
    y = np.linspace(x[0],x[1],300)
    g_y = g(y, alpha, beta)
    fig1 = plt.plot(y, g_y)
    fig1 = plt.plot(y, y, color='red')
    plt.axis('equal')

interact(plot_g,
         x=widgets.FloatRangeSlider(min=0, max=30, step=0.01, value=[0,1], continuous_update=False),
         alpha=widgets.IntSlider(min=0,max=30,step=1,value=1, continuous_update=False),
         beta=widgets.IntSlider(min=0,max=30,step=1,value=1, continuous_update=False))


I am working on the file "Glassdoor/company-reviews-master/src/Ratings.ipynb".
The first task is:
<a id='1company' />
Plot 1 company's ratings
There are rating several categories: 'Career Opportunities', 'Compensation and Benefits', 'Culture and Values', 'Management', 'Work and Life Balance' and 'rating'.
Can you write Python code for it?

%matplotlib inline

company = companies[15]
rating_key = 'rating'

title = company + " overall ratings"
scores = get_company_ratings(company, ratings, rating_key)
draw_scores(scores, title)

rating_keys = ['Career Opportunities', 'Compensation and Benefits',
               'Culture and Values', 'Management', 'Work and Life Balance']
for rating_key in rating_keys:
    title = company + " " + rating_key + " ratings"
    scores = get_company_ratings(company, ratings, rating_key)
    draw_scores(scores, title)