I am working on the file "04-Pandas-Data-Tables/APD-Crime-Data_class.ipynb".
The first task is:
Load data (don't change this if you're running the notebook on the cluster)
We have two files
- /home/data/APD/COBRA083016_2015.xlsx for 2015
- /home/data/APD/COBRA083016.xlsx from 2009 to current date
Can you write Python code for it?

%%time
df = pd.read_excel('/home/data/APD/COBRA083016_2015.xlsx', sheetname='Query')

df.shape

for c in df.columns:
    print(c)

df[0:5]

df.describe()

df.offense_id.min(), df.offense_id.max()

crime_summary = df.groupby(['UC2 Literal', 'Zone']).offense_id.count()

crime_summary.index

crime_summary.reset_index().head()

df["Zone"] = df.beat // 100

df


I am working on the file "04-Pandas-Data-Tables/APD-Crime-Data_class.ipynb".
The first task is:
Normalized over the annual average
Can you write Python code for it?

c = 3 ## 'BURGLARY-RESIDENCE'
fig = plt.figure(figsize=(20,10))
for y in years:
    avg = resburglaries.loc[y].mean()
    plt.hlines(avg, 1, 13, linestyle='dashed')
    plt.plot(resburglaries.loc[y].index, resburglaries.loc[y], label=("%4.0f"%y))
plt.legend()
plt.title("Seasonal Trends - %s (with annuale averages)"%crime_types[c], fontsize=20)
plt.xticks(list(range(1,13)), ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'])
plt.xlim(0,13)
None

c = 3 ## 'BURGLARY-RESIDENCE'
fig = plt.figure(figsize=(20,10))
for y in years:
    avg = resburglaries.loc[y].mean()
    std = resburglaries.loc[y].std()
    ##plt.hlines(avg, 1, 13, linestyle='dashed')
    plt.plot(resburglaries.loc[y].index, (resburglaries.loc[y]-avg)/std, label=("%4.0f"%y))
plt.legend()
plt.title("Seasonal Trends - %s (normalized)"%crime_types[c], fontsize=20)
plt.xticks(list(range(1,13)), ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'])
plt.xlim(0,13)
plt.ylabel("Standard deviations $\sigma_y$")
None

seasonal_adjust = resburglaries.reset_index().groupby(by=['occur_month']).offense_id.agg('mean')


I am working on the file "04-Pandas-Data-Tables/APD-Crime-Data_class.ipynb".
The first task is:
Part 1 - Observations from the data
Part 2 - Seasonal Model
Can you write Python code for it?

## load complete dataset
dff = pd.read_excel('/home/data/APD/COBRA083016.xlsx', sheetname='Query')

dff.shape

for evt in ['occur', 'poss']:
    dff['%s_ts'%evt] = pd.to_datetime(dff['%s_date'%evt]+' '+dff['%s_time'%evt])
dff['rpt_ts'] = pd.to_datetime(dff.rpt_date)

', '.join(dff.columns)

dff['occur_year'] = dff.occur_ts.dt.year
dff['occur_month'] = dff.occur_ts.dt.month
dff['occur_dayweek'] = dff.occur_ts.dt.dayofweek


I am working on the file "04-Pandas-Data-Tables/APD-Crime-Data_class.ipynb".
The first task is:
Part 2 - Seasonal Model
Can you write Python code for it?


I am working on the file "04-Pandas-Data-Tables/APD-Crime-Data_orig.ipynb".
The first task is:
Atlanta Police Department
The Atlanta Police Department provides Part 1 crime data at http://www.atlantapd.org/crimedatadownloads.aspx
A recent copy of the data file is stored in the cluster. Please, do not copy this data file into your home directory!
Can you write Python code for it?

### Load libraries
%matplotlib inline
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

help(plt.legend)


I am working on the file "04-Pandas-Data-Tables/APD-Crime-Data_orig.ipynb".
The first task is:
Convert into date-time type
Can you write Python code for it?

df['occur_ts'] = pd.to_datetime(df.occur_date+' '+df.occur_time)

#df[['offense_id', 'occur_date', 'occur_time', 'occur_ts', 'rpt_date']][1:10]

df['occur_ts'] = pd.to_datetime(df.occur_date+' '+df.occur_time)
df['occur_month'] = df['occur_ts'].map(lambda x: x.month)
df['occur_woy'] = df.occur_ts.dt.weekofyear

df.describe()

resdf = df.groupby(['UC2 Literal', 'occur_month']).offense_id.count()
resdf

resdf['BURGLARY-RESIDENCE'].as_matrix()

resdf['BURGLARY-RESIDENCE'].iloc(0)

%matplotlib inline
fig = plt.figure(figsize=(10,6)) # 10inx10in
#plt.plot(resdf['BURGLARY-RESIDENCE'].index, resdf['BURGLARY-RESIDENCE'])
plt.scatter(resdf['BURGLARY-RESIDENCE'].index, resdf['BURGLARY-RESIDENCE'], marker='x')
plt.scatter(resdf['BURGLARY-NONRES'].index, resdf['BURGLARY-NONRES'], marker='o')

plt.ylim(0, 500)
plt.title('BURGLARY-RESIDENCE')
plt.xticks(range(13), ['', 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'])
fig.savefig('BurglaryResidence_over_month.svg')
x = 1

df = pd.read_excel('/home/data/APD/COBRA083016_2015.xlsx', sheetname='Query')
df['occur_ts'] = pd.to_datetime(df.occur_date+' '+df.occur_time)
df['occur_month'] = df['occur_ts'].map(lambda x: x.month)
df['occur_woy'] = df.occur_ts.dt.weekofyear

%matplotlib inline
resdf = df.groupby(['UC2 Literal', 'occur_month']).offense_id.count()
fig = plt.figure(figsize=(10,6))
plt.scatter(resdf['BURGLARY-RESIDENCE'].index, resdf['BURGLARY-RESIDENCE'], marker='x')
plt.ylim(0, 500)
plt.title('BURGLARY-RESIDENCE')
plt.xticks(range(13), ['', 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'])
plt.savefig('quiz3-burglary-residence.png')

''

plt.savefig('quiz3-burglary-residence.png')


I am working on the file "04-Pandas-Data-Tables/APD-Crime-Data_orig.ipynb".
The first task is:
Crime per year
Let's look at the
Can you write Python code for it?

crime_year = dff[dff.occur_year.between(2009, 2015)].groupby(by=['UC2 Literal', 'occur_year']).offense_id.count()

%matplotlib inline
fig = plt.figure(figsize=(40,30))
crime_types = crime_year.index.levels[0]
years = crime_year.index.levels[1]
for c in range(len(crime_types)):
    y_max = max(crime_year.loc[crime_types[c]])

    plt.subplot(4,3,c+1)
    plt.hlines(crime_year.loc[crime_types[c]].iloc[-1]*100/y_max, years[0], years[-1], linestyles="dashed", color="r")
    plt.bar(crime_year.loc[crime_types[c]].index, crime_year.loc[crime_types[c]]*100/y_max, label=crime_types[c], alpha=0.5)
    ##plt.legend()
    plt.ylim(0, 100)
    plt.xticks(years+0.4, [str(int(y)) for y in years], rotation=0, fontsize=24)
    plt.yticks([0,20,40,60,80,100], ['0%','20%','40%','60%','80%','100%'], fontsize=24)
    plt.title(crime_types[c], fontsize=30)
    None


I am working on the file "04-Pandas-Data-Tables/APD-Crime-Data_orig.ipynb".
The first task is:
Exploring Dates
Can you write Python code for it?

df[['offense_id', 'occur_date', 'occur_time', 'rpt_date']][1:10]


I am working on the file "04-Pandas-Data-Tables/APD-Crime-Data_orig.ipynb".
The first task is:
Fitting the regression line
Suppose there are $n$ data points {{math|{(''xi'', ''yi''), ''i'' {{=}} 1, ..., ''n''}.}} The function that describes x and y is:
$$y_i = \alpha + \beta x_i + \varepsilon_i.$$
The goal is to find the equation of the straight line
$$y = \alpha + \beta x,$$
which would provide a "best" fit for the data points. Here the "best" will be understood as in the [[Ordinary least squares|least-squares]] approach: a line that minimizes the sum of squared residuals of the linear regression model. In other words, {{mvar|α}} (the {{mvar|y}}-intercept) and {{mvar|β}} (the slope) solve the following minimization problem:
$$\text{Find }\min_{\alpha,\,\beta} Q(\alpha, \beta), \qquad \text{for } Q(\alpha, \beta) = \sum_{i=1}^n\varepsilon_i^{\,2} = \sum_{i=1}^n (y_i - \alpha - \beta x_i)^2\ $$
By using either [[calculus]], the geometry of [[inner product space]]s, or simply expanding to get a quadratic expression in {{mvar|α}} and {{mvar|β}}, it can be shown that the values of {{mvar|α}} and {{mvar|β}} that minimize the objective function {{mvar|Q}}<ref>Kenney, J. F. and Keeping, E. S. (1962) "Linear Regression and Correlation." Ch. 15 in ''Mathematics of Statistics'', Pt. 1, 3rd ed. Princeton, NJ: Van Nostrand, pp. 252–285</ref> are
: <math>\begin{align}
  \hat\beta &= \frac{ \sum_{i=1}^{n} (x_i - \bar{x})(y_i - \bar{y}) }{ \sum_{i=1}^n (x_i - \bar{x})^2 } \[6pt]
            &= \frac{ \sum_{i=1}^{n} (x_i y_i - x_i \bar{y} - \bar{x} y_i + \bar{x} \bar{y})} { \sum_{i=1}^n (x_i^2 - 2 x_i \bar{x} + \bar{x}^2) } \[6pt]
            &= \frac{ \sum_{i=1}^{n} (x_i y_i) - \bar{y} \sum_{i=1}^{n}  x_i  - \bar{x} \sum_{i=1}^{n} y_i + n  \bar{x} \bar{y}} { \sum_{i=1}^n (x_i^2) - 2 \bar{x}  \sum_{i=1}^n  x_i  + n \bar{x}^2 } \[6pt]
            &= \frac{ \frac{1}{n} \sum_{i=1}^{n} x_i y_i - \bar{x} \bar{y} }{ \frac{1}{n}\sum_{i=1}^n {x_i^2} -   \overline{x}^2 } \[6pt]
            &= \frac{ \overline{xy} - \bar{x}\bar{y} }{ \overline{x^2} - \bar{x}^2 } = \frac{ \operatorname{Cov}[x, y] }{...
Can you write Python code for it?

### in case we want to save a DataFrame
#writer = pd.ExcelWriter('myresults.xlsx')
#df.to_excel(writer,'Results')
#writer.save()

resdf


I am working on the file "04-Pandas-Data-Tables/APD-Crime-Data_orig.ipynb".
The first task is:
Let's look at residential burglary.
Can you write Python code for it?

c = 3
crime_types[c]

crime_year_month = dff[dff.occur_year.between(2009, 2015)].groupby(by=['UC2 Literal', 'occur_year', 'occur_month']).offense_id.count()

c = 3 ## 'BURGLARY-RESIDENCE'
resburglaries = crime_year_month.loc[crime_types[c]]
fig = plt.figure(figsize=(20,10))
for y in years:
    plt.plot(resburglaries.loc[y].index, resburglaries.loc[y], label=("%4.0f"%y))
plt.legend()
plt.title("Seasonal Trends - %s"%crime_types[c], fontsize=20)
plt.xticks(range(13), ['', 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'])
plt.xlim(0,13)
None


I am working on the file "04-Pandas-Data-Tables/APD-Crime-Data_orig.ipynb".
The first task is:
Load data (don't change this if you're running the notebook on the cluster)
We have two files
- /home/data/APD/COBRA083016_2015.xlsx for 2015
- /home/data/APD/COBRA083016.xlsx from 2009 to current date
Can you write Python code for it?

%%time
df = pd.read_excel('/home/data/APD/COBRA083016_2015.xlsx', sheetname='Query')

df.shape

for c in df.columns:
    print(c)

df[0:5]

df.describe()

df.offense_id.min(), df.offense_id.max()

df.groupby(['UC2 Literal']).offense_id.count()


I am working on the file "04-Pandas-Data-Tables/APD-Crime-Data_orig.ipynb".
The first task is:
Normalized over the annual average
Can you write Python code for it?


I am working on the file "04-Pandas-Data-Tables/HW04/CheckHomework04.ipynb".
The first task is:
Now, import your solutions from hw4_answers.py. The following code looks a bit redundant. However, we do this to allow reloading the hw4_answers.py in case you made some changes. Normally, Python assumes that modules don't change and therefore does not try to import them again.
Can you write Python code for it?

import hw4_answers
reload(hw4_answers)
from hw4_answers import *


I am working on the file "04-Pandas-Data-Tables/HW04/CheckHomework04.ipynb".
The first task is:
The output should be
<pre>
274, "Jiang, Stephen Y."
999, "Smith, Chadwick"
102, "Mu, Zheng W."
</pre>

Problem 3
Define a function isSales that takes the job title of an employee as string as an argument and return either True if the job title indicates this person works in sales, and False otherwise.
Argument:
- jobtitle (str)
Returns:
- True or False
Can you write Python code for it?

for jt in ['Chief Data Scientist', 'Sales Manager', 'Vice President of Sales']:
    if isSales(jt):
        print "The job title '%s' is part of the Sales Department." % jt
    else:
        print "The job title '%s' belongs to a different department." % jt


I am working on the file "04-Pandas-Data-Tables/HW04/CheckHomework04.ipynb".
The first task is:
The output should be
<pre>
Number of rows: 18
Number of cols: 25

Head of index: Int64Index([278, 283, 274, 276, 286, 284, 287, 281, 280, 285], dtype='int64', name=u'EmployeeID')

Record of sales employee with ID=280

ManagerID                             274
TerritoryID                             1
Title                                 NaN
FirstName                          Pamela
MiddleName                              O
LastName                     Ansman-Wolfe
Suffix                                NaN
JobTitle             Sales Representative
NationalIDNumber                 61161660
BirthDate                      1969-01-06
MaritalStatus                           S
Gender                                  F
HireDate              2005-10-01 00:00:00
SalariedFlag                            1
VacationHours                          22
SickLeaveHours                         31
PhoneNumber                  340-555-0193
PhoneNumberType                      Cell
EmailAddress            pamela0@yahoo.com
AddressLine1            636 Vine Hill Way
AddressLine2                          NaN
City                             Portland
StateProvinceName                  Oregon
PostalCode                          97205
CountryName                 United States
Name: 280, dtype: object
</pre>

Problem 5
Define a function getEmailList with that returns a Series of strings of all email addresses of employees in this state or province. The email addresses should be separated by a given character, usually a comma ',' or semicolon ';'.
Arguments:
- employees (DataFrame)
- delimiter (str)
Returns:
- Series of email addresses, concatenated by the given delimiter. The Series is indexed by the state or province.
Can you write Python code for it?

emails = getEmailListByState(sales_df, ", ")
for state in sorted(emails.index):
    print "%15s: %s" % (state, emails[state])


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
A where operation for getting.
Can you write Python code for it?


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
A where operation with setting.
Can you write Python code for it?


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
Another simple example:
Can you write Python code for it?

df3 = pd.DataFrame(np.random.rand(3,4), columns=['ValuesA','ValuesB','C','D'], index=[x**2 for x in xrange(3)])
df3['CplusD'] = df3.C+df3.D
df3


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
Append
Append rows to a dataframe. See the Appending
Can you write Python code for it?

df = pd.DataFrame(np.random.randn(8, 4), columns=['A','B','C','D'])
df

s = df.iloc[3]
df.append(s, ignore_index=True)


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
Apply
Applying functions to the data
Can you write Python code for it?

df

df.apply(np.cumsum)

df

df.max() ##df.apply(max)

df.Ann.max()-df.Ann.min()

df.apply(lambda x: x.max() - x.min())


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
Boolean Indexing
Using a single column’s values to select data.
Can you write Python code for it?

df.Ann

flt = (df.Ann >= 0) & (df.Ann < 1.5)
flt

df[flt]

df[(df.Ann >= 0) & (df.Ann < 1.5)]


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
By lists of integer position locations, similar to the numpy/python style
Can you write Python code for it?


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
Categoricals
Since version 0.15, pandas can include categorical data in a DataFrame. For full docs, see the categorical introduction and the API documentation.
Can you write Python code for it?


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
Convert to another time zone
Can you write Python code for it?


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
Converting between period and timestamp enables some convenient arithmetic functions to be used. In the following example, we convert a quarterly frequency with year ending in November to 9am of the end of the month following the quarter end:
Can you write Python code for it?

prng = pd.period_range('1990Q1', '2000Q4', freq='Q-NOV')
ts = pd.Series(np.random.randn(len(prng)), prng)
ts.index = (prng.asfreq('M', 'e') + 1).asfreq('H', 's') + 9
ts.head()


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
Creating a DataFrame by passing a numpy array, with a datetime index and labeled columns:
Can you write Python code for it?

dates = pd.date_range('20130101', periods=6, freq='M')
dates

df = pd.DataFrame(np.random.randn(6,4), index=dates, columns = ['Ann', "Bob", "Charly", "Don"])
                  ## columns=list('ABCD'))
df


dates = pd.date_range('20130101', periods=6, freq='M')
dates

df = pd.DataFrame(np.random.randn(6,4), index=dates, columns = ['Ann', "Bob", "Charly", "Don"])
                  ## columns=list('ABCD'))
df

df.Charly+df.Don

df2 = pd.DataFrame({ 'A' : 1.,
   ....:                      'B' : pd.Timestamp('20130102'),
   ....:                      'C' : pd.Series(1,index=list(range(4)),dtype='float32'),
   ....:                      'D' : np.array([3] * 4,dtype='int32'),
   ....:                      'E' : pd.Categorical(["test","train","test","train"]),
   ....:                      'F' : 'foo' })
   ....:

df2


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
Excel
Reading and writing to MS Excel
Writing to an excel file
Can you write Python code for it?


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
Filling missing data
Can you write Python code for it?


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
For getting a scalar value
Can you write Python code for it?


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
For getting a value explicitly
Can you write Python code for it?

%time print df.iloc[1,1]

#For getting fast access to a scalar (equiv to the prior method)

%time print df.iat[1,1]

# in-class exercise:
# Show all values of "Bob" and "Don" for March-May 2013.

df

dates[np.logical_and(dates>='20130301', dates<'20130601')]

df.ix[dates[np.logical_and(dates>='20130301', dates<'20130601')], ['Bob','Don']]


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
For getting fast access to a scalar (equiv to the prior method)
Can you write Python code for it?


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
For slicing columns explicitly
Can you write Python code for it?


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
For slicing rows explicitly
Can you write Python code for it?


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
Getting Data In/Out
CSV
Writing to a csv file
Can you write Python code for it?


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
Gotchas
If you are trying an operation and you see an exception like:
<pre>
>>> if pd.Series([False, True, False]):
    print("I was true")
Traceback
    ...

ValueError: The truth value of an array is ambiguous. Use a.empty, a.any() or a.all().
</pre>

See Comparisons for an explanation and what to do.
See Gotchas as well.
Can you write Python code for it?


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
Grouping
By “group by” we are referring to a process involving one or more of the following steps
- Splitting the data into groups based on some criteria
- Applying a function to each group independently
- Combining the results into a data structure
See the Grouping section
Can you write Python code for it?


df = pd.DataFrame({'A' : ['foo', 'bar', 'foo', 'bar', 'foo', 'bar', 'foo', 'foo'],
                   'B' : ['one', 'one', 'two', 'three', 'two', 'two', 'one', 'three'],
                   'C' : np.random.randn(8),
                   'D' : np.random.randn(8)})

df


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
Grouping and then applying a function sum to the resulting groups.
Can you write Python code for it?


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
Grouping by a categorical column shows also empty categories.
Can you write Python code for it?


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
Grouping by multiple columns forms a hierarchical index, which we then apply the function.
Can you write Python code for it?


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
HDF5
Reading and writing to HDFStores
Writing to a HDF5 Store
Can you write Python code for it?


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
Histogramming
See more at Histogramming and Discretization
Can you write Python code for it?

s = pd.Series(np.random.randint(0, 7, size=10))
s

s.value_counts()


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
If you’re using IPython, tab completion for column names (as well as public attributes) is automatically enabled. Here’s a subset of the attributes that will be completed:
<pre>
In [13]: df2.<TAB>
df2.A                  df2.boxplot
df2.abs                df2.C
df2.add                df2.clip
df2.add_prefix         df2.clip_lower
df2.add_suffix         df2.clip_upper
df2.align              df2.columns
df2.all                df2.combine
df2.any                df2.combineAdd
df2.append             df2.combine_first
df2.apply              df2.combineMult
df2.applymap           df2.compound
df2.as_blocks          df2.consolidate
df2.asfreq             df2.convert_objects
df2.as_matrix          df2.copy
df2.astype             df2.corr
df2.at                 df2.corrwith
df2.at_time            df2.count
df2.axes               df2.cov
df2.B                  df2.cummax
df2.between_time       df2.cummin
df2.bfill              df2.cumprod
df2.blocks             df2.cumsum
df2.bool               df2.D
</pre>
As you can see, the columns A, B, C, and D are automatically tab completed. E is there as well; the rest of the attributes have been truncated for brevity.
Viewing Data
See the Basics section
See the top & bottom rows of the frame
Can you write Python code for it?

df.head(3)

df.tail(3)

df.index

df.columns

df.values

df.describe()

df2.columns

df2.dtypes

df2.describe()


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
Missing Data
pandas primarily uses the value np.nan to represent missing data. It is by default not included in computations.
See the Missing Data section
Reindexing allows you to change/add/delete the index on a specified axis. This returns a copy of the data.
Can you write Python code for it?

df1 = df.reindex(index=dates[0:4], columns=list(df.columns) + ['E'])
df1

df1.loc[dates[0]:dates[1],'E'] = 1
df1


df1 = df.reindex(index=dates[0:4], columns=list(df.columns) + ['E'])
df1.loc[dates[0]:dates[1],'E'] = 1
df1


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
Object Creation
See the Data Structure Intro section.
When we worked with Numpy arrays, one limitation was that a numpy ndarray can only hold one type of data (there are ways of getting around that, but they take away some of the nice advantages of numpy!). More often than not, real-world data sets contain mixed data that can have strings, dates, and numerical values for each "element". That is where pandas data frames show their real advantage.
Pandas has Series (1-dim labeled lists) and DataTables (2-dim tables with column names and row labels).
We'll start off by looking as Series.
Creating a Series by passing a list of values, letting pandas create a default integer index:
Can you write Python code for it?

s = pd.Series([1,3,5,np.nan,6,8], index=[x**2 for x in range(6)])
s


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
On DataFrame, plot() is a convenience to plot all of the columns with labels:
Can you write Python code for it?

df = pd.DataFrame(np.random.randn(1000, 4), index=ts.index,
   .....:                   columns=['A', 'B', 'C', 'D'])
   .....:

df = df.cumsum()

%matplotlib inline
plt.figure(); df.plot(); plt.legend(loc='best')


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
Operating with objects that have different dimensionality and need alignment. In addition, pandas automatically broadcasts along the specified dimension.
Can you write Python code for it?

s = pd.Series([1,3,5,np.nan,6,8], index=dates).shift(2)
s

df.sub(s, axis='index')


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
Pivot Tables
See the section on Pivot Tables.
In [100]:
Can you write Python code for it?


df = pd.DataFrame({'ModelNumber' : ['one', 'one', 'two', 'three'] * 3,
                   'Submodel' : ['A', 'B', 'C'] * 4,
                   'Type' : ['foo', 'foo', 'foo', 'bar', 'bar', 'bar'] * 2,
                   'Xval' : np.random.randn(12),
                   'Yval' : np.random.randn(12)})

df


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
Plotting
Plotting docs.
Can you write Python code for it?

ts = pd.Series(np.random.randn(1000), index=pd.date_range('1/1/2000', periods=1000))
ts = ts.cumsum()

%matplotlib inline
ts.plot()


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
Reading from a HDF5 Store
Can you write Python code for it?


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
Reading from a csv file
Can you write Python code for it?


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
Reduction in the dimensions of the returned object
Can you write Python code for it?


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
Rename the categories to more meaningful names (assigning to Series.cat.categories is inplace!)
Can you write Python code for it?

df["grade"].cat.categories = ["very good", "good", "very bad"]


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
Reorder the categories and simultaneously add the missing categories (methods under Series .cat return a new Series per default).
Can you write Python code for it?

df["grade"] = df["grade"].cat.set_categories(["very bad", "bad", "medium", "good", "very good"])
df["grade"]


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
Same operation on the other axis
Can you write Python code for it?


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
Selecting on a multi-axis by label
Can you write Python code for it?


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
Selecting via [], which slices the rows.
Can you write Python code for it?


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
Selection
Note: While standard Python / Numpy expressions for selecting and setting are intuitive and come in handy for interactive work, for production code, we recommend the optimized pandas data access methods, .at, .iat, .loc, .iloc and .ix.
See the indexing documentation Indexing and Selecting Data and MultiIndex / Advanced Indexing
Getting
Selecting a single column, which yields a Series, equivalent to df.A
Can you write Python code for it?


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
Selection
Note: While standard Python / Numpy expressions for selecting and setting are intuitive and come in handy for interactive work, for production code, we recommend the optimized pandas data access methods, .at, .iat, .loc, .iloc and .ix.
See the indexing documentation Indexing and Selecting Data and MultiIndex / Advanced Indexing.
Basically: loc is for label-based indexing; iloc is for indexing by row number; ix first tries to use labels, and if it fails, it goes to integer row-number indexing. These methods can all be used to select several elements at once. By contrast, at provides fast label-based indexing for a single element, and iat provides fast row-number indexing for a single element.
Getting
Selecting a single column, which yields a Series, equivalent to df.Ann
Can you write Python code for it?


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
Selection by Label
See more in Selection by Label
For getting a cross section using a label
Can you write Python code for it?


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
Selection by Position
See more in Selection by Position
Select via the position of the passed integers
Can you write Python code for it?


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
Series is equipped with a set of string processing methods in the str attribute that make it easy to operate on each element of the array, as in the code snippet below. Note that pattern-matching in str generally uses regular expressions by default (and in some cases always uses them). See more at Vectorized String Methods.
Can you write Python code for it?

s = pd.Series(['A', 'B', 'C', 'Aaba', 'Baca', np.nan, 'CABA', 'dog', 'cat'])
s.str.lower()


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
Setting
Setting a new column automatically aligns the data by the indexes
Can you write Python code for it?

s1 = pd.Series([1,2,3,4,5,6], index=pd.date_range('20130102', periods=6))
s1

df['F'] = s1
df['G'] = df['Ann']-df['Bob']
df

# in-class exercise: how would you change the index in s1 so that we get values for F in df?
#(note: that's probably not a good way to manipulate real data!)

s1.index = df.index

df['F'] = s1
df


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
Setting by assigning with a numpy array
Can you write Python code for it?


df.loc[:,'Fives'] = np.array([5] * len(df))
df


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
Setting values by label
Can you write Python code for it?


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
Setting values by position
Can you write Python code for it?


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
Showing label slicing, both endpoints are included
Can you write Python code for it?


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
Sorting by an axis
Can you write Python code for it?


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
Sorting by values
Can you write Python code for it?


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
Sorting is per order in the categories, not lexical order.
Can you write Python code for it?

df.sort_values(by="grade")

Out[128]:
   id raw_grade      grade
5   6         e   very bad
1   2         b       good
2   3         b       good
0   1         a  very good
3   4         a  very good
4   5         a  very good


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
The result of the prior setting operations
Can you write Python code for it?


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
The stack() method “compresses” a level in the DataFrame’s columns.
Can you write Python code for it?


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
Time Series
pandas has simple, powerful, and efficient functionality for performing resampling operations during frequency conversion (e.g., converting secondly data into 5-minutely data). This is extremely common in, but not limited to, financial applications.
See the Time Series section
Can you write Python code for it?

rng = pd.date_range('1/1/2012', periods=100, freq='S')
ts = pd.Series(np.random.randint(0, 500, len(rng)), index=rng)
ts.resample('5Min').sum()

Out[105]:
2012-01-01    25083
Freq: 5T, dtype: int64


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
To drop any rows that have missing data.
Can you write Python code for it?


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
To get the boolean mask where values are nan
Can you write Python code for it?

pd.isnull(df1)

df.set_value(dates[:3], 'Ann', [0,1,2])


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
Transposing your data
Can you write Python code for it?


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
Using the isin() method for filtering:
Can you write Python code for it?

df2 = df.copy()
df2['E'] = ['one', 'one','two','three','four','three']
df2

df

df2[df2['E'].isin(['two','three'])]

# in-class exercise:
# Display all rows of df where Ann > Charly.

df[df.Ann>df.Charly]


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
We can produce pivot tables from this data very easily:
Can you write Python code for it?

pd.pivot_table(df, values='D', index=['A', 'B'], columns=['C'])


pd.pivot_table(df, values='Xval', index=['ModelNumber', 'Submodel'], columns=['Type'])


I am working on the file "04-Pandas-Data-Tables/Ten-Minute-Tutorial_class.ipynb".
The first task is:
You can operate on the columns:
Can you write Python code for it?


I am working on the file "04-Project-Fletcher/Phases/Phase_3/Phase_3_Notebook.ipynb".
The first task is:
Gensim LDA Process
Fantastic - at this point, we have everything we need to proceed with LDA from the Gensim Library.
LDA via the Gensim library requires that our data be in a very specific format.
Broadly, LDA requires a Dictionary object that is later used to create a matrix called a corpus.
The Gensim LDA Dictionary will require that we pass in a list of lists. Every sublist will be a tweet that has been split.
Let's look at my first tweet as an example.
Before:
['great turnout today hope able join slide available link video webinar come soon', tweet 2, tweet 3, ...]
Correct Gensim Format:
[['great', 'turnout', 'today', 'hope', 'able', 'join', 'slide', 'available','link', 'video', 'webinar', 'come', 'soon'], [tweet 2 in split form], [...],...]
Can you write Python code for it?

list_of_tweets_gabr = clean_gabr_tweets[0]['gabr_ibrahim']['tokenized_tweets']

gensim_format_tweets = []
for tweet in list_of_tweets_gabr:
    list_form = tweet.split()
    gensim_format_tweets.append(list_form)

gensim_format_tweets[:5]

gensim_dictionary = Dictionary(gensim_format_tweets)


I am working on the file "04-Project-Fletcher/Phases/Phase_3/Phase_3_Notebook.ipynb".
The first task is:
I will not turn the dictionary back into a dataframe, run it through the filtration function before re-casting the dataframe into a dictionary.
This time, we are running the filtration process on the tokenized tweets column and not the content column.
NLP models are very sensitive - ensuring consistent cleaning is important!
Can you write Python code for it?

clean_gabr_tweets_df = pd.DataFrame.from_dict(clean_gabr_tweets[0], orient='index')

clean_gabr_tweets_df.head()

clean_gabr_tweets_df = filtration(clean_gabr_tweets_df, "tokenized_tweets")

clean_gabr_tweets = dataframe_to_dict(clean_gabr_tweets_df)

clean_gabr_tweets[0]['gabr_ibrahim']['tokenized_tweets'][:5]


I am working on the file "04-Project-Fletcher/Phases/Phase_3/Phase_3_Notebook.ipynb".
The first task is:
I will then place this LDA Counter Object back into our dictionary!
We will then pickle this object - we will use it again for our TF-IDF analysis!
Be sure to look at the file called lda.py to see how I stuructured the code to run through the 2nd degree connections!
Can you write Python code for it?

clean_gabr_tweets[0]['gabr_ibrahim'].keys()

clean_gabr_tweets[0]['gabr_ibrahim']['LDA'] = temp

pickle_object(clean_gabr_tweets, "gabr_ibrahim_tweets_LDA_Complete")


I am working on the file "04-Project-Fletcher/Phases/Phase_3/Phase_3_Notebook.ipynb".
The first task is:
Let's now proceed to tokenize these tweets in addition to lemmatizing them! This will help improve the performance of our LDA model!
I will utilise spacy for this process as it is a production grade NLP library that is exceptionally fast!
Can you write Python code for it?

tokenized_tweets = []
for tweet in gabr_tweets:
    tokenized_tweet = nlp(tweet)

    tweet = "" # we want to keep each tweet seperate

    for token in tokenized_tweet:
        if token.is_space:
            continue
        elif token.is_punct:
            continue
        elif token.is_stop:
            continue
        elif token.is_digit:
            continue
        elif len(token) == 1:
            continue
        elif len(token) == 2:
            continue
        else:
            tweet += str(token.lemma_) + " " #creating lemmatized version of tweet

    tokenized_tweets.append(tweet)
tokenized_tweets = list(map(str.strip, tokenized_tweets)) # strip whitespace
tokenized_tweets = [x for x in tokenized_tweets if x != ""] # remove empty entries

tokenized_tweets[:5] # you can see how this is different to the raw tweets!


I am working on the file "04-Project-Fletcher/Phases/Phase_3/Phase_3_Notebook.ipynb".
The first task is:
Now for the LDA part!
I will be using the LDAMulticore class from gensim!
I set the passess parameter to 100 and the chunksize to 2000.
The chunksie will ensure it use's all the documents at once, and the passess parameter will ensure it looks at all the documents 100 times before converging.
As I am using my ENTIRE tweet history, I will create 30 topics!
I will adjust this to 10 when running lda on 2nd degree connections, as I will only have 200 of their tweets!
Can you write Python code for it?

lda = LdaMulticore(corpus, num_topics=30, id2word=gensim_dictionary, chunksize=2000, workers=100, passes=100)


I am working on the file "04-Project-Fletcher/Phases/Phase_3/Phase_3_Notebook.ipynb".
The first task is:
Now, I will now filter out extreme words - that is words that appear far too often and words that are rare.
Can you write Python code for it?

gensim_dictionary.filter_extremes(no_below=10, no_above=0.4)
gensim_dictionary.compactify() # remove gaps after words that were removed


I am working on the file "04-Project-Fletcher/Phases/Phase_3/Phase_3_Notebook.ipynb".
The first task is:
Step 1: Obtain my tweets!
I will obtain my entire tweet history! Note: For 2nd degree potential followers, I only extract 200 of their most recent tweets!
Can you write Python code for it?

gabr_tweets = extract_users_tweets("gabr_ibrahim", 2000)


I am working on the file "04-Project-Fletcher/Phases/Phase_3/Phase_3_Notebook.ipynb".
The first task is:
Step 3: Create a dataframe from my tweets
We will now turn this dictionary into a dataframe - I do this as it allows me to utilise pandas in cleaning the content of my tweets!
After the cleaning on the 'content' column, I will convert the dataframe back into a dictionary.
Can you write Python code for it?

gabr_tweets_df = pd.DataFrame.from_dict(gabr_dict, orient='index')

gabr_tweets_df.head()

clean_gabr_tweets = filtration(gabr_tweets_df, "content")

clean_gabr_tweets = dataframe_to_dict(clean_gabr_tweets)

clean_gabr_tweets #this is a list of 1 dictionary


I am working on the file "04-Project-Fletcher/Phases/Phase_4/Phase_4_Notebook.ipynb".
The first task is:
I will now create a TF-IDF model for my tweets.
Using K-Means Clustering with TF-IDF, I will cluster my tweet's into 20 centroids. From each of these centroids, I will extract 20 words. These words will be placed in a counter dictionary.
TF-IDF KMeans - segemented by individual tweet!
I will make use of spacy again in order to ensure we are giving the 'purest' for of our tweets to the tf-idf vectorizer.
You will see two lists below relating to vocabulary. I will use these lists later to create a usefull dictionary that will help identify particular words within a centroid by index!
Can you write Python code for it?

temp_gabr_df = pd.DataFrame.from_dict(gabr_tweets[0], orient="index")

temp_gabr_df = filtration(temp_gabr_df, "content")

gabr_tweets_filtered_1 = dataframe_to_dict(temp_gabr_df)

clean_tweet_list = []
totalvocab_tokenized = []
totalvocab_stemmed = []

for tweet in gabr_tweets_filtered_1[0]['gabr_ibrahim']['content']:
    clean_tweet = ""
    to_process = nlp(tweet)

    for token in to_process:
        if token.is_space:
            continue
        elif token.is_punct:
            continue
        elif token.is_stop:
            continue
        elif token.is_digit:
            continue
        elif len(token) == 1:
            continue
        elif len(token) == 2:
            continue
        else:
            clean_tweet += str(token.lemma_) + ' '
            totalvocab_tokenized.append(str(token.lemma_))
            totalvocab_stemmed.append(str(token.lemma_))

    clean_tweet_list.append(clean_tweet)

#just going to add this to the dictionary so we can do the second round of filtration
gabr_tweets_filtered_1[0]['gabr_ibrahim']['temp_tfidf'] = clean_tweet_list

temp_gabr_df = pd.DataFrame.from_dict(gabr_tweets_filtered_1[0], orient='index')

temp_gabr_df = filtration(temp_gabr_df, 'temp_tfidf')

gabr_tweets_filtered_2 = dataframe_to_dict(temp_gabr_df)

clean_tweet_list = gabr_tweets_filtered_2[0]['gabr_ibrahim']["temp_tfidf"]
del gabr_tweets_filtered_2[0]["gabr_ibrahim"]["temp_tfidf"] # we will add back TF-IDF analysis later!

vocab_frame = pd.DataFrame({'words': totalvocab_tokenized}, index = totalvocab_stemmed)
print('There are ' + str(vocab_frame.shape[0]) + ' items in vocab_frame')

#define vectorizer parameters
tfidf_vectorizer = TfidfVectorizer(max_features=200000, stop_words='english', ngram_range=(0,2))

tfidf_matrix = tfidf_vectorizer.fit_transform(clean_tweet_list) #fit the vectorizer to synopses

print(tfidf_matrix.shape)

terms = tfidf_vectorizer.get_feature_names()

num_clusters = 20

km = KMeans(n_clusters=num_clusters, n_jobs=-1,...


I am working on the file "04-Project-Fletcher/Phases/Phase_4/Phase_4_Notebook.ipynb".
The first task is:
So far, we have two databases:

2nd degree connection database where all handles have valid LDA Analysis.

A database with my tweets and associated LDA analysis.

The LDA method was quite powerful for potential followers, distilling down their entire corpus to a few key terms.
Let's now do some TF-IDF and KMeans clustering to see if we find similar results to LDA.
In fact, later in the notebook, I will take the intersection of the LDA Analysis results and TF-IDF results. This intersection will represent words/topics that were picked up by BOTH models for a particular handle's tweets. This will give us the most robust results!
Can you write Python code for it?

gabr_tweets = unpickle_object("gabr_ibrahim_tweets_LDA_Complete.pkl")

gabr_tweets[0]['gabr_ibrahim'].keys() #just to refresh our mind of the keys in the sub-dictionary


I am working on the file "04-SingularValueDecomposition.ipynb".
The first task is:
Clearly the first two rows are linearly dependent and the rank of this matrix is 2. We can verify this using NumPy:
Can you write Python code for it?

print("Rank of A is: {}".format(np.linalg.matrix_rank(A)))


I am working on the file "04-SingularValueDecomposition.ipynb".
The first task is:
Combine compressed channels:
Can you write Python code for it?

fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(20, 20/1.77))
axes[0].set_title('Image with largest 20% of brightness singular values retained and 0.5% colours')
im = Image.fromarray(np.uint8(compressed), mode="YCbCr")
axes[0].imshow(im)
axes[1].set_title('Original image')
axes[1].imshow(img_colour);


I am working on the file "04-SingularValueDecomposition.ipynb".
The first task is:
Compress each channel, and display compressed channels in gray scale:
Can you write Python code for it?

# Compress each component separately
compressed = [compress_image(U[0], s[0], V[0], 0.05),
              compress_image(U[1], s[1], V[1], 0.005),
              compress_image(U[2], s[2], V[2], 0.005)]
# Reconstruct 3D YCbCr array
compressed = np.dstack(compressed)
fig, axes = plt.subplots(nrows=1, ncols=3, figsize=(20, 20/1.77))
Y = compressed[:,:,0]
axes[0].imshow(Y, cmap='gray');

Cb = compressed[:,:,1]
axes[1].imshow(Cb, cmap='gray');

Cr = compressed[:,:,2]
axes[2].imshow(Cr, cmap='gray');


I am working on the file "04-SingularValueDecomposition.ipynb".
The first task is:
Effective rank
Determining the rank of a matrix is not a binary question in the context of floating point arithmetic or measurement errors. The SVD can be used to determine the 'effective rank' of a matrix.
Consider the matrix:
Can you write Python code for it?


I am working on the file "04-SingularValueDecomposition.ipynb".
The first task is:
Even with only 10% if the singular values retains, it is hard to perceive a difference between the images. Next we try keeping only 2%:
Can you write Python code for it?

# Compress image/matrix
compressed = compress_image(U, s, V, 0.02)

# Plot compressed and original image
fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(20, 20/1.77))
axes[0].set_title('Compressed image with largest 2% of singular values retained')
axes[0].imshow(compressed, cmap='gray');
axes[1].set_title('Original image')
axes[1].imshow(img_array, cmap='gray');

plt.savefig("bw-0-02.pdf")


I am working on the file "04-SingularValueDecomposition.ipynb".
The first task is:
If we define the effective rank as the number of singular values that are greater than the noise level, the effective rank of $\boldsymbol{A}$ is 2.
Rank deficient least-squares problems
For least squares problem, we have seen before that we solve
$$
\boldsymbol{A}^{T} \boldsymbol{A} \hat{\boldsymbol{x}} = \boldsymbol{A}^{T} \boldsymbol{b}
$$
or
$$
\begin{align}
\hat{\boldsymbol{x}} &= (\boldsymbol{A}^{T} \boldsymbol{A})^{-1} \boldsymbol{A}^{T} \boldsymbol{b}
\
&= \boldsymbol{A}^{+}\boldsymbol{b}
\end{align}
$$
Everything is fine as long as $\boldsymbol{A}$ is full rank. The problem is that we might have data that leads to $\boldsymbol{A}$ not being full rank. For example, if we try to fit a polynomial in $x$ and $y$, but the data lies on a line.
We have covered in the lectures how to handle least-squares problems that are rank deficient. Here we present an example.
Example: fitting points in a two-dimensional space
Say we are given four data points that depend on $x$ and $y$, and we are asked to fit a polynomial of the form
$$
f(x, y) = c_{00} + c_{10}x + c_{01}y + c_{11}xy
$$
to the data points. Normally, we would expect to be able to fit the above polynomial to four data points by interpolation, i.e. solving $\boldsymbol{A} \boldsymbol{c} = \boldsymbol{f}$ where
$\boldsymbol{A}$ a square Vandermonde matrix. However, if the points happened to lie on a line, then $\boldsymbol{A}$ will be singular. If the points happen to almost lie on a line, then $\boldsymbol{A}$ will be close to singular.
A possibility is to exclude zero or small singular values from the process, thereby finding a least-squares fit with minimal $\|\boldsymbol{c}\|_{2}$. We test this for the data set
\begin{equation}
f_{1}(1, 0) = 3, \
f_{2}(2, 0) = 5, \
f_{3}(3, 0) = 7, \
f_{4}(4, 0) = 9.
\end{equation}
The data lies on the line $y = 0$, and is in fact is linear in $x$.
We create arrays to hold this data, and visualise the points:
Can you write Python code for it?

x, y, f = np.zeros(4), np.zeros(4), np.zeros(4)
x[0], y[0], f[0] = 1.0, 0.0, 3.0
x[1], y[1], f[1] = 2.0, 0.0, 5.0
x[2], y[2], f[2] = 3.0, 0.0, 7.0
x[3], y[3], f[3] = 4.0, 0.0, 9.0

from mpl_toolkits.mplot3d import Axes3D
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
ax.set_xlabel('x')
ax.set_ylabel('y')
ax.set_zlabel('f')
ax.scatter(x, y, f)
plt.show()


I am working on the file "04-SingularValueDecomposition.ipynb".
The first task is:
Interactive compression
We'll now create an interactive image with sliders to interactively control the compression level.
Can you write Python code for it?

from ipywidgets import widgets
from ipywidgets import interact

url = "https://github.com/garth-wells/notebooks-3M1/raw/master/photo/IMG_20190117_141222563.png"
img = Image.open(urlopen(url))
img_colour_ycbcr = np.array(img.convert("YCbCr"))

# Compute SVD for each channel
U0, s0, V0 = [0]*3, [0]*3, [0]*3
for i in range(3):
    U0[i], s0[i], V0[i] = np.linalg.svd(img_colour_ycbcr[:, :, i], full_matrices=False)

@interact(ratio_Y=(0.005, 0.4, 0.02),
          ratio_Cb=(0.001, 0.1, 0.01),
          ratio_Cr=(0.001, 0.1, 0.01))
def plot_image(ratio_Y=0.1, ratio_Cb=0.01, ratio_Cr=0.01):

    compressed = [compress_image(U0[0], s0[0], V0[0], ratio_Y),
                  compress_image(U0[1], s0[1], V0[1], ratio_Cb),
                  compress_image(U0[2], s0[2], V0[2], ratio_Cr)]

    # Reconstruct 3D YCbCr array
    compressed = np.dstack(compressed)
    img_compressed = Image.fromarray(np.uint8(compressed), mode="YCbCr")

    # Show
    fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(20, 20/1.77))

    axes[0].set_title('Compressed image')
    axes[0].imshow(img_compressed)

    axes[1].set_title('Original image')
    axes[1].imshow(img)


I am working on the file "04-SingularValueDecomposition.ipynb".
The first task is:
Let's try retaining just 10% of the singular values:
Can you write Python code for it?

# Compress image/matrix
compressed = compress_image(U, s, V, 0.1)

# Plot compressed and original image
fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(20, 20/1.77))
axes[0].set_title('Compressed image with largest 10% of singular values retained')
axes[0].imshow(compressed, cmap='gray');
axes[1].set_title('Original image')
axes[1].imshow(img_array, cmap='gray');

plt.savefig("bw-0-10.pdf")


I am working on the file "04-SingularValueDecomposition.ipynb".
The first task is:
Note: If you run this notebook yourself it can take sometime because it computes number of moderate size SVD problems.
Low rank approximations
Recall that we can represent a matrix as a sum of rank-1 matrices:
$$
\boldsymbol{A} = \sum_{i} \sigma_{i} \boldsymbol{u}{i} \boldsymbol{v}^{T}{i}
$$
where $\sigma_{i}$ is the $i$th singular value and $\boldsymbol{u}{i}$ and $\boldsymbol{v}{i}$ are the $i$th columns vectors of $\boldsymbol{U}$ and $\boldsymbol{V}$, respectively from the SVD. Clearly, for any $\sigma_{i} = 0$ we can avoid storing the data that makes no contribution. If $\sigma_{i}$ is small, then the contribution of $\boldsymbol{u}{i} \boldsymbol{v}^{T}{i}$ is small and we discard it and introduce only a small 'error' to the matrix. We will use low rank approximations in a number of examples in this notebook.
Data compression
We start with a $100 \times 200$ matrix that has entries equal to one or zero. We create a matrix with all entries set to zero, and we then set some entries equal to one in the pattern of rectangle.
Can you write Python code for it?


I am working on the file "04-SingularValueDecomposition.ipynb".
The first task is:
Now, maybe we can discard information associated with small singular values without perceiving any visual change in the image. To explore this, we compute the SVD of the gray scale image:
Can you write Python code for it?

U, s, V = np.linalg.svd(img_array, full_matrices=False)


I am working on the file "04-SingularValueDecomposition.ipynb".
The first task is:
Performing an SVD on the matrix with noise and printing the singular values:
Can you write Python code for it?


I am working on the file "04-SingularValueDecomposition.ipynb".
The first task is:
Plotting the data set and the principal directions:
Can you write Python code for it?

ax = plt.axes()
ax.set_aspect(1.0);
ax.set_ylim(-4.0, 4.0);
ax.set_xlabel('$x_{1}$')
ax.set_ylabel('$x_{2}$')
ax.quiver(V[0, 0], V[0, 1], angles='xy',scale_units='xy',scale=0.3);
ax.quiver(V[1, 0], V[1, 1], angles='xy',scale_units='xy',scale=1);
ax.scatter(X[:,0], X[:,1], alpha=0.2);


I am working on the file "04-SingularValueDecomposition.ipynb".
The first task is:
The argument full_matrices=False tells NumPy to not store all the redundant zero terms in the $\boldsymbol{\Sigma}$ array. This is the normal approach in practice, but not in most text books. Note that NumPy return the singular values as a one-dimendional array, not as a matrix.
We now print the largest and smallest singular values, and plot all the singular values $\sigma_{i}$ on a log-scale:
Can you write Python code for it?

print("Number of singular values: {}".format(len(s)))
print("Max, min singular values: {}, {}".format(s[0], s[-1]))

plt.xlabel('$i$')
plt.ylabel('$\sigma_i$')
plt.title('Singular values')
plt.yscale('log')
plt.plot(s, 'bo');

plt.savefig("bw-svd.pdf")


I am working on the file "04-SingularValueDecomposition.ipynb".
The first task is:
The image quality is now quite poor.
Colour image: RGB
We'll now try compressing a colour image.
Can you write Python code for it?

print("Image array shape: {}".format(img_colour.size))

plt.figure(figsize=(20,20/1.77))
plt.title('This is a photo of 2020 3M1 class members')
plt.imshow(img_colour);


I am working on the file "04-SingularValueDecomposition.ipynb".
The first task is:
We can convert the image to a regular matrix with values between 0 and 255, with each entry corresponding to a pixel in the image. Creating the matrix and inspecting first four rows and three columns (top left corner of the image):
Can you write Python code for it?

img_array = np.array(img_bw)
print("Image shape:", img_array.shape)
print(img_array[:4, :3])


I am working on the file "04-SingularValueDecomposition.ipynb".
The first task is:
We collect the data in a $200 \times 2$ matrix $\boldsymbol{X}$ (200 measurements, 2 variables):
Can you write Python code for it?


I am working on the file "04-SingularValueDecomposition.ipynb".
The first task is:
We could work with the colour image, but it is simpler to work with a gray scale image because then we have only one value for the colour intensity at each pixel rather than three (red/green/blue).
Can you write Python code for it?

img_bw = img_colour.convert('L')

plt.figure(figsize=(15, 15/1.77))
plt.imshow(img_bw, cmap='gray');
print("Image array shape: {}".format(img_bw.size))

plt.savefig("bw.pdf")


I am working on the file "04-SingularValueDecomposition.ipynb".
The first task is:
We now compute an SVD for the matrix of each colour:
Can you write Python code for it?

# Compute SVD for each colour
U, s, V = [0]*3, [0]*3, [0]*3
for i in range(3):
    U[i], s[i], V[i] = np.linalg.svd(img_array[:, :, i], full_matrices=False)


I am working on the file "04-SingularValueDecomposition.ipynb".
The first task is:
We now see some image clear degradation, but the image is sill recognisable. We'll try one more case where we retain only 0.5% of the singular values.
Can you write Python code for it?

# Compress image/matrix
compressed = compress_image(U, s, V, 0.005)

# Plot compressed and original image
fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(20, 20/1.77))
axes[0].set_title('Compressed image with largest 0.5% of singular values retained')
axes[0].imshow(compressed, cmap='gray');
axes[1].set_title('Original image')
axes[1].imshow(img_array, cmap='gray');

plt.savefig("bw-0-005.pdf")


I am working on the file "04-SingularValueDecomposition.ipynb".
The first task is:
We now try adding some noise to the sample positions and the measured values. The Vandermonde matrix is no longer singular so we can solve $\boldsymbol{A} \boldsymbol{c} = \boldsymbol{f}$ to get the polynomial coefficients:
Can you write Python code for it?

np.random.seed(20)
xn = x + 1.0e-3*(1.0 - np.random.rand(len(x)))
yn = y + 1.0e-3*(1.0 - np.random.rand(len(y)))
fn = f + 1.0e-3*(1.0 - np.random.rand(len(f)))

A = np.polynomial.polynomial.polyvander2d(yn, xn, [1, 1])
c = np.linalg.solve(A, fn)
print(c)


I am working on the file "04-SingularValueDecomposition.ipynb".
The first task is:
We see that the fitting polynomial is very close to the noise-free case.
Principal component analysis
Principal component analysis finds a transformation such that covariance of a data set is zero in the transformed directions, and the variance in these directions is greatest. From a dataset this tells us which are the 'important' parameters in a system.
Consider taking $N = 200$ measurements of two quantities $x_{1}$ and $x_{2}$. We model the system by:
Can you write Python code for it?

np.random.seed(1)
x0 = np.random.randn(200) + 5.0
x1 = 1.5*x0 + np.random.rand(len(x0))

ax = plt.axes()
ax.scatter(x0, x1, alpha=0.5);
ax.set_xlabel('$x_{1}$');
ax.set_ylabel('$x_{2}$');


I am working on the file "04-SingularValueDecomposition.ipynb".
The first task is:
We see that two of the values are considerably small than the others. If we set these to zero and follow the least-squares procedure for rank-deficient problem:
Can you write Python code for it?

# Create view of U with last two columns removed
U1 = U[:, :2]

# Create view of V with last two columns removed
V1 = V[:2,:]

# Create \Sigma^{+}
S1 = np.diag(1.0/s[:-2])

c = np.transpose(V1).dot(S1.dot(U1.T).dot(f))
print(c)


I am working on the file "04-SingularValueDecomposition.ipynb".
The first task is:
With only three nonzero singular values, we could reconstruct the matrix with very little data - just three singular values and six vectors.
Removing noise
We consider the same matrix problem again, this time with some back ground noise in the white regions.
Can you write Python code for it?


I am working on the file "04-Stylometry/01-Ad-Hoc-Stylometry.ipynb".
The first task is:
And perform mathematical operations:
Can you write Python code for it?


I am working on the file "04-Stylometry/01-Ad-Hoc-Stylometry.ipynb".
The first task is:
Cool! But we need it within a line, and Thornbury specifically does it for each Fitt. What's a "Fitt"? It's a further division in poetry constituted by a group of lines. Luckily this is nicely delimited by double line breaks (\n\n).
Can you write Python code for it?

cs_fitts = christ_and_satan.split('\n\n')
cs_fitts

%matplotlib inline
import matplotlib.pyplot as plt

plt.figure(figsize = (10,10))

# iterate through fitts
for i in range(len(cs_fitts)):

    # lowercase the string and get the tokens for each line back
    fitt_tokens = [l.split() for l in cs_fitts[i].lower().split('\n')]

    # collect letter of most freq alliteration
    most_freq_allit = []

    # cycle through lines
    for l in fitt_tokens:

        # get first letter of all words in line
        first_letters = [x[0] if x[0] not in ['a','e','i','o','u','y'] else 'a' for x in l]

        # count freq of all first letters
        allit_freq = Counter(first_letters).most_common()
        try:
            # append most freq letter (alliterated letter) to list for all lines
            most_freq_allit.append(allit_freq[0][0])
        except:
            pass

    # use Counter to get the most common alliterations
    allit_freq = Counter(most_freq_allit).most_common()

    # need keys for x axis
    common_keys = [x[0] for x in allit_freq]

    # need values for y axes
    common_values = [x[1] for x in allit_freq]

    # normalize so we can compare across Fitts despite different number of words
    normed_values = [x[1]/sum(common_values) for x in allit_freq]

    # add up to get cumulative alliteration of the four most preferred patterns
    cumulative_values = np.cumsum(normed_values)

    # add the Fitt to the plot
    plt.xticks(range(4), ['1st','2nd','3rd','4th'], rotation='vertical')
    plt.plot(cumulative_values[:4], color = plt.cm.bwr(i*.085), lw=3)
plt.legend(labels=['Fitt '+str(i+1) for i in range(12)], loc=0)
plt.show()


I am working on the file "04-Stylometry/01-Ad-Hoc-Stylometry.ipynb".
The first task is:
Does it feel like time for a list comprehension? It should.
Challenge 3
Write a list comprehension to remove line numbers and punctuation.

Python comes with the convenient Counter method from the collections library. It returns a dictionary like object that will return the frequency of a particular key.
Can you write Python code for it?

from collections import Counter
cs_dict = Counter(tokens)

cs_dict

cs_dict.keys()

cs_dict.values()

cs_dict.most_common()


I am working on the file "04-Stylometry/01-Ad-Hoc-Stylometry.ipynb".
The first task is:
Homework: Acrostics
In poetry, an acrostic is a message created by taking certain letters in a pattern over lines. One 9th century German writer, Otfrid of Weissenburg, was notorius for his early use of acrostics, one instance of which is in the text below: Salomoni episcopo Otfridus. His message can be found by taking the first character of every other line. Print Otfrid's message!
Source: http://titus.uni-frankfurt.de/texte/etcs/germ/ahd/otfrid/otfri.htm
Can you write Python code for it?

text = '''si sálida gimúati      sálomones gúati,
     ther bíscof ist nu édiles      kóstinzero sédales;
     allo gúati gidúe thio sín,      thio bíscofa er thar hábetin,
     ther ínan zi thiu giládota,      in hóubit sinaz zuívalta!
     lékza ih therera búachi      iu sentu in suábo richi,
     thaz ir irkíaset ubar ál,      oba siu frúma wesan scal;
     oba ir hiar fíndet iawiht thés      thaz wírdig ist thes lésannes:
     iz iuer húgu irwállo,      wísduames fóllo.
     mir wárun thio iuo wízzi      ju ófto filu núzzi,
     íueraz wísduam;      thes duan ih míhilan ruam.
     ófto irhugg ih múates      thes mánagfalten gúates,
     thaz ír mih lértut hárto      íues selbes wórto.
     ni thaz míno dohti      giwérkon thaz io móhti,
     odo in thén thingon      thio húldi so gilángon;
     iz datun gómaheiti,      thio íues selbes gúati,
     íueraz giráti,      nales míno dati.
     emmizen nu ubar ál      ih druhtin férgon scal,
     mit lón er iu iz firgélte      joh sínes selbes wórte;
     páradyses résti      gébe iu zi gilústi;
     ungilónot ni biléip      ther gotes wízzode kleip.
     in hímilriches scóne      so wérde iz iu zi lóne
     mit géltes ginúhti,      thaz ír mir datut zúhti.
     sínt in thesemo búache,      thes gómo theheiner rúache;
     wórtes odo gúates,      thaz lích iu iues múates:
     chéret thaz in múate      bi thia zúhti iu zi gúate,
     joh zellet tház ana wánc      al in íuweran thanc.
     ofto wírdit, oba gúat      thes mannes júngoro giduat,
     thaz es líwit thráto      ther zúhtari gúato.
     pétrus ther rícho      lono iu es blídlicho,
     themo zi rómu druhtin gráp      joh hús inti hóf gap;
     óbana fon hímile      sént iu io zi gámane
     sálida gimýato      selbo kríst ther gúato!
     oba ih irbálden es gidár,      ni scal ih firlázan iz ouh ál,
     nub ih ío bi iuih gerno      gináda sina férgo,
     thaz hóh er iuo wírdi      mit sínes selbes húldi,
     joh iu féstino in thaz múat      thaz sinaz mánagfalta...


I am working on the file "04-Stylometry/01-Ad-Hoc-Stylometry.ipynb".
The first task is:
Looks like a decent start. But we still have verse numbering in there, as well as some punctuation. What if we just want the words?
Can you write Python code for it?

from string import punctuation, digits

punctuation

digits


I am working on the file "04-Stylometry/01-Ad-Hoc-Stylometry.ipynb".
The first task is:
Now you know why list comprehensions are one of the best parts of Python!
Especially for text analysis, these will come in handy when we want to parse and sift through text.
Challenge 2

Concatenate the first three lines of Christ and Satan.
Create a new list that contains only the words whose last letter is "e"
Create a new list that contains the first letter of each word.
Create a new list that contains only words longer than two letters.

Word Frequencies
Can you write Python code for it?

with open('data/christ-and-satan.txt', 'r') as f:
    christ_and_satan = f.read()

tokens = christ_and_satan.split()

tokens


I am working on the file "04-Stylometry/01-Ad-Hoc-Stylometry.ipynb".
The first task is:
Otfrid was more skillful than to settle for the first letter of every other line. What happens if you extract the last letter of the last word of each line, for every other line starting on the second line?
Can you write Python code for it?

# HINT: first remove punctuation, tab is represented by \t
from string import punctuation


I am working on the file "04-Stylometry/01-Ad-Hoc-Stylometry.ipynb".
The first task is:
We can assign these to variables too!
Can you write Python code for it?

first_hemistich = ["þæt", "wearð", "underne"]
second_hemistich = ["eorðbuendum"]
print(first_hemistich)
print(second_hemistich)


I am working on the file "04-Stylometry/01-Ad-Hoc-Stylometry.ipynb".
The first task is:
You can get the length of a list using the len function:
Can you write Python code for it?


I am working on the file "04-Units/Units_Quantities.ipynb".
The first task is:

 Challenges

 Create a function that calculates potential energy where *g* defaults to Earth value,
  but could be used for different planets.
  Test it for any of the *g* values for any other
  planet.

Can you write Python code for it?

#4

# run it for some values

# on Mars:


I am working on the file "04-Units/Units_Quantities.ipynb".
The first task is:

 Challenges

 Convert the area calculated before `rectangle_area` in Hectare
  (1 hectare = 100 ares; 1 are = 100 m2).

Can you write Python code for it?


I am working on the file "04-Units/Units_Quantities.ipynb".
The first task is:

 Challenges

 Convert the speed in imperial units (miles/hour) using:

 ```from astropy.units import imperial```

 Calculate whether a pint is more than half litre

 <emph>You can compare quantities as comparing variables.</emph>

 Something strange? Check what deffinition of pint astropy is using.

 Does units work with areas? calculate the area of a rectangle of 3 km of side and 5 meter of width. Show them in m^2 and convert them to yards^2

Can you write Python code for it?


I am working on the file "04-Units/Units_Quantities.ipynb".
The first task is:
Arrays
Quantities can also be applied to arrays
Can you write Python code for it?

# different ways of defining a quantity for a single value

# now with lists
# and arrays
# and its arithmetics

# angles are smart!


I am working on the file "04-Units/Units_Quantities.ipynb".
The first task is:
Composed units
Many units are compositions of others, for example, one could create new combinationes for ease of use:
Can you write Python code for it?

# create a composite unit

# and in the imperial system


I am working on the file "04-Units/Units_Quantities.ipynb".
The first task is:
Creating functions with quantities as units
We want to have functions that contain the information of the untis, and with them we can be sure that we will be always have the right result.
Can you write Python code for it?

# Create a function for the Kinetic energy

# run with and without units


I am working on the file "04-Units/Units_Quantities.ipynb".
The first task is:
Equivalencies
Some conversions are not done by a conversion factor as between miles and kilometers, for example converting between wavelength and frequency.
Can you write Python code for it?

# converting spectral quantities

# but doing it right


I am working on the file "04-Units/Units_Quantities.ipynb".
The first task is:
Other built-in equivalencies are:
 - parallax()
 - Doppler (dopplr_radio, doppler_optical, doppler_relativistic)
 - spectral flux density
 - brigthness temperature
 - temperature energy
 - and you can build your own
Can you write Python code for it?

# finding the equivalencies

# but also using other systems


I am working on the file "04-Units/Units_Quantities.ipynb".
The first task is:
Plotting quantities
To work nicely with matplotlib we need to do as follows:
Can you write Python code for it?

# allowing for plotting
from astropy.visualization import quantity_support
quantity_support()

# loading matplotlib
%matplotlib inline
from matplotlib import pyplot as plt

# Ploting the previous array


I am working on the file "04-Units/Units_Quantities.ipynb".
The first task is:
Printing the quantities
Can you write Python code for it?

# Printing values with different formats


I am working on the file "04-Units/Units_Quantities.ipynb".
The first task is:
Quantities can also be combined, for example to measure speed
Can you write Python code for it?


I am working on the file "04-Units/Units_Quantities.ipynb".
The first task is:
Quantities can be converted to other units systems or factors by using to()
Can you write Python code for it?


I am working on the file "04-Units/Units_Quantities.ipynb".
The first task is:
Sometime we get no units quantitites
Can you write Python code for it?


I am working on the file "04-Units/Units_Quantities.ipynb".
The first task is:
We can do arithmetic operations when the quantities have the compatible units:
Can you write Python code for it?


I am working on the file "04-Units/Units_Quantities.ipynb".
The first task is:
and others are already a composition:
Can you write Python code for it?

# what can be converted from s-1?

# or Jules?

# Unity of R


I am working on the file "04-Units/Units_Quantities_Instructor.ipynb".
The first task is:

 Challenges

 Create a function that calculates potential energy where *g* defaults to Earth value,
  but could be used for different planets.
  Test it for any of the *g* values for any other
  planet.

Can you write Python code for it?

#4
@u.quantity_input(mass=u.kg, height=u.m, g=u.m / u.s ** 2)
def potential(mass, height, g=9.8 * u.m / u.s **2):
    return (0.5 * mass * g * height).to(u.joule)

# run it for some values
potential(5 * u.kg, 30 *u.cm)

# on Mars:
potential(5 * u.kg, 1 * u.m, g=3.75 * u.m/u.s**2)


I am working on the file "04-Units/Units_Quantities_Instructor.ipynb".
The first task is:

 Challenges

 Convert the speed in imperial units (miles/hour) using:

 ```from astropy.units import imperial```

 Calculate whether a pint is more than half litre

 <emph>You can compare quantities as comparing variables.</emph>

 Something strange? Check what deffinition of pint astropy is using.

 Does units work with areas? calculate the area of a rectangle of 3 km of side and 5 meter of width. Show them in m^2 and convert them to yards^2

Can you write Python code for it?

#1
from astropy.units import imperial
print(speed.to(imperial.mile/u.hour))

#2
imperial.pint > 0.5 * u.l
 # A liquid pint in US is 473 ml; in UK is 568 ml

#3
rectangle_area = 3 * u.km * 5 * u.m
print(rectangle_area)
print(rectangle_area.decompose())
print(rectangle_area.to(imperial.yard ** 2))


I am working on the file "04-Units/Units_Quantities_Instructor.ipynb".
The first task is:
Arrays
Quantities can also be applied to arrays
Can you write Python code for it?

# different ways of defining a quantity for a single value
length = 44 * u.m
time = u.Quantity(23, u.s)
speed = length / time
speed

# now with lists
length_list = [1, 2, 3] * u.m

# and arrays
import numpy as np
time_array = np.array([1, 2, 3]) * u.s

# and its arithmetics
length_list / time_array

# angles are smart!
angle = u.Quantity(np.arange(180), u.deg)
print(angle[[0, -1]])
print(np.sin(angle[[0, -1]]))


I am working on the file "04-Units/Units_Quantities_Instructor.ipynb".
The first task is:
Composed units
Many units are compositions of others, for example, one could create new combinationes for ease of use:
Can you write Python code for it?

# create a composite unit
cms = u.cm / u.s
speed.to(cms)

# and in the imperial system
mph = imperial.mile / u.hour
speed.to(mph)


I am working on the file "04-Units/Units_Quantities_Instructor.ipynb".
The first task is:
Create your own units
Some times we want to create our own units:
Can you write Python code for it?

# Create units for a laugh scale
titter = u.def_unit('titter')
chuckle = u.def_unit('chuckle', 5 * titter)
laugh = u.def_unit('laugh', 4 * chuckle)
guffaw = u.def_unit('guffaw', 3 * laugh)
rofl = u.def_unit('rofl', 4 * guffaw)
death_by_laughing = u.def_unit('death_by_laughing', 10 * rofl)
print((1 * rofl).to(titter))


I am working on the file "04-Units/Units_Quantities_Instructor.ipynb".
The first task is:
Creating functions with quantities as units
We want to have functions that contain the information of the untis, and with them we can be sure that we will be always have the right result.
Can you write Python code for it?

# Create a function for the Kinetic energy
@u.quantity_input(mass=u.kg, speed=u.m/u.s)
def kinetic(mass, speed):
    return (mass * speed ** 2 / 2.).to(u.joule)

# run with and without units
kinetic(5, 10) # Fails! it doesn't specify the units.
kinetic(5 * u.kg, 100 * cms)


I am working on the file "04-Units/Units_Quantities_Instructor.ipynb".
The first task is:
Plotting quantities
To work nicely with matplotlib we need to do as follows:
Can you write Python code for it?

# allowing for plotting
from astropy.visualization import quantity_support
quantity_support()

# loading matplotlib
%matplotlib inline
from matplotlib import pyplot as plt

# Ploting the previous array
plt.plot(angle, np.sin(angle))


I am working on the file "04-Units/Units_Quantities_Instructor.ipynb".
The first task is:
Quantities can be converted to other units systems or factors by using to()
Can you write Python code for it?

# Convert it to: km, lyr
print(length.to(u.km))
print(length.to(u.lightyear))


I am working on the file "04-Units/Units_Quantities_Instructor.ipynb".
The first task is:
Sometime we get no units quantitites
Can you write Python code for it?

# no units
nounits = 20. * u.cm / (1. * u.m)
nounits


I am working on the file "04-Units/Units_Quantities_Instructor.ipynb".
The first task is:
We can do arithmetic operations when the quantities have the compatible units:
Can you write Python code for it?

# arithmetic with distances
distance_start = 10 * u.mm
distance_end = 23 * u.km
length = distance_end - distance_start
print(length)


I am working on the file "04-Units/Units_Quantities_Instructor.ipynb".
The first task is:
What happen if we add a number to this?
Can you write Python code for it?

# arithmetic with no units
nounits + 3

# final value of a no unit quantity
nounits.decompose() # It's a unitless quantity


I am working on the file "04-Units/Units_Quantities_Instructor.ipynb".
The first task is:
and others are already a composition:
Can you write Python code for it?

# what can be converted from s-1?
(u.s ** -1).compose()

# or Jules?
(u.joule).compose()

# Unity of R
(13.605692 * u.eV).to(u.Ry)


I am working on the file "04-VotingProfile/Clustering/1-DistanceComputationPeople.ipynb".
The first task is:
In the next step, we create a new dataframe with people as indexes, and all the voting Bill / Business title as column.
The profile matrix contains all the voting decisions of each person (one column per person) for all the subjects (one row per subject), with conventions :
- 0 : Yes
- 1 : No
- 0.5 : did not participate
- -1 : was not part of the parliament at the time of votation
Can you write Python code for it?

def processVote(vote):
    if vote == 1 or vote == 2:
        return vote-1
    return 0.5

profile_df = pd.DataFrame(data = -1, index = people, columns = texts)
#profile_df.loc[people[0], voting_df_copy.loc[people[0]].index] = voting_df_copy.loc[people[0]].Decision
for p in people:
    profile_df.loc[p, voting_df_copy.loc[p].index] = [processVote(x) for x in voting_df_copy.loc[p].Decision]
profile_df.head()

profile_df.to_csv("profileMatrix.csv")
print(profile_df.loc['Brigitta M. Gadient'].values)
profile_df.loc['Duri Campell'].values

profile_df.loc[people[0]].values

def distance(p1, p2):
    d = 0.0
    nCommonVotes = 0
    for i in range(len(p1)):
        if not (p1[i] == -1 or p2[i] == -1):
            nCommonVotes += 1
            d += (p1[i] - p2[i]) * (p1[i] - p2[i])
    if nCommonVotes == 0:
        return 100
    return np.sqrt(d / nCommonVotes)

n = people.shape[0]
distanceMatrix = np.zeros((n,n))

for i in range(n):
    if i % 10 == 0:
        print("Compute distances from person " + str(i))
    for j in range(n):
        distanceMatrix[i][j] = distance(profile_df.loc[people[i]].values,
                                        profile_df.loc[people[j]].values)

import networkx as nx
G = nx.from_numpy_matrix(distanceMatrix)
nx.draw(G)

import pylab as plt
plt.show()

print("Mean distance : {d}".format(d = np.mean(distanceMatrix)))

import pandas as pd
df = pd.DataFrame(distanceMatrix, index = people, columns = people)
df.to_csv("distanceMatrix.csv")
df.head()

#voting_df['ParlGroupName']
groupId = {"Groupe conservateur-catholique" : 1, "Groupe socialiste" : 2, "Groupe des Paysans, Artisans et Bourgeois" : 3,
           "Groupe radical-démocratique" : 4, "Groupe écologiste" : 5, "Groupe BD" : 6, "Groupe vert'libéral" : 7, "Non inscrit" : 8}
GroupPeople_df = voting_df.drop_duplicates(['Name'], keep = 'last')
GroupPeople_df = GroupPeople_df.set_index('Name')
GroupPeople_df['ParlGroupName'] = [groupId[x] for x in GroupPeople_df['ParlGroupName'].values]
GroupPeople_df =...


I am working on the file "04-VotingProfile/Clustering/1-DistanceComputationPeople.ipynb".
The first task is:
We now want to detect people that vote very differently from their own groups.
Can you write Python code for it?

groupId_inv

list(voting_df[voting_df['Name'] == 'Didier Berberat']['Name'].unique())

GroupPeople_df.index

partite = "Groupe écologiste"
GroupPeople_df[GroupPeople_df == groupId[partite]]

maxDist = 0
furthestPerson = ""
for p in GroupPeople_df[GroupPeople_df == groupId[partite]].index:
    dist = GroupDistance(list(voting_df[voting_df['ParlGroupName'] == partite]['Name'].unique()),
                        list(voting_df[voting_df['Name'] == p]['Name'].unique()))
    if dist > maxDist:
        maxDist = dist
        furthestPerson = p
    print("Mean distance of {person} to its partite {part} : {d}".format(person = p, part = partite, d = dist))

print("The person in partite {part} which is the furthest of the others in terms of voting is : {p}"
     .format(part=partite, p = furthestPerson))

partite = "Groupe écologiste"
GroupPeople_df[GroupPeople_df == groupId[partite]]

maxDist = 0
furthestPerson = ""
for partite in groupDistance_df.index:
    maxDist = 0
    for p in GroupPeople_df[GroupPeople_df == groupId[partite]].index:
        dist = GroupDistance(list(voting_df[voting_df['ParlGroupName'] == partite]['Name'].unique()),
                            list(voting_df[voting_df['Name'] == p]['Name'].unique()))
        if dist > maxDist:
            maxDist = dist
            furthestPerson = p
            #print("{pers} {d}".format(pers=p, d=dist))
        #print("Mean distance of {person} to its partite {part} : {d}".format(person = p, part = partite, d = dist))

    print("The person in partite {part} which is the furthest of the others in terms of voting is : {p} with distance {d}"
         .format(part=partite, p = furthestPerson, d = maxDist))


I am working on the file "04-VotingProfile/Clustering/1-DistanceComputationPeople.ipynb".
The first task is:
We observe that the average distance between all parliament members is 0.53, and that average distance between people belonging to a same partite is always smaller than 0.43, thus it indicates as expected that people belonging to a same partite vote in a similar way.
We then want to observe the difference between each groups. The difference between 2 groups is defined as the sum of all distances between people from one group to people from the other group (see function GroupDistance).
Can you write Python code for it?

# Computes the mean distance of people within group1 from one within group2 (lists of string)
def GroupDistance(group1, group2):
    d = 0.0
    nbPairs = 0
    for p1 in group1:
        for p2 in group2:
            nbPairs += 1
            d += distance(profile_df.loc[p1].values,
                          profile_df.loc[p2].values)
    return d / nbPairs

partite = "Groupe socialiste"
for p in Group_df.index:
    #dist = meanDistance(list(voting_df[voting_df['ParlGroupName'] == p]['Name'].unique()))
    dist = GroupDistance(list(voting_df[voting_df['ParlGroupName'] == partite]['Name'].unique()),
                        list(voting_df[voting_df['ParlGroupName'] == p]['Name'].unique()))
    print("Mean distance between partite {p1} and {p2} is : {d}".format(
        p1 = partite, p2 = p, d = dist))


I am working on the file "04-VotingProfile/Clustering/2-DistanceComputationSubjects.ipynb".
The first task is:
We save the matrix. We observe as expected that the diagonal of the distance matrix contains only 0 as the distance between some subject and itself is 0.
Can you write Python code for it?

import pandas as pd
df = pd.DataFrame(distanceMatrix, index = voting_subjects, columns = voting_subjects)
df.to_csv("distanceMatrixSubjects.csv")
df.head()


I am working on the file "04-VotingProfile/Clustering/2-DistanceComputationSubjects.ipynb".
The first task is:
We then apply it to every pairs of subjects in order to compute the distance matrix.
Can you write Python code for it?

n = voting_subjects.shape[0]
distanceMatrix = np.zeros((n,n))

for i in range(n):
    if i % 10 == 0:
        print("Compute distances from subject " + str(i))
    for j in range(n):
        distanceMatrix[i][j] = distance(voting_df_copy.loc[voting_subjects[i]].values,
                                        voting_df_copy.loc[voting_subjects[j]].values)
print("Mean distance : {d}".format(d = np.mean(distanceMatrix)))


I am working on the file "04-VotingProfile/Clustering/3-Clustering.ipynb".
The first task is:
Analysis of the clustering
We would like to observe whether the obtained clustering spearates well the different political partites. To do so, we compute for each cluster the percentage of people in each partite.
Can you write Python code for it?

ratio_df = pd.DataFrame(index = voting_df.ParlGroupName.unique())
ratio_df['ratio'] = 0
np.array(ratio_df.index)

def ratioPartite(cluster, clusterDist):
    # Compute the partites distribution for all people within this cluster
    people = distances.index[clusterDist == cluster]
    size = len(people)
    ratio_df = pd.DataFrame(index = voting_df.ParlGroupName.unique())
    ratio_df['ratio'] = 1.0
    for group in np.array(ratio_df.index):
        print(group)
        peopleGroup = [p for p in people[voting_df.loc[people].ParlGroupName == group]]
        print(len(peopleGroup) / float(size))
        ratio_df.set_value(group, 'ratio', len(peopleGroup) / float(size))
    return ratio_df

ratio_df = pd.DataFrame(index = voting_df.ParlGroupName.unique(), columns = range(nbClust))
ratio_df[0] = range(8)
ratio_df

ratio_df = pd.DataFrame(index = voting_df.ParlGroupName.unique(), columns = range(nbClust))
for cluster in range(nbClust):
    ratio = ratioPartite(cluster, clusterDist)
    ratio_df[cluster] = ratio.values

ratio_df


I am working on the file "04-VotingProfile/Clustering/3-Clustering.ipynb".
The first task is:
Collect the data
Can you write Python code for it?

path = '../../datas/nlp_results/'
voting_df = pd.read_csv(path+'voting_with_topics.csv')
print('Entries in the DataFrame',voting_df.shape)

#Dropping the useless column
voting_df = voting_df.drop('Unnamed: 0',1)

#Putting numerical values into the columns that should have numerical values
#print(voting_df.columns.values)

num_cols = ['Decision', ' armée', ' asile / immigration', ' assurances', ' budget', ' dunno', ' entreprise/ finance',
           ' environnement', ' famille / enfants', ' imposition', ' politique internationale', ' retraite  ']
voting_df[num_cols] = voting_df[num_cols].apply(pd.to_numeric)

#Inserting the full name at the second position
voting_df.insert(2,'Name', voting_df['FirstName'] + ' ' + voting_df['LastName'])

voting_df = voting_df.drop_duplicates(['Name'], keep = 'last')
voting_df = voting_df.set_index(['Name'])
voting_df.head(3)

profileMatrixFile = 'profileMatrix.csv'
profileMatrix = pd.read_csv(profileMatrixFile, index_col = 0)
profileArray = profileMatrix.values
print(profileArray.shape)
profileMatrix.head()

distanceMatrixFile = 'distanceMatrix.csv'
distances = pd.read_csv(distanceMatrixFile, index_col = 0)
distances = distances.replace(-0.001, 0)
distancesArray = distances.values
print(distancesArray.shape)
distances.head()


I am working on the file "04-VotingProfile/Clustering/3-Clustering.ipynb".
The first task is:
Spectral Clustering
We now apply the clustering algorithm to the adjacency matrix. This matrix is likely not to be symmetric, but the algorithm will symmetrize it, which does make sense in this case.
Can you write Python code for it?

nbClust = 4
clusterDist = sklearn.cluster.spectral_clustering(affinity = distances.values, n_clusters = nbClust)
clusterDist


I am working on the file "04-VotingProfile/PartyAnalysis.ipynb".
The first task is:
2.0 Dealing with a single individual at a time.
2.1 Formatting the DF
Can you write Python code for it?

def format_individual_voting_profile(voting_unique):

    # Setting Name, Party and Topic as indices
    voting_deputee = voting_unique.set_index(['Name','ParlGroupName','Topic'])[['Decision']]

    # Functions to count number of yes/no/absentions, same principle as before
    count_yes = lambda x: np.sum(x==1)/(len(x))
    count_no = lambda x: np.sum(x==2)/(len(x))
    count_abstention = lambda x: np.sum(x==3)/(len(x))

    voting_deputee = voting_deputee.groupby(level=['ParlGroupName','Name','Topic']).agg({'Decision':
                                  {'Yes': count_yes, 'No': count_no,'Abstention': count_abstention}})
    voting_deputee.columns = voting_deputee.columns.droplevel(0)

    return voting_deputee

voting_deputee = format_individual_voting_profile(voting_unique)
voting_deputee.head()


I am working on the file "04-VotingProfile/PartyAnalysis.ipynb".
The first task is:
2.3 Plotting the stats w.r.t to the party for each individual
Can you write Python code for it?

def plot_df(df,item,topic):

    #Setting the size of the plots
    fig_size = plt.rcParams["figure.figsize"]
    fig_size[0] = 25
    fig_size[1] = 6
    plt.rcParams["figure.figsize"] = fig_size

    df_item = df.sort_values(item,ascending=False)
    y = np.array(df_item[item])

    plt.bar(range(df_item.shape[0]), df_item[item], align='center', tick_label=df_item.index)
    plt.ylabel("Deviation from"+item+"mean of party\n item :"+ topic)

    plt.xticks(rotation=90, ha='left')
    plt.xlabel("Deputee")

    plt.show()

party = 'Groupe des Paysans, Artisans et Bourgeois'
test = voting_deputee.xs((party,topics[5]),level=('ParlGroupName','Topic'),drop_level=True).sort_values('Yes',ascending=False)
plot_df(test,'Abstention',topics[5])


I am working on the file "04-VotingProfile/PartyAnalysis.ipynb".
The first task is:
2.4 Computing the stats w.r.t to the party for each individual while aggregating all topics
Can you write Python code for it?

voting_deputee_party.head()

def format_individual_global_voting_profile(voting_unique):

    # Setting the desired multiIndex
    voting_indiv = voting_unique.set_index(['ParlGroupName','Name'])[['Decision']]

    #2. Counting yes/no/abstention

    # Splitting the df by each party and topic, and then aggregating by yes/no/abstention
    # Normalising every entry
    count_yes = lambda x: np.sum(x==1)/(len(x))
    count_no = lambda x: np.sum(x==2)/(len(x))
    count_abstention = lambda x: np.sum(x==3)/(len(x))

    voting_indiv = voting_indiv.groupby(level=['ParlGroupName','Name']).agg({'Decision':
                                  {'Yes': count_yes, 'No': count_no,'Abstention': count_abstention}})

    voting_indiv.columns = voting_indiv.columns.droplevel(0)

    return voting_indiv

def format_party_global_profile(voting_unique):

    #2. Counting yes/no/abstention

    # Splitting the df by each party and topic, and then aggregating by yes/no/abstention
    # Normalising every entry
    count_yes = lambda x: np.sum(x==1)/(len(x))
    count_no = lambda x: np.sum(x==2)/(len(x))
    count_abstention = lambda x: np.sum(x==3)/(len(x))

    overall_party = voting_unique.groupby('ParlGroupName').agg({'Decision':
                                  {'Yes': count_yes, 'No': count_no,'Abstention': count_abstention}})

    overall_party.columns = overall_party.columns.droplevel(0)

    return overall_party
voting_indiv = format_individual_global_voting_profile(voting_unique)
overall_party = format_party_global_profile(voting_unique)

voting_indiv.head()

overall_party.loc[overall_party.index == party]

def compute_overall_party_distance(voting_indiv,overall_party):

    # Retrieve the unique parties and topics
    parties = voting_party.index.get_level_values('ParlGroupName').unique()
    # Extract the features of each party in a more convenient and easily accessible fashion
    party_vote_dict = {}
    for party in parties:
        party_vote = overall_party.loc[overall_party.index...


I am working on the file "04-VotingProfile/PartyAnalysis.ipynb".
The first task is:
Drop duplicates here
Can you write Python code for it?

voting_unique = voting_df.drop_duplicates(['text','Name'], keep = 'last')
voting_unique.head()


I am working on the file "04-VotingProfile/PartyAnalysis.ipynb".
The first task is:
Extracting a party
The function below allows us to select one party and extract the DataFrame from the whole DataFrame
Can you write Python code for it?

party_vote = voting_party.xs('Groupe des Paysans, Artisans et Bourgeois', level='ParlGroupName', drop_level=True)
party_vote.Yes


I am working on the file "04-VotingProfile/PartyAnalysis.ipynb".
The first task is:
Formatting the voting_party DF
Can you write Python code for it?

#voting_party = voting_party.unstack()
#voting_party.columns = voting_party.columns.swaplevel(1, 2)
#voting_party.sortlevel(0,axis=1,inplace=True)

voting_party = format_party_voting_profile(voting_unique)
voting_party.head()


I am working on the file "04-VotingProfile/PartyAnalysis.ipynb".
The first task is:
One element left below for a given people and entry. We will work with the voting_unique df.
Can you write Python code for it?

gp = voting_unique.loc[voting.Name=='Guy Parmelin']
gpt = gp.loc[gp.text == "Arrêté fédéral concernant la contribution de la Suisse en faveur de la Bulgarie et de la Roumanie au titre de la réduction des disparités économiques et sociales dans l'Union européenne élargie Réduction des disparités économiques et sociales dans l'UE. Contribution de la Suisse en faveur de la Roumanie et de la Bulgarie"]
gpt[['Decision','VoteEnd']]


I am working on the file "04-VotingProfile/PartyAnalysis.ipynb".
The first task is:
Some statistics about the whole DF.
Retrieve all topics and parties
Can you write Python code for it?

parties = voting_party.index.get_level_values('ParlGroupName').unique()
topics = voting_party.index.get_level_values('Topic').unique()
topics


I am working on the file "04-VotingProfile/PartyAnalysis.ipynb".
The first task is:
The first reduction of our DataFrame is to suppress all the entries of the Decision field, which contain either a 4, a 5, a 6 or a 7. It basically means that the person did not take part to the voting, and that is hence not useful to our purpose.
Can you write Python code for it?

voting_df = voting[((voting.Decision != 4) & (voting.Decision != 5) & (voting.Decision != 6) & (voting.Decision != 7))]
print(voting_df.shape)
#print('Top number of entries in the df :\n', voting_df.Name.value_counts())


I am working on the file "04-VotingProfile/PartyAnalysis.ipynb".
The first task is:
We also drop the duplicates in the votes, keeping only the last one. Plot first an example of vote of a subject (Guy Parmelin)
Can you write Python code for it?

gp = voting.loc[voting.Name=='Guy Parmelin']
gpt = gp.loc[gp.text == "Arrêté fédéral concernant la contribution de la Suisse en faveur de la Bulgarie et de la Roumanie au titre de la réduction des disparités économiques et sociales dans l'Union européenne élargie Réduction des disparités économiques et sociales dans l'UE. Contribution de la Suisse en faveur de la Roumanie et de la Bulgarie"]
gpt[['Decision','VoteEnd']]


I am working on the file "04-VotingProfile/PersonAnalysis/SplitDeputy.ipynb".
The first task is:
1. Loading and formatting the Vote DataFrame
Convert date into datetime format with pd.to_datetime allows us to sort the dates the way we want.

Attributes of datetime object: year, month, day, hour, minute, second, microsecond, and tzinfo.

A few useful German words unknown to me.
Antrag : demande
Can you write Python code for it?

VOTE_DICT  = {1:'Yes', 2:'No', 3:'Abstention', 5:'Absent', 6:'Excused', 7:'President'}

vote_df = load_vote_df()


I am working on the file "04-VotingProfile/PersonAnalysis/SplitDeputy.ipynb".
The first task is:
Aggregating further up at a yearly level and displaying the result.
Can you write Python code for it?

test = test.drop('SessionName',1).groupby('Date').apply(sum).drop('Date',1)
test['Presence'] = test['Yes']+test['No']+test['Abstention']+test['President']
test['Absence'] = test['Excused']+test['Absent']

test.plot.bar(x=test.index, y=['Yes','No','Abstention','Excused','Absent','President'])
#test.plot.bar(x=test.index,y=['Presence','Absence'])


I am working on the file "04-VotingProfile/PersonAnalysis/SplitDeputy.ipynb".
The first task is:
Applying the functions we previously defined to make the information about the vote of each deputee complete.
Can you write Python code for it?

deputee_vote = format_result_vote(voting_df)
deputee_vote = format_party_voting(deputee_vote)
deputee_vote = deputee_vote.fillna('Not specified')
deputee_vote = deputee_vote.drop(['ParlGroupName','ParlGroupCode','IdSession'],1)


I am working on the file "04-VotingProfile/PersonAnalysis/SplitDeputy.ipynb".
The first task is:
First of all, count the overall result of the vote for the whole councile considered
Can you write Python code for it?

def format_result_vote(voting_df):
    # Setting Name, Party and Topic as indices
    result_df = voting_df.set_index('IdVote')[['Decision']]

    result_df = result_df.groupby(level='IdVote')

    # Functions to count number of yes/no/absentions, same principle as before
    count_total = lambda x: sum(x==VOTE_DICT[1])+sum(x==VOTE_DICT[2])+sum(x==VOTE_DICT[3])
    count_yes = lambda x: int(round(100*np.sum(x==VOTE_DICT[1])/count_total(x),0))
    count_no = lambda x: int(round(100*np.sum(x==VOTE_DICT[2])/count_total(x),0))
    count_abstention = lambda x: int(round(100*np.sum(x==VOTE_DICT[3])/count_total(x),0))

    result_df = result_df.agg({'Decision':{'Total_Yes': count_yes, 'Total_No': count_no,'Total_Abstention': count_abstention,
                                          'Total_Total': count_total}})
    result_df.columns = result_df.columns.droplevel(0)
    result_df.reset_index(inplace=True)
    return pd.merge(voting_df, result_df, on= 'IdVote')


I am working on the file "04-VotingProfile/PersonAnalysis/SplitDeputy.ipynb".
The first task is:
Secondly, compute the voting result aggregated to a party level, to be able to know whetether someone voted like his party
Can you write Python code for it?

def format_party_voting(voting_df):
    # Setting Name, Party and Topic as indices
    party_voting_df = voting_df.set_index(['ParlGroupCode','IdVote'])[['Decision']]

    party_voting_df = party_voting_df.groupby(level=['ParlGroupCode','IdVote'])

    # Functions to count number of yes/no/absentions, same principle as before
    count_total = lambda x: sum(x==VOTE_DICT[1])+sum(x==VOTE_DICT[2])+sum(x==VOTE_DICT[3]) if sum(x==VOTE_DICT[1])+sum(x==VOTE_DICT[2])+sum(x==VOTE_DICT[3]) != 0 else 1
    count_yes = lambda x: int(round(100*np.sum(x==VOTE_DICT[1])/count_total(x),0))
    count_no = lambda x: int(round(100*np.sum(x==VOTE_DICT[2])/count_total(x),0))
    count_abstention = lambda x: int(round(100*np.sum(x==VOTE_DICT[3])/count_total(x),0))

    party_voting_df = party_voting_df.agg({'Decision':{'Party_Yes': count_yes, 'Party_No': count_no,'Party_Abstention': count_abstention,
                                                      'Party_Total':count_total}})
    party_voting_df.columns = party_voting_df.columns.droplevel(0)
    party_voting_df.reset_index(inplace=True)
    return pd.merge(voting_df, party_voting_df, on=['ParlGroupCode', 'IdVote'])


I am working on the file "04-VotingProfile/PersonAnalysis/SplitDeputy.ipynb".
The first task is:
Simply trying the aggregation at a session level for a deputy in particular, it will have to be done for every single one after, then exporting the resulting DataFrame to a .csv or .json to make it readable with javascript.
Can you write Python code for it?

test = voting_dict['Filippo Leutenegger']
test.head()

voting_dict['Filippo Leutenegger'].loc[voting_dict['Filippo Leutenegger'].IdSession==4913].SessionName.unique()[0]

test = voting_dict['Filippo Leutenegger'].groupby(['IdSession','SessionName','Date'])

count_yes = lambda x: np.sum(x==VOTE_DICT[1])
count_no = lambda x: np.sum(x==VOTE_DICT[2])
count_abstention = lambda x: np.sum(x==VOTE_DICT[3])
count_absent = lambda x: np.sum(x==VOTE_DICT[5])
count_excused = lambda x: np.sum(x==VOTE_DICT[6])
count_president = lambda x: np.sum(x==VOTE_DICT[7])
name_session = lambda x: x.unique()[0]
year_session = lambda x: x.unique()[0]

test = test.agg({'Decision':{'Yes': count_yes, 'No': count_no,'Abstention': count_abstention,
                'Excused':count_excused, 'Absent':count_absent, 'President':count_president}})

test.columns = test.columns.droplevel(0)
#test_grouped = test_grouped.join(test['SessionName','Year'])
test = test.reset_index().set_index('IdSession')
test = test[['Yes','No','Abstention','Absent','Excused','President','SessionName','Date']]
#test['Total'] = test.sum(axis=1)
test


I am working on the file "04-VotingProfile/PersonAnalysis/SplitDeputy.ipynb".
The first task is:
Still has a lot of data, but we tried to keep a minimum of it to make it readable by a human as well as having all the information we need to display.
3. Format the DataFrame
a) Session level statistics for each of the deputies.
Now, format the way we want to for exporting for each person
First of all : Global stats -> Yes/No/Abstention/Absence/Excused/President at a session level.
Can you write Python code for it?

voting_dict = split_df_dict(voting_df, 'Name')