# -*- coding: utf-8 -*-
# ---
# jupyter:
#   '@webio':
#     lastCommId: a8ab2762cccf499696a7ef0a86be4d18
#     lastKernelId: 261999dd-7ee7-4ad4-9a26-99a84a77979b
#   cite2c:
#     citations:
#       6202365/8AH9AXN2:
#         URL: http://econ.jhu.edu/people/ccarroll/papers/BufferStockTheory.pdf
#         author:
#         - family: Carroll
#           given: Christopher
#         container-title: Manuscript, Department of Economics, Johns Hopkins University
#         id: 6202365/8AH9AXN2
#         issued:
#           month: 2
#           year: 2019
#         note: "Available at http://econ.jhu.edu/people/ccarroll/papers/BufferStockTheory\
#           \ \nCitation Key: carrollBufferStockTheory \nbibtex*[extra=bibtex:carrollBufferStockTheory]"
#         title: Theoretical Foundations of Buffer Stock Saving
#         type: article-journal
#       6202365/TGG4U7J4:
#         author:
#         - family: Clarida
#           given: Richard H.
#         container-title: International Economic Review
#         issued:
#           date-parts:
#           - - 1987
#         page: "339\u2013351"
#         title: Consumption, Liquidity Constraints, and Asset Accumulation in the Face
#           of Random Fluctuations in Income
#         type: article-journal
#         volume: XXVIII
#       undefined:
#         URL: http://econ.jhu.edu/people/ccarroll/papers/BufferStockTheory.pdf
#         author:
#         - family: Carroll
#           given: Christopher
#         container-title: Manuscript, Department of Economics, Johns Hopkins University
#         issued:
#           date-parts:
#           - - '2019'
#             - 2
#         note: "Available at http://econ.jhu.edu/people/ccarroll/papers/BufferStockTheory\
#           \ \nCitation Key: carrollBufferStockTheory \nbibtex*[extra=bibtex:carrollBufferStockTheory]"
#         title: Theoretical Foundations of Buffer Stock Saving
#         type: article-journal
#   jupytext:
#     formats: ipynb,py:percent
#     text_representation:
#       extension: .py
#       format_name: percent
#       format_version: '1.1'
#       jupytext_version: 0.8.3
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
#   language_info:
#     codemirror_mode:
#       name: ipython
#       version: 3
#     file_extension: .py
#     mimetype: text/x-python
#     name: python
#     nbconvert_exporter: python
#     pygments_lexer: ipython3
#     version: 3.6.7
#   varInspector:
#     cols:
#       lenName: 16
#       lenType: 16
#       lenVar: 40
#     kernels_config:
#       python:
#         delete_cmd_postfix: ''
#         delete_cmd_prefix: 'del '
#         library: var_list.py
#         varRefreshCmd: print(var_dic_list())
#       r:
#         delete_cmd_postfix: ') '
#         delete_cmd_prefix: rm(
#         library: var_list.r
#         varRefreshCmd: 'cat(var_dic_list()) '
#     types_to_exclude:
#     - module
#     - function
#     - builtin_function_or_method
#     - instance
#     - _Feature
#     window_display: false
# ---

# %% [markdown]
# # Buffer Stock Saving in HARK and dolo
#

# %% [markdown]
# This notebook compares the solutions to a standard buffer stock saving model obtained by the [Econ-ARK/HARK](https://github.com/econ-ark/HARK) toolkit and the [dolo](https://github.com/EconForge/dolo) modeling system.

# %% {"code_folding": [0]}
# This cell does some setup and imports generic tools

Generator=True # Is this notebook the master or is it generated?
# Import related generic python packages
import numpy as np
from time import clock
mystr = lambda number : "{:.4f}".format(number)

# This is a jupytext paired notebook that autogenerates BufferStockTheory.py
# which can be executed from a terminal command line via "ipython BufferStockTheory.py"
# But a terminal does not permit inline figures, so we need to test jupyter vs terminal
# Google "how can I check if code is executed in the ipython notebook"

from IPython import get_ipython # In case it was run from python instead of ipython
def in_ipynb():
    try:
        if str(type(get_ipython())) == "<class 'ipykernel.zmqshell.ZMQInteractiveShell'>":
            return True
        else:
            return False
    except NameError:
        return False

# Determine whether to make the figures inline (for spyder or jupyter)
# vs whatever is the automatic setting that will apply if run from the terminal
if in_ipynb():
    # %matplotlib inline generates a syntax error when run from the shell
    # so do this instead
    get_ipython().run_line_magic('matplotlib', 'inline')
else:
    get_ipython().run_line_magic('matplotlib', 'auto')
    print('You appear to be running from a terminal')
    print('By default, figures will appear one by one')
    print('Close the visible figure in order to see the next one')

# Import the plot-figure library matplotlib

import matplotlib.pyplot as plt

# In order to use LaTeX to manage all text layout in our figures, we import rc settings from matplotlib.
from matplotlib import rc
plt.rc('font', family='serif')

# LaTeX is huge and takes forever to install on mybinder
# so if it is not installed then do not use it
from distutils.spawn import find_executable
iflatexExists=False
if find_executable('latex'):
    iflatexExists=True

plt.rc('font', family='serif')
plt.rc('text', usetex=iflatexExists)

# The warnings package allows us to ignore some harmless but alarming warning messages
import warnings
warnings.filterwarnings("ignore")

# The tools for navigating the filesystem
import sys
import os

sys.path.insert(0, os.path.abspath('../../lib')) # REMARKs directory is two down from root

from HARK.utilities import plotFuncsDer, plotFuncs
from copy import copy, deepcopy

# Define (and create, if necessary) the figures directory "Figures"
if Generator:
    my_file_path = os.path.dirname(os.path.abspath("BufferStockTheory.ipynb")) # Find pathname to this file:
    Figures_HARK_dir = os.path.join(my_file_path,"Figures/") # LaTeX document assumes figures will be here
    Figures_HARK_dir = os.path.join(my_file_path,"/tmp/Figures/") # Uncomment to make figures outside of git path
    if not os.path.exists(Figures_HARK_dir):
        os.makedirs(Figures_HARK_dir)

# %% [markdown]
# ## [The Problem](http://econ.jhu.edu/people/ccarroll/papers/BufferStockTheory/#The-Problem)
#
# [This paper](http://econ.jhu.edu/people/ccarroll/papers/BufferStockTheory/#The-Problem) defines a buffer stock saving model and calibrates parameters:
#
# | Parameter | Description | Code | Value |
# | :---: | ---         | ---  | :---: |
# | $\newcommand{\PermGroFac}{\Gamma}\PermGroFac$ | Permanent Income Growth Factor | $\texttt{PermGroFac}$ | 1.03 |
# | $\newcommand{\Rfree}{\mathrm{\mathsf{R}}}\Rfree$ | Interest Factor | $\texttt{Rfree}$ | 1.04 |
# | $\newcommand{\DiscFac}{\beta}\DiscFac$ | Time Preference Factor | $\texttt{DiscFac}$ | 0.96 |
# | $\newcommand{\CRRA}{\rho}\CRRA$ | Coeﬃcient of Relative Risk Aversion| $\texttt{CRRA}$ | 2 |
# | $\newcommand{\UnempPrb}{\wp}\UnempPrb$ | Probability of Unemployment | $\texttt{UnempPrb}$ | 0.005 |
# | $\newcommand{\IncUnemp}{\mu}\IncUnemp$ | Income when Unemployed | $\texttt{IncUnemp}$ | 0. |
# | $\newcommand{\PermShkStd}{\sigma_\psi}\PermShkStd$ | Std Dev of Log Permanent Shock| $\texttt{PermShkStd}$ | 0.1 |
# | $\newcommand{\TranShkStd}{\sigma_\theta}\TranShkStd$ | Std Dev of Log Transitory Shock| $\texttt{TranShkStd}$ | 0.1 |
#
# For a microeconomic consumer with 'Market Resources' (net worth plus current income) $M_{t}$, end-of-period assets $A_{t}$ will be the amount remaining after consumption of $C_{t}$.
# \begin{eqnarray}
# A_{t}   &=&M_{t}-C_{t}  \label{eq:DBCparts} \\
# %B_{t+1}   & = & A_{t} R \notag \\
# \end{eqnarray}
#
# The consumer's permanent noncapital income $P$ grows by a predictable factor $\PermGroFac$ and is subject to an unpredictable lognormally distributed multiplicative shock $\mathbb{E}_{t}[\psi_{t+1}]=1$,
# \begin{eqnarray}
# P_{t+1} & = & P_{t} \PermGroFac \psi_{t+1}
# \end{eqnarray}
#
# and actual income is permanent income multiplied by a logormal multiplicative transitory shock, $\mathbb{E}_{t}[\theta_{t+1}]=1$, so that next period's market resources are
# \begin{eqnarray}
# %M_{t+1} &=& B_{t+1} +P_{t+1}\theta_{t+1},  \notag
# M_{t+1} &=& A_{t}R +P_{t+1}\theta_{t+1}.  \notag
# \end{eqnarray}
#
# When the consumer has a CRRA utility function $u(c)=\frac{c^{1-\rho}}{1-\rho}$, the paper shows that the problem can be written in terms of ratios of money variables to permanent income, e.g. $m_{t} \equiv M_{t}/P_{t}$, and the Bellman form of [the problem reduces to](http://econ.jhu.edu/people/ccarroll/papers/BufferStockTheory/#The-Related-Problem):
#
# \begin{eqnarray*}
# v_t(m_t) &=& \max_{c_t}~~ u(c_t) + \beta~\mathbb{E}_{t} [(\Gamma\psi_{t+1})^{1-\rho} v_{t+1}(m_{t+1}) ] \\
# & s.t. & \\
# a_t &=& m_t - c_t \\
# m_{t+1} &=& R/(\Gamma \psi_{t+1}) a_t + \theta_{t+1} \\
# \end{eqnarray*}
#
# and the Euler equation for this model is
#
# \begin{eqnarray*}
# c_{t}^{-\rho} & = & R \beta \mathbb{E}_{t}[(\Gamma \psi c_{t+1})^{-\rho})] \\
# 0 & = & R \beta \mathbb{E}_{t}[(\Gamma \psi c_{t+1}/c_{t})^{-\rho})]-1
# \end{eqnarray*}
#
#
# For the purposes of this notebook, the paper's baseline parameterization is changed as follows:
#
# 1. The unemployment (zero-income event) shocks are turned off
# 2. An explicit liqudity constraint is added ($c_{t} \leq m_{t}$)

# %% [markdown]
# # Dolo
#
# [Installation instructions](https://github.com/EconForge/dolo/wiki/Installation) for dolo involve a number of dependencies, including the dolo language "dolang."  This notebook assumes all these have been installed.

# %%
from dolo import *

# %% [markdown]
# Dolo defines models using "model files" whose syntax is specified in the documentation

# %%
model_dolo = yaml_import("../models/bufferstock.yaml")
print( model_dolo )

# %%
# Set a maximum value of the market resources ratio m for use in both models
max_m = 500
model_dolo.data['calibration']['max_m'] = max_m
model_dolo.data['domain']['m'] = [0,'max_m']

# Obtain the decision rule by time iteration
dr = time_iteration(model_dolo,tol=1e-08,verbose=True)

# %% [markdown]
# # HARK
# The HARK tool used here is $\texttt{ConsIndShockModel.py}$.  For an introduction to this module, see the [ConsIndShockModel.ipynb](https://econ-ark.org/notebooks) notebook at the [Econ-ARK](https://econ-ark.org) website.

# %% {"code_folding": [0]}
# Define a parameter dictionary with baseline parameter values

# Set the baseline parameter values
PermGroFac = 1.03
Rfree      = 1.04
DiscFac    = 0.96
CRRA       = 2.00
UnempPrb   = 0.00
IncUnemp   = 0.0
PermShkStd = 0.1
TranShkStd = 0.1
# Import default parameter values
import HARK.ConsumptionSaving.ConsumerParameters as Params

# Make a dictionary containing all parameters needed to solve the model
base_params = Params.init_idiosyncratic_shocks

# Set the parameters for the baseline results in the paper
# using the variable values defined in the cell above
base_params['PermGroFac'] = [PermGroFac]   # Permanent income growth factor
base_params['Rfree']      = Rfree          # Interest factor on assets
base_params['DiscFac']    = DiscFac        # Time Preference Factor
base_params['CRRA']       = CRRA           # Coefficient of relative risk aversion
base_params['UnempPrb']   = UnempPrb       # Probability of unemployment (e.g. Probability of Zero Income in the paper)
base_params['IncUnemp']   = IncUnemp       # Induces natural borrowing constraint
base_params['PermShkStd'] = [PermShkStd]   # Standard deviation of log permanent income shocks
base_params['TranShkStd'] = [TranShkStd]   # Standard deviation of log transitory income shocks

# Some technical settings that are not interesting for our purposes
base_params['LivPrb']       = [1.0]   # 100 percent probability of living to next period
base_params['CubicBool']    = True    # Use cubic spline interpolation
base_params['T_cycle']      = 1       # No 'seasonal' cycles
base_params['BoroCnstArt']  = None    # No artificial borrowing constraint

from HARK.utilities import plotFuncsDer, plotFuncs
from HARK.ConsumptionSaving.ConsIndShockModel import IndShockConsumerType
# %%
# Create a model identical to the dolo model
# Start with the HARK baseline parameters and modify
# to be like the dolo model
base_params_dolo = dict(base_params)
base_params_dolo['BoroCnstArt']  = 0.0    # Liquidity constraint at 0
base_params_dolo['UnempPrb']     = 0      # No point-mass on unemployment state
base_params_dolo['TranShkCount'] = 5      # Default number of nodes in dolo
base_params_dolo['PermShkCount'] = 5
base_params_dolo['aXtraMax']     = max_m  # Use same maximum
base_params_dolo['aXtraCount']   = 100    # How dense to make the grid
base_params_dolo['DiscFac']      = 0.96
#base_params_dolo['CubicBool']    = False
model_HARK = IndShockConsumerType(**base_params_dolo,cycles=0) # cycles=0 indicates infinite horizon

# %%
# Solve the HARK model
model_HARK.updateIncomeProcess()
model_HARK.solve()
model_HARK.UnempPrb = 0.05
model_HARK.unpackcFunc()

# %%

# Plot the results: Green is perfect foresight, red is HARK, black is dolo
tab = tabulate(model_dolo, dr, 'm')
plt.plot(tab['m'],tab['c'])     # This is pretty cool syntax
m = tab.iloc[:,2]
c_m  = model_HARK.cFunc[0](m)
# cPF uses the analytical formula for the perfect foresight solution
cPF = (np.array(m)-1+1/(1-PermGroFac/Rfree))*((Rfree-(Rfree * DiscFac)**(1/CRRA))/Rfree)
plt.plot(tab['m'],c_m,color="red")
plt.plot(m,cPF,color="green")


#!/usr/bin/env python
# coding: utf-8

# In[1]:

import os
from sympy import *
import pandas as pd
import numpy as np
import scipy.fftpack
import matplotlib as mpl
import matplotlib.pyplot as plt
plt.style.use("seaborn-paper")

def find_nearest(array, value):
    array = np.asarray(array)
    idx = (np.abs(array - value)).argmin()
    return idx

# ## Canvas palette

# In[2]:

#Canvas for single plot
x = np.linspace(0,10,100)
y = np.sin(x)
plt.figure(figsize=[14,6])
plt.grid(True)
plt.title("Change-me!",fontsize=20)
plt.plot(x,y,label="testvalue")
plt.legend(fontsize=16)
plt.xlabel("XLABEL (unit)",fontsize=18)
plt.ylabel("YLABEL (unit)",fontsize=18)
plt.show()

# In[3]:

#Canvas for side by side
fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(14,6))
fig.suptitle("test",y=1.05,fontsize=20)

axes[0].grid(True)
axes[0].plot(x,y,label="testvalue")
axes[0].legend(fontsize=16)
axes[0].set_title("TESTTITLE",fontsize=18)
axes[0].set_xlabel("XLABEL (unit)",fontsize=18)
axes[0].set_ylabel("YLABEL (unit)",fontsize=18)
axes[0].legend(fontsize=16)
axes[0].tick_params(axis='both', which='major', labelsize=15)

axes[1].grid(True)
axes[1].plot(x,y,label="testvalue")
axes[1].legend(fontsize=16)
axes[1].set_title("TESTTITLE",fontsize=18)
axes[1].set_xlabel("XLABEL (unit)",fontsize=18)
axes[1].set_ylabel("YLABEL (unit)",fontsize=18)
axes[1].legend(fontsize=16)
axes[1].tick_params(axis='both', which='major', labelsize=15)

fig.tight_layout()
plt.show()

# In[4]:

#Canvas for side by side
fig, axes = plt.subplots(nrows=2, ncols=4, figsize=(14,6))
fig.suptitle("test",y=1.05,fontsize=20)

axes[0,0].grid(True)
axes[0,0].plot(x,y,label="testvalue")
axes[0,0].legend(fontsize=16)
axes[0,0].set_title("TESTTITLE",fontsize=18)
axes[0,0].set_xlabel("XLABEL (unit)",fontsize=18)
axes[0,0].set_ylabel("YLABEL (unit)",fontsize=18)
axes[0,0].legend(fontsize=16)
axes[0,0].tick_params(axis='both', which='major', labelsize=15)

axes[0,1].grid(True)
axes[0,1].plot(x,y,label="testvalue")
axes[0,1].legend(fontsize=16)
axes[0,1].set_title("TESTTITLE",fontsize=18)
axes[0,1].set_xlabel("XLABEL (unit)",fontsize=18)
axes[0,1].set_ylabel("YLABEL (unit)",fontsize=18)
axes[0,1].legend(fontsize=16)
axes[0,1].tick_params(axis='both', which='major', labelsize=15)

fig.tight_layout()
plt.show()

# ## Read data

# In[5]:

#Folder and paths definitions
main_path  = os.getcwd()
datafolder_path = main_path+"/results"
results_dir = "/output_py"
output_dir = main_path+results_dir
try:
    os.mkdir(output_dir)
except OSError:
    print ("Creation of the directory %s failed" % results_dir)
else:
    print ("Successfully created the directory %s " % results_dir)

# In[ ]:

# In[6]:

#Simulation parameters
N = 5000
T = 10000
n_runs = 20
dt = .01
freq = "gfreq"
MF = "MF"

if(freq =="gfreq"):
    freq_plot="$\\mathcal{N}(0,1)$ natural freqs"
else:
    freq_plot="Uniformly distributed freqs $\\in[-.5,.5]$"
if(MF =="MF"):
    MF_plot="MeanField"
else:
    MF_plot="non-MeanField"

# In[13]:

#OutputFileNames
#S/N --> |r(t)|/sigma(r(t))
sn_name = "S_N"
#(Mod&Phase)(t)
modphase_name = "ModPhase_t"
#Spectrum
spectrum_name = "Spectrum"
#r_inf
rinf_name = "r_inf"
#Configuration-specific name
config_name= "/N%d_nruns%d_freq=%s_"%(N,n_runs,freq)

# In[19]:

#K for simulation
K_r0 = np.arange(0,1.4,.2) #da 0 a 1.2 a step di .2, note the last step is not included!
K_r1 = np.arange(1.21,2.01,.01)
K_r2 = np.arange(2.1,5.1,.1)
Kvalues = np.concatenate((K_r0,K_r1,K_r2))
Kvalues = np.unique(Kvalues, axis=0)
print(len(Kvalues))
Kvalues

# In[9]:

#Create dataframe dictionary. For each entry, first value is the K of the dataframe (second value)
data = []
for i in range(0,len(Kvalues)):
    filename = datafolder_path + "/%s_uphase_N%d_%s_T%d_dt%.4f_nruns%d_K%.3f.tsv"%(freq,N,MF,T,dt,n_runs,Kvalues[i])
    #cols refers to timestep, avgmod, stdmod, avgphase,stdphase (of order parameter)
    df = pd.read_csv(filename,sep="\t",header=None)
    data.append([format(Kvalues[i],'.3f'),df])

# In[ ]:

# ## Plots

# In[18]:

#Plot settings
alph = 1
tmax =100

selected_index = [0,3,17,21,25,47]
fig = plt.figure(figsize=[14,6])
plt.grid(True)
plt.title("N = %d, dt = %.3f, %s, n_runs = %d" % (N,dt,freq_plot,n_runs),fontsize=20)
for i in selected_index:
    plt.plot(data[i][1][0],data[i][1][1]/data[i][1][2],ls='--',marker='.',markersize=.5,label="K=%s"%(data[i][0]),alpha=alph)
    plt.semilogy()
    plt.legend(fontsize=16)
plt.xlabel("t",fontsize=18)
plt.ylabel("$\\frac{|r(t)|}{\\sigma(r(t))}$",fontsize=20,rotation=0)
plt.xlim(0,tmax)

fig.tight_layout()
plt.savefig(output_dir+config_name+sn_name)
plt.show()

# In[11]:

#Plot settings
alph = 1
tmax =80

fig, axes = plt.subplots(nrows=2, ncols=1, figsize=(14,6))

fig.suptitle("N = %d, dt = %.3f, %s, n_runs = %d"%(N,dt,freq_plot, n_runs),y=.95,fontsize=20)

axes[0].grid(True)

for i in selected_index:
    axes[0].errorbar(data[i][1][0],data[i][1][1],yerr=data[i][1][2],ls='--',linewidth=.5,fmt='.',markersize=.05, elinewidth=.5, capthick=.5,label="K=%s"%(data[i][0]),alpha=alph)
    axes[1].errorbar(data[i][1][0],data[i][1][3],yerr=data[i][1][4],ls='--',linewidth=.5,fmt='.',markersize=.05, elinewidth=.5, capthick=.5,label="K=%s"%(data[i][0]),alpha=alph)

axes[0].legend(fontsize=16)
axes[0].set_title("|r(t)|",fontsize=18)
axes[0].set_xlabel("t",fontsize=18)
axes[0].set_ylabel("",fontsize=18)
axes[0].legend(fontsize=16,ncol=2)
axes[0].set_xlim(0,tmax)
axes[0].tick_params(axis='both', which='major', labelsize=15)

axes[1].grid(True)
axes[1].legend(fontsize=16)
axes[1].set_title("Arg(r(t))",fontsize=18)
axes[1].set_xlabel("t",fontsize=18)
axes[1].set_ylabel("",fontsize=18)
axes[1].legend(fontsize=16,ncol=2)
axes[1].set_xlim(0,tmax)
axes[1].tick_params(axis='both', which='major', labelsize=15)
fig.tight_layout()

plt.subplots_adjust(top=.85)
plt.savefig(output_dir+config_name+modphase_name)
plt.show()

# In[12]:

max_K_in_plot = 8
max_n_of_K_in_plot = 5
idx_max_K_in_plot = find_nearest(Kvalues,max_K_in_plot)

fig, axes = plt.subplots(nrows=2, ncols=3, figsize=(14,6))
fig.suptitle("$\\mathcal{F}(r(t))$,\nN = %d, dt = %.3f, %s, n_runs = %d"%(N,dt,freq_plot, n_runs),y=1,fontsize=20)
counter1 = 0
counter2 = 0
for i in selected_index:
    if(counter1<len(selected_index)/2):
        axes[0,counter1].grid(True)
        im = axes[0,counter1].specgram(data[i][1][1],Fs=1/dt)
        axes[0,counter1].set_title("K = %s"%(data[i][0]),fontsize=18)
        axes[0,counter1].set_xlabel("t",fontsize=18)
        axes[0,counter1].set_ylabel("Freq.",fontsize=18)
        axes[0,counter1].tick_params(axis='both', which='major', labelsize=15)
        counter1 = counter1+1
    else:
        axes[1,counter2].grid(True)
        im = axes[1,counter2].specgram(data[i][1][1],Fs=1/dt)
        axes[1,counter2].set_title("K = %s"%(data[i][0]),fontsize=18)
        axes[1,counter2].set_xlabel("t",fontsize=18)
        axes[1,counter2].set_ylabel("Freq.",fontsize=18)
        axes[1,counter2].tick_params(axis='both', which='major', labelsize=15)
        counter2 = counter2+1

fig.tight_layout()
plt.subplots_adjust(top=.85)

plt.savefig(output_dir+config_name+spectrum_name)

plt.show()

# In[20]:

#for r_inf evaluation
Kval_list = []
r_inf = []
r_inf_err = []
last_percent = .9
for i in range(0,len(Kvalues)):
    r_inf.append([np.mean(data[i][1][1][int(len(data[i][1][1])*last_percent):])])
    r_inf_err.append([np.std(data[i][1][1][int(len(data[i][1][1])*last_percent):])])
    Kval_list.append([Kvalues[i]])

# # ADD ERRORBARS

# In[ ]:

fig = plt.figure(figsize=[14,6])
plt.grid(True)
plt.title("N = %d, dt = %.3f, %s, n_runs = %d"%(N,dt,freq_plot, n_runs),y=1,fontsize=20)
#plt.errorbar(Kval_list,r_inf,y_err=r_inf_err,label="testvalue")
plt.errorbar(Kval_list,r_inf,ls='--',linewidth=.5,fmt='.',markersize=5, elinewidth=.5, capthick=.5,label="Average on last %d steps"%((1-last_percent)*T+1))
plt.legend(fontsize=16)
plt.xlabel("K",fontsize=18)
plt.ylabel("$r_{\\infty}$",fontsize=18,rotation=0)

fig.tight_layout()
plt.savefig(output_dir+config_name+rinf_name)

plt.show()


#!/usr/bin/env python
# coding: utf-8

# # Recomendation system : Book Recommendation

# ### Oleh : [Ahmad Habib Husaini](https://www.linkedin.com/in/ahmad-habib-husaini-1705711b0/)
#
# #### Pendahuluan
# Pada proyek ini, topik yang dibahas adalah mengenai pembuatan sistem rekomendasi dengan data book . Proyek ini dibuat untuk proyek Submission 2 - Machine Learning Terapan Dicoding.

# # 1. Import important package

# In[1]:

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import opendatasets
import wget
import zipfile
from tqdm import tqdm
import os

# # 2. Data loading

# ## 2.1 Data Acquisition

# In[2]:

if os.path.exists('book-recommendation-dataset'):
    print("file sudah ada")
else:
    opendatasets.download_kaggle_dataset(dataset_url='https://www.kaggle.com/arashnic/book-recommendation-dataset', data_dir='')

# Pada tahapan *data loading* berisikan akuisisi data dengan mengunduh dataset pada [link](https://www.kaggle.com/arashnic/book-recommendation-dataset) dengan menggunakan library `opendatasets` dengan sintaks seperti diatas. Ketika merunning code tersebut akan diminta mengisikan username dan key.
# 1. username silahkan disisi dengan username *account* kaggle
# 2. key didapatkan dengan :
#     1. Buka website [kaggle](https://www.kaggle.com/),
#     2. login dengan akun masing-masing.
#     3. Pilih your profile pada kanan atas
#     4. Pilih *account*
#     5. Scroll sedikit kebawah maka akan ada pilihan `Create new API token`
#

#     6. Setelah menekan pilihan tersebut akan terunduh file kaggle.json yang berisikan username dan key

# ## 2.2 Memuat data dalam format Dataframe

# ### 2.2.1 Book Dataframe

# In[3]:

df_book = pd.read_csv('book-recommendation-dataset/Books.csv')
df_book.head(3)

# ### 2.2.2 Rating Dataframe

# In[4]:

df_rating = pd.read_csv('book-recommendation-dataset/Ratings.csv')
df_rating.head(3)

# ### 2.2.4 User Dataframe

# In[5]:

df_user = pd.read_csv('book-recommendation-dataset/Users.csv')
df_user.tail(3)

# # 3. Data Understanding

# ## 3.1 Informasi dataset

# ![image/kaggle_dataset.png](attachment:image.png)

# Pada berkas yang dapat diunduh pada link [berikut](https://www.kaggle.com/arashnic/book-recommendation-dataset?select=Users.csv) berisikan dataset berjumlah 3 buah file yakni Book.csv, Rating.csv dan User.csv
#

# ### 3.1.1 Book
#
# Berisikan data sebanyak 271360 baris dan 8 kolom yang terdiri dari:

#
# Kolom  | Keterangan
# :------------- | :-------------
# Sumber  | https://www.kaggle.com/arashnic/book-recommendation-dataset?select=Books.csv
# ISBN | International Standard Book Number merupakan kode unik masing masing buku
# Book-title | Merupakan judul dari tiap buku
# Book-Author | Merupakan penulis atau pengarang tiap buku
# Year-Of_Publication |Merupakan tahun terbit tiap buku
# Publisher | Merupakan lebaga penerbit tiap buku
# Image-URL-S | Link foto dari tiap buku berukuran kecil
# Image-URL-M | Link foto dari tiap buku berukuran sedang
# Image-URL-L | Link foto dari tiap buku berukuran besar

# ### 3.1.2 Rating
# Berisikan data sebanyak 1149780 baris dan 3 kolom yang terdiri dari:
#
# Kolom  | Keterangan
# :------------- | :-------------
# Sumber  | https://www.kaggle.com/arashnic/book-recommendation-dataset?select=Ratings.csv
# User-ID | Merupakan kode unik tiap user
# ISBN | International Standard Book Number merupakan kode unik masing masing buku
# Book-Rating |Merupakan rating dari tiap buku

# ### 3.1.3 User
#
# Berisikan data sebanyak 278858 baris dan 4 kolom yang terdiri dari:

#
# Kolom  | Keterangan
# :------------- | :-------------
# Sumber  | https://www.kaggle.com/arashnic/book-recommendation-dataset?select=Users.csv
# User-ID | Merupakan kode unik tiap user
# Age | Merupakan umur tiap user
# Location |Merupakan lokasi tiap user

# ## 3.2 Cek Missing Values, tipe data & summary statistical descriptive

# ### 3.2.1 Book

# In[6]:

pd.DataFrame(df_book.isnull().sum(), columns=['missing_values'])

# In[7]:

df_book.info()

# In[8]:

df_book.loc[:,:'Publisher'].describe(include=object)

# ### 3.2.2 Rating

# In[9]:

pd.DataFrame(df_rating.isnull().sum(), columns=['missing_values'])

# In[10]:

df_rating.info()

# In[11]:

df_rating[['Book-Rating']].describe()

# In[12]:

df_rating['Book-Rating'] = df_rating['Book-Rating'].astype(object)

# In[13]:

df_rating.describe(include=[object])

# In[14]:

df_rating.info()

# ### 3.2.3 User

# In[15]:

df_user.head()

# In[16]:

df_user.Location.unique()

# In[17]:

df_user.Location.str.split(',',expand=True).loc[:,:2][0].unique()

# In[18]:

len(df_user.Location.str.split(',',expand=True).loc[:,:2][0].unique())

# In[19]:

df_user.Location.str.split(',',expand=True).loc[:,:2][1].unique()

# In[20]:

len(df_user.Location.str.split(',',expand=True).loc[:,:2][1].unique())

# In[21]:

df_user['city'] = df_user.Location.str.split(',',expand=True)[0]
# df_user['nation_state'] = df_user.Location.str.split(',',expand=True)[1]
df_user['country'] = df_user.Location.str.split(',',expand=True)[2]
df_user.head()

# In[22]:

pd.DataFrame(df_user.isnull().sum(), columns=['missing_values'])

# In[23]:

df_user.info()

# In[24]:

pd.set_option('display.max_rows',105)

# In[25]:

df_user.groupby('country')[['country']].count().head(10)

# In[26]:

df_user.drop(columns=['country'], inplace=True)

# In[27]:

df_user[['Age']].describe()

# In[28]:

df_user.describe(include=object)

# `Kesimpulan dari Tahapan ini :`
# 1. Book
#     - Buku dengan judul terbanyak adalah Selected Poems dengan jumlah 27 buah
#     - Penulis terbanyak adalah Agatha Christie dengan total 632 karya
#     - Buku terbitan tahun 2002 menjadi buku terbanyak
#     - Lembaga penerbit yang paling banayk menerbitkan buku adalah Harlequin
# 2. Rating
#     - masih banyak buku yang belum dirating, yakni sebanyak 716109
# 3. User
#     - Banyak data umur yang kosong kemungkinan tidak akan digunakan
#     - london menjadi kota user terbanyak sebanyak 139187 user

# ## 3.3 Visualisasi Data

# ### 3.3.1 Book

# In[29]:

count_book = {key: len(df_book[key].unique()) for key in df_book.columns[1:5]}

# In[30]:

plt.figure(figsize=(15,9))
ax = sns.barplot(x=list(count_book.keys()), y=list(count_book.values()))
for p in ax.patches:
        ax.annotate(format(p.get_height(), '.1f'), (p.get_x() + p.get_width() / 2., p.get_height()), ha = 'center', va = 'center', xytext = (0, 10), textcoords = 'offset points')
ax.set_title("Banyak Data Unik Tiap Kolom Dataset Book")
plt.show()

# ### 3.3.2 Rating

# In[31]:

count_rating = {key: len(df_rating[key].unique()) for key in df_rating.columns}

# In[32]:

plt.figure(figsize=(15,9))
ax = sns.barplot(x=list(count_rating.keys()), y=list(count_rating.values()))
for p in ax.patches:
        ax.annotate(format(p.get_height(), '.1f'), (p.get_x() + p.get_width() / 2., p.get_height()), ha = 'center', va = 'center', xytext = (0, 10), textcoords = 'offset points')
ax.set_title("Banyak Data Unik Tiap Kolom Dataset Rating")
plt.show()

# In[33]:

plt.figure(figsize=(15,9))
ax = sns.countplot(data=df_rating, x='Book-Rating')
for p in ax.patches:
        ax.annotate(format(p.get_height(), '.1f'), (p.get_x() + p.get_width() / 2., p.get_height()), ha = 'center', va = 'center', xytext = (0, 10), textcoords = 'offset points')
ax.set_title("Banyak Data Unik Kolom Rating")
plt.show()

# ### 3.3.3 Users

# In[34]:

df_user.head()

# In[35]:

count_users = {key: len(df_user[key].unique()) for key in df_user.columns}

# In[36]:

plt.figure(figsize=(12,9))
ax = sns.barplot(x=list(count_users.keys()), y=list(count_users.values()))
for p in ax.patches:
        ax.annotate(format(p.get_height(), '.1f'), (p.get_x() + p.get_width() / 2., p.get_height()), ha = 'center', va = 'center', xytext = (0, 10), textcoords = 'offset points')
ax.set_title("Banyak Data Unik Tiap Kolom Dataset User")
plt.show()

# In[37]:

city_top10 = df_user.city.value_counts().head(10).reset_index()
city_top10.columns = ['city', 'count']
city_top10

# In[38]:

plt.figure(figsize=(12,9))
ax = sns.barplot(x=city_top10.city, y=city_top10['count'])
for p in ax.patches:
        ax.annotate(format(p.get_height(), '.1f'), (p.get_x() + p.get_width() / 2., p.get_height()), ha = 'center', va = 'center', xytext = (0, 10), textcoords = 'offset points')
ax.set_title("Top 10 Kota Terbanyak")
plt.show()

# # 4. Data Preparation & Data Preprocessing

# ## 4.1 Memilih fitur untuk sistem rekomendasi

# In[39]:

df_fix = df_book.iloc[:,0:4]
df_fix.head()

# In[40]:

pd.DataFrame({
    'missing_values':df_fix.isnull().sum(),
    'presentase':df_fix.isnull().sum()/len(df_fix)
})

# ## 4.2 Hapus Missing Value dan data duplicate pada dataset fix

# In[41]:

df_fix.dropna(inplace=True)
df_fix.drop_duplicates(inplace=True)

# In[42]:

df_fix.shape

# ## 4.3 Create Metadata

# In[43]:

df_fix['Year-Of-Publication'] = df_fix['Year-Of-Publication'].astype('str')

# In[44]:

df_fix['metadata'] = ""
for col in ['Book-Title', 'Book-Author', 'Year-Of-Publication']:
    df_fix['metadata'] += df_fix[col]
    df_fix['metadata'] += " "

# In[45]:

df_fix.tail()

# ## 4.4 Encoding dengan TF-IDF

# In[46]:

from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer

# In[47]:

bow = CountVectorizer()
bank = bow.fit_transform(df_fix.metadata)

# In[48]:

tfidf = TfidfVectorizer()
bank2 = tfidf.fit_transform(df_fix.metadata)

# # 5. Modeling

# ## 5.1 Encoding metadata index

# In[49]:

index = 0
content = df_fix.loc[index, 'metadata']
content

# In[50]:

code1 = bow.transform([content])
code1

# In[51]:

code2 = tfidf.transform([content])

# ## 5.2 Document Search

# In[52]:

from sklearn.metrics.pairwise import cosine_distances

# In[53]:

distance1 = cosine_distances(code1, bank)
distance1

# In[54]:

rec_idx = distance1.argsort()[0, 1:]
rec_idx

# In[55]:

df_fix.loc[rec_idx[:5], :]

# In[56]:

distance2 = cosine_distances(code2, bank)
distance2

# In[57]:

rec_idx2 = distance2.argsort()[0, 1:]
rec_idx2

# In[58]:

df_fix.loc[rec_idx2[:10], :]

# ## 5.3 Bungkus code agar rapih

# In[59]:

class RecommendationSystems():
    def __init__(self,data, metadata_col):
        self.df = data
        self.metadata = metadata_col
        self.encoder = None
        self.bank = None

    def fit(self, encoder='tfidf'):
        if encoder not in ['bow', 'tfidf']:
            print("hanya support BoW dan TF-IDF")
        else:
            self.encoder = TfidfVectorizer()
            if encoder=='bow':
                self.encoder = CountVectorizer()
            self.bank = self.encoder.fit_transform(self.df[self.metadata])
    def recommender(self, index, topn=10):
        content = self.df.loc[index, self.metadata]
        code = self.encoder.transform([content])
        distance = cosine_distances(code, self.bank)
        rec_idx = distance.argsort()[0, 1:]
        return self.df.loc[rec_idx[:topn], :]

# In[60]:

recsys = RecommendationSystems(df_fix, 'metadata')
recsys.fit()

# In[61]:

rec = recsys.recommender(100)
rec

# In[62]:

df_fix.loc[100,:]

# In[63]:

recsys2 = RecommendationSystems(df_fix, 'metadata')
recsys2.fit('bow')

# In[64]:

rec2 = recsys2.recommender(100)
rec2

# ## 5.4 Scoring

# ## 5.4.1 Try

# In[65]:

from sklearn.metrics.pairwise import cosine_similarity
import numpy as np

# In[66]:

book = pd.DataFrame(dict(df_fix.loc[100,:]), index=[0])
book

# In[67]:

rec2

# In[68]:

np.mean(cosine_similarity(bow.transform(book.metadata.values), bow.transform(rec.metadata)))

# In[69]:

np.mean(cosine_similarity(tfidf.transform(book.metadata.values), tfidf.transform(rec.metadata)))

# ## 5.4.2 Bungkus kembali kedalam class

# In[70]:

class RecommendationSystems():
    def __init__(self,data, metadata_col):
        self.df = data
        self.metadata = metadata_col
        self.encoder = None
        self.bank = None
        self.content = None
        self.index = None
        self.topn = None

    def fit(self, encoder='tfidf'):
        if encoder not in ['bow', 'tfidf']:
            print("hanya support BoW dan TF-IDF")
        else:
            self.encoder = TfidfVectorizer()
            if encoder=='bow':
                self.encoder = CountVectorizer()
            self.bank = self.encoder.fit_transform(self.df[self.metadata])
    def recommender(self, index, topn=10, include_book_search=False):
        self.index = index
        self.topn = topn
        self.content = self.df.loc[index, self.metadata]
        code = self.encoder.transform([self.content])
        distance = cosine_distances(code, self.bank)
        rec_idx = distance.argsort()[0, :]
        if include_book_search:
            topn+=1
            return self.df.loc[rec_idx[:topn], :]
        return self.df.loc[rec_idx[1:topn], :]

    def score(self):
        rec = self.recommender(index=self.index, topn=self.topn, include_book_search=True)
        bank = self.encoder.transform(rec.metadata)
        code = self.encoder.transform([self.content])
        return np.mean(cosine_similarity(code, bank)[:,1:])

# In[71]:

recsys3 = RecommendationSystems(df_fix, 'metadata')
recsys3.fit()
recsys3.recommender(index=100, include_book_search=True)

# In[72]:

recsys3.score()

# In[73]:

import time

# In[74]:

start = time.time()
recsys4 = RecommendationSystems(df_fix, 'metadata')
recsys4.fit('bow')
recsys4.recommender(index=100)
stop = time.time()
print(f"Training time: {stop - start} s")
print("Score : ",recsys4.score())

# ## 5.4.3 Scoring Model dengan *Bag Of Word* top 5 rekomendasi

# In[75]:

recsys = RecommendationSystems(data=df_fix, metadata_col='metadata')
recsys.fit(encoder='bow')

# In[76]:

recsys.recommender(index=100, topn=5, include_book_search=True)

# In[77]:

recsys.score()

# In[78]:

recsys.recommender(index=1000, topn=5, include_book_search=True)

# In[79]:

recsys.score()

# ## 5.4.4 Scoring Model Tf-idf

# In[80]:

recsys = RecommendationSystems(data=df_fix, metadata_col='metadata')
recsys.fit(encoder='tfidf')

# In[81]:

recsys.recommender(index=100, topn=5, include_book_search=True)

# In[82]:

recsys.score()

# In[83]:

recsys.recommender(index=1000, topn=5, include_book_search=True)

# In[84]:

recsys.score()

# # 6 Evaluasi

# ## 6.1 Generate data yang akan dijadikan sampel sebanyak 500 data

# In[85]:

np.random.randint(0,10, 5)

# In[86]:

idx_sample_list = np.random.randint(0,len(df_fix), 500)
idx_sample_list

# In[87]:

len(idx_sample_list)

# In[88]:

idx_sample_list2 = [i for i in np.random.randint(0,len(df_fix), 500) if i not in idx_sample_list]
len(idx_sample_list2)

# In[89]:

from tqdm import tqdm

# ## 6.2 Define Loop Function

# In[90]:

def loop_rec(data, metadata_col='metadata', encoder='tfidf', sample_list=[0,2,10],topn=10, return_dict=False):
    score_list = []
    score_dict = {}
    idx_done = []
    recsys = RecommendationSystems(data, metadata_col)
    recsys.fit(encoder)
    for i in tqdm(sample_list):
        while True:
            idx = np.random.randint(low=0, high=len(data))
            if idx not in idx_done:
                idx_done.append(idx)
                break
        recsys.recommender(index=idx, topn=topn)
        score = recsys.score()
        score_list.append(score)
        score_dict[idx] = score
    if return_dict:
        return score_dict, np.mean(score_list)
    return np.mean(score_list)

# ## 6.3 Model Bag-of-word

# In[91]:

dict_score_bow, score_bow = loop_rec(data=df_fix,
                                     metadata_col='metadata',
                                     encoder='bow',
                                     topn=5,
                                     sample_list=idx_sample_list,
                                     return_dict=True)

# In[92]:

score_bow

# In[93]:

dict_score_bow2, score_bow2 = loop_rec(data=df_fix,
                                     metadata_col='metadata',
                                     encoder='bow',
                                     topn=5,
                                     sample_list=idx_sample_list2,
                                     return_dict=True)

# In[94]:

score_bow2

# ## 6.4 Model Tf-idf

# In[95]:

dict_score_tfidf, score_tfidf = loop_rec(data=df_fix,
                                     metadata_col='metadata',
                                     encoder='tfidf',
                                     topn=5,
                                     sample_list=idx_sample_list,
                                     return_dict=True)

# In[96]:

score_tfidf

# In[97]:

dict_score_tfidf2, score_tfidf2 = loop_rec(data=df_fix,
                                     metadata_col='metadata',
                                     encoder='tfidf',
                                     topn=5,
                                     sample_list=idx_sample_list2,
                                     return_dict=True)

# In[98]:

score_tfidf2

# In[100]:

pd.DataFrame({
    'Tf-Idf':[score_tfidf, score_tfidf2],
    'BoW':[score_bow, score_bow2]
})

# In[ ]:


r"""
Elements of posets, lattices, semilattices, etc.
"""
#*****************************************************************************
#       Copyright (C) 2008 Peter Jipsen <jipsen@chapman.edu>,
#                          Franco Saliola <saliola@gmail.com>
#
#  Distributed under the terms of the GNU General Public License (GPL)
#
#    This code is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#    General Public License for more details.
#
#  The full text of the GPL is available at:
#
#                  http://www.gnu.org/licenses/
#*****************************************************************************
from sage.structure.element import Element
from sage.structure.element import have_same_parent

class PosetElement(Element):

    def __init__(self, poset, element, vertex):
        r"""
        Establishes the parent-child relationship between ``poset``
        and ``element``, where ``element`` is associated to the
        vertex ``vertex`` of the Hasse diagram of the poset.

        INPUT:

        - ``poset`` - a poset object

        - ``element`` - any object

        - ``vertex`` - a vertex of the Hasse diagram of the poset

        TESTS::

            sage: from sage.combinat.posets.elements import PosetElement
            sage: P = Poset([[1,2],[4],[3],[4],[]], facade = False)
            sage: e = P(0)
            sage: e.parent() is P
            True
            sage: TestSuite(e).run()
        """
        Element.__init__(self, poset)
        if isinstance(element, self.parent().element_class):
            self.element = element.element
        else:
            self.element = element
        self.vertex = vertex

    def __hash__(self):
        r"""
        TESTS::

            sage: P = Poset([[1,2],[4],[3],[4],[]], facade = False)
            sage: e = P(0)
            sage: hash(e)
            0
        """
        return hash(self.element)

    def _repr_(self):
        """
        TESTS::

            sage: Poset([[1,2],[4],[3],[4],[]], facade = False)(0)._repr_()
            '0'
        """
        return "%s" %str(self.element)

    def _latex_(self):
        r"""
        Return the latex code of the poset element.

        EXAMPLES::

            sage: m = matrix(2,[1,2,3,4])
            sage: m.set_immutable()
            sage: P = Poset(([m],[]), facade = False)
            sage: [e] = P
            sage: type(e)
            <class 'sage.combinat.posets.elements.FinitePoset_with_category.element_class'>
            sage: latex(e)                 #indirect doctest
            \left(\begin{array}{rr}
            1 & 2 \\
            3 & 4
            \end{array}\right)
        """
        from sage.misc.latex import latex
        return latex(self.element)

    def __eq__(self,other):
        """
        TESTS::

            sage: P = Poset([["a","b"],["d"],["c"],["d"],[]], facade = False)
            sage: Q = Poset([["a","b"],["d"],["c"],[],[]], facade = False)
            sage: P(0).__eq__(P(4))
            False
            sage: from sage.combinat.posets.elements import PosetElement
            sage: PosetElement(P,0,"c") == PosetElement(P,0,"c")
            True
            sage: PosetElement(P,0,"c") == PosetElement(Q,0,"c")
            False
            sage: PosetElement(P,0,"b") == PosetElement(P,0,"c")
            False

        .. warning:: as an optimization, this only compares the parent
           and vertex, using the invariant that, in a proper poset
           element, ``self.element == other.element`` if and only
           ``self.vertex == other.vertex``::

            sage: PosetElement(P,1,"c") == PosetElement(P,0,"c")
            True

        Test that :trac:`12351` is fixed::

            sage: P(0) == int(0)
            False
        """
        # This should instead exploit unique representation, using
        # self is other, or best inherit __eq__ from there. But there
        # are issues around pickling and rich comparison functions.
        return have_same_parent(self, other) \
            and self.vertex == other.vertex

    def __ne__(self,other):
        r"""
        TESTS::

            sage: P = Poset([[1,2],[4],[3],[4],[]])
            sage: P = Poset([["a","b"],["d"],["c"],["d"],[]])
            sage: P(0).__ne__(P(4))
            True
            sage: from sage.combinat.posets.elements import PosetElement
            sage: PosetElement(P,0,"c") != PosetElement(P,0,"c")
            False
            sage: PosetElement(P,0,"b") != PosetElement(P,0,"c")
            True

        For this one, see comment in :meth:`__eq__`::

            sage: PosetElement(P,1,"c") != PosetElement(P,0,"c")
            False
        """
        return not self == other

    def _cmp(self,other):
        """
        TESTS::

            sage: P = Poset([[1,2],[4],[3],[4],[]], facade = False)
            sage: P(0)._cmp(P(4))
            -1
            sage: P(4)._cmp(P(0))
            1
            sage: P(0)._cmp(P(0))
            0
            sage: P(1)._cmp(P(2))

        """
        return self.parent().compare_elements(self,other)

    def __cmp__(self, other):
        r"""
        A default comparison of ``self`` with ``other``.

        .. note::

           The rich comparison methods have been implemented for poset
           elements, so when a user asks for ``x < y``, for example, rich
           comparison is used (that is, ``x.__lt__(y)`` is returned). This
           method is implemented because ``PosetElement`` inherits from
           ``Element``, which requires ``__cmp__`` to enable sorting by the
           ``cmp`` method.

           If both ``self`` and ``other`` have the same parent poset,
           then the comparison is done in the poset. If the elements
           are incomparable in the poset, then 0 is returned. Note that,
           in particular, ``cmp(a,b) == cmp(b,a)`` if ``a`` and ``b`` are
           equal or incomparable in the poset.

        TESTS::

            sage: P = Poset([[1,2],[4],[3],[4],[]], facade = False)
            sage: P(0).__cmp__(P(4))
            -1
            sage: P(4).__cmp__(P(0))
            1
            sage: P(0).__cmp__(P(0))
            0
            sage: P(1).__cmp__(P(2))
            0
            sage: cmp(P(0),P(4))
            -1
            sage: cmp(P(4),P(0))
            1
            sage: cmp(P(0),P(0))
            0
            sage: cmp(P(1),P(2))
            0
            sage: cmp(P(2),P(1))
            0
        """
        if isinstance(other, type(self)):
            r = self.parent().compare_elements(self,other)
            if r is None:
                return 0
            else:
                return r
        else:
            return cmp(type(other), type(self))

    def __lt__(self,other):
        """
        TESTS

        ::

            sage: dag = DiGraph({0:[2,3], 1:[3,4], 2:[5], 3:[5], 4:[5]})
            sage: P = Poset(dag, facade = False)
            sage: P(0) < P(1)
            False
            sage: P(4) < P(1)
            False
            sage: P(0) < P(0)
            False
        """
        return self._cmp(other) == -1 or False

    def __le__(self,other):
        """
        TESTS

        ::

            sage: dag = DiGraph({0:[2,3], 1:[3,4], 2:[5], 3:[5], 4:[5]})
            sage: P = Poset(dag, facade = False)
            sage: P(1) <= P(0)
            False
            sage: P(0) <= P(1)
            False
            sage: P(0) <= P(3)
            True
            sage: P(0) <= P(0)
            True
        """
        return self == other or self._cmp(other) == -1 or False

    def __gt__(self,other):
        """
        TESTS

        ::

            sage: dag = DiGraph({0:[2,3], 1:[3,4], 2:[5], 3:[5], 4:[5]})
            sage: P = Poset(dag)
            sage: P(0).__gt__(P(5))
            False
            sage: P(5).__gt__(P(0))
            True
            sage: P(0).__gt__(P(0))
            False
        """
        return self._cmp(other) == 1 or False

    def __ge__(self,other):
        """
        TESTS

        ::

            sage: dag = DiGraph({0:[2,3], 1:[3,4], 2:[5], 3:[5], 4:[5]})
            sage: P = Poset(dag)
            sage: P(0).__ge__(P(5))
            False
            sage: P(5).__ge__(P(0))
            True
            sage: P(0).__ge__(P(0))
            True
        """
        return self == other or self._cmp(other) == 1 or False

class MeetSemilatticeElement(PosetElement):
    def __mul__(self,other):
        r"""
        Return the meet of ``self`` and ``other`` in the lattice.

        EXAMPLES::

            sage: D = Posets.DiamondPoset(5,facade=False)
            sage: D(1) * D(2)
            0
            sage: D(1) * D(1)
            1
            sage: D(1) * D(0)
            0
            sage: D(1) * D(4)
            1
        """
        return self.parent().meet(self,other)

class JoinSemilatticeElement(PosetElement):
    def __add__(self,other):
        r"""
        Return the join of ``self`` and ``other`` in the lattice.

        EXAMPLES::

            sage: D = Posets.DiamondPoset(5,facade=False)
            sage: D(1) + D(2)
            4
            sage: D(1) + D(1)
            1
            sage: D(1) + D(4)
            4
            sage: D(1) + D(0)
            1
        """
        return self.parent().join(self,other)

class LatticePosetElement(MeetSemilatticeElement,JoinSemilatticeElement):
    pass


#!/usr/bin/env python
# coding: utf-8

# # Classifying Pulsars from the High Time Resolution Universe Survey (HTRU2) - Exploratory Data Analysis

# ## Overview & Citation

# The purpose of this project is to analyze telescope data to correctly separate actual pulsar data from radio frequency noise. The dataset was retrieved from the UC Irvine Machine Learning Repository at the following link: <https://archive.ics.uci.edu/ml/datasets/HTRU2#>
#
# The data comes from the High Time Resolution Universe Survey (South), known as HTRU2. The dataset was donated to the UCI Repository by Dr. Robert Lyon of The University of Manchester, United Kingdom. The two papers requested for citation in the description are listed below:
# * R. J. Lyon, B. W. Stappers, S. Cooper, J. M. Brooke, J. D. Knowles, Fifty Years of Pulsar Candidate Selection: From simple filters to a new principled real-time classification approach, Monthly Notices of the Royal Astronomical Society 459 (1), 1104-1123, DOI: 10.1093/mnras/stw656
# * R. J. Lyon, HTRU2, DOI: 10.6084/m9.figshare.3080389.v1.
#
# Below is an edited description of the dataset, taken from the website. The citation numbers refer to the publications listed on the website:
#
# *HTRU2 is a data set which describes a sample of pulsar candidates collected during the High Time Resolution Universe Survey (South) [1]. Pulsars are a rare type of Neutron star that produce radio emission detectable here on Earth. They are of considerable scientific interest as probes of space-time, the inter-stellar medium, and states of matter (see [2] for more uses).*
#
# *As pulsars rotate, their emission beam sweeps across the sky, and when this crosses our line of sight, produces a detectable pattern of broadband radio emission. As pulsars
# rotate rapidly, this pattern repeats periodically. Thus pulsar search involves looking for periodic radio signals with large radio telescopes. Each pulsar produces a slightly different emission pattern, which varies slightly with each rotation (see [2] for an introduction to pulsar astrophysics to find out why). Thus a potential signal detection known as a 'candidate', is averaged over many rotations of the pulsar, as determined by the length of an observation. In the absence of additional info, each candidate could potentially describe a real pulsar. However in practice almost all detections are caused by radio frequency interference (RFI) and noise, making legitimate signals hard to find.*
#
# *Machine learning tools are now being used to automatically label pulsar candidates to facilitate rapid analysis. Classification systems in particular are being widely adopted,
# (see [4,5,6,7,8,9]) which treat the candidate data sets as binary classification problems. Here the legitimate pulsar examples are a minority positive class, and spurious examples the majority negative class. At present multi-class labels are unavailable, given the costs associated with data annotation. The data set shared here contains 16,259 spurious examples caused by RFI/noise, and 1,639 real pulsar examples. These examples have all been checked by human annotators.*

# ## Feature Names

# The dataset was downloaded from the UC Irvine Repository as a CSV file. Column names were not provided in the original dataset. The features are listed below according to the feature names listed on the website, with the annotated column names used in this project listed in parentheses:
#
# 1. Mean of the integrated profile. (IP_Mean)
# 2. Standard deviation of the integrated profile. (IP_StdDev)
# 3. Excess kurtosis of the integrated profile. (IP_Kurtosis)
# 4. Skewness of the integrated profile. (IP_Skewness)
# 5. Mean of the DM-SNR curve. (DM_Mean)
# 6. Standard deviation of the DM-SNR curve. (DM_StdDev)
# 7. Excess kurtosis of the DM-SNR curve. (DM_Kurtosis)
# 8. Skewness of the DM-SNR curve. (DM_Skewness)
# 9. Class (Class)
#
# The XLSX and CSV files with the added column names are provided in the GitHub repository. We will import and explore the data in the following sections.

# ## Exploratory Data Analysis

# ### Import the Relevant Libraries

# In[1]:

import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
get_ipython().run_line_magic('matplotlib', 'inline')
sns.set()

# ### Import & Check the Data

# In[3]:

df = pd.read_csv('2020_1125_Pulsar_Data.csv')
pulsar_data = df.copy()

# In[4]:

pulsar_data.head()

# In[9]:

# Showing only data for input variables, since the statistics for binary variables in Class are meaningless
pulsar_data.drop('Class',axis=1,inplace=False).describe()

# ### Data Visualization

# The visualization below shows paired plots for every input and outut variable.

# In[10]:

sns.pairplot(pulsar_data)

# ## Modeling Recommendations

# This dataset poses a binary classification problem. All input values are numerical, and the output variable is categorical. Since there are no null values in the dataset, no additional preprocessing is requred prior to modeling. The following classification models should be explored and compared for performance:
# * Multiple Logistic Regression
# * Decision Tree Classification
# * Random Forest Classification
# * Support Vector Machines (SVM)
# * Deep Artificial Neural Networks (DNN)


# coding: utf-8

# # Analisi del problema di Quora
# ---
#
# Il problema è proposto da http://www.quora.com. Il sito propone un sistema per condividere le proprie conoscenze in vari ambiti e per espandere queste stesse conoscenze. Questo grazie ad un sistema di semplici domande e risposte, che sono però organizzate in vari topic (interessi) che le persone possono seguire per personalizzare la loro esperienza. E’ presente un profilo personale che mostra domande e risposte e le conoscenze principali di ogni persona.
# Il problema posto dall’azienda è il riconoscimento di domande che hanno lo stesso significato: sono molte infatti le domande che vengono ripetute (per pigrizia degli utenti o anche a causa della ricerca). Diventerebbe quindi importante ottimizzare il numero di domande unendo quelle uguali (con stesso scopo), ma anche (e soprattutto) unire le risposte, per rendere più completa possibile l’esperienza dell’utente riguardo lo specifico argomento della domanda.
# Attualmente il sistema utilizza un modello RandomForest per identificare le domande uguali: la sfida lanciata è quella di trovare degli algoritmi migliori di rilevamento usando tecniche avanzate di Machine Learning e di Natural Language Processing.
#
# Il problema si presenta come una classificazione binaria con l'uso di addestramento supervisionato
#
# # Analisi  Training Set
# ---
#
# Abbiamo iniziato ad analizzare il dataset partendo dal Training Set (`train.csv`).
# L'header indica la denominazione delle 5 features e del label:
# 1. `id` = id della coppia di domande
# 2. `qid1` = id della prima domanda
# 3. `qid2` = id della seconda domanda
# 4. `question1` = testo della prima domanda
# 5. `question2` = testo della seconda domanda
# 6. `is_duplicate` = indica se le due domande sono uguali (è il label)

# In[1]:

get_ipython().run_cell_magic('time', '', "\nimport pandas as pd\nimport numpy as np\nimport time\n\n#Read dataset\ntrain = pd.read_csv('/usr/local/share/kaggle/Quora Question Pairs/uncompressed/train.csv')\n\n#Print dataframe columns types infos\nprint(train.dtypes)\n\n#Print some examples\ntrain.head()\nprint(len(train))")

# # Elaborazione dei dati
# ---
#
# Inizialmente, seguendo i suggerimenti trovati in alcuni notebook presenti su kaggle, si era deciso di ripulire il dataset, eliminando tutto ciò che non fosse lettere o numeri.
# Questa decisione si è rivelata poi controproducente in quanto l'algoritmo risulta meno performante su di un dataset "pulito".
#
# Si noti la differenza tra una frase "sporca" e una "pulita"

# ### FASE 0A: Creazione dizionari
#
# Inizialmente è utile costruire delle strutture dati che poi saranno impiegate nel corso dell'elaborazione dei dati; anche se esse vengono utilizzate in fasi diverse dell'elaborazione risulta utile crearle tutte insime in un unica operazione.
# Le strutture dati sono le seguenti
#
# * `dictionary`: Dizionario che associa a ciascuna parola un id intero
# * `f_list`: lista che associa a ciascuna parola la propria frequenza, nello specifico contiene coppie (`id_parola`, `frequenza_parola`); la lista è ordinata in ordine decrescente di frequenza
# * `n_data`: dataset formato da due colonne contenente le coppie di domande, dove ciascuna domanda è rappresentata da un array di `id_parola`
# * `id_train`: `n_data` con l'aggiunta di una prima colonna contenente l'id della coppia di domande
#

# In[2]:

get_ipython().run_cell_magic('time', '', "\ndef build_dictionary(data):\n    #Initialization\n    dictionary = {} #dizionario: stringa -> id\n    c = 0\n    frequency_list = [] #lista frequenze: list[id] = (id_parola, frequenza_parola)\n    converted_data = [] #dataset che contiene numeri al posto delle parole \n    #Takes every word in every couple and creates a dictionary\n    for col in ['question1', 'question2']:\n        new_col = []\n        for sentence in data[col]:\n            sentence_converted = []\n            sentence = str(sentence).split(' ')\n            for p in sentence:\n                #Checks if word is already in dictionary\n                if str(p) not in dictionary.keys():\n                    #Checks length of word and if it's a space, then add\n                    if not (len(str(p)) == 1 and ord(str(p)) == 32):\n                        dictionary[str(p)] = c\n                        frequency_list.append((c,1))\n                        sentence_converted.append(c)\n                        c = c+1\n\n                else:\n                    c_old = dictionary[str(p)]\n                    (n, freq) = frequency_list[c_old]\n                    frequency_list[c_old] = (n, freq+1)\n                    sentence_converted.append(c_old)\n\n            new_col.append(sentence_converted)\n\n        converted_data.append(new_col)            \n\n    sorted_frequency_list = sorted(frequency_list, key=lambda pair: pair[1], reverse=True)                \n\n    return dictionary, sorted_frequency_list, converted_data\n\ndictionary, f_list, n_data = build_dictionary(train)\n\nid_train = pd.DataFrame({\n        'id': train['id'],\n        'question1': n_data[0],\n        'question2': n_data[1]\n        })")

# ### Fase 0B: Inizializzazione rete di test
# ---
#
# La rete prescelta per la soluzione di questo problema è `MLPClassifier` presente all'interno del package `sklearn`.
# Come funzione di attivazione è stata scelta `logistic` in quanto la probabilità che le due domande siano un duplicato è rappresentata da un numero reale compreso tra 0 e 1 (come da istruzioni presenti sul sito)
# Come algoritmo solver è stato scelto `adam` perchè, dopo aver effettuato dei test di confronto con gli altri algoritmi, è risultato essere quello che dava il risultato migliore
# Come metrica di valutazione è stata scelta la funzione `LogLoss` in quanto è la stessa utilizzata da Kaggle
#
# In questa fase seleziono anche il numero di elementi da usare come insieme di train e quanti invece come insieme di test

# In[3]:

from sklearn.neural_network import MLPClassifier
from sklearn.metrics import log_loss

from keras.models import Sequential
from keras.layers import Dropout, Dense, BatchNormalization
from keras.losses import binary_crossentropy
import keras.utils as kUtils

#labels
duplicate = train['is_duplicate']

#initialize algorithm
alg = MLPClassifier(solver='adam', activation='logistic')

# split dataset train/test
split = np.random.rand(len(train)) < 0.8

#funzione di test
def test(features, k=0):

    features_train = features[split]
    features_test = features[~split]

    labels_train = duplicate[split]
    labels_test = duplicate[~split]

    alg.fit(features_train, labels_train)
    predict = alg.predict_proba(features_test)
    logLoss = log_loss(labels_test, predict)

    print('Sklearn Log Loss: {}'.format(logLoss))

    plot(predict[:,1], labels_test)

    ###KERAS NETWORK###
    features_train = np.asmatrix(features_train)
    features_test = np.asmatrix(features_test)

    labels_train = np.asmatrix(labels_train)
    labels_test = np.asmatrix(labels_test)

    labels_train = kUtils.to_categorical(labels_train, 2)
    labels_test = kUtils.to_categorical(labels_test, 2)

    model = Sequential()
    model.add(Dense(200, activation='relu', input_dim = features_train.shape[1]))
    model.add(Dropout(0.1))
    model.add(BatchNormalization())
    model.add(Dense(100, activation='relu'))
    model.add(Dropout(0.1))
    model.add(BatchNormalization())
    model.add(Dense(2, activation='sigmoid'))

    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])

    model.fit(features_train, labels_train,
          batch_size=128,
          epochs=20,
          verbose=0,
          validation_data=(features_test, labels_test))

    score = model.evaluate(features_test, labels_test, verbose=0)
    keras_predict = model.predict_proba(features_test)

    print('Keras Log loss: {}'.format(score[0]))
    #plot(keras_predict[:,1], labels_test)

    ####################

    if (k == 1):
        return predict[:,1]

import matplotlib.pyplot as plt

#funzione per mostrare la distribuzione delle predizioni rispetto alle label di test
def plot(p, labels_test):

    t = []
    f = []

    for p, l in zip(p, labels_test):
        if l == True:
            t.append(p)
        else:
            f.append(p)

    plt.hist(f, bins=20, normed=True, label='Not Duplicate')
    plt.hist(t, bins=20, normed=True, alpha=0.7, label='Duplicate')
    plt.show()

# ### FASE 0C: Multithreading
# ---
#
# Scrivo una funzione che suddivida il lavoro su 4 core in modo da velocizzare l'esecuzione dell'algoritmo

# In[4]:

from threading import Thread
import numpy as np

def split_work(fun, train):
    dataframes = np.array_split(train, 4)

    threads = []
    results = {}
    i = 0

    for data in dataframes:
        t = Thread(target=fun, args=(data, results, i))
        threads.append(t)
        t.start()
        i = i+1

    for t in threads:
        t.join()

    features = []
    for key in sorted(results):
        features.extend(results[key])

    return features

# ### FASE 0D: Salvataggio e caricamento delle strutture dati
# ---

# In[5]:

def save(s, o):
    obj = np.asarray(o)
    np.save(s, obj)

def load(s):
    return np.load(s)

def compute(fun, data, file):
    if not os.path.exists(file):
        res = split_work(fun, data)
        save(file, res)
    else:
        res = load(file)

    return res

# ### FASE 1: Estrazione Features di base
# ---
#
# Come prima cosa risulta utile estrarre le feature più semplici che possono caratterizzare una frase:
#
# * `q1_words`: numero di parole contenute nella prima domanda
# * `q1_letters`: numero di caratteri contenuti nella prima domanda
# * `q2_words`: numero di parole contenute nella seconda domanda
# * `q2_letters`: numero di lettere contenute nella seconda domanda
#
# Infatti risulta piuttosto intuitivo che frasi di simile lunghezza hanno più probabilità di avere lo stesso significato

# In[6]:

get_ipython().run_cell_magic('time', '', "import os\n\ndef base_extraction(data):\n\n    if not os.path.exists('base_features.csv'):\n        #Count number of words and letters for each 'question1'\n        q1_words = data['question1'].apply(lambda x: len(str(x).split(' ')))\n        q1_letters = data['question1'].apply(lambda x: len(str(x)))\n\n        #Count number of words and letters for each 'question2'\n        q2_words = data['question2'].apply(lambda x: len(str(x).split(' ')))\n        q2_letters = data['question2'].apply(lambda x: len(str(x)))\n\n        features = pd.DataFrame({\n                    'q1_words': q1_words,\n                    'q1_letters': q1_letters,\n                    'q2_words': q2_words,\n                    'q2_letters': q2_letters,\n                    })\n        \n        features.to_csv('base_features.csv', index=False, header=list(features))\n    else:\n        features = pd.read_csv('base_features.csv')\n    \n    return features\n\nfeatures = base_extraction(train)\n\n#TEST\n\n#test(features)")

# ### Fase 2: LCS
# ---
#
# Dovendo stabilire la similarità tra due frasi un buon indicatore può essere la presenza di sottosequenze comuni, ho quindi deciso di prendere come parametro la lunghezza della più lunga sottostringa comune tra le due domande.

# In[7]:

get_ipython().run_cell_magic('time', '', "\ndef LCSLength(data, results, k):\n    res =[]\n    \n    for row in data.itertuples():\n        X = getattr(row, 'question1')\n        Y = getattr(row, 'question2')\n        m = len(X)\n        n = len(Y)\n        C = np.zeros((m,n), dtype=np.int8);\n        for i in range(m):\n            for j in range(n):\n                if X[i] == Y[j]:\n                    C[i,j] = C[i-1,j-1] + 1\n                else:\n                    C[i,j] = max(C[i,j-1], C[i-1,j])    \n        res.append(C[m-1,n-1])\n    results[k] = res\n\nLCSs = compute(LCSLength, id_train, 'LCS.npy')\nfeatures = features.assign(**{'LCS' : LCSs})\n\n#TEST\n\n#test(features)")

# ### Fase 3: Word Similar Share
# ---
#
# Appoggiandomi alla libreria `Spacy`, libreria specifica per il Natural Language Processing, ho calcolato un indice di similarità tra le due domande come somma degli indici di similarità tra ciascuna coppia di parole (`w1`,`w2`) dove `w1` appartiene alla prima domanda e `w2` alla seconda
#
# Inizialmente avevo normalizzato l'indice rispetto alla lunghezza delle due domande ma la predizione risultava più precisa senza normalizzazione

# In[8]:

import spacy
nlp = spacy.load('en_core_web_md')

# In[9]:

def word_similar_share(data, results, i):
    res = []
    for row in data.itertuples():
        q1 = getattr(row, 'question1')
        q2 = getattr(row, 'question2')

        q1words = nlp(str(q1))
        q2words = nlp(str(q2))

        jac = 0

        for w1 in q1words:
            s_w1 = str(w1)
            if not (len(s_w1) == 1 and ord(s_w1) == 32):
                for w2 in q2words:
                    s_w2 = str(w2)
                    if not (len(s_w2) == 1 and ord(s_w2) == 32):
                        jac = jac + w1.similarity(w2)

        res.append(jac)
    results[i] = res
############################################

#word_match = split_work(word_similar_share, train)
#features_tmp = features.assign(**{'Word_Match' : word_match})

#TEST

#test(features_tmp)

# ### Fase 4: Common/Rare Words
# ---
#
# In miglioramento dell'algoritmo visto alla fase 3 è l'introduzione del concetto di parola comune e parola rara
# Una parola è considerata rara se la sua frequenza è bassa, e similmente è considerata comune se la sua frequenza è alta
#
# Si ottengono quindi due algoritmi distinti, entrambi ignorano le parole comuni
# * `word_match_rare_function`: prendendo in considerazione le parole rare si ottiene un indice di similarità che viene incrementato di 1 per ogni parola rara che le due domande hanno in comune e decrementato di 1 per ogni parola rara che compare in una sola delle due domande
# * `word_similar_share_cr`: come il `word_similar_share` visto al punto 3
#
# Anche in questo caso normalizzare gli indici ne riduce la capacità predittiva

# In[10]:

def word_match_rare_function(data, results, i):
    res = []
    for row in data.itertuples():
        q1words = getattr(row, 'question1')
        q2words = getattr(row, 'question2')

        rare_jac = 0
        #n_rare = 0
        q1_rares = set()
        q2_rares = set()

        for w1 in q1words:
            if w1 in rare_words:
                q1_rares.add(w1)

        for w2 in q2words:
            if w2 in rare_words:
                q2_rares.add(w2)

        for w1r in q1_rares:
            if w1r in q2_rares:
                rare_jac = rare_jac + 1
            else:
                rare_jac = rare_jac - 1

        for w2r in q2_rares:
            if not w2r in q1_rares:
                rare_jac = rare_jac - 1
        '''
        try:
            rare_jac = rare_jac/n_rare
        except:
            rare_jac = 0
        '''
        res.append(rare_jac)
    results[i] = res

def word_similar_share_cr(data, results, i):
    res = []
    for row in data.itertuples():
        q1 = getattr(row, 'question1')
        q2 = getattr(row, 'question2')

        q1words = nlp(str(q1))
        q2words = nlp(str(q2))

        jac = 0

        for w1 in q1words:
            s_w1 = str(w1)
            if not (len(s_w1) == 1 and ord(s_w1) == 32 and s_w1 not in common_words):
                for w2 in q2words:
                    s_w2 = str(w2)
                    if not (len(s_w2) == 1 and ord(s_w2) == 32 and s_w2 not in common_words):
                        jac = jac + w1.similarity(w2)
                try:
                    jac = jac/(len(q1words)+len(q2words))
                except:
                    jac = 0
        res.append(jac)
    results[i] = res

common_words = set()
rare_words = set()

#build rare words set
for (p,f) in f_list:
    if f<=20:
        rare_words.add(p)

#Build common words set
for (p,f) in f_list[:10]:
    common_words.add(p)

#Find similarity with Jaccard for each question pair
#word_match_cr = split_work(word_similar_share_cr, train)
#word_match_rare = split_work(word_match_rare_function, id_train)

#features_tmp = features.assign(**{'Word_Match_cr' : word_match_cr})
#features_tmp = features.assign(**{'Word_Match_rare' : word_match_rare})

#TEST

#test(features_tmp)

# ### Fase 4B: vari valori per Common e Rare
# ---
#
# I valori di frequenza per le parole comuni e rare al punto 4 erano stati scelti arbitrariamente, quindi in questa fase si testeranno vari valori per tali frequenze per determinare le migliori

# In[11]:

# start = time.time()

# for F in range(20, 110, 10):
#     for n in range(4, 11):
#         data_tmp = features.copy()
#         C = pow(2,n)

#         common_words = set()
#         rare_words = set()

#         #build rare words list
#         for (p,f) in f_list:
#             if f<=F:
#                 rare_words.add(p)

#         #Build common words list
#         for (p,f) in f_list[:C]:
#             common_words.add(p)

#         #Find similarity with Jaccard for each question pair
#         word_match_cr = train.apply(word_similar_share_cr, axis=1, raw=True)
#         word_match_rare = id_train.apply(word_match_rare_function, axis=1, raw=True)

#         data_tmp = data_tmp.assign(**{'Word_Match_cr' : word_match_cr})
#         data_tmp = data_tmp.assign(**{'Word_Match_rare' : word_match_rare})

#         #TEST

#         features_train = features_tmp[split]
#         features_test = features_tmp[~split]

#         labels_train = duplicate[split]
#         labels_test = duplicate[~split]

#         alg.fit(features_train, labels_train)
#         predict = alg.predict_proba(features_test)
#         logLoss = log_loss(labels_test, predict)

#         results = []
#         results.append((C,F,logLoss))
#         print("Log loss C: {}, F: {} = {}".format(C,F,logLoss))

# sorted_results = sorted(results, key=lambda pair: pair[2])
# print(sorted_results[:5])
# end = time.time()
# print('time: {}'.format(end - start))

# ### Fase 4C: deduzioni
# ---
#
# Dall'esecuzione della fase 5 si deduce che i parametri che danno il risultato migliore sono `C = 64 ` e `F = 50`. Non è detto che si rivelino buoni parametri anche per il set di test ma dato che comunque l'algoritmo basato sulle parole rare si ha dimostrato di dare buoni risultati terrò questi parametri validi anche per il caso di test.
# In ogni caso il valore dei parametri incide poco sul risultato finale, in quanto il valore migliore risulta essere `0.4996397287043904` mentre il peggiore è `0.5068634596789862` per una differenza di `0.00722373097`

# In[12]:

get_ipython().run_cell_magic('time', '', "\nF = 50\nC = 64\n\ndef word_match_rare_function(data, results, i):\n    res = []\n    for row in data.itertuples():   \n        q1words = getattr(row, 'question1')\n        q2words = getattr(row, 'question2')\n\n        rare_jac = 0\n        #n_rare = 0\n        q1_rares = set()\n        q2_rares = set()\n\n        for w1 in q1words:\n            if w1 in rare_words:\n                q1_rares.add(w1)\n\n        for w2 in q2words:\n            if w2 in rare_words:\n                q2_rares.add(w2)\n\n        for w1r in q1_rares:\n            if w1r in q2_rares:\n                rare_jac = rare_jac + 1\n            else:\n                rare_jac = rare_jac - 1\n\n        for w2r in q2_rares:\n            if not w2r in q1_rares:\n                rare_jac = rare_jac - 1\n        '''                \n        try:\n            rare_jac = rare_jac/n_rare\n        except:\n            rare_jac = 0                            \n        '''\n        res.append(rare_jac)\n    results[i] = res\n\ndef word_similar_share_cr(data, results, i):\n    res = []\n    for row in data.itertuples():   \n        q1 = getattr(row, 'question1')\n        q2 = getattr(row, 'question2')\n    \n        q1words = nlp(str(q1))\n        q2words = nlp(str(q2))\n\n        jac = 0\n\n        for w1 in q1words:\n            s_w1 = str(w1)\n            if not (len(s_w1) == 1 and ord(s_w1) == 32 and s_w1 not in common_words):\n                for w2 in q2words:\n                    s_w2 = str(w2)\n                    if not (len(s_w2) == 1 and ord(s_w2) == 32 and s_w2 not in common_words):\n                        jac = jac + w1.similarity(w2)\n                '''\n                try:\n                    jac = jac/(len(q1words)+len(q2words))\n                except:\n                    jac = 0\n                '''    \n        res.append(jac)\n    results[i] = res\n\ncommon_words = set()\nrare_words = set()\n\n#build rare words set\nfor (p,f) in f_list:\n    if f<=F:\n        rare_words.add(p)\n\n#Build common words set\nfor (p,f) in f_list[:C]:\n    common_words.add(p)\n\n#Find similarity with Jaccard for each question pair\nword_match_cr = compute(word_similar_share_cr, train, 'word_match_cr.npy')\nword_match_rare = compute(word_match_rare_function, id_train, 'word_match_rare.npy')\n    \nfeatures = features.assign(**{'Word_Match_cr' : word_match_cr})\nfeatures = features.assign(**{'Word_Match_rare' : word_match_rare})\n\n#TEST\n\n#test(features)")

# ### Fase 5: Cosine Similarity
# ---
#
# Un altro utile parametro per la similarità tra frasi è la Cosine similarity: una tecnica euristica per la misurazione della similitudine tra due vettori effettuata calcolando il coseno tra di loro.
#
# Per poterla applicare è quindi necessario convertire ciascuna frase in una `Bag of words`, cioè un'array di dimensione pari al numero di parole distinte presenti all'interno del dataset. In questo array la posizione i-esima è settata a 1 se la parola comprare nella frase, 0 altrimenti.
# In pratica ciascuna frase è rappresentata come un vettore in uno spazio n-dimensionale (con n pari alla dimensione di ciascuna `Bag of words`)
# A questo punto si applica semplicemente la cosine similarity tra i due vettori ottenuti

# In[13]:

# start = time.time()

# #bag words size
# dim = len(dictionary.keys())
# print("bag words size: {}".format(dim))

# #converte ogni frase in una bag of words da usare per la cosine similarity
# def cosineConversion(sentence):
#     s = np.zeros(dim)
#     for w in sentence:
#         s[w] = 1
#     return s

# #simple wrap for cosineConversion
# def cosineConversion_wrap(data):
#     c1 = []
#     c2 = []
#     for sent1 in data['question1']:
#         s1 = cosineConversion(sent1)
#         c1.append(s1)

#     for sent2 in data['question2']:
#         s2 = cosineConversion(sent2)
#         c2.append(s2)

#     return c1, c2

# #calculate Cosine Similarity
# def cosineSimilarity(row):
#     a = row['question1']
#     b = row['question2']

#     cos_sim = dot(a, b)/(norm(a)*norm(b))

#     return cos_sim

# cosine1, cosine2 = cosineConversion_wrap(id_train)

# cosine_data = pd.DataFrame({
#             'id': train['id'],
#             'question1': cosine1,
#             'question2': cosine2
#             })

# mid = time.time()

# print('Cosine Similarity dataset: {}'.format(mid - start))

# cosine_similarity = cosine_data.apply(cosineSimilarity, axis=1, raw=True)

# features_tmp = features.copy().assign(**{'Cosine Similarity' : cosine_similarity})

# end = time.time()
# print('Cosine Similarity Algorithm: {}'.format(end - mid))

# #TEST

# test(features_tmp)

# Sfortunatamente questa prima versione dell'algoritmo occupa troppa memoria

# ### Fase 5B: cosine similarity migliorata
# ---
# Questa versione dell'algoritmo è più efficiente rispetto alla precedente e occupa molta meno memoria. Inoltre invece che far assumere a ciascun elemento della `Bag of words` i valori 0/1, `Sentence[w]` varrà  `n` cioè il numero di apparizioni della data parola all'interno della frase

# In[14]:

from numpy.linalg import norm

#bag words size
dim = len(dictionary.keys())

def cosineConversion(sentence):
    s = np.zeros(dim, dtype=np.int8)
    for w in sentence:
        s[w] = s[w] + 1
    return s

def cosineSimilarity(data, results, i):
    res = []
    for row in data.itertuples():
        q1 = getattr(row, 'question1')
        q2 = getattr(row, 'question2')

        a = cosineConversion(q1)
        b = cosineConversion(q2)

        cos_sim = np.dot(a, b)/(norm(a)*norm(b))

        res.append(cos_sim)
    results[i] = res

#cosine_similarity = split_work(cosineSimilarity, id_train)
#features_tmp = features.assign(**{'Cosine Similarity' : cosine_similarity})

#TEST

#test(features_tmp)

# ### Fase 5C: Cosine Similarity Sklearn
# ---
#
# La `Cosine Similarity` sembrava un buon parametro ma l'algoritmo precedentemente utilizzato non dava il risultato sperato, così mi sono appoggiato all'algoritmo presente all'interno del pacchetto `Sklearn`

# In[15]:

get_ipython().run_cell_magic('time', '', "from sklearn.feature_extraction.text import TfidfVectorizer as TF\nfrom sklearn.metrics.pairwise import cosine_similarity\n\ndef cosineSimilarity(data, results, i):\n    res = []\n    for row in data.itertuples():   \n        q1 = str(getattr(row, 'question1'))\n        q2 = str(getattr(row, 'question2'))\n\n        vect = TF(min_df=1)\n        tfidf = vect.fit_transform([q1, q2])\n\n        res.append((tfidf * tfidf.T).A[0,1])\n    results[i] = res\n    \ndef cosineSimilarity2(data):\n    vect = TF(min_df=1)\n    l1 = data['question1']\n    l2 = data['question2']\n    res = []\n    \n    vect.fit(data['question1'] + data['question2'])\n    for row in data.itertuples():   \n        q1 = str(getattr(row, 'question1'))\n        q2 = str(getattr(row, 'question2'))\n\n        tf1 = vect.transform(q1)\n        tf2 = vect.transform(q2)\n\n        cs = cosine_similarity(tf1,tf2)\n        res.append(cs)\n\n#cos_sim = compute(cosineSimilarity, train, 'cosine_similarity.npy')\ncos_sim = cosineSimilarity2(train)\nfeatures = features.assign(**{'cos_sim': cos_sim})\n\n#TEST\n#test(features)")

# La cosine similarity calcolata da Sklearn risulta essere migliore rispetto a quella calcolata precedente

# ### Fase 6: Spacy Sentence similarity
# ---
#
# La libreria Spacy prevede la possibilità di calcolare direttamente un indice di similarità tra frasi, questo si va ad aggiungere ai parametri precedenti

# In[16]:

get_ipython().run_cell_magic('time', '', "\ndef sentenceSimilarity(data, results, i):\n    res = []\n    for row in data.itertuples():   \n        q1s = getattr(row, 'question1')\n        q2s = getattr(row, 'question2')\n    \n        q1 = nlp(str(q1s))\n        q2 = nlp(str(q2s))\n\n        res.append(q1.similarity(q2))\n    results[i] = res\n\nspacy_sim = compute(sentenceSimilarity, train, 'spacy_similarity.npy')\nfeatures = features.assign(**{'spacy_sim': spacy_sim})     \n\n#TEST\n\n#test(features)\n\nfeatures_set = features")

# ## Recap
# ---
#
# I parametri finora individuati sono:
# * features di base
# * LCS
# * jaccard common/rare
# * cosine similarity (Sklearn)
# * Spacy sentence similarity
#
# essi risultano ottimi per l'individuazione di frasi non duplicate, risulta invece più difficile individuare i doppioni

# ## GridSearch

# In[18]:

from sklearn.model_selection import GridSearchCV
from sklearn.metrics import make_scorer
'''
X_train = features[split]
X_test = features[~split]

y_train = duplicate[split]
y_test = duplicate[~split]

possible_parameters = {
    'activation': ['identity', 'logistic', 'tanh', 'relu'],
    'solver': ['lbfgs', 'sgd', 'adam']
}

alg = MLPClassifier()
loss_f = make_scorer(log_loss, greater_is_better=False, needs_proba=True)

# The GridSearchCV is itself a classifier
# we fit the GridSearchCV with the training data
# and then we use it to predict on the test set
clf = GridSearchCV(alg, possible_parameters, n_jobs=4, scoring=loss_f) # n_jobs=4 means we parallelize the search over 4 threads
clf.fit(X_train, y_train)

y_pred = clf.predict_proba(X_test)
logLoss = log_loss(y_test, y_pred)

logLoss
'''

# In[ ]:

'''
cv = clf.cv_results_
tab = pd.DataFrame({
    'mean_fit_time': cv['mean_fit_time'],
    'mean_score_time': cv['mean_score_time'],
    'mean_test_score': cv['mean_test_score'],
    'mean_train_score': cv['mean_train_score'],
    'param_activation': cv['param_activation'],
    'param_solver': cv['param_solver'],
    'rank_test_score': cv['rank_test_score']
})

tab.sort_values(['rank_test_score']).head(3)
'''

# In[26]:

get_ipython().run_cell_magic('time', '', "\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.metrics import make_scorer\n\nalg = MLPClassifier(solver='adam', activation='logistic')\nloss_f = make_scorer(log_loss, greater_is_better=False, needs_proba=True)\n\nX_train = features[split]\nX_test = features[~split]\n\ny_train = duplicate[split]\ny_test = duplicate[~split]\n\npossible_parameters = {\n    'alpha': [0.0001, 0.00005],\n    'batch_size': [100, 50],\n    'learning_rate_init': [0.001, 0.0005],\n    'beta_1': [0.9, 0.99],\n    'beta_2': [0.999, 0.5],\n    'epsilon': [5e-9, 2e-9]\n}\n\nclf2 = GridSearchCV(alg, possible_parameters, n_jobs=4, scoring=loss_f) # n_jobs=4 means we parallelize the search over 4 threads\nclf2.fit(X_train, y_train)\n\ny_pred = clf2.predict_proba(X_test)\nlogLoss = log_loss(y_test, y_pred)\n\nlogLoss")

# In[28]:

cv = clf2.cv_results_
tab = pd.DataFrame({
    'mean_fit_time': cv['mean_fit_time'],
    'mean_score_time': cv['mean_score_time'],
    'mean_test_score': cv['mean_test_score'],
    'mean_train_score': cv['mean_train_score'],
    'param_alpha': cv['param_alpha'],
    'param_batch_size': cv['param_batch_size'],
    'param_learning_rate_init': cv['param_learning_rate_init'],
    'rank_test_score': cv['rank_test_score'],
    'param_beta_1': cv['param_beta_1'],
    'param_epsilon': cv['param_epsilon']
})

tab.sort_values(['rank_test_score']).head(3)

# ## Random Forest Classifier

# In[39]:

get_ipython().run_cell_magic('time', '', "from sklearn.ensemble import RandomForestClassifier\n\nrf = RandomForestClassifier(random_state=0, n_jobs=4)\n\npossible_parameters = {\n    'max_depth': [x for x in range(1, 20, 1)],\n    'criterion': ['gini', 'entropy'],\n    'max_features': ['auto', 'log2', None]\n}\n\nclf = GridSearchCV(rf, possible_parameters, n_jobs=4, scoring=loss_f)\n\nclf.fit(X_train, y_train)")

# In[42]:

y_pred = clf.predict_proba(X_test)
logLoss = log_loss(y_test, y_pred)

logLoss

# Max_depth = 2, gini: 0.5456

# In[41]:

cv_rf = clf.cv_results_
tab = pd.DataFrame({
    'mean_fit_time': cv_rf['mean_fit_time'],
    'mean_score_time': cv_rf['mean_score_time'],
    'mean_test_score': cv_rf['mean_test_score'],
    'mean_train_score': cv_rf['mean_train_score'],
    'param_max_depth': cv_rf['param_max_depth'],
    'param_criterion': cv_rf['param_criterion'],
    'param_max_features': cv_rf['param_max_features'],
    'rank_test_score': cv_rf['rank_test_score'],
})

tab.sort_values(['rank_test_score']).head(3)

# ## Normalize Data

# In[22]:

from sklearn.preprocessing import MinMaxScaler
import warnings
warnings.filterwarnings('ignore')

features_scaled = pd.DataFrame(columns = list(features))

letters_scaler = MinMaxScaler()
words_scaler = MinMaxScaler()

letters_scaler.fit(features['q1_letters'] + features['q2_letters'])
features_scaled['q1_letters'] = letters_scaler.transform(features['q1_letters'])
features_scaled['q2_letters'] = letters_scaler.transform(features['q2_letters'])

words_scaler.fit(features['q1_words'] + features['q2_words'])
features_scaled['q1_words'] = words_scaler.transform(features['q1_words'])
features_scaled['q2_words'] = words_scaler.transform(features['q2_words'])

for feat in  ['LCS', 'Word_Match_cr', 'Word_Match_rare', 'cos_sim', 'spacy_sim']:
    features_scaled[feat] = MinMaxScaler().fit_transform(features[feat])

# ### MLPClassifier

# In[24]:

from sklearn.model_selection import GridSearchCV
from sklearn.metrics import make_scorer

alg = MLPClassifier(solver='adam', activation='logistic')
loss_f = make_scorer(log_loss, greater_is_better=False, needs_proba=True)

X_train = features_scaled[split]
X_test = features_scaled[~split]

y_train = duplicate[split]
y_test = duplicate[~split]

possible_parameters = {
    'alpha': [0.0001, 0.00005],
    'batch_size': [100, 50],
    'learning_rate_init': [0.001, 0.0005],
    'beta_1': [0.9, 0.99],
    'beta_2': [0.999, 0.5],
    'epsilon': [5e-9, 2e-9]
}

mlp = GridSearchCV(alg, possible_parameters, n_jobs=4, scoring=loss_f) # n_jobs=4 means we parallelize the search over 4 threads
mlp.fit(X_train, y_train)

# In[25]:

cv = mlp.cv_results_
tab = pd.DataFrame({
    'mean_fit_time': cv['mean_fit_time'],
    'mean_score_time': cv['mean_score_time'],
    'mean_test_score': cv['mean_test_score'],
    'mean_train_score': cv['mean_train_score'],
    'param_alpha': cv['param_alpha'],
    'param_batch_size': cv['param_batch_size'],
    'param_learning_rate_init': cv['param_learning_rate_init'],
    'rank_test_score': cv['rank_test_score'],
    'param_beta_1': cv['param_beta_1'],
    'param_epsilon': cv['param_epsilon']
})

tab.sort_values(['rank_test_score']).head(3)

# ### Random Forest

# In[29]:

from sklearn.ensemble import RandomForestClassifier

rf = RandomForestClassifier(random_state=0, n_jobs=4)

possible_parameters = {
    'max_depth': [x for x in range(1, 20, 1)],
    'criterion': ['gini', 'entropy'],
    'max_features': ['auto', 'log2', None]
}

possible_parameters['max_depth'].append(None)

clf = GridSearchCV(rf, possible_parameters, n_jobs=4, scoring=loss_f)

clf.fit(X_train, y_train)

# In[30]:

cv_rf = clf.cv_results_
tab = pd.DataFrame({
    'mean_fit_time': cv_rf['mean_fit_time'],
    'mean_score_time': cv_rf['mean_score_time'],
    'mean_test_score': cv_rf['mean_test_score'],
    'mean_train_score': cv_rf['mean_train_score'],
    'param_max_depth': cv_rf['param_max_depth'],
    'param_criterion': cv_rf['param_criterion'],
    'param_max_features': cv_rf['param_max_features'],
    'rank_test_score': cv_rf['rank_test_score'],
})

tab.sort_values(['rank_test_score']).head(3)

# In[21]:

X_train = features[split]
X_test = features[~split]

y_train = duplicate[split]
y_test = duplicate[~split]

# ### Naive Bayes Gaussian

# In[33]:

from sklearn.naive_bayes import GaussianNB

gnb = GaussianNB()
gnb.fit(X_train, y_train)

y_pred = gnb.predict_proba(X_test)
logLoss = log_loss(y_test, y_pred)

logLoss

# ### Quadratic Discriminant

# In[34]:

from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis

qda = QuadraticDiscriminantAnalysis()
qda.fit(X_train, y_train)

y_pred = qda.predict_proba(X_test)
logLoss = log_loss(y_test, y_pred)

logLoss

# ### Support Vector Classifier

# In[ ]:

get_ipython().run_cell_magic('time', '', '\nfrom sklearn import svm\n\nsvc = svm.SVC()\nsvc.fit(X_train, y_train)\n\ny_pred = svc.predict_proba(X_test)\nlogLoss = log_loss(y_test, y_pred)\n\nlogLoss')

# ## FC Network

# In[17]:

get_ipython().run_cell_magic('time', '', "\nfeatures_train = features[split]\nfeatures_test = features[~split]\n\nlabels_train = duplicate[split]\nlabels_test = duplicate[~split]\n\nfeatures_train = np.asmatrix(features_train)\nfeatures_test = np.asmatrix(features_test)\n\nlabels_train = np.asmatrix(labels_train)\nlabels_test = np.asmatrix(labels_test)\n\nlabels_train = kUtils.to_categorical(labels_train, 2)\nlabels_test = kUtils.to_categorical(labels_test, 2)\n\n#architecture\nmodel = Sequential()\nmodel.add(Dense(300, activation= 'elu', input_dim = features_train.shape[1]))\n#model.add(Dropout(0.2))\nmodel.add(BatchNormalization())\nfor i in range(8):\n    model.add(Dense(200, activation= 'elu'))\n    #model.add(Dropout(0.2))\n    #model.add(BatchNormalization())\nmodel.add(Dense(2, activation= 'softmax'))\n\nmodel.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n\nhistory= model.fit(features_train, labels_train, batch_size=128, epochs=20, verbose=2, validation_data=(features_test, labels_test))\n\nscore = model.evaluate(features_test, labels_test, verbose=0)\n\nprint('Log loss: {}'.format(score[0]))\nprint('Accuracy: {}'.format(score[1]))")

# ### Add Dropout and Normalization

# In[19]:

get_ipython().run_cell_magic('time', '', "\nfeatures_train = features[split]\nfeatures_test = features[~split]\n\nlabels_train = duplicate[split]\nlabels_test = duplicate[~split]\n\nfeatures_train = np.asmatrix(features_train)\nfeatures_test = np.asmatrix(features_test)\n\nlabels_train = np.asmatrix(labels_train)\nlabels_test = np.asmatrix(labels_test)\n\nlabels_train = kUtils.to_categorical(labels_train, 2)\nlabels_test = kUtils.to_categorical(labels_test, 2)\n\n#architecture\nmodel = Sequential()\nmodel.add(Dense(300, activation= 'elu', input_dim = features_train.shape[1]))\nmodel.add(Dropout(0.2))\nmodel.add(BatchNormalization())\nfor i in range(8):\n    model.add(Dense(200, activation= 'elu'))\n    model.add(Dropout(0.2))\n    model.add(BatchNormalization())\nmodel.add(Dense(2, activation= 'softmax'))\n\nmodel.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n\nhistory= model.fit(features_train, labels_train, batch_size=128, epochs=20, verbose=2, validation_data=(features_test, labels_test))\n\nscore = model.evaluate(features_test, labels_test, verbose=0)\n\nprint('Log loss: {}'.format(score[0]))\nprint('Accuracy: {}'.format(score[1]))")

# ### Esperimenti

# In[ ]:

get_ipython().run_cell_magic('time', '', "\nfeatures_train = features[split]\nfeatures_test = features[~split]\n\nlabels_train = duplicate[split]\nlabels_test = duplicate[~split]\n\nfeatures_train = np.asmatrix(features_train)\nfeatures_test = np.asmatrix(features_test)\n\nlabels_train = np.asmatrix(labels_train)\nlabels_test = np.asmatrix(labels_test)\n\nlabels_train = kUtils.to_categorical(labels_train, 2)\nlabels_test = kUtils.to_categorical(labels_test, 2)\n\n#architecture\nmodel = Sequential()\nmodel.add(Dense(400, activation= 'elu', input_dim = features_train.shape[1]))\nfor i in range(13):\n    model.add(Dense(400, activation= 'elu'))\nmodel.add(Dense(2, activation= 'softmax'))\n\nmodel.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n\nhistory= model.fit(features_train, labels_train, batch_size=128, epochs=20, verbose=2, validation_data=(features_test, labels_test))\n\nscore = model.evaluate(features_test, labels_test, verbose=0)\n\nprint('Log loss: {}'.format(score[0]))\nprint('Accuracy: {}'.format(score[1]))")