Kaggle competition: 2019  ML competition with KISTI
data.isna().sum(axis=0 )<filter>
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.preprocessing import MinMaxScaler
from sklearn.preprocessing import StandardScaler
from sklearn.utils import shuffle
from sklearn.linear_model import LogisticRegression
from sklearn.linear_model import Perceptron
from sklearn import svm
from sklearn.ensemble import SVC
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import train_test_split
from sklearn import metrics
from sklearn.metrics import confusion_matrix
from sklearn.ensemble import VotingClassifier
from sklearn.ensemble import AdaBoostClassifier
from sklearn.neural_network import MLPClassifier
from sklearn.model_selection import KFold
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import cross_val_predict
from sklearn.model_selection import cross_validate
from sklearn.model_selection import GridSearchCV
from sklearn import svm, tree, linear_model, neighbors, naive_bayes, ensemble, discriminant_analysis, gaussian_process
from sklearn.preprocessing import OneHotEncoder, LabelEncoder
from sklearn import feature_selection
from sklearn import model_selection
from sklearn import metrics
import matplotlib as mpl
import matplotlib.pyplot as plt
import matplotlib.pylab as pylab
import seaborn as sns
from pandas.tools.plotting import scatter_matrix
data['ConfirmedCases'][data['Country_Region']==''][51:]<count_values>
train_df = pd.read_csv(".. /input/train.csv")
 test_df = pd.read_csv(".. /input/test.csv")
 data_df = train_df.append(test_df )
data['ConfirmedCases'][data['Country_Region']=='India'].value_counts()<create_dataframe>
data_df['Title'] = data_df['Name']
for name_string in data_df['Name']:
data_df['Title'] = data_df['Name'].str.extract('([A-Za-z]+)\.', expand=True)
pd.DataFrame(data.loc[data['Country_Region']=='India',['ConfirmedCases']] )<data_type_conversions>
mapping = {'Mlle': 'Miss', 'Major': 'Mr', 'Col': 'Mr', 'Sir': 'Mr', 'Don': 'Mr', 'Mme': 'Miss',
'Jonkheer': 'Mr', 'Lady': 'Mrs', 'Capt': 'Mr', 'Countess': 'Mrs', 'Ms': 'Miss', 'Dona': 'Mrs'}
data_df.replace({'Title': mapping}, inplace=True)
 titles = ['Dr', 'Master', 'Miss', 'Mr', 'Mrs', 'Rev']
datetime_str = '03/22/20 00:00:00'
datetime_object = datetime.strptime(datetime_str, '%m/%d/%y %H:%M:%S' )<import_modules>
for title in titles:
age_to_impute = data_df.groupby('Title')['Age'].median() [titles.index(title)]
data_df.loc[(data_df['Age'].isnull())&(data_df['Title'] == title), 'Age'] = age_to_impute
train_df['Age'] = data_df['Age'][:891]
test_df['Age'] = data_df['Age'][891:]
data_df.drop('Title', axis = 1, inplace = True )
from datetime import timedelta<import_modules>
data_df['Family_Size'] = data_df['Parch'] + data_df['SibSp']
train_df['Family_Size'] = data_df['Family_Size'][:891]
test_df['Family_Size'] = data_df['Family_Size'][891:]
import math<compute_test_metric>
data_df['Last_Name'] = data_df['Name'].apply(lambda x: str.split(x, ",")[0])
 data_df['Fare'].fillna(data_df['Fare'].mean() , inplace=True)
 data_df['Fare'].fillna(data_df['Fare'].mean() , inplace=True )
def rmsle(y, y_pred):
assert len(y)== len(y_pred)
 terms_to_sum = [(math.log(y_pred[i] + 1)- math.log(y[i] + 1)) ** 2.0 for i,pred in enumerate(y_pred)]
return(sum(terms_to_sum)*(1.0/len(y)))** 0.5<train_model>
train_size = len(train_df)
 test_size = len(test_df)
 fare_df = data_df.loc[data_df['Family_Size']>1, ["Last_Name", "Fare", "Family_Size"]].iloc[:train_size]
fare_diff =(((fare_df.groupby(['Last_Name', 'Family_Size'] ).max()
- fare_df.groupby(['Last_Name', 'Family_Size'] ).min())!=0 ).sum() /train_size * 100)
 print(( "Percentage of families with different fares is: %.1f" %(fare_diff.values[0])) + '%')
def evaluate_arima_model(X,forecast_days, arima_order):
X=[x for x in X]
train_size = int(len(X)* 0.9)
 train, test1 = X[0:train_size], X[train_size:]
history=train
model = ARIMA(history, order=arima_order)
 model_fit = model.fit(disp=0)
 predictions = list()
predictions =model_fit.forecast(steps=len(test1)) [0]
model = ARIMA(X, order=arima_order)
 model_fit = model.fit(disp=0)
 if np.isnan(model_fit.forecast(steps=forecast_days)[0] ).sum() >0:
return float('inf')
 error = rmsle(test1, predictions)
 return error<find_best_params>
train_temp_df = data_df.iloc[:train_size]
family_df_grpby = train_temp_df[train_temp_df['Family_Size']>1][
['Last_Name', 'Fare', 'Family_Size', 'Survived']].groupby(['Last_Name', 'Fare'])
 family_df = pd.DataFrame(data=family_df_grpby.size() , columns=['Size in train'])
 family_df['Survived total'] = family_df_grpby['Survived'].sum().astype(int)
 family_df['Family_Size'] = family_df_grpby['Family_Size'].mean().astype(int)
 print("Whole family survived: %.1f"
%(100*len(family_df[family_df['Size in train']==family_df['Survived total'] ])/len(family_df)) +'%')
 print("Whole family perished: %.1f"
%(100*len(family_df[family_df['Survived total'] == 0])/len(family_df)) +'%')
def evaluate_models(dataset,forcast_days, p_values, d_values, q_values):
best_score, best_cfg = float("inf"),(0,0,0)
 for p in p_values:
for d in d_values:
for q in q_values:
order =(p,d,q)
 try:
mse = evaluate_arima_model(dataset,forcast_days, order)
 if mse < best_score:
best_score, best_cfg = mse, order
except:
continue
print('Best ARIMA%s MSE=%.3f' %(best_cfg, best_score))
return best_cfg, best_score<set_options>
DEFAULT_SURVIVAL_VALUE = 0.5
data_df['Family_Survival'] = DEFAULT_SURVIVAL_VALUE
for grp, grp_df in data_df[['Survived','Name', 'Last_Name', 'Fare', 'Ticket', 'PassengerId',
'SibSp', 'Parch', 'Age', 'Cabin']].groupby(['Last_Name', 'Fare']):
if(len(grp_df)!= 1):
for ind, row in grp_df.iterrows() :
smax = grp_df.drop(ind)['Survived'].max()
smin = grp_df.drop(ind)['Survived'].min()
passID = row['PassengerId']
if(smax == 1.0):
data_df.loc[data_df['PassengerId'] == passID, 'Family_Survival'] = 1
elif(smin==0.0):
data_df.loc[data_df['PassengerId'] == passID, 'Family_Survival'] = 0
print("Number of passengers with family survival information:",
data_df.loc[data_df['Family_Survival']!=0.5].shape[0])
warnings.filterwarnings('ignore' )<feature_engineering>
for _, grp_df in data_df.groupby('Ticket'):
if(len(grp_df)!= 1):
for ind, row in grp_df.iterrows() :
if(row['Family_Survival'] == 0)|(row['Family_Survival']== 0.5):
smax = grp_df.drop(ind)['Survived'].max()
smin = grp_df.drop(ind)['Survived'].min()
passID = row['PassengerId']
if(smax == 1.0):
data_df.loc[data_df['PassengerId'] == passID, 'Family_Survival'] = 1
elif(smin==0.0):
data_df.loc[data_df['PassengerId'] == passID, 'Family_Survival'] = 0
print("Number of passenger with family/group survival information: "
+str(data_df[data_df['Family_Survival']!=0.5].shape[0]))
train_df['Family_Survival'] = data_df['Family_Survival'][:891]
test_df['Family_Survival'] = data_df['Family_Survival'][891:]
test['ConfirmedCases']=0
test['Fatalities']=0<filter>
data_df['FareBin'] = pd.qcut(data_df['Fare'], 5)
sliced_data=data.loc[(data['Province_State']=='')&(data['Country_Region']=='India'),:]<import_modules>
label = LabelEncoder()
data_df['FareBin_Code'] = label.fit_transform(data_df['FareBin'] )
from pandas import read_csv
from pandas import datetime
from matplotlib import pyplot
from statsmodels.tsa.arima_model import ARIMA
from sklearn.metrics import mean_squared_error
from math import sqrt
from time import time
from sklearn.metrics import mean_squared_error<remove_duplicates>
train_df['FareBin_Code'] = data_df['FareBin_Code'][:891]
test_df['FareBin_Code'] = data_df['FareBin_Code'][891:]
train_df.drop(['Fare'], 1, inplace=True)
 test_df.drop(['Fare'], 1, inplace=True )
country='India'
state=''
sliced_data=data.loc[(data['Province_State']==state)&(data['Country_Region']==country),:]
test_sliced=test.loc[(test['Province_State']==state)&(test['Country_Region']==country),:]
sliced_data=sliced_data.drop_duplicates()
sliced_data=sliced_data.reset_index(drop=True)
 sliced_data=sliced_data.sort_values(by='Date')
 if sliced_data.loc[sliced_data['ConfirmedCases']>0,:].shape[0]>0:
sliced_data=sliced_data.loc[sliced_data['ConfirmedCases']>0,:]
sliced_data=sliced_data.reset_index(drop=True)
 max_date_train=sliced_data['Date'].max()
max_date_test=test_sliced['Date'].max()
forcast_days=int(( max_date_test-max_date_train)/np.timedelta64(1, 'D'))
history=sliced_data['ConfirmedCases'].to_list()
if len(history)==1:
history.append(history[0])
 best_cfg,best_score=evaluate_models(history,forcast_days,range(10),range(7),range(7))
preds=[]
model = ARIMA(history, order=best_cfg)
 model_fit = model.fit(disp=0)
 preds=model_fit.forecast(steps=forcast_days)[0]
preds=[round(p)if p>0 else 0 for p in preds]
dates=[max_date_train+timedelta(days=day+1)for day in range(forcast_days)]
predictions=pd.DataFrame()
predictions['Date']=dates
predictions['ConfirmedCases']=preds
test_sliced=test_sliced.merge(sliced_data[['Date','ConfirmedCases']], on='Date',how='left')
 test_sliced['ConfirmedCases']=test_sliced['ConfirmedCases_y']
del test_sliced['ConfirmedCases_y']
del test_sliced['ConfirmedCases_x']
test_sliced=test_sliced.merge(predictions, on='Date',how='left')
 test_sliced['ConfirmedCases_x'][test_sliced['ConfirmedCases_x'].isna() ]=test_sliced['ConfirmedCases_y'][test_sliced['ConfirmedCases_x'].isna() ]
test_sliced['ConfirmedCases']=test_sliced['ConfirmedCases_x']
del test_sliced['ConfirmedCases_y']
del test_sliced['ConfirmedCases_x']
sliced_data_bck=sliced_data.copy()
if sliced_data.loc[sliced_data['Fatalities']>0,:].shape[0]>0:
sliced_data=sliced_data.loc[sliced_data['Fatalities']>0,:]
sliced_data=sliced_data.reset_index(drop=True)
 max_date_train=sliced_data['Date'].max()
max_date_test=test_sliced['Date'].max()
forcast_days=int(( max_date_test-max_date_train)/np.timedelta64(1, 'D'))
history=sliced_data['Fatalities'].to_list()
if len(history)==1:
history.append(history[0])
 best_cfg,best_score=evaluate_models(history,forcast_days,range(5),range(5),range(5))
preds=[]
model=None
model = ARIMA(history, order=best_cfg)
 model_fit = model.fit(disp=0)
 preds=model_fit.forecast(steps=forcast_days)[0]
preds=[round(p)if p>0 else 0 for p in preds]
dates=[max_date_train+timedelta(days=day+1)for day in range(forcast_days)]
predictions_f=pd.DataFrame()
predictions_f['Date']=dates
predictions_f['Fatalities']=preds
test_sliced=test_sliced.merge(sliced_data_bck[['Date','Fatalities']], on='Date',how='left')
 test_sliced['Fatalities']=test_sliced['Fatalities_y']
del test_sliced['Fatalities_y']
del test_sliced['Fatalities_x']
test_sliced=test_sliced.merge(predictions_f, on='Date',how='left')
 test_sliced['Fatalities_x'][test_sliced['Fatalities_x'].isna() ]=test_sliced['Fatalities_y'][test_sliced['Fatalities_x'].isna() ]
test_sliced['Fatalities']=test_sliced['Fatalities_x']
del test_sliced['Fatalities_y']
del test_sliced['Fatalities_x']
test=test.merge(test_sliced,on='ForecastId',how='left')
 test['ConfirmedCases_x'][test['ConfirmedCases_y'].notna() ]=test['ConfirmedCases_y'][test['ConfirmedCases_y'].notna() ]
test['Fatalities_x'][test['Fatalities_y'].notna() ]=test['Fatalities_y'][test['Fatalities_y'].notna() ]
new_cols=[]
for col in test.columns:
if col[-2:]=='_y':
del test[col]
elif col[-2:]=='_x':
new_cols.append(col[:-2])
 else:
new_cols.append(col)
 test.columns=new_cols
test.loc[(test['Province_State']==state)&(test['Country_Region']==country),:].head()
plt.plot('Date', 'ConfirmedCases', data=sliced_data, color='blue', linewidth=2)
 plt.plot('Date','ConfirmedCases',data=test_sliced,color='orange',linewidth=2)
 plt.plot('Date', 'Fatalities', data=sliced_data, color='purple', linewidth=2)
 plt.plot('Date','Fatalities',data=test_sliced,color='red',linewidth=2)
 plt.show()<filter>
data_df['AgeBin'] = pd.qcut(data_df['Age'], 4)
 label = LabelEncoder()
data_df['AgeBin_Code'] = label.fit_transform(data_df['AgeBin'])
 train_df['AgeBin_Code'] = data_df['AgeBin_Code'][:891]
test_df['AgeBin_Code'] = data_df['AgeBin_Code'][891:]
train_df.drop(['Age'], 1, inplace=True)
 test_df.drop(['Age'], 1, inplace=True )
test.loc[(test['Province_State']==state)&(test['Country_Region']==country),['Country_Region','Date','ConfirmedCases','Fatalities']]<feature_engineering>
train_df['Sex'].replace(['male','female'],[0,1],inplace=True)
 test_df['Sex'].replace(['male','female'],[0,1],inplace=True)
 train_df.drop(['Name', 'PassengerId', 'SibSp', 'Parch', 'Ticket', 'Cabin',
'Embarked'], axis = 1, inplace = True)
 test_df.drop(['Name','PassengerId', 'SibSp', 'Parch', 'Ticket', 'Cabin',
'Embarked'], axis = 1, inplace = True )
test['ConfirmedCases']=0
test['Fatalities']=0<remove_duplicates>
X = train_df.drop('Survived', 1)
 y = train_df['Survived']
X_test = test_df.copy()
for country in countries:
for state in countries[country]:
print('Country : '+country,'State : '+state)
 sliced_data=data.loc[(data['Province_State']==state)&(data['Country_Region']==country),:]
test_sliced=test.loc[(test['Province_State']==state)&(test['Country_Region']==country),:]
sliced_data=sliced_data.drop_duplicates()
sliced_data=sliced_data.reset_index(drop=True)
 sliced_data=sliced_data.sort_values(by='Date')
 if sliced_data.loc[sliced_data['ConfirmedCases']>0,:].shape[0]>0:
sliced_data=sliced_data.loc[sliced_data['ConfirmedCases']>0,:]
sliced_data=sliced_data.reset_index(drop=True)
 max_date_train=sliced_data['Date'].max()
max_date_test=test_sliced['Date'].max()
forcast_days=int(( max_date_test-max_date_train)/np.timedelta64(1, 'D'))
history=sliced_data['ConfirmedCases'].to_list()
if len(history)==1:
history.append(history[0])
 best_cfg,best_score=evaluate_models(history,forcast_days,range(5),range(5),range(5))
preds=[]
model = ARIMA(history, order=best_cfg)
 model_fit = model.fit(disp=0)
 preds=model_fit.forecast(steps=forcast_days)[0]
preds=[round(p)if p>0 else 0 for p in preds]
dates=[max_date_train+timedelta(days=day+1)for day in range(forcast_days)]
predictions=pd.DataFrame()
predictions['Date']=dates
predictions['ConfirmedCases']=preds
test_sliced=test_sliced.merge(sliced_data[['Date','ConfirmedCases']], on='Date',how='left')
 test_sliced['ConfirmedCases']=test_sliced['ConfirmedCases_y']
del test_sliced['ConfirmedCases_y']
del test_sliced['ConfirmedCases_x']
test_sliced=test_sliced.merge(predictions, on='Date',how='left')
 test_sliced['ConfirmedCases_x'][test_sliced['ConfirmedCases_x'].isna() ]=test_sliced['ConfirmedCases_y'][test_sliced['ConfirmedCases_x'].isna() ]
test_sliced['ConfirmedCases']=test_sliced['ConfirmedCases_x']
del test_sliced['ConfirmedCases_y']
del test_sliced['ConfirmedCases_x']
sliced_data_bck=sliced_data.copy()
if sliced_data.loc[sliced_data['Fatalities']>0,:].shape[0]>0:
sliced_data=sliced_data.loc[sliced_data['Fatalities']>0,:]
sliced_data=sliced_data.reset_index(drop=True)
 max_date_train=sliced_data['Date'].max()
max_date_test=test_sliced['Date'].max()
forcast_days=int(( max_date_test-max_date_train)/np.timedelta64(1, 'D'))
history=sliced_data['Fatalities'].to_list()
if len(history)==1:
history.append(history[0])
 best_cfg,best_score=evaluate_models(history,forcast_days,range(5),range(5),range(5))
preds=[]
model=None
model = ARIMA(history, order=best_cfg)
 model_fit = model.fit(disp=0)
 preds=model_fit.forecast(steps=forcast_days)[0]
preds=[round(p)if p>0 else 0 for p in preds]
dates=[max_date_train+timedelta(days=day+1)for day in range(forcast_days)]
predictions_f=pd.DataFrame()
predictions_f['Date']=dates
predictions_f['Fatalities']=preds
test_sliced=test_sliced.merge(sliced_data_bck[['Date','Fatalities']], on='Date',how='left')
 test_sliced['Fatalities']=test_sliced['Fatalities_y']
del test_sliced['Fatalities_y']
del test_sliced['Fatalities_x']
test_sliced=test_sliced.merge(predictions_f, on='Date',how='left')
 test_sliced['Fatalities_x'][test_sliced['Fatalities_x'].isna() ]=test_sliced['Fatalities_y'][test_sliced['Fatalities_x'].isna() ]
test_sliced['Fatalities']=test_sliced['Fatalities_x']
del test_sliced['Fatalities_y']
del test_sliced['Fatalities_x']
test=test.merge(test_sliced,on='ForecastId',how='left')
 test['ConfirmedCases_x'][test['ConfirmedCases_y'].notna() ]=test['ConfirmedCases_y'][test['ConfirmedCases_y'].notna() ]
test['Fatalities_x'][test['Fatalities_y'].notna() ]=test['Fatalities_y'][test['Fatalities_y'].notna() ]
new_cols=[]
for col in test.columns:
if col[-2:]=='_y':
del test[col]
elif col[-2:]=='_x':
new_cols.append(col[:-2])
 else:
new_cols.append(col)
 test.columns=new_cols
print(test.loc[(test['Province_State']==state)&(test['Country_Region']==country),:].head())
 plt.plot('Date', 'ConfirmedCases', data=sliced_data, color='blue', linewidth=2)
 plt.plot('Date','ConfirmedCases',data=test_sliced,color='orange',linewidth=2)
 plt.plot('Date', 'Fatalities', data=sliced_data, color='purple', linewidth=2)
 plt.plot('Date','Fatalities',data=test_sliced,color='red',linewidth=2)
 plt.show()<save_to_csv>
std_scaler = StandardScaler()
X = std_scaler.fit_transform(X)
 X_test = std_scaler.transform(X_test )
sumb=pd.read_csv('/kaggle/input/covid19-global-forecasting-week-2/submission.csv')
 output=pd.DataFrame()
output['ForecastId']=test['ForecastId'].astype(int)
 output['ConfirmedCases']=test['ConfirmedCases'].astype(int)
 output['Fatalities']=test['Fatalities'].astype(int)
 output.to_csv('submission.csv',index=False )<load_from_csv>
from sklearn.model_selection import GridSearchCV
mnist_test = pd.read_csv(".. /input/mnist-in-csv/mnist_test.csv")
 mnist_train = pd.read_csv(".. /input/mnist-in-csv/mnist_train.csv" )<load_from_csv>
sample_submission = pd.read_csv(".. /input/digit-recognizer/sample_submission.csv")
 test = pd.read_csv(".. /input/digit-recognizer/test.csv")
 train = pd.read_csv(".. /input/digit-recognizer/train.csv" )<feature_engineering>
test['dataset'] = 'test'<feature_engineering>
train['dataset'] = 'train'<concatenate>
n_neighbors = [6,7,8,9,10,11,12,14,16,18,20,22]
algorithm = ['auto']
weights = ['uniform', 'distance']
leaf_size = list(range(1,50,5))
hyperparams = {'algorithm': algorithm, 'weights': weights, 'leaf_size': leaf_size,
'n_neighbors': n_neighbors}
gd=GridSearchCV(estimator = KNeighborsClassifier() , param_grid = hyperparams, verbose=True,
cv=10, scoring = "accuracy")
 gd.fit(X, y)
 print(gd.best_score_)
 print(gd.best_estimator_)
dataset = pd.concat([train.drop('label', axis=1), test] ).reset_index()<concatenate>
gd.best_estimator_.fit(X, y)
 y_pred = gd.best_estimator_.predict(X_test )
<sort_values><EOS> 
submission = pd.read_csv(".. /input/sample_submission.csv")
 submission['Survived'] = y_pred
submission.to_csv(".. /working/submission0615_9.csv", index = False )