Kaggle competition: Titanic - Machine Learning from Disaster
country = 'Italy'
if country not in train['Country_Region'].unique() :
country_pd_train = train[train['Province_State']==country]
else:
country_pd_train = train[train['Country_Region']==country]
a,b = fit_model_new(country_pd_train,country,make_plot=True )<split>
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import cross_val_score
from sklearn.metrics import accuracy_score, roc_auc_score
from sklearn.metrics import confusion_matrix
from sklearn.ensemble import RandomForestClassifier
from collections import Counter
country = 'New York'
if country not in train['Country_Region'].unique() :
country_pd_train = train[train['Province_State']==country]
else:
country_pd_train = train[train['Country_Region']==country]
a,b = fit_model_new(country_pd_train,country,make_plot=True )<feature_engineering>
train=pd.read_csv('/kaggle/input/titanic/train.csv')
 test=pd.read_csv('/kaggle/input/titanic/test.csv' )
country = 'Afghanistan'
if country not in train['Country_Region'].unique() :
country_pd_train = train[train['Province_State']==country]
else:
country_pd_train = train[train['Country_Region']==country]
a,b = fit_model_new(country_pd_train,country,make_plot=True )<feature_engineering>
df = pd.concat([train,test] ).set_index('PassengerId' )
country = 'US'
country_pd_train = train[train['Country_Region']==country]
country_pd_train2 = country_pd_train.groupby(['Date'] ).sum().reset_index()
country_pd_train2['Date_datetime'] = country_pd_train2['Date'].apply(lambda x:(datetime.datetime.strptime(x, '%Y-%m-%d')))
 a,b = fit_model_new(country_pd_train2,country,make_plot=True )<data_type_conversions>
df.isnull().sum()
country = 'Global'
country_pd_train2 = train.groupby(['Date'] ).sum().reset_index()
country_pd_train2['Date_datetime'] = country_pd_train2['Date'].apply(lambda x:(datetime.datetime.strptime(x, '%Y-%m-%d')))
 a,b = fit_model_new(country_pd_train2,country,make_plot=True )<count_unique_values>
df.Fare = df.Fare.fillna(df.Fare.loc[df.Pclass==3].median() )
validation_scores = []
validation_county = []
validation_country = []
for country in tqdm(train['Country_Region'].unique()):
country_pd_train = train[train['Country_Region']==country]
if len(country_pd_train['Province_State'].unique())<2:
predict_test, score = fit_model_new(country_pd_train,country,make_plot=False)
 if score ==0:
print(f'{country} no case')
 validation_scores.append(score)
 validation_county.append(country)
 validation_country.append(country)
 test.loc[test['Country_Region']==country,'ConfirmedCases'] = predict_test['ConfirmedCases'].tolist()
test.loc[test['Country_Region']==country,'Fatalities'] = predict_test['Fatalities'].tolist()
else:
for state in country_pd_train['Province_State'].unique() :
if state != state:
state_pd = country_pd_train[country_pd_train['Province_State'].isna() ]
predict_test, score = fit_model_new(state_pd,state,make_plot=False)
 if score ==0:
print(f'{country} / {state} no case')
 validation_scores.append(score)
 validation_county.append(state)
 validation_country.append(country)
 test.loc[(test['Country_Region']==country)&(test['Province_State'].isna()),'ConfirmedCases'] = predict_test['ConfirmedCases'].tolist()
test.loc[(test['Country_Region']==country)&(test['Province_State'].isna()),'Fatalities'] = predict_test['Fatalities'].tolist()
else:
state_pd = country_pd_train[country_pd_train['Province_State']==state]
predict_test, score = fit_model_new(state_pd,state,make_plot=False)
 if score ==0:
print(f'{country} / {state} no case')
 validation_scores.append(score)
 validation_county.append(state)
 validation_country.append(country)
 test.loc[(test['Country_Region']==country)&(test['Province_State']==state),'ConfirmedCases'] = predict_test['ConfirmedCases'].tolist()
test.loc[(test['Country_Region']==country)&(test['Province_State']==state),'Fatalities'] = predict_test['Fatalities'].tolist()
print(f'Mean validation score: {np.average(validation_scores):0.5f}' )<prepare_output>
Counter(df.Embarked )
validation_scores = pd.DataFrame({'country/state':validation_country,'country':validation_county,'MSLE':validation_scores})
 validation_scores.sort_values(by=['MSLE'], ascending=False ).head(20 )<filter>
df.Embarked = df.Embarked.fillna('S' )
large_msle = validation_scores[validation_scores['MSLE']>1]<define_variables>
age = list()
for i in df.index:
if df.Age.isnull() [i]:
s = df['Pclass'][i]
age.append(df['Age'].loc[df['Pclass']== s].median())
 else : age.append(df['Age'][i])
 df.Age=age
test_end = datetime.datetime.strptime('2020-05-14','%Y-%m-%d')
 test_start = datetime.datetime.strptime('2020-04-02','%Y-%m-%d')
 train_max = train.Date_datetime.max()
train_min = train.Date_datetime.min()
delta_days =(test_end - train_max ).days
all_days =(test_end - train_min ).days
delta_days,all_days<filter>
def Cabin_cla(x):
if x=='A' : return 4
elif x=='B'or'C': return 3
elif x=='D'or'D': return 2
elif x=='F'or'G': return 1
else : return 0
df['Cabin']=list(map(Cabin_cla,df.Cabin))
for country in large_msle['country'].unique() :
if(country!= country)==False:
country_pd_train = train[train['Country_Region']==country]
country_pd_test = test[test['Country_Region']==country]
if len(country_pd_train)==0:
country_pd_train = train[train['Province_State']==country]
country_pd_test = test[test['Province_State']==country]
test_end = datetime.datetime.strptime('2020-05-14','%Y-%m-%d')
 test_start = datetime.datetime.strptime('2020-04-02','%Y-%m-%d')
 train_max = train.Date_datetime.max()
train_min = train.Date_datetime.min()
delta_days =(test_end - train_max ).days
x = np.array(range(len(country_pd_train)) ).reshape(( -1,1)) [:-7]
valid_x = np.array(range(len(country_pd_train)) ).reshape(( -1,1)) [-7:]
y = country_pd_train['ConfirmedCases'][:-7]
valid_y = country_pd_train['ConfirmedCases'][-7:]
y_fat = country_pd_train['Fatalities'][:-7]
valid_y_fat = country_pd_train['Fatalities'][-7:]
model = Pipeline([('poly', PolynomialFeatures(degree=2)) , 
('linear', LinearRegression(fit_intercept=False)) ])
 model = model.fit(x, y)
 model_fat = Pipeline([('poly', PolynomialFeatures(degree=2)) , 
('linear', LinearRegression(fit_intercept=False)) ])
 model_fat = model_fat.fit(x, y_fat)
 predict_y = model.predict(valid_x)
 predict_yfat = model_fat.predict(valid_x)
 score = mean_squared_log_error(np.clip(valid_y,0,np.inf), np.clip(predict_y,0,np.inf))
score_fat = mean_squared_log_error(np.clip(valid_y_fat,0,np.inf), np.clip(predict_yfat,0,np.inf))
score =(score+score_fat)/2
print(f'{country} {score:0.5f}')
 if score < large_msle[large_msle['country']==country]['MSLE'].tolist() [0]:
validation_scores.loc[validation_scores['country']==country,'MSLE'] = score
predict_x =(np.array(range(all_days)) ).reshape(( -1,1))
predict_result = model.predict(predict_x)
 predict_fat_result = model_fat.predict(predict_x)
 test.loc[test['Province_State']==country,'ConfirmedCases'] = country_pd_train[country_pd_train.Date_datetime>=test_start]['ConfirmedCases'].tolist() +predict_result[-(delta_days):].tolist()
test.loc[test['Province_State']==country,'Fatalities'] = country_pd_train[country_pd_train.Date_datetime>=test_start]['Fatalities'].tolist() +predict_fat_result[-(delta_days):].tolist()
else:
x = np.array(range(len(country_pd_train)) ).reshape(( -1,1)) [:-7]
valid_x = np.array(range(len(country_pd_train)) ).reshape(( -1,1)) [-7:]
y = country_pd_train['ConfirmedCases'][:-7]
valid_y = country_pd_train['ConfirmedCases'][-7:]
y_fat = country_pd_train['Fatalities'][:-7]
valid_y_fat = country_pd_train['Fatalities'][-7:]
model = Pipeline([('poly', PolynomialFeatures(degree=2)) , 
('linear', LinearRegression(fit_intercept=False)) ])
 model = model.fit(x, y)
 model_fat = Pipeline([('poly', PolynomialFeatures(degree=2)) , 
('linear', LinearRegression(fit_intercept=False)) ])
 model_fat = model_fat.fit(x, y_fat)
 predict_y = model.predict(valid_x)
 predict_yfat = model_fat.predict(valid_x)
 score = mean_squared_log_error(np.clip(valid_y,0,np.inf), np.clip(predict_y,0,np.inf))
score_fat = mean_squared_log_error(np.clip(valid_y_fat,0,np.inf), np.clip(predict_yfat,0,np.inf))
score =(score+score_fat)/2
print(f'{country} {score:0.5f}')
 if score < large_msle[large_msle['country']==country]['MSLE'].tolist() [0]:
validation_scores.loc[validation_scores['country']==country,'MSLE'] = score
predict_x =(np.array(range(all_days)) ).reshape(( -1,1))
predict_result = model.predict(predict_x)
 predict_fat_result = model_fat.predict(predict_x)
 test.loc[test['Country_Region']==country,'ConfirmedCases'] = country_pd_train[country_pd_train.Date_datetime>=test_start]['ConfirmedCases'].tolist() +predict_result[-(delta_days):].tolist()
test.loc[test['Country_Region']==country,'Fatalities'] = country_pd_train[country_pd_train.Date_datetime>=test_start]['Fatalities'].tolist() +predict_fat_result[-(delta_days):].tolist()
MSLE_score = validation_scores['MSLE'].tolist()
print(f'Mean validation score: {np.average(MSLE_score):0.5f}')
<save_to_csv>
title = [i.split(' ')[1].split('.')[0].strip() for i in df.Name]
Counter(title )
submit = pd.read_csv('.. /input/covid19-global-forecasting-week-4/submission.csv')
 submit['Fatalities'] = test['Fatalities'].astype('float')
 submit['ConfirmedCases'] = test['ConfirmedCases'].astype('float')
 submit.to_csv('submission.csv',index=False )<define_variables>
def Name_cla(x):
if x=='Mr': return 1
elif x=='Master': return 3
elif x=='Miss': return 4
elif x=='Mrs': return 5
else : return 2
df.Name = list(map(Name_cla,title))
N = 5<predict_on_test>
df=df.drop(['Parch','SibSp'],axis=1 )
def get_preds_lin_reg(series, pred_min, H):

 regr = LinearRegression(fit_intercept=True)
 pred_list = []
X_train = np.array(range(len(series)))
 y_train = np.array(series)
 X_train = X_train.reshape(-1, 1)
 y_train = y_train.reshape(-1, 1)
 regr.fit(X_train, y_train)
 pred = regr.predict(np.array(range(len(series),len(series)+H)).reshape(-1,1))
pred = pred.reshape(H,)
 pred[pred < pred_min] = pred_min
return np.around(pred )<load_from_csv>
df.Embarked = [{'S':1, 'Q':2, 'C':3}[i] for i in df.Embarked]
train = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/train.csv')
 train.columns = [col.lower() for col in train.columns]
train['date'] = pd.to_datetime(train['date'], format='%Y-%m-%d')
 train<load_from_csv>
y_train=df.Survived.dropna()
X_train=df[df.Survived.notnull() ].drop('Survived',axis=1)
 X_test=df[df.Survived.isnull() ].drop('Survived',axis=1 )
test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv')
 test.columns = [col.lower() for col in test.columns]
test['date'] = pd.to_datetime(test['date'], format='%Y-%m-%d')
 test<load_from_csv>
cv_score = list()
auc_score = list()
for i in [5,6,7,8,9]:
rfc=RandomForestClassifier(criterion='gini',random_state=4, max_depth=i)
 rfc.fit(X_train, y_train)
 tree_predicted = rfc.predict(X_train)
 auc_score.append(roc_auc_score(y_train,tree_predicted))
scores = cross_val_score(rfc, X_train, y_train,cv=5)
 cv_score.append(scores.mean())
 df_score=pd.DataFrame({'max_depth':[5,6,7,8,9],'cv_score':cv_score,'auc_score':auc_score} )
submission = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/submission.csv')
 submission<count_missing_values>
rfc=RandomForestClassifier(criterion='gini',random_state=4, max_depth=8)
 rfc.fit(X_train, y_train)
 tree_predicted = rfc.predict(X_train)
 roc_auc_score(y_train,tree_predicted )
train.isnull().sum(axis=0 )<count_unique_values>
rfc=RandomForestClassifier(criterion='gini',random_state=4, max_depth=7)
 rfc.fit(X_train, y_train)
 tree_predicted = rfc.predict(X_train)
 y_test = rfc.predict(X_test )
print(len(train['province_state'].unique()))
train['province_state'].unique()<count_unique_values>
sum(y_test )
print(len(train['country_region'].unique()))
train['country_region'].unique()<count_values>
PassengerId = list(X_test.reset_index() ['PassengerId'])
 result = pd.DataFrame({'PassengerId':PassengerId,'Survived':y_test})
 result['Survived']=[int(i)for i in result.Survived]
result.to_csv('result.csv', index=False )