Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<merge>
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
data = pd.read_csv("/kaggle/input/titanic/train.csv")
 test=pd.read_csv("/kaggle/input/titanic/test.csv")
output_columns = ['ConfirmedCases', 'Fatalities']
tmp_output_columns = ['ConfirmedCases_y', 'Fatalities_y']
last_training_day = df['DayNum'].max()
first_test_day = df_test['DayNum'].min()
train_test_keys = ['Country/Region', 'Province/State', 'DayNum']
df_test_pop_train = pd.merge(df_test_pop, df_pop[df_pop['DayNum']>=first_test_day][train_test_keys + ['PreviousDay-0ExposedDensity', 'ExposedDensity'] + previous_days_cols + output_columns],
on=train_test_keys, how='left')
 df_test_pop_train 
<concatenate>
name=[]
cnt=[]
for column in data.columns:
name.append(column)
 cnt.append(data[column].isna().sum())
 tmp=pd.DataFrame(cnt,name)
 print('Count of Na values in dataset')
 tmp
df_test_final = pd.concat([df_pop[df_pop['DayNum']<first_test_day], df_test_pop_train] ).reset_index(drop=True)
 df_test_final[(df_test_final['Country/Region']=='Poland')&(df_test_final['DayNum']<=last_training_day+1)&(df_test_final['DayNum']>last_training_day-10)]<prepare_output>
name=[]
cnt=[]
for column in test.columns:
name.append(column)
 cnt.append(test[column].isna().sum())
 tmp=pd.DataFrame(cnt,name)
 print('Count of Na values in dataset')
 tmp
df_test_final['PopulationOrg'] = df_test_final['Population'].copy()
df_test_final['Land Area Org'] = df_test_final['Land Area'].copy()<feature_engineering>
data.Age=data.Age.fillna(data.Age.mean())
 data.Embarked=data.Embarked.fillna(data.Embarked.mode() [0])
 test.Age=test.Age.fillna(test.Age.mean() )
model_cc = load_model(model_file_cc)
 model_f = load_model(model_file_f)
 last_test_day = df_test['DayNum'].max()
for day in range(last_training_day+1, last_test_day+1):
print('predicting day {}({} to go)'.format(day, last_test_day-day))
add_extra_features_from_previous_days(df_test_final, TAIL)
 df_test_final['PreviousDay-0ConfirmedCases_NotScaled'] = df_test_final['PreviousDay-0ConfirmedCases'].copy()
df_test_final['PreviousDay-0Fatalities_NotScaled'] = df_test_final['PreviousDay-0Fatalities'].copy()
df_test_final['PreviousDay-0ExposedDensity'] = df_test_final.groupby(['Country/Region', 'Province/State'])['ExposedDensity'].shift(periods=1, fill_value=np.nan)
 df_test_final['PreviousDay-0ExposedDensity'] = df_test_final.apply(
 lambda row: row['Density'] if np.isnan(row['PreviousDay-0ExposedDensity'])else row['PreviousDay-0ExposedDensity'],
axis=1 
)
 current_day = df_test_final[df_test_final['DayNum']==day].copy()
current_day_scaled = scale_data(current_day)
 current_day_scaled['ConfirmedCasesDelta'] = predict_output(current_day_scaled[model_x_columns].drop(columns=['DayNum']), model_cc)
 current_day_scaled['FatalitiesDelta'] = predict_output(current_day_scaled[model_x_columns].drop(columns=['DayNum']), model_f)
 current_day_scaled['ConfirmedCases'] = current_day_scaled['PreviousDay-0ConfirmedCases_NotScaled'] + current_day_scaled['ConfirmedCasesDelta']
current_day_scaled['Fatalities'] = current_day_scaled['PreviousDay-0Fatalities_NotScaled'] + current_day_scaled['FatalitiesDelta']
current_day_scaled['ExposedDensity'] =(current_day_scaled['PopulationOrg'] - np.power(current_day_scaled['ConfirmedCases'], 1.43)) /current_day_scaled['Land Area Org']
cond = df_test_final['DayNum']==day
df_test_final.loc[cond, output_columns+['ExposedDensity']] = current_day_scaled[output_columns+['ExposedDensity']].copy()
df_test_final.loc[(cond)&(df_test_final[cond]['ConfirmedCases']<current_day_scaled['ConfirmedCases']), 'ConfirmedCases'] = df_test_final['PreviousDay-0ConfirmedCases']
df_test_final.loc[(cond)&(df_test_final[cond]['Fatalities']<current_day_scaled['Fatalities']), 'Fatalities'] = df_test_final['PreviousDay-0Fatalities']
df_test_final.columns 
<save_to_csv>
( data.Cabin.isnull().sum() /(data.shape[0])) *100
submission_columns = ['ForecastId', 'ConfirmedCases', 'Fatalities']
df_test_final.loc[df_test_final['ForecastId'].isna() , 'ForecastId'] = 0
df_test_final[submission_columns] = df_test_final[submission_columns].astype(int)
 df_test_final[df_test_final['DayNum']>=first_test_day][submission_columns].to_csv('submission.csv', index=False)
 df_test_final[df_test_final['DayNum']>=last_training_day][submission_columns]<set_options>
data=data.drop(['Cabin'],1)
 data.sample(3 )
warnings.filterwarnings("ignore" )<load_from_csv>
a1=data[(data.Age <= 10)&(data.Survived == 1)].count() [0]
a2=data[(( data.Age>10)&(data.Age<=20)) &(data.Survived==1)].count() [0]
a3=data[(( data.Age>20)&(data.Age<=30)) &(data.Survived==1)].count() [0]
a4=data[(( data.Age>30)&(data.Age<=40)) &(data.Survived==1)].count() [0]
a5=data[(( data.Age>40)&(data.Age<=50)) &(data.Survived==1)].count() [0]
a6=data[(( data.Age>50)&(data.Age<=60)) &(data.Survived==1)].count() [0]
a7=data[(( data.Age>60)&(data.Age<=70)) &(data.Survived==1)].count() [0]
a8=data[(( data.Age>70)&(data.Age<=80)) &(data.Survived==1)].count() [0]
a9=data[(data.Age>80)&(data.Survived==1)].count() [0]
plt.figure(figsize=(8,6))
plt.pie([a1,a2,a3,a4,a5,a6,a7,a8,a9],labels=['0-10 ','10-20','20-30','30-40','40-50','50-60','60-70','70-80','80 above'],autopct='%1.1f%%', shadow=True, startangle=140,radius=1.6)
 plt.show()
df_train=pd.read_csv(".. /input/covid19-global-forecasting-week-4/train.csv")
 df_test=pd.read_csv(".. /input/covid19-global-forecasting-week-4/test.csv")
 df_sub=pd.read_csv(".. /input/covid19-global-forecasting-week-4/submission.csv")
 print(df_train.shape)
 print(df_test.shape)
 print(df_sub.shape )<count_unique_values>
data=data.drop(['Name','Ticket','Fare','Parch'],1)
 test=test.drop(['Name','Ticket','Fare','Parch','Cabin'],1 )
print(f"Unique Countries: {len(df_train.Country_Region.unique())}" )<feature_engineering>
le = LabelEncoder()
data['Sex']= le.fit_transform(data['Sex'])
 data['Embarked']= le.fit_transform(data['Embarked'])
 test['Sex']= le.fit_transform(test['Sex'])
 test['Embarked']= le.fit_transform(test['Embarked'])
train_dates=list(df_train.Date.unique())
 latest_date=df_train.Date.max()
print(f"Period : {len(df_train.Date.unique())} days")
 print(f"From : {df_train.Date.min() } To : {df_train.Date.max() }" )<count_unique_values>
x=data.drop(['Survived'],1)
 y=data['Survived']
print(f"Unique Regions: {df_train.shape[0]/len(df_train.Date.unique())}" )<count_values>
model = RandomForestClassifier(n_estimators=1300, min_samples_leaf=16,bootstrap = True,
max_features = 'sqrt')
 model.fit(x,y)
 prediction=model.predict(test )
df_train.Country_Region.value_counts()<count_missing_values>
sub=pd.read_csv('.. /input/titanic/gender_submission.csv')
 Yt=sub['Survived'].values
print(f"Number of rows without Country_Region : {df_train.Country_Region.isna().sum() }" )<drop_column>
print("Accuaracy:",accuracy_score(Yt, prediction))
print("Precision:", precision_score(Yt, prediction))
df_train.drop(labels=["Id","Province_State","Country_Region"], axis=1, inplace=True)
 df_train<feature_engineering>
testf=pd.read_csv('.. /input/titanic/test.csv' )
<filter><EOS> 
submission = pd.DataFrame({'PassengerId':testf['PassengerId'],'Survived':prediction})
 submission.head()
filename = 'Titanic1.csv'
submission.to_csv(filename,index=False)
 print('Saved file: ' + filename )