Kaggle competition: San Francisco Crime Classification
df['Confirmedlast43'] = df['ConfirmedCases'].shift(43)
 df['Fatalitieslast43'] = df['Fatalities'].shift(43 )<feature_engineering>
train = pd.read_csv('/kaggle/input/sf-crime/train.csv', parse_dates=['Dates'])
 test = pd.read_csv('/kaggle/input/sf-crime/test.csv', parse_dates=['Dates'] )
df['is_valid'] = np.where(df['Date']<'2020-03-29', False, True )<groupby>
train['year'] = train['Dates'].dt.year
train['month'] = train['Dates'].dt.month
train['day'] = train['Dates'].dt.day
train['DayOfWeek'] = train['Dates'].dt.weekday
train['hour'] = train['Dates'].dt.hour
train['minute'] = train['Dates'].dt.minute
df.groupby('is_valid' ).size()<feature_engineering>
train['Block'] = train['Address'].str.contains('Block', case=False)
 train.head()
df['ConfirmedLog'] = np.log(df['ConfirmedCases']+1)
 df['FatalitiesLog'] = np.log(df['Fatalities']+1 )<define_variables>
train['AV'] = train['Address'].str.contains('AV', case=False)
 train.head()
cat_vars = ['Province_State','Country_Region','province_flag']
cont_vars = ['Elapsed',
'days_first_case',
'days_first_fatality',
'pop_density',
'population',
'area',
'lat_min',
'lat_max',
'lon_min',
'lon_max',
'centroid_x',
'centroid_y',
'wdi_country_population',
'wdi_country_arrivals',
'wdi_arrivals_per_capita',
'wdi_gini',
'wdi_perc_urban_pop',
'wdi_perc_handwashing',
'wdi_uhc_coverage',
'wdi_hospital_beds_p1000',
'wdi_smoke_prevalence',
'wdi_diabetes_prevalence',
'wdi_gdp_per_capita_ppp',
'wdi_perc_death_comm_diseases',
'wdi_perc_death_non_comm_diseases',
'wdi_death_rate_p1000',
'wdi_perc_basic_sanitation',
'wdi_dom_govmt_healt_exped_gdp',
'wdi_dom_govmt_healt_exped_per_cap',
'wdi_perc_females',
'wdi_perc_males',
'wdi_perc_female_20_29',
'wdi_perc_female_30_39',
'wdi_perc_female_40_49',
'wdi_perc_female_50_59',
'wdi_perc_female_60_69',
'wdi_perc_female_70_79',
'wdi_perc_female_80p',
'wdi_perc_male_20_29',
'wdi_perc_male_30_39',
'wdi_perc_male_40_49',
'wdi_perc_male_50_59',
'wdi_perc_male_60_69',
'wdi_perc_male_70_79',
'wdi_perc_male_80p',
'wdi_pop_denisty',
'wdi_perc_anual_growth_pop','Month','Week','Dayofyear']
dep_vars = ['ConfirmedLog']<categorify>
train['ST'] = train['Address'].str.contains('ST', case=False)
 train.head()
y_block = TransformBlock()<create_dataframe>
test['year'] = test['Dates'].dt.year
test['month'] = test['Dates'].dt.month
test['day'] = test['Dates'].dt.day
test['DayOfWeek'] = test['Dates'].dt.weekday
test['hour'] = test['Dates'].dt.hour
test['minute'] = test['Dates'].dt.minute
to = TabularPandas(df, procs, cat_names=cat_vars, cont_names=cont_vars, y_names=dep_vars, y_block=RegressionBlock() , splits=splits )<load_pretrained>
test['Block'] = test['Address'].str.contains('Block', case=False)
 test.head()
dls = to.dataloaders(bs=512 )<define_variables>
test['AV'] = test['Address'].str.contains('AV', case=False)
 test.head()
dls.show_batch()<train_model>
test['ST'] = test['Address'].str.contains('ST', case=False)
 test.head()
cats, conts, y = next(iter(dls.train))<prepare_x_and_y>
train.groupby(['Category'])['AV'].mean()
max_log_y = np.log(1.2)+ np.max(df['ConfirmedLog'])
 y_range =(0, max_log_y)
 y_range<choose_model_class>
train.groupby(['Category'])['ST'].mean()
learn = tabular_learner(dls, layers=[300,250], loss_func=MSELossFlat() ,
config=tabular_config(ps=[0.005,0.015]), y_range=y_range,
metrics=[exp_rmspe, rmse] )<find_best_params>
train['Category'].value_counts()
learn.model<load_pretrained>
y = train['Category']
learn.load('/kaggle/input/covid19-week2-external-data/best18_2day' )<predict_on_test>
le = LabelEncoder()
le.fit(train['PdDistrict'])
 le.classes_
preds, y = learn.get_preds()<prepare_output>
train['PdDistrict'] = le.transform(train['PdDistrict'] )
np.exp(preds)-1<prepare_output>
test['PdDistrict'] = le.transform(test['PdDistrict'])
 test.head()
np.exp(y)-1<train_model>
train.Address.nunique()
dl = learn.dls.test_dl(df )<predict_on_test>
test.Address.nunique()
raw_test_preds = learn.get_preds(dl=dl )<prepare_output>
le.fit(list(train['Address'])+ list(test['Address']))
preds = np.exp(to_np(raw_test_preds[0])) -1<create_dataframe>
train['Address'] = le.transform(train['Address'])
 test['Address'] = le.transform(test['Address'] )
preds = pd.DataFrame(preds)
 preds.columns = ['pred_confirmed']<concatenate>
train['Dates'].dt.week
df = pd.concat([df, preds], axis=1 )<train_model>
test['Dates'].dt.week
dl = learn.dls.test_dl(df_test )<predict_on_test>
y = train['Category']
train = train.drop(['Dates', 'Resolution', 'Descript', 'Category'], axis=1)
 test = test.drop(['Id', 'Dates'], axis=1 )
raw_test_preds = learn.get_preds(dl=dl )<prepare_output>
preds_confirmed_test = np.exp(to_np(raw_test_preds[0])) -1<define_variables>
cat = CatBoostClassifier(task_type='GPU')
 cat.fit(train, y )
cat_vars = ['Province_State','Country_Region','province_flag']
cont_vars = ['Elapsed',
'days_first_case',
'days_first_fatality',
'pop_density',
'population',
'area',
'lat_min',
'lat_max',
'lon_min',
'lon_max',
'centroid_x',
'centroid_y',
'wdi_country_population',
'wdi_country_arrivals',
'wdi_arrivals_per_capita',
'wdi_gini',
'wdi_perc_urban_pop',
'wdi_perc_handwashing',
'wdi_uhc_coverage',
'wdi_hospital_beds_p1000',
'wdi_smoke_prevalence',
'wdi_diabetes_prevalence',
'wdi_gdp_per_capita_ppp',
'wdi_perc_death_comm_diseases',
'wdi_perc_death_non_comm_diseases',
'wdi_death_rate_p1000',
'wdi_perc_basic_sanitation',
'wdi_dom_govmt_healt_exped_gdp',
'wdi_dom_govmt_healt_exped_per_cap',
'wdi_perc_females',
'wdi_perc_males',
'wdi_perc_female_20_29',
'wdi_perc_female_30_39',
'wdi_perc_female_40_49',
'wdi_perc_female_50_59',
'wdi_perc_female_60_69',
'wdi_perc_female_70_79',
'wdi_perc_female_80p',
'wdi_perc_male_20_29',
'wdi_perc_male_30_39',
'wdi_perc_male_40_49',
'wdi_perc_male_50_59',
'wdi_perc_male_60_69',
'wdi_perc_male_70_79',
'wdi_perc_male_80p',
'wdi_pop_denisty',
'wdi_perc_anual_growth_pop','Month','Week','Dayofyear','pred_confirmed']
dep_vars = ['FatalitiesLog']<create_dataframe>
preds = cat.predict_proba(test )
to = TabularPandas(df, procs, cat_names=cat_vars, cont_names=cont_vars, y_names=dep_vars, y_block=RegressionBlock() , splits=splits )<load_pretrained>
sample = pd.read_csv('/kaggle/input/sf-crime/sampleSubmission.csv')
 sample.head()
dls = to.dataloaders(bs=512 )<define_variables>
sample.iloc[:,1:] = preds
sample.head()
max_log_y = np.log(1.2)+ np.max(df['FatalitiesLog'])
 y_range =(0, max_log_y)
 y_range<choose_model_class>
sample.to_csv("baseline_lgbm.csv", index=False )
<load_pretrained><EOS>