Kaggle competition: San Francisco Crime Classification
<SOS> metric: MulticlassLoss Kaggle data source: san-francisco-crime-classification<install_modules>
import pandas as pd
from shapely.geometry import Point
import geopandas as gpd
import matplotlib.pyplot as plt
import numpy as np
from sklearn.impute import SimpleImputer
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split
import seaborn as sns
from matplotlib import cm
import urllib.request
import shutil
import zipfile
import os
import re
import contextily as ctx
import geoplot as gplt
import lightgbm as lgb
import eli5
from eli5.sklearn import PermutationImportance
from lightgbm import LGBMClassifier
from matplotlib import pyplot as plt
from pdpbox import pdp, get_dataset, info_plots
import shap
!pip install -q fastprogress fastai2 fastcore fast_tabnet --upgrade<import_modules>
train = pd.read_csv('.. /input/train.csv', parse_dates=['Dates'])
 test = pd.read_csv('.. /input/test.csv', parse_dates=['Dates'], index_col='Id' )
from fastai2.basics import *
from fastai2.tabular.all import *
from fast_tabnet.core import *<load_from_csv>
train.duplicated().sum()
df = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-2/train.csv' )<load_from_csv>
train.drop_duplicates(inplace=True)
 train.replace({'X': -120.5, 'Y': 90.0}, np.NaN, inplace=True)
 test.replace({'X': -120.5, 'Y': 90.0}, np.NaN, inplace=True)
 imp = SimpleImputer(strategy='mean')
 for district in train['PdDistrict'].unique() :
train.loc[train['PdDistrict'] == district, ['X', 'Y']] = imp.fit_transform(
 train.loc[train['PdDistrict'] == district, ['X', 'Y']])
 test.loc[test['PdDistrict'] == district, ['X', 'Y']] = imp.transform(
 test.loc[test['PdDistrict'] == district, ['X', 'Y']])
 train_gdf = create_gdf(train )
df_test = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-2/test.csv')
 df_test.head()<feature_engineering>
url = 'https://data.sfgov.org/api/geospatial/wkhw-cjsf?method=export&format=Shapefile'
with urllib.request.urlopen(url)as response, open('pd_data.zip', 'wb')as out_file:
shutil.copyfileobj(response, out_file)
 with zipfile.ZipFile('pd_data.zip', 'r')as zip_ref:
zip_ref.extractall('pd_data')
 for filename in os.listdir('./pd_data/'):
if re.match(".+\.shp", filename):
pd_districts = gpd.read_file('./pd_data/'+filename)
 break
pd_districts.crs={'init': 'epsg:4326'}
pd_districts = pd_districts.merge(
 train.groupby('PdDistrict' ).count().iloc[:, [0]].rename(
 columns={'Dates': 'Incidents'}),
how='inner',
left_on='district',
right_index=True,
suffixes=('_x', '_y'))
pd_districts = pd_districts.to_crs({'init': 'epsg:3857'})
 train_days = train.groupby('Date' ).count().shape[0]
pd_districts['inc_per_day'] = pd_districts.Incidents/train_days
fig, ax = plt.subplots(figsize=(10, 10))
pd_districts.plot(
 column='inc_per_day',
cmap='Reds',
alpha=0.6,
edgecolor='r',
linestyle='-',
linewidth=1,
legend=True,
ax=ax)
 def add_basemap(ax, zoom, url='http://tile.stamen.com/terrain/tileZ/tileX/tileY.png'):

 xmin, xmax, ymin, ymax = ax.axis()
basemap, extent = ctx.bounds2img(xmin, ymin, xmax, ymax, zoom=zoom, url=url)
 ax.imshow(basemap, extent=extent, interpolation='bilinear')
 ax.axis(( xmin, xmax, ymin, ymax))
add_basemap(ax, zoom=11, url=ctx.sources.ST_TONER_LITE)
 for index in pd_districts.index:
plt.annotate(
 pd_districts.loc[index].district, 
(pd_districts.loc[index].geometry.centroid.x,
pd_districts.loc[index].geometry.centroid.y),
color='
fontsize='large',
fontweight='heavy',
horizontalalignment='center' 
)
 ax.set_axis_off()
plt.show()
df['key'] = df['Country_Region'] + '
df['province_flag'] = np.where(df['Province_State'].isnull() ,0,1)
 df['Province_State'] = df['Province_State'].fillna(df['Country_Region'] )<feature_engineering>
naive_vals = train.groupby('Category' ).count().iloc[:,0]/train.shape[0]
n_rows = test.shape[0]
submission = pd.DataFrame(
 np.repeat(np.array(naive_vals), n_rows ).reshape(39, n_rows ).transpose() ,
columns=naive_vals.index )
df_test['key'] = df_test['Country_Region'] + '
df_test['province_flag'] = np.where(df_test['Province_State'].isnull() ,0,1)
 df_test['Province_State'] = df_test['Province_State'].fillna(df_test['Country_Region'] )<groupby>
def feature_engineering(data):
data['Date'] = pd.to_datetime(data['Dates'].dt.date)
 data['n_days'] =(
 data['Date'] - data['Date'].min() ).apply(lambda x: x.days)
 data['Day'] = data['Dates'].dt.day
data['DayOfWeek'] = data['Dates'].dt.weekday
data['Month'] = data['Dates'].dt.month
data['Year'] = data['Dates'].dt.year
data['Hour'] = data['Dates'].dt.hour
data['Minute'] = data['Dates'].dt.minute
data['Block'] = data['Address'].str.contains('block', case=False)
 data.drop(columns=['Dates','Date','Address'], inplace=True)
 return data
firstconfirmed = df[(df['ConfirmedCases']>0)].groupby(['Province_State','Country_Region'])['Date'].min().reset_index()<groupby>
train = feature_engineering(train)
 train.drop(columns=['Descript','Resolution'], inplace=True)
 test = feature_engineering(test)
 train.head()
firstfatality = df[(df['Fatalities']>0)].groupby(['Province_State','Country_Region'])['Date'].min().reset_index() 
<rename_columns>
le1 = LabelEncoder()
train['PdDistrict'] = le1.fit_transform(train['PdDistrict'])
 test['PdDistrict'] = le1.transform(test['PdDistrict'])
 le2 = LabelEncoder()
y = le2.fit_transform(train.pop('Category'))
train_X, val_X, train_y, val_y = train_test_split(train, y)
 model =LGBMClassifier(objective='multiclass', num_class=39 ).fit(train_X, train_y)
 perm = PermutationImportance(model ).fit(val_X, val_y)
 eli5.show_weights(perm, feature_names=val_X.columns.tolist() )
firstconfirmed.columns = ['Province_State','Country_Region','FirstCaseDate']
firstfatality.columns = ['Province_State','Country_Region','FirstFatalityDate']<merge>
train = pd.read_csv('.. /input/train.csv', parse_dates=['Dates'])
 test = pd.read_csv('.. /input/test.csv', parse_dates=['Dates'], index_col='Id')
 train.drop_duplicates(inplace=True)
 train.replace({'X': -120.5, 'Y': 90.0}, np.NaN, inplace=True)
 test.replace({'X': -120.5, 'Y': 90.0}, np.NaN, inplace=True)
 imp = SimpleImputer(strategy='mean')
 for district in train['PdDistrict'].unique() :
train.loc[train['PdDistrict'] == district, ['X', 'Y']] = imp.fit_transform(
 train.loc[train['PdDistrict'] == district, ['X', 'Y']])
 test.loc[test['PdDistrict'] == district, ['X', 'Y']] = imp.transform(
 test.loc[test['PdDistrict'] == district, ['X', 'Y']])
 train_data = lgb.Dataset(
 train, label=y, categorical_feature=['PdDistrict'], free_raw_data=False)
 train = feature_engineering(train)
 train.drop(columns=['Descript','Resolution'], inplace=True)
 test = feature_engineering(test)
 le1 = LabelEncoder()
train['PdDistrict'] = le1.fit_transform(train['PdDistrict'])
 test['PdDistrict'] = le1.transform(test['PdDistrict'])
 le2 = LabelEncoder()
X = train.drop(columns=['Category'])
 y= le2.fit_transform(train['Category'])
 train_data = lgb.Dataset(
 X, label=y, categorical_feature=['PdDistrict'])
 params = {'boosting':'gbdt',
'objective':'multiclass',
'num_class':39,
'max_delta_step':0.9,
'min_data_in_leaf': 21,
'learning_rate': 0.4,
'max_bin': 465,
'num_leaves': 41
}
bst = lgb.train(params, train_data, 100)
 predictions = bst.predict(test)
 submission = pd.DataFrame(
 predictions,
columns=le2.inverse_transform(np.linspace(0, 38, 39, dtype='int16')) ,
index=test.index)
 submission.to_csv(
 'LGBM_final.csv', index_label='Id' )
df = df.merge(firstconfirmed, left_on=['Province_State','Country_Region'],right_on=['Province_State','Country_Region'],how='left')
 df = df.merge(firstfatality, left_on=['Province_State','Country_Region'],right_on=['Province_State','Country_Region'],how='left' )<merge>
data_for_prediction = test.loc[[846262]]
data_for_prediction