Kaggle competition: San Francisco Crime Classification
import pandas as pd
import datetime
import lightgbm as lgb
import numpy as np
from sklearn import preprocessing<load_from_csv>
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
train = pd.read_csv(".. /input/covid19-global-forecasting-week-2/train.csv")
 test = pd.read_csv(".. /input/covid19-global-forecasting-week-2/test.csv")
 sub = pd.read_csv(".. /input/covid19-global-forecasting-week-2/submission.csv" )<concatenate>
train_data = pd.read_csv("/kaggle/input/sf-crime/train.csv")
 test_data = pd.read_csv("/kaggle/input/sf-crime/test.csv" )
train = train.append(test[test['Date']>'2020-03-31'] )<data_type_conversions>
print(train_data.isnull().sum())
 print(test_data.isnull().sum() )
train['Date'] = pd.to_datetime(train['Date'], format='%Y-%m-%d' )<feature_engineering>
train_data = train_data.drop(["Descript", "Resolution"], axis = 1 )
train['day_dist'] = train['Date']-train['Date'].min()<feature_engineering>
def transformDataset(dataset):
dataset['Dates'] = pd.to_datetime(dataset['Dates'])
 dataset['Date'] = dataset['Dates'].dt.date
dataset['n_days'] =(dataset['Date'] - dataset['Date'].min() ).apply(lambda x: x.days)
 dataset['Year'] = dataset['Dates'].dt.year
dataset['DayOfWeek'] = dataset['Dates'].dt.dayofweek
dataset['WeekOfYear'] = dataset['Dates'].dt.weekofyear
dataset['Month'] = dataset['Dates'].dt.month
dataset['Hour'] = dataset['Dates'].dt.hour
dataset['Block'] = dataset['Address'].str.contains('block', case=False)
 dataset['Block'] = dataset['Block'].map(lambda x: 1 if x == True else 0)
 dataset = dataset.drop('Dates', 1)
 dataset = dataset.drop('Date', 1)
 dataset = dataset.drop('Address', 1)
 dataset = pd.get_dummies(data=dataset, columns=[ 'PdDistrict'], drop_first = True)
 return dataset
train['day_dist'] = train['day_dist'].dt.days<define_variables>
train_data = transformDataset(train_data )
cat_cols = train.dtypes[train.dtypes=='object'].keys()
cat_cols<correct_missing_values>
test_data = transformDataset(test_data )
for cat_col in cat_cols:
train[cat_col].fillna('no_value', inplace = True )<feature_engineering>
le = LabelEncoder()
train_data["Category"] = le.fit_transform(train_data["Category"])
train['place'] = train['Province_State']+'_'+train['Country_Region'] 
<define_variables>
X = train_data.drop("Category",axis=1 ).values
y = train_data["Category"].values
cat_cols = train.dtypes[train.dtypes=='object'].keys()
cat_cols<categorify>
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.10 )
for cat_col in ['place']:
le = preprocessing.LabelEncoder()
le.fit(train[cat_col])
 train[cat_col]=le.transform(train[cat_col] )<define_variables>
dtree = DecisionTreeClassifier()
dtree.fit(X_train,y_train )
drop_cols = ['Id','ForecastId', 'ConfirmedCases','Date', 'Fatalities',
'day_dist', 'Province_State', 'Country_Region']<filter>
predictions = dtree.predict(X_test)
val = train[(train['Date']>='2020-03-12')&(train['Id'].isnull() ==False)] 
<prepare_x_and_y>
print(classification_report(y_test,predictions))
y_ft = train["Fatalities"]
y_val_ft = val["Fatalities"]
y_cc = train["ConfirmedCases"]
y_val_cc = val["ConfirmedCases"] 
<compute_test_metric>
rfc = RandomForestClassifier(n_estimators=40,min_samples_split=100)
 rfc.fit(X_train, y_train )
def rmsle(y_true, y_pred):
return np.sqrt(np.mean(np.power(np.log1p(y_pred)- np.log1p(y_true), 2)) )<compute_test_metric>
rfc_pred = rfc.predict(X_test)
 print(classification_report(y_test,rfc_pred))
def mape(y_true, y_pred):
return np.mean(np.abs(y_pred -y_true)*100/(y_true+1))<filter>
keys = le.classes_
values = le.transform(le.classes_)
 keys
dates = dates[dates>'2020-03-31']<init_hyperparams>
dictionary = dict(zip(keys, values))
print(dictionary )
params = {
"objective": "regression",
"boosting": 'gbdt',
"num_leaves": 1280,
"learning_rate": 0.05,
"feature_fraction": 0.9,
"reg_lambda": 2,
"metric": "rmse",
'min_data_in_leaf':20
}<filter>
test_data = test_data.drop('Id', 1 )
train[train['Date']==date]<filter>
y_pred_proba = rfc.predict_proba(test_data)
 y_pred_proba
test[test['Country_Region']=='Italy']<filter>
result = pd.DataFrame(y_pred_proba, columns=keys)
 result.head()
test[(test['Country_Region']=='China')&(test['Province_State']=='Zhejiang')]<filter>
result.to_csv(path_or_buf="rfc_predict_4.csv",index=True, index_label = 'Id' )