Kaggle competition: San Francisco Crime Classification
df_test = df_test.merge(firstconfirmed, left_on=['Province_State','Country_Region'],right_on=['Province_State','Country_Region'],how='left')
 df_test = df_test.merge(firstfatality, left_on=['Province_State','Country_Region'],right_on=['Province_State','Country_Region'],how='left' )<data_type_conversions>
test_set_full_raw = pd.read_csv(".. /input/sf-crime/test.csv.zip", compression="zip", index_col='Id')
 train_set_full_raw = pd.read_csv(".. /input/sf-crime/train.csv.zip", compression="zip" )
df['Date']=pd.to_datetime(df['Date'], infer_datetime_format=True)
 df['FirstCaseDate']=pd.to_datetime(df['FirstCaseDate'], infer_datetime_format=True)
 df['FirstFatalityDate']=pd.to_datetime(df['FirstFatalityDate'], infer_datetime_format=True )<data_type_conversions>
ts = train_set_full_raw.copy()
ts = ts[ts["Y"]<90]
df_test['Date']=pd.to_datetime(df_test['Date'], infer_datetime_format=True)
 df_test['FirstCaseDate']=pd.to_datetime(df_test['FirstCaseDate'], infer_datetime_format=True)
 df_test['FirstFatalityDate']=pd.to_datetime(df_test['FirstFatalityDate'], infer_datetime_format=True )<feature_engineering>
ts["Category"].value_counts()
df['days_first_case']=(df['Date']-df['FirstCaseDate'] ).dt.days
df['days_first_fatality']=(df['Date']-df['FirstFatalityDate'] ).dt.days<feature_engineering>
print("Most common resolutions for each category in percentage
")
 for i in ts.groupby(["Category"])["Resolution"]:
print('\033[95m'+i[0]+'\033[0m')
 print(round(i[1].value_counts() [:3]/i[1].count() *100,1))
print()
df_test['days_first_case']=(df_test['Date']-df_test['FirstCaseDate'] ).dt.days
df_test['days_first_fatality']=(df_test['Date']-df_test['FirstFatalityDate'] ).dt.days<feature_engineering>
ts["Hour"] = ts.Dates.apply(lambda date_string: date_string[11:-6])
 ts.head()
df['days_first_case']=np.where(df['days_first_case']<0,0,df['days_first_case'].fillna(0))
df['days_first_fatality']=np.where(df['days_first_fatality']<0,0,df['days_first_fatality'].fillna(0))<correct_missing_values>
ts.groupby(["DayOfWeek"])["Hour"].value_counts()
df_test['days_first_case']=np.where(df_test['days_first_case']<0,0,df_test['days_first_case'].fillna(0))
df_test['days_first_fatality']=np.where(df_test['days_first_fatality']<0,0,df_test['days_first_fatality'].fillna(0))<feature_engineering>
train_full = ts.copy()
X_train_full, y_train_full = np.array(train_full[["DayOfWeek", "PdDistrict", "X", "Y", "Hour"]]), np.array(train_full[["Category"]])
 y_train_full = y_train_full.ravel()
X_test = test_set_full_raw.copy()
X_test["Hour"] = X_test.Dates.apply(lambda date_string: date_string[11:-6])
 X_test = X_test.drop(columns=["Dates", "Address"])
 X_test = np.array(X_test )
add_datepart(df,'Date',drop=False )<feature_engineering>
sss = StratifiedShuffleSplit(n_splits=1, test_size=0.25, random_state=42)
 for train_index, test_index in sss.split(X_train_full, y_train_full):
X_train, y_train = X_train_full[train_index], y_train_full[train_index]
X_val, y_val = X_train_full[test_index], y_train_full[test_index]
add_datepart(df_test,'Date',drop=False )<load_from_csv>
num_attribs = [2,3]
cat_attribs = [0,1,4]
num_pipeline = Pipeline([ 
('std_scaler', StandardScaler())
 ])
 full_pipeline = ColumnTransformer([ 
('num', num_pipeline, num_attribs), 
('cat', OneHotEncoder() , cat_attribs)
 ])
 X_train_prepared = full_pipeline.fit_transform(X_train)
 X_val_prepared = full_pipeline.transform(X_val)
 X_test_prepared = full_pipeline.transform(X_test )
external = pd.read_csv('/kaggle/input/covid19-week2-external-data/external_data.csv',sep=';',decimal=',' )<merge>
from sklearn.svm import LinearSVC
from sklearn.ensemble import RandomForestClassifier,VotingClassifier
import xgboost
df = df.merge(external, left_on='key',right_on='key',how='left' )<merge>
ss = StratifiedShuffleSplit(n_splits=1, train_size=100_000, random_state=42)
 for train_index, _ in ss.split(X_train_prepared, y_train):
X_train_prepared_small, y_train_small = X_train_prepared[train_index], y_train[train_index].ravel()
X_train_prepared_small.shape, y_train_small.shape
df_test = df_test.merge(external, left_on='key',right_on='key',how='left' )<create_dataframe>
rf_clf = RandomForestClassifier(max_depth=16, random_state=42, n_jobs=-1, verbose=3)
 xg_clf = xgboost.XGBClassifier()
estimators = [ 
("rf", rf_clf), 
("xg", xg_clf)
 ]
voting_clf = VotingClassifier(estimators, n_jobs=-1, voting="soft")
 voting_clf.fit(X_train_prepared_small, y_train_small)
 voting_clf.score(X_val_prepared, y_val )
df.pivot_table(index='Country_Region', columns='Date', values='ConfirmedCases', aggfunc=np.sum, fill_value=0 )<feature_engineering>
y_pred = voting_clf.predict_proba(X_test_prepared)
 pred_df = pd.DataFrame(y_pred, columns=[voting_clf.classes_])
 pred_df["Id"]= list(range(pred_df.shape[0]))
pred_df.to_csv("crime_pred_02.zip", compression="zip", index=False)
 pred_df.head()