Kaggle competition: San Francisco Crime Classification
<SOS> metric: MulticlassLoss Kaggle data source: san-francisco-crime-classification<train_model>
!pip install folium
dl = learn.dls.test_dl(df_test )<predict_on_test>
%matplotlib inline
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
raw_test_preds = learn.get_preds(dl=dl )<prepare_output>
df_train = pd.read_csv('.. /input/sf-crime/train.csv')
 df_test = pd.read_csv('.. /input/sf-crime/test.csv' )
preds_fatalities_test = np.exp(to_np(raw_test_preds[0])) -1<create_dataframe>
print(df_train.duplicated().sum())
 df_train.drop_duplicates(inplace=True)
 print(df_train.duplicated().sum() )
preds_fatalities_test = pd.DataFrame(preds_fatalities_test )<rename_columns>
y = df_train['Category']
df_train_description = df_train['Descript']
df_train_resolution = df_train['Resolution']
preds_fatalities_test.columns=['Fatalities']<concatenate>
test_ID = df_test["Id"]
df_test.drop("Id", axis=1, inplace=True )
df_test = pd.concat([df_test, preds_fatalities_test], axis=1 )<load_from_csv>
y.value_counts()
sub_ex = df = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-2/submission.csv' )<prepare_output>
le = LabelEncoder()
y = le.fit_transform(y)
 print(le.classes_ )
sub = df_test[['ForecastId','ConfirmedCases','Fatalities']]<save_to_csv>
def create_update_time_of_day_features(df_, dic_labels):

 try:
print('
Creating or updating period feature 
...early morning from 0H to 6H 
...morning from 6H to 12H 
...afternoon from 12H to 18H 
...evening from 18H to 24H')
 conditions = [(df_[dic_labels['datetime']].dt.hour >= 0)&(df_[dic_labels['datetime']].dt.hour < 6), 
(df_[dic_labels['datetime']].dt.hour >= 6)&(df_[dic_labels['datetime']].dt.hour < 12), 
(df_[dic_labels['datetime']].dt.hour >= 12)&(df_[dic_labels['datetime']].dt.hour < 18), 
(df_[dic_labels['datetime']].dt.hour >= 18)&(df_[dic_labels['datetime']].dt.hour < 24)]
choices = ['early morning', 'morning', 'afternoon', 'evening']
df_['PeriodOfDay'] = np.select(conditions, choices, 'undefined')
 print('...the period of day feature was created')
 except Exception as e:
raise e
sub.to_csv('submission.csv',index=False )<load_from_csv>
df_train['Dates'] = pd.to_datetime(df_train['Dates'])
 df_test['Dates'] = pd.to_datetime(df_test['Dates'])
 create_update_time_of_day_features(df_train,dic_labels={'datetime' : 'Dates'})
 create_update_time_of_day_features(df_test,dic_labels={'datetime' : 'Dates'} )
train = pd.read_csv('/kaggle/input/titanic/train.csv')
 test = pd.read_csv('/kaggle/input/titanic/test.csv' )<count_missing_values>
df_train['year'] = df_train['Dates'].dt.year
df_train['month'] = df_train['Dates'].dt.month
df_train['day'] = df_train['Dates'].dt.day
df_train['hour'] = df_train['Dates'].dt.hour
df_train['minute'] = df_train['Dates'].dt.minute
df_train["n_days"] =(df_train['Dates'] - df_train['Dates'].min() ).apply(lambda x: x.days)
 df_train.drop("Dates", axis=1, inplace=True )
train.isnull().sum()<count_missing_values>
df_test['year'] = df_test['Dates'].dt.year
df_test['month'] = df_test['Dates'].dt.month
df_test['day'] = df_test['Dates'].dt.day
df_test['hour'] = df_test['Dates'].dt.hour
df_test['minute'] = df_test['Dates'].dt.minute
df_test['special_time'] = df_train['minute'].isin([0, 30] ).astype(int)
 df_test["n_days"] =(df_test['Dates'] - df_test['Dates'].min() ).apply(lambda x: x.days)
 df_test.drop("Dates", axis=1, inplace=True )
train.isnull().sum()<count_missing_values>
df_train["DayOfWeek"].value_counts()
test.isnull().sum()<count_missing_values>
df_train["DayOfWeek"].value_counts()
test.isnull().sum()<set_options>
df_train.drop(["Category", "Descript", "Resolution", "Address"], axis=1, inplace=True)
 df_test.drop("Address", axis=1, inplace=True )
%matplotlib inline
sns.set()<feature_engineering>
categorical_features = ["DayOfWeek", "PdDistrict", "PeriodOfDay"]
ct = ColumnTransformer(transformers=[("categorical_features", OrdinalEncoder() , categorical_features)],
remainder="passthrough")
 df_train = ct.fit_transform(df_train )
train_test_data = [train, test]
for dataset in train_test_data:
dataset['Title']= dataset['Name'].str.extract('([A-Za-z]+)\.', expand=False )<count_values>
categorical_features = ["DayOfWeek", "PdDistrict","PeriodOfDay"]
ct = ColumnTransformer(transformers=[("categorical_features", OrdinalEncoder() , categorical_features)],
remainder="passthrough")
 df_test = ct.fit_transform(df_test )
train['Title'].value_counts()<count_values>
from sklearn.model_selection import train_test_split
test['Title'].value_counts()<categorify>
from sklearn.model_selection import train_test_split
title_mapping = {"Mr": 0, "Miss": 1, "Mrs": 2,
"Master": 3, "Dr": 3, "Rev": 3, "Col": 3, "Major": 3, "Mlle": 3,"Countess": 3,
"Ms": 3, "Lady": 3, "Jonkheer": 3, "Don": 3, "Dona" : 3, "Mme": 3,"Capt": 3,"Sir": 3 }
for dataset in train_test_data:
dataset['Title'] = dataset['Title'].map(title_mapping )<drop_column>
X_train, X_test, y_train, y_test = train_test_split(df_train, y, test_size=0.25, random_state=42,stratify=y )
train.drop('Name', axis=1, inplace=True)
 test.drop('Name', axis=1, inplace=True )<categorify>
from sklearn.preprocessing import StandardScaler
sex_mapping = {"male": 0, "female": 1}
for dataset in train_test_data:
dataset['Sex'] = dataset['Sex'].map(sex_mapping )<categorify>
std = StandardScaler()
X_train_std = std.fit_transform(X_train)
 X_test_std = std.transform(X_test)
 X_predict_std = StandardScaler().fit_transform(df_test )
train["Age"].fillna(train.groupby("Title")["Age"].transform("median"), inplace=True)
 test["Age"].fillna(test.groupby("Title")["Age"].transform("median"), inplace=True )<categorify>
cbc = catboost.CatBoostClassifier(n_estimators=5000, learning_rate=0.05,
random_seed=0, task_type="GPU", verbose=50 )
train.groupby("Title")["Age"].transform("median" )<count_missing_values>
cbc.fit(X_train_std, y_train)
 prob = cbc.predict_proba(X_test_std )
train.isnull().sum()<count_missing_values>
from sklearn.linear_model import SGDClassifier
train.isnull().sum()<feature_engineering>
SGDC = SGDClassifier(max_iter=100,tol=0.01 )
for dataset in train_test_data:
dataset.loc[ dataset['Age'] <= 16, 'Age'] = 0,
dataset.loc[(dataset['Age'] > 16)&(dataset['Age'] <= 26), 'Age'] = 1,
dataset.loc[(dataset['Age'] > 26)&(dataset['Age'] <= 36), 'Age'] = 2,
dataset.loc[(dataset['Age'] > 36)&(dataset['Age'] <= 62), 'Age'] = 3,
dataset.loc[ dataset['Age'] > 62, 'Age'] = 4<count_values>
SGDC.fit(X_train_std,y_train)
 y_pred_SGDC = SGDC.predict(X_test_std )
Pclass1 = train[train['Pclass']==1]['Embarked'].value_counts()
Pclass2 = train[train['Pclass']==2]['Embarked'].value_counts()
Pclass3 = train[train['Pclass']==3]['Embarked'].value_counts()
df = pd.DataFrame([Pclass1, Pclass2, Pclass3])
 df.index = ['1st class','2nd class', '3rd class']
df.plot(kind='bar',stacked=True, figsize=(10,5))<data_type_conversions>
from sklearn.metrics import accuracy_score
from sklearn.metrics import precision_score
from sklearn.metrics import mean_absolute_error
from sklearn.metrics import mean_squared_error
for dataset in train_test_data:
dataset['Embarked']=dataset['Embarked'].fillna('S' )<categorify>
print(' MAE|',mean_absolute_error(y_test,y_pred_SGDC))
print(' MSE|',mean_squared_error(y_test,y_pred_SGDC))
print(' RMSE|',np.sqrt(mean_squared_error(y_test,y_pred_SGDC)) )
embarked_mapping = {"S": 0, "C": 1, "Q": 2}
for dataset in train_test_data:
dataset['Embarked'] = dataset['Embarked'].map(embarked_mapping )<categorify>
y_train = np.float64(y_train )
train["Fare"].fillna(train.groupby("Pclass")["Fare"].transform("median"), inplace=True)
 test["Fare"].fillna(test.groupby("Pclass")["Fare"].transform("median"), inplace=True)
 train.head(50 )<feature_engineering>
from sklearn.ensemble import RandomForestClassifier
for dataset in train_test_data:
dataset.loc[ dataset['Fare'] <= 17, 'Fare'] = 0,
dataset.loc[(dataset['Fare'] > 17)&(dataset['Fare'] <= 30), 'Fare'] = 1,
dataset.loc[(dataset['Fare'] > 30)&(dataset['Fare'] <= 100), 'Fare'] = 2,
dataset.loc[ dataset['Fare'] > 100, 'Fare'] = 3<count_values>
RF = RandomForestClassifier(n_estimators=50,criterion='entropy',max_depth=10)
 RF.fit(X_train_std,y_train )
train.Cabin.value_counts()<feature_engineering>
y_pred_RF = RF.predict(X_test_std )
for dataset in train_test_data:
dataset['Cabin']=dataset['Cabin'].str[:1]<categorify>
print(' Accuracy|',accuracy_score(y_test,y_pred_RF))
print('Precision|',precision_score(y_test,y_pred_RF,average='macro'))
print(' MAE|',mean_absolute_error(y_test,y_pred_RF))
print(' MSE|',mean_squared_error(y_test,y_pred_RF))
print(' RMSE|',np.sqrt(mean_squared_error(y_test,y_pred_RF)))
cabin_mapping = {"A": 0, "B": 0.4, "C": 0.8, "D": 1.2, "E": 1.6, "F": 2, "G": 2.4, "T": 2.8}
for dataset in train_test_data:
dataset['Cabin'] = dataset['Cabin'].map(cabin_mapping )<categorify>
RF2 = RandomForestClassifier(n_estimators=50,criterion='entropy',max_depth=10)
 RF2.fit(X_train_std,y_train )
train["Cabin"].fillna(train.groupby("Pclass")["Cabin"].transform("median"), inplace=True)
 test["Cabin"].fillna(test.groupby("Pclass")["Cabin"].transform("median"), inplace=True )<feature_engineering>
from keras.models import Sequential
from keras.layers import SimpleRNN, Embedding, Dense
train["FamilySize"] = train["SibSp"] + train["Parch"] + 1
test["FamilySize"] = test["SibSp"] + test["Parch"] + 1<categorify>
model = Sequential()
family_mapping = {1: 0, 2: 0.4, 3: 0.8, 4: 1.2, 5: 1.6, 6: 2, 7: 2.4, 8: 2.8, 9: 3.2, 10: 3.6, 11: 4}
for dataset in train_test_data:
dataset['FamilySize'] = dataset['FamilySize'].map(family_mapping )<drop_column>
model.add(Dense(26,input_dim=26, activation='relu'))
model.add(Dense(13,activation='relu'))
model.add(Dense(6,activation='relu'))
model.add(Dense(1,activation='relu'))
features_drop = ['Ticket', 'SibSp', 'Parch']
train = train.drop(features_drop, axis=1)
 test = test.drop(features_drop, axis=1)
 train = train.drop(['PassengerId'], axis=1 )<import_modules>
model.compile(optimizer='sgd',loss='mse',metrics=['mae','acc'] )
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.svm import SVC
import numpy as np<choose_model_class>
from keras.callbacks import EarlyStopping
k_fold = KFold(n_splits=10, shuffle=True, random_state=0 )<compute_train_metric>
callback_early = EarlyStopping(min_delta=0.01,patience=20 )
clf = KNeighborsClassifier(n_neighbors = 13)
 scoring = 'accuracy'
score = cross_val_score(clf, train_data, target, cv=k_fold, n_jobs=1, scoring=scoring)
 print(score )<compute_test_metric>
history = model.fit(x=X_train_std,y=y_train, batch_size=10, epochs=50,validation_split=0.1,callbacks=[callback_early] )
round(np.mean(score)*100, 2 )<compute_train_metric>
loss, mae, acc = model.evaluate(X_test_std,y_test)
 loss, mae, acc
clf = DecisionTreeClassifier()
scoring = 'accuracy'
score = cross_val_score(clf, train_data, target, cv=k_fold, n_jobs=1, scoring=scoring)
 print(score )<compute_test_metric>
ypred_RN = model.predict(X_test_std )
round(np.mean(score)*100, 2 )<compute_train_metric>
print(' MAE|',mean_absolute_error(y_test,ypred_RN))
print(' MSE|',mean_squared_error(y_test,ypred_RN))
print(' RMSE|',np.sqrt(mean_squared_error(y_test,ypred_RN)) )
clf = RandomForestClassifier(n_estimators=13)
 scoring = 'accuracy'
score = cross_val_score(clf, train_data, target, cv=k_fold, n_jobs=1, scoring=scoring)
 print(score )<compute_test_metric>
X2 = df_train.drop('Category', axis=1 ).values
y2 = df_train['Category'].values
round(np.mean(score)*100, 2 )<compute_train_metric>
X_train, X_test, y_train, y_test = train_test_split(X2, y2, test_size=0.15, random_state=42,stratify=y )
clf = GaussianNB()
scoring = 'accuracy'
score = cross_val_score(clf, train_data, target, cv=k_fold, n_jobs=1, scoring=scoring)
 print(score )<compute_test_metric>
test = df_test.drop('Id',axis=1)
round(np.mean(score)*100, 2 )<compute_train_metric>
test = test.values
clf = SVC()
scoring = 'accuracy'
score = cross_val_score(clf, train_data, target, cv=k_fold, n_jobs=1, scoring=scoring)
 print(score )<compute_test_metric>
std = StandardScaler()
X_train = std.fit_transform(X_train)
 X_test = std.transform(X_test )
round(np.mean(score)*100,2 )<predict_on_test>
test = StandardScaler().fit_transform(test);
clf = SVC()
clf.fit(train_data, target)
 test_data = test.drop("PassengerId", axis=1 ).copy()
prediction = clf.predict(test_data )<save_to_csv>
RF = RandomForestClassifier(n_estimators=50,criterion='entropy',max_depth=10 )
submission = pd.DataFrame({
"PassengerId": test["PassengerId"],
"Survived": prediction
})
 submission.to_csv('submission.csv', index=False )<load_from_csv>
RF.fit(X_train,y_train )
folder = '/kaggle/input/covid19-global-forecasting-week-3/'
submission = pd.read_csv(folder+'submission.csv',index_col=0)
 test = pd.read_csv(folder+'test.csv',index_col=0)
 test['Date'] = pd.to_datetime(test['Date'],format='%Y-%m-%d')
 test['Province_State'] = test['Province_State'].fillna(value='NaN')
 data_cases = format_kaggle(folder,'ConfirmedCases')
 data_deaths = format_kaggle(folder,'Fatalities' )<save_to_csv>
y_pred_RF = RF.predict(X_predict_std )
params_cases = fit_all(data_cases,plot=True,ylabel='Confirmed Cases',p0=1e2,prior=(8,500))
params_cases.to_csv('params_cases.csv' )<save_to_csv>
Y = df_train['Category'].astype('category' )
params_deaths = fit_all(data_deaths,plot=True,ylabel='Fatalities',p0=10,prior=(8,200))
params_deaths.to_csv('params_deaths.csv' )<feature_engineering>
np.count_nonzero(Y.cat.categories.unique()), np.count_nonzero(np.unique(y_pred_RF))
for item in submission.index:
country = test.loc[item,'Country_Region']
province = test.loc[item,'Province_State']
t_abs = test.loc[item,'Date']
t =(t_abs-tref)/pd.to_timedelta(1,unit='days')
 th, logK, sigma = params_cases[['th','logK','sigma']].loc[country,province]
if not np.isnan(th):
tau =(t-th)/(np.sqrt(2)*sigma)
 submission.loc[item,'ConfirmedCases'] = np.exp(logK)*(1+erf(tau)) /2
else:
if t_abs in data_cases.index:
submission.loc[item,'ConfirmedCases'] = data_cases[country,province].loc[t_abs]
else:
submission.loc[item,'ConfirmedCases'] = data_cases[country,province].max()
th, logK, sigma = params_deaths[['th','logK','sigma']].loc[country,province]
if not np.isnan(th):
tau =(t-th)/(np.sqrt(2)*sigma)
 submission.loc[item,'Fatalities'] = np.exp(logK)*(1+erf(tau)) /2
else:
if t_abs in data_deaths.index:
submission.loc[item,'Fatalities'] = data_deaths[country,province].loc[t_abs]
else:
submission.loc[item,'Fatalities'] = data_deaths[country,province].max()
submission.to_csv('submission.csv' )<define_variables>
submit = pd.DataFrame({'Id': df_test.Id.tolist() })
 for category in Y.cat.categories:
submit[category] = np.where(y_pred_RF == category, 1, 0 )
warnings.filterwarnings('ignore')
 daybasecount = 4
baseday = 98 - float(daybasecount-1)/2.
 exponent = 1./float(daybasecount)
 fatalityBaseDayShift = 10
maxincrease = 140
maxDeadPrDay = 1500<load_from_csv>
submit.to_csv('San_Francisco_Crime_RF.csv', index = False )