Kaggle competition: San Francisco Crime Classification
daily_log_confirmed = train_clean.pivot('Location', 'Date', 'LogConfirmed' ).reset_index()
daily_log_confirmed = daily_log_confirmed.sort_values(TRAIN_END, ascending=False)
 daily_log_confirmed.to_csv('daily_log_confirmed.csv', index=False)
 for i, d in tqdm(enumerate(pd.date_range(add_days(TRAIN_END, 1), add_days(TEST_END, 1)))) :
new_day = str(d ).split(' ')[0]
last_day = dt.datetime.strptime(new_day, DATEFORMAT)- dt.timedelta(days=1)
 last_day = last_day.strftime(DATEFORMAT)
 for loc in confirmed_deltas.Location.values:
confirmed_delta = confirmed_deltas.loc[confirmed_deltas.Location == loc, 'DELTA'].values[0]
daily_log_confirmed.loc[daily_log_confirmed.Location == loc, new_day] = daily_log_confirmed.loc[daily_log_confirmed.Location == loc, last_day] + \
confirmed_delta * DECAY ** i<save_to_csv>
train_data = pd.read_csv(".. /input/train.csv")
 train_data.head()
print("train_data shape: ", train_data.shape )
train_clean['Geo
latest = train_clean[train_clean.Date == TRAIN_END][[
'Geo
daily_death_deltas = train_clean[train_clean.Date >= '2020-03-17'].pivot(
 'Geo
daily_death_deltas = latest.merge(daily_death_deltas, on='Geo
daily_death_deltas.shape
daily_death_deltas.head()
daily_death_deltas.to_csv('daily_death_deltas.csv', index=False )<feature_engineering>
X = train_data.drop(train_data.columns[[1, 2, 5, 6]], axis = 1)
 X.head()
death_deltas = train.groupby(['Location', 'Country_Region', 'continent'])[[
'Id']].count().reset_index()
GLOBAL_DELTA = 0.07
death_deltas['DELTA'] = GLOBAL_DELTA
death_deltas.loc[death_deltas.continent=='Oceania', 'DELTA'] = 0.06
death_deltas.loc[death_deltas.continent=='Africa', 'DELTA'] = 0.06
death_deltas.loc[death_deltas.Country_Region=='China', 'DELTA'] = 0.005
death_deltas.loc[death_deltas.Country_Region=='Korea South', 'DELTA'] = 0.025
death_deltas.loc[death_deltas.Country_Region=='Japan', 'DELTA'] = 0.05
death_deltas.loc[death_deltas.Country_Region=='Singapore', 'DELTA'] = 0.05
death_deltas.loc[death_deltas.Country_Region=='Taiwan*', 'DELTA'] = 0.03
death_deltas.loc[death_deltas.Country_Region=='US', 'DELTA'] = 0.12
death_deltas.loc[death_deltas.Country_Region=='Switzerland', 'DELTA'] = 0.07
death_deltas.loc[death_deltas.Country_Region=='Norway', 'DELTA'] = 0.10
death_deltas.loc[death_deltas.Country_Region=='Sweden', 'DELTA'] = 0.15
death_deltas.loc[death_deltas.Country_Region=='Belgium', 'DELTA'] = 0.12
death_deltas.loc[death_deltas.Country_Region=='Netherlands', 'DELTA'] = 0.10
death_deltas.loc[death_deltas.Country_Region=='Poland', 'DELTA'] = 0.13
death_deltas.loc[death_deltas.Country_Region=='Hungary', 'DELTA'] = 0.13
death_deltas.loc[death_deltas.Country_Region=='Romania', 'DELTA'] = 0.11
death_deltas.loc[death_deltas.Country_Region=='Iceland', 'DELTA'] = 0.02
death_deltas.loc[death_deltas.Country_Region=='Austria', 'DELTA'] = 0.08
death_deltas.loc[death_deltas.Country_Region=='Italy', 'DELTA'] = 0.045
death_deltas.loc[death_deltas.Country_Region=='Spain', 'DELTA'] = 0.045
death_deltas.loc[death_deltas.Country_Region=='Portugal', 'DELTA'] = 0.09
death_deltas.loc[death_deltas.Country_Region=='Ireland', 'DELTA'] = 0.1
death_deltas.loc[death_deltas.Country_Region=='Israel', 'DELTA'] = 0.1
death_deltas.loc[death_deltas.Country_Region=='Iran', 'DELTA'] = 0.05
death_deltas.loc[death_deltas.Country_Region=='Germany', 'DELTA'] = 0.10
death_deltas.loc[death_deltas.Country_Region=='Malaysia', 'DELTA'] = 0.05
death_deltas.loc[death_deltas.Country_Region=='Russia', 'DELTA'] = 0.13
death_deltas.loc[death_deltas.Country_Region=='Ukraine', 'DELTA'] = 0.13
death_deltas.loc[death_deltas.Country_Region=='Brazil', 'DELTA'] = 0.15
death_deltas.loc[death_deltas.Country_Region=='Turkey', 'DELTA'] = 0.16
death_deltas.loc[death_deltas.Country_Region=='Philippines', 'DELTA'] = 0.08
death_deltas.loc[death_deltas.Country_Region=='Saudi Arabia', 'DELTA'] = 0.1
death_deltas.loc[death_deltas.Location=='France-', 'DELTA'] = 0.08
death_deltas.loc[death_deltas.Location=='United Kingdom-', 'DELTA'] = 0.12
death_deltas.loc[death_deltas.Location=='Diamond Princess-', 'DELTA'] = 0.00
death_deltas.loc[death_deltas.Location=='China-Hong Kong', 'DELTA'] = 0.01
death_deltas.loc[death_deltas.Location=='San Marino-', 'DELTA'] = 0.04
death_deltas.loc[death_deltas.Location=='Netherlands-', 'DELTA'] = 0.09
death_deltas.loc[death_deltas.Country_Region=='Mexico', 'DELTA'] = 0.18
death_deltas.loc[death_deltas.Country_Region=='Peru', 'DELTA'] = 0.18
death_deltas.loc[death_deltas.Country_Region=='India', 'DELTA'] = 0.14
death_deltas.loc[death_deltas.Country_Region=='Pakistan', 'DELTA'] = 0.16
death_deltas.shape
death_deltas.DELTA.mean()
death_deltas[death_deltas.DELTA != GLOBAL_DELTA].shape
death_deltas[death_deltas.DELTA != GLOBAL_DELTA].DELTA.mean()
death_deltas[death_deltas.DELTA != GLOBAL_DELTA]
death_deltas.describe()<save_to_csv>
y = train_data.iloc[:, 1]
y.head()
daily_log_deaths = train_clean.pivot('Location', 'Date', 'LogFatalities' ).reset_index()
daily_log_deaths = daily_log_deaths.sort_values(TRAIN_END, ascending=False)
 daily_log_deaths.to_csv('daily_log_deaths.csv', index=False)
 for i, d in tqdm(enumerate(pd.date_range(add_days(TRAIN_END, 1), add_days(TEST_END, 1)))) :
new_day = str(d ).split(' ')[0]
last_day = dt.datetime.strptime(new_day, DATEFORMAT)- dt.timedelta(days=1)
 last_day = last_day.strftime(DATEFORMAT)
 for loc in death_deltas.Location:
death_delta = death_deltas.loc[death_deltas.Location == loc, 'DELTA'].values[0]
daily_log_deaths.loc[daily_log_deaths.Location == loc, new_day] = daily_log_deaths.loc[daily_log_deaths.Location == loc, last_day] + \
death_delta * DECAY ** i<feature_engineering>
def preprocess_data(dataset):
dataset['Dates'] = pd.to_datetime(dataset['Dates'])
 dataset['Month'] = dataset.Dates.apply(lambda x: x.month)
 dataset['Day'] = dataset.Dates.apply(lambda x: x.day)
 dataset['Hour'] = dataset.Dates.apply(lambda x: x.hour)
 dataset['Minute'] = dataset.Dates.apply(lambda x: x.minute)
 dataset = dataset.drop('Dates', 1)
 dataset = pd.get_dummies(data=dataset, columns=['DayOfWeek', 'PdDistrict'])
 return dataset
confirmed = []
fatalities = []
for id, d, loc in tqdm(test[['ForecastId', 'Date', 'Location']].values):
c = to_exp(daily_log_confirmed.loc[daily_log_confirmed.Location == loc, d].values[0])
 f = to_exp(daily_log_deaths.loc[daily_log_deaths.Location == loc, d].values[0])
 confirmed.append(c)
 fatalities.append(f )<prepare_output>
y = y.to_frame()
le = LabelEncoder()
y["Category"] = le.fit_transform(y["Category"])
 print(y.head())
my_submission = test.copy()
my_submission['ConfirmedCases'] = confirmed
my_submission['Fatalities'] = fatalities
my_submission.shape
my_submission.head() 
<save_to_csv>
keys = le.classes_
values = le.transform(le.classes_)
 dictionary = dict(zip(keys, values))
print(dictionary )
my_submission[[
'ForecastId', 'ConfirmedCases', 'Fatalities'
]].to_csv('submission.csv', index=False)
 print(DECAY)
 my_submission.head()
my_submission.tail()
my_submission.shape<train_model>
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 0)
end = dt.datetime.now()
print('Finished', end,(end - start ).seconds, 's' )<import_modules>
rf = RandomForestClassifier(max_depth = 10, n_estimators = 256)
 rf.fit(X_train.values, y_train.values.ravel() )
from sklearn.model_selection import train_test_split
import random
from sklearn.impute import SimpleImputer
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import LabelEncoder
from sklearn.compose import ColumnTransformer
from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import GridSearchCV<import_modules>
y_pred = rf.predict(X_test)
 y_pred.shape
y_test.shape
print("Train Accuracy: ", accuracy_score(y_train, rf.predict(X_train)))
 print("Test Accuracy: ", accuracy_score(y_test, y_pred))
import itertools
import matplotlib.pyplot as plt
import lightgbm as lgb
from sklearn import metrics<import_modules>
print(classification_report(y_test, y_pred))
from sklearn.feature_selection import SelectKBest, f_classif<import_modules>
test_data = pd.read_csv('.. /input/test.csv')
 test_data.head()
import itertools<categorify>
test_data = preprocess_data(test_data)
 test_data.head()
def Feature_selection(X):
cat_features = [y for y in X.columns if X[y].dtypes == 'object']
cat_features = [col for col in cat_features if col not in ['Name', 'Ticket', 'Cabin']]
interactions = pd.DataFrame(index=X.index)
 for col1, col2 in itertools.combinations(cat_features, 2):
new_col_name = '_'.join([col1, col2])
 new_values = X[col1].map(str)+ "_" + X[col2].map(str)
 encoder = LabelEncoder()
X[new_col_name] = encoder.fit_transform(new_values)
 return X<import_modules>
test_data = test_data.drop('Id', 1)
 test_data = test_data.drop('Address', 1 )
import category_encoders as ce<categorify>
y_pred_proba = rf.predict_proba(test_data )
<categorify><EOS> 
colmn = ["ARSON","ASSAULT","BAD CHECKS","BRIBERY","BURGLARY","DISORDERLY CONDUCT","DRIVING UNDER THE INFLUENCE","DRUG/NARCOTIC","DRUNKENNESS","EMBEZZLEMENT","EXTORTION","FAMILY OFFENSES","FORGERY/COUNTERFEITING","FRAUD","GAMBLING","KIDNAPPING","LARCENY/THEFT","LIQUOR LAWS","LOITERING","MISSING PERSON","NON-CRIMINAL","OTHER OFFENSES","PORNOGRAPHY/OBSCENE MAT","PROSTITUTION","RECOVERED VEHICLE","ROBBERY","RUNAWAY","SECONDARY CODES","SEX OFFENSES FORCIBLE","SEX OFFENSES NON FORCIBLE","STOLEN PROPERTY","SUICIDE","SUSPICIOUS OCC","TREA","TRESPASS","VANDALISM","VEHICLE THEFT","WARRANTS","WEAPON LAWS"]
result = pd.DataFrame(y_pred_proba, columns=colmn)
 result.to_csv(path_or_buf="rf_predict.csv",index=True, index_label = 'Id' )