Kaggle competition: San Francisco Crime Classification
cols =('ExterCond','HeatingQC', 'BsmtFinType1',
'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope',
'LotShape', 'PavedDrive', 'CentralAir', 'MSSubClass', 'OverallCond',
'YrSold', 'MoSold','GarageYrBlt','YearBuilt','YearRemodAdd', 'BsmtHalfBath','BsmtFullBath', 'GarageCars')
 for c in cols:
lbl = LabelEncoder()
lbl.fit(list(all_data[c].values))
all_data[c] = lbl.transform(list(all_data[c].values))<data_type_conversions>
pd.set_option("display.max_columns", None)
 %matplotlib inline
all_data.FireplaceQu = all_data.FireplaceQu.astype('category', ordered=True, categories=['None','Po','Fa','TA','Gd','Ex'] ).cat.codes
all_data.BsmtQual = all_data.BsmtQual.astype('category', ordered=True, categories=['None','Fa','TA','Gd','Ex'] ).cat.codes
all_data.BsmtCond = all_data.BsmtCond.astype('category', ordered=True, categories=['None','Po','Fa','TA','Gd'] ).cat.codes
all_data.GarageQual = all_data.GarageQual.astype('category', ordered=True, categories=['None','Po','Fa','TA','Gd','Ex'] ).cat.codes
all_data.GarageCond = all_data.GarageCond.astype('category', ordered=True, categories=['None','Po','Fa','TA','Gd','Ex'] ).cat.codes
all_data.ExterQual = all_data.ExterQual.astype('category', ordered=True, categories=['Fa','TA','Gd','Ex'] ).cat.codes
all_data.KitchenQual = all_data.KitchenQual.astype('category', ordered=True, categories=['Fa','TA','Gd','Ex'] ).cat.codes<feature_engineering>
dateparse = lambda x: pd.datetime.strptime(x, '%Y-%m-%d %H:%M:%S')
 train = pd.read_csv('/kaggle/input/sf-crime/train.csv.zip',parse_dates=['Dates'], date_parser=dateparse)
 test = pd.read_csv('/kaggle/input/sf-crime/test.csv.zip',parse_dates=['Dates'], date_parser=dateparse)
 print('done' )
for feat in skewness[abs(skewness)>0.5].index:
all_data[feat] = boxcox1p(all_data[feat], 0.15 )<prepare_x_and_y>
print(train.duplicated().sum())
 train.drop_duplicates(inplace=True)
 assert train.duplicated().sum() ==0
print('done' )
all_data = pd.get_dummies(all_data)
 all_data.drop(["Neighborhood_Veenker", "RoofMatl_WdShngl", "RoofStyle_Shed" , "SaleCondition_Partial" , "SaleType_WD"], axis=1, inplace=True)
 train_data = all_data[:train_data.shape[0]]
test_data = all_data[train_data.shape[0]:]
y_train = transform_log<import_modules>
print(train.columns.difference(test.columns))
from sklearn.linear_model import ElasticNet, Lasso, BayesianRidge, LassoLarsIC
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.kernel_ridge import KernelRidge
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import RobustScaler
from sklearn.base import BaseEstimator, TransformerMixin, RegressorMixin, clone
from sklearn.model_selection import KFold, cross_val_score, train_test_split
from sklearn.metrics import mean_squared_error
import xgboost as xgb
import lightgbm as lgb<compute_train_metric>
print(train_gdf.loc[train_gdf.Y > 50].count() [0])
n_folds = 10
def rmsle_cv(model):
kf = KFold(n_folds, shuffle=True, random_state=42 ).get_n_splits(train_data.values)
 rmse= np.sqrt(-cross_val_score(model, train_data.values, y_train, scoring="neg_mean_squared_error", cv = kf))
return(rmse )<compute_train_metric>
train.replace({'X': -120.5, 'Y': 90.0}, np.NaN, inplace=True)
 test.replace({'X': -120.5, 'Y': 90.0}, np.NaN, inplace=True)
 imp = SimpleImputer(strategy='most_frequent')
 for district in train['PdDistrict'].unique() :
train.loc[train['PdDistrict'] == district, ['X', 'Y']] = imp.fit_transform(
 train.loc[train['PdDistrict'] == district, ['X', 'Y']])
 test.loc[test['PdDistrict'] == district, ['X', 'Y']] = imp.transform(
 test.loc[test['PdDistrict'] == district, ['X', 'Y']])
 train_gdf = create_gdf(train)
 print('done' )
KRR = KernelRidge(alpha=0.6, kernel='polynomial', degree=2, coef0=2.5)
 score = rmsle_cv(KRR)
 print("Kernel Ridge score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<compute_test_metric>
dateparse = lambda x: pd.datetime.strptime(x, '%Y-%m-%d %H:%M:%S')
 train = pd.read_csv('/kaggle/input/sf-crime/train.csv.zip',parse_dates=['Dates'], date_parser=dateparse)
 test = pd.read_csv('/kaggle/input/sf-crime/test.csv.zip',parse_dates=['Dates'], date_parser=dateparse)
 print('done' )
lasso = make_pipeline(RobustScaler() , Lasso(alpha =0.0005, random_state=1))
score = rmsle_cv(lasso)
 print("Lasso score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<compute_train_metric>
train.replace({'X': -120.5, 'Y': 90.0}, np.NaN, inplace=True)
 test.replace({'X': -120.5, 'Y': 90.0}, np.NaN, inplace=True)
 imp = SimpleImputer(strategy='most_frequent')
 for district in train['PdDistrict'].unique() :
train.loc[train['PdDistrict'] == district, ['X', 'Y']] = imp.fit_transform(
 train.loc[train['PdDistrict'] == district, ['X', 'Y']])
 test.loc[test['PdDistrict'] == district, ['X', 'Y']] = imp.transform(
 test.loc[test['PdDistrict'] == district, ['X', 'Y']])
 print('done')
ENet = make_pipeline(RobustScaler() , ElasticNet(alpha=0.0005, l1_ratio=.9, random_state=3))
score = rmsle_cv(ENet)
 print("ElasticNet score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<compute_train_metric>
y_train = train['Category']
train_des = train['Descript']
train_res = train['Resolution']
train.drop(["Category", "Descript", "Resolution"], axis=1, inplace=True)
 print('done')
GBoost = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05,
max_depth=4, max_features='sqrt',
min_samples_leaf=15, min_samples_split=10,
loss='huber', random_state =5)
 score = rmsle_cv(GBoost)
 print("Gradient Boosting score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<choose_model_class>
test_ID = test["Id"]
test.drop("Id", axis=1, inplace=True)
 print('done' )
model_xgb = xgb.XGBRegressor(colsample_bytree=0.4603, gamma=0.0468,
learning_rate=0.05, max_depth=3,
min_child_weight=1.7817, n_estimators=2200,
reg_alpha=0.4640, reg_lambda=0.8571,
subsample=0.5213, silent=1,
random_state =7, nthread = -1)
 score = rmsle_cv(model_xgb)
 print("Xgboost score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<choose_model_class>
le = LabelEncoder()
y_train = le.fit_transform(y_train)
 print(le.classes_)
 print('done' )
model_lgb = lgb.LGBMRegressor(objective='regression',num_leaves=5,
learning_rate=0.05, n_estimators=720,
max_bin = 55, bagging_fraction = 0.8,
bagging_freq = 5, feature_fraction = 0.2319,
feature_fraction_seed=9, bagging_seed=9,
min_data_in_leaf =6, min_sum_hessian_in_leaf = 11)
 score = rmsle_cv(model_lgb)
 print("LGBM score: {:.4f}({:.4f})
 ".format(score.mean() , score.std()))<train_model>
LassoMd = lasso.fit(train_data.values,y_train)
 ENetMd = ENet.fit(train_data.values,y_train)
 KRRMd = KRR.fit(train_data.values,y_train)
 GBoostMd = GBoost.fit(train_data.values,y_train)
 XGBMd = model_xgb.fit(train_data.values, y_train)
 LGBMd = model_lgb.fit(train_data.values, y_train )<predict_on_test>
num_train = train.shape[0]
all_data = pd.concat(( train, test), ignore_index=True)
 print('done' )
finalMd =(np.expm1(LassoMd.predict(test_data.values)) +
np.expm1(ENetMd.predict(test_data.values)) +
np.expm1(KRRMd.predict(test_data.values)) +
np.expm1(GBoostMd.predict(test_data.values)) +
np.expm1(XGBMd.predict(test_data.values)) +
np.expm1(LGBMd.predict(test_data.values)))/ 6
finalMd<save_to_csv>
def feature_engineering(data):
date = pd.to_datetime(all_data['Dates'])
 data['year'] = date.dt.year
data['month'] = date.dt.month
data['day'] = date.dt.day
data['hour'] = date.dt.hour
data['minute'] = date.dt.minute
data['special_time'] = data['minute'].isin([0, 30] ).astype(int)
 data["n_days"] =(date - date.min() ).apply(lambda x: x.days)
 data.drop("Dates", axis=1, inplace=True)
 data['block'] = data["Address"].str.contains("block", case=False)
 data['ST'] = data['Address'].str.contains('ST', case=False)
 return data
all_data1 = feature_engineering(all_data)
 print('done' )
my_submission = pd.DataFrame({'Id': test_ID, 'SalePrice': finalMd})
 my_submission.to_csv('submission.csv', index=False )<set_options>
def feature_engineering2(data):
data["X+Y"] = data["X"] + data["Y"]
data["X-Y"] = data["X"] - data["Y"]
data["XY30_1"] = data["X"] * np.cos(np.pi / 6)+ data["Y"] * np.sin(np.pi / 6)
 data["XY30_2"] = data["Y"] * np.cos(np.pi / 6)- data["X"] * np.sin(np.pi / 6)
 data["XY60_1"] = data["X"] * np.cos(np.pi / 3)+ data["Y"] * np.sin(np.pi / 3)
 data["XY60_2"] = data["Y"] * np.cos(np.pi / 3)- data["X"] * np.sin(np.pi / 3)
 data["XY1"] =(data["X"] - data["X"].min())** 2 +(data["Y"] - data["Y"].min())** 2
data["XY2"] =(data["X"].max() - data["X"])** 2 +(data["Y"] - data["Y"].min())** 2
data["XY3"] =(data["X"] - data["X"].min())** 2 +(data["Y"].max() - data["Y"])** 2
data["XY4"] =(data["X"].max() - data["X"])** 2 +(data["Y"].max() - data["Y"])** 2
pca = PCA(n_components=2 ).fit(data[["X", "Y"]])
 XYt = pca.transform(data[["X", "Y"]])
 data["XYpca1"] = XYt[:, 0]
data["XYpca2"] = XYt[:, 1]
clf = GaussianMixture(n_components=150, covariance_type="diag",random_state=0 ).fit(data[["X", "Y"]])
 data["XYcluster"] = clf.predict(data[["X", "Y"]])
 return data
all_data2 = feature_engineering2(all_data1)
 print('done' )
sns.set(style='whitegrid', palette='muted', font_scale=1.5)
 rcParams['figure.figsize'] = 10, 6
RANDOM_SEED = 42
np.random.seed(RANDOM_SEED)
 def print_metrics(y_true,y_pred):
conf_mx = confusion_matrix(y_true,y_pred)
 print("------------------------------------------")
 print(" Accuracy : ", accuracy_score(y_true,y_pred))
print(" Precision : ", precision_score(y_true,y_pred))
print(" Sensitivity : ", recall_score(y_true,y_pred))
print("------------------------------------------")
 print(classification_report(y_true, y_pred))
print("------------------------------------------")
 class_names = [0,1]
fig,ax = plt.subplots()
tick_marks = np.arange(len(class_names))
plt.xticks(tick_marks,class_names)
 plt.yticks(tick_marks,class_names)
 sns.heatmap(pd.DataFrame(conf_mx),annot=True,cmap="Blues",fmt="d",cbar=False)
 ax.xaxis.set_label_position('top')
 plt.tight_layout()
plt.ylabel('Actual label')
 plt.xlabel('Predicted label')
 plt.show()<load_from_csv>
vectorizer = TfidfVectorizer(lowercase=True, preprocessor=None,tokenizer=lambda x : x.split() , analyzer='word', stop_words=None, token_pattern='(?u)\b\w\w+\b', ngram_range=(1, 3), max_df=0.75, min_df=5, max_features=12500)
 train_vect = vectorizer.fit_transform(train['Address'])
 test_vect = vectorizer.transform(test['Address'])
 print('done')
train_df = pd.read_csv('/kaggle/input/titanic/train.csv')
 test_df = pd.read_csv('/kaggle/input/titanic/test.csv')
 submit_df = pd.read_csv('/kaggle/input/titanic/gender_submission.csv')
 train_df.head(10 )<feature_engineering>
sentences = []
for s in all_data2["Address"]:
sentences.append(s.split(" "))
address_model = gensim.models.Word2Vec(sentences, min_count=1)
 encoded_address = np.zeros(( all_data2.shape[0], 100))
for i in range(len(sentences)) :
for j in range(len(sentences[i])) :
encoded_address[i] += address_model.wv[sentences[i][j]]
encoded_address[i] /= len(sentences[i])
 print('done' )
def feature_eng(df,columnshoice):
df['TicketLetter'] = df['Ticket'].apply(lambda x : str(x)[0])
 df['TicketLetter'] = df['TicketLetter'].apply(lambda x : re.sub('[0-9]','N',x))
df['Title'] = df.Name.apply(lambda name: name.split(',')[1].split('.')[0].strip())
 normalized_titles = {
"Capt": "Officer",
"Col": "Officer",
"Major": "Officer",
"Jonkheer": "Royalty",
"Don": "Royalty",
"Sir" : "Royalty",
"Dr": "Officer",
"Rev": "Officer",
"the Countess":"Royalty",
"Dona": "Royalty",
"Mme": "Mrs",
"Mlle": "Miss",
"Ms": "Mrs",
"Mr" : "Mr",
"Mrs" : "Mrs",
"Miss" : "Miss",
"Master" : "Master",
"Lady" : "Royalty"
}
df['Title'] = df['Title'].map(normalized_titles)
 df['NameLength'] = df['Name'].apply(lambda x : len(x))
df['NameLength'] =(( df.NameLength)/15 ).astype(np.int64)+1
df["FamilySize"] = df['SibSp'] + df['Parch'] + 1
bins = [-1,1,4, np.inf]
labels = ['ONE','SMALL','BIG']
df['FamilyGroup'] = pd.cut(df["FamilySize"], bins, labels = labels)
 df['IsAlone'] = 'Y'
df.loc[df['FamilySize'] > 1,'IsAlone'] = 'N'
df["Embarked"] = df["Embarked"].fillna("S")
 df["Age"] = df.groupby(['Sex','Title'])["Age"].transform(lambda x: x.fillna(x.median()))
df["Cabin"] = df["Cabin"].str[0:1]
df["Cabin"] = df["Cabin"].fillna('T')
 df['Fare'] = df['Fare'].fillna(-1)
 return df[columnshoice]
columnshoice = ['Pclass', 'Sex', 'Age', 'SibSp',
'Parch', 'Fare', 'Embarked', 'TicketLetter', 'Title',
'FamilySize', 'IsAlone','NameLength']
y = train_df["Survived"]
X = feature_eng(train_df,columnshoice)
 test = feature_eng(test_df,columnshoice)
 print(X.columns)
 print(X.shape)
 numerical_features = list(X.select_dtypes(include=['int64', 'float64', 'int32'] ).columns)
 scaler = preprocessing.StandardScaler()
X[numerical_features] = scaler.fit_transform(X[numerical_features])
 test[numerical_features] = scaler.transform(test[numerical_features])
 X = pd.get_dummies(X)
 test = pd.get_dummies(test )<choose_model_class>
all_data2.drop(['Address'],axis=1,inplace=True)
 categorical_features = ["DayOfWeek", "PdDistrict", "block", "special_time", "XYcluster",'ST']
ct = ColumnTransformer(transformers=[("categorical_features", OrdinalEncoder() , categorical_features)],
remainder="passthrough")
 all_data3 = ct.fit_transform(all_data2)
 print('done' )
model = tf.keras.models.Sequential([
tf.keras.layers.Dense(units=64, activation='relu',input_dim=X.shape[1]),
tf.keras.layers.Dense(units=32, activation='relu'),
tf.keras.layers.Dense(units=32, activation='relu'),
tf.keras.layers.Dense(units=1, activation='sigmoid')
 ])
 early_stop = keras.callbacks.EarlyStopping(
 monitor='val_loss',
patience=5 
)
 model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['accuracy'])
<train_model>
all_data4 = np.hstack(( all_data3,encoded_address))
print('done' )
history = model.fit(
 x=X,
y=y,
shuffle=True,
epochs=100,
validation_split=0.1,
verbose=0,
callbacks=[early_stop]
)<predict_on_test>
train2 = all_data3[:num_train]
test2 = all_data3[num_train:]
print('done' )
predictions = model.predict(X)
 predictions = tf.round(predictions ).numpy().flatten().astype(int )<compute_test_metric>
train_2 = sparse.hstack(( train2.astype(float),train_vect))
test_2 = sparse.hstack(( test2.astype(float),test_vect))
print('done' )
print_metrics(y,predictions)
<save_to_csv>
model = LGBMClassifier(objective="multiclass", num_class=39, max_bin = 465, max_delta_step = 0.9,
learning_rate=0.4, num_leaves = 42, n_estimators=100, verbose=50)
 model.fit(train2, y_train)
 preds = model.predict_proba(test2)
 submission = pd.DataFrame(preds, columns=le.inverse_transform(np.linspace(0, 38, 39, dtype='int16')) , index=test.index)
 submission.to_csv('LGBM_final1.csv', index_label='Id')
 print('done' )
y_sub= model.predict(test)
 y_sub = tf.round(y_sub ).numpy().flatten().astype(int)
 submission = pd.DataFrame({
"PassengerId": test_df["PassengerId"],
"Survived": y_sub
})
 submission.to_csv('titanic.csv', index=False )<categorify>
model2 = LGBMClassifier(objective="multiclass", num_class=39, max_bin = 465, max_delta_step = 0.9,
learning_rate=0.4, num_leaves = 42, n_estimators=100, verbose=50)
 model2.fit(train_2, y_train)
 preds2 = model2.predict_proba(test_2)
 submission2 = pd.DataFrame(preds2, columns=le.inverse_transform(np.linspace(0, 38, 39, dtype='int16')) , index=test.index)
 submission2.to_csv('LGBM_final2.csv', index_label='Id')
 print('done' )
<load_from_csv><EOS> 
sub_tot =(preds+preds2+prob)/3
submission4 = pd.DataFrame(sub_tot, columns=le.inverse_transform(np.linspace(0, 38, 39, dtype='int16')) , index=test.index)
 submission4.to_csv('final.csv', index_label='Id')
 print('done' )