Kaggle competition: Titanic - Machine Learning from Disaster
train.isnull().sum()<count_missing_values>
train_data = pd.read_csv("/kaggle/input/titanic/train.csv")
 test_data = pd.read_csv("/kaggle/input/titanic/test.csv" )
train.isnull().sum()<feature_engineering>
Col_with_missing = [col for col in train_data.columns if train_data[col].isnull().any() ]
print(Col_with_missing )
pd.options.display.max_columns=100
train = pd.read_csv('.. /input/train.csv', parse_dates=['Dates'])
 test = pd.read_csv('.. /input/test.csv', parse_dates=['Dates'], index_col='Id')
 def feature_engineering(data):
data['Date'] = pd.to_datetime(data['Dates'].dt.date)
 data['n_days'] =(data['Date'] - data['Date'].min() ).apply(lambda x: x.days)
 data['Day'] = data['Dates'].dt.day
data['DayOfWeek'] = data['Dates'].dt.weekday
data['Month'] = data['Dates'].dt.month
data['Year'] = data['Dates'].dt.year
data['Hour'] = data['Dates'].dt.hour
data['Minute'] = data['Dates'].dt.minute
data['Block'] = data['Address'].str.contains('block', case=False ).apply(lambda x: 1 if x == True else 0)
 data["X_Y"] = data["X"] - data["Y"]
data["XY"] = data["X"] + data["Y"]
data.drop(columns=['Dates','Date','Address'], inplace=True)
 return data
train = feature_engineering(train)
 test = feature_engineering(test)
 train.drop(columns=['Descript','Resolution'], inplace=True )<feature_engineering>
Col_with_missing = [col for col in test_data.columns if test_data[col].isnull().any() ]
print(Col_with_missing )
pd.options.display.max_columns=100
train = pd.read_csv('.. /input/train.csv', parse_dates=['Dates'])
 test = pd.read_csv('.. /input/test.csv', parse_dates=['Dates'], index_col='Id')
 def feature_engineering(data):
data['Date'] = pd.to_datetime(data['Dates'].dt.date)
 data['n_days'] =(data['Date'] - data['Date'].min() ).apply(lambda x: x.days)
 data['Day'] = data['Dates'].dt.day
data['DayOfWeek'] = data['Dates'].dt.weekday
data['Month'] = data['Dates'].dt.month
data['Year'] = data['Dates'].dt.year
data['Hour'] = data['Dates'].dt.hour
data['Minute'] = data['Dates'].dt.minute
data['Block'] = data['Address'].str.contains('block', case=False ).apply(lambda x: 1 if x == True else 0)
 data["X_Y"] = data["X"] - data["Y"]
data["XY"] = data["X"] + data["Y"]
data.drop(columns=['Dates','Date','Address'], inplace=True)
 return data
train = feature_engineering(train)
 test = feature_engineering(test)
 train.drop(columns=['Descript','Resolution'], inplace=True )<categorify>
s =(train_data.dtypes == 'object')
 object_cols = list(s[s].index)
 print("Categorical variables:")
 print(object_cols )
le1 = LabelEncoder()
train['PdDistrict'] = le1.fit_transform(train['PdDistrict'])
 test['PdDistrict'] = le1.transform(test['PdDistrict'])
 le2 = LabelEncoder()
X = train.drop(columns=['Category'])
 y= le2.fit_transform(train['Category'] )<create_dataframe>
feature_name=['Pclass','Sex','Age','SibSp','Parch','Fare','Embarked']
X=train_data[feature_name]
y=train_data["Survived"]
X_test=test_data[feature_name]
train_data = lgb.Dataset(X, label=y, categorical_feature=['PdDistrict', ])
 params = {'boosting':'gbdt',
'objective':'multiclass',
'num_class':39,
'max_delta_step':0.9,
'min_data_in_leaf': 21,
'learning_rate': 0.4,
'max_bin': 465,
'num_leaves': 41,
'verbose' : 1}
bst = lgb.train(params, train_data, 120)
 predictions = bst.predict(test)
 submission = pd.DataFrame(predictions, columns=le2.inverse_transform(np.linspace(0, 38, 39, dtype='int16')) , index=test.index)
 submission.to_csv('LGBM_final.csv', index_label='Id' )<import_modules>
X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.2 )
import pandas as pd
from shapely.geometry import Point
import geopandas as gpd
import matplotlib.pyplot as plt
import numpy as np
from sklearn.impute import SimpleImputer
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split
import seaborn as sns
from matplotlib import cm
import urllib.request
import shutil
import zipfile
import os
import re
import contextily as ctx
import geoplot as gplt
import lightgbm as lgb
import eli5
from eli5.sklearn import PermutationImportance
from lightgbm import LGBMClassifier
from matplotlib import pyplot as plt
from pdpbox import pdp, get_dataset, info_plots
import shap<load_from_csv>
my_imputer=SimpleImputer(strategy="most_frequent")
 imputed_X_train= pd.DataFrame(my_imputer.fit_transform(X_train))
imputed_X_test=pd.DataFrame(my_imputer.transform(X_test))
imputed_X_valid=pd.DataFrame(my_imputer.transform(X_valid))
imputed_X_train.index = X_train.index
imputed_X_valid.index = X_valid.index
imputed_X_test.index = X_test.index
imputed_X_train.columns=X_train.columns
imputed_X_valid.columns=X_valid.columns
imputed_X_test.columns=X_test.columns
train = pd.read_csv('.. /input/train.csv', parse_dates=['Dates'])
 test = pd.read_csv('.. /input/test.csv', parse_dates=['Dates'], index_col='Id' )<count_duplicates>
Col_with_missing_2 = [col for col in imputed_X_test.columns if imputed_X_test[col].isnull().any() ]
print(Col_with_missing_2 )
train.duplicated().sum()<categorify>
New_feature_train = imputed_X_train['Sex'] + "_" + imputed_X_train['Embarked']
New_feature_valid = imputed_X_valid['Sex'] + "_" + imputed_X_valid['Embarked']
New_feature_test = imputed_X_test['Sex'] + "_" + imputed_X_test['Embarked']
train.drop_duplicates(inplace=True)
 train.replace({'X': -120.5, 'Y': 90.0}, np.NaN, inplace=True)
 test.replace({'X': -120.5, 'Y': 90.0}, np.NaN, inplace=True)
 imp = SimpleImputer(strategy='mean')
 for district in train['PdDistrict'].unique() :
train.loc[train['PdDistrict'] == district, ['X', 'Y']] = imp.fit_transform(
 train.loc[train['PdDistrict'] == district, ['X', 'Y']])
 test.loc[test['PdDistrict'] == district, ['X', 'Y']] = imp.transform(
 test.loc[test['PdDistrict'] == district, ['X', 'Y']])
 train_gdf = create_gdf(train )<load_pretrained>
imputed_X_train["Sex_Embarked"]=New_feature_train
imputed_X_valid["Sex_Embarked"]=New_feature_valid
imputed_X_test["Sex_Embarked"]=New_feature_test
url = 'https://data.sfgov.org/api/geospatial/wkhw-cjsf?method=export&format=Shapefile'
with urllib.request.urlopen(url)as response, open('pd_data.zip', 'wb')as out_file:
shutil.copyfileobj(response, out_file)
 with zipfile.ZipFile('pd_data.zip', 'r')as zip_ref:
zip_ref.extractall('pd_data')
 for filename in os.listdir('./pd_data/'):
if re.match(".+\.shp", filename):
pd_districts = gpd.read_file('./pd_data/'+filename)
 break
pd_districts.crs={'init': 'epsg:4326'}
pd_districts = pd_districts.merge(
 train.groupby('PdDistrict' ).count().iloc[:, [0]].rename(
 columns={'Dates': 'Incidents'}),
how='inner',
left_on='district',
right_index=True,
suffixes=('_x', '_y'))
pd_districts = pd_districts.to_crs({'init': 'epsg:3857'})
 train_days = train.groupby('Date' ).count().shape[0]
pd_districts['inc_per_day'] = pd_districts.Incidents/train_days
fig, ax = plt.subplots(figsize=(10, 10))
pd_districts.plot(
 column='inc_per_day',
cmap='Reds',
alpha=0.6,
edgecolor='r',
linestyle='-',
linewidth=1,
legend=True,
ax=ax)
 def add_basemap(ax, zoom, url='http://tile.stamen.com/terrain/tileZ/tileX/tileY.png'):

 xmin, xmax, ymin, ymax = ax.axis()
basemap, extent = ctx.bounds2img(xmin, ymin, xmax, ymax, zoom=zoom, url=url)
 ax.imshow(basemap, extent=extent, interpolation='bilinear')
 ax.axis(( xmin, xmax, ymin, ymax))
add_basemap(ax, zoom=11, url=ctx.sources.ST_TONER_LITE)
 for index in pd_districts.index:
plt.annotate(
 pd_districts.loc[index].district, 
(pd_districts.loc[index].geometry.centroid.x,
pd_districts.loc[index].geometry.centroid.y),
color='
fontsize='large',
fontweight='heavy',
horizontalalignment='center' 
)
 ax.set_axis_off()
plt.show()<create_dataframe>
Cat_cols=['Sex','Embarked','Sex_Embarked']
naive_vals = train.groupby('Category' ).count().iloc[:,0]/train.shape[0]
n_rows = test.shape[0]
submission = pd.DataFrame(
 np.repeat(np.array(naive_vals), n_rows ).reshape(39, n_rows ).transpose() ,
columns=naive_vals.index )<feature_engineering>
OH_encoder = OneHotEncoder(handle_unknown='ignore', sparse=False)
 OH_cols_train = pd.DataFrame(OH_encoder.fit_transform(imputed_X_train[Cat_cols]))
OH_cols_valid = pd.DataFrame(OH_encoder.transform(imputed_X_valid[Cat_cols]))
OH_cols_test = pd.DataFrame(OH_encoder.transform(imputed_X_test[Cat_cols]))
OH_cols_train.index = imputed_X_train.index
OH_cols_valid.index = imputed_X_valid.index
OH_cols_test.index = imputed_X_test.index
num_X_train = imputed_X_train.drop(Cat_cols, axis =1)
 num_X_valid = imputed_X_valid.drop(Cat_cols, axis =1)
 num_X_test = imputed_X_test.drop(Cat_cols, axis =1)
 OH_X_train = pd.concat([num_X_train, OH_cols_train], axis=1)
 OH_X_valid = pd.concat([num_X_valid, OH_cols_valid], axis=1)
 OH_X_test = pd.concat([num_X_test, OH_cols_test], axis=1 )
def feature_engineering(data):
data['Date'] = pd.to_datetime(data['Dates'].dt.date)
 data['n_days'] =(
 data['Date'] - data['Date'].min() ).apply(lambda x: x.days)
 data['Day'] = data['Dates'].dt.day
data['DayOfWeek'] = data['Dates'].dt.weekday
data['Month'] = data['Dates'].dt.month
data['Year'] = data['Dates'].dt.year
data['Hour'] = data['Dates'].dt.hour
data['Minute'] = data['Dates'].dt.minute
data['Block'] = data['Address'].str.contains('block', case=False)
 data.drop(columns=['Dates','Date','Address'], inplace=True)
 return data<drop_column>
OH_X_train = OH_X_train.apply(pd.to_numeric)
 OH_X_valid = OH_X_valid.apply(pd.to_numeric)
 OH_X_test = OH_X_test.apply(pd.to_numeric)
 OH_X_train=OH_X_train.rename(columns={0:"Sex1", 1:"Sex2"})
 OH_X_train=OH_X_train.rename(columns={2:"C", 3:"Q",4:"S"})
 OH_X_valid=OH_X_valid.rename(columns={0:"Sex1", 1:"Sex2"})
 OH_X_valid=OH_X_valid.rename(columns={2:"C", 3:"Q",4:"S"})
 OH_X_test=OH_X_test.rename(columns={0:"Sex1", 1:"Sex2"})
 OH_X_test=OH_X_test.rename(columns={2:"C", 3:"Q",4:"S"} )
train = feature_engineering(train)
 train.drop(columns=['Descript','Resolution'], inplace=True)
 test = feature_engineering(test)
 train.head()<categorify>
my_model = XGBClassifier(n_estimators=1000, learning_rate=0.001)
 my_model.fit(OH_X_train, y_train, early_stopping_rounds=50,
eval_set=[(OH_X_valid, y_valid)], verbose=False)
 my_model.fit(OH_X_train, y_train)
 y_pred5 = my_model.predict(OH_X_valid)
 print("Accuracy:",metrics.accuracy_score(y_valid, y_pred5))
le1 = LabelEncoder()
train['PdDistrict'] = le1.fit_transform(train['PdDistrict'])
 test['PdDistrict'] = le1.transform(test['PdDistrict'])
 le2 = LabelEncoder()
y = le2.fit_transform(train.pop('Category'))
train_X, val_X, train_y, val_y = train_test_split(train, y)
 model =LGBMClassifier(objective='multiclass', num_class=39 ).fit(train_X, train_y)
 perm = PermutationImportance(model ).fit(val_X, val_y)
 eli5.show_weights(perm, feature_names=val_X.columns.tolist() )<load_from_csv>
predictions2 = my_model.predict(OH_X_test)
 output = pd.DataFrame({'PassengerId': test_data.PassengerId, 'Survived': predictions2})
 output.to_csv('my_submission_02_06.csv', index=False)
 print("Your submission was successfully saved!" )