Kaggle competition: San Francisco Crime Classification
<SOS> metric: MulticlassLoss Kaggle data source: san-francisco-crime-classification<categorify>
%matplotlib inline
def Categorical_Encoding_test(X, encoder):
cat_features = [y for y in X.columns if X[y].dtypes == 'object']
print(cat_features)
 cat_features = [col for col in cat_features if col not in ['Name']]
numerical_col = [y for y in X.columns if X[y].dtypes != 'object']
data = X[numerical_col].join(encoder.transform(X[cat_features]))
print(data.head())
 return data<import_modules>
train = pd.read_csv(".. /input/sf-crime/train.csv.zip", parse_dates=['Dates'])
 test = pd.read_csv(".. /input/sf-crime/test.csv.zip", parse_dates=['Dates'], index_col='Id')
 sampleSubmission = pd.read_csv(".. /input/sf-crime/sampleSubmission.csv.zip" )
from sklearn.experimental import enable_iterative_imputer
from sklearn.impute import IterativeImputer<feature_engineering>
fe_name = list(test)
 df_train = train[fe_name]
df = pd.concat(( df_train,test))
def iterative_imputer(X):
numerical_feature = [y for y in X.columns if X[y].dtypes != 'object'
and X[y].isnull().sum() != 0]
imp_mean = IterativeImputer(max_iter=10, verbose=0)
 X[numerical_feature] = imp_mean.fit_transform(X[numerical_feature])
 return X<prepare_x_and_y>
target = train['Category']
def numerical_imputer(X):
numerical_feature = [y for y in X.columns if X[y].dtypes != 'object'
and X[y].isnull().sum() != 0]
my_imputer = SimpleImputer(missing_values=np.nan, strategy='most_frequent')
 X[numerical_feature] = pd.DataFrame(my_imputer.fit_transform(X[numerical_feature]))
return X<import_modules>
target.value_counts()
import lightgbm as lgb<predict_on_test>
LB = LabelEncoder()
target = LB.fit_transform(target)
 print(LB.classes_ )
def train_and_predict(train_X, train_y, valid_X, valid_y, X_test):
dtrain = lgb.Dataset(train_X, label=train_y)
 dvalid = lgb.Dataset(valid_X, label=valid_y)
 param = {'num_leaves': 64, 'objective': 'binary'}
param['metric'] = 'auc'
num_round = 1000
bst = lgb.train(param, dtrain, num_round,
valid_sets=[dvalid], early_stopping_rounds=10, verbose_eval=False)
 y_pred = bst.predict(X_test)
 return y_pred<load_from_csv>
date = pd.to_datetime(df['Dates'])
 df['Date'] = date.dt.date
df['Year'] = date.dt.year
df['Month'] = date.dt.month
df['Day'] = date.dt.day
df['Hour'] = date.dt.hour
if __name__ == '__main__':
seed = 123
random.seed(seed)
 print('Loading Training Data')
 baseline_data = pd.read_csv('/kaggle/input/titanic/train.csv')
 baseline_data = Feature_selection(baseline_data)
 baseline_data = numerical_imputer(baseline_data)
 encoded_data, encoder = Categorical_Encoding_train(baseline_data)
 cols = [col for col in encoded_data.columns if col not in ['Survived','PassengerId']]
X = encoded_data[cols]
y = baseline_data['Survived']
valid_fraction = 0.1
valid_size = int(len(encoded_data)* valid_fraction)
 train_X = X[:-2 * valid_size]
train_y = y[:-2 * valid_size]
valid_X = X[-2 * valid_size:]
valid_y = y[-2 * valid_size:]
print('Loading Testing Data')
 test_data = pd.read_csv('/kaggle/input/titanic/test.csv')
 test_data = Feature_selection(test_data)
 test_data = numerical_imputer(test_data)
 encoded_data_test = Categorical_Encoding_test(test_data, encoder)
 X_test = encoded_data_test[cols].iloc[:,:]
print(X_test.head())
 y_pred = train_and_predict(train_X, train_y, valid_X, valid_y, X_test)
 y_pred = np.around(y_pred)
 y_pred = y_pred.astype(int)
 output = pd.DataFrame({'PassengerId': test_data.PassengerId,
'Survived': y_pred})
 output.to_csv('submission_grid_search.csv', index=False)
<categorify>
df.drop("Dates", axis = 1, inplace = True )
<groupby>
date = df.groupby('Date' ).count().iloc[:, 0]
baseline_data.groupby('Pclass')['Survived'].count()<groupby>
lb = LabelEncoder()
df['PdDistrict'] = lb.fit_transform(df["PdDistrict"] )
baseline_data.groupby('Embarked')['Survived'].count()<save_to_csv>
df["PdDistrict"].value_counts()
def create_download_link(df, title = "Download CSV file", filename = "data.csv"):
csv = df.to_csv()
b64 = base64.b64encode(csv.encode())
 payload = b64.decode()
html = '<a download="{filename}" href="data:text/csv;base64,{payload}" target="_blank">{title}</a>'
html = html.format(payload=payload,title=title,filename=filename)
 return HTML(html)
 df = pd.DataFrame(np.random.randn(50, 4), columns=list('ABCD')) 
<load_from_csv>
lb = LabelEncoder()
df['PdDistrict'] = lb.fit_transform(df["PdDistrict"] )
train_data = pd.read_csv(".. /input/train.csv")
 test_data = pd.read_csv(".. /input/test.csv" )<define_variables>
df['DayOfWeek'] = lb.fit_transform(df["DayOfWeek"] )
train_ID = train_data['Id']
test_ID = test_data['Id']<drop_column>
df["Address"].value_counts().head(20 )
train_data.drop("Id", axis = 1, inplace = True)
 test_data.drop("Id", axis = 1, inplace = True )<feature_engineering>
df['block'] = df['Address'].str.contains('block', case=False)
 df['ST'] = df['Address'].str.contains('ST', case=False )
transform_log = np.log1p(train_data["SalePrice"])
 plot_dist_norm(transform_log, 'log(SalePrice)Distribution')
 train_data["SalePrice"] = transform_log<filter>
df['block'] = lb.fit_transform(df["block"])
 df['ST'] = lb.fit_transform(df["ST"] )
train_data[train_data.GrLivArea > 4500] 
<filter>
df.drop("Address", axis = 1, inplace = True )
train_data = train_data[train_data.GrLivArea < 4500]<drop_column>
print(len(df.loc[df["X"] >= -120.5, "X"]))
print(len(df.loc[df["Y"] >= 90, "Y"]))
transform_log = transform_log.drop([523 , 1298], axis=0 )<concatenate>
X_median = df[df["X"] < -120.5]["X"].median()
Y_median = df[df["Y"] < 90]["Y"].median()
df.loc[df["X"] >= -120.5, "X"] = X_median
df.loc[df["Y"] >= 90, "Y"] = Y_median
all_data = pd.concat(( train_data, test_data)).reset_index(drop=True )<drop_column>
df["X+Y"] = df["X"] + df["Y"]
df["X-Y"] = df["X"] - df["Y"]
all_data.drop(['SalePrice'], axis=1, inplace=True )<sort_values>
df.drop("Date", axis = 1, inplace = True )
total = all_data.isnull().sum().sort_values(ascending=False)
 percent =(all_data.isnull().sum() /all_data.isnull().count() ).sort_values(ascending=False)
 missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
 missing_data[total > 0]<categorify>
new_train = df[:train.shape[0]]
new_test = df[train.shape[0]:]
for col in ['SaleType', 'KitchenQual' , 'Exterior2nd' , 'Exterior1st' , 'Electrical' , 'Functional' , 'MSZoning' ] :
all_data[col] = all_data[col].fillna(all_data[col].mode() [0])
 for col in ['BsmtFinSF1', 'BsmtFinSF2' , 'GarageCars' , 'GarageArea' , 'TotalBsmtSF' , 'BsmtUnfSF' , 'BsmtHalfBath' , 'BsmtFullBath' , 'MasVnrArea' ]:
all_data[col]=all_data[col].fillna(0)
 for col in ['MasVnrType','FireplaceQu' , 'Fence' , 'PoolQC' , 'MiscFeature' ,'Alley' , 'BsmtFinType1', 'BsmtFinType2', 'BsmtQual', 'BsmtCond', 'BsmtExposure', 'GarageType', 'GarageFinish' , 'GarageCond' , 'GarageQual'] :
all_data[col]=all_data[col].fillna('None' )<drop_column>
train_data = lgb.Dataset(new_train, label=target, categorical_feature=["PdDistrict", "DayOfWeek"])
 params = {'boosting':'gbdt',
'objective':'multiclass',
'num_class':39,
'max_delta_step':0.9,
'min_data_in_leaf': 21,
'learning_rate': 0.4,
'max_bin': 465,
'num_leaves': 41,
'verbose' : 1}
bst = lgb.train(params, train_data, 120 )
all_data.drop(['Utilities', 'Street' , 'PoolQC'], axis=1, inplace=True )<feature_engineering>
predictions = bst.predict(new_test )
all_data['Total_sqr_footage'] =(all_data['BsmtFinSF1'] + all_data['BsmtFinSF2'] + all_data['1stFlrSF'] + all_data['2ndFlrSF'])
 all_data['Total_Bathrooms'] =(all_data['FullBath'] +(0.5 * all_data['HalfBath'])+ all_data['BsmtFullBath'] +(0.5 * all_data['BsmtHalfBath']))
all_data['Total_porch_sf'] =(all_data['OpenPorchSF'] + all_data['3SsnPorch'] + all_data['EnclosedPorch'] + all_data['ScreenPorch'] + all_data['WoodDeckSF'])
<feature_engineering>
submission = pd.DataFrame(predictions,columns=LB.inverse_transform(np.linspace(0, 38, 39, dtype='int16')) ,index=new_test.index)
all_data['haspool'] = all_data['PoolArea'].apply(lambda x: 1 if x > 0 else 0)
 all_data['has2ndfloor'] = all_data['2ndFlrSF'].apply(lambda x: 1 if x > 0 else 0)
 all_data['hasgarage'] = all_data['GarageArea'].apply(lambda x: 1 if x > 0 else 0)
 all_data['hasbsmt'] = all_data['TotalBsmtSF'].apply(lambda x: 1 if x > 0 else 0)
 all_data['hasfireplace'] = all_data['Fireplaces'].apply(lambda x: 1 if x > 0 else 0 )<filter>
train_xgb = xgb.DMatrix(new_train, label=target)
 test_xgb = xgb.DMatrix(new_test )
all_data['GarageYrBlt'][all_data['GarageYrBlt']>2150]<filter>
params = {
'max_depth': 4,
'eta': 0.3,
'silent': 1,
'objective': 'multi:softprob',
'num_class': 39,
}
xg = xgb.cv(params, train_xgb, nfold=3, early_stopping_rounds=10, metrics='mlogloss', verbose_eval=True )
all_data['GarageYrBlt']=all_data['GarageYrBlt'].fillna(all_data['YearBuilt'][ all_data['GarageYrBlt'].isnull() ])
 all_data['GarageYrBlt'][all_data['GarageYrBlt']>2018] = all_data['YearBuilt'][all_data['GarageYrBlt']>2018]<categorify>
train_xgb = xgb.train(params, train_xgb, 10)
 pred_xgb = train_xgb.predict(test_xgb )
all_data["LotFrontage"] = all_data.groupby("Neighborhood")["LotFrontage"].transform(lambda x: x.fillna(x.median()))<data_type_conversions>
submission1 = pd.DataFrame(pred_xgb,columns=LB.inverse_transform(np.linspace(0, 38, 39, dtype='int16')) ,index=new_test.index)
all_data['MSSubClass'] = all_data['MSSubClass'].astype(str)
 all_data['OverallCond'] = all_data['OverallCond'].astype(str)
 all_data['BsmtFullBath'] = all_data['BsmtFullBath'].astype(str)
 all_data['BsmtHalfBath'] = all_data['BsmtHalfBath'].astype(str)
 all_data['YrSold'] = all_data['YrSold'].astype(str)
 all_data['MoSold'] = all_data['MoSold'].astype(str)
 all_data['GarageYrBlt'] = all_data['GarageYrBlt'].astype(str)
 all_data['YearBuilt'] = all_data['YearBuilt'].astype(str)
 all_data['YearRemodAdd'] = all_data['YearRemodAdd'].astype(str)
 all_data['GarageCars'] = all_data['GarageCars'].astype(str )<feature_engineering>
ensemble = 0.9*predictions + 0.1*pred_xgb
all_data['TotalSF']=all_data['TotalBsmtSF']+all_data['1stFlrSF']+all_data['2ndFlrSF']<categorify>
sub = pd.DataFrame(ensemble,columns=LB.inverse_transform(np.linspace(0, 38, 39, dtype='int16')) ,index=new_test.index)
 sub.to_csv('submission.csv', index_label='Id' )