Kaggle competition: Categorical Feature Encoding Challenge
test = pd.read_csv(os.path.join(INPUT_DIR, 'test_simplified.csv'))
test.head()
x_test = df_to_image_array_xd(test, size)
 print(test.shape, x_test.shape)
 print('Test array memory {:.2f} GB'.format(x_test.nbytes / 1024.**3))<predict_on_test>
submission = pd.read_csv("/kaggle/input/cat-in-the-dat/sample_submission.csv")
 train = pd.read_csv("/kaggle/input/cat-in-the-dat/train.csv")
 test = pd.read_csv("/kaggle/input/cat-in-the-dat/test.csv" )
test_predictions = model.predict(x_test, batch_size=128, verbose=1)
 top3 = preds2catids(test_predictions)
 top3.head()
top3.shape
cats = list_all_categories()
id2cat = {k: cat.replace(' ', '_')for k, cat in enumerate(cats)}
top3cats = top3.replace(id2cat)
 top3cats.head()
top3cats.shape<save_to_csv>
test_id = test["id"]
test.drop(['id'],inplace=True, axis = 1)
 test.head()
test['word'] = top3cats['a'] + ' ' + top3cats['b'] + ' ' + top3cats['c']
submission = test[['key_id', 'word']]
submission.to_csv('gs_mn_submission_{}.csv'.format(int(map3 * 10**4)) , index=False)
 submission.head()
submission.shape<train_model>
y = train['target']
X = train.drop(['target', 'id'], axis = 1)
 X.head()
end = dt.datetime.now()
print('Latest run {}.
 Total time {}s'.format(end,(end - start ).seconds))<define_variables>
cat_columns = [cols for cols in train.columns if train[cols].dtype == 'object']
print(cat_columns )
LABELS = ["HasDetections"]<load_from_csv>
encoder = LabelEncoder()
for col in cat_columns:
X[col] = pd.DataFrame(encoder.fit_transform(X[col]))
test[col] = pd.DataFrame(encoder.fit_transform(test[col]))
predict_list = []
predict_list.append(pd.read_csv(".. /input/aimalware/nffm_submission.csv")[LABELS].values)
 predict_list.append(pd.read_csv(".. /input/last-stages/submission_ashish_v2.csv")[LABELS].values)
 predict_list.append(pd.read_csv(".. /input/laststages/submission_ashish_v3.csv")[LABELS].values)
 predict_list.append(pd.read_csv(".. /input/last-stages/submission_ashish_v4.csv")[LABELS].values)
 predict_list.append(pd.read_csv(".. /input/last-stages/submission_ashish_vb1.csv")[LABELS].values)
 predict_list.append(pd.read_csv(".. /input/finalai/blending.csv")[LABELS].values )<save_to_csv>
Xtrain,Xtest,ytrain,ytest = train_test_split(X,y,random_state=0 )
print("Rank averaging on ", len(predict_list), " files")
 predictions = np.zeros_like(predict_list[0])
 for predict in predict_list:
for i in range(1):
predictions[:, i] = np.add(predictions[:, i], rankdata(predict[:, i])/predictions.shape[0])
 predictions /= len(predict_list)
 submission = pd.read_csv('.. /input/microsoft-malware-prediction/sample_submission.csv')
 submission[LABELS] = predictions
submission.to_csv('super_blend.csv', index=False )<load_from_csv>
model = XGBClassifier(n_estimators=500,scale_pos_weight=2,random_state=1,colsample_bytree=0.5)
 model.fit(Xtrain,ytrain)
 pred = model.predict_proba(Xtest)[:, 1]
score = roc_auc_score(ytest, pred)
 print("score: ", score )
submission = pd.read_csv('.. /input/microsoft-malware-prediction/sample_submission.csv')
 sample_3 = pd.read_csv(".. /input/last-stages/submission_ashish_v2.csv")
 sample_4 = pd.read_csv(".. /input/laststages/submission_ashish_v3.csv")
 sample_5 = pd.read_csv(".. /input/last-stages/submission_ashish_v4.csv")
 sample_6 = pd.read_csv(".. /input/last-stages/submission_ashish_vb1.csv")
 sample_7 = pd.read_csv(".. /input/aimalware/nffm_submission.csv")
 sample_8 = pd.read_csv(".. /input/finalai/blending.csv")
 submission['HasDetections'] = sample_6['HasDetections'] * 0.3 + sample_8['HasDetections'] * 0.3 + sample_3['HasDetections'] * 0.2 + sample_7['HasDetections'] * 0.2
submission.to_csv('super_blend_2.csv', index=False )<categorify>
test = pd.read_csv("/kaggle/input/cat-in-the-dat/test.csv")
 y2 = train['target']
X2 = train.drop(['target', 'id'], axis = 1)
 test.drop(['id'],inplace=True, axis = 1)
 test.head()
def info(function):
def wrapper(data,*args,**kargs):
tic = datetime.datetime.now()
result = function(data,*args,**kargs)
 toc = datetime.datetime.now()
print(function.__name__,' took ', toc-tic)
 print('Shape: ',data.shape,' ----> ', result.shape)
 print('NaN value: ', result.isna().sum() [result.isna().sum() != 0])
 print('
')
 return result
return wrapper<load_from_csv>
traintest = pd.concat([X2, test])
 dummies = pd.get_dummies(traintest, columns=traintest.columns, sparse=True )
train = pd.read_csv('/kaggle/input/rossmann-store-sales/train.csv')
 test = pd.read_csv('/kaggle/input/rossmann-store-sales/test.csv')
 stores = pd.read_csv('/kaggle/input/rossmann-store-sales/store.csv')
 sample = pd.read_csv('/kaggle/input/rossmann-store-sales/sample_submission.csv' )<compute_test_metric>
X2 = X2.sparse.to_coo().tocsr()
test = test.sparse.to_coo().tocsr()
def ToWeight(y):
w = np.zeros(y.shape, dtype=float)
 ind = y != 0
w[ind] = 1./(y[ind]**2)
 return w
def rmspe(yhat, y):
w = ToWeight(y)
 rmspe = np.sqrt(np.mean(w *(y - yhat)**2))
return rmspe<feature_engineering>
Xtrain,Xtest,ytrain,ytest = train_test_split(X2,y,random_state=0 )
@info
def log_transf(df):
df.Sales = np.log1p(df.Sales)
 df.Customers = np.log1p(df.Customers)
 return df
@info
def remove_outliers(df,column='Sales'):
q1 = df[column].quantile(0.2)
 q3 = df[column].quantile(0.8)
 iqr = q3-q1
iqr_lower = q1 - 1.5*iqr
iqr_upper = q3 + 1.5*iqr
df = df.loc[(df[column] > iqr_lower)&(df[column]< iqr_upper),:]
return df<feature_engineering>
model = LogisticRegression(solver='lbfgs', C = 0.1, max_iter=1000)
 model.fit(Xtrain,ytrain)
 pred = model.predict_proba(Xtest)[:, 1]
score = roc_auc_score(ytest, pred)
 print("score: ", score )
@info
def timeseries_features(df):
df.Date = pd.to_datetime(df.Date)
 df = df.sort_values('Date' ).reset_index(drop = True)
 df['month'] = df.Date.dt.month
df['dayofmonth'] = df.Date.dt.day
df['dayofyear'] = df.Date.dt.dayofyear
df['year'] = df.Date.dt.year
df['is_weekday'] = df.DayOfWeek.apply(lambda x: 0 if x in(6,7)else 1)
 df['is_month_start'] = df.Date.dt.is_month_start.astype(int)
 df['is_month_end'] = df.Date.dt.is_month_end.astype(int)
 holidays = calendar().holidays(start = df.Date.min() , end = df.Date.max())
 df['is_holiday'] = df.Date.isin(holidays ).astype(int)
 return df
@info
def clean_main(df):
df = df.loc[df.Sales != 0,:].reset_index(drop = True)
 df = df.drop(['Open'],axis = 1)
 df.StateHoliday = df.StateHoliday.astype(str)
 return df
@info
def clean_store(df):
df.CompetitionDistance.fillna(df.CompetitionDistance.mean() ,inplace = True)
 df.drop(['CompetitionOpenSinceMonth','CompetitionOpenSinceYear','Promo2SinceWeek','Promo2SinceYear'],axis = 1,inplace = True)
 month_dict = {v: k for k,v in enumerate(calendar.month_abbr)}
del month_dict['']
del month_dict['Sep']
month_dict['NaN'] = 0
month_dict['Sept'] = 9
df.PromoInterval = df.PromoInterval.fillna('NaN')
 df.PromoInterval = df.PromoInterval.str.split(',')
 df.PromoInterval = df.PromoInterval.apply(lambda x: [month_dict[value] for value in x if month_dict.get(value)])
 df['promo_len'] = df.PromoInterval.apply(lambda x: len(x))
return df
train_prep =(train 
.copy() 
.pipe(log_transf)
.pipe(remove_outliers)
.pipe(timeseries_features)
.pipe(clean_main)
)
 store_prep =(stores 
.copy() 
.pipe(clean_store)
)
 data_prep = pd.merge(train_prep,store_prep,how='left',on='Store')
 data_prep['is_promo'] = data_prep.apply(lambda x: 1 if x['month'] in x['PromoInterval'] else 0,axis = 1)
 data_prep = data_prep.drop('PromoInterval',axis=1 ).reset_index(drop=True )<groupby>
submission["id"] = test_id
submission["target"] = model.predict_proba(test )
test_bs = data_prep[data_prep.year == 2015]
train_bs = data_prep[data_prep.year < 2015]
predict_bs =(train_bs 
.groupby(['Store','month','dayofmonth'] ).Sales.mean().reset_index().rename({'Sales':'predictions'},axis = 1)
.merge(test_bs,how='right',on = ['Store','month','dayofmonth'])
.fillna(train_bs.Sales.mean())
.sort_values('Date')
)
 print('Baseline to overcome = {:.2f}'.format(rmspe(np.expm1(predict_bs.Sales),np.expm1(predict_bs.predictions))))
fig,ax = plt.subplots(1,3,figsize =(30,10))
rnd_store = np.random.randint(min(predict_bs.Store),max(predict_bs.Store),3)
 for idx,store in enumerate(rnd_store):
ax[idx].plot(predict_bs[predict_bs.Store == store].Date,np.expm1(predict_bs[predict_bs.Store == store].Sales), color = 'blue' ,label = 'Observed')
 ax[idx].plot(predict_bs[predict_bs.Store == store].Date,np.expm1(predict_bs[predict_bs.Store == store].predictions),color = 'red',label = 'Predicted')
 ax[idx].legend()
ax[idx].set_title('Store '+str(store)) 
<categorify>
submission.to_csv("submission.csv", index=False )
<categorify><EOS> 
submission.to_csv("submission.csv", index=False )