Kaggle competition: Avito Demand Prediction Challenge
no data cleaning
===========
2121.7477345466614
score on train:0.5734535197684092
score on test:0.5407060294406468
===========<compute_test_metric>
print("
Data Load Stage")
 training = pd.read_csv('.. /input/train.csv', index_col = "item_id", parse_dates = ["activation_date"])
 traindex = training.index
testing = pd.read_csv('.. /input/test.csv', index_col = "item_id", parse_dates = ["activation_date"])
 testdex = testing.index
y = training.deal_probability.copy()
training.drop("deal_probability",axis=1, inplace=True)
 print('Train shape: {} Rows, {} Columns'.format(*training.shape))
print('Test shape: {} Rows, {} Columns'.format(*testing.shape))
99.62418842315674
score on train:0.5464178263136221
score on test:0.5404374837245638
149.2367594242096
score on train:0.5468616551042119
score on test:0.5408792455637811
195.48552250862122
score on train:0.5472481323281481
score on test:0.5410125491363168
253.72855687141418
score on train:0.547716245115429
score on test:0.5410125491363168<feature_engineering>
df = pd.concat([training,testing],axis=0)
 del training, testing
gc.collect()
print('
All Data shape: {} Rows, {} Columns'.format(*df.shape))
print("Feature Engineering")
 df["price"] = np.log(df["price"]+0.001)
 df["price"].fillna(-999,inplace=True)
 df["image_top_1"].fillna(-999,inplace=True)
 print("
Create Time Variables")
 df["Weekday"] = df['activation_date'].dt.weekday
df["Week of Year"] = df['activation_date'].dt.week
df["Day of Month"] = df['activation_date'].dt.day
print('Create combined param len')
 df['param_combined'] = df.apply(lambda row:' '.join([str(row['param_1']),str(row['param_2']),str(row['param_3'])]),axis = 1)
 df['param_combined'].fillna(' ',inplace = True)
 df['param_combined_len'] = df['param_combined'].apply(lambda x:len(x.split()))
df.drop(["activation_date","image",'param_combined','Week of Year','Day of Month'],axis=1,inplace=True)
 print("
Encode Variables")
 categorical = ["user_id","region","city","parent_category_name","category_name","item_seq_number","user_type","image_top_1",'param_combined_len']
messy_categorical = ["param_1","param_2","param_3","title","description"]
print("Encoding :",categorical + messy_categorical)
test_dataframe = pd.DataFrame()
days = env.get_prediction_days()
for(market_obs_df, news_obs_df, predictions_template_df)in days:
market_obs_df['MA_15MA'] = market_obs_df['close'].rolling(window=15 ).mean()
market_obs_df['MA_30MA'] = market_obs_df['close'].rolling(window=30 ).mean()
market_obs_df['MA_45MA'] = market_obs_df['close'].rolling(window=60 ).mean()
market_obs_df['MA_60MA'] = market_obs_df['close'].rolling(window=45 ).mean()
market_obs_df['MA_80MA'] = market_obs_df['close'].rolling(window=80 ).mean()
X_test = market_obs_df[best_cols]
pred_1 = model.predict_proba(X_test)
 pred_2 = model_2.predict_proba(X_test)
 pred_3 = model_3.predict_proba(X_test)
 pred_4 = model_4.predict_proba(X_test)
 bagged_predictions = np.zeros(( X_test.shape[0],2))
bagged_predictions += pred_1
bagged_predictions += pred_2
bagged_predictions += pred_3
bagged_predictions += pred_4
probability = bagged_predictions/4
confidence = 2* probability[:,1] -1
predictions_template_df.confidenceValue = confidence
env.predict(predictions_template_df)
 print('Done!')
 env.write_submission_file()<import_modules>
lbl = preprocessing.LabelEncoder()
for col in categorical + messy_categorical:
df[col] = lbl.fit_transform(df[col].astype(str))
print("
Catboost Modeling Stage")
 X = df.loc[traindex,:].copy()
print("Training Set shape",X.shape)
 test = df.loc[testdex,:].copy()
print("Submission Set Shape: {} Rows, {} Columns".format(*test.shape))
del df
gc.collect()
import numpy as np
import lightgbm as lgb
import pandas as pd
from kaggle.competitions import twosigmanews
import matplotlib.pyplot as plt
import random
from datetime import datetime, date
from xgboost import XGBClassifier
from sklearn import model_selection
from sklearn.metrics import mean_squared_error
import time<import_modules>
X_train, X_valid, y_train, y_valid = train_test_split(
 X, y, test_size=0.10, random_state=23)
 def column_index(df, query_cols):
cols = df.columns.values
sidx = np.argsort(cols)
 return sidx[np.searchsorted(cols,query_cols,sorter=sidx)]
categorical_features_pos = column_index(X,categorical + messy_categorical)
env = twosigmanews.make_env()
print('Done!' )<train_model>
print("Train CatBoost Decision Tree")
 modelstart= time.time()
cb_model = CatBoostRegressor(iterations=700,
learning_rate=0.02,
depth=12,
eval_metric='RMSE',
random_seed = 23,
bagging_temperature = 0.2,
od_type='Iter',
metric_period = 75,
od_wait=100)
 cb_model.fit(X_train, y_train,
eval_set=(X_valid,y_valid),
cat_features=categorical_features_pos,
use_best_model=True,
verbose=True )
<feature_engineering><EOS> 
print("Model Evaluation Stage")
 print(cb_model.get_params())
 print('RMSE:', np.sqrt(metrics.mean_squared_error(y_valid, cb_model.predict(X_valid))))
catpred = cb_model.predict(test)
 catsub = pd.DataFrame(catpred,columns=["deal_probability"],index=testdex)
 catsub['deal_probability'].clip(0.0, 1.0, inplace=True)
 catsub.to_csv("submission.csv",index=True,header=True)
 print("Model Runtime: %0.2f Minutes"%(( time.time() - modelstart)/60))
print("Notebook Runtime: %0.2f Minutes"%(( time.time() - notebookstart)/60))