Kaggle competition: Avito Demand Prediction Challenge
<SOS> metric: RMSE Kaggle data source: avito-demand-prediction-challenge<create_dataframe>
color = sns.color_palette()
%matplotlib inline
pd.options.mode.chained_assignment = None
pd.options.display.max_columns = 999
print(os.listdir(".. /input"))
clean_dataset = market_train_df.copy()<categorify>
train_df = pd.read_csv(".. /input/train.csv", parse_dates=["activation_date"])
 test_df = pd.read_csv(".. /input/test.csv", parse_dates=["activation_date"])
 print("Train file rows and columns are : ", train_df.shape)
 print("Test file rows and columns are : ", test_df.shape)
 testdex = test_df.index
traindex = train_df.index
code_train = clean_dataset['assetCode'].unique().tolist()
code_test = diff_values_train
final_list = code_train + code_test
encoder = preprocessing.LabelEncoder()
encoder.fit(final_list)
 clean_dataset['assetCode'] = encoder.transform(clean_dataset['assetCode'] )<filter>
train_df["activation_weekday"] = train_df["activation_date"].dt.weekday
test_df["activation_weekday"] = test_df["activation_date"].dt.weekday
train_df["activation_month"] = train_df["activation_date"].dt.month
test_df["activation_month"] = test_df["activation_date"].dt.month
train_df["title_nwords"] = train_df["title"].apply(lambda x: len(x.split()))
test_df["title_nwords"] = test_df["title"].apply(lambda x: len(x.split()))
train_df["description"].fillna("NA", inplace=True)
 test_df["description"].fillna("NA", inplace=True)
 train_df["desc_nwords"] = train_df["description"].apply(lambda x: len(x.split()))
test_df["desc_nwords"] = test_df["description"].apply(lambda x: len(x.split()))
train_df['param123'] = train_df['param_1'].fillna('')+ " " + train_df['param_2'].fillna('')+ " " + train_df['param_3'].fillna('')
 test_df['param123'] = test_df['param_1'].fillna('')+ " " + test_df['param_2'].fillna('')+ " " + test_df['param_3'].fillna('')
 enc = train_df.groupby('category_name')['image_top_1'].agg(lambda x:x.value_counts().index[0] ).astype(np.float32 ).reset_index()
enc.columns = ['category_name' ,'image_top_1_impute']
train_df = pd.merge(train_df, enc, how='left', on='category_name')
 test_df = pd.merge(test_df, enc, how='left', on='category_name')
 del enc
gc.collect()
train_df['image_top_1'].fillna(train_df['image_top_1_impute'], inplace=True)
 test_df['image_top_1'].fillna(test_df['image_top_1_impute'], inplace=True)
 test_df['image'] = test_df['image'].map(lambda x: 1 if len(str(x)) >0 else 0)
 train_df['image'] = train_df['image'].map(lambda x: 1 if len(str(x)) >0 else 0)
 train_df['city'] = train_df['city'] + "_" + train_df['region']
test_df['city'] = test_df['city'] + "_" + test_df['region']
train_df['price'].fillna(0, inplace=True)
 test_df['price'].fillna(0, inplace=True)
 train_df['price'] = np.log1p(train_df['price'])
 test_df['price'] = np.log1p(test_df['price'])
 price_mean = train_df['price'].mean()
price_std = train_df['price'].std()
train_df['price'] =(train_df['price'] - price_mean)/ price_std
test_df['price'] =(test_df['price'] - price_mean)/ price_std
cat_cols = ['category_name', 'image_top_1']
num_cols = ['price', 'deal_probability']
for c in cat_cols:
for c2 in num_cols:
enc = train_df.groupby(c)[c2].agg(['median'] ).astype(np.float32 ).reset_index()
enc.columns = ['_'.join([str(c), str(c2), str(c3)])if c3 != c else c for c3 in enc.columns]
train_df = pd.merge(train_df, enc, how='left', on=c)
 test_df = pd.merge(test_df, enc, how='left', on=c)
 del cat_cols, num_cols, c, c2, enc
gc.collect()
clean_dataset = clean_dataset.loc[clean_dataset['time'] >= '2009-01-01 22:00:00+0000']<define_variables>
def cleanName(text):
try:
textProc = text.lower()
textProc = re.sub('[!@
textProc = " ".join(textProc.split())
 return textProc
except:
return "name error"
best_cols = ['volume', 'close', 'open', 'returnsClosePrevRaw1', 'returnsOpenPrevRaw1',
'returnsClosePrevMktres1',
'returnsClosePrevRaw10', 'returnsOpenPrevRaw10',
'returnsClosePrevMktres10', 'returnsOpenPrevMktres10',
'MA_15MA','MA_30MA', 'MA_45MA', 'MA_60MA', 'MA_80MA']<feature_engineering>
textfeats = ["description", "title"]
train_df['desc_punc'] = train_df['description'].apply(lambda x: len([c for c in str(x)if c in string.punctuation]))
test_df['desc_punc'] = test_df['description'].apply(lambda x: len([c for c in str(x)if c in string.punctuation]))
train_df['title'] = train_df['title'].apply(lambda x: cleanName(x))
train_df["description"] = train_df["description"].apply(lambda x: cleanName(x))
test_df['title'] = test_df['title'].apply(lambda x: cleanName(x))
test_df["description"] = test_df["description"].apply(lambda x: cleanName(x))
for cols in textfeats:
train_df[cols] = train_df[cols].astype(str)
 train_df[cols] = train_df[cols].astype(str ).fillna('missing')
 train_df[cols] = train_df[cols].str.lower()
train_df[cols + '_num_words'] = train_df[cols].apply(lambda comment: len(comment.split()))
train_df[cols + '_num_unique_words'] = train_df[cols].apply(lambda comment: len(set(w for w in comment.split())))
 train_df[cols + '_words_vs_unique'] = train_df[cols+'_num_unique_words'] / train_df[cols+'_num_words'] * 100
train_df[cols + '_num_letters'] = train_df[cols].apply(lambda comment: len(comment))
test_df[cols] = test_df[cols].astype(str)
 test_df[cols] = test_df[cols].astype(str ).fillna('missing')
 test_df[cols] = test_df[cols].str.lower()
test_df[cols + '_num_words'] = test_df[cols].apply(lambda comment: len(comment.split()))
test_df[cols + '_num_unique_words'] = test_df[cols].apply(lambda comment: len(set(w for w in comment.split())))
 test_df[cols + '_words_vs_unique'] = test_df[cols+'_num_unique_words'] / test_df[cols+'_num_words'] * 100
test_df[cols + '_num_letters'] = test_df[cols].apply(lambda comment: len(comment))
train_df['title_desc_len_ratio'] = train_df['title_num_letters']/train_df['description_num_letters']
test_df['title_desc_len_ratio'] = test_df['title_num_letters']/test_df['description_num_letters']
print(train_df.head())
 print(test_df.head() )
clean_dataset['MA_15MA'] = clean_dataset['close'].rolling(window=15 ).mean()
clean_dataset['MA_30MA'] = clean_dataset['close'].rolling(window=30 ).mean()
clean_dataset['MA_45MA'] = clean_dataset['close'].rolling(window=45 ).mean()
clean_dataset['MA_60MA'] = clean_dataset['close'].rolling(window=60 ).mean()
clean_dataset['MA_80MA'] = clean_dataset['close'].rolling(window=80 ).mean()<prepare_x_and_y>
cat_vars = ["region", "city", "parent_category_name", "category_name", "user_type", "param_1", "param123"]
for col in cat_vars:
lbl = preprocessing.LabelEncoder()
lbl.fit(list(train_df[col].values.astype('str')) + list(test_df[col].values.astype('str')))
 train_df[col] = lbl.transform(list(train_df[col].values.astype('str')))
 test_df[col] = lbl.transform(list(test_df[col].values.astype('str')))
 cols_to_drop = ["item_id", "user_id", "activation_date", "image", "param_2", "param_3"
, "image_top_1_impute"]
train_X = train_df.drop(cols_to_drop + ["deal_probability"], axis=1)
 test_X = test_df.drop(cols_to_drop, axis=1)
 train_y = train_df["deal_probability"].values
test_id = test_df["item_id"].values
del train_df, test_df
gc.collect()
X_train = clean_dataset[best_cols]
y_train = clean_dataset.returnsOpenNextMktres10 >= 0
y_train = y_train.values<split>
russian_stop = set(stopwords.words('russian'))
tfidf_para = {
"stop_words": russian_stop,
"analyzer": 'word',
"token_pattern": r'\w{1,}',
"sublinear_tf": True,
"dtype": np.float32,
"norm": 'l2',
"smooth_idf":False
}
def get_col(col_name): return lambda x: x[col_name]
vectorizer = FeatureUnion([ 
('description',TfidfVectorizer(
 ngram_range=(1, 2),
max_features=17000,
**tfidf_para,
preprocessor=get_col('description'))), 
('title',CountVectorizer(
 ngram_range=(1, 2),
stop_words = russian_stop,
preprocessor=get_col('title')))
 ])
 df = pd.concat([train_X,test_X],axis=0)
 vectorizer.fit(df.to_dict('records'))
ready_df = vectorizer.transform(df.to_dict('records'))
tfvocab = vectorizer.get_feature_names()
df.drop(textfeats, axis=1,inplace=True )
X_train_valid, X_test_valid, y_train_valid, y_test_valid = train_test_split(X_train, y_train, shuffle=False )<compute_test_metric>
sparse_m = csr_matrix(df.values)
 complete = hstack([sparse_m,ready_df])
 tfvocab = df.columns.tolist() + tfvocab
complete = complete.tocsr()
print(complete.shape)
 print(ready_df.shape)
 del sparse_m, ready_df, df
gc.collect()
no assetCode
===========
128.73414850234985
deph 4
score on train:0.5521080261083056
score on test:0.543682274151361
===========
===========
136.2291419506073
deph 5
score on train:0.5536006706720693
score on test:0.5433445431289702
===========
===========
143.36471796035767
deph 6
score on train:0.5557504141799238
score on test:0.5434164007933087
===========<compute_test_metric>
train_X = complete[0:traindex.shape[0],]
test_X = complete[traindex.shape[0]:,]
del complete
gc.collect()
train_X.shape
min_child_weight =3
===========
2060.7648026943207
score on train:0.5740279461005274
score on test:0.5400018681829286
===========<train_model>
dev_X = train_X[:-200000,:]
val_X = train_X[-200000:,:]
dev_y = train_y[:-200000]
val_y = train_y[-200000:]
print(dev_X.shape, val_X.shape, test_X.shape)
 del train_X, train_y
gc.collect()
begin_time = time.time()
model = XGBClassifier(n_jobs=-1,n_estimators=350,max_depth=7,eta=1, random_seed = 42,subsample=1, min_child_weight =2)
 model.fit(X_train, y_train)
 model_2 = XGBClassifier(n_jobs=-1,n_estimators=350,max_depth=7,eta=1, random_seed = 43,subsample=1, min_child_weight =2)
 model_2.fit(X_train, y_train)
 model_3 = XGBClassifier(n_jobs=-1,n_estimators=350,max_depth=7,eta=1, random_seed = 44,subsample=1, min_child_weight =2)
 model_3.fit(X_train, y_train)
 model_4 = XGBClassifier(n_jobs=-1,n_estimators=350,max_depth=7,eta=1, random_seed = 45,subsample=1, min_child_weight =2)
 model_4.fit(X_train, y_train)
 print('===========')
 print(time.time() - begin_time)
 print("score on train:{}".format(model.score(X_train_valid, y_train_valid)))
 print("score on test:{}".format(model.score(X_test_valid, y_test_valid)))
 print('===========' )<compute_test_metric>
def run_lgb(train_X, train_y, val_X, val_y, test_X):
params = {
"objective" : "regression",
"metric" : "rmse",
"num_leaves" : 300,
"learning_rate" : 0.01,
"bagging_fraction" : 0.75,
"feature_fraction" : 0.5,
"bagging_freq" : 2,
"bagging_seed" : 2018,
"verbosity" : -1,
"max_depth": 18,
"min_child_samples":100
}
lgtrain = lgb.Dataset(train_X, label=train_y, feature_name=tfvocab)
 lgval = lgb.Dataset(val_X, label=val_y, feature_name=tfvocab)
 del train_X, val_X; gc.collect()
evals_result = {}
model = lgb.train(params, lgtrain, 10000, valid_sets=[lgval], early_stopping_rounds=50, verbose_eval=200, evals_result=evals_result)
 pred_test_y = model.predict(test_X, num_iteration=model.best_iteration)
 del lgtrain, lgval; gc.collect()
return pred_test_y, model, evals_result
last result
===========
2141.4117183685303
score on train:0.5735349533832473
score on test:0.5402126375389803
===========<compute_test_metric>
pred_test, model, evals_result = run_lgb(dev_X, dev_y, val_X, val_y, test_X )
clean data
===========
2124.9790363311768
score on train:0.5740413980318968
score on test:0.5404774223218648
===========<compute_test_metric>
pred_test[pred_test>1] = 1
pred_test[pred_test<0] = 0
sub_df = pd.DataFrame({"item_id":test_id})
 sub_df["deal_probability"] = pred_test
sub_df.to_csv("baseline_lgb.csv", index=False )