Kaggle competition: Avito Demand Prediction Challenge
<SOS> metric: RMSE Kaggle data source: avito-demand-prediction-challenge<filter>
sns.set()
%matplotlib inline
train = full_lstm_df.loc[full_lstm_df['time'].dt.year < 2016]
valid = full_lstm_df.loc[full_lstm_df['time'].dt.year >= 2016]
valid = valid.loc[valid['universe']==1] 
<prepare_x_and_y>
used_cols = ['item_id', 'user_id']
train = pd.read_csv('.. /input/train.csv', usecols=used_cols)
 train_active = pd.read_csv('.. /input/train_active.csv', usecols=used_cols)
 test = pd.read_csv('.. /input/test.csv', usecols=used_cols)
 test_active = pd.read_csv('.. /input/test_active.csv', usecols=used_cols)
 train_periods = pd.read_csv('.. /input/periods_train.csv', parse_dates=['date_from', 'date_to'])
 test_periods = pd.read_csv('.. /input/periods_test.csv', parse_dates=['date_from', 'date_to'])
 train.head()
valid_X, valid_y = valid.values[:, :-5], valid['TARGET'].values
valid_returns = valid.values[:, -2]
valid_universe = valid.values[:, -3]
valid_time = valid.values[:, -4]<set_options>
all_samples = pd.concat([
train,
train_active,
test,
test_active
] ).reset_index(drop=True)
 all_samples.drop_duplicates(['item_id'], inplace=True)
 del train_active
del test_active
gc.collect()
del valid
gc.collect()<set_options>
all_periods = pd.concat([
train_periods,
test_periods
])
 del train_periods
del test_periods
gc.collect()
all_periods.head()
%whos<prepare_x_and_y>
all_periods['days_up'] = all_periods['date_to'].dt.dayofyear - all_periods['date_from'].dt.dayofyear
train_X, train_y = train.values[:, :-5], train['TARGET'].values<set_options>
gp = all_periods.groupby(['item_id'])[['days_up']]
gp_df = pd.DataFrame()
gp_df['days_up_sum'] = gp.sum() ['days_up']
gp_df['times_put_up'] = gp.count() ['days_up']
gp_df.reset_index(inplace=True)
 gp_df.rename(index=str, columns={'index': 'item_id'})
 gp_df.head()
del train
gc.collect()<prepare_x_and_y>
all_periods.drop_duplicates(['item_id'], inplace=True)
 all_periods = all_periods.merge(gp_df, on='item_id', how='left')
 all_periods.head()
USE_SIGMOID_ACTIVATION_LAYER = True 
<import_modules>
del gp
del gp_df
gc.collect()
from keras import callbacks<compute_train_metric>
all_periods = all_periods.merge(all_samples, on='item_id', how='left')
 all_periods.head()
class Metrics(callbacks.Callback):
def on_train_begin(self, logs={}):
self._data = []
def on_epoch_end(self, batch, logs={}):
X_val, y_val = self.validation_data[0], self.validation_data[1]
y_predict =(pd.DataFrame(model.predict(X_val)) * 2)- 1
_sigmascore = sigma_scorelstm(y_val, y_predict)
 print(" — sigmascore: %f" %(_sigmascore))
self._data.append({
'val_sigmascore': _sigmascore
})
 return
def get_data(self):
return self._data
metrics = Metrics()<compute_test_metric>
gp = all_periods.groupby(['user_id'])[['days_up_sum', 'times_put_up']].mean().reset_index() \ 
.rename(index=str, columns={
'days_up_sum': 'avg_days_up_user',
'times_put_up': 'avg_times_up_user'
})
 gp.head()
def sigma_scorelstm(y_true, y_pred):
x_t_i = y_pred * pd.DataFrame(valid_returns)* pd.DataFrame(valid_universe)
 data = pd.concat([pd.DataFrame(valid_time), x_t_i], axis=1)
 data.columns = ['day','x_t_i']
x_t = data.groupby('day' ).sum().values.flatten()
mean = np.mean(x_t)
 std = np.std(x_t)
 score_valid = mean / std
return score_valid<create_dataframe>
n_user_items = all_samples.groupby(['user_id'])[['item_id']].count().reset_index() \ 
.rename(index=str, columns={
'item_id': 'n_user_items'
})
 gp = gp.merge(n_user_items, on='user_id', how='outer')
 gp.head()
x_t_i = yhat_lstm * pd.DataFrame(valid_returns)* pd.DataFrame(valid_universe)
 data = pd.concat([pd.DataFrame(valid_time), x_t_i], axis=1)
 data.columns = ['day','x_t_i']
x_t = data.groupby('day' ).sum().values.flatten()
mean = np.mean(x_t)
 std = np.std(x_t)
 score_valid = mean / std
print(score_valid )<drop_column>
gp.to_csv('aggregated_features.csv', index=False )
del history, train_X, train_y, valid_X, valid_y
gc.collect()<prepare_output>
del all_samples
del all_periods
del train
del test
gc.collect()
x_t_i = pd.DataFrame(( yhat_lgbm + yhat_goss + yhat_dart)/3)* pd.DataFrame(valid_returns)* pd.DataFrame(valid_universe)
 data = pd.concat([pd.DataFrame(valid_time), x_t_i], axis=1)
 data.columns = ['day','x_t_i']
x_t = data.groupby('day' ).sum().values.flatten()
mean = np.mean(x_t)
 std = np.std(x_t)
 score_valid = mean / std
print(score_valid )<prepare_output>
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 train = train.merge(gp, on='user_id', how='left')
 test = test.merge(gp, on='user_id', how='left')
 agg_cols = list(gp.columns)[1:]
del gp
gc.collect()
train.head()
x_t_i =(yhat_lstm + pd.DataFrame(yhat_lgbm + yhat_goss + yhat_dart)) /4 * pd.DataFrame(valid_returns)* pd.DataFrame(valid_universe)
 data = pd.concat([pd.DataFrame(valid_time), x_t_i], axis=1)
 data.columns = ['day','x_t_i']
x_t = data.groupby('day' ).sum().values.flatten()
mean = np.mean(x_t)
 std = np.std(x_t)
 score_valid = mean / std
print(score_valid )<create_dataframe>
train[agg_cols].isna().any(axis=1 ).sum() / len(train)* 100
yhat_lgbm = pd.DataFrame(( yhat_lgbm + 1)/ 2)
 yhat_goss = pd.DataFrame(( yhat_goss + 1)/ 2)
 yhat_dart = pd.DataFrame(( yhat_dart + 1)/ 2)
 yhat_lstm =(yhat_lstm + 1)/ 2<feature_engineering>
test[agg_cols].isna().any(axis=1 ).sum() / len(test)* 100
truth =(np.sign(valid_returns)+ 1)/ 2<data_type_conversions>
count = lambda l1,l2: sum([1 for x in l1 if x in l2])
 for df in [train, test]:
df['description'].fillna('unknowndescription', inplace=True)
 df['title'].fillna('unknowntitle', inplace=True)
 df['weekday'] = pd.to_datetime(df['activation_date'] ).dt.day
for col in ['description', 'title']:
df['num_words_' + col] = df[col].apply(lambda comment: len(comment.split()))
df['num_unique_words_' + col] = df[col].apply(lambda comment: len(set(w for w in comment.split())))
 df['words_vs_unique_title'] = df['num_unique_words_title'] / df['num_words_title'] * 100
df['words_vs_unique_description'] = df['num_unique_words_description'] / df['num_words_description'] * 100
df['city'] = df['region'] + '_' + df['city']
df['num_desc_punct'] = df['description'].apply(lambda x: count(x, set(string.punctuation)))
 for col in agg_cols:
df[col].fillna(-1, inplace=True )
ensemble = pd.concat([yhat_lgbm, yhat_goss, yhat_dart, yhat_lstm], axis=1)
 ensemble.columns = ['lgbm','goss','dart', 'lstm']
truth = pd.DataFrame(truth ).astype('int')
 truth.columns = ['y']<train_model>
count_vectorizer_title = CountVectorizer(stop_words=stopwords.words('russian'), lowercase=True, min_df=25)
 title_counts = count_vectorizer_title.fit_transform(train['title'].append(test['title']))
train_title_counts = title_counts[:len(train)]
test_title_counts = title_counts[len(train):]
count_vectorizer_desc = TfidfVectorizer(stop_words=stopwords.words('russian'),
lowercase=True, ngram_range=(1, 2),
max_features=15000)
 desc_counts = count_vectorizer_desc.fit_transform(train['description'].append(test['description']))
train_desc_counts = desc_counts[:len(train)]
test_desc_counts = desc_counts[len(train):]
train_title_counts.shape, train_desc_counts.shape
logreg = LogisticRegression()
logreg.fit(ensemble, truth)
 logreg.score(ensemble, truth )<prepare_output>
target = 'deal_probability'
predictors = [
'num_desc_punct',
'words_vs_unique_description', 'num_unique_words_description', 'num_unique_words_title', 'num_words_description', 'num_words_title',
'avg_times_up_user', 'avg_days_up_user', 'n_user_items',
'price', 'item_seq_number'
]
categorical = [
'image_top_1', 'param_1', 'param_2', 'param_3',
'city', 'region', 'category_name', 'parent_category_name', 'user_type'
]
predictors = predictors + categorical
yhat_ens =(pd.DataFrame(logreg.predict_proba(ensemble)[:,1])* 2)- 1
x_t_i = yhat_ens * pd.DataFrame(valid_returns)* pd.DataFrame(valid_universe)
 data = pd.concat([pd.DataFrame(valid_time), x_t_i], axis=1)
 data.columns = ['day','x_t_i']
x_t = data.groupby('day' ).sum().values.flatten()
mean = np.mean(x_t)
 std = np.std(x_t)
 score_valid = mean / std
print(score_valid )<define_variables>
for feature in categorical:
print(f'Transforming {feature}...')
 encoder = LabelEncoder()
encoder.fit(train[feature].append(test[feature] ).astype(str))
train[feature] = encoder.transform(train[feature].astype(str))
test[feature] = encoder.transform(test[feature].astype(str))
days = env.get_prediction_days()<drop_column>
rounds = 16000
early_stop_rounds = 500
params = {
'objective' : 'regression',
'metric' : 'rmse',
'num_leaves' : 32,
'max_depth': 15,
'learning_rate' : 0.02,
'feature_fraction' : 0.6,
'verbosity' : -1
}
feature_names = np.hstack([
count_vectorizer_desc.get_feature_names() ,
count_vectorizer_title.get_feature_names() ,
predictors
])
 print('Number of features:', len(feature_names))
del active_history['universe']
del active_history['returnsOpenNextMktres10']
del active_history['TARGET']<define_variables>
train_index, valid_index = train_test_split(np.arange(len(train)) , test_size=0.1, random_state=42)
 x_train = scipy.sparse.hstack([
train_desc_counts[train_index],
train_title_counts[train_index],
train.loc[train_index, predictors]
], format='csr')
 y_train = train.loc[train_index, target]
x_valid = scipy.sparse.hstack([
train_desc_counts[valid_index],
train_title_counts[valid_index],
train.loc[valid_index, predictors]
], format='csr')
 y_valid = train.loc[valid_index, target]
x_test = scipy.sparse.hstack([
test_desc_counts,
test_title_counts,
test.loc[:, predictors]
], format='csr')
 dtrain = lgb.Dataset(x_train, label=y_train,
feature_name=list(feature_names),
categorical_feature=categorical)
 dvalid = lgb.Dataset(x_valid, label=y_valid,
feature_name=list(feature_names),
categorical_feature=categorical )
LSTM_COLUMNS_TO_USE = [col for col in LSTM_COLUMNS_TO_USE if(col!='universe' and col!='returnsOpenNextMktres10')]<data_type_conversions>
evals_result = {}
model = lgb.train(params, dtrain,
valid_sets=[dtrain, dvalid],
valid_names=['train', 'valid'],
num_boost_round=rounds,
early_stopping_rounds=early_stop_rounds,
verbose_eval=500 )
<save_to_csv><EOS> 
subm = pd.read_csv('.. /input/sample_submission.csv')
 subm['deal_probability'] = np.clip(model.predict(x_test), 0, 1)
 subm.to_csv('submission.csv', index=False )