Kaggle competition: Avito Demand Prediction Challenge
<SOS> metric: RMSE Kaggle data source: avito-demand-prediction-challenge<prepare_output>
import pandas as pd
import numpy as np
import re
import gc
from keras.preprocessing import text, sequence
import math
from xgboost.sklearn import XGBRegressor
import xgboost as xgb
from sklearn import metrics
import matplotlib.pyplot as plt
preds = train_and_eval()<load_from_csv>
train = pd.read_csv('.. /input/readyforuse/trainforuse_mix.csv')
 test = pd.read_csv('.. /input/readyforuse/testforuse_mix.csv')
 submission = pd.read_csv('.. /input/avito-demand-prediction/sample_submission.csv')
 train = train.drop(columns = ['item_id'])
 train['date'] = pd.to_datetime(train['activation_date'] ).dt.weekday.astype('int')
 train = train.drop(columns = ['activation_date','image'])
 test = test.drop(columns = ['item_id'])
 test['date'] = pd.to_datetime(test['activation_date'] ).dt.weekday.astype('int')
 test = test.drop(columns = ['activation_date','image'] )
sub = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/sample_submission.csv', index_col='id')
 sub['prediction'] = preds
sub.reset_index(drop=False, inplace=True)
 sub.head()<save_to_csv>
encoding_predictors = ['user_id', 'region', 'city', 'parent_category_name', 'category_name', 'param_1', 'param_2', 'param_3', 'item_seq_number', 'user_type', 'date']
sub.to_csv('submission.csv', index=False )<save_to_csv>
for i in encoding_predictors:
print(str(i)+': ' + str(len(set(train[i]))))
print('Na contains? ' + str(train[i].isnull().values.any()))
sub.to_csv('submission.csv', index=False )<set_options>
for i in encoding_predictors:
print(str(i)+': ' + str(len(set(test[i]))))
print('Na contains? ' + str(test[i].isnull().values.any()))
tqdm.pandas()<load_from_csv>
l_region = dict(zip(list(set(train['region'])) ,range(1,29)))
 l_parent_category_name = dict(zip(list(set(train['parent_category_name'])) ,range(1,10)))
 l_category_name = dict(zip(list(set(train['category_name'])) ,range(1,48)))
 l_user_type = dict(zip(list(set(train['user_type'])) ,range(1,4)))
 l_date = dict(zip(list(set(train['date'])) ,range(1,8)))
 l_param1 = dict(zip(list(set(train['param_1'])) ,range(1,373)))
 l_param2 = dict(zip(list(set(train['param_2'])) ,range(1,273)))
 l_city = dict(zip(list(set(train['city'])) ,range(1,1734)))
 l_param3 = dict(zip(list(set(train['param_3'])) ,range(1,1221)) )
TEXT_COL = 'comment_text'
EMB_PATH = '.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec'
train = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv', index_col='id')
 test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv', index_col='id')
<categorify>
train['region'] = train['region'].replace(l_region)
 train['parent_category_name'] = train['parent_category_name'].replace(l_parent_category_name)
 train['category_name'] = train['category_name'].replace(l_category_name)
 train['user_type'] = train['user_type'].replace(l_user_type)
 train['date'] = train['date'].replace(l_date)
 train['param_1'] = train['param_1'].replace(l_param1)
 train['param_2'] = train['param_2'].replace(l_param2)
 train['city'] = train['city'].replace(l_city)
 train['param_3'] = train['param_3'].replace(l_param3 )
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 def load_embeddings(embed_dir=EMB_PATH):
embedding_index = dict(get_coefs(*o.strip().split(" ")) for o in tqdm(open(embed_dir)))
 return embedding_index
def build_embedding_matrix(word_index, embeddings_index, max_features, lower = True, verbose = True):
embedding_matrix = np.zeros(( max_features, 300))
for word, i in tqdm(word_index.items() ,disable = not verbose):
if lower:
word = word.lower()
if i >= max_features: continue
try:
embedding_vector = embeddings_index[word]
except:
embedding_vector = embeddings_index["unknown"]
if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
return embedding_matrix
def build_matrix(word_index, embeddings_index):
embedding_matrix = np.zeros(( len(word_index)+ 1,300))
for word, i in word_index.items() :
try:
embedding_matrix[i] = embeddings_index[word]
except:
embedding_matrix[i] = embeddings_index["unknown"]
return embedding_matrix<prepare_x_and_y>
test['region'] = test['region'].map(l_region)
 test['parent_category_name'] = test['parent_category_name'].map(l_parent_category_name)
 test['category_name'] = test['category_name'].map(l_category_name)
 test['user_type'] = test['user_type'].map(l_user_type)
 test['date'] = test['date'].map(l_date)
 test['param_1'] = test['param_1'].map(l_param1)
 test['param_2'] = test['param_2'].map(l_param2)
 test['city'] = test['city'].map(l_city)
 test['param_3'] = test['param_3'].map(l_param3 )
maxlen = 220
max_features = 100000
embed_size = 300
tokenizer = Tokenizer(num_words=max_features, lower=True)
 print('fitting tokenizer')
 tokenizer.fit_on_texts(list(train[TEXT_COL])+ list(test[TEXT_COL]))
word_index = tokenizer.word_index
X_train = tokenizer.texts_to_sequences(list(train[TEXT_COL]))
y_train = train['target'].values
X_test = tokenizer.texts_to_sequences(list(test[TEXT_COL]))
X_train = pad_sequences(X_train, maxlen=maxlen)
 X_test = pad_sequences(X_test, maxlen=maxlen)
 del tokenizer
gc.collect() 
<load_pretrained>
def modelfit(alg,dtrain,predictors,useTrainCV = True, cv_folds = 5, early_stopping_rounds = 50):
if useTrainCV:
xgb_param = alg.get_xgb_params()
xgtrain = xgb.DMatrix(dtrain[predictors].values, label = dtrain['deal_probability'].values, feature_names = predictors)
 cvresult = xgb.cv(xgb_param, xgtrain, num_boost_round = alg.get_params() ['n_estimators'], nfold = cv_folds, metrics = 'rmse', early_stopping_rounds = early_stopping_rounds)
 alg.set_params(n_estimators = cvresult.shape[0])
 print('Best n_estimator = ' + str(cvresult.shape[0]))
alg.fit(dtrain[predictors], dtrain['deal_probability'], eval_metric = 'rmse')
 dtrain_predictions = alg.predict(dtrain[predictors])
 print('
Model Report:')
 print('RMSE: %f' % math.sqrt(metrics.mean_squared_error(dtrain['deal_probability'].values, dtrain_predictions)) )
embeddings_index = load_embeddings() 
<prepare_x_and_y>
train.loc[:,'image_item_n'] = train.groupby(['image_top_1','item_seq_number'] ).user_id.transform('nunique')
 train.loc[:,'image_city_n'] = train.groupby(['city','image_top_1'] ).user_id.transform('nunique')
 train.loc[:,'image_region_n'] = train.groupby(['region','image_top_1'] ).user_id.transform('nunique')
 train.loc[:,'image_categoryname_n'] = train.groupby(['category_name','image_top_1'] ).user_id.transform('nunique')
 train.loc[:,'image_param2_n'] = train.groupby(['image_top_1','param_2'] ).user_id.transform('nunique')
 train.loc[:,'image_parentcategoryname_n'] = train.groupby(['parent_category_name','image_top_1'] ).user_id.transform('nunique')
 train.loc[:,'image_date_n'] = train.groupby(['image_top_1','date'] ).user_id.transform('nunique')
 train.loc[:,'image_usertype_n'] = train.groupby(['image_top_1','user_type'] ).user_id.transform('nunique')
 train.loc[:,'image_param1_n'] = train.groupby(['image_top_1','param_1'] ).user_id.transform('nunique')
 train.loc[:,'image_param3_n'] = train.groupby(['image_top_1','param_3'] ).user_id.transform('nunique' )
embedding_matrix = build_matrix(word_index, embeddings_index )<set_options>
test.loc[:,'image_item_n'] = test.groupby(['image_top_1','item_seq_number'] ).user_id.transform('nunique')
 test.loc[:,'image_city_n'] = test.groupby(['city','image_top_1'] ).user_id.transform('nunique')
 test.loc[:,'image_region_n'] = test.groupby(['region','image_top_1'] ).user_id.transform('nunique')
 test.loc[:,'image_categoryname_n'] = test.groupby(['category_name','image_top_1'] ).user_id.transform('nunique')
 test.loc[:,'image_param2_n'] = test.groupby(['image_top_1','param_2'] ).user_id.transform('nunique')
 test.loc[:,'image_parentcategoryname_n'] = test.groupby(['parent_category_name','image_top_1'] ).user_id.transform('nunique')
 test.loc[:,'image_date_n'] = test.groupby(['image_top_1','date'] ).user_id.transform('nunique')
 test.loc[:,'image_usertype_n'] = test.groupby(['image_top_1','user_type'] ).user_id.transform('nunique')
 test.loc[:,'image_param1_n'] = test.groupby(['image_top_1','param_1'] ).user_id.transform('nunique')
 test.loc[:,'image_param3_n'] = test.groupby(['image_top_1','param_3'] ).user_id.transform('nunique' )
del embeddings_index
gc.collect()<import_modules>
def clean(string):
string = re.sub(r'
', ' ', string)
 string = re.sub(r'\t', ' ', string)
 string = re.sub('[\W]', ' ', string)
 string = re.sub(r'\s{2,}', ' ', string.lower())
 return string
def find_punc(string):
string = re.sub(r'\s','',string)
 string = re.findall('[\W]',string)
 l = len(string)
 return l
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<choose_model_class>
train_t = train['title'].apply(clean)
 test_t = test['title'].apply(clean)
 train_d = train['description'].astype(str ).apply(clean)
 test_d = test['description'].astype(str ).apply(clean )
def build_model(verbose = False, compile = True):
sequence_input = L.Input(shape=(maxlen,), dtype='int32')
 embedding_layer = L.Embedding(len(word_index)+ 1,
300,
weights=[embedding_matrix],
input_length=maxlen,
trainable=False)
 x = embedding_layer(sequence_input)
 x = L.SpatialDropout1D(0.2 )(x)
 x = L.Bidirectional(L.CuDNNLSTM(64, return_sequences=True))(x)
 att = Attention(maxlen )(x)
 avg_pool1 = L.GlobalAveragePooling1D()(x)
 max_pool1 = L.GlobalMaxPooling1D()(x)
 x = L.concatenate([att,avg_pool1, max_pool1])
 preds = L.Dense(1, activation='sigmoid' )(x)
 model = Model(sequence_input, preds)
 if verbose:
model.summary()
if compile:
model.compile(loss='binary_crossentropy',optimizer=Adam(0.005),metrics=['acc'])
 return model 
<train_on_grid>
train_t_len = []
for line in train_t:
train_t_len.append(len(line.split()))
train_d_len = []
for line in train_d:
train_d_len.append(len(line.split()))
test_t_len = []
for line in test_t:
test_t_len.append(len(line.split()))
test_d_len = []
for line in test_d:
test_d_len.append(len(line.split()))
splits = list(KFold(n_splits=5 ).split(X_train,y_train))
BATCH_SIZE = 2048
NUM_EPOCHS = 100
oof_preds = np.zeros(( X_train.shape[0]))
test_preds = np.zeros(( X_test.shape[0]))
for fold in [0,1,2,3,4]:
K.clear_session()
tr_ind, val_ind = splits[fold]
ckpt = ModelCheckpoint(f'gru_{fold}.hdf5', save_best_only = True)
 es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=3)
 model = build_model()
model.fit(X_train[tr_ind],
y_train[tr_ind]>0.5,
batch_size=BATCH_SIZE,
epochs=NUM_EPOCHS,
validation_data=(X_train[val_ind], y_train[val_ind]>0.5),
callbacks = [es,ckpt])
 oof_preds[val_ind] += model.predict(X_train[val_ind])[:,0]
test_preds += model.predict(X_test)[:,0]
test_preds /= 5<compute_test_metric>
train['t_per'] = np.array(train['title'].apply(find_punc)) /np.array(train_t_len)
 test['t_per'] = np.array(test['title'].apply(find_punc)) /np.array(test_t_len)
 train['d_n'] = train_d_len
test['d_n'] = test_d_len
roc_auc_score(y_train>0.5,oof_preds )<load_from_csv>
train = train.drop(columns = ['title','description'])
 test = test.drop(columns = ['title','description'] )
submission = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/sample_submission.csv', index_col='id')
 submission['prediction'] = test_preds
submission.reset_index(drop=False, inplace=True)
 submission.head() 
<save_to_csv>
ready_train = train[['price','image_top_1','param_1','item_seq_number','city','region','parent_category_name','category_name','user_type','date','param_2','deal_probability','param_3','mix'] + list(train.columns[15:])]
ready_test = test[['price','image_top_1','param_1','item_seq_number','city','region','parent_category_name','category_name','user_type','date','param_2','param_3','mix'] + list(test.columns[14:])]
submission.to_csv('submission.csv', index=False )<save_to_csv>
predictors = ready_train.columns[ready_train.columns != 'deal_probability']
submission.to_csv('submission.csv', index=False )<set_options>
xgb1 = XGBRegressor(objective = 'reg:logistic', learning_rate = 0.1, n_estimators = 1000, max_depth = 5, min_child_weight = 6, gamma = 0, subsample = 0.9, colsample_bytree = 0.7, reg_alpha = 1.4, seed = 2018 )
%matplotlib inline
pd.set_option('max_colwidth',400)
 print(os.listdir(".. /input")) 
<load_from_csv>
modelfit(xgb1, ready_train, predictors, useTrainCV = False )
train = pd.read_csv('.. /input/jigsaw-public-files/train.csv')
 test = pd.read_csv('.. /input/jigsaw-public-files/test.csv')
 train['comment_text'] = train['comment_text'].fillna('')
 test['comment_text'] = test['comment_text'].fillna('')
 sub = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/sample_submission.csv' )<train_model>
pred = xgb1.predict(ready_test[predictors] )
<define_variables><EOS> 
submission['deal_probability'] = pred
submission.to_csv('submission.csv',index=False )