Kaggle competition: Avito Demand Prediction Challenge
<SOS> metric: RMSE Kaggle data source: avito-demand-prediction-challenge<compute_test_metric>
os.environ['TF_CPP_MIN_LOG_LEVEL']='2'
warnings.simplefilter(action='ignore', category=FutureWarning)
 @contextmanager
def timer(name):
t0 = time.time()
yield
print(f'[{name}] done in {time.time() - t0:.0f} s')
 with timer('reading data'):
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 with timer('imputation'):
train['param_1'].fillna('missing', inplace=True)
 test['param_1'].fillna('missing', inplace=True)
 train['param_2'].fillna('missing', inplace=True)
 test['param_2'].fillna('missing', inplace=True)
 train['param_3'].fillna('missing', inplace=True)
 test['param_3'].fillna('missing', inplace=True)
 train['image_top_1'].fillna(0, inplace=True)
 test['image_top_1'].fillna(0, inplace=True)
 train['price'].fillna(0, inplace=True)
 test['price'].fillna(0, inplace=True)
 train['price'] = np.log1p(train['price'])
 test['price'] = np.log1p(test['price'])
 price_mean = train['price'].mean()
price_std = train['price'].std()
train['price'] =(train['price'] - price_mean)/ price_std
test['price'] =(test['price'] - price_mean)/ price_std
train['description'].fillna('', inplace=True)
 test['description'].fillna('', inplace=True)
 train['city'] = train['city'] + '_' + train['region']
test['city'] = test['city'] + '_' + test['region']
with timer('add new features'):
cat_cols = ['region', 'city', 'parent_category_name', 'category_name', 'param_1', 'param_2', 'param_3', 'user_type']
num_cols = ['price', 'deal_probability']
for c in cat_cols:
for c2 in num_cols:
enc = train.groupby(c)[c2].agg(['mean'] ).astype(np.float32 ).reset_index()
enc.columns = ['_'.join([str(c), str(c2), str(c3)])if c3 != c else c for c3 in enc.columns]
train = pd.merge(train, enc, how='left', on=c)
 test = pd.merge(test, enc, how='left', on=c)
 del(enc)
 train.head()
roc_auc_score(y_train>0.5,oof_preds )<load_from_csv>
def preprocess(df: pd.DataFrame)-> pd.DataFrame:
ex_col = ['item_id', 'user_id', 'deal_probability', 'title', 'param_1', 'param_2', 'param_3', 'activation_date']
df['description_len'] = df['description'].map(lambda x: len(str(x)) ).astype(np.float16)
 df['description_wc'] = df['description'].map(lambda x: len(str(x ).split(' ')) ).astype(np.float16)
 df['description'] =(df['parent_category_name'] + ' ' + df['category_name'] + ' ' + df['param_1'] + ' ' + df['param_2'] + ' ' + df['param_3'] + ' ' +
df['title'] + ' ' + df['description'].fillna(''))
df['description'] = df['description'].str.lower().replace(r"[^[:alpha:]]", " ")
 df['description'] = df['description'].str.replace(r"\\s+", " ")
 df['title_len'] = df['title'].map(lambda x: len(str(x)) ).astype(np.float16)
 df['title_wc'] = df['title'].map(lambda x: len(str(x ).split(' ')) ).astype(np.float16)
 df['image'] = df['image'].map(lambda x: 1 if len(str(x)) >0 else 0)
 df['price'] = np.log1p(df['price'].fillna(0))
df['wday'] = pd.to_datetime(df['activation_date'] ).dt.dayofweek
col = [c for c in df.columns if c not in ex_col]
return df[col]
with timer('process train'):
train, valid = train_test_split(train, test_size=0.05, shuffle=True, random_state=37)
 y_train = train['deal_probability'].values
X_train = preprocess(train)
 print(f'X_train: {X_train.shape}')
 with timer('process valid'):
X_valid = preprocess(valid)
 print(f'X_valid: {X_valid.shape}')
 with timer('process test'):
X_test = preprocess(test)
 print(f'X_test: {X_test.shape}')
 X_train.head()
submission = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/sample_submission.csv', index_col='id')
 submission['prediction'] = test_preds
submission.reset_index(drop=False, inplace=True)
 submission.head() 
<save_to_csv>
desc_len_mean = X_train['description_len'].mean()
desc_len_std = X_train['description_len'].std()
X_train['description_len'] =(X_train['description_len'] - desc_len_mean)/ desc_len_std
X_valid['description_len'] =(X_valid['description_len'] - desc_len_mean)/ desc_len_std
X_test['description_len'] =(X_test['description_len'] - desc_len_mean)/ desc_len_std
desc_wc_mean = X_train['description_wc'].mean()
desc_wc_std = X_train['description_wc'].std()
X_train['description_wc'] =(X_train['description_wc'] - desc_wc_mean)/ desc_wc_std
X_valid['description_wc'] =(X_valid['description_wc'] - desc_wc_mean)/ desc_wc_std
X_test['description_wc'] =(X_test['description_wc'] - desc_wc_mean)/ desc_wc_std
title_len_mean = X_train['title_len'].mean()
title_len_std = X_train['title_len'].std()
X_train['title_len'] =(X_train['title_len'] - title_len_mean)/ title_len_std
X_valid['title_len'] =(X_valid['title_len'] - title_len_mean)/ title_len_std
X_test['title_len'] =(X_test['title_len'] - title_len_mean)/ title_len_std
title_wc_mean = X_train['title_wc'].mean()
title_wc_std = X_train['title_wc'].std()
X_train['title_wc'] =(X_train['title_wc'] - title_wc_mean)/ title_wc_std
X_valid['title_wc'] =(X_valid['title_wc'] - title_wc_mean)/ title_wc_std
X_test['title_wc'] =(X_test['title_wc'] - title_wc_mean)/ title_wc_std
image_top_1_mean = X_train['image_top_1'].mean()
image_top_1_std = X_train['image_top_1'].std()
X_train['image_top_1'] =(X_train['image_top_1'] - image_top_1_mean)/ image_top_1_std
X_valid['image_top_1'] =(X_valid['image_top_1'] - image_top_1_mean)/ image_top_1_std
X_test['image_top_1'] =(X_test['image_top_1'] - image_top_1_mean)/ image_top_1_std
submission.to_csv('submission.csv', index=False )<save_to_csv>
X_train.fillna(0, inplace=True)
 X_valid.fillna(0, inplace=True)
 X_test.fillna(0, inplace=True )
submission.to_csv('submission.csv', index=False )<import_modules>
with timer('TFIDF'):
tfidf = TfidfVectorizer(ngram_range=(1, 2),
max_features=100000,
token_pattern='\w+',
encoding='KOI8-R')
 tfidf_train = tfidf.fit_transform(X_train['description'])
 tfidf_valid = tfidf.transform(X_valid['description'])
 tfidf_test = tfidf.transform(X_test['description'] )
import numpy as np
import pandas as pd
import os
import time
import gc
import random
from tqdm._tqdm_notebook import tqdm_notebook as tqdm
from keras.preprocessing import text, sequence
import torch
from torch import nn
from torch.utils import data
from torch.nn import functional as F<set_options>
with timer('Dummy'):
dummy_cols = ['parent_category_name', 'category_name', 'user_type', 'image_top_1', 'wday', 'region', 'city']
for col in dummy_cols:
le = LabelEncoder()
le.fit(X_train[col] + X_valid[col] + X_test[col])
 le.fit(list(X_train[col].values.astype('str')) + list(X_valid[col].values.astype('str')) + list(X_test[col].values.astype('str')))
 X_train[col] = le.transform(list(X_train[col].values.astype('str')))
 X_valid[col] = le.transform(list(X_valid[col].values.astype('str')))
 X_test[col] = le.transform(list(X_test[col].values.astype('str')))
 with timer('Dropping'):
X_train.drop('description', axis=1, inplace=True)
 X_valid.drop('description', axis=1, inplace=True)
 X_test.drop('description', axis=1, inplace=True)
 with timer('OHE'):
ohe = OneHotEncoder(categorical_features=[X_train.columns.get_loc(c)for c in dummy_cols])
 X_train = ohe.fit_transform(X_train)
 print(f'X_train: {X_train.shape}')
 X_valid = ohe.transform(X_valid)
 print(f'X_valid: {X_valid.shape}')
 X_test = ohe.transform(X_test)
 print(f'X_test: {X_test.shape}' )
def seed_everything(seed=1234):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()<define_variables>
def huber_loss(y_true, y_pred, clip_delta=1.0):
error = y_true - y_pred
cond = tf.keras.backend.abs(error)< clip_delta
squared_loss = 0.5 * tf.keras.backend.square(error)
 linear_loss = clip_delta *(tf.keras.backend.abs(error)- 0.5 * clip_delta)
 return tf.where(cond, squared_loss, linear_loss)
 def fit_predict(xs, y_train, loss_fn='mean_squared_error')-> np.ndarray:
X_train, X_test = xs
config = tf.ConfigProto(
 intra_op_parallelism_threads=4, use_per_session_threads=4, inter_op_parallelism_threads=4)
 with tf.Session(graph=tf.Graph() , config=config)as sess, timer('fit_predict'):
ks.backend.set_session(sess)
 model_in = ks.Input(shape=(X_train.shape[1],), dtype='float32', sparse=True)
 out = ks.layers.Dense(192, activation='relu' )(model_in)
 out = ks.layers.Dense(64, activation='relu' )(out)
 out = ks.layers.Dense(64, activation='relu' )(out)
 out = ks.layers.Dense(1 )(out)
 model = ks.Model(model_in, out)
 model.compile(loss=loss_fn, optimizer=ks.optimizers.Adam(lr=2e-3))
for i in range(3):
with timer(f'epoch {i + 1}'):
model.fit(x=X_train, y=y_train, batch_size=2**(8 + i), epochs=1, verbose=0)
 return model.predict(X_test, batch_size=2**(8 + i)) [:, 0]
X_train = X_train.tocsr()
X_valid = X_valid.tocsr()
X_test = X_test.tocsr()
X_train_1, X_train_2, y_train_1, y_train_2 = train_test_split(X_train, y_train, test_size = 0.5, shuffle = False)
 preds_oofs = []
preds_valids = []
preds_tests = []
for r in range(8):
with timer('Round {}'.format(r)) :
if r % 2 == 0:
loss_name = 'huber_loss'
loss = huber_loss
else:
loss_name = 'mean_squared_error'
loss = 'mean_squared_error'
if r >= 4:
print('Running loss = {}, binary = True'.format(loss_name))
xs = [x.astype(np.bool ).astype(np.float32)for x in [X_train_1, X_train_2]]
y_pred1 = fit_predict(xs, y_train=y_train_1, loss_fn=loss)
 xs = [x.astype(np.bool ).astype(np.float32)for x in [X_train_2, X_train_1]]
y_pred2 = fit_predict(xs, y_train=y_train_2, loss_fn=loss)
 xs = [x.astype(np.bool ).astype(np.float32)for x in [X_train_1, X_valid]]
y_predf = fit_predict(xs, y_train=y_train_1, loss_fn=loss)
 xs = [x.astype(np.bool ).astype(np.float32)for x in [X_train_1, X_test]]
y_predt = fit_predict(xs, y_train=y_train_1, loss_fn=loss)
 else:
print('Running loss = {}, binary = False'.format(loss_name))
xs = [X_train_1, X_train_2]
y_pred1 = fit_predict(xs, y_train=y_train_1, loss_fn=loss)
 xs = [X_train_2, X_train_1]
y_pred2 = fit_predict(xs, y_train=y_train_2, loss_fn=loss)
 xs = [X_train_1, X_valid]
y_predf = fit_predict(xs, y_train=y_train_1, loss_fn=loss)
 xs = [X_train_1, X_test]
y_predt = fit_predict(xs, y_train=y_train_1, loss_fn=loss)
 preds_oof = np.concatenate(( y_pred2, y_pred1), axis=0)
 preds_valid = y_predf
preds_test = y_predt
print('Round {} OOF RMSE: {:.4f}'.format(r, np.sqrt(mean_squared_error(train['deal_probability'], preds_oof))))
print('Round {} Valid RMSE: {:.4f}'.format(r, np.sqrt(mean_squared_error(valid['deal_probability'], preds_valid))))
preds_oofs.append(preds_oof)
 preds_valids.append(preds_valid)
 preds_tests.append(preds_test )
CRAWL_EMBEDDING_PATH = '.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec'
GLOVE_EMBEDDING_PATH = '.. /input/glove840b300dtxt/glove.840B.300d.txt'
NUM_MODELS = 2
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
MAX_LEN = 220<feature_engineering>
preds_oof = np.mean(preds_oofs, axis=0)
 print('Overall OOF RMSE: {:.4f}'.format(np.sqrt(mean_squared_error(train['deal_probability'], preds_oof))))
preds_valid = np.mean(preds_valids, axis=0)
 print('Overall Valid RMSE: {:.4f}'.format(np.sqrt(mean_squared_error(valid['deal_probability'], preds_valid))))
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
 def load_embeddings(path):
with open(path)as f:
return dict(get_coefs(*line.strip().split(' ')) for line in tqdm(f))
def build_matrix(word_index, path):
embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( len(word_index)+ 1, 300))
unknown_words = []
for word, i in word_index.items() :
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
unknown_words.append(word)
 return embedding_matrix, unknown_words<train_model>
np.mean(np.corrcoef(preds_oofs), axis=0 )
def sigmoid(x):
return 1 /(1 + np.exp(-x))
def train_model(model, train, test, loss_fn, output_dim, lr=0.001,
batch_size=512, n_epochs=4,
enable_checkpoint_ensemble=True):
param_lrs = [{'params': param, 'lr': lr} for param in model.parameters() ]
optimizer = torch.optim.Adam(param_lrs, lr=lr)
 scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lambda epoch: 0.6 ** epoch)
 train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)
 test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)
 all_test_preds = []
checkpoint_weights = [2 ** epoch for epoch in range(n_epochs)]
for epoch in range(n_epochs):
start_time = time.time()
scheduler.step()
model.train()
avg_loss = 0.
 for data in tqdm(train_loader, disable=False):
x_batch = data[:-1]
y_batch = data[-1]
y_pred = model(*x_batch)
 loss = loss_fn(y_pred, y_batch)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
avg_loss += loss.item() / len(train_loader)
 model.eval()
test_preds = np.zeros(( len(test), output_dim))
for i, x_batch in enumerate(test_loader):
y_pred = sigmoid(model(*x_batch ).detach().cpu().numpy())
 test_preds[i * batch_size:(i+1)* batch_size, :] = y_pred
all_test_preds.append(test_preds)
 elapsed_time = time.time() - start_time
print('Epoch {}/{} \t loss={:.4f} \t time={:.2f}s'.format(
 epoch + 1, n_epochs, avg_loss, elapsed_time))
if enable_checkpoint_ensemble:
test_preds = np.average(all_test_preds, weights=checkpoint_weights, axis=0)
 else:
test_preds = all_test_preds[-1]
return test_preds<normalization>
with timer('reading data'):
train = pd.read_csv('.. /input/train.csv')
 test = pd.read_csv('.. /input/test.csv')
 target = train['deal_probability']
with timer('imputation'):
train['param_1'].fillna('missing', inplace=True)
 test['param_1'].fillna('missing', inplace=True)
 train['param_2'].fillna('missing', inplace=True)
 test['param_2'].fillna('missing', inplace=True)
 train['param_3'].fillna('missing', inplace=True)
 test['param_3'].fillna('missing', inplace=True)
 train['price'].fillna(0, inplace=True)
 test['price'].fillna(0, inplace=True)
 train['city'] = train['city'] + '_' + train['region']
test['city'] = test['city'] + '_' + test['region']
with timer('FE'):
trainp = preprocess(train)
 testp = preprocess(test)
 for col in ['param_1', 'param_2', 'param_3']:
trainp[col] = train[col]
testp[col] = test[col]
print(train.shape)
 train.head()
class SpatialDropout(nn.Dropout2d):
def forward(self, x):
x = x.unsqueeze(2)
 x = x.permute(0, 3, 2, 1)
 x = super(SpatialDropout, self ).forward(x)
 x = x.permute(0, 3, 2, 1)
 x = x.squeeze(2)
 return x
class NeuralNet(nn.Module):
def __init__(self, embedding_matrix, num_aux_targets):
super(NeuralNet, self ).__init__()
embed_size = embedding_matrix.shape[1]
self.embedding = nn.Embedding(max_features, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = SpatialDropout(0.3)
 self.lstm1 = nn.LSTM(embed_size, LSTM_UNITS, bidirectional=True, batch_first=True)
 self.lstm2 = nn.LSTM(LSTM_UNITS * 2, LSTM_UNITS, bidirectional=True, batch_first=True)
 self.linear1 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)
 self.linear2 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)
 self.linear_out = nn.Linear(DENSE_HIDDEN_UNITS, 1)
 self.linear_aux_out = nn.Linear(DENSE_HIDDEN_UNITS, num_aux_targets)
 def forward(self, x):
h_embedding = self.embedding(x)
 h_embedding = self.embedding_dropout(h_embedding)
 h_lstm1, _ = self.lstm1(h_embedding)
 h_lstm2, _ = self.lstm2(h_lstm1)
 avg_pool = torch.mean(h_lstm2, 1)
 max_pool, _ = torch.max(h_lstm2, 1)
 h_conc = torch.cat(( max_pool, avg_pool), 1)
 h_conc_linear1 = F.relu(self.linear1(h_conc))
h_conc_linear2 = F.relu(self.linear2(h_conc))
hidden = h_conc + h_conc_linear1 + h_conc_linear2
result = self.linear_out(hidden)
 aux_result = self.linear_aux_out(hidden)
 out = torch.cat([result, aux_result], 1)
 return out<categorify>
with timer('drop'):
trainp.drop(['description', 'image'], axis=1, inplace=True)
 testp.drop(['description', 'image'], axis=1, inplace=True)
 print(trainp.shape)
 print(testp.shape )
def preprocess(data):

 punct = "/-'?!.,
def clean_special_chars(text, punct):
for p in punct:
text = text.replace(p, ' ')
 return text
data = data.astype(str ).apply(lambda x: clean_special_chars(x, punct))
return data<prepare_x_and_y>
with timer('To cat'):
trainp['image_top_1'] = trainp['image_top_1'].astype('str' ).fillna('missing')
 testp['image_top_1'] = testp['image_top_1'].astype('str' ).fillna('missing')
 cat_cols = ['region', 'city', 'parent_category_name', 'category_name',
'param_1', 'param_2', 'param_3', 'user_type', 'image_top_1', 'wday']
for col in trainp.columns:
print(col)
 if col in cat_cols:
trainp[col] = trainp[col].astype('category')
 testp[col] = testp[col].astype('category')
 else:
trainp[col] = trainp[col].astype(np.float64)
 testp[col] = testp[col].astype(np.float64 )
train = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv')
 test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv')
 x_train = preprocess(train['comment_text'])
 y_train = np.where(train['target'] >= 0.5, 1, 0)
 y_aux_train = train[['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']]
x_test = preprocess(test['comment_text'] )<define_variables>
with timer('Split'):
train, valid, y_train, y_valid = train_test_split(trainp, target, test_size=0.05, shuffle=True, random_state=37)
 test = testp
print(train.shape)
 print(valid.shape)
 print(test.shape )
max_features = None<string_transform>
with timer('Submodels'):
train_models = pd.DataFrame(np.array(preds_oofs ).transpose())
 valid_models = pd.DataFrame(np.array(preds_valids ).transpose())
 test_models = pd.DataFrame(np.array(preds_tests ).transpose())
 train_models.columns = ['nn_' + str(i + 1)for i in range(train_models.shape[1])]
valid_models.columns = ['nn_' + str(i + 1)for i in range(train_models.shape[1])]
test_models.columns = ['nn_' + str(i + 1)for i in range(train_models.shape[1])]
print(train_models.shape)
 print(valid_models.shape)
 print(test_models.shape )
tokenizer = text.Tokenizer()
tokenizer.fit_on_texts(list(x_train)+ list(x_test))
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_train = sequence.pad_sequences(x_train, maxlen=MAX_LEN)
 x_test = sequence.pad_sequences(x_test, maxlen=MAX_LEN )<define_variables>
with timer('Concat'):
print(train.shape)
 X_train = pd.concat([train.reset_index() , train_models.reset_index() ], axis=1)
 print(X_train.shape)
 print('-')
 print(valid.shape)
 X_valid = pd.concat([valid.reset_index() , valid_models.reset_index() ], axis=1)
 print(X_valid.shape)
 print('-')
 print(test.shape)
 X_test = pd.concat([test.reset_index() , test_models.reset_index() ], axis=1)
 print(X_test.shape)
 X_train.head()
max_features = max_features or len(tokenizer.word_index)+ 1
max_features<choose_model_class>
del trainp
del testp
gc.collect()
crawl_matrix, unknown_words_crawl = build_matrix(tokenizer.word_index, CRAWL_EMBEDDING_PATH)
 print('n unknown words(crawl): ', len(unknown_words_crawl))<choose_model_class>
d_train = lgb.Dataset(X_train, label=y_train)
 d_valid = lgb.Dataset(X_valid, label=y_valid)
 watchlist = [d_train, d_valid]
params = {'application': 'regression',
'metric': 'rmse',
'nthread': 3,
'verbosity': -1,
'data_random_seed': 3,
'learning_rate': 0.05,
'num_leaves': 31,
'bagging_fraction': 0.8,
'feature_fraction': 0.2,
'lambda_l1': 3,
'lambda_l2': 3,
'min_data_in_leaf': 40}
model = lgb.train(params,
train_set=d_train,
num_boost_round=1500,
valid_sets=watchlist,
verbose_eval=100)
 pprint(sorted(list(zip(model.feature_importance() , X_train.columns)) , reverse=True))
print('Done' )
<concatenate><EOS> 
valid_preds = model.predict(X_valid ).clip(0, 1)
 print('Overall Valid RMSE: {:.4f}'.format(np.sqrt(mean_squared_error(y_valid, valid_preds))))
test_preds = model.predict(X_test ).clip(0, 1)
 submission = pd.read_csv('.. /input/test.csv', usecols=["item_id"])
 submission["deal_probability"] = test_preds
submission.to_csv("submit_boosting_mlp.csv", index=False, float_format="%.2g" )