Kaggle competition: Avito Demand Prediction Challenge
<SOS> metric: RMSE Kaggle data source: avito-demand-prediction-challenge<prepare_x_and_y>
%%time
start_time = time.time()
SUBMIT_MODE = True
def rmse(predicted, actual):
return np.sqrt(((predicted - actual)** 2 ).mean() )
x_train_torch = torch.tensor(x_train, dtype=torch.long)
 y_train_torch = torch.tensor(np.hstack([y_train[:, np.newaxis], y_aux_train]), dtype=torch.float32 )<data_type_conversions>
%%time
class TargetEncoder:
def __repr__(self):
return 'TargetEncoder'
def __init__(self, cols, smoothing=1, min_samples_leaf=1, noise_level=0, keep_original=False):
self.cols = cols
self.smoothing = smoothing
self.min_samples_leaf = min_samples_leaf
self.noise_level = noise_level
self.keep_original = keep_original
@staticmethod
def add_noise(series, noise_level):
return series *(1 + noise_level * np.random.randn(len(series)))
 def encode(self, train, test, target):
for col in self.cols:
if self.keep_original:
train[col + '_te'], test[col + '_te'] = self.encode_column(train[col], test[col], target)
 else:
train[col], test[col] = self.encode_column(train[col], test[col], target)
 return train, test
def encode_column(self, trn_series, tst_series, target):
temp = pd.concat([trn_series, target], axis=1)
 averages = temp.groupby(by=trn_series.name)[target.name].agg(["mean", "count"])
 smoothing = 1 /(1 + np.exp(-(averages["count"] - self.min_samples_leaf)/ self.smoothing))
prior = target.mean()
averages[target.name] = prior *(1 - smoothing)+ averages["mean"] * smoothing
averages.drop(['mean', 'count'], axis=1, inplace=True)
 ft_trn_series = pd.merge(
 trn_series.to_frame(trn_series.name),
averages.reset_index().rename(columns={'index': target.name, target.name: 'average'}),
on=trn_series.name,
how='left')['average'].rename(trn_series.name + '_mean' ).fillna(prior)
 ft_trn_series.index = trn_series.index
ft_tst_series = pd.merge(
 tst_series.to_frame(tst_series.name),
averages.reset_index().rename(columns={'index': target.name, target.name: 'average'}),
on=tst_series.name,
how='left')['average'].rename(trn_series.name + '_mean' ).fillna(prior)
 ft_tst_series.index = tst_series.index
return self.add_noise(ft_trn_series, self.noise_level), self.add_noise(ft_tst_series, self.noise_level )
x_test_torch = torch.tensor(x_test, dtype=torch.long )<create_dataframe>
%%time
stopwords = {x: 1 for x in stopwords.words('russian')}
non_alphanums = re.compile(u'[^A-Za-z0-9]+')
 non_alphanumpunct = re.compile(u'[^A-Za-z0-9\.?!,; \(\)\[\]'"\$]+')
 RE_PUNCTUATION = '|'.join([re.escape(x)for x in string.punctuation] )
batch_size = 512
train_dataset = data.TensorDataset(x_train_torch, y_train_torch)
 valid_dataset = data.TensorDataset(x_train_torch[:batch_size], y_train_torch[:batch_size])
 test_dataset = data.TensorDataset(x_test_torch)
 train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
 valid_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=batch_size, shuffle=False)
 databunch = DataBunch(train_dl=train_loader,valid_dl=valid_loader )<choose_model_class>
%%time
train = pd.read_csv('.. /input/train.csv', index_col = "item_id", parse_dates = ["activation_date"])
 test = pd.read_csv('.. /input/test.csv', index_col = "item_id", parse_dates = ["activation_date"])
 print('[{}] Finished load data'.format(time.time() - start_time))
all_test_preds = []
for model_idx in range(NUM_MODELS):
print('Model ', model_idx)
 seed_everything(1234 + model_idx)
 model = NeuralNet(embedding_matrix, y_aux_train.shape[-1])
 learn = Learner(databunch,model,loss_func=nn.BCEWithLogitsLoss(reduction='mean'))
test_preds = train_model(learn,test_dataset,output_dim=y_train_torch.shape[-1])
 all_test_preds.append(test_preds)
 print()<save_to_csv>
%%time
def normalize_text(text):
text = text.lower().strip()
for s in string.punctuation:
text = text.replace(s, ' ')
 text = text.strip().split(' ')
 return u' '.join(x for x in text if len(x)> 1 and x not in stopwords)
 print(train.description[0])
 print(normalize_text(train.description[0]))
submission = pd.DataFrame.from_dict({
'id': test['id'],
'prediction': np.mean(all_test_preds, axis=0)[:, 0]
})
 submission.to_csv('submission.csv', index=False )<set_options>
%%time
train['is_train'] = 1
test['is_train'] = 0
print('[{}] Compiled train / test'.format(time.time() - start_time))
print('Train shape: ', train.shape)
 print('Test shape: ', test.shape)
 y = train.deal_probability.copy()
nrow_train = train.shape[0]
merge = pd.concat([train, test])
 submission = pd.DataFrame(test.index)
 print('[{}] Compiled merge'.format(time.time() - start_time))
print('Merge shape: ', merge.shape)
 del train
del test
gc.collect()
print('[{}] Garbage collection'.format(time.time() - start_time))
tqdm.pandas() 
<define_variables>
%%time
print("Feature Engineering - Part 1")
 merge["price"] = np.log(merge["price"]+0.001)
 merge["price"].fillna(-999,inplace=True)
 merge["image_top_1"].fillna(-999,inplace=True)
 print("
Create Time Variables")
 merge["activation_weekday"] = merge['activation_date'].dt.weekday
print(merge.head(5))
gc.collect()
TEXT_COL = 'comment_text'
VECS_PATH = Path('.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec')
 TRAIN_DATA = '.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv'
TEST_DATA = '.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv'<set_options>
%%time
training_index = merge.loc[merge.activation_date<=pd.to_datetime('2017-04-07')].index
validation_index = merge.loc[merge.activation_date>=pd.to_datetime('2017-04-08')].index
merge.drop(["activation_date","image"],axis=1,inplace=True)
 merge.drop(["user_id"], axis=1,inplace=True )
def seed_torch(seed=1029):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True<define_variables>
%%time
print("
Text Features")
 textfeats = ["description", "title"]
for cols in textfeats:
merge[cols] = merge[cols].astype(str)
 merge[cols] = merge[cols].astype(str ).fillna('missing')
 merge[cols] = merge[cols].str.lower()
merge[cols + '_num_stopwords'] = merge[cols].apply(lambda x: len([w for w in x.split() if w in stopwords]))
merge[cols + '_num_punctuations'] = merge[cols].apply(lambda comment:(comment.count(RE_PUNCTUATION)))
 merge[cols + '_num_alphabets'] = merge[cols].apply(lambda comment: len([c for c in comment if c.isupper() ]))
merge[cols + '_num_digits'] = merge[cols].apply(lambda comment:(comment.count('[0-9]')))
 merge[cols + '_num_letters'] = merge[cols].apply(lambda comment: len(comment))
merge[cols + '_num_words'] = merge[cols].apply(lambda comment: len(comment.split()))
merge[cols + '_num_unique_words'] = merge[cols].apply(lambda comment: len(set(w for w in comment.split())))
 merge[cols + '_words_vs_unique'] = merge[cols+'_num_unique_words'] / merge[cols+'_num_words']
merge[cols + '_letters_per_word'] = merge[cols+'_num_letters'] / merge[cols+'_num_words']
merge[cols + '_punctuations_by_letters'] = merge[cols+'_num_punctuations'] / merge[cols+'_num_letters']
merge[cols + '_punctuations_by_words'] = merge[cols+'_num_punctuations'] / merge[cols+'_num_words']
merge[cols + '_digits_by_letters'] = merge[cols+'_num_digits'] / merge[cols+'_num_letters']
merge[cols + '_alphabets_by_letters'] = merge[cols+'_num_alphabets'] / merge[cols+'_num_letters']
merge[cols + '_stopwords_by_words'] = merge[cols+'_num_stopwords'] / merge[cols+'_num_words']
merge[cols + '_mean'] = merge[cols].apply(lambda x: 0 if len(x)== 0 else float(len(x.split())) / len(x)) * 10
merge['title_desc_len_ratio'] = merge['title_num_letters']/merge['description_num_letters']
misspell_dict = {"aren't": "are not", "can't": "cannot", "couldn't": "could not",
"didn't": "did not", "doesn't": "does not", "don't": "do not",
"hadn't": "had not", "hasn't": "has not", "haven't": "have not",
"he'd": "he would", "he'll": "he will", "he's": "he is",
"i'd": "I had", "i'll": "I will", "i'm": "I am", "isn't": "is not",
"it's": "it is", "it'll": "it will", "i've": "I have", "let's": "let us",
"mightn't": "might not", "mustn't": "must not", "shan't": "shall not",
"she'd": "she would", "she'll": "she will", "she's": "she is",
"shouldn't": "should not", "that's": "that is", "there's": "there is",
"they'd": "they would", "they'll": "they will", "they're": "they are",
"they've": "they have", "we'd": "we would", "we're": "we are",
"weren't": "were not", "we've": "we have", "what'll": "what will",
"what're": "what are", "what's": "what is", "what've": "what have",
"where's": "where is", "who'd": "who would", "who'll": "who will",
"who're": "who are", "who's": "who is", "who've": "who have",
"won't": "will not", "wouldn't": "would not", "you'd": "you would",
"you'll": "you will", "you're": "you are", "you've": "you have",
"'re": " are", "wasn't": "was not", "we'll": " will", "tryin'": "trying"}<string_transform>
%%time
df_test = merge.loc[merge['is_train'] == 0]
df_train = merge.loc[merge['is_train'] == 1]
del merge
gc.collect()
df_test = df_test.drop(['is_train'], axis=1)
 df_train = df_train.drop(['is_train'], axis=1)
 print(df_train.shape)
 print(y.shape)
 if SUBMIT_MODE:
y_train = y
del y
gc.collect()
else:
df_train, df_test, y_train, y_test = train_test_split(df_train, y, test_size=0.2, random_state=144)
 print('[{}] Splitting completed.'.format(time.time() - start_time))
def _get_misspell(misspell_dict):
misspell_re = re.compile('(%s)' % '|'.join(misspell_dict.keys()))
return misspell_dict, misspell_re<categorify>
%%time
wb = wordbatch.WordBatch(normalize_text, extractor=(WordBag, {"hash_ngrams": 2,
"hash_ngrams_weights": [1.5, 1.0],
"hash_size": 2 ** 29,
"norm": None,
"tf": 'binary',
"idf": None,
}), procs=8)
 wb.dictionary_freeze = True
X_name_train = wb.fit_transform(df_train['title'])
 print(X_name_train.shape)
 X_name_test = wb.transform(df_test['title'])
 print(X_name_test.shape)
 del(wb)
 gc.collect()
def replace_typical_misspell(text):
misspellings, misspellings_re = _get_misspell(misspell_dict)
 def replace(match):
return misspellings[match.group(0)]
return misspellings_re.sub(replace, text )<define_variables>
%%time
X_train_1, X_train_2, y_train_1, y_train_2 = train_test_split(X_name_train, y_train,
test_size = 0.5,
shuffle = False)
 print('[{}] Finished splitting'.format(time.time() - start_time))
model = Ridge(solver="sag", fit_intercept=True, random_state=42, alpha=5)
 model.fit(X_train_1, y_train_1)
 print('[{}] Finished to train name ridge(1)'.format(time.time() - start_time))
name_ridge_preds1 = model.predict(X_train_2)
 name_ridge_preds1f = model.predict(X_name_test)
 print('[{}] Finished to predict name ridge(1)'.format(time.time() - start_time))
model = Ridge(solver="sag", fit_intercept=True, random_state=42, alpha=5)
 model.fit(X_train_2, y_train_2)
 print('[{}] Finished to train name ridge(2)'.format(time.time() - start_time))
name_ridge_preds2 = model.predict(X_train_1)
 name_ridge_preds2f = model.predict(X_name_test)
 print('[{}] Finished to predict name ridge(2)'.format(time.time() - start_time))
name_ridge_preds_oof = np.concatenate(( name_ridge_preds2, name_ridge_preds1), axis=0)
 name_ridge_preds_test =(name_ridge_preds1f + name_ridge_preds2f)/ 2.0
print('RMSLE OOF: {}'.format(rmse(name_ridge_preds_oof, y_train)))
 if not SUBMIT_MODE:
print('RMSLE TEST: {}'.format(rmse(name_ridge_preds_test, y_test)))
 gc.collect()
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']',
'>', '%', '=', '
'®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█',
'½', 'à', '…', '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶',
'↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', '▒', '：', '¼',
'⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲',
'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', '∙', '）', '↓', '、', '│', '（', '»', '，', '♪',
'╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√']<string_transform>
%%time
wb = wordbatch.WordBatch(normalize_text, extractor=(WordBag, {"hash_ngrams": 2,
"hash_ngrams_weights": [1.0, 1.0],
"hash_size": 2 ** 28,
"norm": "l2",
"tf": 1.0,
"idf": None}), procs=8)
 wb.dictionary_freeze = True
X_description_train = wb.fit_transform(df_train['description'].fillna(''))
print(X_description_train.shape)
 X_description_test = wb.transform(df_test['description'].fillna(''))
print(X_description_test.shape)
 print('-')
 del(wb)
 gc.collect()
def clean_text(x):
x = str(x)
 for punct in puncts + list(string.punctuation):
if punct in x:
x = x.replace(punct, f' {punct} ')
 return x<define_variables>
X_train_1, X_train_2, y_train_1, y_train_2 = train_test_split(X_description_train, y_train,
test_size = 0.5,
shuffle = False)
 print('[{}] Finished splitting'.format(time.time() - start_time))
model = Ridge(solver="sag", fit_intercept=True, random_state=205, alpha=3.3)
 model.fit(X_train_1, y_train_1)
 print('[{}] Finished to train desc ridge(1)'.format(time.time() - start_time))
desc_ridge_preds1 = model.predict(X_train_2)
 desc_ridge_preds1f = model.predict(X_description_test)
 print('[{}] Finished to predict desc ridge(1)'.format(time.time() - start_time))
model = Ridge(solver="sag", fit_intercept=True, random_state=205, alpha=3.3)
 model.fit(X_train_2, y_train_2)
 print('[{}] Finished to train desc ridge(2)'.format(time.time() - start_time))
desc_ridge_preds2 = model.predict(X_train_1)
 desc_ridge_preds2f = model.predict(X_description_test)
 print('[{}] Finished to predict desc ridge(2)'.format(time.time() - start_time))
desc_ridge_preds_oof = np.concatenate(( desc_ridge_preds2, desc_ridge_preds1), axis=0)
 desc_ridge_preds_test =(desc_ridge_preds1f + desc_ridge_preds2f)/ 2.0
print('RMSLE OOF: {}'.format(rmse(desc_ridge_preds_oof, y_train)))
 if not SUBMIT_MODE:
print('RMSLE TEST: {}'.format(rmse(desc_ridge_preds_test, y_test)))
 gc.collect()
def clean_numbers(x):
return re.sub('\d+', ' ', x )<compute_test_metric>
del X_train_1
del X_train_2
del y_train_1
del y_train_2
del name_ridge_preds1
del name_ridge_preds1f
del name_ridge_preds2
del name_ridge_preds2f
del desc_ridge_preds1
del desc_ridge_preds1f
del desc_ridge_preds2
del desc_ridge_preds2f
gc.collect()
print('[{}] Finished garbage collection'.format(time.time() - start_time))
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32' )<feature_engineering>
%%time
df_train.drop(['deal_probability', 'title', 'description'], axis=1, inplace=True)
 df_test.drop(['title', 'description'], axis=1, inplace=True)
 print('Remerged')
 dummy_cols = ['parent_category_name', 'category_name', 'user_type', 'image_top_1',
'region', 'city', 'param_1', 'param_2', 'param_3', 'activation_weekday']
numeric_cols = list(set(df_train.columns.values)- set(dummy_cols))
print(numeric_cols )
def load_fasttext(word_index):
embeddings_index = dict(get_coefs(*o.strip().split(' ')) for o in open(VECS_PATH))
nb_words = min(max_features, len(word_index))
embedding_matrix = np.zeros(( nb_words, embed_size))
for word, i in word_index.items() :
if i >= max_features:
continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
return embedding_matrix 
<define_variables>
%%time
class Scaler(BaseEstimator, TransformerMixin):
def __init__(self, columns, copy=True, with_mean=True, with_std=True):
self.scaler = StandardScaler(copy, with_mean, with_std)
 self.columns = columns
def fit(self, X, y=None):
self.scaler.fit(X[self.columns], y)
 return self
def transform(self, X, y=None, copy=None):
init_col_order = X.columns
X_scaled = pd.DataFrame(self.scaler.transform(X[self.columns]), columns=self.columns, index=X.index)
 X_not_scaled = X[list(set(init_col_order)- set(self.columns)) ]
return pd.concat([X_not_scaled, X_scaled], axis=1)[init_col_order]
print('Scaler')
 scaler = Scaler(columns=numeric_cols)
 df_train = scaler.fit_transform(df_train)
 df_test = scaler.transform(df_test )
maxlen = 220
max_features = 100000
embed_size = 300
batch_size = 1024
train_epochs = 5
n_splits = 5<define_variables>
%%time
sparse_merge_train = hstack(( X_name_train, X_description_train)).tocsr()
sparse_merge_test = hstack(( X_name_test, X_description_test)).tocsr()
print(sparse_merge_train.shape)
 for col in dummy_cols:
print(col)
 lb = LabelBinarizer(sparse_output=True)
 sparse_merge_train = hstack(( sparse_merge_train, lb.fit_transform(df_train[[col]].fillna('')))).tocsr()
print(sparse_merge_train.shape)
 sparse_merge_test = hstack(( sparse_merge_test, lb.transform(df_test[[col]].fillna('')))).tocsr()
seed = 1024<load_from_csv>
del X_description_test, X_name_test
del X_description_train, X_name_train
del lb, mask
gc.collect()
def load_and_prec() :
train = pd.read_csv(TRAIN_DATA, index_col='id')
 test = pd.read_csv(TEST_DATA, index_col='id')
 train['comment_text'] = train['comment_text'].str.lower()
test['comment_text'] = test['comment_text'].str.lower()
train['comment_text'] = train['comment_text'].apply(replace_typical_misspell)
 test['comment_text'] = test['comment_text'].apply(replace_typical_misspell)
 train['comment_text'] = train['comment_text'].apply(clean_text)
 test['comment_text'] = test['comment_text'].apply(clean_text)
 train['comment_text'] = train['comment_text'].apply(clean_numbers)
 test['comment_text'] = test['comment_text'].apply(clean_numbers)
 train_x = train['comment_text'].fillna('_
test_x = test['comment_text'].fillna('_
tokenizer = Tokenizer(num_words=max_features)
 tokenizer.fit_on_texts(list(train_x))
train_x = tokenizer.texts_to_sequences(train_x)
 test_x = tokenizer.texts_to_sequences(test_x)
 train_x = pad_sequences(train_x, maxlen=maxlen)
 test_x = pad_sequences(test_x, maxlen=maxlen)
 train_y =(train['target'].values > 0.5 ).astype(int)
 np.random.seed(seed)
 train_idx = np.random.permutation(len(train_x))
train_x = train_x[train_idx]
train_y = train_y[train_idx]
return train_x, train_y, test_x, tokenizer.word_index<choose_model_class>
%%time
print("
FM_FTRL Starting...........")
 if SUBMIT_MODE:
iters = 1
else:
iters = 1
rounds = 1
model = FM_FTRL(alpha=0.035, beta=0.001, L1=0.00001, L2=0.15, D=sparse_merge_train.shape[1],
alpha_fm=0.05, L2_fm=0.0, init_fm=0.01,
D_fm=100, e_noise=0, iters=iters, inv_link="identity", threads=4)
 if SUBMIT_MODE:
model.fit(sparse_merge_train, y_train)
 print('[{}] Train FM completed'.format(time.time() - start_time))
predsFM = model.predict(sparse_merge_test)
 print('[{}] Predict FM completed'.format(time.time() - start_time))
else:
for i in range(rounds):
model.fit(sparse_merge_train, y_train)
 predsFM = model.predict(sparse_merge_test)
 print('[{}] Iteration {}/{} -- RMSLE: {}'.format(time.time() - start_time, i + 1, rounds, rmse(predsFM, y_test)))
 del model
gc.collect()
if not SUBMIT_MODE:
print("FM_FTRL dev RMSLE:", rmse(predsFM, y_test))
class Net(nn.Module):
def __init__(self, embedding_matrix):
super(Net, self ).__init__()
lstm_hidden_size = 120
gru_hidden_size = 60
self.gru_hidden_size = gru_hidden_size
self.embedding = nn.Embedding(max_features, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = nn.Dropout2d(0.1)
 self.lstm = nn.LSTM(embed_size, lstm_hidden_size, bidirectional=True, batch_first=True)
 self.gru = nn.GRU(lstm_hidden_size*2, gru_hidden_size, bidirectional=True, batch_first=True)
 self.linear = nn.Linear(gru_hidden_size*6, 16)
 self.relu = nn.ReLU()
self.dropout = nn.Dropout(0.1)
 self.out = nn.Linear(16, 1)
 def forward(self, x):
h_embedding = self.embedding(x)
 h_embedding = torch.unsqueeze(h_embedding.transpose(1, 2), 2)
 h_embedding = torch.squeeze(self.embedding_dropout(h_embedding)).transpose(1, 2)
 h_lstm, _ = self.lstm(h_embedding)
 h_gru, hh_gru = self.gru(h_lstm)
 hh_gru = hh_gru.view(-1, self.gru_hidden_size*2)
 avg_pool = torch.mean(h_gru, 1)
 max_pool, _ = torch.max(h_gru, 1)
 conc = torch.cat(( hh_gru, avg_pool, max_pool), 1)
 conc = self.relu(self.linear(conc))
conc = self.dropout(conc)
 out = self.out(conc)
 return out<set_options>
X_train_1, X_train_2, y_train_1, y_train_2 = train_test_split(sparse_merge_train, y_train,
test_size = 0.5,
shuffle = False)
 print('[{}] Finished splitting'.format(time.time() - start_time))
model = Ridge(solver="sag", fit_intercept=True, random_state=205, alpha=3.3)
 model.fit(X_train_1, y_train_1)
 print('[{}] Finished to train ridge(1)'.format(time.time() - start_time))
ridge_preds1 = model.predict(X_train_2)
 ridge_preds1f = model.predict(sparse_merge_test)
 print('[{}] Finished to predict ridge(1)'.format(time.time() - start_time))
model = Ridge(solver="sag", fit_intercept=True, random_state=205, alpha=3.3)
 model.fit(X_train_2, y_train_2)
 print('[{}] Finished to train ridge(2)'.format(time.time() - start_time))
ridge_preds2 = model.predict(X_train_1)
 ridge_preds2f = model.predict(sparse_merge_test)
 print('[{}] Finished to predict ridge(2)'.format(time.time() - start_time))
ridge_preds_oof = np.concatenate(( ridge_preds2, ridge_preds1), axis=0)
 ridge_preds_test =(ridge_preds1f + ridge_preds2f)/ 2.0
print('RMSLE OOF: {}'.format(rmse(ridge_preds_oof, y_train)))
 if not SUBMIT_MODE:
print('RMSLE TEST: {}'.format(rmse(ridge_preds_test, y_test)) )
warnings.filterwarnings('ignore' )<load_pretrained>
fselect = SelectKBest(f_regression, k=48000)
 train_features = fselect.fit_transform(sparse_merge_train, y_train)
 test_features = fselect.transform(sparse_merge_test)
 print('[{}] Select best completed'.format(time.time() - start_time))
del sparse_merge_train
del sparse_merge_test
gc.collect()
print('[{}] Garbage collection'.format(time.time() - start_time))
with timer('load data'):
train_x, train_y, test_x, word_index = load_and_prec()
embedding_matrix = load_fasttext(word_index )<import_modules>
del ridge_preds1
del ridge_preds1f
del ridge_preds2
del ridge_preds2f
del X_train_1
del X_train_2
del y_train_1
del y_train_2
del model
gc.collect()
print('[{}] Finished garbage collection'.format(time.time() - start_time))
import random
import os<categorify>
df_train['ridge'] = ridge_preds_oof
df_train['name_ridge'] = name_ridge_preds_oof
df_train['desc_ridge'] = desc_ridge_preds_oof
df_test['ridge'] = ridge_preds_test
df_test['name_ridge'] = name_ridge_preds_test
df_test['desc_ridge'] = desc_ridge_preds_test
print('[{}] Finished adding submodels'.format(time.time() - start_time))
_camel_re1 = re.compile(' (.)([A-Z][a-z]+)')
 _camel_re2 = re.compile('([a-z0-9] )([A-Z])')
 def camel2snake(name):
s1 = re.sub(_camel_re1, r'\1_\2', name)
 return re.sub(_camel_re2, r'\1_\2', s1 ).lower()<train_model>
f_cats = ["region","city","parent_category_name","category_name","user_type","image_top_1"]
target_encode = TargetEncoder(min_samples_leaf=100, smoothing=10, noise_level=0.01,
keep_original=True, cols=f_cats)
 df_train, df_test = target_encode.encode(df_train, df_test, y_train)
 print('[{}] Finished target encoding'.format(time.time() - start_time))
class TrainEvalCallback(Callback):
def begin_fit(self):
self.run.n_epochs = 0
self.run.n_iter = 0
def after_batch(self):
if not self.in_train: return
self.run.n_epochs+=1./self.iters
self.run.n_iter+=1
def begin_epoch(self):
self.run.n_epochs = self.epoch
self.model.train()
self.run.in_train = True
def begin_validate(self):
self.model.eval()
self.run.in_train = False
class CancelTrainException(Exception): pass
class CancelEpochException(Exception): pass
class CancelBatchException(Exception): pass<import_modules>
del ridge_preds_oof
del ridge_preds_test
gc.collect()
print('[{}] Finished garbage collection'.format(time.time() - start_time))
from collections import Iterable<categorify>
cols = ['region_te', 'city_te', 'parent_category_name_te', 'category_name_te',
'user_type_te', 'image_top_1_te', 'desc_ridge', 'name_ridge', 'ridge']
train_dummies = csr_matrix(df_train[cols].values)
 print('[{}] Finished dummyizing model 1/5'.format(time.time() - start_time))
test_dummies = csr_matrix(df_test[cols].values)
 print('[{}] Finished dummyizing model 2/5'.format(time.time() - start_time))
del df_train
del df_test
gc.collect()
print('[{}] Finished dummyizing model 3/5'.format(time.time() - start_time))
train_features = hstack(( train_features, train_dummies)).tocsr()
print('[{}] Finished dummyizing model 4/5'.format(time.time() - start_time))
test_features = hstack(( test_features, test_dummies)).tocsr()
print('[{}] Finished dummyizing model 5/5'.format(time.time() - start_time))
def listify(o):
if o is None: return []
if isinstance(o, list): return o
if isinstance(o, str): return [o]
if isinstance(o, Iterable): return list(o)
 return [o]<find_best_params>
d_train = lgb.Dataset(train_features, label=y_train)
 del train_features
gc.collect()
if SUBMIT_MODE:
watchlist = [d_train]
else:
d_valid = lgb.Dataset(test_features, label=y_test)
 watchlist = [d_train, d_valid]
params = {
'learning_rate': 0.15,
'application': 'regression',
'max_depth': 13,
'num_leaves': 400,
'verbosity': -1,
'metric': 'RMSE',
'data_random_seed': 1,
'bagging_fraction': 0.8,
'feature_fraction': 0.6,
'nthread': 4,
'lambda_l1': 10,
'lambda_l2': 10
}
print('[{}] Finished compiling LGB'.format(time.time() - start_time))
class Runner() :
def __init__(self, cbs=None, cb_funcs=None):
cbs = listify(cbs)
 for cbf in listify(cb_funcs):
cb = cbf()
setattr(self, cb.name, cb)
 cbs.append(cb)
 self.stop, self.cbs = False, [TrainEvalCallback() ] + cbs
@property
def opt(self): return self.learn.opt
@property
def model(self): return self.learn.model
@property
def loss_func(self): return self.learn.loss_func
@property
def data(self): return self.learn.data
def one_batch(self, xb, yb):
try:
self.xb, self.yb = xb, yb
self('begin_batch')
 self.pred = self.model(self.xb)
 self('after_pred')
 self.loss = self.loss_func(self.pred, self.yb)
 self('after_loss')
 if not self.in_train: return
self.loss.backward()
self('after_backward')
 self.opt.step()
self('after_step')
 self.opt.zero_grad()
except CancelBatchException: self('after_cancel_batch')
 finally: self('after_batch')
 def all_batches(self, dl):
self.iters = len(dl)
 try:
for xb, yb in progress_bar(dl, leave=False): self.one_batch(xb, yb)
 except CancelEpochException: self('after_cancel_epoch')
 def fit(self, epochs, learn):
self.epochs, self.learn, self.loss = epochs, learn, torch.tensor(0.)
try:
for cb in self.cbs: cb.set_runner(self)
 self('begin_fit')
 for epoch in range(epochs):
self.epoch = epoch
if not self('begin_epoch'): self.all_batches(self.data.train_dl)
 with torch.no_grad() :
if not self('begin_validate'): self.all_batches(self.data.valid_dl)
 self('after_epoch')
 except CancelTrainException: self('after_cancel_train')
 finally:
self('after_fit')
 self.learn = None
def __call__(self, cb_name):
res = False
for cb in sorted(self.cbs, key=lambda x: x._order): res = cb(cb_name)and res
return res<choose_model_class>
%%time
modelL = lgb.train(params,
train_set=d_train,
num_boost_round=400,
valid_sets=watchlist,
verbose_eval=50)
 predsL = modelL.predict(test_features)
 predsL[predsL < 0] = 0
predsL[predsL > 1] = 1
if not SUBMIT_MODE:
print("LGB RMSLE:", rmse(predsL, y_test))
class Learner() :
def __init__(self, model, opt, loss_func, data):
self.model, self.opt, self.loss_func, self.data = model, opt, loss_func, data<choose_model_class>
del d_train
del modelL
if not SUBMIT_MODE:
del d_valid
gc.collect()
def get_model(data, lr=0.005):
model = Net(embedding_matrix ).to(device)
 return model, torch.optim.Adam(model.parameters() , lr )<train_model>
preds_final = predsFM * 0.1 + predsL * 0.9
if not SUBMIT_MODE:
print('Final RMSE: ', rmse(preds_final, y_test))
<split><EOS> 
if SUBMIT_MODE:
submission['deal_probability'] = preds_final
submission['deal_probability'] = submission['deal_probability'].clip(0.0, 1.0)
 submission.to_csv('lgb_and_fm_separate_train_test.csv', index=False)
 print('[{}] Writing submission done'.format(time.time() - start_time))
print(submission.head(5))