Kaggle competition: Avito Demand Prediction Challenge
<SOS> metric: RMSE Kaggle data source: avito-demand-prediction-challenge<feature_engineering>
%matplotlib inline
def get_coefs(word,*arr):
return word, np.asarray(arr, dtype='float32')
 def build_matrix(embedding_path, tokenizer):
embedding_index = dict(get_coefs(*o.strip().split(" ")) for o in open(embedding_path))
word_index = tk.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.zeros(( nb_words + 1, embed_size))
for word, i in word_index.items() :
if i >= max_features:
continue
embedding_vector = embedding_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
return embedding_matrix
embedding_matrix = np.concatenate([build_matrix(embedding_path1, tk), build_matrix(embedding_path2, tk)], axis=-1 )<prepare_x_and_y>
target_col ='deal_probability'
toy = False
validate = True
best_num_boost_round = 100 if toy else 5000
num_boost_round = 100 if toy else 5000
y = np.where(train['target'] >= 0.5, True, False)* 1<feature_engineering>
%%time
dtypes = {
'user_id': 'category',
'region': 'category',
'city': 'category',
'parent_category_name': 'category',
'category_name': 'category',
'param_1': 'category',
'param_2': 'category',
'param_3': 'category',
'title': 'str',
'description': 'str',
'price': 'float',
'item_seq_number': 'int',
'activation_date': 'object',
'user_type': 'category',
'image': 'str',
'image_top_1': 'float',
'deal_probability': 'float'
}
date_cols = ['activation_date']
dtypes_load = {k:('object' if v == 'category' else v)for k, v in dtypes.items() }
df_train = pd.read_csv('.. /input/avito-demand-prediction/train.csv', dtype=dtypes_load, parse_dates=date_cols, index_col="item_id", nrows=100000 if toy else None)
 df_test = pd.read_csv('.. /input/avito-demand-prediction/test.csv', dtype=dtypes_load, parse_dates=date_cols, index_col="item_id", nrows=100000 if toy else None)
 n_train = df_train.shape[0]
identity_columns = ['male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish', 'muslim', 'black', 'white', 'psychiatric_or_mental_illness']
for col in identity_columns + ['target']:
train[col] = np.where(train[col] >= 0.5, True, False)
 n_fold = 5
folds = StratifiedKFold(n_splits=n_fold, shuffle=True, random_state=11 )<compute_test_metric>
def load_imfeatures(folder):
path = PurePath(folder)
 features = sparse.load_npz(str(path / 'features.npz'))
if toy:
features = features[:100000]
return features
SUBGROUP_AUC = 'subgroup_auc'
BPSN_AUC = 'bpsn_auc'
BNSP_AUC = 'bnsp_auc'
def compute_auc(y_true, y_pred):
try:
return metrics.roc_auc_score(y_true, y_pred)
 except ValueError:
return np.nan
def compute_subgroup_auc(df, subgroup, label, oof_name):
subgroup_examples = df[df[subgroup]]
return compute_auc(subgroup_examples[label], subgroup_examples[oof_name])
 def compute_bpsn_auc(df, subgroup, label, oof_name):

 subgroup_negative_examples = df[df[subgroup] & ~df[label]]
non_subgroup_positive_examples = df[~df[subgroup] & df[label]]
examples = subgroup_negative_examples.append(non_subgroup_positive_examples)
 return compute_auc(examples[label], examples[oof_name])
 def compute_bnsp_auc(df, subgroup, label, oof_name):

 subgroup_positive_examples = df[df[subgroup] & df[label]]
non_subgroup_negative_examples = df[~df[subgroup] & ~df[label]]
examples = subgroup_positive_examples.append(non_subgroup_negative_examples)
 return compute_auc(examples[label], examples[oof_name])
 def compute_bias_metrics_for_model(dataset,
subgroups,
model,
label_col,
include_asegs=False):

 records = []
for subgroup in subgroups:
record = {
'subgroup': subgroup,
'subgroup_size': len(dataset[dataset[subgroup]])
 }
record[SUBGROUP_AUC] = compute_subgroup_auc(dataset, subgroup, label_col, model)
 record[BPSN_AUC] = compute_bpsn_auc(dataset, subgroup, label_col, model)
 record[BNSP_AUC] = compute_bnsp_auc(dataset, subgroup, label_col, model)
 records.append(record)
 return pd.DataFrame(records ).sort_values('subgroup_auc', ascending=True)
 def calculate_overall_auc(df, oof_name):
true_labels = df['target']
predicted_labels = df[oof_name]
return metrics.roc_auc_score(true_labels, predicted_labels)
 def power_mean(series, p):
total = sum(np.power(series, p))
return np.power(total / len(series), 1 / p)
 def get_final_metric(bias_df, overall_auc, POWER=-5, OVERALL_MODEL_WEIGHT=0.25):
bias_score = np.average([
power_mean(bias_df[SUBGROUP_AUC], POWER),
power_mean(bias_df[BPSN_AUC], POWER),
power_mean(bias_df[BNSP_AUC], POWER)
 ])
 return(OVERALL_MODEL_WEIGHT * overall_auc)+(( 1 - OVERALL_MODEL_WEIGHT)* bias_score)
<choose_model_class>
ftrain = load_imfeatures('.. /input/vgg16-train-features/' )
class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim
def build_model(X_train, y_train, X_valid, y_valid, max_len, max_features, embed_size, embedding_matrix, lr=0.0, lr_d=0.0, spatial_dr=0.0,
dense_units=128, conv_size=128, dr=0.2, patience=3, fold_id=1):
file_path = f"best_model_fold_{fold_id}.hdf5"
check_point = ModelCheckpoint(file_path, monitor="val_loss", verbose=1,save_best_only=True, mode="min")
 early_stop = EarlyStopping(monitor="val_loss", mode="min", patience=patience)
 inp = Input(shape =(max_len,))
x = Embedding(max_features + 1, embed_size * 2, weights=[embedding_matrix], trainable=False )(inp)
 x1 = SpatialDropout1D(spatial_dr )(x)
 att = Attention(max_len )(x1)
 x = Conv1D(conv_size, 2, activation='relu', padding='same' )(x1)
 x = MaxPooling1D(5, padding='same' )(x)
 x = Conv1D(conv_size, 3, activation='relu', padding='same' )(x)
 x = MaxPooling1D(5, padding='same' )(x)
 x = Flatten()(x)
 x = concatenate([x, att])
 x = Dropout(dr )(Dense(dense_units, activation='relu' )(x))
x = Dense(1, activation="sigmoid" )(x)
 model = Model(inputs=inp, outputs=x)
 model.compile(loss="binary_crossentropy", optimizer=Adam(lr=lr, decay=lr_d), metrics=["accuracy"])
 model.fit(X_train, y_train, batch_size=128, epochs=3, validation_data=(X_valid, y_valid),
verbose=2, callbacks=[early_stop, check_point])
 return model<split>
ftest = load_imfeatures('.. /input/vgg16-test-features/' )
def train_model(X, X_test, y, tokenizer, max_len):
oof = np.zeros(( len(X), 1))
prediction = np.zeros(( len(X_test), 1))
scores = []
test_tokenized = tokenizer.texts_to_sequences(test['comment_text'])
 X_test = pad_sequences(test_tokenized, maxlen = max_len)
 for fold_n,(train_index, valid_index)in enumerate(folds.split(X, y)) :
print('Fold', fold_n, 'started at', time.ctime())
 X_train, X_valid = X.iloc[train_index], X.iloc[valid_index]
y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]
valid_df = X_valid.copy()
train_tokenized = tokenizer.texts_to_sequences(X_train['comment_text'])
 valid_tokenized = tokenizer.texts_to_sequences(X_valid['comment_text'])
 X_train = pad_sequences(train_tokenized, maxlen = max_len)
 X_valid = pad_sequences(valid_tokenized, maxlen = max_len)
 model = build_model(X_train, y_train, X_valid, y_valid, max_len, max_features, embed_size, embedding_matrix,
lr = 1e-3, lr_d = 0, spatial_dr = 0.1, dense_units=128, conv_size=128, dr=0.1, patience=3, fold_id=fold_n)
 pred_valid = model.predict(X_valid)
 oof[valid_index] = pred_valid
valid_df[oof_name] = pred_valid
bias_metrics_df = compute_bias_metrics_for_model(valid_df, identity_columns, oof_name, 'target')
 scores.append(get_final_metric(bias_metrics_df, calculate_overall_auc(valid_df, oof_name)))
 prediction += model.predict(X_test, batch_size = 1024, verbose = 1)
 prediction /= n_fold
return oof, prediction, scores<train_model>
df_target = df_train[target_col]
df_both = pd.concat([df_train, df_test])
 del df_train, df_test
gc.collect() ;
oof_name = 'predicted_target'
max_len = 250
oof, prediction, scores = train_model(X=train, X_test=test, y=train['target'], tokenizer=tk, max_len=max_len)
 print('CV mean score: {0:.4f}, std: {1:.4f}.'.format(np.mean(scores), np.std(scores)) )<save_to_csv>
fboth = sparse.vstack([ftrain, ftest])
 del ftrain, ftest
gc.collect()
fboth.shape
sub['prediction'] = prediction
sub.to_csv('submission.csv', index=False )<set_options>
df_both['im_max_feature'] = fboth.argmax(axis=1)
 df_both['im_min_feature'] = fboth.argmin(axis=1)
 df_both['im_n_features'] = fboth.getnnz(axis=1)
 df_both['im_mean_features'] = fboth.mean(axis=1)
 df_both['im_meansquare_features'] = fboth.power(2 ).mean(axis=1 )
%matplotlib inline
pd.set_option('max_colwidth',400)
 print(os.listdir(".. /input"))<load_from_csv>
%%time
tsvd = TruncatedSVD(32)
 ftsvd = tsvd.fit_transform(fboth)
 del fboth
gc.collect()
train = pd.read_csv('.. /input/jigsaw-public-files/train.csv')
 test = pd.read_csv('.. /input/jigsaw-public-files/test.csv')
 train['comment_text'] = train['comment_text'].fillna('')
 test['comment_text'] = test['comment_text'].fillna('')
 sub = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/sample_submission.csv' )<define_variables>
for col, dtype in dtypes.items() :
if dtype == 'category':
df_both[col] = df_both[col].astype('category' ).cat.codes
full_text = list(train['comment_text'].values)+ list(test['comment_text'].values )<train_model>
cat_cols = ['region', 'city', 'parent_category_name', 'category_name', 'param_1', 'param_2', 'param_3', 'user_type',
'im_max_feature', 'im_min_feature']
num_cols = ['price', 'image_top_1', 'deal_probability']
for cat_col in cat_cols:
df_group = df_both.groupby(cat_col)[num_cols].agg(['mean', 'std'])
 df_group.columns = ['{}_'.format(cat_col)+ '-'.join(col ).strip() for col in df_group.columns.values]
df_both = df_both.join(df_group, on=cat_col, how='left')
 del df_group
print(cat_col)
%%time
tk = Tokenizer(lower = True, filters='', num_words=120000)
 tk.fit_on_texts(full_text )<string_transform>
df_both['activation_dow'] = df_both['activation_date'].dt.dayofweek
train_tokenized = tk.texts_to_sequences(train['comment_text'])
 test_tokenized = tk.texts_to_sequences(test['comment_text'] )<string_transform>
sw = stopwords.words('russian')
 txt_cols = ['title', 'description']
for txt_col in txt_cols:
df_both[txt_col + '_len'] = df_both[txt_col].str.len()
df_both[txt_col + '_wc'] = df_both[txt_col].str.count(' ')
 if txt_col != 'description':
feature_cnt = 50
tfidf = TfidfVectorizer(stop_words=sw, min_df=10, max_df=0.8, dtype=np.float32, max_features=32000)
 X_text = tfidf.fit_transform(df_both[txt_col])
 svr = LinearSVR(C=0.01 ).fit(X_text[:n_train], df_target)
 fnames = sorted(tfidf.vocabulary_, key=tfidf.vocabulary_.__getitem__)
 best_features = np.argsort(svr.coef_)
 selected_features = np.concatenate([best_features[:feature_cnt], best_features[-feature_cnt:]])
 features_names = list(map(fnames.__getitem__, selected_features))
df_both_tfidf = pd.DataFrame(X_text[:, selected_features].todense() , columns=features_names, index=df_both.index)
 del X_text, features_names, selected_features, best_features, svr, fnames, tfidf
df_both = df_both.join(df_both_tfidf)
 del df_both_tfidf
gc.collect()
print(txt_col )
max_len = 230
X_train = pad_sequences(train_tokenized, maxlen = max_len)
 X_test = pad_sequences(test_tokenized, maxlen = max_len )<categorify>
df_ftsvd = pd.DataFrame(ftsvd, index=df_both.index ).add_prefix('im_tsvd_')
 df_both = pd.concat([df_both, df_ftsvd], axis=1)
 del df_ftsvd, ftsvd
gc.collect() ;
y = np.where(train['target'] >= 0.5, True, False)* 1
y_binary = to_categorical(y )<define_variables>
df_train = df_both.iloc[:n_train]
df_test = df_both.iloc[n_train:]
del df_both
gc.collect()
df_train.shape, df_test.shape
embedding_path = ".. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec"<define_variables>
ex_cols = {'item_id', 'user_id', 'deal_probability', 'title', 'description', 'image', 'activation_date'}
used_cols = [c for c in df_train.columns if c not in ex_cols]
print('Used cols:', ', '.join(used_cols))
embed_size = 300
max_features = 120000<feature_engineering>
if validate:
fold_train, fold_valid, target_train, target_valid = train_test_split(df_train[used_cols], df_target, test_size=0.2, random_state=42)
 dtrain = lgb.Dataset(fold_train, target_train, categorical_feature=cat_cols)
 dvalid = lgb.Dataset(fold_valid, target_valid, categorical_feature=cat_cols)
 valid_sets = [dvalid]
valid_names = ['valid']
del fold_train, fold_valid, target_train, target_valid
else:
dtrain = lgb.Dataset(df_train[used_cols], df_target)
 valid_sets = [dtrain]
valid_names = ['train']
assert best_num_boost_round is not None
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embedding_index = dict(get_coefs(*o.strip().split(" ")) for o in open(embedding_path))
word_index = tk.word_index
nb_words = min(max_features, len(word_index))
embedding_matrix = np.zeros(( nb_words + 1, embed_size))
for word, i in word_index.items() :
if i >= max_features: continue
embedding_vector = embedding_index.get(word)
 if embedding_vector is not None: embedding_matrix[i] = embedding_vector<choose_model_class>
%%time
params = {
'learning_rate': 0.02,
'boosting': 'gbdt',
'objective': 'regression',
'metric': ['rmse'],
'is_training_metric': True,
'seed': 19,
'num_leaves': 31,
'feature_fraction': 0.9,
'bagging_fraction': 0.8,
'bagging_freq': 5
}
model = lgb.train(params, dtrain, num_boost_round=num_boost_round, valid_sets=valid_sets, valid_names=valid_names,
verbose_eval=num_boost_round//20, early_stopping_rounds=50 if validate else None)
 if validate:
best_num_boost_round = model.best_iteration
def build_model(lr=0.0, lr_d=0.0, units=0, spatial_dr=0.0, kernel_size1=3, kernel_size2=2, dense_units=128, dr=0.1, conv_size=32):
file_path = "best_model.hdf5"
check_point = ModelCheckpoint(file_path, monitor = "val_loss", verbose = 1,
save_best_only = True, mode = "min")
 early_stop = EarlyStopping(monitor = "val_loss", mode = "min", patience = 3)
 inp = Input(shape =(max_len,))
x = Embedding(max_features + 1, embed_size, weights = [embedding_matrix], trainable = False )(inp)
 x1 = SpatialDropout1D(spatial_dr )(x)
 x_gru = Bidirectional(CuDNNGRU(units, return_sequences = True))(x1)
 x1 = Conv1D(int(units/2), kernel_size=kernel_size1, padding='valid', kernel_initializer='he_uniform' )(x_gru)
 y = Bidirectional(CuDNNLSTM(units, return_sequences = True))(x1)
 y = Conv1D(int(units/2), kernel_size = 2, padding = "valid", kernel_initializer = "he_uniform" )(y)
 avg_pool1 = GlobalAveragePooling1D()(x)
 max_pool1 = GlobalMaxPooling1D()(x)
 avg_pool2 = GlobalAveragePooling1D()(y)
 max_pool2 = GlobalMaxPooling1D()(y)
 x = concatenate([avg_pool1, max_pool1, avg_pool2, max_pool2])
 x = Dense(2, activation = "sigmoid" )(x)
 model = Model(inputs = inp, outputs = x)
 model.compile(loss = "binary_crossentropy", optimizer = Adam(lr = lr, decay = lr_d), metrics = ["accuracy"])
 history = model.fit(X_train, y_binary, batch_size = 128, epochs = 3, validation_split=0.1,
verbose = 2, callbacks = [check_point, early_stop])
 model = load_model(file_path)
 return model<predict_on_test>
del df_train, dtrain
gc.collect()
<choose_model_class><EOS> 
df_test.index.name = 'item_id'
df_test['deal_probability'] = model.predict(df_test[used_cols], num_iteration=best_num_boost_round ).clip(0., 1.)
df_test[['deal_probability']].to_csv('submission.csv', index=True )