Kaggle competition: Categorical Feature Encoding Challenge II
pred_sub = reg_lgbm.predict(sub_stacked)
 lightgbm_submission = submission.copy()
lightgbm_submission['price'] = pd.DataFrame(10 ** pred_sub - 1)
 lightgbm_submission.to_csv('stacked_submission_1.csv', index=False )<set_options>
train = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/train.csv')
 test = pd.read_csv('/kaggle/input/cat-in-the-dat-ii/test.csv')
 test.insert(1, 'target', 0 )
start_time = datetime.now()
%matplotlib inline 
<load_from_csv>
features = [_f for _f in train if _f not in ['id', 'target']]
def factor_encoding(train, test):
assert sorted(train.columns)== sorted(test.columns)
 full = pd.concat([train, test], axis=0, sort=False)
 for f in full:
full[f], _ = pd.factorize(full[f])
 full[f] += 1
return full.iloc[:train.shape[0]], full.iloc[train.shape[0]:]
train_f, test_f = factor_encoding(train[features], test[features] )
train = pd.read_table(".. /input/train.tsv")
 test = pd.read_table(".. /input/test.tsv")
 print(test.head() )<categorify>
class LibFFMEncoder(object):
def __init__(self):
self.encoder = 1
self.encoding = {}
def encode_for_libffm(self, row):
txt = f"{row[0]}"
for i, r in enumerate(row[1:]):
try:
txt += f' {i+1}:{self.encoding[(i, r)]}:1'
except KeyError:
self.encoding[(i, r)] = self.encoder
self.encoder += 1
txt += f' {i+1}:{self.encoding[(i, r)]}:1'
return txt
fold_ids = [
[trn_, val_] for(trn_, val_)in StratifiedKFold(N_Splits,True,SEED ).split(train, train['target'])
 ]
for fold_,(trn_, val_)in enumerate(fold_ids):
encoder = LibFFMEncoder()
libffm_format_trn = pd.concat([train['target'].iloc[trn_], train_f.iloc[trn_]], axis=1 ).apply(
 lambda row: encoder.encode_for_libffm(row), raw=True, axis=1 
)
 libffm_format_val = pd.concat([train['target'].iloc[val_], train_f.iloc[val_]], axis=1 ).apply(
 lambda row: encoder.encode_for_libffm(row), raw=True, axis=1 
)
 libffm_format_tst = pd.concat([test['target'], test_f], axis=1 ).apply(
 lambda row: encoder.encode_for_libffm(row), raw=True, axis=1 
)
 print(train['target'].iloc[trn_].shape, train['target'].iloc[val_].shape, libffm_format_tst.shape)
 libffm_format_trn.to_csv(f'libffm_trn_fold_{fold_+1}.txt', index=False, header=False)
 libffm_format_val.to_csv(f'libffm_val_fold_{fold_+1}.txt', index=False, header=False)
 libffm_format_tst.to_csv(f'libffm_tst_fold_{fold_+1}.txt', index=False, header=False)
def handle_missing(dataset):
dataset.category_name.fillna(value="missing", inplace=True)
 dataset.brand_name.fillna(value="missing", inplace=True)
 dataset.item_description.fillna(value="missing", inplace=True)
 return(dataset)
 train = handle_missing(train)
 test = handle_missing(test)
 print(train.shape)
 print(test.shape )<categorify>
!./ffm-train -p libffm_val_fold_1.txt -r 0.05 -l 0.0002 -k 50 --auto-stop libffm_trn_fold_1.txt libffm_fold_1_model
!./ffm-predict libffm_val_fold_1.txt libffm_fold_1_model val_preds_fold_1.txt
!./ffm-predict libffm_tst_fold_1.txt libffm_fold_1_model tst_preds_fold_1.txt
os.remove('libffm_val_fold_1.txt')
 os.remove('libffm_trn_fold_1.txt')
 os.remove('libffm_fold_1_model')
 os.remove('libffm_tst_fold_1.txt')
(
 log_loss(train['target'].iloc[fold_ids[0][1]], pd.read_csv('val_preds_fold_1.txt', header=None ).values[:,0]),
roc_auc_score(train['target'].iloc[fold_ids[0][1]], pd.read_csv('val_preds_fold_1.txt', header=None ).values[:,0])
)
le = LabelEncoder()
le.fit(np.hstack([train.category_name, test.category_name]))
train.category_name = le.transform(train.category_name)
 test.category_name = le.transform(test.category_name)
 le.fit(np.hstack([train.brand_name, test.brand_name]))
train.brand_name = le.transform(train.brand_name)
 test.brand_name = le.transform(test.brand_name)
 print(train.head() )<string_transform>
!./ffm-train -p libffm_val_fold_2.txt -r 0.05 -l 0.00002 -k 50 --auto-stop libffm_trn_fold_2.txt libffm_fold_2_model
!./ffm-predict libffm_val_fold_2.txt libffm_fold_2_model val_preds_fold_2.txt
!./ffm-predict libffm_tst_fold_2.txt libffm_fold_2_model tst_preds_fold_2.txt
os.remove('libffm_val_fold_2.txt')
 os.remove('libffm_trn_fold_2.txt')
 os.remove('libffm_fold_2_model')
 os.remove('libffm_tst_fold_2.txt')
(
 log_loss(train['target'].iloc[fold_ids[1][1]], pd.read_csv('val_preds_fold_2.txt', header=None ).values[:,0]),
roc_auc_score(train['target'].iloc[fold_ids[1][1]], pd.read_csv('val_preds_fold_2.txt', header=None ).values[:,0])
)
raw_text = np.hstack([train.item_description.str.lower() , train.name.str.lower() ])
 tok_raw = Tokenizer()
tok_raw.fit_on_texts(raw_text )<feature_engineering>
!./ffm-train -p libffm_val_fold_3.txt -r 0.05 -l 0.00002 -k 50 --auto-stop libffm_trn_fold_3.txt libffm_fold_3_model
!./ffm-predict libffm_val_fold_3.txt libffm_fold_3_model val_preds_fold_3.txt
!./ffm-predict libffm_tst_fold_3.txt libffm_fold_3_model tst_preds_fold_3.txt
os.remove('libffm_val_fold_3.txt')
 os.remove('libffm_trn_fold_3.txt')
 os.remove('libffm_fold_3_model')
 os.remove('libffm_tst_fold_3.txt')
(
 log_loss(train['target'].iloc[fold_ids[2][1]], pd.read_csv('val_preds_fold_3.txt', header=None ).values[:,0]),
roc_auc_score(train['target'].iloc[fold_ids[2][1]], pd.read_csv('val_preds_fold_3.txt', header=None ).values[:,0])
)
train["seq_item_description"] = tok_raw.texts_to_sequences(train.item_description.str.lower())
 test["seq_item_description"] = tok_raw.texts_to_sequences(test.item_description.str.lower())
 train["seq_name"] = tok_raw.texts_to_sequences(train.name.str.lower())
 test["seq_name"] = tok_raw.texts_to_sequences(test.name.str.lower())
 print(train.head() )<define_variables>
!./ffm-train -p libffm_val_fold_4.txt -r 0.05 -l 0.00002 -k 50 --auto-stop libffm_trn_fold_4.txt libffm_fold_4_model
!./ffm-predict libffm_val_fold_4.txt libffm_fold_4_model val_preds_fold_4.txt
!./ffm-predict libffm_tst_fold_4.txt libffm_fold_4_model tst_preds_fold_4.txt
os.remove('libffm_val_fold_4.txt')
 os.remove('libffm_trn_fold_4.txt')
 os.remove('libffm_fold_4_model')
 os.remove('libffm_tst_fold_4.txt')
(
 log_loss(train['target'].iloc[fold_ids[3][1]], pd.read_csv('val_preds_fold_4.txt', header=None ).values[:,0]),
roc_auc_score(train['target'].iloc[fold_ids[3][1]], pd.read_csv('val_preds_fold_4.txt', header=None ).values[:,0])
)
max_name_seq_set = 10
max_seq_item_description_set = 75
max_text = np.max([np.max(train.seq_name.max()),
np.max(test.seq_name.max()),
np.max(train.seq_item_description.max()),
np.max(test.seq_item_description.max())])+2
max_category = np.max([train.category_name.max() , test.category_name.max() ])+1
max_brand = np.max([train.brand_name.max() , test.brand_name.max() ])+1
max_cond_id = np.max([train.item_condition_id.max() , test.item_condition_id.max() ])+1<define_variables>
!./ffm-train -p libffm_val_fold_5.txt -r 0.05 -l 0.00002 -k 50 --auto-stop libffm_trn_fold_5.txt libffm_fold_5_model
!./ffm-predict libffm_val_fold_5.txt libffm_fold_5_model val_preds_fold_5.txt
!./ffm-predict libffm_tst_fold_5.txt libffm_fold_5_model tst_preds_fold_5.txt
os.remove('libffm_val_fold_5.txt')
 os.remove('libffm_trn_fold_5.txt')
 os.remove('libffm_fold_5_model')
 os.remove('libffm_tst_fold_5.txt')
(
 log_loss(train['target'].iloc[fold_ids[4][1]], pd.read_csv('val_preds_fold_5.txt', header=None ).values[:,0]),
roc_auc_score(train['target'].iloc[fold_ids[4][1]], pd.read_csv('val_preds_fold_5.txt', header=None ).values[:,0])
)
max_text = np.max([np.max(train.seq_name.max()),
np.max(test.seq_name.max()),
np.max(train.seq_item_description.max()),
np.max(test.seq_item_description.max())])+ 2
max_category = np.max([train.category_name.max() , test.category_name.max() ])+ 1
max_brand = np.max([train.brand_name.max() , test.brand_name.max() ])+ 1
max_cond_id = np.max([train.item_condition_id.max() , test.item_condition_id.max() ])+ 1<split>
!./ffm-train -p libffm_val_fold_6.txt -r 0.05 -l 0.00002 -k 50 --auto-stop libffm_trn_fold_6.txt libffm_fold_6_model
!./ffm-predict libffm_val_fold_6.txt libffm_fold_6_model val_preds_fold_6.txt
!./ffm-predict libffm_tst_fold_6.txt libffm_fold_6_model tst_preds_fold_6.txt
os.remove('libffm_val_fold_6.txt')
 os.remove('libffm_trn_fold_6.txt')
 os.remove('libffm_fold_6_model')
 os.remove('libffm_tst_fold_6.txt')
(
 log_loss(train['target'].iloc[fold_ids[5][1]], pd.read_csv('val_preds_fold_6.txt', header=None ).values[:,0]),
roc_auc_score(train['target'].iloc[fold_ids[5][1]], pd.read_csv('val_preds_fold_6.txt', header=None ).values[:,0])
)
dtrain, dvalid = train_test_split(train, random_state=123, train_size=0.90)
 print(dtrain.shape)
 print(dvalid.shape )<prepare_x_and_y>
!./ffm-train -p libffm_val_fold_7.txt -r 0.05 -l 0.00002 -k 50 --auto-stop libffm_trn_fold_7.txt libffm_fold_7_model
!./ffm-predict libffm_val_fold_7.txt libffm_fold_7_model val_preds_fold_7.txt
!./ffm-predict libffm_tst_fold_7.txt libffm_fold_7_model tst_preds_fold_7.txt
os.remove('libffm_val_fold_7.txt')
 os.remove('libffm_trn_fold_7.txt')
 os.remove('libffm_fold_7_model')
 os.remove('libffm_tst_fold_7.txt')
(
 log_loss(train['target'].iloc[fold_ids[6][1]], pd.read_csv('val_preds_fold_7.txt', header=None ).values[:,0]),
roc_auc_score(train['target'].iloc[fold_ids[6][1]], pd.read_csv('val_preds_fold_7.txt', header=None ).values[:,0])
)
def get_keras_data(dataset):
X = {
'name': pad_sequences(dataset.seq_name, maxlen=max_name_seq_set),
'item_desc': pad_sequences(dataset.seq_item_description, maxlen=max_seq_item_description),
'brand_name': np.array(dataset.brand_name),
'category_name': np.array(dataset.category_name),
'item_condition': np.array(dataset.item_condition_id),
'num_vars': np.array(dataset[["shipping"]])
 }
return X
X_train = get_keras_data(dtrain)
 X_valid = get_keras_data(dvalid)
 X_test = get_keras_data(test)
 print(X_train )<compute_train_metric>
!./ffm-train -p libffm_val_fold_8.txt -r 0.05 -l 0.00002 -k 50 --auto-stop libffm_trn_fold_8.txt libffm_fold_8_model
!./ffm-predict libffm_val_fold_8.txt libffm_fold_8_model val_preds_fold_8.txt
!./ffm-predict libffm_tst_fold_8.txt libffm_fold_8_model tst_preds_fold_8.txt
os.remove('libffm_val_fold_8.txt')
 os.remove('libffm_trn_fold_8.txt')
 os.remove('libffm_fold_8_model')
 os.remove('libffm_tst_fold_8.txt')
(
 log_loss(train['target'].iloc[fold_ids[7][1]], pd.read_csv('val_preds_fold_8.txt', header=None ).values[:,0]),
roc_auc_score(train['target'].iloc[fold_ids[7][1]], pd.read_csv('val_preds_fold_8.txt', header=None ).values[:,0])
)
def get_callbacks(filepath, patience=2):
es = EarlyStopping('val_loss', patience=patience, mode="min")
 msave = ModelCheckpoint(filepath, save_best_only=True)
 return [es, msave]
def rmsle_cust(y_true, y_pred):
first_log = K.log(K.clip(y_pred, K.epsilon() , None)+1.)
second_log = K.log(K.clip(y_true, K.epsilon() , None)+1.)
return K.sqrt(K.mean(K.square(first_log - second_log), axis=-1))
def get_model() :
dr_r = 0.2
name = Input(shape=[X_train["name"].shape[1]], name="name")
 item_desc = Input(shape=[X_train["item_desc"].shape[1]], name="item_desc")
 brand_name = Input(shape=[1], name="brand_name")
 category_name = Input(shape=[1], name="category_name")
 item_condition = Input(shape=[1], name="item_condition")
 num_vars = Input(shape=[X_train["num_vars"].shape[1]], name="num_vars")
 emb_name = Embedding(max_text, 50 )(name)
 emb_item_desc = Embedding(max_text, 50 )(item_desc)
 emb_brand_name = Embedding(max_brand, 10 )(brand_name)
 emb_category_name = Embedding(max_category, 10 )(category_name)
 emb_item_condition = Embedding(max_cond_id, 5 )(item_condition)
 rnn_layer1 = GRU(16 )(emb_item_desc)
 rnn_layer2 = GRU(8 )(emb_name)
 main_l = concatenate([
Flatten()(emb_brand_name),
Flatten()(emb_category_name),
Flatten()(emb_item_condition),
rnn_layer1,
rnn_layer2,
num_vars
])
 main_l = Dropout(dr_r )(Dense(64 )(main_l))
main_l = Activation('relu' )(main_l)
 main_l = Dropout(dr_r )(Dense(32 )(main_l))
main_l = Activation('relu' )(main_l)
 output = Dense(1, activation="linear" )(main_l)
 model = Model([name, item_desc, brand_name, category_name, item_condition, num_vars], output)
 optimizers = Adam(lr=0.001, decay=0.0)
 model.compile(loss="mse", optimizer=optimizers, metrics=["mae", rmsle_cust])
 return model
model = get_model()
model.summary()<train_model>
!./ffm-train -p libffm_val_fold_9.txt -r 0.05 -l 0.00002 -k 50 --auto-stop libffm_trn_fold_9.txt libffm_fold_9_model
!./ffm-predict libffm_val_fold_9.txt libffm_fold_9_model val_preds_fold_9.txt
!./ffm-predict libffm_tst_fold_9.txt libffm_fold_9_model tst_preds_fold_9.txt
os.remove('libffm_val_fold_9.txt')
 os.remove('libffm_trn_fold_9.txt')
 os.remove('libffm_fold_9_model')
 os.remove('libffm_tst_fold_9.txt')
(
 log_loss(train['target'].iloc[fold_ids[8][1]], pd.read_csv('val_preds_fold_9.txt', header=None ).values[:,0]),
roc_auc_score(train['target'].iloc[fold_ids[8][1]], pd.read_csv('val_preds_fold_9.txt', header=None ).values[:,0])
)
model = get_model()
model.fit(X_train, dtrain.target, epochs=1, batch_size=1500,
validation_data=(X_valid, dvalid.target), verbose=1 )<predict_on_test>
!./ffm-train -p libffm_val_fold_10.txt -r 0.05 -l 0.00002 -k 50 --auto-stop libffm_trn_fold_10.txt libffm_fold_10_model
!./ffm-predict libffm_val_fold_10.txt libffm_fold_10_model val_preds_fold_10.txt
!./ffm-predict libffm_tst_fold_10.txt libffm_fold_10_model tst_preds_fold_10.txt
os.remove('libffm_val_fold_10.txt')
 os.remove('libffm_trn_fold_10.txt')
 os.remove('libffm_fold_10_model')
 os.remove('libffm_tst_fold_10.txt')
(
 log_loss(train['target'].iloc[fold_ids[9][1]], pd.read_csv('val_preds_fold_10.txt', header=None ).values[:,0]),
roc_auc_score(train['target'].iloc[fold_ids[9][1]], pd.read_csv('val_preds_fold_10.txt', header=None ).values[:,0])
)
preds = model.predict(X_test, batch_size=1500)
 preds = target_scaler.inverse_transform(preds)
 preds = np.exp(preds)-1<save_to_csv>
!./ffm-train -p libffm_val_fold_11.txt -r 0.05 -l 0.00002 -k 50 --auto-stop libffm_trn_fold_11.txt libffm_fold_11_model
!./ffm-predict libffm_val_fold_11.txt libffm_fold_11_model val_preds_fold_11.txt
!./ffm-predict libffm_tst_fold_11.txt libffm_fold_11_model tst_preds_fold_11.txt
os.remove('libffm_val_fold_11.txt')
 os.remove('libffm_trn_fold_11.txt')
 os.remove('libffm_fold_11_model')
 os.remove('libffm_tst_fold_11.txt')
(
 log_loss(train['target'].iloc[fold_ids[10][1]], pd.read_csv('val_preds_fold_11.txt', header=None ).values[:,0]),
roc_auc_score(train['target'].iloc[fold_ids[10][1]], pd.read_csv('val_preds_fold_11.txt', header=None ).values[:,0])
)
submission = test[["test_id"]]
submission["price"] = preds
submission.to_csv("output_submission.csv", index=False )<define_variables>
!./ffm-train -p libffm_val_fold_12.txt -r 0.05 -l 0.00002 -k 50 --auto-stop libffm_trn_fold_12.txt libffm_fold_12_model
!./ffm-predict libffm_val_fold_12.txt libffm_fold_12_model val_preds_fold_12.txt
!./ffm-predict libffm_tst_fold_12.txt libffm_fold_12_model tst_preds_fold_12.txt
os.remove('libffm_val_fold_12.txt')
 os.remove('libffm_trn_fold_12.txt')
 os.remove('libffm_fold_12_model')
 os.remove('libffm_tst_fold_12.txt')
(
 log_loss(train['target'].iloc[fold_ids[11][1]], pd.read_csv('val_preds_fold_12.txt', header=None ).values[:,0]),
roc_auc_score(train['target'].iloc[fold_ids[11][1]], pd.read_csv('val_preds_fold_12.txt', header=None ).values[:,0])
)
end_time = datetime.now()
execution_time = start_time - end_time
print(execution_time )<feature_engineering>
!./ffm-train -p libffm_val_fold_13.txt -r 0.05 -l 0.00002 -k 50 --auto-stop libffm_trn_fold_13.txt libffm_fold_13_model
!./ffm-predict libffm_val_fold_13.txt libffm_fold_13_model val_preds_fold_13.txt
!./ffm-predict libffm_tst_fold_13.txt libffm_fold_13_model tst_preds_fold_13.txt
os.remove('libffm_val_fold_13.txt')
 os.remove('libffm_trn_fold_13.txt')
 os.remove('libffm_fold_13_model')
 os.remove('libffm_tst_fold_13.txt')
(
 log_loss(train['target'].iloc[fold_ids[12][1]], pd.read_csv('val_preds_fold_13.txt', header=None ).values[:,0]),
roc_auc_score(train['target'].iloc[fold_ids[12][1]], pd.read_csv('val_preds_fold_13.txt', header=None ).values[:,0])
)
import os ; os.environ['OMP_NUM_THREADS'] = '4'<import_modules>
!./ffm-train -p libffm_val_fold_14.txt -r 0.05 -l 0.00002 -k 50 --auto-stop libffm_trn_fold_14.txt libffm_fold_14_model
!./ffm-predict libffm_val_fold_14.txt libffm_fold_14_model val_preds_fold_14.txt
!./ffm-predict libffm_tst_fold_14.txt libffm_fold_14_model tst_preds_fold_14.txt
os.remove('libffm_val_fold_14.txt')
 os.remove('libffm_trn_fold_14.txt')
 os.remove('libffm_fold_14_model')
 os.remove('libffm_tst_fold_14.txt')
(
 log_loss(train['target'].iloc[fold_ids[13][1]], pd.read_csv('val_preds_fold_14.txt', header=None ).values[:,0]),
roc_auc_score(train['target'].iloc[fold_ids[13][1]], pd.read_csv('val_preds_fold_14.txt', header=None ).values[:,0])
)
import pandas as pd
import operator
import matplotlib.pyplot as plt
import numpy as np
import nltk
import time<load_from_csv>
!./ffm-train -p libffm_val_fold_15.txt -r 0.05 -l 0.00002 -k 50 --auto-stop libffm_trn_fold_15.txt libffm_fold_15_model
!./ffm-predict libffm_val_fold_15.txt libffm_fold_15_model val_preds_fold_15.txt
!./ffm-predict libffm_tst_fold_15.txt libffm_fold_15_model tst_preds_fold_15.txt
os.remove('libffm_val_fold_15.txt')
 os.remove('libffm_trn_fold_15.txt')
 os.remove('libffm_fold_15_model')
 os.remove('libffm_tst_fold_15.txt')
(
 log_loss(train['target'].iloc[fold_ids[14][1]], pd.read_csv('val_preds_fold_15.txt', header=None ).values[:,0]),
roc_auc_score(train['target'].iloc[fold_ids[14][1]], pd.read_csv('val_preds_fold_15.txt', header=None ).values[:,0])
)
df = pd.read_csv('.. /input/train.tsv', sep='\t')
 df_sub = pd.read_csv('.. /input/test.tsv', sep='\t')
 submission = pd.DataFrame()
submission['test_id'] = df_sub.test_id.copy()
y_target = list(df.price )<count_missing_values>
!./ffm-train -p libffm_val_fold_16.txt -r 0.05 -l 0.00002 -k 50 --auto-stop libffm_trn_fold_16.txt libffm_fold_16_model
!./ffm-predict libffm_val_fold_16.txt libffm_fold_16_model val_preds_fold_16.txt
!./ffm-predict libffm_tst_fold_16.txt libffm_fold_16_model tst_preds_fold_16.txt
os.remove('libffm_val_fold_16.txt')
 os.remove('libffm_trn_fold_16.txt')
 os.remove('libffm_fold_16_model')
 os.remove('libffm_tst_fold_16.txt')
(
 log_loss(train['target'].iloc[fold_ids[15][1]], pd.read_csv('val_preds_fold_16.txt', header=None ).values[:,0]),
roc_auc_score(train['target'].iloc[fold_ids[15][1]], pd.read_csv('val_preds_fold_16.txt', header=None ).values[:,0])
)
def null_percentage(column):
df_name = column.name
nans = np.count_nonzero(column.isnull().values)
 total = column.size
frac = nans / total
perc = int(frac * 100)
 print('%d%% or %d missing from %s column.' % 
(perc, nans, df_name))
def check_null(df, columns):
for col in columns:
null_percentage(df[col])
 check_null(df, df.columns)
<feature_engineering>
!./ffm-train -p libffm_val_fold_17.txt -r 0.05 -l 0.00002 -k 50 --auto-stop libffm_trn_fold_17.txt libffm_fold_17_model
!./ffm-predict libffm_val_fold_17.txt libffm_fold_17_model val_preds_fold_17.txt
!./ffm-predict libffm_tst_fold_17.txt libffm_fold_17_model tst_preds_fold_17.txt
os.remove('libffm_val_fold_17.txt')
 os.remove('libffm_trn_fold_17.txt')
 os.remove('libffm_fold_17_model')
 os.remove('libffm_tst_fold_17.txt')
(
 log_loss(train['target'].iloc[fold_ids[16][1]], pd.read_csv('val_preds_fold_17.txt', header=None ).values[:,0]),
roc_auc_score(train['target'].iloc[fold_ids[16][1]], pd.read_csv('val_preds_fold_17.txt', header=None ).values[:,0])
)
def merc_imputer(df_temp):
df_temp.brand_name = df_temp.brand_name.replace(np.nan, 'no_brand')
 df_temp.category_name = df_temp.category_name.replace(np.nan, 'uncategorized/uncategorized')
 df_temp.item_description = df_temp.item_description.replace(np.nan, 'No description yet')
 df_temp.item_description = df_temp.item_description.replace('No description yet', 'no_description')
 return df_temp
df = merc_imputer(df)
 df_sub = merc_imputer(df_sub )<train_model>
!./ffm-train -p libffm_val_fold_18.txt -r 0.05 -l 0.00002 -k 50 --auto-stop libffm_trn_fold_18.txt libffm_fold_18_model
!./ffm-predict libffm_val_fold_18.txt libffm_fold_18_model val_preds_fold_18.txt
!./ffm-predict libffm_tst_fold_18.txt libffm_fold_18_model tst_preds_fold_18.txt
os.remove('libffm_val_fold_18.txt')
 os.remove('libffm_trn_fold_18.txt')
 os.remove('libffm_fold_18_model')
 os.remove('libffm_tst_fold_18.txt')
(
 log_loss(train['target'].iloc[fold_ids[17][1]], pd.read_csv('val_preds_fold_18.txt', header=None ).values[:,0]),
roc_auc_score(train['target'].iloc[fold_ids[17][1]], pd.read_csv('val_preds_fold_18.txt', header=None ).values[:,0])
)
print('Training Data')
 check_null(df, df.columns)
 print('Submission Data')
 check_null(df_sub, df_sub.columns )<count_values>
!./ffm-train -p libffm_val_fold_19.txt -r 0.05 -l 0.00002 -k 50 --auto-stop libffm_trn_fold_19.txt libffm_fold_19_model
!./ffm-predict libffm_val_fold_19.txt libffm_fold_19_model val_preds_fold_19.txt
!./ffm-predict libffm_tst_fold_19.txt libffm_fold_19_model tst_preds_fold_19.txt
os.remove('libffm_val_fold_19.txt')
 os.remove('libffm_trn_fold_19.txt')
 os.remove('libffm_fold_19_model')
 os.remove('libffm_tst_fold_19.txt')
(
 log_loss(train['target'].iloc[fold_ids[18][1]], pd.read_csv('val_preds_fold_19.txt', header=None ).values[:,0]),
roc_auc_score(train['target'].iloc[fold_ids[18][1]], pd.read_csv('val_preds_fold_19.txt', header=None ).values[:,0])
)
df.shipping.value_counts()<count_values>
!./ffm-train -p libffm_val_fold_20.txt -r 0.05 -l 0.00002 -k 50 --auto-stop libffm_trn_fold_20.txt libffm_fold_20_model
!./ffm-predict libffm_val_fold_20.txt libffm_fold_20_model val_preds_fold_20.txt
!./ffm-predict libffm_tst_fold_20.txt libffm_fold_20_model tst_preds_fold_20.txt
os.remove('libffm_val_fold_20.txt')
 os.remove('libffm_trn_fold_20.txt')
 os.remove('libffm_fold_20_model')
 os.remove('libffm_tst_fold_20.txt')
(
 log_loss(train['target'].iloc[fold_ids[19][1]], pd.read_csv('val_preds_fold_20.txt', header=None ).values[:,0]),
roc_auc_score(train['target'].iloc[fold_ids[19][1]], pd.read_csv('val_preds_fold_20.txt', header=None ).values[:,0])
)
print('$1 items: ' + str(df.price[df.price == 1].count()))
print('$2 items: ' + str(df.price[df.price == 2].count()))
print('$3 items: ' + str(df.price[df.price == 3].count()))<filter>
!./ffm-train -p libffm_val_fold_21.txt -r 0.05 -l 0.00002 -k 50 --auto-stop libffm_trn_fold_21.txt libffm_fold_21_model
!./ffm-predict libffm_val_fold_21.txt libffm_fold_21_model val_preds_fold_21.txt
!./ffm-predict libffm_tst_fold_21.txt libffm_fold_21_model tst_preds_fold_21.txt
os.remove('libffm_val_fold_21.txt')
 os.remove('libffm_trn_fold_21.txt')
 os.remove('libffm_fold_21_model')
 os.remove('libffm_tst_fold_21.txt')
(
 log_loss(train['target'].iloc[fold_ids[20][1]], pd.read_csv('val_preds_fold_21.txt', header=None ).values[:,0]),
roc_auc_score(train['target'].iloc[fold_ids[20][1]], pd.read_csv('val_preds_fold_21.txt', header=None ).values[:,0])
)
print('Free items: %d, representing %.5f%% of all items.' % 
(df.price[df.price == 0].count() , 
(df.price[df.price == 0].count() / df.shape[0])) )<count_values>
!./ffm-train -p libffm_val_fold_22.txt -r 0.05 -l 0.00002 -k 50 --auto-stop libffm_trn_fold_22.txt libffm_fold_22_model
!./ffm-predict libffm_val_fold_22.txt libffm_fold_22_model val_preds_fold_22.txt
!./ffm-predict libffm_tst_fold_22.txt libffm_fold_22_model tst_preds_fold_22.txt
os.remove('libffm_val_fold_22.txt')
 os.remove('libffm_trn_fold_22.txt')
 os.remove('libffm_fold_22_model')
 os.remove('libffm_tst_fold_22.txt')
(
 log_loss(train['target'].iloc[fold_ids[21][1]], pd.read_csv('val_preds_fold_22.txt', header=None ).values[:,0]),
roc_auc_score(train['target'].iloc[fold_ids[21][1]], pd.read_csv('val_preds_fold_22.txt', header=None ).values[:,0])
)
print('Free items where seller pays shipping: %d.' %
df.price[operator.and_(df.price == 0, df.shipping == 1)].count() )<count_values>
!./ffm-train -p libffm_val_fold_23.txt -r 0.05 -l 0.00002 -k 50 --auto-stop libffm_trn_fold_23.txt libffm_fold_23_model
!./ffm-predict libffm_val_fold_23.txt libffm_fold_23_model val_preds_fold_23.txt
!./ffm-predict libffm_tst_fold_23.txt libffm_fold_23_model tst_preds_fold_23.txt
os.remove('libffm_val_fold_23.txt')
 os.remove('libffm_trn_fold_23.txt')
 os.remove('libffm_fold_23_model')
 os.remove('libffm_tst_fold_23.txt')
(
 log_loss(train['target'].iloc[fold_ids[22][1]], pd.read_csv('val_preds_fold_23.txt', header=None ).values[:,0]),
roc_auc_score(train['target'].iloc[fold_ids[22][1]], pd.read_csv('val_preds_fold_23.txt', header=None ).values[:,0])
)
print('No description:', str(df.item_description[df.item_description == 'no_description'].count()))
print('Uncategorized:',str(df.category_name[df.category_name == 'uncategorized/uncategorized'].count()))<define_variables>
!./ffm-train -p libffm_val_fold_24.txt -r 0.05 -l 0.00002 -k 50 --auto-stop libffm_trn_fold_24.txt libffm_fold_24_model
!./ffm-predict libffm_val_fold_24.txt libffm_fold_24_model val_preds_fold_24.txt
!./ffm-predict libffm_tst_fold_24.txt libffm_fold_24_model tst_preds_fold_24.txt
os.remove('libffm_val_fold_24.txt')
 os.remove('libffm_trn_fold_24.txt')
 os.remove('libffm_fold_24_model')
 os.remove('libffm_tst_fold_24.txt')
(
 log_loss(train['target'].iloc[fold_ids[23][1]], pd.read_csv('val_preds_fold_24.txt', header=None ).values[:,0]),
roc_auc_score(train['target'].iloc[fold_ids[23][1]], pd.read_csv('val_preds_fold_24.txt', header=None ).values[:,0])
)
print('The top 75%% of categories represent %.1f%% of the dataset, and the top 50%% represent %.1f%%.' % 
(( sum([count for count in cat_counts if count > 10])/ len(df)) *100, 
(sum([count for count in cat_counts if count > 76])/ len(df)) *100))<count_values>
!./ffm-train -p libffm_val_fold_25.txt -r 0.05 -l 0.00002 -k 50 --auto-stop libffm_trn_fold_25.txt libffm_fold_25_model
!./ffm-predict libffm_val_fold_25.txt libffm_fold_25_model val_preds_fold_25.txt
!./ffm-predict libffm_tst_fold_25.txt libffm_fold_25_model tst_preds_fold_25.txt
os.remove('libffm_val_fold_25.txt')
 os.remove('libffm_trn_fold_25.txt')
 os.remove('libffm_fold_25_model')
 os.remove('libffm_tst_fold_25.txt')
(
 log_loss(train['target'].iloc[fold_ids[24][1]], pd.read_csv('val_preds_fold_25.txt', header=None ).values[:,0]),
roc_auc_score(train['target'].iloc[fold_ids[24][1]], pd.read_csv('val_preds_fold_25.txt', header=None ).values[:,0])
)
brand_counts = np.sort(df.brand_name.value_counts())
 print(str(len(brand_counts)) + ' brands total.')
 print(str(df.shape[0])+ ' records total.')
 print('Category frequency percentiles, marked by lines:
25%%: %d, 50%%: %d, 75%%: %d, 95%%: %d, 97.5%%: %d.' % 
(brand_counts[int(len(brand_counts)*0.25)],
brand_counts[int(len(brand_counts)*0.5)],
brand_counts[int(len(brand_counts)*0.75)],
brand_counts[int(len(brand_counts)*0.9)],
brand_counts[int(len(brand_counts)*0.95)]))
title = 'Brand Quantity ECDF Without Top 25 Outliers'
plt.figure(title, figsize=(30,10))
plt.title(title, fontsize=32)
 x = np.sort(df.brand_name.value_counts())
 x = x[0:-25]
y = np.arange(1, len(x)+ 1)/ len(x)
 plt.plot(x, y, marker='.', linestyle='none')
 plt.xticks(fontsize=24)
 plt.yticks(fontsize=26)
 plt.axvline(x=x[int(len(x)*0.25)], linewidth=1, color='b')
 plt.axvline(x=x[int(len(x)*0.5)], linewidth=1, color='b')
 plt.axvline(x=x[int(len(x)*0.75)], linewidth=1, color='b')
 plt.axvline(x=x[int(len(x)*0.95)], linewidth=1, color='b')
 plt.axvline(x=x[int(len(x)*0.975)], linewidth=1, color='b')
 plt.show()<define_variables>
oof_preds = np.zeros(train.shape[0])
 for fold_,(_, val_)in enumerate(fold_ids):
oof_preds[val_] = pd.read_csv(f'val_preds_fold_{fold_+1}.txt', header=None ).values[:, 0]
oof_score = roc_auc_score(train['target'], oof_preds)
 print(oof_score )
print('The top 75%% of categories represent %.1f%% of the dataset, and the top 50%% represent %.1f%%.' % 
(( sum([count for count in brand_counts if count > 1])/ len(df)) *100, 
(sum([count for count in brand_counts if count > 4])/ len(df)) *100))<filter>
test_preds = np.zeros(( test.shape[0], N_Splits))
for fold_ in range(N_Splits):
test_preds[:, fold_] = pd.read_csv(f'tst_preds_fold_{fold_+1}.txt', header=None ).values[:, 0]
test_preds_avg = test_preds.mean(axis=1 )
print('%d items, or %.2f%%, are missing a brand name.' % 
(len(df[df.brand_name == 'no_brand']),
len(df[df.brand_name == 'no_brand'])/ len(df)) )<import_modules>
submission = test[['id']].copy()
submission['target'] = test_preds_avg
submission.to_csv('libffm_sub_531.csv', index=False )
nltk.data.path.append(r'D:\Python\Data Sets
ltk_data')
<feature_engineering>
np.save('test_preds_libffm.npy', test_preds_avg)
 np.save('oof_preds_libffm.npy', oof_preds )
cat_vec = CountVectorizer(stop_words=[stopwords, string.punctuation], max_features=int(len(cat_counts)*0.5))
cat_matrix = cat_vec.fit_transform(df.category_name)
 cat_matrix_sub = cat_vec.transform(df_sub.category_name )<define_variables>
subs = [
'/kaggle/input/bestpublicscores/sub_623.csv',
'/kaggle/input/bestpublicscores/sub_634.csv',
'/kaggle/input/bestpublicscores/sub_626.csv',
'/kaggle/input/bestpublicscores/sub_600.csv',
'/kaggle/input/bestpublicscores/sub_590.csv',
'/kaggle/input/otherbestpublicscores/sub_659.csv',
'/kaggle/input/otherbestpublicscores/sub_606.csv',
'/kaggle/input/otherbestpublicscores/sub_563.csv',
'/kaggle/input/otherbestpublicscores/sub_620.csv',
'/kaggle/input/bestpublicscores3/sub_589.csv',
'libffm_sub_531.csv'
]
cat_tokens = list(zip(cat_vec.get_feature_names() , np.array(cat_matrix.sum(axis=0)).ravel()))<categorify>
predictions = pd.concat([pd.read_csv(sub, index_col='id')for sub in subs], axis=1 ).reset_index(drop=True)
 predictions.columns = ['sub_'+str(i)for i in range(11)]
predictions
brand_vec = CountVectorizer(stop_words=[stopwords, string.punctuation], max_features=int(len(brand_counts)*0.5))
brand_matrix = brand_vec.fit_transform(df.brand_name)
 brand_matrix_sub = brand_vec.transform(df_sub.brand_name )<define_variables>
for col in predictions.columns:
predictions[col]=predictions[col].rank() /predictions.shape[0]
<feature_engineering><EOS> 
submission['target'] = blend
submission.to_csv('TopPublicBlend.csv',index=False)
 submission.head()