Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<define_variables>
DEBUG = False
dv_des = 0<load_pretrained>
sys.path.append('.. /input/iterativestratification')
 !cp -r.. /input/pytorchtabnet/tabnet/*./
os.listdir('.. /input/lish-moa')
mms = pickle.load(open('.. /mms.mms', 'rb'))<categorify>
def log_loss(y_true, y_pred):
losses = []
for col in range(y.shape[1]):
losses.append(_log_loss(y_true[:,col], y_pred[:,col], eps=1e-15))
return sum(losses)/ y_true.shape[1]
condition = X_test[:,2]
shipping = X_test[:,5]
print('MinMaxScaler')
 condition_arr_n = mms.transform(condition.reshape(-1,1))
shipping_arr_n = shipping.reshape(-1,1 ).astype('float64' )<define_variables>
n_comp = 80
Dropout_Model = 0.25
QT_n_quantile_min=10,
QT_n_quantile_max=200
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu' )
mms = 0<load_pretrained>
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )
dv_brand = pickle.load(open('.. /dv_brand.dv', 'rb'))<categorify>
assert(train_features['sig_id'] == train_targets_scored['sig_id'] ).all()
train_features.drop('sig_id', axis=1, inplace=True)
 train_targets_scored.drop('sig_id', axis=1, inplace=True)
 train_targets_nonscored.drop('sig_id', axis=1, inplace=True)
 test_features.drop('sig_id', axis=1, inplace=True )
print('brands')
 words_dict = get_dict_without_split(X_test[:,4])
 print('dv_brand')
 brand_vec_matrix = dv_brand.transform(words_dict )<concatenate>
train_cp = train_features['cp_type'] == 'trt_cp'
test_ctl = test_features['cp_type'] == 'ctl_vehicle'
ctl_df = sample_submission[test_ctl].iloc[:,1:]
ctl_df.values[:,:] = 0
print('X_test_modified')
 X_test_modified = hstack(( names_vec_matrix,cat1d_vec_matrix,cat2d_vec_matrix,cat3d_vec_matrix,shipping_arr_n,condition_arr_n,brand_vec_matrix,des_vec_matrix))<define_variables>
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
CATEGORICAL = ['cp_type', 'cp_time', 'cp_dose']
names_vec_matrix = 0
cat1d_vec_matrix = 0
cat2d_vec_matrix = 0
cat3d_vec_matrix = 0
des_vec_matrix = 0
shipping_arr_n = 0
condition_arr_n = 0
words_dict = 0<load_pretrained>
def encode_categorical(data):
data = pd.get_dummies(data, columns=CATEGORICAL)
 return data
train_features.encoded = encode_categorical(train_features)
 test_features.encoded = encode_categorical(test_features )
lgb_r = pickle.load(open('.. /lgb_r.lgb', 'rb'))<predict_on_test>
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
print('y_pred')
 y_pred = lgb_r.predict(X_test_modified )<define_variables>
print(f"Feature count: {len(train_features.columns)}")
 all_features = pd.concat([train_features.encoded, test_features.encoded])
 def reduce_dimensions(df, n_components, whiten=False):

 pca = PCA(n_components=n_components)
 reduced = pca.fit_transform(df)
 print("
Principal variances
-------------------
" + str(pca.explained_variance_))
return reduced, pca.explained_variance_
temp_, variance = reduce_dimensions(all_features, n_components=130)
 train_features.reduced = temp_[:train_features.shape[0]]
test_features.reduced = temp_[-test_features.shape[0]:]
print('y_pred_e')
 y_pred_e = np.array([np.power(np.e,y)-1 for y in y_pred] )<feature_engineering>
vt = VarianceThreshold (.9)
 all_features.threshold = vt.fit_transform(all_features.values)
 train_features.threshold = all_features.threshold[:train_features.shape[0]]
test_features.threshold = all_features.threshold[-test_features.shape[0]:]
for i in np.where(y_pred_e < 0)[0]:
y_pred_e[i] = np.mean(price )<define_variables>
train_features.enhanced = np.concatenate(( train_features.threshold, train_features.reduced), axis=1)
 test_features.enhanced = np.concatenate(( test_features.threshold, test_features.reduced), axis=1 )
submission = df_test[['test_id']]<prepare_output>
def get_folds(X, y, n_splits=7):
mskf = MultilabelStratifiedKFold(n_splits=n_splits)
 return mskf.split(X, y )
submission['price'] = y_pred_e<save_to_csv>
class MoADataset:
def __init__(self, features, targets):
self.features = torch.tensor(features, dtype=torch.float, device=DEVICE)
 self.targets = torch.tensor(targets, dtype=torch.float, device=DEVICE)
 def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : self.features[idx],
'y' : self.targets[idx]
}
return dct
class TestDataset:
def __init__(self, features):
self.features = torch.tensor(features, dtype=torch.float, device=DEVICE)
 def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : self.features[idx]
}
return dct
submission.to_csv('lgbm_submission.csv', index = False )<define_variables>
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss
start_time = time.time()
develop = False
train_set_ratio =.99
split_seed = 123
train_path = '.. /input/train.tsv'
test_path = '.. /input/test.tsv'
def rmsle(y, y_pred):
assert y.shape == y_pred.shape
return np.sqrt(np.square(np.log(y_pred + 1)- np.log(y + 1)).mean() )<load_from_csv>
EPOCHS = 200
BATCH_SIZE = 1024
LEARNING_RATE =.1e-2
WEIGHT_DECAY = 1e-5
NFOLDS = 7
EARLY_STOPPING_STEPS = 100
EARLY_STOP = True
Dropout_model = 0.7
num_features = train_features.reduced.shape[1]
num_targets = len(train_targets_scored.columns)
 hidden_size=200
train_df = pd.read_csv(train_path, sep='\t')
 test_df = pd.read_csv(test_path, sep='\t')
 print('size of train set:', train_df.shape)
 print('size of test set:', test_df.shape)
 def impute_missing(data_df):
data_df.category_name.fillna(value="missing", inplace=True)
 data_df.brand_name.fillna(value="missing", inplace=True)
 data_df.item_description.fillna(value="missing", inplace=True)
 impute_missing(train_df)
 impute_missing(test_df)
 le = LabelEncoder()
le.fit(np.hstack([train_df.category_name, test_df.category_name]))
train_df.category_name = le.transform(train_df.category_name)
 test_df.category_name = le.transform(test_df.category_name)
 le.fit(np.hstack([train_df.brand_name, test_df.brand_name]))
train_df.brand_name = le.transform(train_df.brand_name)
 test_df.brand_name = le.transform(test_df.brand_name)
 del le
print("[{}] text to seq process...".format(time.time() - start_time))
raw_text = np.hstack([
train_df.item_description.str.lower() ,
train_df.name.str.lower()
])
 print("[{}] fitting tokenizer...".format(time.time() - start_time))
tok_raw = Tokenizer()
tok_raw.fit_on_texts(raw_text)
 train_df["seq_item_description"] = tok_raw.texts_to_sequences(train_df.item_description.str.lower())
 test_df["seq_item_description"] = tok_raw.texts_to_sequences(test_df.item_description.str.lower())
 train_df["seq_name"] = tok_raw.texts_to_sequences(train_df.name.str.lower())
 test_df["seq_name"] = tok_raw.texts_to_sequences(test_df.name.str.lower())
 print("[{}] transforming text to seq...".format(time.time() - start_time))
max_name_seq = np.max([np.max(train_df.seq_name.apply(lambda x: len(x))),
np.max(test_df.seq_name.apply(lambda x: len(x)))])
 max_seq_item_description = np.max([np.max(train_df.seq_item_description.apply(lambda x: len(x))),
np.max(test_df.seq_item_description.apply(lambda x: len(x)))])
 print('[{}] finish calculating MAX_seq'.format(time.time() - start_time))
MAX_NAME_SEQ = 10
MAX_ITEM_DESC_SEQ = 75
MAX_TEXT = np.max([np.max(train_df.seq_name.max()),
np.max(test_df.seq_name.max()),
np.max(train_df.seq_item_description.max()),
np.max(test_df.seq_item_description.max())])+ 2
MAX_CATEGORY = np.max([train_df.category_name.max() , test_df.category_name.max() ])+ 1
MAX_BRAND = np.max([train_df.brand_name.max() , test_df.brand_name.max() ])+ 1
MAX_CONDITION = np.max([train_df.item_condition_id.max() , test_df.item_condition_id.max() ])+ 1
train_df["target"] = np.log(train_df.price + 1)
 target_scaler = MinMaxScaler(feature_range=(-1, 1))
train_df["target"] = target_scaler.fit_transform(train_df.target.reshape(-1, 1))
def get_keras_data(dataset):
X = {
'name': pad_sequences(dataset.seq_name, maxlen=MAX_NAME_SEQ)
 , 'item_desc': pad_sequences(dataset.seq_item_description, maxlen=MAX_ITEM_DESC_SEQ)
 , 'brand_name': np.array(dataset.brand_name)
 , 'category_name': np.array(dataset.category_name)
 , 'item_condition': np.array(dataset.item_condition_id)
 , 'num_vars': np.array(dataset[["shipping"]])
 }
return X
if develop:
dtrain, dvalid = train_test_split(train_df, train_size=train_set_ratio, random_state=split_seed)
 X_train = get_keras_data(dtrain)
 X_valid = get_keras_data(dvalid)
 X_test = get_keras_data(test_df)
 else:
dtrain = train_df
X_train = get_keras_data(dtrain)
 X_test = get_keras_data(test_df)
 ids = test_df[["test_id"]]
del train_df, test_df; gc.collect()
print('[{}] finish forming keras data input for rnn model'.format(time.time() - start_time))
BatchNormalization
dr_r =.1
def get_callbacks(filepath, patience=2):
es = EarlyStopping('val_loss', patience=patience, mode="min")
 msave = ModelCheckpoint(filepath, save_best_only=True)
 return [es, msave]
def rmsle_cust(y_true, y_pred):
first_log = K.log(K.clip(y_pred, K.epsilon() , None)+ 1.)
second_log = K.log(K.clip(y_true, K.epsilon() , None)+ 1.)
return K.sqrt(K.mean(K.square(first_log - second_log), axis=-1))
def rmsle2(y_true, y_pred):
return K.sqrt(K.mean(K.square(y_true - y_pred), axis=-1))
def get_model() :
name = Input(shape=[X_train["name"].shape[1]], name="name")
 item_desc = Input(shape=[X_train["item_desc"].shape[1]], name="item_desc")
 brand_name = Input(shape=[1], name="brand_name")
 category_name = Input(shape=[1], name="category_name")
 item_condition = Input(shape=[1], name="item_condition")
 num_vars = Input(shape=[X_train["num_vars"].shape[1]], name="num_vars")
 emb_name = Embedding(MAX_TEXT, 50 )(name)
 emb_item_desc = Embedding(MAX_TEXT, 50 )(item_desc)
 emb_brand_name = Embedding(MAX_BRAND, 10 )(brand_name)
 emb_category_name = Embedding(MAX_CATEGORY, 10 )(category_name)
 emb_item_condition = Embedding(MAX_CONDITION, 5 )(item_condition)
 rnn_layer1 = GRU(16 )(emb_item_desc)
 rnn_layer2 = GRU(8 )(emb_name)
 main_l = concatenate([
Flatten()(emb_brand_name)
 , Flatten()(emb_category_name)
 , Flatten()(emb_item_condition)
 , rnn_layer1
, rnn_layer2
, num_vars
])
 main_l = Dropout(dr_r )(Dense(128, activation='relu' )(main_l))
output = Dense(1, activation="linear" )(main_l)
 model = Model([name, item_desc, brand_name, category_name, item_condition, num_vars], output)
 model.compile(loss="mse", optimizer="adam", metrics=["mae", rmsle_cust])
 return model
model = get_model()
BATCH_SIZE = 2000
epochs = 2
steps = int(len(X_train['name'])/ BATCH_SIZE)* epochs
lr_init, lr_fin = 0.009, 0.006
exp_decay = lambda init, fin, steps:(init / fin)**(1/(steps - 1)) - 1
lr_decay = exp_decay(lr_init, lr_fin, steps)
 log_subdir = '_'.join(['ep', str(epochs),
'bs', str(BATCH_SIZE),
'lrI', str(lr_init),
'lrF', str(lr_fin),
'dr', str(dr_r)
 ])
 model = get_model()
K.set_value(model.optimizer.lr, lr_init)
 K.set_value(model.optimizer.decay, lr_decay)
 if develop:
model.fit(X_train, dtrain.target
, epochs=epochs
, batch_size=BATCH_SIZE
, validation_data=(X_valid, dvalid.target)
 , verbose=1)
 print('[{}] finish fitting rnn model'.format(time.time() - start_time))
else:
model.fit(X_train, dtrain.target
, epochs=epochs
, batch_size=BATCH_SIZE
, validation_split=0
, verbose=1)
 print('[{}] finish fitting rnn model'.format(time.time() - start_time))
if develop:
val_preds_rnn = model.predict(X_valid)
 val_preds_rnn = target_scaler.inverse_transform(val_preds_rnn)
 val_preds_rnn = np.expm1(val_preds_rnn)
 y_true = np.array(dvalid.price.values ).reshape(dvalid.shape[0], 1)
 v_rmsle = rmsle(y_true, val_preds_rnn)
 print('[{}] RMSLE of validation set with RNN model is: {}'.format(
 time.time() - start_time, v_rmsle))
preds_rnn = model.predict(X_test, batch_size=BATCH_SIZE)
 preds_rnn = target_scaler.inverse_transform(preds_rnn)
 preds_rnn = np.expm1(preds_rnn)
 print('[{}] finish prediction with rnn'.format(time.time() - start_time))
if develop:
del dtrain, dvalid, X_train, X_valid, X_test; gc.collect()
else:
del dtrain, X_train, X_test; gc.collect()<compute_test_metric>
class Chunk(nn.Module):
def __init__(self, in_size, out_size, bn=True, dropout=True):
super().__init__()
self.in_size = in_size
self.out_size = out_size
if bn:
self.bn = nn.BatchNorm1d(in_size)
 else:
self.bn = nn.Identity()
if dropout:
self.dropout = nn.Dropout(Dropout_Model)
 else:
self.dropout = nn.Identity()
self.dense = nn.Linear(in_size, out_size)
 def forward(self, x):
x = self.bn(x)
 x = self.dropout(x)
 x = self.dense(x)
 return x
class Bottleneck(nn.Module):
def __init__(self, size, squeeze_factor=4, **kwargs):
super().__init__()
self.size = size
self.bottleneck_size = size//squeeze_factor
self.chunk1 = Chunk(size, self.bottleneck_size)
 self.chunk2 = Chunk(self.bottleneck_size, size, dropout=False)
 def forward(self, x):
x_skip = x
x = F.leaky_relu(self.chunk1(x))
x = self.chunk2(x)
 return torch.cat([x, x_skip], dim=1)
 class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super().__init__()
self.chunk1 = Chunk(num_features, hidden_size)
 self.chunk2 = Chunk(hidden_size, hidden_size)
 self.chunk3 = Chunk(hidden_size, num_targets)
 def forward(self, x):
x = F.leaky_relu(self.chunk1(x))
x = F.leaky_relu(self.chunk2(x))
x = self.chunk3(x)
 return x
class LabelSmoothingLoss(nn.Module):
def __init__(self, classes, smoothing=0.0, dim=-1):
super(LabelSmoothingLoss, self ).__init__()
self.confidence = 1.0 - smoothing
self.smoothing = smoothing
self.cls = classes
self.dim = dim
def forward(self, pred, target):
pred = pred.log_softmax(dim=self.dim)
 with torch.no_grad() :
true_dist = torch.zeros_like(pred)
 true_dist.fill_(self.smoothing /(self.cls - 1))
true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence)
 return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))
NUM_BRANDS = 4500
NUM_CATEGORIES = 1250
def rmsle(y, y0):
assert len(y)== len(y0)
 return np.sqrt(np.mean(np.power(np.log1p(y)- np.log1p(y0), 2)))
 def split_cat(text):
try:
return text.split("/")
 except:
return("No Label", "No Label", "No Label")
 def handle_missing_inplace(dataset):
dataset['general_cat'].fillna(value='missing', inplace=True)
 dataset['subcat_1'].fillna(value='missing', inplace=True)
 dataset['subcat_2'].fillna(value='missing', inplace=True)
 dataset['brand_name'].fillna(value='missing', inplace=True)
 dataset['item_description'].fillna(value='missing', inplace=True)
 def cutting(dataset):
pop_brand = dataset['brand_name'].value_counts().loc[lambda x: x.index != 'missing'].index[:NUM_BRANDS]
dataset.loc[~dataset['brand_name'].isin(pop_brand), 'brand_name'] = 'missing'
pop_category1 = dataset['general_cat'].value_counts().loc[lambda x: x.index != 'missing'].index[:NUM_CATEGORIES]
pop_category2 = dataset['subcat_1'].value_counts().loc[lambda x: x.index != 'missing'].index[:NUM_CATEGORIES]
pop_category3 = dataset['subcat_2'].value_counts().loc[lambda x: x.index != 'missing'].index[:NUM_CATEGORIES]
dataset.loc[~dataset['general_cat'].isin(pop_category1), 'general_cat'] = 'missing'
dataset.loc[~dataset['subcat_1'].isin(pop_category2), 'subcat_1'] = 'missing'
dataset.loc[~dataset['subcat_2'].isin(pop_category3), 'subcat_2'] = 'missing'
def to_categorical(dataset):
dataset['general_cat'] = dataset['general_cat'].astype('category')
 dataset['subcat_1'] = dataset['subcat_1'].astype('category')
 dataset['subcat_2'] = dataset['subcat_2'].astype('category')
 dataset['item_condition_id'] = dataset['item_condition_id'].astype('category')
 stopwords = {x: 1 for x in stopwords.words('english')}
non_alphanums = re.compile(u'[^A-Za-z0-9]+')
 def normalize_text(text):
return u" ".join(
 [x for x in [y for y in non_alphanums.sub(' ', text ).lower().strip().split(" ")] \
if len(x)> 1 and x not in stopwords])
 def construct_features(start_time):
train = pd.read_table(train_path, engine='c')
 test = pd.read_table(test_path, engine='c')
 print('[{}] Finished to load data'.format(time.time() - start_time))
print('Train shape: ', train.shape)
 print('Test shape: ', test.shape)
 nrow_test = train.shape[0]
dftt = train[(train.price < 1.0)]
train = train.drop(train[(train.price < 1.0)].index)
 del dftt['price']
nrow_train = train.shape[0]
y = np.log1p(train["price"])
 merge: pd.DataFrame = pd.concat([train, dftt, test])
 del train
del test
gc.collect()
merge['general_cat'], merge['subcat_1'], merge['subcat_2'] = \
zip(*merge['category_name'].apply(lambda x: split_cat(x)))
 merge.drop('category_name', axis=1, inplace=True)
 print('[{}] Split categories completed.'.format(time.time() - start_time))
handle_missing_inplace(merge)
 print('[{}] Handle missing completed.'.format(time.time() - start_time))
cutting(merge)
 print('[{}] Cut completed.'.format(time.time() - start_time))
to_categorical(merge)
 print('[{}] Convert categorical completed'.format(time.time() - start_time))
wb = wordbatch.WordBatch(normalize_text, extractor=(WordBag, {"hash_ngrams": 2, "hash_ngrams_weights": [1.5, 1.0],
"hash_size": 2 ** 21, "norm": None, "tf": 'binary',
"idf": None,
}), procs=8)
 wb.dictionary_freeze = True
X_name = wb.fit_transform(merge['name'])
 del(wb)
 X_name = X_name[:, np.array(np.clip(X_name.getnnz(axis=0)- 3, 0, 1), dtype=bool)]
print('[{}] Vectorize `name` completed.'.format(time.time() - start_time))
wb = CountVectorizer()
X_category1 = wb.fit_transform(merge['general_cat'])
 X_category2 = wb.fit_transform(merge['subcat_1'])
 X_category3 = wb.fit_transform(merge['subcat_2'])
 print('[{}] Count vectorize `categories` completed.'.format(time.time() - start_time))
wb = wordbatch.WordBatch(normalize_text, extractor=(WordBag, {"hash_ngrams": 2, "hash_ngrams_weights": [1.0, 1.0],
"hash_size": 2 ** 21, "norm": "l2", "tf": 1.0,
"idf": None}), procs=8)
 wb.dictionary_freeze = True
X_description = wb.fit_transform(merge['item_description'])
 del(wb)
 X_description = X_description[:, np.array(np.clip(X_description.getnnz(axis=0)- 3, 0, 1), dtype=bool)]
print('[{}] Vectorize `item_description` completed.'.format(time.time() - start_time))
lb = LabelBinarizer(sparse_output=True)
 X_brand = lb.fit_transform(merge['brand_name'])
 print('[{}] Label binarize `brand_name` completed.'.format(time.time() - start_time))
X_dummies = csr_matrix(pd.get_dummies(merge[['item_condition_id', 'shipping']],
sparse=True ).values)
 print('[{}] Get dummies on `item_condition_id` and `shipping` completed.'.format(time.time() - start_time))
print(X_dummies.shape, X_description.shape, X_brand.shape, X_category1.shape, X_category2.shape, X_category3.shape,
X_name.shape)
 sparse_merge = hstack(( X_dummies, X_description, X_brand, X_category1, X_category2, X_category3, X_name)).tocsr()
print('[{}] Create sparse merge completed'.format(time.time() - start_time))
return sparse_merge, y, nrow_train, nrow_test
def main(start_time):
print(strftime("%Y-%m-%d %H:%M:%S", gmtime()))
sparse_merge, y, nrow_train, nrow_test = construct_features(start_time)
 print(sparse_merge.shape)
 mask = np.array(np.clip(sparse_merge.getnnz(axis=0)- 3, 0, 1), dtype=bool)
 sparse_merge = sparse_merge[:, mask]
X = sparse_merge[:nrow_train]
X_test = sparse_merge[nrow_test:]
print('X size:', X.shape)
 print('X_test size:', X_test.shape)
 print(sparse_merge.shape)
 gc.collect()
if develop:
train_X, valid_X, train_y, valid_y = train_test_split(X, y, train_size=train_set_ratio, random_state=split_seed)
 else:
train_X, train_y = X, y
del X, y; gc.collect()
model = FM_FTRL(alpha=0.01, beta=0.01, L1=0.00001, L2=0.1,
D=sparse_merge.shape[1], alpha_fm=0.01, L2_fm=0.0,
init_fm=0.01, D_fm=200, e_noise=0.0001, iters=17,
inv_link="identity", threads=4)
 model.fit(train_X, train_y)
 print('[{}] Train ridge v2 completed'.format(time.time() - start_time))
if develop:
val_predsFM = np.expm1(model.predict(X=valid_X))
print("FM_FTRL dev RMSLE:", rmsle(np.expm1(valid_y), val_predsFM))
predsFM = np.expm1(model.predict(X_test))
print('[{}] Predict FM_FTRL completed'.format(time.time() - start_time))
del train_X, train_y, sparse_merge
if develop:
del valid_X, X, y
gc.collect()
if develop:
results = val_predsFM, predsFM, valid_y
else:
results = predsFM
return results
results_linear_models = main(start_time )<compute_test_metric>
input_dim = train_features.encoded.shape[1]
output_dim = train_targets_scored.values.shape[1]
def get_tabnet(input_dim=input_dim, output_dim=output_dim):
model = TabNet(input_dim, output_dim,
n_d=24, n_a=24,
n_steps=1, n_independent=2, n_shared=0)
 model.to(DEVICE)
 optimizer = optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 loss_fn = nn.BCEWithLogitsLoss()
loss = loss_fn
return model, optimizer, loss
if develop:
val_predsFM, predsFM, valid_y = results_linear_models
preds_ensemble = preds_rnn.reshape(preds_rnn.shape[0],)*.6 + predsFM *.4
else:
predsFM = results_linear_models
preds_ensemble = preds_rnn.reshape(preds_rnn.shape[0],)*.6 + predsFM *.4<save_to_csv>
def get_scheduler(optimizer, loader, epochs=EPOCHS):
return optim.lr_scheduler.OneCycleLR(optimizer=optimizer,
pct_start=0.1,
div_factor=40,
max_lr=optimizer.defaults['lr'],
epochs=epochs,
steps_per_epoch=len(loader))
submission = ids
submission["price"] = preds_ensemble
submission.to_csv("ensemble_rnn_wordbatch_ridge_submission_v10.csv", index=False)
 print('[{}] finish everything'.format(time.time() -start_time))<set_options>
def get_model(num_features, num_targets, hidden_size):
model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size,
).to(DEVICE)
 optimizer = optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 loss_fn = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing =0.001)
 loss = loss_fn
return model, optimizer, loss
def get_dataloader(*args, batch_size=BATCH_SIZE, shuffle=False):
if len(args)== 1:
dataset = TestDataset(*args)
 elif len(args)== 2:
dataset = MoADataset(*args)
 loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, shuffle=shuffle)
 return loader
py.init_notebook_mode(connected=True)
 %matplotlib inline
InteractiveShell.ast_node_interactivity = "all" 
<data_type_conversions>
def train_epoch(model, optimizer, scheduler, loss_fn, dataloader, device=DEVICE, output=None):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'], data['y']
if output is None:
outputs = model(inputs)
 else:
outputs = model(inputs)[output]
loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
@torch.no_grad()
def valid_epoch(model, loss_fn, dataloader, device=DEVICE, output=None):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 if output is None:
outputs = model(inputs)
 else:
outputs = model(inputs)[output]
loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def prepare_data(df):
def _clean(text):
return re.sub(r'[^\w\s]','',text)
 def _lower(text):
return text.lower()
for column in ['name', 'brand_name', 'item_description', 'item_condition_id']:
df[column] = df[column].astype(str ).apply(_clean ).apply(_lower)
 df['category_name'] = df['category_name'].astype(str ).apply(_lower)
 def _split_cat(text):
cats = text.split("/")
 if len(cats)>=3:
return cats[0:3]
else: return("No Label", "No Label", "No Label")
 df['general_cat'], df['subcat_1'], df['subcat_2'] = zip(*df['category_name'].apply(lambda x: _split_cat(x)))
 def _length(text):
tokens = re.sub(r'[^\w\s]','',text ).lower()
return len(tokens.split())
 df['desc_len'] = df['item_description'].apply(_length)
 def _concat_(df):
text = ""
for col in ['name', 'general_cat', 'subcat_1', 'subcat_2', 'brand_name', 'item_condition_id']:
text = text +(df[col])+ " "
return text
df['other_info'] = df.loc[:, ['name', 'general_cat', 'subcat_1', 'subcat_2', 'brand_name', 'item_condition_id']].apply(_concat_, axis=1)
 df['other_info_len'] = df['other_info'].apply(_length)
 return df<load_from_csv>
def train_session(model, train_data, valid_data, epochs=EPOCHS, scheduler=None, optimizer=None,
loss=None, session_name=None, patience=EARLY_STOPPING_STEPS, output=None):
early_step = 0
best_loss = np.inf
session_stats = {'Session': session_name}
if not hasattr(model, 'saved_states'):
model.saved_states = []
for epoch in range(epochs):
train_loss = train_epoch(model, optimizer, scheduler, loss, train_data, output=output)
 valid_loss, valid_preds = valid_epoch(model, loss, valid_data, output=output)
 session_stats.update(
 Epoch=epoch+1,
train_loss=f"{train_loss:.5f}",
valid_loss=f"{valid_loss:.5f}",
lr=f"{scheduler.get_last_lr() [0]:.2}" 
)
 print(", ".join(f"{key}: {value}" for key, value in session_stats.items()))
if valid_loss < best_loss:
early_step -=1
best_loss = valid_loss
torch.save(model.state_dict() , f"{session_name}.pt")
 elif(EARLY_STOP == True):
early_step += 1
if(early_step >= patience):
break
model.saved_states.append(f"{session_name}.pt")
 def train_kfold(model, optimizer, loss,
x, y,
initialization,
epochs=EPOCHS,
prefix="",
batch_size=BATCH_SIZE,
patience=EARLY_STOPPING_STEPS):
y_pred = np.array(y, dtype=np.float)
 for n, fold in enumerate(get_folds(x, y)) :
session_name = prefix + f" Fold {n+1}" if prefix else f"Fold {n+1}"
train_idx, valid_idx = fold
trainloader = get_dataloader(x[train_idx], y[train_idx], shuffle=True, batch_size=batch_size)
 validloader = get_dataloader(x[valid_idx], y[valid_idx], shuffle=False, batch_size=batch_size)
 model.load_state_dict(initialization)
 scheduler = get_scheduler(optimizer, trainloader, epochs)
 train_session(model, trainloader, validloader,
optimizer=optimizer, loss=loss,
session_name=session_name,
epochs=epochs,
scheduler=scheduler,
patience=patience)
 model.load_state_dict(torch.load(model.saved_states[-1]))
y_pred[valid_idx] = model(validloader.dataset.features ).sigmoid().detach().cpu().numpy()
return y_pred
df = pd.read_csv(".. /input/train.tsv", sep='\t')
 df.head()
df = prepare_data(df )<choose_model_class>
RUN_PRETRAINING=False
if RUN_PRETRAINING:
x, y = train_features.encoded.values, train_targets_nonscored.values
trainloader = get_dataloader(x, y, shuffle=True, batch_size=2048)
 pretrain_tabnet, optimizer, loss = get_tabnet(input_dim=x.shape[1], output_dim=y.shape[1])
 optimizer.defaults['lr'] =.01
scheduler = get_scheduler(optimizer=optimizer, loader=trainloader, epochs=40)
 train_session(pretrain_tabnet, trainloader, trainloader,
optimizer=optimizer,
loss=loss,
scheduler=scheduler,
session_name="pretrain",
epochs=40,
patience=1)
 pretrain = torch.load("pretrain.pt")
 pretrain.pop("tabnet.final_mapping.weight")
 torch.save(pretrain, "pretrain.pt" )
corpus = []
for row in range(len(df)) :
for column in ['other_info', 'item_description']:
tokens =(df.loc[row, column].split())
 corpus.append(tokens)
 embedding_size = 50
vocab_size = 50000
w2v_model = Word2Vec(corpus, size=embedding_size, min_count=0, max_vocab_size=vocab_size, workers=8, negative=5, sg=1, iter=1,
batch_words=10000, sorted_vocab=1)
 words = list(w2v_model.wv.vocab)
 dictionary = {}
word_embeddings = np.zeros(( len(words), embedding_size), dtype=float)
 row = 0
for word in words:
dictionary[word] = len(dictionary)
 word_embeddings[row, :] = w2v_model.wv.get_vector(word)
 row += 1
reverse_dictionary = dict(zip(dictionary.values() , dictionary.keys()))
pickle.dump(word_embeddings, open("word_embeddings.p", "wb"))
del w2v_model, words, corpus<data_type_conversions>
LOAD_PRETRAIN = False
FREEZE_LAYERS = False
tabnet, optimizer, loss = get_tabnet(input_dim=train_features.encoded.shape[1])
 torch.save(tabnet.state_dict() , "init.pt")
 initialization = torch.load("init.pt")
 if LOAD_PRETRAIN:
pretrain = torch.load("pretrain.pt")
 for key in pretrain.keys() :
initialization[key] = pretrain[key]
if FREEZE_LAYERS:
modules = dict(tabnet.tabnet.named_modules())
 unfreeze_modules = ['final_mapping', 'feat_transformers.1', 'att_transformers.1']
unfreeze_params = [param for name in unfreeze_modules for param in modules[name].parameters() ]
optimizer = optim.Adam(unfreeze_params, lr=2e-02, weight_decay=WEIGHT_DECAY)
 optimizer.defaults['lr'] = 2e-02
x, y = train_features.encoded.values, train_targets_scored.values
y_cv = train_kfold(tabnet, optimizer, loss, x, y, initialization=initialization,
prefix="TabNet", epochs=200, patience=1, batch_size=2048 )
word_embeddings = word_embeddings.astype("float32" )<categorify>
y_true = train_targets_scored.values
print(f"CV Score: {log_loss(y_true,y_cv)}" )
max_D, max_O = 75, 20
def to_array(df):
def _to_array(df, column, max_length):
indices = np.zeros(( len(df), max_length))
length = np.zeros(( len(df),1))
for row in range(len(df)) :
tokens = df.loc[row, column].split()
i = 0
for token in tokens:
if token in dictionary:
indices[row, i] = dictionary[token]
i += 1
if i == max_length: break
length[row] = i
return indices, length
desc_ind, desc_len = _to_array(df, 'item_description', max_D)
 other_ind, other_len = _to_array(df, 'other_info', max_O)
 data = np.hstack(( desc_ind, other_ind, desc_len, other_len))
if 'price' in df.columns:
log_price = np.log(df['price']+1 ).values.reshape(( len(df),1)).astype('float32')
 data = np.hstack(( data, log_price))
return data
data = to_array(df)
 del df
data.shape<split>
TRAIN_MLP = True
if TRAIN_MLP:
mlp, optimizer, loss = get_model(num_features=train_features.enhanced.shape[1],
num_targets=train_targets_scored.shape[1], hidden_size=200)
 mlp.saved_states = []
torch.save(mlp.state_dict() , "init.pt")
 optimizer.defaults['lr'] = 0.04
initialization = torch.load("init.pt")
 x, y = train_features.enhanced, train_targets_scored.values
y_cv = train_kfold(mlp, optimizer, loss, x, y, initialization=initialization,
prefix="MLP", epochs=100, patience=1 )
def train_valid_split(array):
train_id_1 = np.arange(len(array)//3)* 3
train_id_2 = np.arange(len(array)//3)* 3 + 1
valid_id = np.arange(len(array)//3)* 3 + 2
train_id = np.hstack(( train_id_1, train_id_2))
train = array[train_id, :]
valid = array[valid_id, :]
return train, valid
train, valid = train_valid_split(data)
 train.shape, valid.shape
del data<define_variables>
y_true = train_targets_scored.values
print(log_loss(y_true,y_cv))
def get_batch(array, start, size):
if start + size > len(array):
end = len(array)
 else: end = start + size
batch = array[start:end, :]
if end == len(array): start = 0
else: start = start + size
return batch, start<init_hyperparams>
def predict(x, model):
y_pred_ = []
state_dicts = [torch.load(saved_state)for saved_state in model.saved_states]
for sd in state_dicts:
model.load_state_dict(sd)
 y_pred__ = model(x ).sigmoid().detach().cpu().numpy()
y_pred_.append(y_pred__)
 return sum(y_pred_)/ len(state_dicts )
<categorify><EOS> 
ENSEMBLE = True
target_cols = list(sample_submission.columns[1:])
 test_idx = sample_submission.index
x = {'TabNet': torch.tensor(test_features.encoded.values, dtype=torch.float, device=DEVICE),
'MLP': torch.tensor(test_features.enhanced, dtype=torch.float, device=DEVICE)}
if ENSEMBLE:
models = {'MLP': mlp, 'TabNet': tabnet}
y_preds = []
for name in models:
y_preds.append(predict(x[name], models[name]))
y_pred = sum(y_preds)/ len(models)
 else:
y_pred = predict(x['TabNet'], tabnet)
 predictions_df = pd.DataFrame(y_pred, columns=target_cols)
 sample_submission.update(predictions_df)
 sample_submission.update(ctl_df)
 sample_submission.to_csv('submission.csv', index=False )