Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<compute_test_metric>
warnings.filterwarnings('ignore' )
rmsle(np.expm1(train_ridge_preds), np.expm1(y_train_ridge))<compute_test_metric>
train_features = pd.read_csv(data_dir + 'train_features.csv')
 train_targets_scored = pd.read_csv(data_dir + 'train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv(data_dir + 'train_targets_nonscored.csv')
 train_drug = pd.read_csv(data_dir + 'train_drug.csv')
 test_features = pd.read_csv(data_dir + 'test_features.csv')
 sample_submission = pd.read_csv(data_dir + 'sample_submission.csv')
 print('train_features: {}'.format(train_features.shape))
print('train_targets_scored: {}'.format(train_targets_scored.shape))
print('train_targets_nonscored: {}'.format(train_targets_nonscored.shape))
print('train_drug: {}'.format(train_drug.shape))
print('test_features: {}'.format(test_features.shape))
print('sample_submission: {}'.format(sample_submission.shape))
rmsle(np.expm1(val_ridge_preds), np.expm1(y_val_ridge))<compute_test_metric>
train_features2=train_features.copy()
test_features2=test_features.copy()
mean_absolute_error(np.expm1(y_train_ridge), np.expm1(train_ridge_preds))<compute_test_metric>
!pip install /kaggle/input/iterative-stratification/iterative-stratification-master/
mean_absolute_error(np.expm1(y_val_ridge), np.expm1(val_ridge_preds))<groupby>
qt = QuantileTransformer(n_quantiles=100,random_state=42,output_distribution='normal')
 data = pd.concat([pd.DataFrame(train_features[GENES+CELLS]), pd.DataFrame(test_features[GENES+CELLS])])
 data2 = qt.fit_transform(data[GENES+CELLS])
 train_features[GENES+CELLS] = pd.DataFrame(data2[:train_features.shape[0]])
 test_features[GENES+CELLS] = pd.DataFrame(data2[-test_features.shape[0]:] )
def aggregate_predicts(Y1, Y2, ratio):
assert Y1.shape == Y2.shape
return Y1 * ratio + Y2 *(1.0 - ratio )<prepare_output>
SEED_VALUE = 42
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=SEED_VALUE )
val_ridge_preds = np.expm1(val_ridge_preds)
 train_ridge_preds = np.expm1(train_ridge_preds )<prepare_output>
n_comp = 600
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[GENES]))
train2 = data2[:train_features.shape[0]];
test2 = data2[-test_features.shape[0]:]
train_gpca = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test_gpca = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train_gpca), axis=1)
 test_features = pd.concat(( test_features, test_gpca), axis=1 )
rnn_val_preds = np.expm1(rnn_val_preds)
 rnn_train_preds = np.expm1(rnn_train_preds )<predict_on_test>
n_comp = 50
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train_cpca = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test_cpca = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train_cpca), axis=1)
 test_features = pd.concat(( test_features, test_cpca), axis=1)
aggregate_val = aggregate_predicts(rnn_val_preds, val_ridge_preds.reshape(-1, 1), 0.15)
 aggregate_train = aggregate_predicts(rnn_train_preds, train_ridge_preds.reshape(-1, 1), 0.15 )<compute_test_metric>
var_thresh = VarianceThreshold(0.85)
 data = train_features.append(test_features)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1 )
rmsle(aggregate_val, np.expm1(y_val_rnn.values.reshape(-1, 1)) )<compute_test_metric>
def fe_cluster(train, test, n_clusters_g = 22, n_clusters_c = 4, SEED = 42):
features_g = GENES
features_c = CELLS
def create_cluster(train, test, features, kind = 'g', n_clusters = n_clusters_g):
train_ = train[features].copy()
test_ = test[features].copy()
data = pd.concat([train_, test_], axis = 0)
 kmeans = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(data)
 train[f'clusters_{kind}'] = kmeans.labels_[:train.shape[0]]
test[f'clusters_{kind}'] = kmeans.labels_[train.shape[0]:]
train = pd.get_dummies(train, columns = [f'clusters_{kind}'])
 test = pd.get_dummies(test, columns = [f'clusters_{kind}'])
 return train, test
train, test = create_cluster(train, test, features_g, kind = 'g', n_clusters = n_clusters_g)
 train, test = create_cluster(train, test, features_c, kind = 'c', n_clusters = n_clusters_c)
 return train, test
train_features2 ,test_features2=fe_cluster(train_features2,test_features2 )
rmsle(aggregate_train, np.expm1(y_train_rnn.values.reshape(-1, 1)) )<compute_test_metric>
train_pca=pd.concat(( train_gpca,train_cpca),axis=1)
 test_pca=pd.concat(( test_gpca,test_cpca),axis=1 )
mean_absolute_error(np.expm1(y_val_rnn.values.reshape(-1, 1)) , aggregate_val )<compute_test_metric>
def fe_cluster_pca(train, test,n_clusters=5,SEED = 42):
data=pd.concat([train,test],axis=0)
 kmeans = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(data)
 train[f'clusters_pca'] = kmeans.labels_[:train.shape[0]]
test[f'clusters_pca'] = kmeans.labels_[train.shape[0]:]
train = pd.get_dummies(train, columns = [f'clusters_pca'])
 test = pd.get_dummies(test, columns = [f'clusters_pca'])
 return train, test
train_cluster_pca ,test_cluster_pca = fe_cluster_pca(train_pca,test_pca )
mean_absolute_error(np.expm1(y_train_rnn.values.reshape(-1, 1)) , aggregate_train )<predict_on_test>
train_cluster_pca = train_cluster_pca.iloc[:,650:]
test_cluster_pca = test_cluster_pca.iloc[:,650:]
%%time
rnn_preds = predict_rnn(rnn_model, x_test_rnn)
 rnn_preds = np.expm1(rnn_preds )<predict_on_test>
train_features_cluster=train_features2.iloc[:,876:]
test_features_cluster=test_features2.iloc[:,876:]
%%time
ridge_preds = predict_ridge(ridge_model, X_test_ridge)
 ridge_preds = np.expm1(ridge_preds )<predict_on_test>
gsquarecols=['g-574','g-211','g-216','g-0','g-255','g-577','g-153','g-389','g-60','g-370','g-248','g-167','g-203','g-177','g-301','g-332','g-517','g-6','g-744','g-224','g-162','g-3','g-736','g-486','g-283','g-22','g-359','g-361','g-440','g-335','g-106','g-307','g-745','g-146','g-416','g-298','g-666','g-91','g-17','g-549','g-145','g-157','g-768','g-568','g-396']
total_preds = aggregate_predicts(rnn_preds, ridge_preds.reshape(-1, 1), 0.15 )<load_from_csv>
def fe_stats(train, test):
features_g = GENES
features_c = CELLS
for df in train, test:
df['g_sum'] = df[features_g].sum(axis = 1)
 df['g_mean'] = df[features_g].mean(axis = 1)
 df['g_std'] = df[features_g].std(axis = 1)
 df['g_kurt'] = df[features_g].kurtosis(axis = 1)
 df['g_skew'] = df[features_g].skew(axis = 1)
 df['c_sum'] = df[features_c].sum(axis = 1)
 df['c_mean'] = df[features_c].mean(axis = 1)
 df['c_std'] = df[features_c].std(axis = 1)
 df['c_kurt'] = df[features_c].kurtosis(axis = 1)
 df['c_skew'] = df[features_c].skew(axis = 1)
 df['gc_sum'] = df[features_g + features_c].sum(axis = 1)
 df['gc_mean'] = df[features_g + features_c].mean(axis = 1)
 df['gc_std'] = df[features_g + features_c].std(axis = 1)
 df['gc_kurt'] = df[features_g + features_c].kurtosis(axis = 1)
 df['gc_skew'] = df[features_g + features_c].skew(axis = 1)
 df['c52_c42'] = df['c-52'] * df['c-42']
df['c13_c73'] = df['c-13'] * df['c-73']
df['c26_c13'] = df['c-23'] * df['c-13']
df['c33_c6'] = df['c-33'] * df['c-6']
df['c11_c55'] = df['c-11'] * df['c-55']
df['c38_c63'] = df['c-38'] * df['c-63']
df['c38_c94'] = df['c-38'] * df['c-94']
df['c13_c94'] = df['c-13'] * df['c-94']
df['c4_c52'] = df['c-4'] * df['c-52']
df['c4_c42'] = df['c-4'] * df['c-42']
df['c13_c38'] = df['c-13'] * df['c-38']
df['c55_c2'] = df['c-55'] * df['c-2']
df['c55_c4'] = df['c-55'] * df['c-4']
df['c4_c13'] = df['c-4'] * df['c-13']
df['c82_c42'] = df['c-82'] * df['c-42']
df['c66_c42'] = df['c-66'] * df['c-42']
df['c6_c38'] = df['c-6'] * df['c-38']
df['c2_c13'] = df['c-2'] * df['c-13']
df['c62_c42'] = df['c-62'] * df['c-42']
df['c90_c55'] = df['c-90'] * df['c-55']
for feature in features_c:
df[f'{feature}_squared'] = df[feature] ** 2
for feature in gsquarecols:
df[f'{feature}_squared'] = df[feature] ** 2
return train, test
train_features2,test_features2=fe_stats(train_features2,test_features2 )
pd.read_csv('submission.csv' )<define_variables>
train_features_stats=train_features2.iloc[:,902:]
test_features_stats=test_features2.iloc[:,902:]
warnings.filterwarnings('ignore')
 start_time = time.time()
tcurrent = start_time
np.random.seed(54)
 NUM_BRANDS = 4550
NUM_CATEGORIES = 1280
MAX_FEATURES_NAME = 10000
MAX_FEATURES_DESC =10000
develop = True<compute_test_metric>
train_features = pd.concat(( train_features, train_features_cluster,train_cluster_pca,train_features_stats), axis=1)
 test_features = pd.concat(( test_features, test_features_cluster,test_cluster_pca,test_features_stats), axis=1 )
def rmsle(y_true, y_pred):
return np.sqrt(mean_squared_log_error(np.expm1(y_true), np.expm1(y_pred)))
 neg_rmsle = make_scorer(rmsle, greater_is_better=False)
 def split_cat(text):
try:
return text.split("/")
 except:
return("No Label", "No Label", "No Label")
 def handle_missing_inplace(dataset):
dataset['general_cat'].fillna(value='missing', inplace=True)
 dataset['subcat_1'].fillna(value='missing', inplace=True)
 dataset['subcat_2'].fillna(value='missing', inplace=True)
 dataset['brand_name'].fillna(value='missing', inplace=True)
 dataset['item_description'].fillna(value='missing', inplace=True)
 def cutting(dataset):
pop_brand = dataset['brand_name'].value_counts().loc[lambda x: x.index != 'missing'].index[:NUM_BRANDS]
dataset.loc[~dataset['brand_name'].isin(pop_brand), 'brand_name'] = 'missing'
pop_category1 = dataset['general_cat'].value_counts().loc[lambda x: x.index != 'missing'].index[:NUM_CATEGORIES]
pop_category2 = dataset['subcat_1'].value_counts().loc[lambda x: x.index != 'missing'].index[:NUM_CATEGORIES]
pop_category3 = dataset['subcat_2'].value_counts().loc[lambda x: x.index != 'missing'].index[:NUM_CATEGORIES]
dataset.loc[~dataset['general_cat'].isin(pop_category1), 'general_cat'] = 'missing'
dataset.loc[~dataset['subcat_1'].isin(pop_category2), 'subcat_1'] = 'missing'
dataset.loc[~dataset['subcat_2'].isin(pop_category3), 'subcat_2'] = 'missing'
def to_categorical(dataset):
dataset['general_cat'] = dataset['general_cat'].astype('category')
 dataset['subcat_1'] = dataset['subcat_1'].astype('category')
 dataset['subcat_2'] = dataset['subcat_2'].astype('category')
 dataset['item_condition_id'] = dataset['item_condition_id'].astype('category' )<prepare_x_and_y>
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train.merge(train_targets_nonscored, on='sig_id')
 train = train.merge(train_drug, on='sig_id')
 train = train[train['cp_type'] != 'ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type'] != 'ctl_vehicle'].reset_index(drop=True )
print(strftime("%Y-%m-%d %H:%M:%S", gmtime()))
train = pd.read_table('.. /input/train.tsv', engine='c')
 test = pd.read_table('.. /input/test.tsv', engine='c')
 train = train.drop(train[(train.price == 0.0)].index)
 y = np.log1p(train["price"])
 print('[{}] Finished to load data'.format(time.time() - start_time))
print('Train shape: ', train.shape)
 print('Test shape: ', test.shape)
 submission: pd.DataFrame = test[['test_id']]
train.head()<feature_engineering>
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1 )
regex = re.compile('[%s]' % re.escape(string.punctuation))
train['item_description'] = train['item_description'].apply(lambda x: regex.sub('',str(x ).lower()))
test['item_description'] = test['item_description'].apply(lambda x: regex.sub('',str(x ).lower()))
train['name'] = train['name'].apply(lambda x: regex.sub('',x.lower()))
test['name'] = test['name'].apply(lambda x: regex.sub('',x.lower()))
train.head()<feature_engineering>
target_cols = [x for x in train_targets_scored.columns if x != 'sig_id']
aux_target_cols = [x for x in train_targets_nonscored.columns if x != 'sig_id']
all_target_cols = target_cols + aux_target_cols
num_targets = len(target_cols)
 num_aux_targets = len(aux_target_cols)
 num_all_targets = len(all_target_cols)
 print('num_targets: {}'.format(num_targets))
print('num_aux_targets: {}'.format(num_aux_targets))
print('num_all_targets: {}'.format(num_all_targets))
train['general_cat'], train['subcat_1'], train['subcat_2'] = \
zip(*train['category_name'].apply(lambda x: split_cat(x)))
 train.drop('category_name', axis=1, inplace=True)
 test['general_cat'], test['subcat_1'], test['subcat_2'] = \
zip(*test['category_name'].apply(lambda x: split_cat(x)))
 test.drop('category_name', axis=1, inplace=True)
 print('[{}] Split categories completed.'.format(time.time() - start_time))
handle_missing_inplace(train)
 handle_missing_inplace(test)
 print('[{}] Handle missing completed.'.format(time.time() - start_time))
cutting(train)
 cutting(test)
 print('[{}] Cut completed.'.format(time.time() - start_time))
to_categorical(train)
 to_categorical(test)
 print('[{}] Convert categorical completed'.format(time.time() - start_time))
train.head()<categorify>
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct
class ItemSelector(BaseEstimator, TransformerMixin):
def __init__(self, key):
self.key = key
def fit(self, x, y=None):
return self
def transform(self, data_dict):
return data_dict[self.key]<choose_model_class>
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
pipeline = Pipeline([ 
('selector', ItemSelector(key='name')) , 
('tfidf', TfidfVectorizer(ngram_range =(1, 3),
strip_accents = 'unicode',
stop_words = 'english',
min_df=20,
max_df=.9,
max_features = MAX_FEATURES_NAME))
])
 X_name = pipeline.fit_transform(train, train['price'])
 X_name_test = pipeline.transform(test)
 X_name.shape<define_variables>
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss
pipeline = Pipeline([ 
('item_description', Pipeline([ 
('selector', ItemSelector(key='item_description')) , 
('tfidf', TfidfVectorizer(ngram_range =(1, 3),
stop_words = 'english',
min_df=20,
max_df=.9,
max_features = MAX_FEATURES_DESC))
]))
])
 X_text = pipeline.fit_transform(train, train['price'])
 X_text_test = pipeline.transform(test)
 X_text.shape<categorify>
class Model(nn.Module):
def __init__(self, num_features, num_targets):
super(Model, self ).__init__()
self.hidden_size = [1500, 1250, 1000, 750]
self.dropout_value = [0.5, 0.35, 0.3, 0.25]
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.Linear(num_features, self.hidden_size[0])
 self.batch_norm2 = nn.BatchNorm1d(self.hidden_size[0])
 self.dropout2 = nn.Dropout(self.dropout_value[0])
 self.dense2 = nn.Linear(self.hidden_size[0], self.hidden_size[1])
 self.batch_norm3 = nn.BatchNorm1d(self.hidden_size[1])
 self.dropout3 = nn.Dropout(self.dropout_value[1])
 self.dense3 = nn.Linear(self.hidden_size[1], self.hidden_size[2])
 self.batch_norm4 = nn.BatchNorm1d(self.hidden_size[2])
 self.dropout4 = nn.Dropout(self.dropout_value[2])
 self.dense4 = nn.Linear(self.hidden_size[2], self.hidden_size[3])
 self.batch_norm5 = nn.BatchNorm1d(self.hidden_size[3])
 self.dropout5 = nn.Dropout(self.dropout_value[3])
 self.dense5 = nn.utils.weight_norm(nn.Linear(self.hidden_size[3], num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = F.leaky_relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.leaky_relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = F.leaky_relu(self.dense3(x))
x = self.batch_norm4(x)
 x = self.dropout4(x)
 x = F.leaky_relu(self.dense4(x))
x = self.batch_norm5(x)
 x = self.dropout5(x)
 x = self.dense5(x)
 return x
class LabelSmoothingLoss(nn.Module):
def __init__(self, classes, smoothing=0.0, dim=-1):
super(LabelSmoothingLoss, self ).__init__()
self.confidence = 1.0 - smoothing
self.smoothing = smoothing
self.cls = classes
self.dim = dim
def forward(self, pred, target):
pred = pred.log_softmax(dim=self.dim)
 with torch.no_grad() :
true_dist = torch.zeros_like(pred)
 true_dist.fill_(self.smoothing /(self.cls - 1))
true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence)
 return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))
wb = CountVectorizer()
X_category1 = wb.fit_transform(train['general_cat'])
 X_category1_test = wb.transform(test['general_cat'])
 X_category2 = wb.fit_transform(train['subcat_1'])
 X_category2_test = wb.transform(test['subcat_1'])
 X_category3 = wb.fit_transform(train['subcat_2'])
 X_category3_test = wb.transform(test['subcat_2'])
 print('[{}] Count vectorize `categories` completed.'.format(time.time() - start_time))
lb = LabelBinarizer(sparse_output=True)
 X_brand = lb.fit_transform(train['brand_name'])
 X_brand_test = lb.transform(test['brand_name'])
 print('[{}] Label binarize `brand_name` completed.'.format(time.time() - start_time))<categorify>
class FineTuneScheduler:
def __init__(self, epochs):
self.epochs = epochs
self.epochs_per_step = 0
self.frozen_layers = []
def copy_without_top(self, model, num_features, num_targets, num_targets_new):
self.frozen_layers = []
model_new = Model(num_features, num_targets)
 model_new.load_state_dict(model.state_dict())
 for name, param in model_new.named_parameters() :
layer_index = name.split('.')[0][-1]
if layer_index == 5:
continue
param.requires_grad = False
if layer_index not in self.frozen_layers:
self.frozen_layers.append(layer_index)
 self.epochs_per_step = self.epochs // len(self.frozen_layers)
 model_new.batch_norm5 = nn.BatchNorm1d(model_new.hidden_size[3])
 model_new.dropout5 = nn.Dropout(model_new.dropout_value[3])
 model_new.dense5 = nn.utils.weight_norm(nn.Linear(model_new.hidden_size[-1], num_targets_new))
model_new.to(DEVICE)
 return model_new
def step(self, epoch, model):
if len(self.frozen_layers)== 0:
return
if epoch % self.epochs_per_step == 0:
last_frozen_index = self.frozen_layers[-1]
for name, param in model.named_parameters() :
layer_index = name.split('.')[0][-1]
if layer_index == last_frozen_index:
param.requires_grad = True
del self.frozen_layers[-1]
X_dummies = csr_matrix(pd.get_dummies(train[['item_condition_id', 'shipping']],
sparse=True ).values)
 X_dummies_test = csr_matrix(pd.get_dummies(test[['item_condition_id', 'shipping']],
sparse=True ).values)
 print('[{}] Get dummies on `item_condition_id` and `shipping` completed.'.format(time.time() - start_time))
print(X_dummies.shape, X_brand.shape, X_category1.shape, X_category2.shape, X_category3.shape,
X_text.shape)
 sparse_merge = hstack(( X_dummies, X_brand, X_name, X_text,
X_category1, X_category2, X_category3)).tocsr()
sparse_merge_test = hstack(( X_dummies_test, X_brand_test, X_name_test, X_text_test,
X_category1_test, X_category2_test, X_category3_test)).tocsr()
print('[{}] Create sparse merge completed'.format(time.time() - start_time))
del X_dummies, lb, X_brand, X_category1, X_category2, X_category3
del X_name, X_text
del X_dummies_test, X_brand_test, X_category1_test, X_category2_test, X_category3_test
del X_name_test, X_text_test
gc.collect()<prepare_x_and_y>
def process_data(data):
data = pd.get_dummies(data, columns=['cp_time','cp_dose'])
 return data
X = sparse_merge
X_test = sparse_merge_test
train_X, train_y = X, y
del X, sparse_merge, sparse_merge_test
gc.collect()<train_on_grid>
feature_cols = [c for c in process_data(train ).columns if c not in all_target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold', 'sig_id', 'drug_id']]
num_features = len(feature_cols)
 num_features
seed = 101
def target(**params):
fit_intercept = int(params['fit_intercept'])
 fit_intercept_dict = {0:False, 1:True}
model = Ridge(alpha = params['alpha'],
fit_intercept = fit_intercept_dict[fit_intercept],
copy_X = True)
 scores = cross_val_score(model, train_X, train_y, scoring=neg_rmsle, cv=3)
 return scores.mean()
params = {'alpha':(1, 4),
'fit_intercept':(0,1.99)}
if develop:
bo = BayesianOptimization(target, params, random_state=seed)
 bo.gp.set_params(alpha=1e-8)
 bo.maximize(init_points=5, n_iter=10, acq='ucb', kappa=2)
 print(bo.res['max']['max_params'] )<save_to_csv>
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 24
BATCH_SIZE = 128
WEIGHT_DECAY = {'ALL_TARGETS': 1e-5, 'SCORED_ONLY': 3e-6}
MAX_LR = {'ALL_TARGETS': 1e-2, 'SCORED_ONLY': 3e-3}
DIV_FACTOR = {'ALL_TARGETS': 1e3, 'SCORED_ONLY': 1e2}
PCT_START = 0.1
model = Ridge(alpha = 3.0656,
fit_intercept = True,
copy_X = True)
 model.fit(train_X, train_y)
 predsR = model.predict(X_test)
 submission['price'] = np.expm1(predsR)
 submission.to_csv("Bayesian_Ridge.csv", index=False )<train_model>
model = Model(num_features, num_all_targets)
 model
nm=(time.time() - start_time)/60
print("Total processing time %s min" % nm )<define_variables>
def make_cv_folds(train, SEEDS, NFOLDS, DRUG_THRESH):
vc = train.drug_id.value_counts()
vc1 = vc.loc[vc <= DRUG_THRESH].index.sort_values()
vc2 = vc.loc[vc > DRUG_THRESH].index.sort_values()
for seed_id in range(SEEDS):
kfold_col = 'kfold_{}'.format(seed_id)
 dct1 = {}
dct2 = {}
skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.groupby('drug_id')[target_cols].mean().loc[vc1]
for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.index[idxV].values}
dct1.update(dd)
 skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.loc[train.drug_id.isin(vc2)].reset_index(drop=True)
 for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.sig_id[idxV].values}
dct2.update(dd)
 train[kfold_col] = train.drug_id.map(dct1)
 train.loc[train[kfold_col].isna() , kfold_col] = train.loc[train[kfold_col].isna() , 'sig_id'].map(dct2)
 train[kfold_col] = train[kfold_col].astype('int8')
 return train
SEEDS = 7
NFOLDS = 7
DRUG_THRESH = 18
train = make_cv_folds(train, SEEDS, NFOLDS, DRUG_THRESH)
 train.head()
t00 = time.time()
stopwords=set(['i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you',
'your', 'yours', 'yourself', 'yourselves', 'he', 'him', 'his', 'himself',
'she', 'her', 'hers', 'herself', 'it', 'its', 'itself', 'they', 'them', 'their',
'theirs', 'themselves', 'what', 'which', 'who', 'whom', 'this', 'that', 'these',
'those', 'am', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has',
'had', 'having', 'do', 'does', 'did', 'doing', 'a', 'an', 'the', 'and', 'but', 'if',
'or', 'because', 'as', 'until', 'while', 'of', 'at', 'by', 'for', 'with', 'about',
'against', 'between', 'into', 'through', 'during', 'before', 'after', 'above', 'below', 'to',
'from', 'up', 'down', 'in', 'out', 'on', 'off', 'over', 'under', 'again', 'further', 'then',
'once', 'here', 'there', 'when', 'where', 'why', 'how', 'all', 'any', 'both', 'each', 'few',
'more', 'most', 'other', 'some', 'such', 'no', 'nor', 'not', 'only', 'own', 'same', 'so', 'than',
'too', 'very', 's', 't', 'can', 'will', 'just', 'don', 'should', 'now', 'd', 'll', 'm', 'o', 're',
've', 'y', 'ain', 'aren', 'couldn', 'didn', 'doesn', 'hadn', 'hasn', 'haven', 'isn', 'ma', 'mightn',
'mustn', 'needn', 'shan', 'shouldn', 'wasn', 'weren', 'won', 'wouldn',
'&','brand new','new','\[rm\]','free ship.*?','free home',
'rm','price firm','no description yet'
])
 pattern = re.compile(r'\b(' + r'|'.join(stopwords)+ r')\b\s*')
 train = pd.read_csv('.. /input/train.tsv', sep="\t",encoding='utf-8',
converters={'item_description':lambda x: pattern.sub('',x.lower()),
'name':lambda x: pattern.sub('',x.lower())} 
)
 test = pd.read_csv('.. /input/test.tsv', sep="\t",encoding='utf-8',
converters={'item_description':lambda x: pattern.sub('',x.lower()),
'name':lambda x: pattern.sub('',x.lower())} 
)
 simulation=Fa
def introduce_new_unseen_words(desc):
desc = desc.split(' ')
 if random.randrange(0, 10)== 0:
new_word = ''.join(random.sample(string.ascii_letters, random.randrange(3, 15)))
 desc.insert(0, new_word)
 return ' '.join(desc)
 if(simulation==True):
test = pd.concat([test.copy() , test.copy() , test.copy() , test.copy() , test.copy() ], axis=0)
 test.item_description = test.item_description.apply(introduce_new_unseen_words)
 train_label = np.log1p(train['price'])
 train_texts = train['name'].tolist()
test_texts = test['name'].tolist()
print('load tsv completed')
 train['category_name'].fillna('other', inplace=True)
 test['category_name'].fillna('other', inplace=True)
 train['brand_name'].fillna('missing', inplace=True)
 test['brand_name'].fillna('missing', inplace=True)
 test['item_description'].fillna('none', inplace=True)
 train['item_description'].fillna('none', inplace=True)
 test['nm_word_len']=list(map(lambda x: len(x.split()), test_texts))
train['nm_word_len']=list(map(lambda x: len(x.split()),train_texts))
test['desc_word_len']=list(map(lambda x: len(x.split()), test['item_description'].tolist()))
train['desc_word_len']=list(map(lambda x: len(x.split()), train['item_description'].tolist()))
test['nm_len']=list(map(lambda x: len(x),test_texts))
train['nm_len']=list(map(lambda x: len(x),train_texts))
test['desc_len']=list(map(lambda x: len(x), test['item_description'].tolist()))
train['desc_len']=list(map(lambda x: len(x), train['item_description'].tolist()))
nrow_train = train.shape[0]
test_id=test['test_id']
def split_cat(text):
try:
cat_nm=text.split("/")
 if len(cat_nm)>=3:
return cat_nm[0],cat_nm[1],cat_nm[2]
if len(cat_nm)==2:
return cat_Nm[0],cat_nm[1],'missing'
if len(cat_nm)==1:
return cat_nm[0],'missing','missing'
except: return("missing", "missing", "missing")
 train['subcat_0'], train['subcat_1'], train['subcat_2'] = \
zip(*train['category_name'].apply(lambda x: split_cat(x)))
 test['subcat_0'], test['subcat_1'], test['subcat_2'] = \
zip(*test['category_name'].apply(lambda x: split_cat(x)))
 NAME_MIN_DF=30
count = CountVectorizer(min_df=NAME_MIN_DF)
 X_name_mix = count.fit_transform(train['name'].append(test['name']))
X_name=X_name_mix[:nrow_train]
X_t_name = X_name_mix[nrow_train:]
MAX_FEATURES_ITEM_DESCRIPTION=25000
tv = TfidfVectorizer(max_features=MAX_FEATURES_ITEM_DESCRIPTION,
ngram_range=(1,3))
X_description_mix = tv.fit_transform(train['item_description'].append(test['item_description']))
X_description=X_description_mix[:nrow_train]
X_t_description = X_description_mix[nrow_train:]
print('make categorical features')
 cat_features=['subcat_2','subcat_1','subcat_0','brand_name','category_name','item_condition_id','shipping']
for c in cat_features:
newlist=train[c].append(test[c])
 le = LabelEncoder()
le.fit(newlist)
 train[c] = le.transform(train[c])
 test[c] = le.transform(test[c])
 enc = OneHotEncoder()
enc.fit(train[cat_features].append(test[cat_features]))
X_cat = enc.transform(train[cat_features])
 X_t_cat = enc.transform(test[cat_features])
 train_feature=['desc_word_len','nm_word_len','desc_len','nm_len']
train_list = [train[train_feature].values,X_description,X_name,X_cat]
test_list = [test[train_feature].values,X_t_description,X_t_name,X_t_cat]
X = ssp.hstack(train_list ).tocsr()
X_test = ssp.hstack(test_list ).tocsr()
print(' finish feature for training')
 NFOLDS = 3
kfold =KFold(n_splits=NFOLDS, shuffle=True, random_state=128)
 num_boost_round = 1000
params = {"objective": "regression",
"min_data_in_leaf":1000,
"boosting_type": "gbdt",
"learning_rate": 0.65,
"num_leaves": 128,
"feature_fraction": 0.5,
"bagging_freq": 10,
"bagging_fraction": 0.9,
"tree_learner":"voting",
"verbosity": 0,
"metric": "l2_root",
"nthread": 4
}
cv_pred = np.zeros(len(test_id))
kf = kfold.split(X)
 for i,(train_fold, test_fold)in enumerate(kf):
train_t0 = time.time()
X_train, X_validate, label_train, label_validate = \
X[train_fold, :], X[test_fold, :], train_label[train_fold], train_label[test_fold]
dtrain = lgbm.Dataset(X_train, label_train)
 dvalid = lgbm.Dataset(X_validate, label_validate, reference=dtrain)
 bst = lgbm.train(params, dtrain, num_boost_round, valid_sets=dvalid, verbose_eval=100,early_stopping_rounds=100)
 cv_pred += bst.predict(X_test, num_iteration=bst.best_iteration)
 print('training & predict time',time.time() -train_t0)
 gc.collect()
cv_pred /= NFOLDS
cv_pred = np.expm1(cv_pred)
 submission = test[["test_id"]]
submission["price"] = cv_pred
submission.to_csv("./myNNsubmission.csv", index=False)
 print('overall time',time.time() -t00)
<import_modules>
def run_training(fold_id, seed_id):
seed_everything(seed_id)
 train_ = process_data(train)
 test_ = process_data(test)
 kfold_col = f'kfold_{seed_id}'
trn_idx = train_[train_[kfold_col] != fold_id].index
val_idx = train_[train_[kfold_col] == fold_id].index
train_df = train_[train_[kfold_col] != fold_id].reset_index(drop=True)
 valid_df = train_[train_[kfold_col] == fold_id].reset_index(drop=True)
 def train_model(model, tag_name, target_cols_now, fine_tune_scheduler=None):
x_train, y_train = train_df[feature_cols].values, train_df[target_cols_now].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols_now].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 optimizer = torch.optim.Adam(model.parameters() , lr=1e-3, weight_decay=WEIGHT_DECAY[tag_name])
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer,
steps_per_epoch=len(trainloader),
pct_start=PCT_START,
div_factor=DIV_FACTOR[tag_name],
max_lr=MAX_LR[tag_name],
epochs=EPOCHS)
 loss_fn = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing=0.0015)
 oof = np.zeros(( len(train), len(target_cols_now)))
 best_loss = np.inf
for epoch in range(EPOCHS):
if fine_tune_scheduler is not None:
fine_tune_scheduler.step(epoch, model)
 train_loss = train_fn(model, optimizer, scheduler, loss_tr, trainloader, DEVICE)
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"SEED: {seed_id}, FOLD: {fold_id}, {tag_name}, EPOCH: {epoch}, train_loss: {train_loss:.6f}, valid_loss: {valid_loss:.6f}")
 if np.isnan(valid_loss):
break
if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"{tag_name}_FOLD{fold_id}_.pth")
 return oof
fine_tune_scheduler = FineTuneScheduler(EPOCHS)
 pretrained_model = Model(num_features, num_all_targets)
 pretrained_model.to(DEVICE)
 train_model(pretrained_model, 'ALL_TARGETS', all_target_cols)
 pretrained_model = Model(num_features, num_all_targets)
 pretrained_model.load_state_dict(torch.load(f"ALL_TARGETS_FOLD{fold_id}_.pth"))
pretrained_model.to(DEVICE)
 final_model = fine_tune_scheduler.copy_without_top(pretrained_model, num_features, num_all_targets, num_targets)
 oof = train_model(final_model, 'SCORED_ONLY', target_cols, fine_tune_scheduler)
 model = Model(num_features, num_targets)
 model.load_state_dict(torch.load(f"SCORED_ONLY_FOLD{fold_id}_.pth"))
model.to(DEVICE)
 x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 predictions = np.zeros(( len(test_), num_targets))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions
import numpy as np
import pandas as pd
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split
from sklearn.linear_model import Ridge
from sklearn.pipeline import FeatureUnion
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.layers import Input, Dropout, Dense, concatenate, GRU, Embedding, Flatten, Activation
from keras.optimizers import Adam
from keras.models import Model
from keras import backend as K<compute_test_metric>
def run_k_fold(NFOLDS, seed_id):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold_id in range(NFOLDS):
oof_, pred_ = run_training(fold_id, seed_id)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions
def rmsle(Y, Y_pred):
assert Y.shape == Y_pred.shape
return np.sqrt(np.mean(np.square(Y_pred - Y)) )<load_from_csv>
from tqdm.notebook import tqdm
%%time
train_df = pd.read_table('.. /input/train.tsv')
 test_df = pd.read_table('.. /input/test.tsv')
 print(train_df.shape, test_df.shape )<categorify>
SEED = [0, 1, 2, 3, 4, 5, 6]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 time_begin = time()
for seed_id in tqdm(SEED):
oof_, predictions_ = run_k_fold(NFOLDS, seed_id)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 time_diff = time() - time_begin
train[target_cols] = oof
test[target_cols] = predictions
def fill_missing_values(df):
df.category_name.fillna(value="Other", inplace=True)
 df.brand_name.fillna(value="missing", inplace=True)
 df.item_description.fillna(value="None", inplace=True)
 return df
train_df = fill_missing_values(train_df)
 test_df = fill_missing_values(test_df )<prepare_x_and_y>
str(timedelta(seconds=time_diff))
train_df["target"] = np.log1p(train_df.price)
 train_df, dev_df = train_test_split(train_df, random_state=347, train_size=0.99)
 Y_train = train_df.target.values.reshape(-11, 1)
 Y_dev = dev_df.target.values.reshape(-1, 1)
 n_trains = train_df.shape[0]
n_devs = dev_df.shape[0]
n_tests = test_df.shape[0]
print("Training on", n_trains, "examples")
 print("Validating on", n_devs, "examples")
 print("Testing on", n_tests, "examples" )<concatenate>
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score += log_loss(y_true[:, i], y_pred[:, i])
 print("CV log_loss: ", score / y_pred.shape[1] )
<categorify><EOS> 
sub = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub.to_csv('submission.csv', index=False )