Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<normalization>
warnings.filterwarnings('ignore' )
X_all_num = pd.concat([X_train_num, X_test_num], axis=0)
 ss = StandardScaler()
ss.fit(X_all_num)
 X_train_ss = ss.transform(X_train_num)
 X_test_ss = ss.transform(X_test_num )<feature_engineering>
!pip install /kaggle/input/iterative-stratification/iterative-stratification-master/
X_train_ss[np.isnan(X_train_ss)] = 0.0
X_test_ss[np.isnan(X_test_ss)] = 0.0<count_unique_values>
train_features = pd.read_csv(data_dir + 'train_features.csv')
 train_targets_scored = pd.read_csv(data_dir + 'train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv(data_dir + 'train_targets_nonscored.csv')
 train_drug = pd.read_csv(data_dir + 'train_drug.csv')
 test_features = pd.read_csv(data_dir + 'test_features.csv')
 sample_submission = pd.read_csv(data_dir + 'sample_submission.csv')
 print('train_features: {}'.format(train_features.shape))
print('train_targets_scored: {}'.format(train_targets_scored.shape))
print('train_targets_nonscored: {}'.format(train_targets_nonscored.shape))
print('train_drug: {}'.format(train_drug.shape))
print('test_features: {}'.format(test_features.shape))
print('sample_submission: {}'.format(sample_submission.shape))
cat_cat = pd.concat([X_train_cat, X_test_cat])
 n_breed1 = cat_cat["Breed1"].nunique()
n_breed2 = cat_cat["Breed2"].nunique()
n_langs = cat_cat["language"].nunique()
n_color1 = cat_cat["Color1"].nunique()
n_color2 = cat_cat["Color2"].nunique()
n_color3 = cat_cat["Color3"].nunique()<categorify>
train_features2=train_features.copy()
test_features2=test_features.copy()
for c in X_train_cat.columns:
le = LabelEncoder()
le.fit(cat_cat[c])
 X_train_cat[c] = le.transform(X_train_cat[c])
 X_test_cat[c] = le.transform(X_test_cat[c] )<statistical_test>
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
def load_fasttext(word_index):
EMBEDDING_FILE = '.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec'
def get_coefs(word, *arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.rstrip().rsplit(' ')) for o in open(EMBEDDING_FILE))
all_embs = np.stack(embeddings_index.values())
 emb_mean,emb_std = all_embs.mean() , all_embs.std()
embed_size = all_embs.shape[1]
nb_words = min(100000, len(word_index)) + 1
embedding_matrix = np.random.normal(emb_mean, emb_std,(nb_words, embed_size))
words_list = []
for word, i in word_index.items() :
if i >= 100000: continue
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
words_list.append(word)
 return embedding_matrix, set(words_list )<train_model>
qt = QuantileTransformer(n_quantiles=100,random_state=42,output_distribution='normal')
 data = pd.concat([pd.DataFrame(train_features[GENES+CELLS]), pd.DataFrame(test_features[GENES+CELLS])])
 data2 = qt.fit_transform(data[GENES+CELLS])
 train_features[GENES+CELLS] = pd.DataFrame(data2[:train_features.shape[0]])
 test_features[GENES+CELLS] = pd.DataFrame(data2[-test_features.shape[0]:] )
tokenizer = Tokenizer(num_words=100000)
 tokenizer.fit_on_texts(X_text["cleaned_text"] + " " + X_text["desc"] + " " + X_text["entity"])
 len(tokenizer.word_index )<string_transform>
def seed_everything(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
seed_everything(42 )
x_train_text = tokenizer.texts_to_sequences(train_text["cleaned_text"])
 x_test_text = tokenizer.texts_to_sequences(test_text["cleaned_text"])
 x_train_text = pad_sequences(x_train_text, maxlen=70)
 x_test_text = pad_sequences(x_test_text, maxlen=70 )<randomize_order>
n_comp = 600
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[GENES]))
train2 = data2[:train_features.shape[0]];
test2 = data2[-test_features.shape[0]:]
train_gpca = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test_gpca = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train_gpca), axis=1)
 test_features = pd.concat(( test_features, test_gpca), axis=1 )
%%time
embedding_matrix, words_set = load_fasttext(tokenizer.word_index )<set_options>
n_comp = 50
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train_cpca = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test_cpca = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train_cpca), axis=1)
 test_features = pd.concat(( test_features, test_cpca), axis=1 )
gc.collect()<split>
var_thresh = VarianceThreshold(0.85)
 data = train_features.append(test_features)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
def calc_vector(text, word_index, words_set, embedding_matrix):
words = set(text.split())
 n_skip = 0
vec = np.zeros(( embedding_matrix.shape[1],))
if len(words)== 0:
return vec
for n_w, word in enumerate(words):
if word in words_set:
idx = word_index.get(word)
 vec_ = embedding_matrix[idx, :]
else:
n_skip += 1
continue
if n_w == 0:
vec = vec_
else:
vec = vec + vec_
vec = vec /(n_w - n_skip + 1)
 return vec<feature_engineering>
def fe_cluster(train, test, n_clusters_g = 22, n_clusters_c = 4, SEED = 42):
features_g = GENES
features_c = CELLS
def create_cluster(train, test, features, kind = 'g', n_clusters = n_clusters_g):
train_ = train[features].copy()
test_ = test[features].copy()
data = pd.concat([train_, test_], axis = 0)
 kmeans = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(data)
 train[f'clusters_{kind}'] = kmeans.labels_[:train.shape[0]]
test[f'clusters_{kind}'] = kmeans.labels_[train.shape[0]:]
train = pd.get_dummies(train, columns = [f'clusters_{kind}'])
 test = pd.get_dummies(test, columns = [f'clusters_{kind}'])
 return train, test
train, test = create_cluster(train, test, features_g, kind = 'g', n_clusters = n_clusters_g)
 train, test = create_cluster(train, test, features_c, kind = 'c', n_clusters = n_clusters_c)
 return train, test
train_features2 ,test_features2=fe_cluster(train_features2,test_features2 )
train_entity = train_text.entity.progress_apply(lambda x: calc_vector(
 x, tokenizer.word_index, words_set, embedding_matrix))
test_entity = test_text.entity.progress_apply(lambda x: calc_vector(
 x, tokenizer.word_index, words_set, embedding_matrix))
train_entity = np.vstack(train_entity.values.tolist())
 test_entity = np.vstack(test_entity.values.tolist())
 train_entity[np.isnan(train_entity)] = 0.0
test_entity[np.isnan(test_entity)] = 0.0
train_desc = train_text.desc.progress_apply(lambda x: calc_vector(
 x, tokenizer.word_index, words_set, embedding_matrix))
test_desc = test_text.desc.progress_apply(lambda x: calc_vector(
 x, tokenizer.word_index, words_set, embedding_matrix))
train_desc = np.vstack(train_desc.values.tolist())
 test_desc = np.vstack(test_desc.values.tolist())
 train_desc[np.isnan(train_desc)] = 0.0
test_desc[np.isnan(test_desc)] = 0.0<normalization>
train_pca=pd.concat(( train_gpca,train_cpca),axis=1)
 test_pca=pd.concat(( test_gpca,test_cpca),axis=1 )
normalize = transforms.Normalize(
 mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225] 
)
 ds_trans = transforms.Compose([transforms.Resize(224),
transforms.CenterCrop(224),
transforms.ToTensor() ,
normalize] )<categorify>
def fe_cluster_pca(train, test,n_clusters=5,SEED = 42):
data=pd.concat([train,test],axis=0)
 kmeans = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(data)
 train[f'clusters_pca'] = kmeans.labels_[:train.shape[0]]
test[f'clusters_pca'] = kmeans.labels_[train.shape[0]:]
train = pd.get_dummies(train, columns = [f'clusters_pca'])
 test = pd.get_dummies(test, columns = [f'clusters_pca'])
 return train, test
train_cluster_pca ,test_cluster_pca = fe_cluster_pca(train_pca,test_pca )
class ImageDataset(data.Dataset):
def __init__(self, pet_ids, root_dir, transform):
self.pet_ids = pet_ids
self.root_dir = root_dir
self.transform = transform
def __len__(self):
return len(self.pet_ids)
 def __getitem__(self, idx):
imgs = torch.zeros(( 4, 3, 224, 224))
for i in range(4):
img_name = f"{self.pet_ids[idx]}-{i+1}.jpg"
fullname = self.root_dir / Path(img_name)
 try:
image = Image.open(fullname ).convert("RGB")
 except FileNotFoundError:
image = np.zeros(( 3, 224, 224), dtype=np.uint8 ).transpose(1, 2, 0)
 image = Image.fromarray(np.uint8(image))
if self.transform:
image = self.transform(image)
 imgs[i, :, :, :] = image
return [self.pet_ids[idx], imgs]<choose_model_class>
train_cluster_pca = train_cluster_pca.iloc[:,650:]
test_cluster_pca = test_cluster_pca.iloc[:,650:]
class Classifier(nn.Module):
def __init__(self):
super(Classifier, self ).__init__()
def forward(self, x):
return x
class ImagePretrained(nn.Module):
def __init__(self, path):
super(ImagePretrained, self ).__init__()
self.densenet121 = models.densenet121()
self.densenet121.load_state_dict(torch.load(path))
self.densenet121.classifier = Classifier()
dense = nn.Sequential(*list(self.densenet121.children())[:-1])
 for param in dense.parameters() :
param.requires_grad = False
def forward(self, x):
converted = torch.zeros(x.size(0), 4, 1024)
 for i in range(4):
out = self.densenet121(x[:, i, :, :, :])
 converted[:, i, :] = out
return converted<create_dataframe>
train_features_cluster=train_features2.iloc[:,876:]
test_features_cluster=test_features2.iloc[:,876:]
train_img_dataset = ImageDataset(train_pet_ids,
".. /input/petfinder-adoption-prediction/train_images/",
transform=ds_trans)
 test_img_dataset = ImageDataset(test_pet_ids,
".. /input/petfinder-adoption-prediction/test_images/",
transform=ds_trans)
 train_img_loader = data.DataLoader(train_img_dataset, batch_size=128, shuffle=False)
 test_img_loader = data.DataLoader(test_img_dataset, batch_size=128, shuffle=False )<categorify>
gsquarecols=['g-574','g-211','g-216','g-0','g-255','g-577','g-153','g-389','g-60','g-370','g-248','g-167','g-203','g-177','g-301','g-332','g-517','g-6','g-744','g-224','g-162','g-3','g-736','g-486','g-283','g-22','g-359','g-361','g-440','g-335','g-106','g-307','g-745','g-146','g-416','g-298','g-666','g-91','g-17','g-549','g-145','g-157','g-768','g-568','g-396']
train_pids = []
train_img_matrix = np.zeros(( len(train_pet_ids), 4, 1024))
model = ImagePretrained("densenet121.pth")
 model.to("cuda:0")
 for i,(pid, tensor)in tqdm(enumerate(train_img_loader)) :
train_pids += [*pid]
tensor = tensor.to("cuda:0")
 pred = model(tensor ).detach().cpu().numpy()
train_img_matrix[i * 128:(i + 1)* 128, :, :] = pred
test_pids = []
test_img_matrix = np.zeros(( len(test_pet_ids), 4, 1024))
for i,(pid, tensor)in tqdm(enumerate(test_img_loader)) :
test_pids += [*pid]
tensor = tensor.to("cuda:0")
 pred = model(tensor ).detach().cpu().numpy()
test_img_matrix[i * 128:(i + 1)* 128, :, :] = pred<set_options>
def fe_stats(train, test):
features_g = GENES
features_c = CELLS
for df in train, test:
df['g_sum'] = df[features_g].sum(axis = 1)
 df['g_mean'] = df[features_g].mean(axis = 1)
 df['g_std'] = df[features_g].std(axis = 1)
 df['g_kurt'] = df[features_g].kurtosis(axis = 1)
 df['g_skew'] = df[features_g].skew(axis = 1)
 df['c_sum'] = df[features_c].sum(axis = 1)
 df['c_mean'] = df[features_c].mean(axis = 1)
 df['c_std'] = df[features_c].std(axis = 1)
 df['c_kurt'] = df[features_c].kurtosis(axis = 1)
 df['c_skew'] = df[features_c].skew(axis = 1)
 df['gc_sum'] = df[features_g + features_c].sum(axis = 1)
 df['gc_mean'] = df[features_g + features_c].mean(axis = 1)
 df['gc_std'] = df[features_g + features_c].std(axis = 1)
 df['gc_kurt'] = df[features_g + features_c].kurtosis(axis = 1)
 df['gc_skew'] = df[features_g + features_c].skew(axis = 1)
 df['c52_c42'] = df['c-52'] * df['c-42']
df['c13_c73'] = df['c-13'] * df['c-73']
df['c26_c13'] = df['c-23'] * df['c-13']
df['c33_c6'] = df['c-33'] * df['c-6']
df['c11_c55'] = df['c-11'] * df['c-55']
df['c38_c63'] = df['c-38'] * df['c-63']
df['c38_c94'] = df['c-38'] * df['c-94']
df['c13_c94'] = df['c-13'] * df['c-94']
df['c4_c52'] = df['c-4'] * df['c-52']
df['c4_c42'] = df['c-4'] * df['c-42']
df['c13_c38'] = df['c-13'] * df['c-38']
df['c55_c2'] = df['c-55'] * df['c-2']
df['c55_c4'] = df['c-55'] * df['c-4']
df['c4_c13'] = df['c-4'] * df['c-13']
df['c82_c42'] = df['c-82'] * df['c-42']
df['c66_c42'] = df['c-66'] * df['c-42']
df['c6_c38'] = df['c-6'] * df['c-38']
df['c2_c13'] = df['c-2'] * df['c-13']
df['c62_c42'] = df['c-62'] * df['c-42']
df['c90_c55'] = df['c-90'] * df['c-55']
for feature in features_c:
df[f'{feature}_squared'] = df[feature] ** 2
for feature in gsquarecols:
df[f'{feature}_squared'] = df[feature] ** 2
return train, test
train_features2,test_features2=fe_stats(train_features2,test_features2 )
gc.collect()<train_on_grid>
train_features_stats=train_features2.iloc[:,902:]
test_features_stats=test_features2.iloc[:,902:]
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4])
 return -cohen_kappa_score(y, preds, weights='quadratic')
 def fit(self, X, y, initial_coef=[]):
loss_partial = partial(self._kappa_loss, X = X, y = y)
 initial_coef = initial_coef
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4])
 return preds
def coefficients(self):
return self.coef_['x']
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):
assert len(rater_a)== len(rater_b)
 if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_rating = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_rating)] for j in range(num_rating)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):
if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):
rater_a = y
rater_b = y_pred
min_rating = None
max_rating = None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert len(rater_a)== len(rater_b)
 min_rating = min(min(rater_a), min(rater_b))
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b, min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(
 hist_rater_a[i] * hist_rater_b[j])/ num_scored_items
d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator )<create_dataframe>
train_features = pd.concat(( train_features, train_features_cluster,train_cluster_pca,train_features_stats), axis=1)
 test_features = pd.concat(( test_features, test_features_cluster,test_cluster_pca,test_features_stats), axis=1 )
class PetDataset(data.Dataset):
def __init__(self,
img_tensor,
cat_features,
num_features,
entity_tensor,
desc_tensor,
seq_tensor,
labels):
self.img_tensor = img_tensor
self.cat_features = cat_features
self.num_features = num_features
self.entity_tensor = entity_tensor
self.desc_tensor = desc_tensor
self.seq_tensor = seq_tensor
if labels is not None:
self.labels = labels
else:
self.labels = None
def __len__(self):
return len(self.cat_features)
 def __getitem__(self, idx):
img_feature = self.img_tensor[idx]
cat_feature = self.cat_features[idx]
num_feature = self.num_features[idx]
entity_feature = self.entity_tensor[idx]
desc_feature = self.desc_tensor[idx]
seq_feature = self.seq_tensor[idx]
if self.labels is not None:
label = self.labels[idx]
return [
img_feature, cat_feature, num_feature,
entity_feature, desc_feature, seq_feature, label]
else:
return [img_feature, cat_feature, num_feature,
entity_feature, desc_feature, seq_feature]<init_hyperparams>
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train.merge(train_targets_nonscored, on='sig_id')
 train = train.merge(train_drug, on='sig_id')
 train = train[train['cp_type'] != 'ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type'] != 'ctl_vehicle'].reset_index(drop=True )
def get_n_params(model):
pp=0
for p in list(model.parameters()):
nn=1
for s in list(p.size()):
nn = nn*s
pp += nn
return pp
def seed_torch(seed=1029):
random.seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.deterministic = True
def run_xgb(params,
X,
y,
X_test,
resc,
n_splits=10,
num_rounds=60000,
early_stop=500,
verbose_eval=1000):
oof_train = np.zeros(( X.shape[0]))
oof_test = np.zeros(( X_test.shape[0], n_splits))
fold = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=1213)
 for i,(trn_index, val_index)in enumerate(fold.split(X, y.astype(int))):
X_tr = X.iloc[trn_index, :]
X_val = X.iloc[val_index, :]
y_tr = y[trn_index]
y_val = y[val_index]
d_train = xgb.DMatrix(
 data=X_tr, label=y_tr, feature_names=X_tr.columns)
 d_valid = xgb.DMatrix(
 data=X_val, label=y_val, feature_names=X_val.columns)
 watchlist = [(d_train, "train"),(d_valid, "valid")]
model = xgb.train(
 params=params,
dtrain=d_train,
num_boost_round=num_rounds,
evals=watchlist,
early_stopping_rounds=early_stop,
verbose_eval=verbose_eval)
 valid_pred = model.predict(
 xgb.DMatrix(X_val, feature_names=X_val.columns),
ntree_limit=model.best_ntree_limit)
 test_pred = model.predict(
 xgb.DMatrix(X_test, feature_names=X_test.columns),
ntree_limit=model.best_ntree_limit)
 oof_train[val_index] = valid_pred
oof_test[:, i] = test_pred
return model, oof_train, oof_test
class Trainer:
def __init__(self,
model,
resc,
n_splits=5,
seed=42,
device="cuda:0",
train_batch=16,
val_batch=32,
kwargs={}):
self.model = model
self.n_splits = n_splits
self.seed = seed
self.device = device
self.train_batch = train_batch
self.val_batch = val_batch
self.kwargs = kwargs
self.resc = resc
self.best_score = None
self.tag = dt.now().strftime("%Y-%m-%d-%H-%M-%S")
 self.loss_fn = nn.MSELoss(reduction="mean" ).to(self.device)
 path = Path(f"bin/{self.tag}")
 path.mkdir(exist_ok=True, parents=True)
 self.path = path
def fit(self,
img_feats,
cat_feats,
num_feats,
entity_feats,
desc_feats,
seq_feats,
answer,
n_epochs=30):
self.train_preds = np.zeros(( train.shape[0]))
answer = answer.values.astype(int)
 fold = StratifiedKFold(n_splits=self.n_splits, shuffle=True, random_state=self.seed)
 cat_feats = cat_feats.values
for i,(trn_idx, val_idx)in enumerate(fold.split(img_feats,
answer)) :
self.fold_num = i
print(f"Fold: {i+1}")
 img_train, img_val = img_feats[trn_idx], img_feats[val_idx]
cat_train, cat_val = cat_feats[trn_idx], cat_feats[val_idx]
num_train, num_val = num_feats[trn_idx], num_feats[val_idx]
ent_train, ent_val = entity_feats[trn_idx], entity_feats[val_idx]
dsc_train, dsc_val = desc_feats[trn_idx], desc_feats[val_idx]
seq_train, seq_val = seq_feats[trn_idx], seq_feats[val_idx]
y_train, y_val = answer[trn_idx] / 4, answer[val_idx] / 4
valid_preds = self._fit(img_train,
cat_train,
num_train,
ent_train,
dsc_train,
seq_train,
y_train,
n_epochs,
img_val,
cat_val,
num_val,
ent_val,
dsc_val,
seq_val,
y_val)
 self.train_preds[val_idx] = valid_preds
def _fit(self,
img,
cat,
num,
ent,
dsc,
seq,
y,
n_epochs,
img_val,
cat_val,
num_val,
ent_val,
dsc_val,
seq_val,
y_val):
seed_torch(self.seed)
 img_tensor = torch.tensor(img, dtype=torch.float32 ).to(self.device)
 cat_tensor = torch.tensor(cat, dtype=torch.long ).to(self.device)
 num_tensor = torch.tensor(num, dtype=torch.float32 ).to(self.device)
 ent_tensor = torch.tensor(ent, dtype=torch.float32 ).to(self.device)
 dsc_tensor = torch.tensor(dsc, dtype=torch.float32 ).to(self.device)
 seq_tensor = torch.tensor(seq, dtype=torch.long ).to(self.device)
 y_tensor = torch.tensor(y[:, np.newaxis], dtype=torch.float32 ).to(self.device)
 train = PetDataset(img_tensor,
cat_tensor,
num_tensor,
ent_tensor,
dsc_tensor,
seq_tensor,
y_tensor)
 train_loader = data.DataLoader(train,
batch_size=self.train_batch, shuffle=True)
 img_eval = torch.tensor(img_val, dtype=torch.float32 ).to(self.device)
 cat_eval = torch.tensor(cat_val, dtype=torch.long ).to(self.device)
 num_eval = torch.tensor(num_val, dtype=torch.float32 ).to(self.device)
 ent_eval = torch.tensor(ent_val, dtype=torch.float32 ).to(self.device)
 dsc_eval = torch.tensor(dsc_val, dtype=torch.float32 ).to(self.device)
 seq_eval = torch.tensor(seq_val, dtype=torch.long ).to(self.device)
 y_eval = torch.tensor(y_val[:, np.newaxis], dtype=torch.float32 ).to(self.device)
 eval_ = PetDataset(img_eval,
cat_eval,
num_eval,
ent_eval,
dsc_eval,
seq_eval,
y_eval)
 eval_loader = data.DataLoader(eval_,
batch_size=self.val_batch, shuffle=False)
 model = self.model(**self.kwargs)
 model = model.to(self.device)
 optimizer = optim.Adam(model.parameters())
 best_score = np.inf
mb = master_bar(range(n_epochs))
for epoch in mb:
model.train()
avg_loss = 0.
 for i_batch, c_batch, n_batch, e_batch, d_batch, s_batch, y_batch in progress_bar(train_loader, parent=mb):
y_pred = model(i_batch, c_batch, n_batch, e_batch, d_batch, s_batch)
 loss = self.loss_fn(y_pred, y_batch)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
avg_loss += loss.item() / len(train_loader)
 valid_preds, avg_val_loss = self._val(eval_loader, model)
 print(f"epoch {epoch+1}/{n_epochs}")
 print(f"avg_loss: {avg_loss:.4f}")
 print(f"avg_val_loss: {avg_val_loss:.4f}")
 if best_score > avg_val_loss:
torch.save(model.state_dict() ,
self.path / f"best{self.fold_num}.pt")
 print(f"Save model on epoch {epoch + 1}")
 best_score = avg_val_loss
model.load_state_dict(torch.load(self.path / f"best{self.fold_num}.pt"))
valid_preds, avg_val_loss = self._val(eval_loader, model)
 print(f"Validation loss: {avg_val_loss}")
 return valid_preds
def _val(self, loader, model):
model.eval()
valid_preds = np.zeros(loader.dataset.cat_features.size(0))
avg_val_loss = 0.
 for i,(i_batch, c_batch, n_batch, e_batch, d_batch, s_batch, y_batch)in enumerate(loader):
with torch.no_grad() :
y_pred = model(i_batch, c_batch, n_batch, e_batch, d_batch, s_batch ).detach()
avg_val_loss += self.loss_fn(y_pred,
y_batch ).item() / len(loader)
 valid_preds[i * self.val_batch:(i + 1)* self.val_batch] = y_pred.cpu().numpy() [:, 0]
return valid_preds, avg_val_loss<choose_model_class>
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1 )
class Attention(nn.Module):
def __init__(self, feature_dims, step_dims, n_middle, n_attention,
**kwargs):
super(Attention, self ).__init__(**kwargs)
 self.support_masking = True
self.feature_dims = feature_dims
self.step_dims = step_dims
self.n_middle = n_middle
self.n_attention = n_attention
self.features_dim = 0
self.lin1 = nn.Linear(feature_dims, n_middle, bias=False)
 self.lin2 = nn.Linear(n_middle, n_attention, bias=False)
 def forward(self, x, mask=None):
step_dims = self.step_dims
eij = self.lin1(x)
 eij = torch.tanh(eij)
 eij = self.lin2(eij)
 a = torch.exp(eij ).reshape(-1, self.n_attention, step_dims)
 if mask is not None:
a = a * mask
a = a / torch.sum(a, 2, keepdim=True)+ 1e-10
weighted_input = torch.bmm(a, x)
 return torch.sum(weighted_input, 1 )<choose_model_class>
target_cols = [x for x in train_targets_scored.columns if x != 'sig_id']
aux_target_cols = [x for x in train_targets_nonscored.columns if x != 'sig_id']
all_target_cols = target_cols + aux_target_cols
num_targets = len(target_cols)
 num_aux_targets = len(aux_target_cols)
 num_all_targets = len(all_target_cols)
 print('num_targets: {}'.format(num_targets))
print('num_aux_targets: {}'.format(num_aux_targets))
print('num_all_targets: {}'.format(num_all_targets))
class NeuralNet(nn.Module):
def __init__(self,
emb_dims,
num_dims,
img_linear,
ent_linear,
dsc_linear,
seq_linear,
linear_size,
embedding_matrix,
hidden_size,
maxlen,
n_attention):
super(NeuralNet, self ).__init__()
self.img_linear = img_linear
n_features, embed_size = embedding_matrix.shape
self.img_lin = nn.Linear(1024, img_linear)
 self.img_attn = Attention(img_linear, 4, 2, 2)
 self.ent_lin = nn.Linear(300, ent_linear)
 self.dsc_lin = nn.Linear(300, dsc_linear)
 self.seq_emb = nn.Embedding(n_features, embed_size)
 self.seq_emb.weight = nn.Parameter(torch.tensor(
 embedding_matrix, dtype=torch.float32))
self.seq_emb.weight.requires_grad = False
self.seq_emb_dropout = nn.Dropout2d(0.2)
 self.lstm = nn.LSTM(
 embed_size, hidden_size, bidirectional=True, batch_first=True)
 self.attn = Attention(hidden_size * 2, maxlen, n_attention,
n_attention)
 self.seq_lin = nn.Linear(hidden_size * 2, seq_linear)
 self.embeddings = nn.ModuleList(
 [nn.Embedding(x, y)for x, y in emb_dims])
 n_emb_out = sum([y for x, y in emb_dims])
 self.fc1 = nn.Linear(
 img_linear + n_emb_out + num_dims + ent_linear + dsc_linear + seq_linear,
linear_size)
 self.bn1 = nn.BatchNorm1d(linear_size)
 self.fc2 = nn.Linear(linear_size, 1)
 self.drop = nn.Dropout(0.2)
 self.relu = nn.ReLU()
self.tanh = nn.Tanh()
def forward(self, i, c, n, e, d, s):
imgs = torch.zeros(i.size(0), 4, self.img_linear ).to("cuda:0")
 for j in range(4):
img_f = self.img_lin(i[:, j, :])
 imgs[:, j, :] = img_f
img_feats = self.drop(self.tanh(self.img_attn(imgs)))
 ent_feats = self.drop(self.tanh(self.ent_lin(e)))
 dsc_feats = self.drop(self.tanh(self.dsc_lin(d)))
 emb = [
emb_layer(c[:, j])for j, emb_layer in enumerate(self.embeddings)
 ]
emb = self.drop(self.tanh(torch.cat(emb, 1)))
 h_seq = self.seq_emb(s)
 h_seq = torch.squeeze(
 self.seq_emb_dropout(torch.unsqueeze(h_seq, 0)))
 h_lstm, _ = self.lstm(h_seq)
 h_attn = self.attn(h_lstm)
 h_lin = self.tanh(self.seq_lin(h_attn))
data = torch.cat([img_feats, emb, n, ent_feats, dsc_feats, h_lin], 1)
 out = self.relu(self.fc1(data))
out = self.bn1(out)
 out = self.fc2(out)
 return out<choose_model_class>
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct
emb_dims = [(2, 1),(n_breed1, 3),(n_breed2, 3),(3, 1),(n_color1, 1), 
(n_color2, 1),(n_color3, 1),(4, 1),(3, 1),(3, 1), 
(3, 1),(3, 1),(3, 1),(19, 1),(14, 1),(n_langs, 1)]
num_dims = X_test_ss.shape[1]
trainer = Trainer(
 NeuralNet,
resc=train_resc,
n_splits=10,
train_batch=64,
val_batch=512,
seed=328,
kwargs={
"emb_dims": emb_dims,
"num_dims": num_dims,
"img_linear": 48,
"linear_size": 144,
"ent_linear": 10,
"dsc_linear": 10,
"seq_linear": 20,
"embedding_matrix": embedding_matrix,
"hidden_size": 64,
"maxlen": 70,
"n_attention": 20
} )<train_model>
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
trainer.fit(train_img_matrix,
X_train_cat,
X_train_ss,
train_entity,
train_desc,
x_train_text, target, 10 )<data_type_conversions>
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss
bin_path = trainer.path
test_preds = np.zeros(( X_test_cat.shape[0]))
i_tensor = torch.tensor(test_img_matrix, dtype=torch.float32 ).to(trainer.device)
 c_tensor = torch.tensor(X_test_cat.values, dtype=torch.long ).to(trainer.device)
 n_tensor = torch.tensor(X_test_ss, dtype=torch.float32 ).to(trainer.device)
 e_tensor = torch.tensor(test_entity, dtype=torch.float32 ).to(trainer.device)
 d_tensor = torch.tensor(test_desc, dtype=torch.float32 ).to(trainer.device)
 s_tensor = torch.tensor(x_test_text, dtype=torch.long ).to(trainer.device)
 test_dataset = PetDataset(i_tensor,
c_tensor,
n_tensor,
e_tensor,
d_tensor,
s_tensor,
labels=None)
 test_loader = data.DataLoader(test_dataset, batch_size=512, shuffle=False)
 for path in bin_path.iterdir() :
print(f"using {str(path)}")
 model = NeuralNet(**trainer.kwargs)
 model.to("cuda:0")
 model.load_state_dict(torch.load(path))
model.eval()
temp = np.zeros(( X_test_cat.shape[0]))
for i,(i_batch, c_batch, n_batch, e_batch, d_batch, s_batch)in enumerate(test_loader):
i_batch = i_batch.to(trainer.device)
 with torch.no_grad() :
y_pred = model(i_batch, c_batch, n_batch, e_batch, d_batch, s_batch ).detach()
temp[i * 512:(i + 1)* 512] = y_pred.cpu().numpy() [:, 0]
test_preds += temp / trainer.n_splits<prepare_output>
class Model(nn.Module):
def __init__(self, num_features, num_targets):
super(Model, self ).__init__()
self.hidden_size = [1500, 1250, 1000, 750]
self.dropout_value = [0.5, 0.35, 0.3, 0.25]
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.Linear(num_features, self.hidden_size[0])
 self.batch_norm2 = nn.BatchNorm1d(self.hidden_size[0])
 self.dropout2 = nn.Dropout(self.dropout_value[0])
 self.dense2 = nn.Linear(self.hidden_size[0], self.hidden_size[1])
 self.batch_norm3 = nn.BatchNorm1d(self.hidden_size[1])
 self.dropout3 = nn.Dropout(self.dropout_value[1])
 self.dense3 = nn.Linear(self.hidden_size[1], self.hidden_size[2])
 self.batch_norm4 = nn.BatchNorm1d(self.hidden_size[2])
 self.dropout4 = nn.Dropout(self.dropout_value[2])
 self.dense4 = nn.Linear(self.hidden_size[2], self.hidden_size[3])
 self.batch_norm5 = nn.BatchNorm1d(self.hidden_size[3])
 self.dropout5 = nn.Dropout(self.dropout_value[3])
 self.dense5 = nn.utils.weight_norm(nn.Linear(self.hidden_size[3], num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = F.leaky_relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.leaky_relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = F.leaky_relu(self.dense3(x))
x = self.batch_norm4(x)
 x = self.dropout4(x)
 x = F.leaky_relu(self.dense4(x))
x = self.batch_norm5(x)
 x = self.dropout5(x)
 x = self.dense5(x)
 return x
class LabelSmoothingLoss(nn.Module):
def __init__(self, classes, smoothing=0.0, dim=-1):
super(LabelSmoothingLoss, self ).__init__()
self.confidence = 1.0 - smoothing
self.smoothing = smoothing
self.cls = classes
self.dim = dim
def forward(self, pred, target):
pred = pred.log_softmax(dim=self.dim)
 with torch.no_grad() :
true_dist = torch.zeros_like(pred)
 true_dist.fill_(self.smoothing /(self.cls - 1))
true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence)
 return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))
class ImageDataset(data.Dataset):
def __init__(self, imat):
self.imat = imat
def __len__(self):
return len(self.imat)
 def __getitem__(self, idx):
image = self.imat[idx]
return [image]<prepare_x_and_y>
class FineTuneScheduler:
def __init__(self, epochs):
self.epochs = epochs
self.epochs_per_step = 0
self.frozen_layers = []
def copy_without_top(self, model, num_features, num_targets, num_targets_new):
self.frozen_layers = []
model_new = Model(num_features, num_targets)
 model_new.load_state_dict(model.state_dict())
 for name, param in model_new.named_parameters() :
layer_index = name.split('.')[0][-1]
if layer_index == 5:
continue
param.requires_grad = False
if layer_index not in self.frozen_layers:
self.frozen_layers.append(layer_index)
 self.epochs_per_step = self.epochs // len(self.frozen_layers)
 model_new.batch_norm5 = nn.BatchNorm1d(model_new.hidden_size[3])
 model_new.dropout5 = nn.Dropout(model_new.dropout_value[3])
 model_new.dense5 = nn.utils.weight_norm(nn.Linear(model_new.hidden_size[-1], num_targets_new))
model_new.to(DEVICE)
 return model_new
def step(self, epoch, model):
if len(self.frozen_layers)== 0:
return
if epoch % self.epochs_per_step == 0:
last_frozen_index = self.frozen_layers[-1]
for name, param in model.named_parameters() :
layer_index = name.split('.')[0][-1]
if layer_index == last_frozen_index:
param.requires_grad = True
del self.frozen_layers[-1]
train_i_tensor = torch.tensor(train_img_matrix, dtype=torch.float32 ).to("cuda:0")
 test_i_tensor = torch.tensor(test_img_matrix, dtype=torch.float32 ).to("cuda:0")
 train_dataset = ImageDataset(train_i_tensor)
 batch = 256
n_img_dim = 48
train_loader = data.DataLoader(train_dataset,
batch_size=batch,
shuffle=False)
 X_train_img = np.zeros(( len(train_pet_ids), n_img_dim))
test_dataset = ImageDataset(test_i_tensor)
 test_loader = data.DataLoader(test_dataset,
batch_size=batch,
shuffle=False)
 X_test_img = np.zeros(( len(test_pet_ids), n_img_dim))
bin_path = trainer.path
for path in bin_path.iterdir() :
model = NeuralNet(**trainer.kwargs)
 model.to("cuda:0")
 model.load_state_dict(torch.load(path))
model.eval()
temp = np.zeros(( len(train_pet_ids), n_img_dim))
for i,(i_batch,)in tqdm(enumerate(train_loader)) :
with torch.no_grad() :
imgs = torch.zeros(i_batch.size(0), 4, n_img_dim ).to("cuda:0")
 for j in range(4):
pre = model.img_lin(i_batch[:, j, :])
 imgs[:, j, :] = pre
y_pred = model.img_attn(imgs ).detach()
temp[i * batch:(i + 1)* batch, :] = y_pred.cpu().numpy()
X_train_img += temp / trainer.n_splits
temp = np.zeros(( len(test_pet_ids), n_img_dim))
for i,(i_batch,)in tqdm(enumerate(test_loader)) :
with torch.no_grad() :
imgs = torch.zeros(i_batch.size(0), 4, n_img_dim ).to("cuda:0")
 for j in range(4):
pre = model.img_lin(i_batch[:, j, :])
 imgs[:, j, :] = pre
y_pred = model.img_attn(imgs ).detach()
temp[i * batch:(i + 1)* batch, :] = y_pred.cpu().numpy()
X_test_img += temp / trainer.n_splits<create_dataframe>
def process_data(data):
data = pd.get_dummies(data, columns=['cp_time','cp_dose'])
 return data
train_img = pd.DataFrame(data=X_train_img, columns=[
f"img{i}" for i in range(X_train_img.shape[1])
 ])
 test_img = pd.DataFrame(data=X_test_img, columns=[
f"img{i}" for i in range(X_test_img.shape[1])
 ] )<concatenate>
feature_cols = [c for c in process_data(train ).columns if c not in all_target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold', 'sig_id', 'drug_id']]
num_features = len(feature_cols)
 num_features
num_columns = X_train_num.columns
X_train_num_df = pd.DataFrame(data=X_train_ss, columns=num_columns)
 X_test_num_df = pd.DataFrame(data=X_test_ss, columns=num_columns)
 X_test_num_df.index = X_test_cat.index
test_img.index = X_test_cat.index
X_train_all = pd.concat([X_train_num_df, X_train_cat, train_img], axis=1)
 X_test_all = pd.concat([X_test_num_df, X_test_cat, test_img], axis=1)
 print(X_train_all.shape, X_test_all.shape)
 X_train_all.head()<import_modules>
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 24
BATCH_SIZE = 128
WEIGHT_DECAY = {'ALL_TARGETS': 1e-5, 'SCORED_ONLY': 3e-6}
MAX_LR = {'ALL_TARGETS': 1e-2, 'SCORED_ONLY': 3e-3}
DIV_FACTOR = {'ALL_TARGETS': 1e3, 'SCORED_ONLY': 1e2}
PCT_START = 0.1
class CategoryDataset(data.Dataset):
def __init__(self, category):
self.category = category
def __len__(self):
return len(self.category)
 def __getitem__(self, idx):
category = self.category[idx, :]
return [category]<create_dataframe>
model = Model(num_features, num_all_targets)
 model
c_train = torch.tensor(X_train_cat.values, dtype=torch.long ).to("cuda:0")
 c_test = torch.tensor(X_test_cat.values, dtype=torch.long ).to("cuda:0")
 train_dataset = CategoryDataset(c_train)
 test_dataset = CategoryDataset(c_test)
 train_loader = data.DataLoader(train_dataset, batch_size=128, shuffle=False)
 test_loader = data.DataLoader(test_dataset, batch_size=128, shuffle=False)
 X_train_cat_ = np.zeros(( len(train_pet_ids), 20))
X_test_cat_ = np.zeros(( len(test_pet_ids), 20))
bin_path = trainer.path
for path in bin_path.iterdir() :
model = NeuralNet(**trainer.kwargs)
 model.to("cuda:0")
 model.load_state_dict(torch.load(path))
model.eval()
temp = np.zeros(( len(train_pet_ids), 20))
for i,(c_batch,)in tqdm(enumerate(train_loader)) :
with torch.no_grad() :
y_pred = [model.embeddings[i](c_batch[:, i])for i in range(len(model.embeddings)) ]
y_pred = torch.cat(y_pred, 1 ).detach()
temp[i * 128:(i + 1)* 128, :] = y_pred.cpu().numpy()
X_train_cat_ += temp / trainer.n_splits
temp = np.zeros(( len(test_pet_ids), 20))
for i,(c_batch,)in tqdm(enumerate(test_loader)) :
with torch.no_grad() :
y_pred = [model.embeddings[i](c_batch[:, i])for i in range(len(model.embeddings)) ]
y_pred = torch.cat(y_pred, 1 ).detach()
temp[i * 128:(i + 1)* 128, :] = y_pred.cpu().numpy()
X_test_cat_ += temp / trainer.n_splits<create_dataframe>
def make_cv_folds(train, SEEDS, NFOLDS, DRUG_THRESH):
vc = train.drug_id.value_counts()
vc1 = vc.loc[vc <= DRUG_THRESH].index.sort_values()
vc2 = vc.loc[vc > DRUG_THRESH].index.sort_values()
for seed_id in range(SEEDS):
kfold_col = 'kfold_{}'.format(seed_id)
 dct1 = {}
dct2 = {}
skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.groupby('drug_id')[target_cols].mean().loc[vc1]
for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.index[idxV].values}
dct1.update(dd)
 skf = MultilabelStratifiedKFold(n_splits=NFOLDS, shuffle=True, random_state=seed_id)
 tmp = train.loc[train.drug_id.isin(vc2)].reset_index(drop=True)
 for fold,(idxT, idxV)in enumerate(skf.split(tmp, tmp[target_cols])) :
dd = {k: fold for k in tmp.sig_id[idxV].values}
dct2.update(dd)
 train[kfold_col] = train.drug_id.map(dct1)
 train.loc[train[kfold_col].isna() , kfold_col] = train.loc[train[kfold_col].isna() , 'sig_id'].map(dct2)
 train[kfold_col] = train[kfold_col].astype('int8')
 return train
SEEDS = 7
NFOLDS = 7
DRUG_THRESH = 18
train = make_cv_folds(train, SEEDS, NFOLDS, DRUG_THRESH)
 train.head()
train_emb = pd.DataFrame(data=X_train_cat_, columns=[
f"emb{i}" for i in range(X_train_cat_.shape[1])
 ])
 test_emb = pd.DataFrame(data=X_test_cat_, columns=[
f"emb{i}" for i in range(X_test_cat_.shape[1])
 ] )<concatenate>
def run_training(fold_id, seed_id):
seed_everything(seed_id)
 train_ = process_data(train)
 test_ = process_data(test)
 kfold_col = f'kfold_{seed_id}'
trn_idx = train_[train_[kfold_col] != fold_id].index
val_idx = train_[train_[kfold_col] == fold_id].index
train_df = train_[train_[kfold_col] != fold_id].reset_index(drop=True)
 valid_df = train_[train_[kfold_col] == fold_id].reset_index(drop=True)
 def train_model(model, tag_name, target_cols_now, fine_tune_scheduler=None):
x_train, y_train = train_df[feature_cols].values, train_df[target_cols_now].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols_now].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 optimizer = torch.optim.Adam(model.parameters() , lr=1e-3, weight_decay=WEIGHT_DECAY[tag_name])
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer,
steps_per_epoch=len(trainloader),
pct_start=PCT_START,
div_factor=DIV_FACTOR[tag_name],
max_lr=MAX_LR[tag_name],
epochs=EPOCHS)
 loss_fn = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing=0.001)
 oof = np.zeros(( len(train), len(target_cols_now)))
 best_loss = np.inf
for epoch in range(EPOCHS):
if fine_tune_scheduler is not None:
fine_tune_scheduler.step(epoch, model)
 train_loss = train_fn(model, optimizer, scheduler, loss_tr, trainloader, DEVICE)
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"SEED: {seed_id}, FOLD: {fold_id}, {tag_name}, EPOCH: {epoch}, train_loss: {train_loss:.6f}, valid_loss: {valid_loss:.6f}")
 if np.isnan(valid_loss):
break
if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"{tag_name}_FOLD{fold_id}_.pth")
 return oof
fine_tune_scheduler = FineTuneScheduler(EPOCHS)
 pretrained_model = Model(num_features, num_all_targets)
 pretrained_model.to(DEVICE)
 train_model(pretrained_model, 'ALL_TARGETS', all_target_cols)
 pretrained_model = Model(num_features, num_all_targets)
 pretrained_model.load_state_dict(torch.load(f"ALL_TARGETS_FOLD{fold_id}_.pth"))
pretrained_model.to(DEVICE)
 final_model = fine_tune_scheduler.copy_without_top(pretrained_model, num_features, num_all_targets, num_targets)
 oof = train_model(final_model, 'SCORED_ONLY', target_cols, fine_tune_scheduler)
 model = Model(num_features, num_targets)
 model.load_state_dict(torch.load(f"SCORED_ONLY_FOLD{fold_id}_.pth"))
model.to(DEVICE)
 x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 predictions = np.zeros(( len(test_), num_targets))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions
test_emb.index = X_test_all.index
X_train_all = pd.concat([X_train_all, train_emb], axis=1)
 X_test_all = pd.concat([X_test_all, test_emb], axis=1)
 print(X_train_all.shape, X_test_all.shape)
 X_train_all.head()<categorify>
def run_k_fold(NFOLDS, seed_id):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold_id in range(NFOLDS):
oof_, pred_ = run_training(fold_id, seed_id)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions
class WordVecDataset(data.Dataset):
def __init__(self, wv):
self.wv = wv
def __len__(self):
return len(self.wv)
 def __getitem__(self, idx):
wv = self.wv[idx, :]
return [wv]<prepare_x_and_y>
SEED = [0, 1, 2, 3, 4, 5, 6]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 time_begin = time()
for seed_id in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed_id)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 time_diff = time() - time_begin
train[target_cols] = oof
test[target_cols] = predictions
e_train = torch.tensor(train_entity, dtype=torch.float32 ).to("cuda:0")
 e_test = torch.tensor(test_entity, dtype=torch.float32 ).to("cuda:0")
 train_dataset = WordVecDataset(e_train)
 test_dataset = WordVecDataset(e_test)
 train_loader = data.DataLoader(train_dataset, batch_size=128, shuffle=False)
 test_loader = data.DataLoader(test_dataset, batch_size=128, shuffle=False)
 X_train_ent = np.zeros(( len(train_pet_ids), 10))
X_test_ent = np.zeros(( len(test_pet_ids), 10))
bin_path = trainer.path
for path in bin_path.iterdir() :
model = NeuralNet(**trainer.kwargs)
 model.to("cuda:0")
 model.load_state_dict(torch.load(path))
model.eval()
temp = np.zeros(( len(train_pet_ids), 10))
for i,(e_batch,)in tqdm(enumerate(train_loader), ascii=True):
with torch.no_grad() :
y_pred = model.ent_lin(e_batch ).detach()
temp[i * 128:(i + 1)* 128, :] = y_pred.cpu().numpy()
X_train_ent += temp / trainer.n_splits
temp = np.zeros(( len(test_pet_ids), 10))
for i,(e_batch,)in tqdm(enumerate(test_loader), ascii=True):
with torch.no_grad() :
y_pred = model.ent_lin(e_batch ).detach()
temp[i * 128:(i + 1)* 128, :] = y_pred.cpu().numpy()
X_test_ent += temp / trainer.n_splits<concatenate>
str(timedelta(seconds=time_diff))
train_ent = pd.DataFrame(data=X_train_ent, columns=[
f"ent{i}" for i in range(X_train_ent.shape[1])
 ])
 test_ent = pd.DataFrame(data=X_test_ent, columns=[
f"ent{i}" for i in range(X_test_ent.shape[1])
 ])
 test_ent.index = X_test_all.index
X_train_all = pd.concat([X_train_all, train_ent], axis=1)
 X_test_all = pd.concat([X_test_all, test_ent], axis=1)
 print(X_train_all.shape, X_test_all.shape)
 X_train_all.head()<prepare_x_and_y>
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score += log_loss(y_true[:, i], y_pred[:, i])
 print("CV log_loss: ", score / y_pred.shape[1] )
d_train = torch.tensor(train_desc, dtype=torch.float32 ).to("cuda:0")
 d_test = torch.tensor(test_desc, dtype=torch.float32 ).to("cuda:0")
 train_dataset = WordVecDataset(d_train)
 test_dataset = WordVecDataset(d_test)
 train_loader = data.DataLoader(train_dataset, batch_size=128, shuffle=False)
 test_loader = data.DataLoader(test_dataset, batch_size=128, shuffle=False)
 X_train_dsc = np.zeros(( len(train_pet_ids), 10))
X_test_dsc = np.zeros(( len(test_pet_ids), 10))
bin_path = trainer.path
for path in bin_path.iterdir() :
model = NeuralNet(**trainer.kwargs)
 model.to("cuda:0")
 model.load_state_dict(torch.load(path))
model.eval()
temp = np.zeros(( len(train_pet_ids), 10))
for i,(d_batch,)in tqdm(enumerate(train_loader), ascii=True):
with torch.no_grad() :
y_pred = model.dsc_lin(d_batch ).detach()
temp[i * 128:(i + 1)* 128, :] = y_pred.cpu().numpy()
X_train_dsc += temp / trainer.n_splits
temp = np.zeros(( len(test_pet_ids), 10))
for i,(d_batch,)in tqdm(enumerate(test_loader), ascii=True):
with torch.no_grad() :
y_pred = model.dsc_lin(d_batch ).detach()
temp[i * 128:(i + 1)* 128, :] = y_pred.cpu().numpy()
X_test_dsc += temp / trainer.n_splits<concatenate>
sub1 = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0 )
train_dsc = pd.DataFrame(data=X_train_dsc, columns=[
f"dsc{i}" for i in range(X_train_dsc.shape[1])
 ])
 test_dsc = pd.DataFrame(data=X_test_dsc, columns=[
f"dsc{i}" for i in range(X_test_dsc.shape[1])
 ])
 test_dsc.index = X_test_all.index
X_train_all = pd.concat([X_train_all, train_dsc], axis=1)
 X_test_all = pd.concat([X_test_all, test_dsc], axis=1)
 print(X_train_all.shape, X_test_all.shape)
 X_train_all.head()<prepare_output>
! pip install.. /input/pytorch16gpu/torch-1.6.0cu101-cp37-cp37m-linux_x86_64.whl
class SeqDataset(data.Dataset):
def __init__(self, seq):
self.seq = seq
def __len__(self):
return len(self.seq)
 def __getitem__(self, idx):
seq = self.seq[idx, :]
return [seq]<prepare_x_and_y>
!pip install.. /input/iterative-stratification-tabnet/iterative_stratification-0.1.6-py3-none-any.whl
s_train = torch.tensor(x_train_text, dtype=torch.long ).to("cuda:0")
 s_test = torch.tensor(x_test_text, dtype=torch.long ).to("cuda:0")
 train_dataset = SeqDataset(s_train)
 test_dataset = SeqDataset(s_test)
 train_loader = data.DataLoader(train_dataset, batch_size=128, shuffle=False)
 test_loader = data.DataLoader(test_dataset, batch_size=128, shuffle=False)
 X_train_seq = np.zeros(( len(train_pet_ids), 20))
X_test_seq = np.zeros(( len(test_pet_ids), 20))
bin_path = trainer.path
for path in bin_path.iterdir() :
model = NeuralNet(**trainer.kwargs)
 model.to("cuda:0")
 model.load_state_dict(torch.load(path))
model.eval()
temp = np.zeros(( len(train_pet_ids), 20))
for i,(s_batch,)in tqdm(enumerate(train_loader), ascii=True):
with torch.no_grad() :
h_emb = model.seq_emb(s_batch)
 h_emb = torch.squeeze(model.seq_emb_dropout(torch.unsqueeze(h_emb, 0)))
 h_lstm, _ = model.lstm(h_emb)
 h_attn = model.attn(h_lstm)
 y_pred = model.seq_lin(h_attn ).detach()
temp[i * 128:(i + 1)* 128, :] = y_pred.cpu().numpy()
X_train_seq += temp / trainer.n_splits
temp = np.zeros(( len(test_pet_ids), 20))
for i,(s_batch,)in tqdm(enumerate(test_loader), ascii=True):
with torch.no_grad() :
h_emb = model.seq_emb(s_batch)
 h_emb = torch.squeeze(model.seq_emb_dropout(torch.unsqueeze(h_emb, 0)))
 h_lstm, _ = model.lstm(h_emb)
 h_attn = model.attn(h_lstm)
 y_pred = model.seq_lin(h_attn ).detach()
temp[i * 128:(i + 1)* 128, :] = y_pred.cpu().numpy()
X_test_seq += temp / trainer.n_splits<concatenate>
os.environ["CUDA_LAUNCH_BLOCKING"] = "1"
train_seq = pd.DataFrame(data=X_train_seq, columns=[
f"seq{i}" for i in range(X_train_seq.shape[1])
 ])
 test_seq = pd.DataFrame(data=X_test_seq, columns=[
f"seq{i}" for i in range(X_test_seq.shape[1])
 ])
 test_seq.index = X_test_all.index
X_train_all = pd.concat([X_train_all, train_seq], axis=1)
 X_test_all = pd.concat([X_test_all, test_seq], axis=1)
 print(X_train_all.shape, X_test_all.shape)
 X_train_all.head()<train_model>
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=42)
 n_comp = 600
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[GENES]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1)
 n_comp = 50
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1)
 var_thresh = VarianceThreshold(0.8)
 data = train_features.append(test_features)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
 train_features.shape
xgb_params = {
"eval_metric": "rmse",
"seed": 1337,
"eta": 0.0123,
"subsample": 0.8,
"colsample_bytree": 0.85,
"tree_method": "gpu_hist",
"device": "gpu",
"silent": 1
}
xgb_X = X_train_all
xgb_y = target
xgb_X_test = X_test_all
model, oof_train_xgb, oof_test_xgb= run_xgb(
 xgb_params,
xgb_X,
xgb_y,
xgb_X_test,
resc=train_resc,
n_splits=10,
num_rounds=10000 )<split>
def get_ratio_labels(df):
columns = list(df.columns)
 columns.pop(0)
 ratios = []
toremove = []
for c in columns:
counts = df[c].value_counts()
if len(counts)!= 1:
ratios.append(counts[0]/counts[1])
 else:
toremove.append(c)
 print(f"remove {len(toremove)} columns")
 for t in toremove:
columns.remove(t)
 return columns, np.array(ratios ).astype(np.int32)
 columns, ratios = get_ratio_labels(train_targets_scored)
 columns_nonscored, ratios_nonscored = get_ratio_labels(train_targets_nonscored )
def run_lgb(params,
X,
y,
X_test,
resc,
cat_features,
n_splits=10,
early_stop=500):
oof_train = np.zeros(( X.shape[0]))
oof_test = np.zeros(( X_test.shape[0], n_splits))
fold = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=229)
 for i,(trn_index, val_index)in enumerate(fold.split(X,
y.astype(int))):
X_tr = X.iloc[trn_index, :]
X_val = X.iloc[val_index, :]
y_tr = y[trn_index]
y_val = y[val_index]
model = lgb.LGBMRegressor(**params)
 model.fit(X_tr,
y_tr,
eval_set=(X_val, y_val),
verbose=500,
early_stopping_rounds=early_stop,
categorical_feature=cat_features)
 valid_pred = model.predict(X_val, num_iteration=model.best_iteration_)
 test_pred = model.predict(X_test, num_iteration=model.best_iteration_)
 oof_train[val_index] = valid_pred
oof_test[:, i] = test_pred
return model, oof_train, oof_test<init_hyperparams>
remove_vehicle=True
def transform_data(train, test, col, normalize=True, removed_vehicle=False):

 mapping = {"cp_type":{"trt_cp": 0, "ctl_vehicle":1},
"cp_time":{48:0, 72:1, 24:2},
"cp_dose":{"D1":0, "D2":1}}
if removed_vehicle:
categories_tr = np.stack([ train[c].apply(lambda x: mapping[c][x] ).values for c in col[1:3]], axis=1)
 categories_test = np.stack([ test[c].apply(lambda x: mapping[c][x] ).values for c in col[1:3]], axis=1)
 else:
categories_tr = np.stack([ train[c].apply(lambda x: mapping[c][x] ).values for c in col[:3]], axis=1)
 categories_test = np.stack([ test[c].apply(lambda x: mapping[c][x] ).values for c in col[:3]], axis=1)
 max_ = 10.
 min_ = -10.
 if removed_vehicle:
numerical_tr = train[col[3:]].values
numerical_test = test[col[3:]].values
else:
numerical_tr = train[col[3:]].values
numerical_test = test[col[3:]].values
if normalize:
numerical_tr =(numerical_tr-min_)/(max_ - min_)
 numerical_test =(numerical_test-min_)/(max_ - min_)
 return categories_tr, categories_test, numerical_tr, numerical_test
col_features = list(train_features.columns)[1:]
cat_tr, cat_test, numerical_tr, numerical_test = transform_data(train_features, test_features, col_features, normalize=False, removed_vehicle=remove_vehicle)
 targets_tr = train_targets_scored[columns].values.astype(np.float32)
 targets2_tr = train_targets_nonscored[columns_nonscored].values.astype(np.float32 )
lgb_params = {
"boosting_type": "gbdt",
"num_leaves": 146,
"max_depth": 12,
"max_bin": 32,
"learning_rate": 0.01,
"n_estimators": 10000,
"subsample": 0.9212945843023237,
"subsample_freq": 2,
"colsample_bytree": 0.6334740217238963,
"reg_lambda": 1.543309192604612,
"min_child_samples": 45,
"min_child_weight": 0.5878240657385082,
"min_split_gain": 0.004619759404679957,
"n_jobs": -1
}
model, oof_train, oof_test = run_lgb(
 lgb_params,
xgb_X,
xgb_y,
xgb_X_test,
train_resc,
cat_features,
n_splits=6,
early_stop=500 )<define_variables>
def inference_fn(model, X ,verbose=True):
with torch.no_grad() :
y_preds = model.predict(X)
 y_preds = torch.sigmoid(torch.as_tensor(y_preds)).numpy()
return y_preds
lgb_xgb_nn = 0.9 * lgb_xgb + 0.1 *(nn_preds * 4)
 lgb_xgb_nn_test = 0.9 * lgb_xgb_test + 0.1 * nn_preds_test * 4<save_to_csv>
def log_loss_score(actual, predicted, eps=1e-15):

 p1 = actual * np.log(predicted+eps)
 p0 =(1-actual)* np.log(1-predicted+eps)
 loss = p0 + p1
return -loss.mean()
opt = OptimizedRounder()
opt.fit(lgb_xgb, target, [1.6, 2.1, 2.8, 3.5])
 coeff = opt.coefficients()
valid_pred = opt.predict(lgb_xgb, coeff)
 qwk = quadratic_weighted_kappa(xgb_y, valid_pred)
 print("QWK = ", qwk)
 coeffs = coeff.copy()
train_predictions = opt.predict(lgb_xgb, coeffs ).astype(np.int8)
 print(f"train_preds: {Counter(train_predictions)}")
 test_predictions = opt.predict(lgb_xgb_test, coeffs ).astype(np.int8)
 print(f"test_preds: {Counter(test_predictions)}")
 submission = pd.DataFrame({"PetID": test.PetID.values, "AdoptionSpeed": test_predictions})
 submission.to_csv("submission_tree.csv", index=False)
 submission.head()<save_to_csv>
def log_loss_multi(y_true, y_pred):
M = y_true.shape[1]
results = np.zeros(M)
 for i in range(M):
results[i] = log_loss_score(y_true[:,i], y_pred[:,i])
 return results.mean()
opt = OptimizedRounder()
opt.fit(lgb_xgb_nn, target, [1.5, 2.0, 2.5, 3.5])
 coeff = opt.coefficients()
valid_pred = opt.predict(lgb_xgb_nn, coeff)
 qwk = quadratic_weighted_kappa(xgb_y, valid_pred)
 print("QWK = ", qwk)
 coeffs = coeff.copy()
train_predictions = opt.predict(lgb_xgb_nn, coeffs ).astype(np.int8)
 print(f"train_preds: {Counter(train_predictions)}")
 test_predictions = opt.predict(lgb_xgb_nn_test, coeffs ).astype(np.int8)
 print(f"test_preds: {Counter(test_predictions)}")
 submission = pd.DataFrame({"PetID": test.PetID.values, "AdoptionSpeed": test_predictions})
 submission.to_csv("submission.csv", index=False)
 submission.head()<save_to_csv>
def auc_multi(y_true, y_pred):
M = y_true.shape[1]
results = np.zeros(M)
 for i in range(M):
try:
results[i] = roc_auc_score(y_true[:,i], y_pred[:,i])
 except:
pass
return results.mean()
opt = OptimizedRounder()
opt.fit(nn_preds * 4, target, [1.5, 2.0, 2.5, 3.5])
 coeff = opt.coefficients()
valid_pred = opt.predict(nn_preds * 4, coeff)
 qwk = quadratic_weighted_kappa(target, valid_pred)
 print("QWK = ", qwk)
 coeffs = coeff.copy()
train_predictions = opt.predict(nn_preds * 4, coeffs ).astype(np.int8)
 print(f"train_preds: {Counter(train_predictions)}")
 test_predictions = opt.predict(nn_preds_test * 4, coeffs ).astype(np.int8)
 print(f"test_preds: {Counter(test_predictions)}")
 submission = pd.DataFrame({"PetID": test.PetID.values, "AdoptionSpeed": test_predictions})
 submission.to_csv("submission_nn.csv", index=False)
 submission.head()<import_modules>
create_dataloaders,
create_explain_matrix)
 class TabModel(BaseEstimator):
def __init__(self, n_d=8, n_a=8, n_steps=3, gamma=1.3, cat_idxs=[], cat_dims=[], cat_emb_dim=1,
n_independent=2, n_shared=2, epsilon=1e-15, momentum=0.02,
lambda_sparse=1e-3, seed=0,
clip_value=1, verbose=1,
optimizer_fn=torch.optim.Adam,
optimizer_params=dict(lr=2e-2),
scheduler_params=None, scheduler_fn=None,
mask_type="sparsemax",
input_dim=None, output_dim=None,
device_name='auto'):

 self.n_d = n_d
self.n_a = n_a
self.n_steps = n_steps
self.gamma = gamma
self.cat_idxs = cat_idxs
self.cat_dims = cat_dims
self.cat_emb_dim = cat_emb_dim
self.n_independent = n_independent
self.n_shared = n_shared
self.epsilon = epsilon
self.momentum = momentum
self.lambda_sparse = lambda_sparse
self.clip_value = clip_value
self.verbose = verbose
self.optimizer_fn = optimizer_fn
self.optimizer_params = optimizer_params
self.device_name = device_name
self.scheduler_params = scheduler_params
self.scheduler_fn = scheduler_fn
self.mask_type = mask_type
self.input_dim = input_dim
self.output_dim = output_dim
self.batch_size = 1024
self.seed = seed
torch.manual_seed(self.seed)
 if device_name == 'auto':
if torch.cuda.is_available() :
device_name = 'cuda'
else:
device_name = 'cpu'
self.device = torch.device(device_name)
 print(f"Device used : {self.device}")
 @abstractmethod
def construct_loaders(self, X_train, y_train, X_valid, y_valid,
weights, batch_size, num_workers, drop_last):

 raise NotImplementedError('users must define construct_loaders to use this base class')
 def init_network(
 self,
input_dim,
output_dim,
n_d,
n_a,
n_steps,
gamma,
cat_idxs,
cat_dims,
cat_emb_dim,
n_independent,
n_shared,
epsilon,
virtual_batch_size,
momentum,
device_name,
mask_type, 
):
self.network = tab_network.TabNet(
 input_dim,
output_dim,
n_d=n_d,
n_a=n_a,
n_steps=n_steps,
gamma=gamma,
cat_idxs=cat_idxs,
cat_dims=cat_dims,
cat_emb_dim=cat_emb_dim,
n_independent=n_independent,
n_shared=n_shared,
epsilon=epsilon,
virtual_batch_size=virtual_batch_size,
momentum=momentum,
device_name=device_name,
mask_type=mask_type ).to(self.device)
 self.reducing_matrix = create_explain_matrix(
 self.network.input_dim,
self.network.cat_emb_dim,
self.network.cat_idxs,
self.network.post_embed_dim)
 def fit(self, X_train, y_train, X_valid=None, y_valid=None, loss_fn=None,
weights=0, max_epochs=100, patience=10, batch_size=1024,
virtual_batch_size=128, num_workers=0, drop_last=False):

 self.update_fit_params(X_train, y_train, X_valid, y_valid, loss_fn,
weights, max_epochs, patience, batch_size,
virtual_batch_size, num_workers, drop_last)
 train_dataloader, valid_dataloader = self.construct_loaders(X_train,
y_train,
X_valid,
y_valid,
self.updated_weights,
self.batch_size,
self.num_workers,
self.drop_last)
 self.init_network(
 input_dim=self.input_dim,
output_dim=self.output_dim,
n_d=self.n_d,
n_a=self.n_a,
n_steps=self.n_steps,
gamma=self.gamma,
cat_idxs=self.cat_idxs,
cat_dims=self.cat_dims,
cat_emb_dim=self.cat_emb_dim,
n_independent=self.n_independent,
n_shared=self.n_shared,
epsilon=self.epsilon,
virtual_batch_size=self.virtual_batch_size,
momentum=self.momentum,
device_name=self.device_name,
mask_type=self.mask_type 
)
 self.optimizer = self.optimizer_fn(self.network.parameters() ,
**self.optimizer_params)
 if self.scheduler_fn:
self.scheduler = self.scheduler_fn(self.optimizer, **self.scheduler_params)
 else:
self.scheduler = None
self.losses_train = []
self.losses_valid = []
self.learning_rates = []
self.metrics_train = []
self.metrics_valid = []
if self.verbose > 0:
print("Will train until validation stopping metric",
f"hasn't improved in {self.patience} rounds.")
 msg_epoch = f'| EPOCH | train | valid | total time(s)'
print('---------------------------------------')
 print(msg_epoch)
 total_time = 0
while(self.epoch < self.max_epochs and
self.patience_counter < self.patience):
starting_time = time.time()
self.learning_rates.append(self.optimizer.param_groups[-1]["lr"])
 fit_metrics = self.fit_epoch(train_dataloader, valid_dataloader)
 self.losses_train.append(fit_metrics['train']['loss_avg'])
 self.losses_valid.append(fit_metrics['valid']['total_loss'])
 self.metrics_train.append(fit_metrics['train']['stopping_loss'])
 self.metrics_valid.append(fit_metrics['valid']['stopping_loss'])
 stopping_loss = fit_metrics['valid']['stopping_loss']
if stopping_loss < self.best_cost:
self.best_cost = stopping_loss
self.patience_counter = 0
self.best_network = deepcopy(self.network)
 has_improved = True
else:
self.patience_counter += 1
has_improved=False
self.epoch += 1
total_time += time.time() - starting_time
if self.verbose > 0:
if self.epoch % self.verbose == 0:
separator = "|"
msg_epoch = f"| {self.epoch:<5} | "
msg_epoch += f" {fit_metrics['train']['stopping_loss']:.5f}"
msg_epoch += f' {separator:<2} '
msg_epoch += f" {fit_metrics['valid']['stopping_loss']:.5f}"
msg_epoch += f' {separator:<2} '
msg_epoch += f" {np.round(total_time, 1):<10}"
msg_epoch += f" {has_improved}"
print(msg_epoch)
 if self.verbose > 0:
if self.patience_counter == self.patience:
print(f"Early stopping occured at epoch {self.epoch}")
 print(f"Training done in {total_time:.3f} seconds.")
 print('---------------------------------------')
 self.history = {"train": {"loss": self.losses_train,
"metric": self.metrics_train,
"lr": self.learning_rates},
"valid": {"loss": self.losses_valid,
"metric": self.metrics_valid}}
self.load_best_model()
self._compute_feature_importances(train_dataloader)
 def save_model(self, path):

 saved_params = {}
for key, val in self.get_params().items() :
if isinstance(val, type):
continue
else:
saved_params[key] = val
Path(path ).mkdir(parents=True, exist_ok=True)
 with open(Path(path ).joinpath("model_params.json"), "w", encoding="utf8")as f:
json.dump(saved_params, f)
 torch.save(self.network.state_dict() , Path(path ).joinpath("network.pt"))
shutil.make_archive(path, 'zip', path)
 shutil.rmtree(path)
 print(f"Successfully saved model at {path}.zip")
 return f"{path}.zip"
def load_model(self, filepath):
try:
try:
with zipfile.ZipFile(filepath)as z:
with z.open("model_params.json")as f:
loaded_params = json.load(f)
 with z.open("network.pt")as f:
try:
saved_state_dict = torch.load(f)
 except io.UnsupportedOperation:
saved_state_dict = torch.load(io.BytesIO(f.read()))
except:
with open(os.path.join(filepath, "model_params.json")) as f:
loaded_params = json.load(f)
 saved_state_dict = torch.load(os.path.join(filepath, "network.pt"), map_location="cpu")
 except KeyError:
raise KeyError("Your zip file is missing at least one component")
 if torch.cuda.is_available() :
device_name = 'cuda'
else:
device_name = 'cpu'
loaded_params["device_name"] = device_name
self.__init__(**loaded_params)
 self.init_network(
 input_dim=self.input_dim,
output_dim=self.output_dim,
n_d=self.n_d,
n_a=self.n_a,
n_steps=self.n_steps,
gamma=self.gamma,
cat_idxs=self.cat_idxs,
cat_dims=self.cat_dims,
cat_emb_dim=self.cat_emb_dim,
n_independent=self.n_independent,
n_shared=self.n_shared,
epsilon=self.epsilon,
virtual_batch_size=1024,
momentum=self.momentum,
device_name=self.device_name,
mask_type=self.mask_type 
)
 self.network.load_state_dict(saved_state_dict)
 self.network.eval()
return
def fit_epoch(self, train_dataloader, valid_dataloader):

 train_metrics = self.train_epoch(train_dataloader)
 valid_metrics = self.predict_epoch(valid_dataloader)
 fit_metrics = {'train': train_metrics,
'valid': valid_metrics}
return fit_metrics
@abstractmethod
def train_epoch(self, train_loader):

 raise NotImplementedError('users must define train_epoch to use this base class')
 @abstractmethod
def train_batch(self, data, targets):

 raise NotImplementedError('users must define train_batch to use this base class')
 @abstractmethod
def predict_epoch(self, loader):

 raise NotImplementedError('users must define predict_epoch to use this base class')
 @abstractmethod
def predict_batch(self, data, targets):

 raise NotImplementedError('users must define predict_batch to use this base class')
 def load_best_model(self):
if self.best_network is not None:
self.network = self.best_network
@abstractmethod
def predict(self, X):

 raise NotImplementedError('users must define predict to use this base class')
 def explain(self, X):

 self.network.eval()
dataloader = DataLoader(PredictDataset(X),
batch_size=self.batch_size, shuffle=False)
 for batch_nb, data in enumerate(dataloader):
data = data.to(self.device ).float()
M_explain, masks = self.network.forward_masks(data)
 for key, value in masks.items() :
masks[key] = csc_matrix.dot(value.cpu().detach().numpy() ,
self.reducing_matrix)
 if batch_nb == 0:
res_explain = csc_matrix.dot(M_explain.cpu().detach().numpy() ,
self.reducing_matrix)
 res_masks = masks
else:
res_explain = np.vstack([res_explain,
csc_matrix.dot(M_explain.cpu().detach().numpy() ,
self.reducing_matrix)])
 for key, value in masks.items() :
res_masks[key] = np.vstack([res_masks[key], value])
 return res_explain, res_masks
def _compute_feature_importances(self, loader):
self.network.eval()
feature_importances_ = np.zeros(( self.network.post_embed_dim))
for data, targets in loader:
data = data.to(self.device ).float()
M_explain, masks = self.network.forward_masks(data)
 feature_importances_ += M_explain.sum(dim=0 ).cpu().detach().numpy()
feature_importances_ = csc_matrix.dot(feature_importances_,
self.reducing_matrix)
 self.feature_importances_ = feature_importances_ / np.sum(feature_importances_)
 class TabNetRegressor(TabModel):
def construct_loaders(self, X_train, y_train, X_valid, y_valid, weights,
batch_size, num_workers, drop_last):

 if isinstance(weights, int):
if weights == 1:
raise ValueError("Please provide a list of weights for regression.")
 if isinstance(weights, dict):
raise ValueError("Please provide a list of weights for regression.")
 train_dataloader, valid_dataloader = create_dataloaders(X_train,
y_train,
X_valid,
y_valid,
weights,
batch_size,
num_workers,
drop_last)
 return train_dataloader, valid_dataloader
def update_fit_params(self, X_train, y_train, X_valid, y_valid, loss_fn,
weights, max_epochs, patience,
batch_size, virtual_batch_size, num_workers, drop_last):
if loss_fn is None:
self.loss_fn = torch.nn.functional.mse_loss
else:
self.loss_fn = loss_fn
assert X_train.shape[1] == X_valid.shape[1], "Dimension mismatch X_train X_valid"
self.input_dim = X_train.shape[1]
if len(y_train.shape)== 1:
raise ValueError()
assert y_train.shape[1] == y_valid.shape[1], "Dimension mismatch y_train y_valid"
self.output_dim = y_train.shape[1]
self.updated_weights = weights
self.max_epochs = max_epochs
self.patience = patience
self.batch_size = batch_size
self.virtual_batch_size = virtual_batch_size
self.patience_counter = 0
self.epoch = 0
self.best_cost = np.inf
self.num_workers = num_workers
self.drop_last = drop_last
def train_epoch(self, train_loader):

 self.network.train()
y_preds = []
ys = []
total_loss = 0
for data, targets in train_loader:
batch_outs = self.train_batch(data, targets)
 y_preds.append(batch_outs["y_preds"].cpu().detach().numpy())
 ys.append(batch_outs["y"].cpu().detach().numpy())
 total_loss += batch_outs["loss"]
y_preds = np.vstack(y_preds)
 ys = np.vstack(ys)
 stopping_loss =log_loss_multi(ys, torch.sigmoid(torch.as_tensor(y_preds)).numpy())
 total_loss = total_loss / len(train_loader)
 epoch_metrics = {'loss_avg': total_loss,
'stopping_loss': total_loss,
}
if self.scheduler is not None:
self.scheduler.step()
return epoch_metrics
def train_batch(self, data, targets):

 self.network.train()
data = data.to(self.device ).float()
targets = targets.to(self.device ).float()
self.optimizer.zero_grad()
output, M_loss = self.network(data)
 loss = self.loss_fn(output, targets)
 loss -= self.lambda_sparse*M_loss
loss.backward()
if self.clip_value:
clip_grad_norm_(self.network.parameters() , self.clip_value)
 self.optimizer.step()
loss_value = loss.item()
batch_outs = {'loss': loss_value,
'y_preds': output,
'y': targets}
return batch_outs
def predict_epoch(self, loader):

 y_preds = []
ys = []
self.network.eval()
total_loss = 0
for data, targets in loader:
batch_outs = self.predict_batch(data, targets)
 total_loss += batch_outs["loss"]
y_preds.append(batch_outs["y_preds"].cpu().detach().numpy())
 ys.append(batch_outs["y"].cpu().detach().numpy())
 y_preds = np.vstack(y_preds)
 ys = np.vstack(ys)
 stopping_loss =log_loss_multi(ys, torch.sigmoid(torch.as_tensor(y_preds)).numpy())
 total_loss = total_loss / len(loader)
 epoch_metrics = {'total_loss': total_loss,
'stopping_loss': stopping_loss}
return epoch_metrics
def predict_batch(self, data, targets):

 self.network.eval()
data = data.to(self.device ).float()
targets = targets.to(self.device ).float()
output, M_loss = self.network(data)
 loss = self.loss_fn(output, targets)
 loss -= self.lambda_sparse*M_loss
loss_value = loss.item()
batch_outs = {'loss': loss_value,
'y_preds': output,
'y': targets}
return batch_outs
def predict(self, X):

 self.network.eval()
dataloader = DataLoader(PredictDataset(X),
batch_size=self.batch_size, shuffle=False)
 results = []
for batch_nb, data in enumerate(dataloader):
data = data.to(self.device ).float()
output, M_loss = self.network(data)
 predictions = output.cpu().detach().numpy()
results.append(predictions)
 res = np.vstack(results)
 return res
import lightgbm as lgb
import optuna.integration.lightgbm as oplgb
from sklearn.model_selection import KFold
from sklearn.metrics import mean_squared_error
from tqdm.notebook import tqdm
import matplotlib.pyplot as plt
import seaborn as sns<load_from_csv>
from iterstrat.ml_stratifiers import MultilabelStratifiedKFold, MultilabelStratifiedShuffleSplit
from sklearn.metrics import roc_auc_score
df_train = pd.read_csv("/kaggle/input/tabular-playground-series-jan-2021/train.csv")
 df_test = pd.read_csv("/kaggle/input/tabular-playground-series-jan-2021/test.csv")
 df_sample = pd.read_csv("/kaggle/input/tabular-playground-series-jan-2021/sample_submission.csv" )<drop_column>
class Config(object):
def __init__(self):
self.num_class = targets_tr.shape[1]
self.verbose=False
self.device = "cuda" if torch.cuda.is_available() else "cpu"
self.SPLITS = 10
self.EPOCHS = 200
self.num_ensembling = 1
self.seed = 0
self.cat_emb_dim=[1] * cat_tr.shape[1]
self.cats_idx = list(range(cat_tr.shape[1]))
self.cat_dims = [len(np.unique(cat_tr[:, i])) for i in range(cat_tr.shape[1])]
self.num_numericals= numerical_tr.shape[1]
self.save_name = ".. /input/tabnet-smoothing/tabnet_raw_step1"
self.strategy = "KFOLD"
cfg = Config()
train_id = df_train["id"]
test_id = df_test["id"]
df_train.drop("id", axis=1, inplace=True)
 df_test.drop("id", axis=1, inplace=True )<define_variables>
X_test = np.concatenate([cat_test, numerical_test ], axis=1 )
feature_cols = [c for c in df_train.columns if c != "target"]<prepare_x_and_y>
SEED = [0,1,2,3,4,5,6]
if cfg.strategy == "KFOLD":
oof_preds_all = []
oof_targets_all = []
scores_all = []
scores_auc_all= []
preds_test = []
for seed in SEED:
print("
mskf = MultilabelStratifiedKFold(n_splits=5, random_state=cfg.seed+seed, shuffle=True)
 oof_preds = []
oof_targets = []
scores = []
scores_auc = []
p = []
for j,(train_idx, val_idx)in enumerate(mskf.split(np.zeros(len(cat_tr)) , targets_tr)) :
print("FOLDS : ", j)
 X_train, y_train = torch.as_tensor(np.concatenate([cat_tr[train_idx], numerical_tr[train_idx] ], axis=1)) , torch.as_tensor(targets_tr[train_idx])
 X_val, y_val = torch.as_tensor(np.concatenate([cat_tr[val_idx], numerical_tr[val_idx] ], axis=1)) , torch.as_tensor(targets_tr[val_idx])
 model = TabNetRegressor(n_d=24, n_a=24, n_steps=1, gamma=1.3, lambda_sparse=0, cat_dims=cfg.cat_dims, cat_emb_dim=cfg.cat_emb_dim, cat_idxs=cfg.cats_idx, optimizer_fn=torch.optim.Adam,
optimizer_params=dict(lr=2e-2), mask_type='entmax', device_name=cfg.device, scheduler_params=dict(milestones=[ 50,100,150], gamma=0.9), scheduler_fn=torch.optim.lr_scheduler.MultiStepLR)
 name = cfg.save_name + f"_fold{j}_{seed}"
model.load_model(name)
 preds = model.predict(X_val)
 preds = torch.sigmoid(torch.as_tensor(preds)).detach().cpu().numpy()
score = log_loss_multi(y_val, preds)
 temp = model.predict(X_test)
 p.append(torch.sigmoid(torch.as_tensor(temp)).detach().cpu().numpy())
 oof_preds.append(preds)
 oof_targets.append(y_val)
 scores.append(score)
 scores_auc.append(auc_multi(y_val,preds))
print(f"validation fold {j} : {score}")
 p = np.stack(p)
 preds_test.append(p)
 oof_preds_all.append(np.concatenate(oof_preds))
oof_targets_all.append(np.concatenate(oof_targets))
scores_all.append(np.array(scores))
scores_auc_all.append(np.array(scores_auc))
preds_test = np.stack(preds_test )
train_x = df_train[feature_cols]
train_y = df_train.target
test_x = df_test<choose_model_class>
if cfg.strategy == "KFOLD":
for i in range(cfg.num_ensembling):
print("CV score fold : ", log_loss_multi(oof_targets_all[i], oof_preds_all[i]))
print("auc mean : ", sum(scores_auc_all[i])/len(scores_auc_all[i]))
folds = KFold(n_splits=10, shuffle=True, random_state=2021 )<train_model>
submission[columns] = preds_test.mean(1 ).mean(0)
 submission.loc[test_features['cp_type']=='ctl_vehicle', submission.columns[1:]] = 0
class FoldsAverageLGBM:
def __init__(self, folds):
self.folds = folds
self.models = []
def fit(self, lgb_params, train_x, train_y):
oof_preds = np.zeros_like(train_y)
 self.train_x = train_x.values
self.train_y = train_y.values
for tr_idx, va_idx in tqdm(folds.split(train_x)) :
tr_x, va_x = self.train_x[tr_idx], self.train_x[va_idx]
tr_y, va_y = self.train_y[tr_idx], self.train_y[va_idx]
lgb_train_dataset = lgb.Dataset(tr_x, tr_y)
 lgb_valid_dataset = lgb.Dataset(va_x, va_y)
 model = lgb.train(lgb_params, lgb_train_dataset, valid_sets=[lgb_valid_dataset], verbose_eval=100)
 self.models.append(model)
 oof_pred = model.predict(va_x)
 oof_preds[va_idx] = oof_pred
self.oof_preds = oof_preds
def predict(self, test_x):
preds = []
for model in tqdm(self.models):
pred = model.predict(test_x)
 preds.append(pred)
 preds = np.mean(preds, axis=0)
 return preds<init_hyperparams>
sub2 = submission
best_lgb_params = {
'seed': 2021,
'objective': 'regression',
'metric': 'rmse',
'verbosity': -1,
'feature_pre_filter': False,
'lambda_l1': 6.540486456085813,
'lambda_l2': 0.01548480538099245,
'num_leaves': 256,
'feature_fraction': 0.52,
'bagging_fraction': 0.6161835249194311,
'bagging_freq': 7,
'min_child_samples': 20
}
best_lgb_params["learning_rate"] = 0.001
best_lgb_params["early_stopping_round"] = 1000
best_lgb_params["num_iterations"] = 20000<statistical_test>
FOLDS = 10
EPOCHS = 80
BATCH_SIZE = 124
LR = 0.001
VERBOSE = 0
SEEDS1 = [1, 2, 3, 4, 5, 6, 7]
SEEDS2 = [8, 9, 10, 11, 12, 13, 14]
SEEDS3 = [15, 16, 17, 18, 19, 20, 21]
SEEDS4 = [22, 23, 24, 25, 26, 27, 28]
SEEDS5 = [29, 30, 31, 32, 33, 34, 35]
def seed_everything(seed):
random.seed(seed)
 np.random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 tf.random.set_seed(seed)
folds_average_lgbm = FoldsAverageLGBM(folds )<train_model>
def mapping_and_filter(train, train_targets, test):
cp_type = {'trt_cp': 0, 'ctl_vehicle': 1}
cp_dose = {'D1': 0, 'D2': 1}
for df in [train, test]:
df['cp_type'] = df['cp_type'].map(cp_type)
 df['cp_dose'] = df['cp_dose'].map(cp_dose)
 train_targets = train_targets[train['cp_type'] == 0].reset_index(drop = True)
 train = train[train['cp_type'] == 0].reset_index(drop = True)
 train_targets.drop(['sig_id'], inplace = True, axis = 1)
 return train, train_targets, test
def scaling(train, test):
features = train.columns[2:]
scaler = RobustScaler()
scaler.fit(pd.concat([train[features], test[features]], axis = 0))
train[features] = scaler.transform(train[features])
 test[features] = scaler.transform(test[features])
 return train, test, features
def fe_pca(train, test, n_components_g = 70, n_components_c = 10, SEED = 123):
features_g = list(train.columns[4:776])
 features_c = list(train.columns[776:876])
 def create_pca(train, test, features, kind = 'g', n_components = n_components_g):
train_ = train[features].copy()
test_ = test[features].copy()
data = pd.concat([train_, test_], axis = 0)
 pca = PCA(n_components = n_components, random_state = SEED)
 data = pca.fit_transform(data)
 columns = [f'pca_{kind}{i + 1}' for i in range(n_components)]
data = pd.DataFrame(data, columns = columns)
 train_ = data.iloc[:train.shape[0]]
test_ = data.iloc[train.shape[0]:].reset_index(drop = True)
 train = pd.concat([train, train_], axis = 1)
 test = pd.concat([test, test_], axis = 1)
 return train, test
train, test = create_pca(train, test, features_g, kind = 'g', n_components = n_components_g)
 train, test = create_pca(train, test, features_c, kind = 'c', n_components = n_components_c)
 return train, test
def fe_stats(train, test):
features_g = list(train.columns[4:776])
 features_c = list(train.columns[776:876])
 for df in [train, test]:
df['g_sum'] = df[features_g].sum(axis = 1)
 df['g_mean'] = df[features_g].mean(axis = 1)
 df['g_std'] = df[features_g].std(axis = 1)
 df['g_kurt'] = df[features_g].kurtosis(axis = 1)
 df['g_skew'] = df[features_g].skew(axis = 1)
 df['c_sum'] = df[features_c].sum(axis = 1)
 df['c_mean'] = df[features_c].mean(axis = 1)
 df['c_std'] = df[features_c].std(axis = 1)
 df['c_kurt'] = df[features_c].kurtosis(axis = 1)
 df['c_skew'] = df[features_c].skew(axis = 1)
 df['gc_sum'] = df[features_g + features_c].sum(axis = 1)
 df['gc_mean'] = df[features_g + features_c].mean(axis = 1)
 df['gc_std'] = df[features_g + features_c].std(axis = 1)
 df['gc_kurt'] = df[features_g + features_c].kurtosis(axis = 1)
 df['gc_skew'] = df[features_g + features_c].skew(axis = 1)
 return train, test
def c_squared(train, test):
features_c = list(train.columns[776:876])
 for df in [train, test]:
for feature in features_c:
df[f'{feature}_squared'] = df[feature] ** 2
return train, test
def mean_log_loss(y_true, y_pred):
y_pred = np.clip(y_pred, 1e-15, 1 - 1e-15)
 metrics = []
for target in range(206):
metrics.append(log_loss(y_true[:, target], y_pred[:, target]))
return np.mean(metrics)
 def create_model_rs(shape1, shape2):
input_1 = tf.keras.layers.Input(shape =(shape1))
input_2 = tf.keras.layers.Input(shape =(shape2))
head_1 = tf.keras.layers.BatchNormalization()(input_1)
 head_1 = tf.keras.layers.Dropout(0.2 )(head_1)
 head_1 = tf.keras.layers.Dense(512, activation = "elu" )(head_1)
 head_1 = tf.keras.layers.BatchNormalization()(head_1)
 input_3 = tf.keras.layers.Dense(256, activation = "elu" )(head_1)
 input_3_concat = tf.keras.layers.Concatenate()([input_2, input_3])
 head_2 = tf.keras.layers.BatchNormalization()(input_3_concat)
 head_2 = tf.keras.layers.Dropout(0.3 )(head_2)
 head_2 = tf.keras.layers.Dense(512, "relu" )(head_2)
 head_2 = tf.keras.layers.BatchNormalization()(head_2)
 head_2 = tf.keras.layers.Dense(512, "elu" )(head_2)
 head_2 = tf.keras.layers.BatchNormalization()(head_2)
 head_2 = tf.keras.layers.Dense(256, "relu" )(head_2)
 head_2 = tf.keras.layers.BatchNormalization()(head_2)
 input_4 = tf.keras.layers.Dense(256, "elu" )(head_2)
 input_4_avg = tf.keras.layers.Average()([input_3, input_4])
 head_3 = tf.keras.layers.BatchNormalization()(input_4_avg)
 head_3 = tf.keras.layers.Dense(256, kernel_initializer = 'lecun_normal', activation = 'selu' )(head_3)
 head_3 = tf.keras.layers.BatchNormalization()(head_3)
 head_3 = tf.keras.layers.Dense(206, kernel_initializer = 'lecun_normal', activation = 'selu' )(head_3)
 head_3 = tf.keras.layers.BatchNormalization()(head_3)
 output = tf.keras.layers.Dense(206, activation = "sigmoid" )(head_3)
 model = tf.keras.models.Model(inputs = [input_1, input_2], outputs = output)
 opt = tf.optimizers.Adam(learning_rate = LR)
 model.compile(optimizer = opt,
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = 0.0015),
metrics = tf.keras.metrics.BinaryCrossentropy())
 return model
def create_model_5l(shape):
inp = tf.keras.layers.Input(shape =(shape))
x = tf.keras.layers.BatchNormalization()(inp)
 x = tf.keras.layers.Dropout(0.4 )(x)
 x = tf.keras.layers.Dense(2560, activation = 'relu' )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.4 )(x)
 x = tf.keras.layers.Dense(2048, activation = 'relu' )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.4 )(x)
 x = tf.keras.layers.Dense(1524, activation = 'relu' )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.4 )(x)
 x = tf.keras.layers.Dense(1012, activation = 'relu' )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.4 )(x)
 x = tf.keras.layers.Dense(780, activation = 'relu' )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.2 )(x)
 out = tf.keras.layers.Dense(206, activation = 'sigmoid' )(x)
 model = tf.keras.models.Model(inputs = inp, outputs = out)
 opt = tf.optimizers.Adam(learning_rate = LR)
 opt = tfa.optimizers.SWA(opt)
 model.compile(optimizer = opt,
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = 0.0020),
metrics = tf.keras.metrics.BinaryCrossentropy())
 return model
def create_model_4l(shape):
inp = tf.keras.layers.Input(shape =(shape))
x = tf.keras.layers.BatchNormalization()(inp)
 x = tf.keras.layers.Dropout(0.4 )(x)
 x = tf.keras.layers.Dense(2048, activation = 'relu' )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.4 )(x)
 x = tf.keras.layers.Dense(1524, activation = 'relu' )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.4 )(x)
 x = tf.keras.layers.Dense(1012, activation = 'relu' )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.4 )(x)
 x = tf.keras.layers.Dense(1012, activation = 'relu' )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.2 )(x)
 out = tf.keras.layers.Dense(206, activation = 'sigmoid' )(x)
 model = tf.keras.models.Model(inputs = inp, outputs = out)
 opt = tf.optimizers.Adam(learning_rate = LR)
 model.compile(optimizer = opt,
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = 0.0020),
metrics = tf.keras.metrics.BinaryCrossentropy())
 return model
def create_model_3l(shape):
inp = tf.keras.layers.Input(shape =(shape))
x = tf.keras.layers.BatchNormalization()(inp)
 x = tf.keras.layers.Dropout(0.4914099166744246 )(x)
 x = tfa.layers.WeightNormalization(tf.keras.layers.Dense(1159, activation = 'relu'))(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.18817607797795838 )(x)
 x = tfa.layers.WeightNormalization(tf.keras.layers.Dense(960, activation = 'relu'))(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.12542057776853896 )(x)
 x = tfa.layers.WeightNormalization(tf.keras.layers.Dense(1811, activation = 'relu'))(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.20175242230280122 )(x)
 out = tfa.layers.WeightNormalization(tf.keras.layers.Dense(206, activation = 'sigmoid'))(x)
 model = tf.keras.models.Model(inputs = inp, outputs = out)
 opt = tf.optimizers.Adam(learning_rate = LR)
 opt = tfa.optimizers.Lookahead(opt, sync_period = 10)
 model.compile(optimizer = opt,
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = 0.0015),
metrics = tf.keras.metrics.BinaryCrossentropy())
 return model
def create_model_2l(shape):
inp = tf.keras.layers.Input(shape =(shape))
x = tf.keras.layers.BatchNormalization()(inp)
 x = tf.keras.layers.Dropout(0.2688628097505064 )(x)
 x = tfa.layers.WeightNormalization(tf.keras.layers.Dense(1292, activation = 'relu'))(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.4598218403250696 )(x)
 x = tfa.layers.WeightNormalization(tf.keras.layers.Dense(983, activation = 'relu'))(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.4703144018483698 )(x)
 out = tfa.layers.WeightNormalization(tf.keras.layers.Dense(206, activation = 'sigmoid'))(x)
 model = tf.keras.models.Model(inputs = inp, outputs = out)
 opt = tf.optimizers.Adam(learning_rate = LR)
 opt = tfa.optimizers.Lookahead(opt, sync_period = 10)
 model.compile(optimizer = opt,
loss = tf.keras.losses.BinaryCrossentropy(label_smoothing = 0.0015),
metrics = tf.keras.metrics.BinaryCrossentropy())
 return model
def train_and_evaluate(train, test, train_targets, features, start_predictors, SEED = 123, MODEL = '3l'):
seed_everything(SEED)
 oof_pred = np.zeros(( train.shape[0], 206))
test_pred = np.zeros(( test.shape[0], 206))
for fold,(trn_ind, val_ind)in enumerate(MultilabelStratifiedKFold(n_splits = FOLDS,
random_state = SEED,
shuffle = True)\ 
.split(train_targets, train_targets)) :
K.clear_session()
if MODEL == '5l':
model = create_model_5l(len(features))
elif MODEL == '4l':
model = create_model_4l(len(features))
elif MODEL == '3l':
model = create_model_3l(len(features))
elif MODEL == '2l':
model = create_model_2l(len(features))
elif MODEL == "rs":
model = create_model_rs(len(features), len(start_predictors))
early_stopping = tf.keras.callbacks.EarlyStopping(monitor = 'val_binary_crossentropy',
mode = 'min',
patience = 10,
restore_best_weights = False,
verbose = VERBOSE)
 reduce_lr = tf.keras.callbacks.ReduceLROnPlateau(monitor = 'val_binary_crossentropy',
mode = 'min',
factor = 0.3,
patience = 3,
verbose = VERBOSE)
 checkpoint = tf.keras.callbacks.ModelCheckpoint(f'{MODEL}_{fold}_{SEED}.h5',
monitor = 'val_binary_crossentropy',
verbose = VERBOSE,
save_best_only = True,
save_weights_only = True)
 x_train, x_val = train[features].values[trn_ind], train[features].values[val_ind]
y_train, y_val = train_targets.values[trn_ind], train_targets.values[val_ind]
if MODEL == "rs":
x_train_, x_val_ = train[start_predictors].values[trn_ind], train[start_predictors].values[val_ind]
model.fit([x_train, x_train_], y_train,
validation_data =([x_val, x_val_], y_val),
epochs = EPOCHS,
batch_size = BATCH_SIZE,
callbacks = [early_stopping, reduce_lr, checkpoint],
verbose = VERBOSE)
 model.load_weights(f'{MODEL}_{fold}_{SEED}.h5')
 oof_pred[val_ind] = model.predict([x_val, x_val_])
 test_pred += model.predict([test[features].values, test[start_predictors].values])/ FOLDS
else:
model.fit(x_train, y_train,
validation_data =(x_val, y_val),
epochs = EPOCHS,
batch_size = BATCH_SIZE,
callbacks = [early_stopping, reduce_lr, checkpoint],
verbose = VERBOSE)
 model.load_weights(f'{MODEL}_{fold}_{SEED}.h5')
 oof_pred[val_ind] = model.predict(x_val)
 test_pred += model.predict(test[features].values)/ FOLDS
oof_score = mean_log_loss(train_targets.values, oof_pred)
 print(f'Our out of folds mean log loss score is {oof_score}')
 return test_pred, oof_pred
def inference(train, test, train_targets, features, start_predictors, SEED = 123, MODEL = '3l', PATH = '.. /input/moa-3layer'):
seed_everything(SEED)
 oof_pred = np.zeros(( train.shape[0], 206))
test_pred = np.zeros(( test.shape[0], 206))
for fold,(trn_ind, val_ind)in enumerate(MultilabelStratifiedKFold(n_splits = FOLDS,
random_state = SEED,
shuffle = True)\ 
.split(train_targets, train_targets)) :
K.clear_session()
if MODEL == '5l':
model = create_model_5l(len(features))
elif MODEL == '4l':
model = create_model_4l(len(features))
elif MODEL == '3l':
model = create_model_3l(len(features))
elif MODEL == '2l':
model = create_model_2l(len(features))
elif MODEL == "rs":
model = create_model_rs(len(features), len(start_predictors))
x_train, x_val = train[features].values[trn_ind], train[features].values[val_ind]
y_train, y_val = train_targets.values[trn_ind], train_targets.values[val_ind]
model.load_weights('.. /input/moa-5seed/Model_Weights'+'/'+MODEL+'_'+str(fold)+'_'+str(SEED)+'.h5')
 if MODEL == "rs":
x_train_, x_val_ = train[start_predictors].values[trn_ind], train[start_predictors].values[val_ind]
oof_pred[val_ind] = model.predict([x_val, x_val_])
 test_pred += model.predict([test[features].values, test[start_predictors].values])/ FOLDS
else:
oof_pred[val_ind] = model.predict(x_val)
 test_pred += model.predict(test[features].values)/ FOLDS
oof_score = mean_log_loss(train_targets.values, oof_pred)
 print(f'Our out of folds mean log loss score is {oof_score}')
 return test_pred, oof_pred
def run_multiple_seeds(train, test, train_targets, features, start_predictors, SEEDS = [123], MODEL = '3l', PATH = '.. /input/moa-3layer'):
test_pred = []
oof_pred = []
for SEED in SEEDS:
print(f'Using model {MODEL} with seed {SEED} for inference')
 print(f'Trained with {len(features)} features')
 test_pred_, oof_pred_ = inference(train, test, train_targets, features, start_predictors, SEED = SEED, MODEL = MODEL, PATH = PATH)
 test_pred.append(test_pred_)
 oof_pred.append(oof_pred_)
 print('-'*50)
 print('
')
 test_pred = np.average(test_pred, axis = 0)
 oof_pred = np.average(oof_pred, axis = 0)
 seed_log_loss = mean_log_loss(train_targets.values, oof_pred)
 print(f'Our out of folds log loss for our seed blend model is {seed_log_loss}')
 return test_pred, oof_pred
def submission(test_pred):
sample_submission.loc[:, train_targets.columns] = test_pred
sample_submission.loc[test['cp_type'] == 1, train_targets.columns] = 0
sample_submission.to_csv('submission.csv', index = False)
 return sample_submission
start_predictors = ["g-0", "g-7", "g-8", "g-10", "g-13", "g-17", "g-20", "g-22", "g-24", "g-26", "g-28", "g-29", "g-30", "g-31", "g-32", "g-34", "g-35", "g-36", "g-37", "g-38",
"g-39","g-41", "g-46", "g-48", "g-50", "g-51", "g-52", "g-55", "g-58", "g-59", "g-61", "g-62", "g-63", "g-65", "g-66", "g-67", "g-68", "g-70", "g-72", "g-74",
"g-75", "g-79", "g-83", "g-84", "g-85", "g-86", "g-90", "g-91", "g-94", "g-95", "g-96", "g-97", "g-98", "g-100", "g-102", "g-105", "g-106", "g-112", "g-113",
"g-114", "g-116", "g-121", "g-123", "g-126", "g-128", "g-131", "g-132", "g-134", "g-135", "g-138", "g-139", "g-140", "g-142", "g-144", "g-145", "g-146",
"g-147", "g-148", "g-152", "g-155", "g-157", "g-158", "g-160", "g-163", "g-164", "g-165", "g-170", "g-173", "g-174", "g-175", "g-177", "g-178", "g-181",
"g-183", "g-185", "g-186", "g-189", "g-192", "g-194", "g-195", "g-196", "g-197", "g-199", "g-201", "g-202", "g-206", "g-208", "g-210", "g-213", "g-214",
"g-215", "g-220", "g-226", "g-228", "g-229", "g-235", "g-238", "g-241", "g-242", "g-243", "g-244", "g-245", "g-248", "g-250", "g-251", "g-254", "g-257",
"g-259", "g-261", "g-266", "g-270", "g-271", "g-272", "g-275", "g-278", "g-282", "g-287", "g-288", "g-289", "g-291", "g-293", "g-294", "g-297", "g-298",
"g-301", "g-303", "g-304", "g-306", "g-308", "g-309", "g-310", "g-311", "g-314", "g-315", "g-316", "g-317", "g-320", "g-321", "g-322", "g-327", "g-328",
"g-329", "g-332", "g-334", "g-335", "g-336", "g-337", "g-339", "g-342", "g-344", "g-349", "g-350", "g-351", "g-353", "g-354", "g-355", "g-357", "g-359",
"g-360", "g-364", "g-365", "g-366", "g-367", "g-368", "g-369", "g-374", "g-375", "g-377", "g-379", "g-385", "g-386", "g-390", "g-392", "g-393", "g-400",
"g-402", "g-406", "g-407", "g-409", "g-410", "g-411", "g-414", "g-417", "g-418", "g-421", "g-423", "g-424", "g-427", "g-429", "g-431", "g-432", "g-433",
"g-434", "g-437", "g-439", "g-440", "g-443", "g-449", "g-458", "g-459", "g-460", "g-461", "g-464", "g-467", "g-468", "g-470", "g-473", "g-477", "g-478",
"g-479", "g-484", "g-485", "g-486", "g-488", "g-489", "g-491", "g-494", "g-496", "g-498", "g-500", "g-503", "g-504", "g-506", "g-508", "g-509", "g-512",
"g-522", "g-529", "g-531", "g-534", "g-539", "g-541", "g-546", "g-551", "g-553", "g-554", "g-559", "g-561", "g-562", "g-565", "g-568", "g-569", "g-574",
"g-577", "g-578", "g-586", "g-588", "g-590", "g-594", "g-595", "g-596", "g-597", "g-599", "g-600", "g-603", "g-607", "g-615", "g-618", "g-619", "g-620",
"g-625", "g-628", "g-629", "g-632", "g-634", "g-635", "g-636", "g-638", "g-639", "g-641", "g-643", "g-644", "g-645", "g-646", "g-647", "g-648", "g-663",
"g-664", "g-665", "g-668", "g-669", "g-670", "g-671", "g-672", "g-673", "g-674", "g-677", "g-678", "g-680", "g-683", "g-689", "g-691", "g-693", "g-695",
"g-701", "g-702", "g-703", "g-704", "g-705", "g-706", "g-708", "g-711", "g-712", "g-720", "g-721", "g-723", "g-724", "g-726", "g-728", "g-731", "g-733",
"g-738", "g-739", "g-742", "g-743", "g-744", "g-745", "g-749", "g-750", "g-752", "g-760", "g-761", "g-764", "g-766", "g-768", "g-770", "g-771", "c-0",
"c-1", "c-2", "c-3", "c-4", "c-5", "c-6", "c-7", "c-8", "c-9", "c-10", "c-11", "c-12", "c-13", "c-14", "c-15", "c-16", "c-17", "c-18", "c-19", "c-20",
"c-21", "c-22", "c-23", "c-24", "c-25", "c-26", "c-27", "c-28", "c-29", "c-30", "c-31", "c-32", "c-33", "c-34", "c-35", "c-36", "c-37", "c-38", "c-39",
"c-40", "c-41", "c-42", "c-43", "c-44", "c-45", "c-46", "c-47", "c-48", "c-49", "c-50", "c-51", "c-52", "c-53", "c-54", "c-55", "c-56", "c-57", "c-58",
"c-59", "c-60", "c-61", "c-62", "c-63", "c-64", "c-65", "c-66", "c-67", "c-68", "c-69", "c-70", "c-71", "c-72", "c-73", "c-74", "c-75", "c-76", "c-77",
"c-78", "c-79", "c-80", "c-81", "c-82", "c-83", "c-84", "c-85", "c-86", "c-87", "c-88", "c-89", "c-90", "c-91", "c-92", "c-93", "c-94", "c-95", "c-96",
"c-97", "c-98", "c-99"]
train = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 test = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 train, train_targets, test = mapping_and_filter(train, train_targets, test)
 train, test = fe_stats(train, test)
 train, test = c_squared(train, test)
 train, test = fe_pca(train, test, n_components_g = 70, n_components_c = 10, SEED = 123)
 train, test, features = scaling(train, test)
 test_pred_5l, oof_pred_5l = run_multiple_seeds(train, test, train_targets, features, start_predictors, SEEDS = SEEDS1, MODEL = '5l', PATH = '.. /input/moaaaaa/5l')
 test_pred_4l, oof_pred_4l = run_multiple_seeds(train, test, train_targets, features, start_predictors, SEEDS = SEEDS2, MODEL = '4l', PATH = '.. /input/moaaaaa/4l')
 test_pred_3l, oof_pred_3l = run_multiple_seeds(train, test, train_targets, features, start_predictors, SEEDS = SEEDS3, MODEL = '3l', PATH = '.. /input/moaaaaa/3l')
 test_pred_2l, oof_pred_2l = run_multiple_seeds(train, test, train_targets, features, start_predictors, SEEDS = SEEDS4, MODEL = '2l', PATH = '.. /input/moaaaaa/2l')
 test_pred_rs, oof_pred_rs = run_multiple_seeds(train, test, train_targets, features, start_predictors, SEEDS = SEEDS5, MODEL = 'rs', PATH = '.. /input/moaaaaa/rs')
 oof_pred = np.average([oof_pred_5l, oof_pred_4l, oof_pred_3l, oof_pred_2l], axis = 0)
 seed_log_loss = mean_log_loss(train_targets.values, oof_pred)
 print(f'Our final out of folds log loss for our classic dnn blend is {seed_log_loss}')
 test_pred = np.average([test_pred_5l, test_pred_4l, test_pred_3l, test_pred_2l], axis = 0)
 oof_pred = np.average([oof_pred, oof_pred_rs], axis = 0)
 seed_log_loss = mean_log_loss(train_targets.values, oof_pred)
 print(f'Our final out of folds log loss for our classic dnn + dnn resnet type model is {seed_log_loss}')
 test_pred = np.average([test_pred, test_pred_rs], axis = 0)
 sample_submission = submission(test_pred)
 sample_submission.head()
folds_average_lgbm.fit(best_lgb_params, train_x, train_y )<compute_test_metric>
sub3 = sample_submission
np.sqrt(mean_squared_error(df_train.target, folds_average_lgbm.oof_preds))<predict_on_test>
submission = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv' )
y_pred = folds_average_lgbm.predict(test_x )<save_to_csv>
test = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 test_ids = list(test['sig_id'].values)
 private_ids = list(set(test_ids)-set(public_ids))
len(private_ids )
sub = df_sample.copy()
sub["target"] = y_pred
sub.to_csv("submission_lgbm_fold_10.csv", index=False)
 sub.head()<set_options>
predictions = [sub3,sub2,sub1]
!pip3 install -qq optuna
sns.set()<load_from_csv>
y_pred = pd.DataFrame()
y_pred['sig_id'] = predictions[0]['sig_id']
for column in target_columns:
column_data = []
for i in range(len(predictions)) :
column_data.append(predictions[i][column])
 y_pred[column] = np.mean(column_data, axis=0)
 y_pred.shape
train = pd.read_csv("/kaggle/input/tabular-playground-series-jan-2021/train.csv")
 test = pd.read_csv("/kaggle/input/tabular-playground-series-jan-2021/test.csv")
 print(train.shape)
 print(test.shape)
 X, y = train.iloc[:,1:-1].values, train.iloc[:,-1].values
feat_names = list(train.columns[1:-1])
 kfold = KFold(n_splits=5,random_state=2021, shuffle=True)
 X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.15, random_state=2021 )<compute_train_metric>
weights = [0.3345993850605896, 0.33387592693072893, 0.3315246880086815]
baseline_score = mean_squared_error(y,np.mean(y)* np.ones(y.shape[0]),squared=False)
 lr = LinearRegression()
lr_scores = cross_val_score(lr,X, y, cv=kfold,scoring='neg_root_mean_squared_error')
 lasso = Lasso()
lasso_scores = cross_val_score(lasso,X, y, cv=kfold,scoring='neg_root_mean_squared_error')
 ridge = Ridge()
ridge_scores = cross_val_score(ridge,X, y, cv=kfold,scoring='neg_root_mean_squared_error')
 print(-lr_scores, np.mean(-lr_scores))
print(-lasso_scores, np.mean(-lasso_scores))
print(-ridge_scores, np.mean(-ridge_scores))
fig, ax = plt.subplots(ncols=4,nrows=2, figsize=(20, 10))
ridge.fit(X, y)X
lr.fit(X,y)
 feat_names2, coef_order = zip(*sorted(zip(feat_names,ridge.coef_), key=lambda k : abs(k[1]), reverse=True))
sns.barplot(list(coef_order),list(feat_names2), ax=ax[0,0])
 sns.scatterplot(y - ridge.predict(X), y,ax=ax[0,1])
 sns.distplot(y - ridge.predict(X),ax=ax[0,2])
 sns.distplot(ridge.predict(X),ax=ax[0,3])
 feat_names2, coef_order2 = zip(*sorted(zip(feat_names,lr.coef_), key=lambda k : abs(k[1]), reverse=True))
sns.barplot(list(coef_order2),list(feat_names2), ax=ax[1,0])
 sns.scatterplot(y - lr.predict(X), y,ax=ax[1,1])
 sns.distplot(y - lr.predict(X),ax=ax[1,2])
 sns.distplot(lr.predict(X),ax=ax[1,3])
 print("Ridge on full dataset",mean_squared_error(y, ridge.predict(X), squared=False))<find_best_model_class>
weighted_y_pred = pd.DataFrame()
weighted_y_pred['sig_id'] = predictions[0]['sig_id']
for column in target_columns:
column_data = []
for i in range(len(predictions)) :
column_data.append(predictions[i][column] * weights[i])
 weighted_y_pred[column] = np.sum(column_data, axis=0)
 weighted_y_pred.shape
%%time
i = 0
for train_index, test_index in tqdm(kfold.split(X)) :
cr = CatBoostRegressor(verbose=False,iterations=2000, learning_rate=0.01,
random_strength=52, max_depth=12,random_seed=2021)
 cr.fit(X[train_index,:], y[train_index])
 train[f'target{i}'] = cr.predict(X)
 test[f'target{i}'] = cr.predict(test.iloc[:,1:].values)
 i += 1
train['target_final'] = train.loc[:,["target"+str(i)for i in range(5)]].mean(axis=1)
 print("In Sample: ", mean_squared_error(y, train['target_final'], squared=False))
test['target'] = test[["target"+str(i)for i in range(5)]].mean(axis=1)
 test[['id','target']].to_csv("submission.csv",index=False )<train_model>
y_pred = weighted_y_pred
<load_from_csv><EOS> 
submission = pd.DataFrame(index = public_ids + private_ids, columns=target_columns)
 submission.index.name = 'sig_id'
submission[:] = 0
submission.loc[y_pred.sig_id,:] = y_pred[target_columns].values
submission.loc[test[test.cp_type == 'ctl_vehicle'].sig_id] = 0
submission.to_csv('submission.csv', index=True )