Kaggle competition: Mechanisms of Action (MoA) Prediction
submit=pd.DataFrame()
submit['PetID']=test['PetID']
submit['AdoptionSpeed']=df.mode(axis=1)[0].astype(int)
 submit.to_csv('submission.csv',index=False )<set_options>
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
 device
np.random.seed(400 )<compute_test_metric>
warnings.filterwarnings('ignore' )
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):

 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_ratings = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_ratings)]
for j in range(num_ratings)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):

 if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):

 rater_a = y
rater_b = y_pred
min_rating=None
max_rating=None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(min(rater_a), min(rater_b))
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b,
min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(hist_rater_a[i] * hist_rater_b[j]
/ num_scored_items)
 d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator )<compute_test_metric>
def seed_everything(seed):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(42 )
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = quadratic_weighted_kappa(y, X_p)
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']<compute_test_metric>
df_train_features = pd.read_csv('/kaggle/input/lish-moa/train_features.csv')
 df_train_targets_scored = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
 df_test_features = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 print('df_train_features Shape',df_train_features.shape)
 print('df_train_targets_scored Shape',df_train_targets_scored.shape)
 print('df_test_features Shape',df_test_features.shape )
def rmse(actual, predicted):
return sqrt(mean_squared_error(actual, predicted)) 
<load_from_csv>
df_train = pd.merge(df_train_features,df_train_targets_scored,on='sig_id')
 df_train = df_train[df_train['cp_type']!= 'ctl_vehicle'].reset_index(drop=True)
 df_train.shape
%%time
print('Train')
 train = pd.read_csv(".. /input/train/train.csv")
 print(train.shape)
 print('Test')
 test = pd.read_csv(".. /input/test/test.csv")
 print(test.shape)
 print('Breeds')
 breeds = pd.read_csv(".. /input/breed_labels.csv")
 print(breeds.shape)
 print('Colors')
 colors = pd.read_csv(".. /input/color_labels.csv")
 print(colors.shape)
 print('States')
 states = pd.read_csv(".. /input/state_labels.csv")
 print(states.shape )<prepare_x_and_y>
df_train['cp_type'] = df_train['cp_type'].map({'trt_cp': 0, 'ctl_vehicle': 1})
 df_train['cp_dose'] = df_train['cp_dose'].map({'D1': 3, 'D2': 4})
 df_train['cp_time'] = df_train['cp_time'].map({24: 0, 48: 1, 72: 2})
 df_test_features['cp_type'] = df_test_features['cp_type'].map({'trt_cp': 0, 'ctl_vehicle': 1})
 df_test_features['cp_dose'] = df_test_features['cp_dose'].map({'D1': 3, 'D2': 4})
 df_test_features['cp_time'] = df_test_features['cp_time'].map({24: 0, 48: 1, 72: 2} )
target = train['AdoptionSpeed']
train_id = train['PetID']
test_id = test['PetID']
train.drop(['AdoptionSpeed', 'PetID'], axis=1, inplace=True)
 test.drop(['PetID'], axis=1, inplace=True )<feature_engineering>
df_train['folds'] = -1
df_train = df_train.sample(frac=1 ).reset_index(drop=True)
 splits = 5
kf = KFold(n_splits=splits,shuffle = False)
 for fold,(train_index, val_idx)in enumerate(kf.split(df_train)) :
df_train.loc[val_idx,'folds'] = int(fold)
 print('Number of Unique folds in dataset',df_train['folds'].unique() )
doc_sent_mag = []
doc_sent_score = []
nf_count = 0
for pet in train_id:
try:
with open('.. /input/train_sentiment/' + pet + '.json', 'r')as f:
sentiment = json.load(f)
 doc_sent_mag.append(sentiment['documentSentiment']['magnitude'])
 doc_sent_score.append(sentiment['documentSentiment']['score'])
 except FileNotFoundError:
nf_count += 1
doc_sent_mag.append(-1)
 doc_sent_score.append(-1)
 train.loc[:, 'doc_sent_mag'] = doc_sent_mag
train.loc[:, 'doc_sent_score'] = doc_sent_score
doc_sent_mag = []
doc_sent_score = []
nf_count = 0
for pet in test_id:
try:
with open('.. /input/test_sentiment/' + pet + '.json', 'r')as f:
sentiment = json.load(f)
 doc_sent_mag.append(sentiment['documentSentiment']['magnitude'])
 doc_sent_score.append(sentiment['documentSentiment']['score'])
 except FileNotFoundError:
nf_count += 1
doc_sent_mag.append(-1)
 doc_sent_score.append(-1)
 test.loc[:, 'doc_sent_mag'] = doc_sent_mag
test.loc[:, 'doc_sent_score'] = doc_sent_score 
<normalization>
class Train_Dataset(Dataset):
def __init__(self, dataframe,features_col,labels):
self.dataframe = dataframe
self.features_col = features_col
self.labels = labels
self.x = self.dataframe[self.features_col].values
self.y = self.dataframe[self.labels].values
def __len__(self):
return self.dataframe.shape[0]
def __getitem__(self, idx):
feat = torch.FloatTensor(self.x[idx])
 lab = torch.FloatTensor(self.y[idx])
 return feat,lab
class Test_Dataset(Dataset):
def __init__(self, dataframe,features_col):
self.dataframe = dataframe
self.features_col = features_col
self.x = self.dataframe[self.features_col].values
def __len__(self):
return self.dataframe.shape[0]
def __getitem__(self, idx):
feat = torch.FloatTensor(self.x[idx])
 return feat
train_desc = train.Description.fillna("none" ).values
test_desc = test.Description.fillna("none" ).values
tfv = TfidfVectorizer(min_df=2, max_features=None,
strip_accents='unicode', analyzer='word', token_pattern=r'(?u)\b\w+\b',
ngram_range=(1, 3), use_idf=1, smooth_idf=1, sublinear_tf=1, 
)
 tfv.fit(list(train_desc))
X = tfv.transform(train_desc)
 X_test = tfv.transform(test_desc)
 svd = TruncatedSVD(n_components=120)
 svd.fit(X)
 print(svd.explained_variance_ratio_.sum())
 print(svd.explained_variance_ratio_)
 X = svd.transform(X)
 X = pd.DataFrame(X, columns=['svd_{}'.format(i)for i in range(120)])
 train = pd.concat(( train, X), axis=1)
 X_test = svd.transform(X_test)
 X_test = pd.DataFrame(X_test, columns=['svd_{}'.format(i)for i in range(120)])
 test = pd.concat(( test, X_test), axis=1 )<normalization>
class MultiHead_Attn(nn.Module):
def __init__(self, embed_dim ,num_head):
super().__init__()
self.embed_dim = embed_dim
self.num_head = num_head
self.head_dim = self.embed_dim // self.num_head
assert self.embed_dim%self.num_head == 0
self.q = nn.Linear(self.embed_dim , self.embed_dim)
 self.k = nn.Linear(self.embed_dim , self.embed_dim)
 self.v = nn.Linear(self.embed_dim , self.embed_dim)
 self.f_linear = nn.Linear(self.embed_dim, self.embed_dim)
 self.dropout = nn.Dropout (.35)
 self.scale = torch.sqrt(torch.FloatTensor([self.head_dim])).to(device)
 def forward(self, x):
batch_size = x.shape[0]
src_len = x.shape[1]
w_k = self.k(x)
 w_q = self.q(x)
 w_v = self.v(x)
 w_q = w_q.view(batch_size,-1,self.head_dim)
 w_k = w_k.view(batch_size,-1,self.head_dim)
 w_v = w_v.view(batch_size,-1,self.head_dim)
 energy = torch.matmul(w_k.permute(0,2,1),w_q)
 energy = energy/self.scale
energy = torch.softmax(energy,-1)
 f_energy = torch.matmul(self.dropout(energy), w_v.permute(0,2,1))
f_energy = f_energy.permute(0, 2, 1)
 f_energy = f_energy.reshape(batch_size,-1)
 out = self.f_linear(f_energy)
 return out
train_desc = train.Description.fillna("none" ).values
test_desc = test.Description.fillna("none" ).values
tfv = TfidfVectorizer(min_df=3, max_features=10000,
strip_accents='unicode', analyzer='word', token_pattern=r'\w{1,}',
ngram_range=(1, 3), use_idf=1, smooth_idf=1, sublinear_tf=1,
stop_words = 'english')
 tfv.fit(list(train_desc))
X = tfv.transform(train_desc)
 X_test = tfv.transform(test_desc)
 svd = TruncatedSVD(n_components=120)
 svd.fit(X)
 print(svd.explained_variance_ratio_.sum())
 print(svd.explained_variance_ratio_)
 X = svd.transform(X)
 X = pd.DataFrame(X, columns=['svd_{}'.format(i)for i in range(120)])
 train = pd.concat(( train, X), axis=1)
 X_test = svd.transform(X_test)
 X_test = pd.DataFrame(X_test, columns=['svd_{}'.format(i)for i in range(120)])
 test = pd.concat(( test, X_test), axis=1 )<prepare_x_and_y>
class PositionwiseFeedforwardLayer(nn.Module):
def __init__(self, embed_dim, pf_dim):
super().__init__()
self.fc_1 = nn.Linear(embed_dim, pf_dim)
 self.fc_2 = nn.Linear(pf_dim, embed_dim)
 self.dropout = nn.Dropout (.35)
 def forward(self, x):
x = self.dropout(torch.relu(self.fc_1(x)))
 x = self.fc_2(x)
 return x
vertex_xs = []
vertex_ys = []
bounding_confidences = []
bounding_importance_fracs = []
dominant_blues = []
dominant_greens = []
dominant_reds = []
dominant_pixel_fracs = []
dominant_scores = []
label_descriptions = []
label_scores = []
nf_count = 0
nl_count = 0
for pet in train_id:
try:
with open('.. /input/train_metadata/' + pet + '-1.json', 'r')as f:
data = json.load(f)
 vertex_x = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['x']
vertex_xs.append(vertex_x)
 vertex_y = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['y']
vertex_ys.append(vertex_y)
 bounding_confidence = data['cropHintsAnnotation']['cropHints'][0]['confidence']
bounding_confidences.append(bounding_confidence)
 bounding_importance_frac = data['cropHintsAnnotation']['cropHints'][0].get('importanceFraction', -1)
 bounding_importance_fracs.append(bounding_importance_frac)
 dominant_blue = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['blue']
dominant_blues.append(dominant_blue)
 dominant_green = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['green']
dominant_greens.append(dominant_green)
 dominant_red = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['red']
dominant_reds.append(dominant_red)
 dominant_pixel_frac = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['pixelFraction']
dominant_pixel_fracs.append(dominant_pixel_frac)
 dominant_score = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['score']
dominant_scores.append(dominant_score)
 if data.get('labelAnnotations'):
label_description = data['labelAnnotations'][0]['description']
label_descriptions.append(label_description)
 label_score = data['labelAnnotations'][0]['score']
label_scores.append(label_score)
 else:
nl_count += 1
label_descriptions.append('nothing')
 label_scores.append(-1)
 except FileNotFoundError:
nf_count += 1
vertex_xs.append(-1)
 vertex_ys.append(-1)
 bounding_confidences.append(-1)
 bounding_importance_fracs.append(-1)
 dominant_blues.append(-1)
 dominant_greens.append(-1)
 dominant_reds.append(-1)
 dominant_pixel_fracs.append(-1)
 dominant_scores.append(-1)
 label_descriptions.append('nothing')
 label_scores.append(-1)
 print(nf_count)
 print(nl_count)
 train.loc[:, 'vertex_x'] = vertex_xs
train.loc[:, 'vertex_y'] = vertex_ys
train.loc[:, 'bounding_confidence'] = bounding_confidences
train.loc[:, 'bounding_importance'] = bounding_importance_fracs
train.loc[:, 'dominant_blue'] = dominant_blues
train.loc[:, 'dominant_green'] = dominant_greens
train.loc[:, 'dominant_red'] = dominant_reds
train.loc[:, 'dominant_pixel_frac'] = dominant_pixel_fracs
train.loc[:, 'dominant_score'] = dominant_scores
train.loc[:, 'label_description'] = label_descriptions
train.loc[:, 'label_score'] = label_scores
vertex_xs = []
vertex_ys = []
bounding_confidences = []
bounding_importance_fracs = []
dominant_blues = []
dominant_greens = []
dominant_reds = []
dominant_pixel_fracs = []
dominant_scores = []
label_descriptions = []
label_scores = []
nf_count = 0
nl_count = 0
for pet in test_id:
try:
with open('.. /input/test_metadata/' + pet + '-1.json', 'r')as f:
data = json.load(f)
 vertex_x = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['x']
vertex_xs.append(vertex_x)
 vertex_y = data['cropHintsAnnotation']['cropHints'][0]['boundingPoly']['vertices'][2]['y']
vertex_ys.append(vertex_y)
 bounding_confidence = data['cropHintsAnnotation']['cropHints'][0]['confidence']
bounding_confidences.append(bounding_confidence)
 bounding_importance_frac = data['cropHintsAnnotation']['cropHints'][0].get('importanceFraction', -1)
 bounding_importance_fracs.append(bounding_importance_frac)
 dominant_blue = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['blue']
dominant_blues.append(dominant_blue)
 dominant_green = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['green']
dominant_greens.append(dominant_green)
 dominant_red = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['color']['red']
dominant_reds.append(dominant_red)
 dominant_pixel_frac = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['pixelFraction']
dominant_pixel_fracs.append(dominant_pixel_frac)
 dominant_score = data['imagePropertiesAnnotation']['dominantColors']['colors'][0]['score']
dominant_scores.append(dominant_score)
 if data.get('labelAnnotations'):
label_description = data['labelAnnotations'][0]['description']
label_descriptions.append(label_description)
 label_score = data['labelAnnotations'][0]['score']
label_scores.append(label_score)
 else:
nl_count += 1
label_descriptions.append('nothing')
 label_scores.append(-1)
 except FileNotFoundError:
nf_count += 1
vertex_xs.append(-1)
 vertex_ys.append(-1)
 bounding_confidences.append(-1)
 bounding_importance_fracs.append(-1)
 dominant_blues.append(-1)
 dominant_greens.append(-1)
 dominant_reds.append(-1)
 dominant_pixel_fracs.append(-1)
 dominant_scores.append(-1)
 label_descriptions.append('nothing')
 label_scores.append(-1)
 print(nf_count)
 test.loc[:, 'vertex_x'] = vertex_xs
test.loc[:, 'vertex_y'] = vertex_ys
test.loc[:, 'bounding_confidence'] = bounding_confidences
test.loc[:, 'bounding_importance'] = bounding_importance_fracs
test.loc[:, 'dominant_blue'] = dominant_blues
test.loc[:, 'dominant_green'] = dominant_greens
test.loc[:, 'dominant_red'] = dominant_reds
test.loc[:, 'dominant_pixel_frac'] = dominant_pixel_fracs
test.loc[:, 'dominant_score'] = dominant_scores
test.loc[:, 'label_description'] = label_descriptions
test.loc[:, 'label_score'] = label_scores<drop_column>
class TransformerBlock(nn.Module):
def __init__(self, embed_dim,num_heads, pf_dim):
super().__init__()
self.self_attn_layer_norm = nn.LayerNorm(embed_dim)
 self.ff_layer_norm = nn.LayerNorm(embed_dim)
 self.attn = MultiHead_Attn(embed_dim ,num_heads)
 self.positionwise_feedforward = PositionwiseFeedforwardLayer(embed_dim,pf_dim)
 self.dropout = nn.Dropout (.35)
 def forward(self,src):
attn = self.attn(src)
 src = self.self_attn_layer_norm(src + self.dropout(attn))
_src = self.positionwise_feedforward(src)
 src = self.ff_layer_norm(src + self.dropout(_src))
return src
%%time
train.drop(['Name', 'RescuerID', 'Description'], axis=1, inplace=True)
 test.drop(['Name', 'RescuerID', 'Description'], axis=1, inplace=True )<data_type_conversions>
class Encorder(nn.Module):
def __init__(self, src_dim , embed_dim ,num_heads , pf_dim , target_dim ,depth):
super(Encorder, self ).__init__()
self.embed_layer = nn.Linear(src_dim , embed_dim)
 self.Transformer_layers = TransformerBlock(embed_dim , num_heads ,pf_dim)
 tblocks = []
for i in range(depth):
tblocks.append(self.Transformer_layers)
 self.tblocks = nn.Sequential(*tblocks)
 self.dropout = nn.Dropout (.35)
 self.final_linear = nn.Sequential(
 nn.utils.weight_norm(nn.Linear(embed_dim , embed_dim //2))
,nn.BatchNorm1d(embed_dim //2)
 ,nn.Dropout (.35)
 ,nn.ReLU()
,nn.Linear(embed_dim //2 , target_dim))
def forward(self , x):
batch_size,seq_len = x.shape
embed = self.embed_layer(x)
 for layer in self.tblocks:
x = self.dropout(self.Transformer_layers(embed))
out = self.final_linear(x)
 return out
numeric_cols = ['Age', 'Quantity', 'Fee', 'VideoAmt', 'PhotoAmt', 'AdoptionSpeed', 'doc_sent_mag', 'doc_sent_score', 'dominant_score', 'dominant_pixel_frac', 'dominant_red', 'dominant_green', 'dominant_blue', 'bounding_importance', 'bounding_confidence', 'vertex_x', 'vertex_y', 'label_score'] + ['svd_{}'.format(i)for i in range(120)]
cat_cols = list(set(train.columns)- set(numeric_cols))
train.loc[:, cat_cols] = train[cat_cols].astype('category')
 test.loc[:, cat_cols] = test[cat_cols].astype('category')
 print(train.shape)
 print(test.shape)
 train.head()<train_on_grid>
def train(loader, model , optimizer , loss_func ,scheduler):
model.train()
train_running_loss = 0
for index,(feat,label)in enumerate(loader):
x,y = feat.to(device),label.to(device)
 optimizer.zero_grad()
pred = model(x)
 loss = loss_func(pred,y)
 loss.backward()
optimizer.step()
train_running_loss += loss.item()
return train_running_loss/len(loader)
 def valid(loader, model , loss_func):
valid_running_loss = 0
pred_list = list()
with torch.no_grad() :
model.eval()
for val_index,(feat,label)in enumerate(loader):
x,y = feat.to(device),label.to(device)
 pred = model(x)
 pred_list.append(pred)
 loss = loss_func(pred,y)
 valid_running_loss += loss.item()
return torch.cat(pred_list ).sigmoid().detach().cpu().numpy() ,valid_running_loss/len(loader)
 def test(loader,model):
labels = []
with torch.no_grad() :
model.eval()
for index,feat in enumerate(loader):
x = feat.to(device)
 output = model(x)
 labels.append(output.sigmoid().detach().cpu().numpy())
 labels = np.concatenate(labels)
 return labels
n_repeats = 2
n_splits = 5
def run_cv_model(train, test, target, model_fn, params={}, eval_fn=None, label='model'):
kf = RepeatedStratifiedKFold(n_splits=n_splits, random_state=42, n_repeats = n_repeats)
 fold_splits = kf.split(train, target)
 cv_scores = []
qwk_scores = []
pred_full_test = 0
pred_train = np.zeros(( train.shape[0], n_splits*n_repeats))
all_coefficients = np.zeros(( n_splits*n_repeats, 4))
feature_importance_df = pd.DataFrame()
i = 1
for dev_index, val_index in fold_splits:
print('Started ' + label + ' fold ' + str(i)+ '/'+str(n_splits*n_repeats))
if isinstance(train, pd.DataFrame):
dev_X, val_X = train.iloc[dev_index], train.iloc[val_index]
dev_y, val_y = target[dev_index], target[val_index]
else:
dev_X, val_X = train[dev_index], train[val_index]
dev_y, val_y = target[dev_index], target[val_index]
params2 = params.copy()
pred_val_y, pred_test_y, importances, coefficients, qwk = model_fn(dev_X, dev_y, val_X, val_y, test, params2)
 pred_full_test = pred_full_test + pred_test_y
pred_train[val_index] = pred_val_y
all_coefficients[i-1, :] = coefficients
if eval_fn is not None:
cv_score = eval_fn(val_y, pred_val_y)
 cv_scores.append(cv_score)
 qwk_scores.append(qwk)
 print(label + ' cv score {}: RMSE {} QWK {}'.format(i, cv_score, qwk))
fold_importance_df = pd.DataFrame()
fold_importance_df['feature'] = train.columns.values
fold_importance_df['importance'] = importances
fold_importance_df['fold'] = i
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 i += 1
print('{} cv RMSE scores : {}'.format(label, cv_scores))
print('{} cv mean RMSE score : {}'.format(label, np.mean(cv_scores)))
 print('{} cv std RMSE score : {}'.format(label, np.mean(cv_scores)))
 print('{} cv QWK scores : {}'.format(label, qwk_scores))
print('{} cv mean QWK score : {}'.format(label, np.mean(qwk_scores)))
 print('{} cv std QWK score : {}'.format(label, np.std(qwk_scores)))
 pred_full_test = pred_full_test /(n_splits*n_repeats)
 results = {'label': label,
'train': pred_train, 'test': pred_full_test,
'cv': cv_scores, 'qwk': qwk_scores,
'importance': feature_importance_df,
'coefficients': all_coefficients}
return results
params = {'application': 'regression',
'boosting': 'gbdt',
'metric': 'rmse',
'num_leaves': 80,
'max_depth': 9,
'learning_rate': 0.01,
'bagging_fraction': 0.85,
'feature_fraction': 0.8,
'min_split_gain': 0.01,
'min_child_samples': 150,
'min_child_weight': 0.1,
'verbosity': -1,
'data_random_seed': 3,
'early_stop': 100,
'verbose_eval': 100,
'num_rounds': 10000}
def runLGB(train_X, train_y, test_X, test_y, test_X2, params):
print('Prep LGB')
 d_train = lgb.Dataset(train_X, label=train_y)
 d_valid = lgb.Dataset(test_X, label=test_y)
 watchlist = [d_train, d_valid]
print('Train LGB')
 num_rounds = params.pop('num_rounds')
 verbose_eval = params.pop('verbose_eval')
 early_stop = None
if params.get('early_stop'):
early_stop = params.pop('early_stop')
 model = lgb.train(params,
train_set=d_train,
num_boost_round=num_rounds,
valid_sets=watchlist,
verbose_eval=verbose_eval,
early_stopping_rounds=early_stop)
 print('Predict 1/2')
 pred_test_y = model.predict(test_X, num_iteration=model.best_iteration)
 optR = OptimizedRounder()
optR.fit(pred_test_y, test_y)
 coefficients = optR.coefficients()
pred_test_y_k = optR.predict(pred_test_y, coefficients)
 print("Valid Counts = ", Counter(test_y))
print("Predicted Counts = ", Counter(pred_test_y_k))
print("Coefficients = ", coefficients)
 qwk = quadratic_weighted_kappa(test_y, pred_test_y_k)
 print("QWK = ", qwk)
 print('Predict 2/2')
 pred_test_y2 = model.predict(test_X2, num_iteration=model.best_iteration)
 return pred_test_y.reshape(-1, 1), pred_test_y2.reshape(-1, 1), model.feature_importance() , coefficients, qwk
results = run_cv_model(train, test, target, runLGB, params, rmse, 'lgb' )<compute_train_metric>
def train_run(seed , Batch_size , src_dim ,embed_dim , num_heads , pf_dim ,target_dim , depth , n_epoch , patience , learning_rate):
val_preds = np.zeros(( df_train[y_cols].shape))
test_pred = np.zeros(( len(df_test_features), y_cols.shape[0]))
for fold_num in range(splits):
seed_everything(seed)
 print('seed value', seed)
 print('='*30,'*****','Fold',fold_num+1,'*****','='*30)
 trn_idx = df_train[df_train['folds'] != fold_num].index
val_idx = df_train[df_train['folds'] == fold_num].index
df_trn = df_train.loc[trn_idx].reset_index(drop=True)
 df_val = df_train.loc[val_idx].reset_index(drop=True)
 train_dataset = Train_Dataset(df_trn,x_cols,y_cols)
 train_loader = DataLoader(train_dataset, batch_size=Batch_size, shuffle=True)
 valid_dataset = Train_Dataset(df_val,x_cols,y_cols);
valid_loader = DataLoader(valid_dataset, batch_size=Batch_size, shuffle=False)
 model = Encorder(src_dim , embed_dim ,num_heads , pf_dim , target_dim ,depth ).to(device)
 pytorch_total_params = sum(p.numel() for p in model.parameters() if p.requires_grad)
 print('Number of parameters',pytorch_total_params)
 optimizer = torch.optim.Adam(model.parameters() , lr=learning_rate, weight_decay= 1e-5)
 scheduler = ReduceLROnPlateau(optimizer=optimizer, mode='max', patience=3, verbose=True, factor=0.2)
 loss_func = nn.BCEWithLogitsLoss()
best_score = np.inf
counter = 0
for epoch in range(n_epoch):
print("Epoch: {}/{}.. ".format(epoch+1, n_epoch))
train_loss = train(train_loader, model , optimizer , loss_func ,scheduler)
 v_out,valid_loss = valid(valid_loader, model , loss_func)
 print(f'\tTrain Loss: {train_loss:.5f}')
 print(f'\t Val.Loss: {valid_loss:.5f}')
 if valid_loss < best_score:
best_score = valid_loss
val_preds[val_idx] = v_out
torch.save(model.state_dict() , f"fold_{fold_num}_seed_{seed}.pth")
 else:
print('patience starts.........')
 counter +=1
print('.. counter.. ',counter)
 if(counter >= patience):
break;
scheduler.step(valid_loss)
 print(f'Inferenceing the test data at epoch {epoch+1} and fold {fold_num+1} and seed {seed}.....')
 model = Encorder(src_dim , embed_dim ,num_heads , pf_dim , target_dim ,depth ).to(device)
 model.load_state_dict(torch.load(f"fold_{fold_num}_seed_{seed}.pth"))
test_dataset = Test_Dataset(df_test_features,x_cols)
 test_loader = DataLoader(test_dataset, batch_size=Batch_size, shuffle=False)
 test_pred += test(test_loader,model)/splits
print(f'Inferenceing completed for fold {fold_num+1}')
 return test_pred , val_preds
optR = OptimizedRounder()
coefficients_ = np.mean(results['coefficients'], axis=0)
 print(coefficients_)
 train_predictions = [r[0] for r in results['train']]
train_predictions = optR.predict(train_predictions, coefficients_ ).astype(int)
 Counter(train_predictions )<predict_on_test>
random_seed = [33 , 66 , 99 , 132]
Batch_size = 32
src_dim = 875
embed_dim = 1024
num_heads = 8
pf_dim = 512
target_dim = 206
depth = 8
n_epoch = 30
patience = 10
learning_rate =.0005
test_pred = np.zeros(( len(df_test_features), y_cols.shape[0]))
val_preds = np.zeros(( df_train[y_cols].shape))
for seed in random_seed:
tp , vp = train_run(seed , Batch_size , src_dim ,embed_dim , num_heads , pf_dim ,target_dim , depth , n_epoch , patience , learning_rate)
 test_pred += tp/len(random_seed)
 val_preds += vp/len(random_seed )
optR = OptimizedRounder()
test_predictions = [r[0] for r in results['test']]
test_predictions = optR.predict(test_predictions, coefficients_ ).astype(int)
 Counter(test_predictions )<create_dataframe>
score = 0
for i in range(df_train[y_cols].shape[1]):
_score = log_loss(df_train[y_cols].iloc[:,i], val_preds[:,i])
 score += _score / df_train[y_cols].shape[1]
print(f"oof score: {score}" )
pd.DataFrame(sk_cmatrix(target, train_predictions), index=list(range(5)) , columns=list(range(5)))
<compute_test_metric>
sub = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 sub[y_cols] = test_pred
sub.loc[df_test_features['cp_type'] == 1 , y_cols ] = 0
quadratic_weighted_kappa(target, train_predictions )<compute_test_metric>
sub.to_csv('submission.csv', index=False )
rmse(target, [r[0] for r in results['train']] )<prepare_output>
sub.to_csv('submission.csv', index=False )