Kaggle competition: Mechanisms of Action (MoA) Prediction
score=[]
magnitude=[]
petid=[]
for filename in sentimental_analysis:
with open(filename, 'r')as f:
sentiment_file = json.load(f)
 file_sentiment = sentiment_file['documentSentiment']
file_score = np.asarray(sentiment_file['documentSentiment']['score'])
 file_magnitude =np.asarray(sentiment_file['documentSentiment']['magnitude'])
 score.append(file_score)
 magnitude.append(file_magnitude)
 petid.append(filename.replace('.json','' ).replace('.. /input/train_sentiment/', ''))
sentimental_analysis = pd.concat([ pd.DataFrame(petid, columns =['PetID']),pd.DataFrame(score, columns =['sentiment_document_score']),
pd.DataFrame(magnitude, columns =['sentiment_document_magnitude'])],axis =1 )<sort_values>
warnings.filterwarnings('ignore')
 pd.set_option('max_columns', 2000 )
image_metadata = sorted(glob.glob('.. /input/train_metadata/*.json'))
print('num of train metadata: {}'.format(len(image_metadata)) )<load_pretrained>
train_features = pd.read_csv("/kaggle/input/lish-moa/train_features.csv")
 train_targets_scored = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
 test_features = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv' )
description=[]
topicality=[]
imageid=[]
for filename in image_metadata:
with open(filename, 'r')as f:
d = json.load(f)
 file_keys = list(d.keys())
 if 'labelAnnotations' in file_keys:
file_annots = d['labelAnnotations']
file_topicality = np.asarray([x['topicality'] for x in file_annots])
 file_description = [x['description'] for x in file_annots]
description.append(file_description)
 topicality.append(file_topicality)
 imageid.append(filename.replace('.json','' ).replace('.. /input/train_metadata/',''))
description=pd.DataFrame(description)
 topicality=pd.DataFrame(topicality)
 new_names = [(i,'metadata_description_'+str(i)) for i in description.iloc[:, 0:].columns.values]
description.rename(columns = dict(new_names), inplace=True)
 new_names = [(i,'metadata_topicality_'+str(i)) for i in topicality.iloc[:, 0:].columns.values]
topicality.rename(columns = dict(new_names), inplace=True)
 image_labelannot = pd.concat([ pd.DataFrame(imageid, columns =['ImageId']),topicality,description],axis =1)
 image_labelannot['PetID'] = image_labelannot['ImageId'].str.split('-' ).str[0] 
<categorify>
n_comp_GENES = int(0.6*610)
 n_comp_CELLS = int(0.6*55)
 VarianceThreshold_for_FS = 0.82
image_labelannot['metadata_topicality_mean'] = image_labelannot.iloc[:,1:10].mean(axis=1)
 image_labelannot['metadata_topicality_mean'] = image_labelannot.groupby(['PetID'])['metadata_topicality_mean'].transform('mean')
 image_labelannot['metadata_topicality_max'] = image_labelannot.iloc[:,1:10].max(axis=1)
 image_labelannot['metadata_topicality_max'] = image_labelannot.groupby(['PetID'])['metadata_topicality_max'].transform(max)
 image_labelannot['metadata_topicality_min'] = image_labelannot.iloc[:,1:10].min(axis=1)
 image_labelannot['metadata_topicality_min'] = image_labelannot.groupby(['PetID'])['metadata_topicality_min'].transform(min)
 image_labelannot['metadata_topicality_0_mean'] = image_labelannot.groupby(['PetID'])['metadata_topicality_0'].transform('mean')
 image_labelannot['metadata_topicality_0_max'] = image_labelannot.groupby(['PetID'])['metadata_topicality_0'].transform(max)
 image_labelannot['metadata_topicality_0_min'] = image_labelannot.groupby(['PetID'])['metadata_topicality_0'].transform(min)
 image_labelannot['L_metadata_0_cat']=image_labelannot['metadata_description_0'].str.contains("cat" ).astype(int)
 image_labelannot['L_metadata_0_dog'] =image_labelannot['metadata_description_0'].str.contains("dog" ).astype(int)
 image_labelannot['L_metadata_any_cat']=image_labelannot.apply(lambda row: row.astype(str ).str.contains('cat' ).any() , axis=1)
 image_labelannot['L_metadata_any_dog']=image_labelannot.apply(lambda row: row.astype(str ).str.contains('dog' ).any() , axis=1)
 image_labelannot['L_metadata_0_cat_sum'] = image_labelannot.groupby(image_labelannot['PetID'])['L_metadata_0_cat'].transform('sum')
 image_labelannot['L_metadata_0_dog_sum'] = image_labelannot.groupby(image_labelannot['PetID'])['L_metadata_0_dog'].transform('sum')
 image_labelannot['L_metadata_any_cat_sum'] = image_labelannot.groupby(image_labelannot['PetID'])['L_metadata_any_cat'].transform('sum')
 image_labelannot['L_metadata_any_dog_sum'] = image_labelannot.groupby(image_labelannot['PetID'])['L_metadata_any_dog'].transform('sum')
 image_labelannot = image_labelannot[['PetID','metadata_topicality_max','metadata_topicality_mean','metadata_topicality_min','metadata_topicality_0_mean','metadata_topicality_0_max','metadata_topicality_0_min','L_metadata_0_cat_sum','L_metadata_0_dog_sum','L_metadata_any_cat_sum','L_metadata_any_dog_sum']]
image_labelannot=image_labelannot.drop_duplicates('PetID' )<define_variables>
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
color_score_mean=[]
color_score_min=[]
color_score_max=[]
color_pixelfrac_mean=[]
color_pixelfrac_min=[]
color_pixelfrac_max=[]
imageid=[]
for filename in image_metadata:
with open(filename, 'r')as f:
d = json.load(f)
 file_keys = list(d.keys())
 if 'imagePropertiesAnnotation' in file_keys:
file_colors = d['imagePropertiesAnnotation']['dominantColors']['colors']
file_color_score_mean = np.asarray([x['score'] for x in file_colors] ).mean()
file_color_pixelfrac_mean = np.asarray([x['pixelFraction'] for x in file_colors] ).mean()
file_color_score_min = np.asarray([x['score'] for x in file_colors] ).min()
file_color_pixelfrac_min = np.asarray([x['pixelFraction'] for x in file_colors] ).min()
file_color_score_max = np.asarray([x['score'] for x in file_colors] ).max()
file_color_pixelfrac_max = np.asarray([x['pixelFraction'] for x in file_colors] ).max()
imageid.append(filename.replace('.json','' ).replace('.. /input/train_metadata/', ''))
color_score_mean.append(file_color_score_mean)
 color_score_min.append(file_color_score_min)
 color_score_max.append(file_color_score_max)
 color_pixelfrac_mean.append(file_color_pixelfrac_mean)
 color_pixelfrac_min.append(file_color_pixelfrac_min)
 color_pixelfrac_max.append(file_color_pixelfrac_max)
 image_properties = pd.concat([pd.DataFrame({'ImageId':imageid}),pd.DataFrame({'metadata_color_pixelfrac_mean':color_pixelfrac_mean}), pd.DataFrame({'metadata_color_pixelfrac_min':color_pixelfrac_min}),pd.DataFrame({'metadata_color_pixelfrac_max':color_pixelfrac_max}),pd.DataFrame({'metadata_color_score_mean':color_score_mean}),pd.DataFrame({'metadata_color_score_min':color_score_min}),pd.DataFrame({'metadata_color_score_max':color_score_max})],axis=1)
 image_properties['PetID'] = image_properties['ImageId'].str.split('-' ).str[0]
image_properties['metadata_color_pixelfrac_mean'] = image_properties.groupby(['PetID'])['metadata_color_pixelfrac_mean'].transform('mean')
 image_properties['metadata_color_pixelfrac_min'] = image_properties.groupby(['PetID'])['metadata_color_pixelfrac_min'].transform(min)
 image_properties['metadata_color_pixelfrac_max'] = image_properties.groupby(['PetID'])['metadata_color_pixelfrac_max'].transform(max)
 image_properties['metadata_color_score_mean'] = image_properties.groupby(['PetID'])['metadata_color_score_mean'].transform('mean')
 image_properties['metadata_color_score_min'] = image_properties.groupby(['PetID'])['metadata_color_score_min'].transform(min)
 image_properties['metadata_color_score_max'] = image_properties.groupby(['PetID'])['metadata_color_score_max'].transform(max)
 image_properties=image_properties.drop_duplicates('PetID')
 image_properties = image_properties.drop(['ImageId'], 1 )<load_from_csv>
train_len = len(train_features.values)
 test_len = len(test_features.values)
 for col in(GENES + CELLS):
epsilon = 1e-6
train_1col = train_features[col].values.reshape(train_len, 1)
 test_1col = test_features[col].values.reshape(test_len, 1)
 np_arr_total = np.vstack(( train_1col, test_1col))
res = np_arr_total[:,0].argsort().argsort()
res =(res/res.max() -0.5)*2
res = np.clip(res,-1+epsilon,1-epsilon)
 res = erfinv(res)
 np_arr_total_r = np_arr_total
np_arr_total_r[:,0] = res * np.sqrt(2)
 train_features[col] =(np_arr_total_r[:train_len] ).reshape(1, train_len)[0]
test_features[col] =(np_arr_total_r[train_len:] ).reshape(1, test_len)[0]
train = pd.read_csv('.. /input/train/train.csv')
 test = pd.read_csv('.. /input/test/test.csv')
 sample_submission = pd.read_csv('.. /input/test/sample_submission.csv')
 breed =pd.read_csv('.. /input/breed_labels.csv',usecols=["BreedID", "BreedName"])
 color =pd.read_csv('.. /input/color_labels.csv')
 state =pd.read_csv('.. /input/state_labels.csv')
 train =(pd.merge(train, breed.rename(columns={"BreedName": "BreedName1"}), how='left', left_on=['Breed1'], right_on = ['BreedID'] ).drop('BreedID', axis=1))
train =(pd.merge(train, breed.rename(columns={"BreedName": "BreedName2"}), how='left', left_on=['Breed2'], right_on = ['BreedID'] ).drop('BreedID', axis=1))
train =(pd.merge(train, color.rename(columns={"ColorName": "ColorName1"}), how='left', left_on=['Color1'], right_on = ['ColorID'] ).drop('ColorID', axis=1))
train =(pd.merge(train, color.rename(columns={"ColorName": "ColorName2"}), how='left', left_on=['Color2'], right_on = ['ColorID'] ).drop('ColorID', axis=1))
train =(pd.merge(train, color.rename(columns={"ColorName": "ColorName3"}), how='left', left_on=['Color3'], right_on = ['ColorID'] ).drop('ColorID', axis=1))
train =(pd.merge(train, state, how='inner', left_on=['State'], right_on = ['StateID'] ).drop('StateID', axis=1))
train =(pd.merge(train, sentimental_analysis, how='left', left_on=['PetID'], right_on = ['PetID']))
train =(pd.merge(train, image_properties, how='left', left_on=['PetID'], right_on = ['PetID']))
train =(pd.merge(train, image_labelannot, how='left', left_on=['PetID'], right_on = ['PetID']))
train =(pd.merge(train, image_moments, how='left', left_on=['PetID'], right_on = ['PetID']))
train =(pd.merge(train, image_quality, how='left', left_on=['PetID'], right_on = ['PetID'])) 
<data_type_conversions>
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=42 )
train['L_Color1'] =(pd.isnull(train['ColorName3'])& pd.isnull(train['ColorName2'])& pd.notnull(train['ColorName1'])).astype(int)
 train['L_Color2'] =(pd.isnull(train['ColorName3'])& pd.notnull(train['ColorName2'])& pd.notnull(train['ColorName1'])).astype(int)
 train['L_Color3'] =(pd.notnull(train['ColorName3'])& pd.notnull(train['ColorName2'])& pd.notnull(train['ColorName1'])).astype(int)
 train['L_Breed1'] =(pd.isnull(train['BreedName2'])& pd.notnull(train['BreedName1'])).astype(int)
 train['L_Breed2'] =(pd.notnull(train['BreedName2'])& pd.notnull(train['BreedName1'])).astype(int)
 train['Name_Length']=train['Name'].str.len()
train['L_Name_missing'] =(pd.isnull(train['Name'])).astype(int)
 train['L_Breed1_Siamese'] =(train['BreedName1']=='Siamese' ).astype(int)
 train['L_Breed1_Persian']=(train['BreedName1']=='Persian' ).astype(int)
 train['L_Breed1_Labrador_Retriever']=(train['BreedName1']=='Labrador Retriever' ).astype(int)
 train['L_Breed1_Terrier']=(train['BreedName1']=='Terrier' ).astype(int)
 train['L_Breed1_Golden_Retriever ']=(train['BreedName1']=='Golden Retriever' ).astype(int)
 train['Description_Length']=train['Description'].str.len()
train['L_Fee_Free'] =(train['Fee']==0 ).astype(int)
 pets_total = train.groupby(['RescuerID'] ).size().reset_index(name='N_pets_total')
 train= pd.merge(train, pets_total, left_on='RescuerID', right_on='RescuerID', how='inner')
 train.count()
train['L_NoPhoto'] =(train['PhotoAmt']==0 ).astype(int)
 train['L_NoVideo'] =(train['VideoAmt']==0 ).astype(int)
 train['Log_Age']= np.log(train.Age+1)
 train['L_scoreneg'] =(train['sentiment_document_score']<0 ).astype(int)
 train.loc[train['Quantity'] > 5, 'Quantity'] = 5<feature_engineering>
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 =(PCA(n_components=n_comp_GENES, random_state=42 ).fit_transform(data[GENES]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp_GENES)])
 test2 = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp_GENES)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )
train['Description'] =train['Description'].fillna("<MISSING>")
 train['Description'] = train['Description'].str.replace('\d+', '')
 train['Description'] = train['Description'].str.lower()
train["Description"] = train['Description'].str.replace('[^\w\s]','')
 stop = stopwords.words('english')
 pat = r'\b(?:{})\b'.format('|'.join(stop))
train['Description'] = train['Description'].str.replace(pat, '')
 train['Description'] = train['Description'].str.replace(r'\s+', ' ')
 train['Description'] = train['Description'].astype(str ).str.split()
porter_stemmer = PorterStemmer()
train['Description']=train['Description'].apply(lambda x : [porter_stemmer.stem(y)for y in x])
 train['Description']=train['Description'].apply(lambda x : " ".join(x))
def get_top_n_words(corpus, n=None):
vec = CountVectorizer().fit(corpus)
 bag_of_words = vec.transform(corpus)
 sum_words = bag_of_words.sum(axis=0)
 words_freq = [(word, sum_words[0, idx])for word, idx in vec.vocabulary_.items() ]
words_freq =sorted(words_freq, key = lambda x: x[1], reverse=True)
 test=pd.DataFrame(words_freq[:n], columns=['words','freq'])
 sns.barplot(x='words', y='freq', data=test)
 get_top_n_words(train['Description'],10)
 svd_ = TruncatedSVD(
 n_components=5, random_state=1337)
 nmf_ = NMF(
 n_components=5, random_state=1337)
 tfidf_col = TfidfVectorizer().fit_transform(train['Description'])
 svd_col = svd_.fit_transform(tfidf_col)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('SVD_')
 nmf_col = nmf_.fit_transform(tfidf_col)
 nmf_col = pd.DataFrame(nmf_col)
 nmf_col = nmf_col.add_prefix('NMF_')
 train = pd.concat([train,nmf_col,svd_col],axis=1)
<filter>
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 =(PCA(n_components=n_comp_CELLS, random_state=42 ).fit_transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp_CELLS)])
 test2 = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp_CELLS)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )
train.loc[train['FurLength'] == 0]
train.loc[train['Health'] == 0 ]<sort_values>
var_thresh = VarianceThreshold(VarianceThreshold_for_FS)
 data = train_features.append(test_features)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
 train_features.shape
Top4 =rescuer_analysis.sort_values('counts',ascending=False ).head(4)
 Top4<groupby>
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target = train[train_targets_scored.columns]
train.groupby('AdoptionSpeed')['PhotoAmt'].mean()<groupby>
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1 )
train.groupby('AdoptionSpeed')['sentiment_document_score'].mean()<groupby>
target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()
train.groupby('AdoptionSpeed')['sentiment_document_magnitude'].mean()<string_transform>
folds = train.copy()
mskf = MultilabelStratifiedKFold(n_splits=7)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=target)) :
folds.loc[v_idx, 'kfold'] = int(f)
 folds['kfold'] = folds['kfold'].astype(int)
 folds
train_dog=train.loc[train['Type'] == 1]
train_cat=train.loc[train['Type'] == 2]
text_cat = ','.join(str(v)for v in train_cat.Name)
 text_dog = ','.join(str(v)for v in train_dog.Name)
 print("There are {} words in the combination of all Name.".format(len(text_dog)) )<drop_column>
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct
train_analysis = train.drop(["Name","Description","BreedName2","ColorName3",'Name','Breed1','Breed2','RescuerID','Description',
'BreedName1','Color1', 'Color2', 'Color3','Age','State','ImageId'],axis=1)
 for col in ['sentiment_document_score', 'sentiment_document_magnitude','metadata_topicality_max','metadata_topicality_mean','metadata_topicality_min',
'metadata_topicality_0_mean','metadata_topicality_0_max','metadata_topicality_0_min','L_metadata_0_dog_sum',
'L_metadata_0_cat_sum','L_metadata_any_dog_sum','L_metadata_any_cat_sum','pixel_mean','pixel_min','pixel_max','pixel_sum',
'blur_min','blur_max','blur_sum','blur_mean','metadata_color_pixelfrac_mean','metadata_color_pixelfrac_min',
'metadata_color_pixelfrac_max','metadata_color_score_mean','metadata_color_score_min','metadata_color_score_max',
'Description_Length','Name_Length','huMoments0','huMoments1','huMoments2','huMoments3','huMoments4','huMoments5','huMoments6']:
train_analysis[col].fillna(( train_analysis[col].median()), inplace=True)
 train_analysis["ColorName2"].fillna("No Color", inplace = True )<categorify>
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
train_analysis = pd.concat([train_analysis.drop('StateName', axis=1),pd.get_dummies(train_analysis['StateName'], prefix='State')], axis=1)
 col=['ColorName1','ColorName2','Health', 'Gender', 'Dewormed','Type','MaturitySize', 'Sterilized','Vaccinated','FurLength']
for i in col:
train_analysis = pd.concat([train_analysis.drop(i, axis=1),pd.get_dummies(train_analysis[i], prefix=i)], axis=1 )<compute_test_metric>
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):

 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_ratings = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_ratings)]
for j in range(num_ratings)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):

 if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):

 rater_a = y
rater_b = y_pred
min_rating=None
max_rating=None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(min(rater_a), min(rater_b))
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b,
min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(hist_rater_a[i] * hist_rater_b[j]
/ num_scored_items)
 d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator)
<compute_test_metric>
def process_data(data):
data = pd.get_dummies(data, columns=['cp_time','cp_dose'])
 return data
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = quadratic_weighted_kappa(y, X_p)
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.6, 1.7, 2.6, 3.6]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']
def rmse(actual, predicted):
return sqrt(mean_squared_error(actual, predicted))<compute_test_metric>
feature_cols = [c for c in process_data(folds ).columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold','sig_id']]
len(feature_cols )
def evaluate(y_pred, y_true):
cohen_kappa= cohen_kappa_score(y_true, y_pred)
 accuracy=accuracy_score(y_true,y_pred)
 f1=f1_score(y_true,y_pred,average='micro')
 classification=classification_report(y_true,y_pred)
 cm = confusion_matrix(y_true, y_pred)
 plt.figure(figsize=(20,6))
sns.heatmap(cm, annot=True)
 plt.title('Confusion matrix')
 plt.figure(figsize =(5,4))
plt.show()
print('Cohen Kappa: {:0.2f}.'.format(cohen_kappa))
print('Accuracy Score: {:0.2f}%.'.format(accuracy))
print('F1 Score: {:0.2f}%.'.format(f1)) 
<drop_column>
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 22
BATCH_SIZE = 128
LEARNING_RATE = 1e-3
WEIGHT_DECAY = 1e-5
NFOLDS = 7
EARLY_STOPPING_STEPS = 10
EARLY_STOP = False
num_features=len(feature_cols)
 num_targets=len(target_cols)
ids=train_analysis[['PetID']]
train_analysis=train_analysis.drop(['PetID','State_Labuan'],axis=1 )<split>
class Model(nn.Module):
def recalibrate_layer(self, layer):
if(torch.isnan(layer.weight_v ).sum() > 0):
layer.weight_v = torch.nn.Parameter(torch.where(torch.isnan(layer.weight_v), torch.zeros_like(layer.weight_v), layer.weight_v))
layer.weight_v = torch.nn.Parameter(layer.weight_v + 1e-7)
 if(torch.isnan(layer.weight ).sum() > 0):
layer.weight = torch.where(torch.isnan(layer.weight), torch.zeros_like(layer.weight), layer.weight)
 layer.weight += 1e-7
def __init__(self, num_features, num_targets, hidden_size_1d, dropout_rate_1d):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size_1d[0]))
self.batch_norm2 = nn.BatchNorm1d(hidden_size_1d[0])
 self.dropout2 = nn.Dropout(dropout_rate_1d[0])
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size_1d[0], hidden_size_1d[1]))
self.batch_norm3 = nn.BatchNorm1d(hidden_size_1d[1])
 self.dropout3 = nn.Dropout(dropout_rate_1d[1])
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size_1d[1], num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = F.leaky_relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.leaky_relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x
class LabelSmoothingLoss(nn.Module):
def __init__(self, classes, smoothing=0.0, dim=-1):
super(LabelSmoothingLoss, self ).__init__()
self.confidence = 1.0 - smoothing
self.smoothing = smoothing
self.cls = classes
self.dim = dim
def forward(self, pred, target):
pred = pred.log_softmax(dim=self.dim)
 with torch.no_grad() :
true_dist = torch.zeros_like(pred)
 true_dist.fill_(self.smoothing /(self.cls - 1))
true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence)
 return torch.mean(torch.sum(-true_dist * pred, dim=self.dim))
X, y = train_analysis.loc[:, train_analysis.columns != 'AdoptionSpeed'], train_analysis['AdoptionSpeed']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=1, stratify=y)
<split>
def run_training(fold, seed, hidden_size, dropout_rate):
seed_everything(seed)
 train = process_data(folds)
 test_ = process_data(test)
 trn_idx = train[train['kfold'] != fold].index
val_idx = train[train['kfold'] == fold].index
train_df = train[train['kfold'] != fold].reset_index(drop=True)
 valid_df = train[train['kfold'] == fold].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size_1d=hidden_size,
dropout_rate_1d=dropout_rate, 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1e-2, epochs=EPOCHS, steps_per_epoch=len(trainloader))
loss_fn = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing =0.001)
 early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
best_loss = np.inf
for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer,scheduler, loss_tr, trainloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}")
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"FOLD{fold}_.pth")
 elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size_1d=hidden_size,
dropout_rate_1d=dropout_rate, 
)
 model.load_state_dict(torch.load(f"FOLD{fold}_.pth"))
model.to(DEVICE)
 predictions = np.zeros(( len(test_), target.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions
X_train =X_train.loc[:,features_selection]
X_test = X_test.loc[:,features_selection]<train_model>
def run_network(fold, seed, NFOLDS):
hidden_size_arr = [[1000, 1500], [800, 1400], [1500, 1000], [1500, 1500]]
dropout_rate_arr = [[1/6, 0.25], [2/15, 7/30], [0.25, 1/6], [0.25, 0.25]]
weight_rate_arr = [[0.3], [0.3], [0.2], [0.2]]
oof_this_fold = np.zeros(( len(train), len(target_cols)))
 predictions_this_fold = np.zeros(( len(test), len(target_cols)))
 for this_network_i in range(len(hidden_size_arr)) :
print("network:",this_network_i)
 oof_, pred_ = run_training(fold, seed, hidden_size = hidden_size_arr[this_network_i], dropout_rate = dropout_rate_arr[this_network_i])
 if np.amax(oof_)== 0 and np.amin(oof_)== 0:
print("max network: ", np.amax(oof_))
print("min network: ", np.amin(oof_))
predictions_this_fold += pred_ * weight_rate_arr[this_network_i]
oof_this_fold += oof_ * weight_rate_arr[this_network_i]
return oof_this_fold, predictions_this_fold
print('Original dataset shape %s' % Counter(y_train))
sampling_strategy= {4: 3148, 2: 3028, 3: 2444, 1: 2317, 0: 1000}
ros = RandomOverSampler(sampling_strategy= sampling_strategy, random_state=42)
 X_res, y_res = ros.fit_resample(X_train, y_train)
 print('Resampled dataset shape %s' % Counter(y_res))<train_model>
def run_k_fold(NFOLDS, seed):
oof_this_seed = np.zeros(( len(train), len(target_cols)))
 predictions_this_seed = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
print("fold:", fold)
 oof_, pred_ = run_network(fold, seed, NFOLDS)
 if np.amax(oof_)== 0 and np.amin(oof_)== 0:
print("max fold:", np.amax(oof_))
print("min fold:", np.amin(oof_))
predictions_this_seed += pred_ / NFOLDS
oof_this_seed += oof_
return oof_this_seed, predictions_this_seed
def cross_val(model,X_train,y_train):
X = X_train
y = y_train
coeff = np.empty(( 1,4))
cv_scores=[]
fold=1
skf = StratifiedKFold(n_splits=5, random_state=42, shuffle=True)
 print(skf.get_n_splits(X, y))
for train_index, val_index in skf.split(X, y):
xtrain, xvalid = X[train_index], X[val_index]
ytrain, yvalid = y[train_index], y[val_index]
model.fit(
 xtrain, ytrain,
eval_set=[(xvalid, yvalid)],
verbose=100,
early_stopping_rounds=100 
)
 valid_preds = model.predict(xvalid, num_iteration=model.best_iteration_)
 yvalid = np.array(yvalid ).tolist()
optR = OptimizedRounder()
optR.fit(valid_preds, yvalid)
 coefficients = optR.coefficients()
valid_p = optR.predict(valid_preds, coefficients)
 scr = quadratic_weighted_kappa(yvalid, valid_p)
 cv_scores.append(scr)
 print("QWK = {}.Coef = {}".format(scr, coefficients))
coefficients.reshape(( 4, 1))
coeff = np.vstack([coeff, coefficients])
 fold += 1
coeff = np.delete(coeff,(0), axis=0)
 global coefficient_mean
coefficient_mean = coeff.mean(axis=0)
 print("Coef Mean ={}".format(coefficient_mean))<find_best_model_class>
from pytz import timezone
lgb_params = {
'boosting_type': 'gbdt',
'objective': 'regression',
'learning_rate': 0.005,
'subsample':.8,
'colsample_bytree': 0.8,
'min_split_gain': 0.006,
'min_child_samples': 150,
'min_child_weight': 0.1,
'n_estimators': 1000,
'num_leaves': 80,
'silent': -1,
'verbose': -1,
'max_depth': 11,
'random_state': 2018
}
lgb_model = lgb.LGBMRegressor(**lgb_params)
 cross_val(lgb_model,X_res,y_res)
 y_pred=lgb_model.predict(X_train.values)
<compute_test_metric>
datetime.now(timezone('America/Toronto'))
print(coefficient_mean )<create_dataframe>
SEED = [0, 1, 2, 3, 4, 5, 6]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
print("seed:", seed)
 oof_, predictions_ = run_k_fold(NFOLDS, seed)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 train[target_cols] = oof
test[target_cols] = predictions
optR=OptimizedRounder()
y_true = pd.DataFrame(y_train)
 y_true.reset_index(inplace=True,drop=False)
 predictions = optR.predict(y_pred, coefficient_mean ).astype(int)
 pred_lgb = pd.concat([pd.DataFrame(y_pred),y_true,pd.DataFrame(predictions)],axis=1,ignore_index=True)
 pred_lgb.columns = ['y_pred','index','y_true','y_pred_class']
pred_lgb['y_pred_class'].value_counts()<compute_test_metric>
datetime.now(timezone('America/Toronto'))
evaluate(pred_lgb['y_pred_class'].tolist() , pred_lgb['y_true'].tolist() )<prepare_output>
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score_ = log_loss(y_true[:, i], y_pred[:, i])
 score += score_ / target.shape[1]
print("CV log_loss: ", score )
y_pred=lgb_model.predict(X_test.values)
 predictions = optR.predict(y_pred, coefficient_mean ).astype(int)
 y_test =pd.DataFrame(y_test)
 y_test.reset_index(drop=False, inplace=True)
 pred_lgb = pd.concat([pd.DataFrame(y_pred),y_test,pd.DataFrame(predictions)],axis=1,ignore_index=True)
 pred_lgb.columns = ['y_pred','index','y_true','y_pred_class']
pred_lgb['y_pred_class'].value_counts()<compute_test_metric>
pd.DataFrame(
 {
"1st hidden layer": [1500, 1500, 1000, 800],
"2nd hidden layer": [1000, 1500, 1500, 1400],
"1st dropout rate": [0.25, 0.25, 1/6, 2/15],
"2nd dropout rate": [1/6, 0.25, 0.25, 7/30],
"Kaggle Public Score": [0.01841, 0.01841, 0.01840, 0.01840],
"Ratio": [0.3, 0.3, 0.2, 0.2]
} 
)
evaluate(pred_lgb['y_pred_class'].tolist() , pred_lgb['y_true'].tolist() )<train_model>
sub = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left')
lgb_params = {
'boosting_type': 'gbdt',
'objective': 'regression',
'learning_rate': 0.005,
'subsample':.8,
'colsample_bytree': 0.8,
'min_split_gain': 0.006,
'min_child_samples': 150,
'min_child_weight': 0.1,
'n_estimators': 1000,
'num_leaves': 80,
'silent': -1,
'verbose': -1,
'max_depth': 11,
'random_state': 20
}
lgb_model2 = lgb.LGBMRegressor(**lgb_params)
 lgb_model2.fit(X_res, y_res)
 y_pred2=lgb_model2.predict(X_test )<predict_on_test>
np.isnan(sub.values[:, 1:].astype('float64')).any()
optR=OptimizedRounder()
predictions2 = optR.predict(y_pred2, coefficient_mean ).astype(int)
 pred_lgb2 = pd.concat([pd.DataFrame(y_pred2),y_true,pd.DataFrame(predictions2)],axis=1,ignore_index=True)
 pred_lgb2.columns = ['y_pred_lgb2','index','y_true','y_pred_class']
pred_lgb2['y_pred_class'].value_counts()<train_model>
sub = sub.fillna(0)
 sub.to_csv('submission.csv', index=False )