Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<rename_columns>
!pip install --no-index --find-links /kaggle/input/pytorchtabnet/pytorch_tabnet-2.0.0-py3-none-any.whl pytorch-tabnet
train_feats = train_feats.reset_index()
train_feats.rename({'index': 'PetID'}, axis='columns', inplace=True)
 test_feats = test_feats.reset_index()
test_feats.rename({'index': 'PetID'}, axis='columns', inplace=True )<concatenate>
warnings.filterwarnings("ignore")
 os.environ["CUDA_LAUNCH_BLOCKING"] = '1'
all_ids = pd.concat([train, test], axis=0, ignore_index=True, sort=False)[['PetID']]
all_ids.shape<concatenate>
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 train_drug = pd.read_csv('.. /input/lish-moa/train_drug.csv' )
n_components = 32
svd_ = TruncatedSVD(n_components=n_components, random_state=1337)
 features_df = pd.concat([train_feats, test_feats], axis=0)
 features = features_df[[f'pic_{i}' for i in range(256)]].values
svd_col = svd_.fit_transform(features)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('IMG_SVD_')
 img_features = pd.concat([all_ids, svd_col], axis=1 )<load_from_csv>
train_features2=train_features.copy()
test_features2=test_features.copy()
labels_breed = pd.read_csv('.. /input/petfinder-adoption-prediction/breed_labels.csv')
 labels_state = pd.read_csv('.. /input/petfinder-adoption-prediction/color_labels.csv')
 labels_color = pd.read_csv('.. /input/petfinder-adoption-prediction/state_labels.csv' )<define_variables>
print('Training Features Samples')
 display(train_features.head(3))
print('Training Features Description')
 display(train_features.describe() )
train_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_images/*.jpg'))
train_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_metadata/*.json'))
train_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/train_sentiment/*.json'))
print(f'num of train images files: {len(train_image_files)}')
 print(f'num of train metadata files: {len(train_metadata_files)}')
 print(f'num of train sentiment files: {len(train_sentiment_files)}')
 test_image_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_images/*.jpg'))
test_metadata_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_metadata/*.json'))
test_sentiment_files = sorted(glob.glob('.. /input/petfinder-adoption-prediction/test_sentiment/*.json'))
print(f'num of test images files: {len(test_image_files)}')
 print(f'num of test metadata files: {len(test_metadata_files)}')
 print(f'num of test sentiment files: {len(test_sentiment_files)}' )<load_pretrained>
print('Training Features Samples')
 display(test_features.head(3))
print('Training Features Description')
 display(test_features.describe() )
class PetFinderParser(object):
def __init__(self, debug=False):
self.debug = debug
self.sentence_sep = ' '
self.extract_sentiment_text = False
def open_json_file(self, filename):
with open(filename, 'r', encoding='utf-8')as f:
json_file = json.load(f)
 return json_file
def parse_sentiment_file(self, file):

 file_sentiment = file['documentSentiment']
file_entities = [x['name'] for x in file['entities']]
file_entities = self.sentence_sep.join(file_entities)
 file_sentences_sentiment = [x['sentiment'] for x in file['sentences']]
file_sentences_sentiment = pd.DataFrame.from_dict(
 file_sentences_sentiment, orient='columns')
 file_sentences_sentiment_df = pd.DataFrame(
 {
'magnitude_sum': file_sentences_sentiment['magnitude'].sum(axis=0),
'score_sum': file_sentences_sentiment['score'].sum(axis=0),
'magnitude_mean': file_sentences_sentiment['magnitude'].mean(axis=0),
'score_mean': file_sentences_sentiment['score'].mean(axis=0),
'magnitude_var': file_sentences_sentiment['magnitude'].var(axis=0),
'score_var': file_sentences_sentiment['score'].var(axis=0),
}, index=[0] 
)
 df_sentiment = pd.DataFrame.from_dict(file_sentiment, orient='index' ).T
df_sentiment = pd.concat([df_sentiment, file_sentences_sentiment_df], axis=1)
 df_sentiment['entities'] = file_entities
df_sentiment = df_sentiment.add_prefix('sentiment_')
 return df_sentiment
def parse_metadata_file(self, file):

 file_keys = list(file.keys())
 if 'labelAnnotations' in file_keys:
file_annots = file['labelAnnotations']
file_top_score = np.asarray([x['score'] for x in file_annots] ).mean()
file_top_desc = [x['description'] for x in file_annots]
else:
file_top_score = np.nan
file_top_desc = ['']
file_colors = file['imagePropertiesAnnotation']['dominantColors']['colors']
file_crops = file['cropHintsAnnotation']['cropHints']
file_color_score = np.asarray([x['score'] for x in file_colors] ).mean()
file_color_pixelfrac = np.asarray([x['pixelFraction'] for x in file_colors] ).mean()
file_crop_conf = np.asarray([x['confidence'] for x in file_crops] ).mean()
if 'importanceFraction' in file_crops[0].keys() :
file_crop_importance = np.asarray([x['importanceFraction'] for x in file_crops] ).mean()
else:
file_crop_importance = np.nan
df_metadata = {
'annots_score': file_top_score,
'color_score': file_color_score,
'color_pixelfrac': file_color_pixelfrac,
'crop_conf': file_crop_conf,
'crop_importance': file_crop_importance,
'annots_top_desc': self.sentence_sep.join(file_top_desc)
 }
df_metadata = pd.DataFrame.from_dict(df_metadata, orient='index' ).T
df_metadata = df_metadata.add_prefix('metadata_')
 return df_metadata
def extract_additional_features(pet_id, mode='train'):
sentiment_filename = f'.. /input/petfinder-adoption-prediction/{mode}_sentiment/{pet_id}.json'
try:
sentiment_file = pet_parser.open_json_file(sentiment_filename)
 df_sentiment = pet_parser.parse_sentiment_file(sentiment_file)
 df_sentiment['PetID'] = pet_id
except FileNotFoundError:
df_sentiment = []
dfs_metadata = []
metadata_filenames = sorted(glob.glob(f'.. /input/petfinder-adoption-prediction/{mode}_metadata/{pet_id}*.json'))
if len(metadata_filenames)> 0:
for f in metadata_filenames:
metadata_file = pet_parser.open_json_file(f)
 df_metadata = pet_parser.parse_metadata_file(metadata_file)
 df_metadata['PetID'] = pet_id
dfs_metadata.append(df_metadata)
 dfs_metadata = pd.concat(dfs_metadata, ignore_index=True, sort=False)
 dfs = [df_sentiment, dfs_metadata]
return dfs
pet_parser = PetFinderParser()<categorify>
train_missing = train_features.isnull().sum().sum()
test_missing = test_features.isnull().sum().sum()
if train_missing & test_missing == 0:
print("Train and Test Files have no missing values")
 else: print("Train and Test Files have missing values" )
aggregates = ['sum', 'mean', 'var']
sent_agg = ['sum']
train_metadata_desc = train_dfs_metadata.groupby(['PetID'])['metadata_annots_top_desc'].unique()
train_metadata_desc = train_metadata_desc.reset_index()
train_metadata_desc[
'metadata_annots_top_desc'] = train_metadata_desc[
'metadata_annots_top_desc'].apply(lambda x: ' '.join(x))
prefix = 'metadata'
train_metadata_gr = train_dfs_metadata.drop(['metadata_annots_top_desc'], axis=1)
 for i in train_metadata_gr.columns:
if 'PetID' not in i:
train_metadata_gr[i] = train_metadata_gr[i].astype(float)
 train_metadata_gr = train_metadata_gr.groupby(['PetID'] ).agg(aggregates)
 train_metadata_gr.columns = pd.Index([f'{c[0]}_{c[1].upper() }' for c in train_metadata_gr.columns.tolist() ])
 train_metadata_gr = train_metadata_gr.reset_index()
train_sentiment_desc = train_dfs_sentiment.groupby(['PetID'])['sentiment_entities'].unique()
train_sentiment_desc = train_sentiment_desc.reset_index()
train_sentiment_desc[
'sentiment_entities'] = train_sentiment_desc[
'sentiment_entities'].apply(lambda x: ' '.join(x))
prefix = 'sentiment'
train_sentiment_gr = train_dfs_sentiment.drop(['sentiment_entities'], axis=1)
 for i in train_sentiment_gr.columns:
if 'PetID' not in i:
train_sentiment_gr[i] = train_sentiment_gr[i].astype(float)
 train_sentiment_gr = train_sentiment_gr.groupby(['PetID'] ).agg(sent_agg)
 train_sentiment_gr.columns = pd.Index([f'{c[0]}' for c in train_sentiment_gr.columns.tolist() ])
 train_sentiment_gr = train_sentiment_gr.reset_index()
test_metadata_desc = test_dfs_metadata.groupby(['PetID'])['metadata_annots_top_desc'].unique()
test_metadata_desc = test_metadata_desc.reset_index()
test_metadata_desc[
'metadata_annots_top_desc'] = test_metadata_desc[
'metadata_annots_top_desc'].apply(lambda x: ' '.join(x))
prefix = 'metadata'
test_metadata_gr = test_dfs_metadata.drop(['metadata_annots_top_desc'], axis=1)
 for i in test_metadata_gr.columns:
if 'PetID' not in i:
test_metadata_gr[i] = test_metadata_gr[i].astype(float)
 test_metadata_gr = test_metadata_gr.groupby(['PetID'] ).agg(aggregates)
 test_metadata_gr.columns = pd.Index([f'{c[0]}_{c[1].upper() }' for c in test_metadata_gr.columns.tolist() ])
 test_metadata_gr = test_metadata_gr.reset_index()
test_sentiment_desc = test_dfs_sentiment.groupby(['PetID'])['sentiment_entities'].unique()
test_sentiment_desc = test_sentiment_desc.reset_index()
test_sentiment_desc[
'sentiment_entities'] = test_sentiment_desc[
'sentiment_entities'].apply(lambda x: ' '.join(x))
prefix = 'sentiment'
test_sentiment_gr = test_dfs_sentiment.drop(['sentiment_entities'], axis=1)
 for i in test_sentiment_gr.columns:
if 'PetID' not in i:
test_sentiment_gr[i] = test_sentiment_gr[i].astype(float)
 test_sentiment_gr = test_sentiment_gr.groupby(['PetID'] ).agg(sent_agg)
 test_sentiment_gr.columns = pd.Index([f'{c[0]}' for c in test_sentiment_gr.columns.tolist() ])
 test_sentiment_gr = test_sentiment_gr.reset_index()<merge>
plt.figure(figsize=(10,5))
sns.countplot(train_features['cp_time'], color='violet')
 plt.title("Train: Treatment duration ", fontsize=15, weight='bold')
 plt.show()
train_proc = train.copy()
train_proc = train_proc.merge(
 train_sentiment_gr, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_metadata_gr, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_metadata_desc, how='left', on='PetID')
 train_proc = train_proc.merge(
 train_sentiment_desc, how='left', on='PetID')
 test_proc = test.copy()
test_proc = test_proc.merge(
 test_sentiment_gr, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_metadata_gr, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_metadata_desc, how='left', on='PetID')
 test_proc = test_proc.merge(
 test_sentiment_desc, how='left', on='PetID')
 print(train_proc.shape, test_proc.shape)
 assert train_proc.shape[0] == train.shape[0]
assert test_proc.shape[0] == test.shape[0]<merge>
GENES = [g for g in train_features.columns if g.startswith("g-")]
print(f"Number of gene features: {len(GENES)}")
 CELLS = [c for c in train_features.columns if c.startswith("c-")]
print(f"Number of cell features: {len(CELLS)}" )
train_breed_main = train_proc[['Breed1']].merge(
 labels_breed, how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))
train_breed_main = train_breed_main.iloc[:, 2:]
train_breed_main = train_breed_main.add_prefix('main_breed_')
 train_breed_second = train_proc[['Breed2']].merge(
 labels_breed, how='left',
left_on='Breed2', right_on='BreedID',
suffixes=('', '_second_breed'))
train_breed_second = train_breed_second.iloc[:, 2:]
train_breed_second = train_breed_second.add_prefix('second_breed_')
 train_proc = pd.concat(
 [train_proc, train_breed_main, train_breed_second], axis=1)
 test_breed_main = test_proc[['Breed1']].merge(
 labels_breed, how='left',
left_on='Breed1', right_on='BreedID',
suffixes=('', '_main_breed'))
test_breed_main = test_breed_main.iloc[:, 2:]
test_breed_main = test_breed_main.add_prefix('main_breed_')
 test_breed_second = test_proc[['Breed2']].merge(
 labels_breed, how='left',
left_on='Breed2', right_on='BreedID',
suffixes=('', '_second_breed'))
test_breed_second = test_breed_second.iloc[:, 2:]
test_breed_second = test_breed_second.add_prefix('second_breed_')
 test_proc = pd.concat(
 [test_proc, test_breed_main, test_breed_second], axis=1)
 print(train_proc.shape, test_proc.shape )<concatenate>
def treated(a):
treated= a[a['cp_type']=='trt_cp']
return treated
X = pd.concat([train_proc, test_proc], ignore_index=True, sort=False )<define_variables>
target_cols_scored=[col for col in train_targets_scored.columns if col not in ['sig_id']]
target_cols_nonscored=[col for col in train_targets_nonscored.columns if col not in ['sig_id']]
X_temp = X.copy()
text_columns = ['Description', 'metadata_annots_top_desc', 'sentiment_entities']
categorical_columns = ['main_breed_BreedName', 'second_breed_BreedName']
to_drop_columns = ['PetID', 'Name', 'RescuerID']<feature_engineering>
all_targets=train_targets_scored.merge(train_targets_nonscored, on='sig_id',how='left')
 corrs(all_targets[target_cols_nonscored+target_cols_scored],'Target 1', 'Target 2',rows=8 )
X_temp['name_length'] = X['Name'].apply(lambda x: len(x)if type(x)is str else 0)
<merge>
plt.figure(figsize=(10,5))
sns.countplot(test_features['cp_time'], color='violet')
 plt.title("Train: Treatment duration ", fontsize=15, weight='bold')
 plt.show()
rescuer_count = X.groupby(['RescuerID'])['PetID'].count().reset_index()
rescuer_count.columns = ['RescuerID', 'RescuerID_COUNT']
X_temp = X_temp.merge(rescuer_count, how='left', on='RescuerID' )<feature_engineering>
print('First observation:')
 print(f"Number of rows of the Control vehicle is {len(train_features[train_features['cp_type']=='ctl_vehicle'])}")
 print(f"Number of rows of the Drug cacb2b860 is {train_drug.drug_id.value_counts() [0]}" )
for i in categorical_columns:
X_temp.loc[:, i] = pd.factorize(X_temp.loc[:, i])[0]<feature_engineering>
drug_count = train_drug[['drug_id']].value_counts().to_frame()
drug_count = drug_count.rename(columns={0:'drug_count'})
 drug_count2=drug_count['drug_count'].value_counts().to_frame().reset_index()
drug_count2=drug_count2.rename(columns={'index': 'Samples per drug', 'drug_count':'Number of Drugs'})
 drug_count2[:12]
X_text = X_temp[text_columns]
for i in X_text.columns:
X_text.loc[:, i] = X_text.loc[:, i].fillna('none' )<categorify>
train_features['target'] = 0
test_features['target'] = 1
X = pd.concat([train_features.loc[:,GENES+CELLS], test_features.loc[:,GENES+CELLS]] ).reset_index(drop=True)
 y = pd.concat([train_features.loc[:, 'target'], test_features.loc[:, 'target']] ).reset_index(drop=True )
X_temp['Length_Description'] = X_text['Description'].map(len)
 X_temp['Length_metadata_annots_top_desc'] = X_text['metadata_annots_top_desc'].map(len)
 X_temp['Lengths_sentiment_entities'] = X_text['sentiment_entities'].map(len )<feature_engineering>
k=5
skf=StratifiedKFold(n_splits=k,random_state=721991,shuffle=True)
 scores=[]
oof_predictions = pd.DataFrame(np.zeros(( X.shape[0], 1)) , columns=['target'])
 feature_importance = pd.DataFrame(np.zeros(( X.shape[1], k)) , columns=[f'Fold_{i}_Importance' for i in range(1, k + 1)], index=X.columns)
 parameters = {
'num_iterations': 500,
'early_stopping_round': 50,
'num_leaves': 2 ** 5,
'learning_rate': 0.05,
'bagging_fraction': 0.9,
'bagging_freq': 1,
'feature_fraction': 0.9,
'feature_fraction_bynode': 0.9,
'lambda_l1': 0,
'lambda_l2': 0,
'max_depth': -1,
'objective': 'binary',
'seed': 721991,
'feature_fraction_seed': 721991,
'bagging_seed': 721991,
'drop_seed': 721991,
'data_random_seed': 721991,
'boosting_type': 'gbdt',
'verbose': 1,
'metric': 'auc',
'n_jobs': -1,
}
print('Running LightGBM Adversarial Validation Model
' +('-' * 45)+ '
')
 for fold,(train_idx,val_idx)in enumerate(skf.split(X, y), 1):
trn_data=lgb.Dataset(X.iloc[train_idx,:],label=y.iloc[train_idx])
 val_data=lgb.Dataset(X.iloc[val_idx,:],label=y.iloc[val_idx])
 model=lgb.train(parameters,trn_data,valid_sets=[trn_data, val_data], verbose_eval=50)
 feature_importance.iloc[:,fold-1]=model.feature_importance(importance_type='gain')
 predictions = model.predict(X.iloc[val_idx, :], num_iteration=model.best_iteration)
 oof_predictions.loc[val_idx, 'target'] = predictions
score=roc_auc_score(y.iloc[val_idx],predictions)
 scores.append(score)
 print(f'
Fold {fold} - ROC AUC Score {score:.6}
')
 oof_score = roc_auc_score(y, oof_predictions)
 print(f'
{"-" * 30}
LightGBM Adversarial Validation Model Mean ROC AUC Score {np.mean(scores):.6} [STD:{np.std(scores):.6}]')
 print(f'LightGBM Adversarial Validation Model OOF ROC AUC Score: {oof_score:.6}
{"-" * 30}')
 plt.figure(figsize=(20, 20))
feature_importance['Mean_Importance'] = feature_importance.sum(axis=1)/ k
feature_importance.sort_values(by='Mean_Importance', inplace=True, ascending=False)
 sns.barplot(x='Mean_Importance', y=feature_importance.index[:50], data=feature_importance[:50])
 plt.xlabel('')
 plt.tick_params(axis='x', labelsize=18)
 plt.tick_params(axis='y', labelsize=18)
 plt.title('LightGBM Adversarial Validation Model Top 50 Feature Importance(Gain)', size=20, pad=20)
 plt.show()
del X, y, oof_predictions, feature_importance, parameters, scores, oof_score
corpus = [nltk.word_tokenize(text)for text in tqdm(X_tempy['Description'].fillna('<MISSING>' ).values)]
def word2vec_agg(description):
vectors = Magnitude(".. /input/magnitude-embeddings/glove.6B.300d.magnitude")
 return(np.mean(vectors.query(description),axis=0))
descriptions = Parallel(n_jobs=6, verbose=1 )(
 delayed(word2vec_agg )(description)for description in corpus)
 n_components=10
svd_ = TruncatedSVD(n_components=n_components, random_state=1337)
 word2vec_cols = svd_.fit_transform(descriptions)
 word2vec_cols = pd.DataFrame(word2vec_cols)
 word2vec_cols = word2vec_cols.add_prefix('SVD_{}_'.format("word2vecs"))
X_temp = pd.concat([X_temp, word2vec_cols], axis=1 )<feature_engineering>
seed = 42
def seed_everything(seed):
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
seed_everything(seed )
n_components = 16
text_features = []
for i in X_text.columns:
print(f'generating features from: {i}')
 tfv = TfidfVectorizer(min_df=2, max_features=None,
strip_accents='unicode', analyzer='word', token_pattern=r'(?u)\b\w+\b',
ngram_range=(1, 3), use_idf=1, smooth_idf=1, sublinear_tf=1)
 svd_ = TruncatedSVD(
 n_components=n_components, random_state=1337)
 tfidf_col = tfv.fit_transform(X_text.loc[:, i].values)
 svd_col = svd_.fit_transform(tfidf_col)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix('TFIDF_{}_'.format(i))
text_features.append(svd_col)
 text_features = pd.concat(text_features, axis=1)
 X_temp = pd.concat([X_temp, text_features], axis=1)
 for i in X_text.columns:
X_temp = X_temp.drop(i, axis=1 )<merge>
qt = QuantileTransformer(n_quantiles=100,random_state=42,output_distribution='normal')
 train_features[GENES+CELLS] = qt.fit_transform(train_features[GENES+CELLS])
 test_features[GENES+CELLS] = qt.transform(test_features[GENES+CELLS] )
X_temp = X_temp.merge(img_features, how='left', on='PetID' )<define_variables>
n_comp = 600
gpca= load(open('.. /input/moa-tabnet-last/gpca.pkl', 'rb'))
train2=(gpca.transform(train_features[GENES]))
test2 =(gpca.transform(test_features[GENES]))
train_gpca = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test_gpca = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train_gpca), axis=1)
 test_features = pd.concat(( test_features, test_gpca), axis=1)
train_df_ids = train[['PetID']]
test_df_ids = test[['PetID']]
train_df_imgs = pd.DataFrame(train_image_files)
 train_df_imgs.columns = ['image_filename']
train_imgs_pets = train_df_imgs['image_filename'].apply(lambda x: x.split(split_char)[-1].split('-')[0])
 test_df_imgs = pd.DataFrame(test_image_files)
 test_df_imgs.columns = ['image_filename']
test_imgs_pets = test_df_imgs['image_filename'].apply(lambda x: x.split(split_char)[-1].split('-')[0])
 train_df_imgs = train_df_imgs.assign(PetID=train_imgs_pets)
 test_df_imgs = test_df_imgs.assign(PetID=test_imgs_pets)
 def getSize(filename):
st = os.stat(filename)
 return st.st_size
def getDimensions(filename):
img_size = Image.open(filename ).size
return img_size
train_df_imgs['image_size'] = train_df_imgs['image_filename'].apply(getSize)
 train_df_imgs['temp_size'] = train_df_imgs['image_filename'].apply(getDimensions)
 train_df_imgs['width'] = train_df_imgs['temp_size'].apply(lambda x : x[0])
 train_df_imgs['height'] = train_df_imgs['temp_size'].apply(lambda x : x[1])
 train_df_imgs = train_df_imgs.drop(['temp_size'], axis=1)
 test_df_imgs['image_size'] = test_df_imgs['image_filename'].apply(getSize)
 test_df_imgs['temp_size'] = test_df_imgs['image_filename'].apply(getDimensions)
 test_df_imgs['width'] = test_df_imgs['temp_size'].apply(lambda x : x[0])
 test_df_imgs['height'] = test_df_imgs['temp_size'].apply(lambda x : x[1])
 test_df_imgs = test_df_imgs.drop(['temp_size'], axis=1)
 aggs = {
'image_size': ['sum', 'mean', 'var'],
'width': ['sum', 'mean', 'var'],
'height': ['sum', 'mean', 'var'],
}
agg_train_imgs = train_df_imgs.groupby('PetID' ).agg(aggs)
 new_columns = [
k + '_' + agg for k in aggs.keys() for agg in aggs[k]
]
agg_train_imgs.columns = new_columns
agg_train_imgs = agg_train_imgs.reset_index()
agg_test_imgs = test_df_imgs.groupby('PetID' ).agg(aggs)
 new_columns = [
k + '_' + agg for k in aggs.keys() for agg in aggs[k]
]
agg_test_imgs.columns = new_columns
agg_test_imgs = agg_test_imgs.reset_index()
agg_imgs = pd.concat([agg_train_imgs, agg_test_imgs], axis=0 ).reset_index(drop=True )<merge>
n_comp = 50
cpca= load(open('.. /input/moa-tabnet-last/cpca.pkl', 'rb'))
train2=(cpca.transform(train_features[CELLS]))
test2 =(cpca.transform(test_features[CELLS]))
train_cpca = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test_cpca = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train_cpca), axis=1)
 test_features = pd.concat(( test_features, test_cpca), axis=1)
X_temp = X_temp.merge(agg_imgs, how='left', on='PetID' )<drop_column>
print(f'Number of features before Variance thresholding - {train_features.shape[1]}' )
X_temp = X_temp.drop(to_drop_columns, axis=1 )<define_variables>
c_n = [f for f in list(train_features.columns)if f not in ['sig_id', 'cp_type', 'cp_time', 'cp_dose']]
mask =(train_features[c_n].var() >= 0.85 ).values
tmp = train_features[c_n].loc[:, mask]
train_features = pd.concat([train_features[['sig_id', 'cp_type', 'cp_time', 'cp_dose']], tmp], axis=1)
 tmp = test_features[c_n].loc[:, mask]
test_features = pd.concat([test_features[['sig_id', 'cp_type', 'cp_time', 'cp_dose']], tmp], axis=1 )
X_train_non_null = X_train.fillna(-1)
 X_test_non_null = X_test.fillna(-1 )<count_missing_values>
print(f'Number of features after Variance thresholding - {train_features.shape[1]}' )
X_train_non_null.isnull().any().any() , X_test_non_null.isnull().any().any()<import_modules>
def fe_cluster_genes(train, test, n_clusters_g = 22, SEED = 42):
features_g = GENES
def create_cluster(train, test, features, kind = 'g', n_clusters = n_clusters_g):
train_ = train[features].copy()
test_ = test[features].copy()
kmeans_genes = load(open('.. /input/moa-tabnet-last/kmeans_genes.pkl', 'rb'))
train[f'clusters_{kind}'] = kmeans_genes.predict(train_.values)
 test[f'clusters_{kind}'] = kmeans_genes.predict(test_.values)
 train = pd.get_dummies(train, columns = [f'clusters_{kind}'])
 test = pd.get_dummies(test, columns = [f'clusters_{kind}'])
 return train, test
train, test = create_cluster(train, test, features_g, kind = 'g', n_clusters = n_clusters_g)
 return train, test
train_features2 ,test_features2=fe_cluster_genes(train_features2,test_features2 )
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):

 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_ratings = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_ratings)]
for j in range(num_ratings)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):

 if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):

 rater_a = y
rater_b = y_pred
min_rating=None
max_rating=None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(min(rater_a), min(rater_b))
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b,
min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(hist_rater_a[i] * hist_rater_b[j]
/ num_scored_items)
 d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator )<train_on_grid>
def fe_cluster_cells(train, test, n_clusters_c = 4, SEED = 42):
features_c = CELLS
def create_cluster(train, test, features, kind = 'c', n_clusters = n_clusters_c):
train_ = train[features].copy()
test_ = test[features].copy()
kmeans_cells = load(open('.. /input/moa-tabnet-last/kmeans_cells.pkl', 'rb'))
train[f'clusters_{kind}'] = kmeans_cells.predict(train_.values)
 test[f'clusters_{kind}'] = kmeans_cells.predict(test_.values)
 train = pd.get_dummies(train, columns = [f'clusters_{kind}'])
 test = pd.get_dummies(test, columns = [f'clusters_{kind}'])
 return train, test
train, test = create_cluster(train, test, features_c, kind = 'c', n_clusters = n_clusters_c)
 return train, test
train_features2 ,test_features2=fe_cluster_cells(train_features2,test_features2 )
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4])
 return -cohen_kappa_score(y, preds, weights='quadratic')
 def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X = X, y = y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4])
 return preds
def coefficients(self):
return self.coef_['x']<init_hyperparams>
train_pca=pd.concat(( train_gpca,train_cpca),axis=1)
 test_pca=pd.concat(( test_gpca,test_cpca),axis=1 )
xgb_params = {
'eval_metric': 'rmse',
'seed': 1337,
'eta': 0.0123,
'subsample': 0.8,
'colsample_bytree': 0.85,
'tree_method': 'gpu_hist',
'device': 'gpu',
'silent': 1,
}<prepare_x_and_y>
def fe_cluster_pca(train, test,n_clusters=5,SEED = 42):
kmeans_pca = load(open('.. /input/moa-tabnet-last/kmeans_pca.pkl', 'rb'))
train[f'clusters_pca'] = kmeans_pca.predict(train.values)
 test[f'clusters_pca'] = kmeans_pca.predict(test.values)
 train = pd.get_dummies(train, columns = [f'clusters_pca'])
 test = pd.get_dummies(test, columns = [f'clusters_pca'])
 return train, test
train_cluster_pca ,test_cluster_pca = fe_cluster_pca(train_pca,test_pca )
def run_xgb(params, X_train, X_test):
n_splits = 10
verbose_eval = 1000
num_rounds = 60000
early_stop = 500
kf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=1337)
 oof_train = np.zeros(( X_train.shape[0]))
oof_test = np.zeros(( X_test.shape[0], n_splits))
i = 0
for train_idx, valid_idx in kf.split(X_train, X_train['AdoptionSpeed'].values):
X_tr = X_train.iloc[train_idx, :]
X_val = X_train.iloc[valid_idx, :]
y_tr = X_tr['AdoptionSpeed'].values
X_tr = X_tr.drop(['AdoptionSpeed'], axis=1)
 y_val = X_val['AdoptionSpeed'].values
X_val = X_val.drop(['AdoptionSpeed'], axis=1)
 d_train = xgb.DMatrix(data=X_tr, label=y_tr, feature_names=X_tr.columns)
 d_valid = xgb.DMatrix(data=X_val, label=y_val, feature_names=X_val.columns)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
model = xgb.train(dtrain=d_train, num_boost_round=num_rounds, evals=watchlist,
early_stopping_rounds=early_stop, verbose_eval=verbose_eval, params=params)
 valid_pred = model.predict(xgb.DMatrix(X_val, feature_names=X_val.columns), ntree_limit=model.best_ntree_limit)
 test_pred = model.predict(xgb.DMatrix(X_test, feature_names=X_test.columns), ntree_limit=model.best_ntree_limit)
 oof_train[valid_idx] = valid_pred
oof_test[:, i] = test_pred
i += 1
return model, oof_train, oof_test<train_model>
train_cluster_pca = train_cluster_pca.iloc[:,650:]
test_cluster_pca = test_cluster_pca.iloc[:,650:]
model, oof_train, oof_test = run_xgb(xgb_params, X_train_non_null, X_test_non_null )<compute_train_metric>
train_features_cluster=train_features2.iloc[:,876:]
test_features_cluster=test_features2.iloc[:,876:]
optR = OptimizedRounder()
optR.fit(oof_train, X_train['AdoptionSpeed'].values)
 coefficients = optR.coefficients()
valid_pred = optR.predict(oof_train, coefficients)
 qwk = quadratic_weighted_kappa(X_train['AdoptionSpeed'].values, valid_pred)
 print("QWK = ", qwk )<predict_on_test>
gsquarecols=['g-574','g-211','g-216','g-0','g-255','g-577','g-153','g-389','g-60','g-370','g-248','g-167','g-203','g-177','g-301','g-332','g-517','g-6','g-744','g-224','g-162','g-3','g-736','g-486','g-283','g-22','g-359','g-361','g-440','g-335','g-106','g-307','g-745','g-146','g-416','g-298','g-666','g-91','g-17','g-549','g-145','g-157','g-768','g-568','g-396']
coefficients_ = coefficients.copy()
train_predictions = optR.predict(oof_train, coefficients_ ).astype(np.int8)
 print(f'train pred distribution: {Counter(train_predictions)}')
 test_predictions = optR.predict(oof_test.mean(axis=1), coefficients_ ).astype(np.int8)
 print(f'test pred distribution: {Counter(test_predictions)}' )<count_values>
def fe_stats(train, test):
features_g = GENES
features_c = CELLS
for df in train, test:
df['g_sum'] = df[features_g].sum(axis = 1)
 df['g_mean'] = df[features_g].mean(axis = 1)
 df['g_std'] = df[features_g].std(axis = 1)
 df['g_kurt'] = df[features_g].kurtosis(axis = 1)
 df['g_skew'] = df[features_g].skew(axis = 1)
 df['c_sum'] = df[features_c].sum(axis = 1)
 df['c_mean'] = df[features_c].mean(axis = 1)
 df['c_std'] = df[features_c].std(axis = 1)
 df['c_kurt'] = df[features_c].kurtosis(axis = 1)
 df['c_skew'] = df[features_c].skew(axis = 1)
 df['gc_sum'] = df[features_g + features_c].sum(axis = 1)
 df['gc_mean'] = df[features_g + features_c].mean(axis = 1)
 df['gc_std'] = df[features_g + features_c].std(axis = 1)
 df['gc_kurt'] = df[features_g + features_c].kurtosis(axis = 1)
 df['gc_skew'] = df[features_g + features_c].skew(axis = 1)
 df['c52_c42'] = df['c-52'] * df['c-42']
df['c13_c73'] = df['c-13'] * df['c-73']
df['c26_c13'] = df['c-26'] * df['c-13']
df['c33_c6'] = df['c-33'] * df['c-6']
df['c11_c55'] = df['c-11'] * df['c-55']
df['c38_c63'] = df['c-38'] * df['c-63']
df['c38_c94'] = df['c-38'] * df['c-94']
df['c13_c94'] = df['c-13'] * df['c-94']
df['c4_c52'] = df['c-4'] * df['c-52']
df['c4_c42'] = df['c-4'] * df['c-42']
df['c13_c38'] = df['c-13'] * df['c-38']
df['c55_c2'] = df['c-55'] * df['c-2']
df['c55_c4'] = df['c-55'] * df['c-4']
df['c4_c13'] = df['c-4'] * df['c-13']
df['c82_c42'] = df['c-82'] * df['c-42']
df['c66_c42'] = df['c-66'] * df['c-42']
df['c6_c38'] = df['c-6'] * df['c-38']
df['c2_c13'] = df['c-2'] * df['c-13']
df['c62_c42'] = df['c-62'] * df['c-42']
df['c90_c55'] = df['c-90'] * df['c-55']
df['c26_c38'] = df['c-26'] * df['c-38']
df['c90_c13'] = df['c-90'] * df['c-13']
df['c85_c31'] = df['c-85'] * df['c-31']
df['c63_c42'] = df['c-63'] * df['c-42']
df['c94_c11'] = df['c-94'] * df['c-11']
df['c94_c60'] = df['c-94'] * df['c-60']
df['c55_c42'] = df['c-55'] * df['c-42']
df['g37_c50'] = df['g-37'] * df['g-50']
for feature in features_c:
df[f'{feature}_squared'] = df[feature] ** 2
for feature in gsquarecols:
df[f'{feature}_squared'] = df[feature] ** 2
return train, test
train_features2,test_features2=fe_stats(train_features2,test_features2 )
Counter(train_predictions )<count_values>
train_features_stats=train_features2.iloc[:,902:]
test_features_stats=test_features2.iloc[:,902:]
Counter(test_predictions )<create_dataframe>
train_features = pd.concat(( train_features, train_features_cluster,train_cluster_pca,train_features_stats), axis=1)
 test_features = pd.concat(( test_features, test_features_cluster,test_cluster_pca,test_features_stats), axis=1 )
submission1 = pd.DataFrame({'PetID': test['PetID'].values, 'AdoptionSpeed': test_predictions} )<set_options>
cols2 = train_targets_nonscored.columns.to_list()
uniques2 = {col: train_targets_nonscored[col].nunique() for col in cols2}
uniques2=pd.DataFrame(uniques2, index=[0] ).T
uniques2=uniques2.rename(columns={0:'count'})
 uniques2= uniques2.drop('sig_id', axis=0)
 print(f"{len(uniques2[uniques2['count']==1])} targets without ANY mechanism of action in the nonscored dataset" )
%matplotlib inline
np.random.seed(seed=1337)
 warnings.filterwarnings('ignore')
 split_char = '/'
os.listdir('.. /input' )<load_from_csv>
nonmoacols=uniques2[uniques2['count']==1].index
train_targets_nonscored_columns = [col for col in list(train_targets_nonscored.columns)if col not in nonmoacols]
train_targets_nonscored=train_targets_nonscored[train_targets_nonscored_columns]
<load_from_csv>
train = train_features.merge(train_targets_nonscored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target = train[train_targets_nonscored.columns]
target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1)
 target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()
target=target[target_cols]
train = pd.get_dummies(train, columns=['cp_time','cp_dose'])
 test_ = pd.get_dummies(test, columns=['cp_time','cp_dose'])
 feature_cols = [c for c in train.columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['sig_id']]
<load_pretrained>
train_features_tabnet=train[feature_cols]
X_test_tabnet=test_[feature_cols].values
<data_type_conversions>
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct
<merge>
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
<feature_engineering>
class NNModel(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(NNModel, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dropout1 = nn.Dropout(0.2)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(0.2)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(0.2)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = self.dropout1(x)
 x = F.leaky_relu(self.dense1(x), 1e-3)
 x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x
<define_variables>
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 NFOLDS = 7
num_features=len(feature_cols)
 num_targets=len(target_cols)
 hidden_size=2048
BATCH_SIZE=256
X_train = X_temp.loc[np.isfinite(X_temp.AdoptionSpeed), :]
X_test = X_temp.loc[~np.isfinite(X_temp.AdoptionSpeed), :]
X_test = X_test.drop(['AdoptionSpeed'], axis=1)
 assert X_train.shape[0] == train.shape[0]
assert X_test.shape[0] == test.shape[0]
train_cols = X_train.columns.tolist()
train_cols.remove('AdoptionSpeed')
 test_cols = X_test.columns.tolist()
assert np.all(train_cols == test_cols)
 X_train_non_null = X_train.fillna(-1)
 X_test_non_null = X_test.fillna(-1 )<import_modules>
def run_training(fold, seed):
seed_everything(seed)
 x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = NNModel(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.load_state_dict(torch.load(f".. /input/moa-pretrained-non-scored-targets-as-meta-features/SEED{seed}_FOLD{fold}_nonscored.pth",map_location=torch.device(DEVICE)))
 model.to(DEVICE)
 predictions = np.zeros(( len(test_), target.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 return predictions
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):

 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_ratings = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_ratings)]
for j in range(num_ratings)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):

 if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):

 rater_a = y
rater_b = y_pred
min_rating=None
max_rating=None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(min(rater_a), min(rater_b))
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b,
min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(hist_rater_a[i] * hist_rater_b[j]
/ num_scored_items)
 d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator )<train_on_grid>
def run_k_fold(NFOLDS, seed):
predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
pred_ = run_training(fold, seed)
 predictions += pred_ / NFOLDS
return predictions
class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4])
 return -cohen_kappa_score(y, preds, weights='quadratic')
 def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X = X, y = y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
preds = pd.cut(X, [-np.inf] + list(np.sort(coef)) + [np.inf], labels = [0, 1, 2, 3, 4])
 return preds
def coefficients(self):
return self.coef_['x']<split>
SEED = [0,1,2,3,4,5,6]
predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
predictions_ = run_k_fold(NFOLDS, seed)
 predictions += predictions_ / len(SEED)
 test_[target_cols] = predictions
xgb_params = {
'eval_metric': 'rmse',
'seed': 1337,
'silent': 1,
}
def run_xgb(params, X_train, X_test):
n_splits = 5
verbose_eval = 1000
num_rounds = 30000
early_stop = 500
kf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=1337)
 oof_train = np.zeros(( X_train.shape[0]))
oof_test = np.zeros(( X_test.shape[0], n_splits))
i = 0
for train_idx, valid_idx in kf.split(X_train, X_train['AdoptionSpeed'].values):
X_tr = X_train.iloc[train_idx, :]
X_val = X_train.iloc[valid_idx, :]
y_tr = X_tr['AdoptionSpeed'].values
X_tr = X_tr.drop(['AdoptionSpeed'], axis=1)
 y_val = X_val['AdoptionSpeed'].values
X_val = X_val.drop(['AdoptionSpeed'], axis=1)
 d_train = xgb.DMatrix(data=X_tr, label=y_tr, feature_names=X_tr.columns)
 d_valid = xgb.DMatrix(data=X_val, label=y_val, feature_names=X_val.columns)
 watchlist = [(d_train, 'train'),(d_valid, 'valid')]
model = xgb.train(dtrain=d_train, num_boost_round=num_rounds, evals=watchlist,
early_stopping_rounds=early_stop, verbose_eval=verbose_eval, params=params)
 valid_pred = model.predict(xgb.DMatrix(X_val, feature_names=X_val.columns), ntree_limit=model.best_ntree_limit)
 test_pred = model.predict(xgb.DMatrix(X_test, feature_names=X_test.columns), ntree_limit=model.best_ntree_limit)
 oof_train[valid_idx] = valid_pred
oof_test[:, i] = test_pred
i += 1
return model, oof_train, oof_test
model, oof_train, oof_test = run_xgb(xgb_params, X_train_non_null, X_test_non_null )<compute_train_metric>
train = train.merge(train_targets_scored, on='sig_id')
 target = train[train_targets_scored.columns]
target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()
optR = OptimizedRounder()
optR.fit(oof_train, X_train['AdoptionSpeed'].values)
 coefficients = optR.coefficients()
valid_pred = optR.predict(oof_train, coefficients)
 qwk = quadratic_weighted_kappa(X_train['AdoptionSpeed'].values, valid_pred)
 print("QWK = ", qwk )<predict_on_test>
feature_cols_NN = [c for c in train.columns if c not in target_cols]
feature_cols_NN = [c for c in feature_cols_NN if c not in ['sig_id','kfold']]
coefficients_ = coefficients.copy()
train_predictions = optR.predict(oof_train, coefficients_ ).astype(np.int8)
 test_predictions = optR.predict(oof_test.mean(axis=1), coefficients_ ).astype(np.int8)
 submission2 = pd.DataFrame({'PetID': test['PetID'].values, 'AdoptionSpeed': test_predictions} )<load_from_csv>
num_features=len(feature_cols_NN)
 num_targets=len(target_cols)
 hidden_size=2048
train_df = pd.read_csv('.. /input/petfinder-adoption-prediction/train/train.csv')
 img_size = 256
batch_size = 16<define_variables>
def run_training(fold, seed):
seed_everything(seed)
 x_test = test_[feature_cols_NN].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = NNModel(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.load_state_dict(torch.load(f".. /input/moa-pretrained-non-scored-targets-as-meta-features/SEED{seed}_FOLD{fold}_scored.pth",map_location=torch.device(DEVICE)))
 model.to(DEVICE)
 predictions = np.zeros(( len(test_), target.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 return predictions
pet_ids = train_df['PetID'].values
n_batches = len(pet_ids)// batch_size + 1 
<define_search_model>
def run_k_fold(NFOLDS, seed):
predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
pred_ = run_training(fold, seed)
 predictions += pred_ / NFOLDS
return predictions
inp = Input(( 256,256,3))
backbone = DenseNet121(input_tensor = inp,
weights = ".. /input/densenet121weights/densenet121_weights_tf_dim_ordering_tf_kernels_notop.h5",
include_top = False)
 x = backbone.output
x = GlobalAveragePooling2D()(x)
 x = Lambda(lambda x: K.expand_dims(x,axis = -1))(x)
 x = AveragePooling1D(4 )(x)
 out = Lambda(lambda x: x[:,:,0] )(x)
 m = Model(inp,out )<predict_on_test>
SEED = [0,1,2,3,4,5,6]
predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
predictions_ = run_k_fold(NFOLDS, seed)
 predictions += predictions_ / len(SEED)
 preds_pretrain = predictions
features = {}
for b in tqdm_notebook(range(n_batches)) :
start = b*batch_size
end =(b+1)*batch_size
batch_pets = pet_ids[start:end]
batch_images = np.zeros(( len(batch_pets),img_size,img_size,3))
for i,pet_id in enumerate(batch_pets):
try:
batch_images[i] = load_image(".. /input/petfinder-adoption-prediction/train_images/", pet_id)
 except:
pass
batch_preds = m.predict(batch_images)
 for i,pet_id in enumerate(batch_pets):
features[pet_id] = batch_preds[i]<create_dataframe>
class ResNetModel(nn.Module):
def __init__(self, num_features, num_targets):
super(ResNetModel, self ).__init__()
self.hidden_size = [2048, 1500, 1250, 1000]
self.dropout_value = [0.3, 0.3, 0.3, 0.3]
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.Linear(num_features, self.hidden_size[0])
 self.batch_norm2 = nn.BatchNorm1d(self.hidden_size[0])
 self.dropout2 = nn.Dropout(self.dropout_value[0])
 self.dense2 = nn.Linear(self.hidden_size[0], self.hidden_size[1])
 self.batch_norm3 = nn.BatchNorm1d(self.hidden_size[1])
 self.dropout3 = nn.Dropout(self.dropout_value[1])
 self.dense3 = nn.Linear(self.hidden_size[1], self.hidden_size[2])
 self.batch_norm4 = nn.BatchNorm1d(self.hidden_size[2])
 self.dropout4 = nn.Dropout(self.dropout_value[2])
 self.dense4 = nn.Linear(self.hidden_size[2], self.hidden_size[3])
 self.batch_norm5 = nn.BatchNorm1d(self.hidden_size[3])
 self.dropout5 = nn.Dropout(self.dropout_value[3])
 self.dense5 = nn.utils.weight_norm(nn.Linear(self.hidden_size[3], num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = F.leaky_relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.leaky_relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = F.leaky_relu(self.dense3(x))
x = self.batch_norm4(x)
 x = self.dropout4(x)
 x = F.leaky_relu(self.dense4(x))
x = self.batch_norm5(x)
 x = self.dropout5(x)
 x = self.dense5(x)
 return x
train_feats = pd.DataFrame.from_dict(features, orient='index' )<load_from_csv>
num_features = len(feature_cols)
 num_features
test_df = pd.read_csv('.. /input/petfinder-adoption-prediction/test/test.csv')
 pet_ids = test_df['PetID'].values
n_batches = len(pet_ids)// batch_size + 1
features = {}
for b in tqdm_notebook(range(n_batches)) :
start = b*batch_size
end =(b+1)*batch_size
batch_pets = pet_ids[start:end]
batch_images = np.zeros(( len(batch_pets),img_size,img_size,3))
for i,pet_id in enumerate(batch_pets):
try:
batch_images[i] = load_image(".. /input/petfinder-adoption-prediction/test_images/", pet_id)
 except:
pass
batch_preds = m.predict(batch_images)
 for i,pet_id in enumerate(batch_pets):
features[pet_id] = batch_preds[i]
test_feats = pd.DataFrame.from_dict(features, orient='index' )<train_model>
def run_training(fold_id, seed_id):
seed_everything(seed_id)
 model = ResNetModel(num_features, num_targets)
 model.load_state_dict(torch.load(f".. /input/pytorch-transfer-learning-last/SCORED_ONLY_SEED{seed_id}_FOLD{fold_id}_.pth",map_location=torch.device(DEVICE)))
 model.to(DEVICE)
 x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=128, shuffle=False)
 predictions = np.zeros(( len(test_), num_targets))
predictions = inference_fn(model, testloader, DEVICE)
 return predictions
train_feats.reset_index(inplace=True)
 test_feats.reset_index(inplace=True )<load_from_csv>
def run_k_fold(NFOLDS, seed_id):
predictions = np.zeros(( len(test), len(target_cols)))
 for fold_id in range(NFOLDS):
pred_ = run_training(fold_id, seed_id)
 predictions += pred_ / NFOLDS
return predictions
def impact_coding(data, feature, target='y'):

 n_folds = 20
n_inner_folds = 10
impact_coded = pd.Series()
oof_default_mean = data[target].mean()
kf = KFold(n_splits=n_folds, shuffle=True)
 oof_mean_cv = pd.DataFrame()
split = 0
for infold, oof in kf.split(data[feature]):
impact_coded_cv = pd.Series()
kf_inner = KFold(n_splits=n_inner_folds, shuffle=True)
 inner_split = 0
inner_oof_mean_cv = pd.DataFrame()
oof_default_inner_mean = data.iloc[infold][target].mean()
for infold_inner, oof_inner in kf_inner.split(data.iloc[infold]):
oof_mean = data.iloc[infold_inner].groupby(by=feature)[target].mean()
impact_coded_cv = impact_coded_cv.append(data.iloc[infold].apply(
 lambda x: oof_mean[x[feature]]
if x[feature] in oof_mean.index
else oof_default_inner_mean
, axis=1))
inner_oof_mean_cv = inner_oof_mean_cv.join(pd.DataFrame(oof_mean), rsuffix=inner_split, how='outer')
 inner_oof_mean_cv.fillna(value=oof_default_inner_mean, inplace=True)
 inner_split += 1
oof_mean_cv = oof_mean_cv.join(pd.DataFrame(inner_oof_mean_cv), rsuffix=split, how='outer')
 oof_mean_cv.fillna(value=oof_default_mean, inplace=True)
 split += 1
impact_coded = impact_coded.append(data.iloc[oof].apply(
 lambda x: inner_oof_mean_cv.loc[x[feature]].mean()
if x[feature] in inner_oof_mean_cv.index
else oof_default_mean
, axis=1))
return impact_coded, oof_mean_cv.mean(axis=1), oof_default_mean
def frequency_encoding(df, col_name):
new_name = "{}_counts".format(col_name)
 new_col_name = "{}_freq".format(col_name)
 grouped = df.groupby(col_name ).size().reset_index(name=new_name)
 df = df.merge(grouped, how = "left", on = col_name)
 df[new_col_name] = df[new_name]/df[new_name].count()
del df[new_name]
return df
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):

 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_ratings = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_ratings)]
for j in range(num_ratings)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):

 if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):

 rater_a = y
rater_b = y_pred
min_rating=None
max_rating=None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(min(rater_a), min(rater_b))
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b,
min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(hist_rater_a[i] * hist_rater_b[j]
/ num_scored_items)
 d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator)
 class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = quadratic_weighted_kappa(y, X_p)
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']
def rmse(actual, predicted):
return sqrt(mean_squared_error(actual, predicted))
def train_lightgbm(X_train, X_test, params, n_splits, num_rounds, verbose_eval, early_stop):
kfold = StratifiedKFold(n_splits=n_splits, random_state=1337)
 oof_train = np.zeros(( X_train.shape[0]))
oof_test = np.zeros(( X_test.shape[0], n_splits))
i = 0
for train_index, valid_index in kfold.split(X_train, X_train['AdoptionSpeed'].values):
X_tr = X_train.iloc[train_index, :]
X_val = X_train.iloc[valid_index, :]
y_tr = X_tr['AdoptionSpeed'].values
X_tr = X_tr.drop(['AdoptionSpeed'], axis=1)
 y_val = X_val['AdoptionSpeed'].values
X_val = X_val.drop(['AdoptionSpeed'], axis=1)
 print('
y_tr distribution: {}'.format(Counter(y_tr)))
 d_train = lgb.Dataset(X_tr, label=y_tr)
 d_valid = lgb.Dataset(X_val, label=y_val)
 watchlist = [d_train, d_valid]
print('training LGB:')
 model = lgb.train(params,
train_set=d_train,
num_boost_round=num_rounds,
valid_sets=watchlist,
verbose_eval=verbose_eval,
early_stopping_rounds=early_stop)
 val_pred = model.predict(X_val, num_iteration=model.best_iteration)
 test_pred = model.predict(X_test, num_iteration=model.best_iteration)
 oof_train[valid_index] = val_pred
oof_test[:, i] = test_pred
i += 1
return oof_train, oof_test
pet_parser = PetFinderParser()
def main() :
X_train = X_temp.loc[np.isfinite(X_temp.AdoptionSpeed), :]
X_test = X_temp.loc[~np.isfinite(X_temp.AdoptionSpeed), :]
X_test = X_test.drop(['AdoptionSpeed'], axis=1)
 print('X_train shape: {}'.format(X_train.shape))
print('X_test shape: {}'.format(X_test.shape))
assert X_train.shape[0] == train.shape[0]
assert X_test.shape[0] == test.shape[0]
train_cols = X_train.columns.tolist()
train_cols.remove('AdoptionSpeed')
 test_cols = X_test.columns.tolist()
np.random.seed(13)
 categorical_features = ["Type", "Breed1", "Breed2", "Color1" ,"Color2", "Color3", "State"]
impact_coding_map = {}
for f in categorical_features:
print("Impact coding for {}".format(f))
X_train["impact_encoded_{}".format(f)], impact_coding_mapping, default_coding = impact_coding(X_train, f, target="AdoptionSpeed")
 impact_coding_map[f] =(impact_coding_mapping, default_coding)
 mapping, default_mean = impact_coding_map[f]
X_test["impact_encoded_{}".format(f)] = X_test.apply(lambda x: mapping[x[f]] if x[f] in mapping
else default_mean, axis=1)
 for cat in categorical_features:
X_train = frequency_encoding(X_train, cat)
 X_test = frequency_encoding(X_test, cat)
 params = {'application': 'regression',
'boosting': 'gbdt',
'metric': 'rmse',
'num_leaves': 70,
'max_depth': 9,
'learning_rate': 0.01,
'bagging_fraction': 0.85,
'feature_fraction': 0.8,
'min_split_gain': 0.02,
'min_child_samples': 150,
'min_child_weight': 0.02,
'lambda_l2': 0.0475,
'verbosity': -1,
'data_random_seed': 17}
early_stop = 500
verbose_eval = 100
num_rounds = 10000
n_splits = 5
oof_train, oof_test = train_lightgbm(X_train, X_test, params, n_splits, num_rounds, verbose_eval, early_stop)
 optR = OptimizedRounder()
optR.fit(oof_train, X_train['AdoptionSpeed'].values)
 coefficients = optR.coefficients()
pred_test_y_k = optR.predict(oof_train, coefficients)
 print("
Valid Counts = ", Counter(X_train['AdoptionSpeed'].values))
print("Predicted Counts = ", Counter(pred_test_y_k))
print("Coefficients = ", coefficients)
 qwk = quadratic_weighted_kappa(X_train['AdoptionSpeed'].values, pred_test_y_k)
 print("QWK = ", qwk)
 coefficients_ = coefficients.copy()
train_predictions = optR.predict(oof_train, coefficients_ ).astype(int)
 print('train pred distribution: {}'.format(Counter(train_predictions)))
 test_predictions = optR.predict(oof_test.mean(axis=1), coefficients_)
 print('test pred distribution: {}'.format(Counter(test_predictions)))
 submission = pd.DataFrame({'PetID': test['PetID'].values, 'AdoptionSpeed': test_predictions.astype(np.int32)})
 return submission
submission3 = main()<save_to_csv>
SEED = [0,1,2,3,4,5,6]
predictions = np.zeros(( len(test), len(target_cols)))
 for seed_id in SEED:
predictions_ = run_k_fold(NFOLDS, seed_id)
 predictions += predictions_ / len(SEED)
 preds_transfer=predictions
submission1["AdoptionSpeed"] = submission1.AdoptionSpeed*0.4 + submission2.AdoptionSpeed*0.2 + submission3.AdoptionSpeed*0.4
submission1["AdoptionSpeed"] = submission1["AdoptionSpeed"].round().astype(int)
 submission1.to_csv('submission.csv', index=False )<set_options>
test_cv_preds_1 = []
NB_SPLITS = 7
SEED = [0,1,2,3,4,5,6]
for s in SEED:
for fold_nb,(train_idx, val_idx)in enumerate(MultilabelStratifiedKFold(n_splits = NB_SPLITS, random_state = s ).split(train_features_tabnet, target)) :
model1 = TabNetRegressor()
model1.load_model(f".. /input/moa-tabnet-train-inference/TabNet_seed_{s}_fold_{fold_nb+1}.zip")
 preds_test_1 = model1.predict(X_test_tabnet)
 test_cv_preds_1.append(1 /(1 + np.exp(-preds_test_1)))
 test_preds_all_1 = np.stack(test_cv_preds_1)
 preds_tabnet_1 = test_preds_all_1.mean(axis = 0 )
tqdm.pandas()
gc.collect()
np.random.seed(369 )<load_from_csv>
test_cv_preds_2 = []
NB_SPLITS = 7
SEED = [0,1,2,3,4,5,6]
for s in SEED:
for fold_nb,(train_idx, val_idx)in enumerate(MultilabelStratifiedKFold(n_splits = NB_SPLITS, random_state = s ).split(train_features_tabnet, target)) :
model1 = TabNetRegressor()
model2 = TabNetRegressor()
model2.load_model(f".. /input/moa-tabnet-last/TabNet_seed_{s}_fold_{fold_nb+1}.zip")
 preds_test_2 = model2.predict(X_test_tabnet)
 test_cv_preds_2.append(1 /(1 + np.exp(-preds_test_2)))
 test_preds_all_2 = np.stack(test_cv_preds_2)
 preds_tabnet_2 = test_preds_all_2.mean(axis = 0 )
df_train = pd.read_csv('.. /input/train/train.csv')
 df_breed = pd.read_csv('.. /input/breed_labels.csv')
 df_color = pd.read_csv('.. /input/color_labels.csv')
 df_state = pd.read_csv('.. /input/state_labels.csv')
 df_test = pd.read_csv('.. /input/test/test.csv' )<count_missing_values>
predictions_ = 0.25*preds_pretrain + 0.25*preds_tabnet_1 + 0.25*preds_transfer + 0.25*preds_tabnet_2
<compute_test_metric><EOS> 
df = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 public_id = list(df['sig_id'].values)
 test_id = list(test_features['sig_id'].values)
 private_id = list(set(test_id)-set(public_id))
df_submit = pd.DataFrame(index = public_id+private_id, columns=target_cols)
 df_submit.index.name = 'sig_id'
df_submit[:] = 0
df_submit.loc[test.sig_id,:] = predictions_
df_submit.loc[test_features[test_features.cp_type=='ctl_vehicle'].sig_id]= 0
df_submit.to_csv('submission.csv',index=True )