Kaggle competition: Mechanisms of Action (MoA) Prediction
train['Description'].str.count(" " )<init_hyperparams>
warnings.filterwarnings('ignore' )
res = []; n_train = train.shape[0]; output = 0
params = {
'boosting': 'gbdt',
'objective': 'regression',
'metric': 'rmse',
'learning_rate': 0.01,
'num_leaves': 30,
'max_depth': 15,
'colsample_bytree': 0.2,
'min_data_in_leaf':50,
'bagging_freq': 10,
'bagging_fraction': 0.7,
'min_data_per_group': 20,
'seed': 100
}
train['len_des_1'] = train['Description'].fillna("" ).astype(str ).str.len()
test['len_des_1'] = test['Description'].fillna("" ).astype(str ).str.len()
train['len_word'] = train['Description'].fillna("" ).str.count(" ")
 test['len_word'] = test['Description'].fillna("" ).str.count(" ")
 train['NoName'] =(( train['Name'].fillna("NULL" ).str.contains("No Name")) | train['Name'].isnull() ).astype('int')
 test['NoName'] =(( test['Name'].fillna("NULL" ).str.contains("No Name")) | test['Name'].isnull() ).astype('int')
 kf = StratifiedKFold(n_splits=5, shuffle=True, random_state=500)
 predictors = ['magnitude','Age','Type','Breed1','Breed2','Color1','Color2','Color3','PhotoAmt','VideoAmt','State',\
'Sterilized','Vaccinated','Quantity','Fee','Fee_v1','MaturitySize', 'Gender','FurLength','Health',\
'cntPet','len_des','len_des_1','min_age','max_age','mean_age','min_Fee','max_Fee',\
'mean_Fee','min_PhotoAmt','max_PhotoAmt','mean_PhotoAmt',\
'min_Quantity','max_Quantity','mean_Quantity','min_VideoAmt','max_VideoAmt','mean_VideoAmt',
'min_MaturitySize_1','max_MaturitySize_1','mean_MaturitySize_1','min_Health','max_Health','mean_Health',\
'min_score_labelAnnotations','max_score_labelAnnotations','mean_score_labelAnnotations',\
'min_score_labelAnnotations_1','max_score_labelAnnotations_1','mean_score_labelAnnotations_1',\
'min_x','max_x', 'mean_x','min_y','max_y','mean_y',
'min_pixelscore','max_pixelscore', 'mean_pixelscore','min_pixelFraction','max_pixelFraction','mean_pixelFraction',\
'min_red_0','max_red_0', 'mean_red_0','min_green_0','max_green_0','mean_green_0',\
'min_blue_0','max_blue_0', 'mean_blue_0',
'min_len_des','max_len_des','mean_len_des',
] + list(count_cols)
 categorical = ['Type','State','Gender','Breed1','Breed2','Color1','Color2','Color3','Fee_v1','MaturitySize', 'FurLength','Health']
Xspr_train = csr_matrix(hstack([csr_matrix(train[predictors]), train_count_des, train_labelAnnotations_des,\
X_Breed1[:n_train], X_Breed2[:n_train], X_State[:n_train],\
X_Color1[:n_train], X_Color2[:n_train], X_Color3[:n_train]]))
Xspr_test = csr_matrix(hstack([csr_matrix(test[predictors]), test_count_des, test_labelAnnotations_des,\
X_Breed1[n_train:], X_Breed2[n_train:], X_State[n_train:],\
X_Color1[n_train:], X_Color2[n_train:], X_Color3[n_train:]]))
for dev_index, val_index in(kf.split(user_ID, user_ID['class'])) :
dev_user, val_user = user_ID.iloc[dev_index].RescuerID, user_ID.iloc[val_index].RescuerID
X_train_idx, X_val_idx = list(train[train['RescuerID'].isin(dev_user)].index.values),\
list(train[train['RescuerID'].isin(val_user)].index.values)
 X_val = train.iloc[X_val_idx]
print(len(X_train_idx), len(X_val_idx))
dtrain = lgb.Dataset(Xspr_train[X_train_idx], label=train.iloc[X_train_idx]['AdoptionSpeed'].values,
feature_name=predictors+ ['tfidf_' + str(i)for i in range(\
train_count_des.shape[1]+ train_labelAnnotations_des.shape[1])] +[str(i)for i in range(more_dim)],
categorical_feature=categorical 
)
 dvalid = lgb.Dataset(Xspr_train[X_val_idx], label=train.iloc[X_val_idx]['AdoptionSpeed'].values,
feature_name=predictors + ['tfidf_' for i in range(
 train_count_des.shape[1] + train_labelAnnotations_des.shape[1])] +[str(i)for i in range(more_dim)],
categorical_feature=categorical 
)
 lgb_model = lgb.train(params, dtrain,
valid_sets=[dtrain, dvalid], valid_names=['train','valid'],
num_boost_round= 3000,early_stopping_rounds=300,
verbose_eval=100,)
 oof = pd.DataFrame()
oof['PetID'] = X_val['PetID']
oof['target'] = X_val['AdoptionSpeed']
oof['preds'] = lgb_model.predict(Xspr_train[X_val_idx], num_iteration=lgb_model.best_iteration)
 res.append(oof)
 output += lgb_model.predict(Xspr_test, num_iteration=lgb_model.best_iteration)
 oof = pd.concat(res, ignore_index=True)
 quadratic_weighted_kappa(oof['target'], get_adoptionspeed(oof['preds']))<create_dataframe>
from sklearn.preprocessing import QuantileTransformer
submission = pd.DataFrame({'PetID': test['PetID'].values, 'RescuerID': test['RescuerID'].values, \
'AdoptionSpeed': get_adoptionspeed(output)} )<merge>
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 df = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )
tmp = submission.groupby(['RescuerID'])['AdoptionSpeed'].apply(lambda x: x.mode() [0] ).reset_index()
submission.drop(['AdoptionSpeed'], axis=1, inplace = True)
 submission = submission.merge(tmp, on=['RescuerID'], how='left' )<save_to_csv>
train_features2=train_features.copy()
test_features2=test_features.copy()
submission[['PetID','AdoptionSpeed']].to_csv('submission.csv', index=False )<count_values>
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
submission['AdoptionSpeed'].value_counts()<import_modules>
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]
pd.options.display.max_columns = 128
torch.multiprocessing.set_start_method("spawn" )<load_from_csv>
def seed_everything(seed_value):
random.seed(seed_value)
 np.random.seed(seed_value)
 torch.manual_seed(seed_value)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed_value)
 torch.cuda.manual_seed_all(seed_value)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
seed_everything(42 )
input_dir = Path(".. /input/petfinder-adoption-prediction/")
 train = pd.read_csv(input_dir / "train/train.csv")
 test = pd.read_csv(input_dir / "test/test.csv")
 sample_submission = pd.read_csv(input_dir / "test/sample_submission.csv")
 train_pet_ids = train.PetID.unique()
test_pet_ids = test.PetID.unique()<load_pretrained>
n_comp = 600
pca_g = PCA(n_components=n_comp, random_state=42)
 data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 gpca=(pca_g.fit(data[GENES]))
train2=(gpca.transform(train_features[GENES]))
test2 =(gpca.transform(test_features[GENES]))
train_gpca = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test_gpca = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train_gpca), axis=1)
 test_features = pd.concat(( test_features, test_gpca), axis=1)
 dump(gpca, open('gpca.pkl', 'wb'))
def jopen(path):
with open(path, "r", encoding="utf-8")as f:
json_file = json.load(f)
 return json_file
def parse_sentiment_file(path):
file = jopen(path)
 language: str = file["language"]
sentiment: list = file["documentSentiment"]
entities: list = [x["name"] for x in file["entities"]]
entity = " ".join(entities)
 sentence_sentiment: list = [x["sentiment"] for x in file["sentences"]]
magnitude: np.ndarray = np.array(
 [x["magnitude"] for x in sentence_sentiment])
 score: np.ndarray = np.array([x["score"] for x in sentence_sentiment])
 return_js = {
"magnitude_sum": magnitude.sum() ,
"magnitude_mean": magnitude.mean() ,
"magnitude_var": magnitude.var() ,
"score_sum": score.sum() ,
"score_mean": score.mean() ,
"score_var": score.var() ,
"language": language,
"entity": entity,
"document_magnitude": sentiment["magnitude"],
"document_score": sentiment["score"]
}
return return_js
def parse_metadata(path):
file: dict = jopen(path)
 file_keys = list(file.keys())
 name_specified = 0
if "labelAnnotations" in file_keys:
file_annots = file["labelAnnotations"]
file_mean_score = np.asarray([x["score"] for x in file_annots] ).mean()
file_desc = " ".join([x["description"] for x in file_annots])
 if "cat" in file_desc or "dog" in file_desc:
name_specified = 1
else:
file_mean_score = np.nan
file_desc = ""
file_colors: list = file["imagePropertiesAnnotation"]["dominantColors"][
"colors"]
file_crops: list = file["cropHintsAnnotation"]["cropHints"]
color_score = np.asarray([x["score"] for x in file_colors] ).mean()
pixel_frac = np.asarray([x["pixelFraction"] for x in file_colors] ).mean()
crop_conf = np.asarray([x["confidence"] for x in file_crops] ).mean()
if "importanceFraction" in file_crops[0].keys() :
crop_importance = np.asarray(
 [x["importanceFraction"] for x in file_crops] ).mean()
else:
crop_importance = np.nan
metadata = {
"annot_score": file_mean_score,
"color_score": color_score,
"pixel_frac": pixel_frac,
"crop_conf": crop_conf,
"crop_importance": crop_importance,
"desc": file_desc,
"specified": name_specified
}
return metadata
def additinal_features_per_id(pet_id, sentiment_path: Path, meta_path: Path):
sentiment_path = sentiment_path / f"{pet_id}.json"
try:
sentiment = parse_sentiment_file(sentiment_path)
 sentiment["pet_id"] = pet_id
except FileNotFoundError:
sentiment = {}
meta_files = sorted(meta_path.glob(f"{pet_id}*.json"))
metadata_list = []
if len(meta_files)> 0:
for f in meta_files:
metadata = parse_metadata(f)
 metadata["pet_id"] = pet_id
metadata_list.append(metadata)
 return sentiment, metadata_list
def load_additional_features(ped_ids: list, sentiment_path: Path,
meta_path: Path):
features = Parallel(
 n_jobs=-1, verbose=1 )(
 delayed(additinal_features_per_id )(i, sentiment_path, meta_path)
 for i in ped_ids)
 sentiments = [x[0] for x in features if len(x[0])> 0]
metadatas = [x[1] for x in features if len(x[1])> 0]
sentiment_keys = sentiments[0].keys()
metadata_keys = metadatas[0][0].keys()
sentiment_dict = {}
metadata_dict = {}
for key in sentiment_keys:
sentiment_dict[key] = [x[key] for x in sentiments]
for key in metadata_keys:
meta_list = []
for meta_per_pid in metadatas:
meta_list += [meta[key] for meta in meta_per_pid]
metadata_dict[key] = meta_list
sentiment_df = pd.DataFrame(sentiment_dict)
 metadata_df = pd.DataFrame(metadata_dict)
 return sentiment_df, metadata_df
def aggregate_metadata(metadata_df: pd.DataFrame,
aggregates=["sum", "mean", "var"]):
meta_desc: pd.DataFrame = metadata_df.groupby(["pet_id"])["desc"].unique()
meta_desc = meta_desc.reset_index()
meta_desc["desc"] = meta_desc["desc"].apply(lambda x: " ".join(x))
meta_gr: pd.DataFrame = metadata_df.drop(["desc"], axis=1)
 for i in meta_gr.columns:
if "pet_id" not in i:
meta_gr[i] = meta_gr[i].astype(float)
 meta_gr = meta_gr.groupby(["pet_id"] ).agg(aggregates)
 meta_gr.columns = pd.Index(
 [f"{c[0]}_{c[1].upper() }" for c in meta_gr.columns.tolist() ])
 meta_gr = meta_gr.reset_index()
return meta_gr, meta_desc
def aggregate_sentiment(sentiment_df: pd.DataFrame, aggregates=["sum"]):
sentiment_desc: pd.DataFrame = sentiment_df.groupby(
 ["pet_id"])["entity"].unique()
sentiment_desc = sentiment_desc.reset_index()
sentiment_desc["entity"] = sentiment_desc["entity"].apply(
 lambda x: " ".join(x))
sentiment_lang = sentiment_df.groupby(
 ["pet_id"])["language"].unique()
sentiment_lang = sentiment_lang.reset_index()
sentiment_lang["language"] = sentiment_lang["language"].apply(
 lambda x: " ".join(x))
sentiment_desc = sentiment_desc.merge(
 sentiment_lang, how="left", on="pet_id")
 sentiment_gr: pd.DataFrame = sentiment_df.drop(["entity", "language"],
axis=1)
 for i in sentiment_gr.columns:
if "pet_id" not in i:
sentiment_gr[i] = sentiment_gr[i].astype(float)
 sentiment_gr = sentiment_gr.groupby(["pet_id"] ).agg(aggregates)
 sentiment_gr.columns = pd.Index(
 [f"{c[0]}" for c in sentiment_gr.columns.tolist() ])
 sentiment_gr = sentiment_gr.reset_index()
return sentiment_gr, sentiment_desc<load_from_csv>
n_comp = 50
pca_c = PCA(n_components=n_comp, random_state=42)
 data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 cpca=(pca_c.fit(data[CELLS]))
train2=(cpca.transform(train_features[CELLS]))
test2 =(cpca.transform(test_features[CELLS]))
train_cpca = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test_cpca = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train_cpca), axis=1)
 test_features = pd.concat(( test_features, test_cpca), axis=1)
 dump(cpca, open('cpca.pkl', 'wb'))
input_dir = Path(".. /input/petfinder-adoption-prediction/")
 train = pd.read_csv(input_dir / "train/train.csv")
 test = pd.read_csv(input_dir / "test/test.csv")
 sample_submission = pd.read_csv(input_dir / "test/sample_submission.csv" )<define_variables>
c_n = [f for f in list(train_features.columns)if f not in ['sig_id', 'cp_type', 'cp_time', 'cp_dose']]
mask =(train_features[c_n].var() >= 0.85 ).values
tmp = train_features[c_n].loc[:, mask]
train_features = pd.concat([train_features[['sig_id', 'cp_type', 'cp_time', 'cp_dose']], tmp], axis=1)
 tmp = test_features[c_n].loc[:, mask]
test_features = pd.concat([test_features[['sig_id', 'cp_type', 'cp_time', 'cp_dose']], tmp], axis=1 )
sp_train = input_dir / Path("train_sentiment/")
 mp_train = input_dir / Path("train_metadata/")
 sp_test = input_dir / Path("test_sentiment/")
 mp_test = input_dir / Path("test_metadata/" )<feature_engineering>
def fe_cluster_genes(train, test, n_clusters_g = 22, SEED = 42):
features_g = GENES
def create_cluster(train, test, features, kind = 'g', n_clusters = n_clusters_g):
train_ = train[features].copy()
test_ = test[features].copy()
data = pd.concat([train_, test_], axis = 0)
 kmeans_genes = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(data)
 dump(kmeans_genes, open('kmeans_genes.pkl', 'wb'))
train[f'clusters_{kind}'] = kmeans_genes.predict(train_.values)
 test[f'clusters_{kind}'] = kmeans_genes.predict(test_.values)
 train = pd.get_dummies(train, columns = [f'clusters_{kind}'])
 test = pd.get_dummies(test, columns = [f'clusters_{kind}'])
 return train, test
train, test = create_cluster(train, test, features_g, kind = 'g', n_clusters = n_clusters_g)
 return train, test
train_features2 ,test_features2=fe_cluster_genes(train_features2,test_features2 )
train_sentiment_df, train_metadata_df = load_additional_features(
 train_pet_ids, sp_train, mp_train)
 test_sentiment_df, test_metadata_df = load_additional_features(
 test_pet_ids, sp_test, mp_test )<categorify>
def fe_cluster_cells(train, test, n_clusters_c = 4, SEED = 42):
features_c = CELLS
def create_cluster(train, test, features, kind = 'c', n_clusters = n_clusters_c):
train_ = train[features].copy()
test_ = test[features].copy()
data = pd.concat([train_, test_], axis = 0)
 kmeans_cells = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(data)
 dump(kmeans_cells, open('kmeans_cells.pkl', 'wb'))
train[f'clusters_{kind}'] = kmeans_cells.predict(train_.values)
 test[f'clusters_{kind}'] = kmeans_cells.predict(test_.values)
 train = pd.get_dummies(train, columns = [f'clusters_{kind}'])
 test = pd.get_dummies(test, columns = [f'clusters_{kind}'])
 return train, test
train, test = create_cluster(train, test, features_c, kind = 'c', n_clusters = n_clusters_c)
 return train, test
train_features2 ,test_features2=fe_cluster_cells(train_features2,test_features2 )
train_meta_gr, train_meta_desc = aggregate_metadata(train_metadata_df)
 test_meta_gr, test_meta_desc = aggregate_metadata(test_metadata_df)
 train_sentiment_gr, train_sentiment_desc = \
aggregate_sentiment(train_sentiment_df)
 test_sentiment_gr, test_sentiment_desc = \
aggregate_sentiment(test_sentiment_df )<merge>
train_pca=pd.concat(( train_gpca,train_cpca),axis=1)
 test_pca=pd.concat(( test_gpca,test_cpca),axis=1 )
train_proc = train.copy()
train_proc = train_proc.merge(
 train_sentiment_gr, how="left", left_on="PetID", right_on="pet_id")
 train_proc = train_proc.merge(
 train_meta_gr, how="left", left_on="PetID", right_on="pet_id")
 train_proc = train_proc.merge(
 train_sentiment_desc, how="left", left_on="PetID", right_on="pet_id")
 train_proc = train_proc.merge(
 train_meta_desc, how="left", left_on="PetID", right_on = "pet_id")
 test_proc = test.copy()
test_proc = test_proc.merge(
 test_sentiment_gr, how="left", left_on="PetID", right_on="pet_id")
 test_proc = test_proc.merge(
 test_meta_gr, how="left", left_on="PetID", right_on="pet_id")
 test_proc = test_proc.merge(
 test_sentiment_desc, how="left", left_on="PetID", right_on="pet_id")
 test_proc = test_proc.merge(
 test_meta_desc, how="left", left_on="PetID", right_on = "pet_id" )<drop_column>
def fe_cluster_pca(train, test,n_clusters=5,SEED = 42):
data=pd.concat([train,test],axis=0)
 kmeans_pca = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(data)
 dump(kmeans_pca, open('kmeans_pca.pkl', 'wb'))
train[f'clusters_pca'] = kmeans_pca.predict(train.values)
 test[f'clusters_pca'] = kmeans_pca.predict(test.values)
 train = pd.get_dummies(train, columns = [f'clusters_pca'])
 test = pd.get_dummies(test, columns = [f'clusters_pca'])
 return train, test
train_cluster_pca ,test_cluster_pca = fe_cluster_pca(train_pca,test_pca )
train_proc.drop(train_proc.filter(
 regex="pet_id", axis=1 ).columns.tolist() ,
axis=1,
inplace=True)
 test_proc.drop(test_proc.filter(
 regex="pet_id", axis=1 ).columns.tolist() ,
axis=1,
inplace=True)
 train_proc.head()<categorify>
train_cluster_pca = train_cluster_pca.iloc[:,650:]
test_cluster_pca = test_cluster_pca.iloc[:,650:]
train_proc.language.fillna("", inplace=True)
 test_proc.language.fillna("", inplace=True)
 langs = train_proc.language.unique()
encode_dict = {k: i for i, k in enumerate(langs)}
train_proc.language = train_proc.language.map(encode_dict)
 test_proc.language = test_proc.language.map(encode_dict )<load_from_csv>
train_features_cluster=train_features2.iloc[:,876:]
test_features_cluster=test_features2.iloc[:,876:]
labels_breed = pd.read_csv(".. /input/petfinder-adoption-prediction/breed_labels.csv")
 labels_state = pd.read_csv(".. /input/petfinder-adoption-prediction/state_labels.csv")
 labels_color = pd.read_csv(".. /input/petfinder-adoption-prediction/color_labels.csv" )<merge>
gsquarecols=['g-574','g-211','g-216','g-0','g-255','g-577','g-153','g-389','g-60','g-370','g-248','g-167','g-203','g-177','g-301','g-332','g-517','g-6','g-744','g-224','g-162','g-3','g-736','g-486','g-283','g-22','g-359','g-361','g-440','g-335','g-106','g-307','g-745','g-146','g-416','g-298','g-666','g-91','g-17','g-549','g-145','g-157','g-768','g-568','g-396']
train_breed_main = train_proc[["Breed1"]].merge(
 labels_breed, how="left",
left_on="Breed1", right_on="BreedID",
suffixes=("", "_main_breed"))
train_breed_main = train_breed_main.iloc[:, 2:]
train_breed_main = train_breed_main.add_prefix("main_breed_")
 train_breed_second = train_proc[["Breed2"]].merge(
 labels_breed, how="left",
left_on="Breed2", right_on="BreedID",
suffixes=("", "_second_breed"))
train_breed_second = train_breed_second.iloc[:, 2:]
train_breed_second = train_breed_second.add_prefix("second_breed_")
 train_proc = pd.concat([
train_proc, train_breed_main, train_breed_second
], axis=1)
 test_breed_main = test_proc[["Breed1"]].merge(
 labels_breed, how="left",
left_on="Breed1", right_on="BreedID",
suffixes=("", "_main_breed"))
test_breed_main = test_breed_main.iloc[:, 2:]
test_breed_main = test_breed_main.add_prefix("main_breed_")
 test_breed_second = test_proc[["Breed2"]].merge(
 labels_breed, how="left",
left_on="Breed2", right_on="BreedID",
suffixes=("", "_second_breed"))
test_breed_second = test_breed_second.iloc[:, 2:]
test_breed_second = test_breed_second.add_prefix("second_breed_")
 test_proc = pd.concat([
test_proc, test_breed_main, test_breed_second
], axis=1)
 print(train_proc.shape, test_proc.shape)
 train_proc.head()<concatenate>
def fe_stats(train, test):
features_g = GENES
features_c = CELLS
for df in train, test:
df['g_sum'] = df[features_g].sum(axis = 1)
 df['g_mean'] = df[features_g].mean(axis = 1)
 df['g_std'] = df[features_g].std(axis = 1)
 df['g_kurt'] = df[features_g].kurtosis(axis = 1)
 df['g_skew'] = df[features_g].skew(axis = 1)
 df['c_sum'] = df[features_c].sum(axis = 1)
 df['c_mean'] = df[features_c].mean(axis = 1)
 df['c_std'] = df[features_c].std(axis = 1)
 df['c_kurt'] = df[features_c].kurtosis(axis = 1)
 df['c_skew'] = df[features_c].skew(axis = 1)
 df['gc_sum'] = df[features_g + features_c].sum(axis = 1)
 df['gc_mean'] = df[features_g + features_c].mean(axis = 1)
 df['gc_std'] = df[features_g + features_c].std(axis = 1)
 df['gc_kurt'] = df[features_g + features_c].kurtosis(axis = 1)
 df['gc_skew'] = df[features_g + features_c].skew(axis = 1)
 df['c52_c42'] = df['c-52'] * df['c-42']
df['c13_c73'] = df['c-13'] * df['c-73']
df['c26_c13'] = df['c-26'] * df['c-13']
df['c33_c6'] = df['c-33'] * df['c-6']
df['c11_c55'] = df['c-11'] * df['c-55']
df['c38_c63'] = df['c-38'] * df['c-63']
df['c38_c94'] = df['c-38'] * df['c-94']
df['c13_c94'] = df['c-13'] * df['c-94']
df['c4_c52'] = df['c-4'] * df['c-52']
df['c4_c42'] = df['c-4'] * df['c-42']
df['c13_c38'] = df['c-13'] * df['c-38']
df['c55_c2'] = df['c-55'] * df['c-2']
df['c55_c4'] = df['c-55'] * df['c-4']
df['c4_c13'] = df['c-4'] * df['c-13']
df['c82_c42'] = df['c-82'] * df['c-42']
df['c66_c42'] = df['c-66'] * df['c-42']
df['c6_c38'] = df['c-6'] * df['c-38']
df['c2_c13'] = df['c-2'] * df['c-13']
df['c62_c42'] = df['c-62'] * df['c-42']
df['c90_c55'] = df['c-90'] * df['c-55']
df['c26_c38'] = df['c-26'] * df['c-38']
df['c90_c13'] = df['c-90'] * df['c-13']
df['c85_c31'] = df['c-85'] * df['c-31']
df['c63_c42'] = df['c-63'] * df['c-42']
df['c94_c11'] = df['c-94'] * df['c-11']
df['c94_c60'] = df['c-94'] * df['c-60']
df['c55_c42'] = df['c-55'] * df['c-42']
df['g37_c50'] = df['g-37'] * df['g-50']
for feature in features_c:
df[f'{feature}_squared'] = df[feature] ** 2
for feature in gsquarecols:
df[f'{feature}_squared'] = df[feature] ** 2
return train, test
train_features2,test_features2=fe_stats(train_features2,test_features2 )
X = pd.concat([train_proc, test_proc], ignore_index=True, sort=False)
 X_temp = X.copy()
text_columns = [
"Description",
"entity",
"desc"]
categorical_columns = [
"Type", "Breed1", "Breed2", "Gender",
"Color1", "Color2", "Color3", "MaturitySize",
"FurLength", "Vaccinated", "Dewormed", "Sterilized",
"State", "language", "main_breed_BreedName", "second_breed_BreedName"
]
cat_c = ["main_breed_BreedName", "second_breed_BreedName"]
drop_columns = [
"PetID", "Name", "RescuerID"
]<feature_engineering>
train_features_stats=train_features2.iloc[:,902:]
test_features_stats=test_features2.iloc[:,902:]
for i in cat_c:
X_temp.loc[:, i] = pd.factorize(X_temp.loc[:, i])[0]<data_type_conversions>
train_features = pd.concat(( train_features, train_features_cluster,train_cluster_pca,train_features_stats), axis=1)
 test_features = pd.concat(( test_features, test_features_cluster,test_cluster_pca,test_features_stats), axis=1 )
X_text = X_temp[text_columns]
for i in X_text.columns:
X_text[i] = X_text[i].fillna("none" )<categorify>
cols2 = train_targets_nonscored.columns.to_list()
uniques2 = {col: train_targets_nonscored[col].nunique() for col in cols2}
uniques2=pd.DataFrame(uniques2, index=[0] ).T
uniques2=uniques2.rename(columns={0:'count'})
 uniques2= uniques2.drop('sig_id', axis=0)
 print(f"{len(uniques2[uniques2['count']==1])} targets without ANY mechanism of action in the nonscored dataset" )
X_temp["len_description"] = X_text["Description"].map(len)
 X_temp["len_meta_desc"] = X_text["desc"].map(len)
 X_temp["len_entity"] = X_text["entity"].map(len )<define_variables>
nonmoacols=uniques2[uniques2['count']==1].index
train_targets_nonscored_columns = [col for col in list(train_targets_nonscored.columns)if col not in nonmoacols]
train_targets_nonscored=train_targets_nonscored[train_targets_nonscored_columns]
puncts = [',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$', '&', '/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…',
'“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─',
'▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',
'∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]
def clean_text(x):
x = str(x)
 for punct in puncts:
x = x.replace(punct, f' {punct} ')
 return x
def clean_numbers(x):
x = re.sub('[0-9]{5,}', '
x = re.sub('[0-9]{4}', '
x = re.sub('[0-9]{3}', '
x = re.sub('[0-9]{2}', '
return x
mispell_dict = {"aren't" : "are not",
"can't" : "cannot",
"couldn't" : "could not",
"didn't" : "did not",
"doesn't" : "does not",
"don't" : "do not",
"hadn't" : "had not",
"hasn't" : "has not",
"haven't" : "have not",
"he'd" : "he would",
"he'll" : "he will",
"he's" : "he is",
"i'd" : "I would",
"i'd" : "I had",
"i'll" : "I will",
"i'm" : "I am",
"isn't" : "is not",
"it's" : "it is",
"it'll":"it will",
"i've" : "I have",
"let's" : "let us",
"mightn't" : "might not",
"mustn't" : "must not",
"shan't" : "shall not",
"she'd" : "she would",
"she'll" : "she will",
"she's" : "she is",
"shouldn't" : "should not",
"that's" : "that is",
"there's" : "there is",
"they'd" : "they would",
"they'll" : "they will",
"they're" : "they are",
"they've" : "they have",
"we'd" : "we would",
"we're" : "we are",
"weren't" : "were not",
"we've" : "we have",
"what'll" : "what will",
"what're" : "what are",
"what's" : "what is",
"what've" : "what have",
"where's" : "where is",
"who'd" : "who would",
"who'll" : "who will",
"who're" : "who are",
"who's" : "who is",
"who've" : "who have",
"won't" : "will not",
"wouldn't" : "would not",
"you'd" : "you would",
"you'll" : "you will",
"you're" : "you are",
"you've" : "you have",
"'re": " are",
"wasn't": "was not",
"we'll":" will",
"didn't": "did not",
"tryin'":"trying"}
def _get_mispell(mispell_dict):
mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))
return mispell_dict, mispell_re
mispellings, mispellings_re = _get_mispell(mispell_dict)
 def replace_typical_misspell(text):
def replace(match):
return mispellings[match.group(0)]
return mispellings_re.sub(replace, text )<feature_engineering>
train = train_features.merge(train_targets_nonscored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target = train[train_targets_nonscored.columns]
X_text["cleaned_text"] = X_text["Description"].map(lambda x: x.lower())
 X_text["cleaned_text"] = X_text["cleaned_text"].map(lambda x: clean_text(x))
X_text["cleaned_text"] = X_text["cleaned_text"].map(lambda x: clean_numbers(x))
X_text["cleaned_text"] = X_text["cleaned_text"].map(lambda x: replace_typical_misspell(x))<feature_engineering>
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1 )
eng_stopwords = set(stopwords.words("english"))
X_temp["len_description"] = X_text["Description"].map(len)
 X_temp["len_meta_desc"] = X_text["desc"].map(len)
 X_temp["len_entity"] = X_text["entity"].map(len)
 X_temp["num_description_words"] = X_text["Description"].map(lambda x: len(str(x ).split()))
X_temp["num_desc_words"] = X_text["desc"].map(lambda x: len(str(x ).split()))
X_temp["num_entity_words"] = X_text["entity"].map(lambda x: len(str(x ).split()))
X_temp["uniq_description_words"] = X_text["Description"].map(lambda x: len(set(str(x ).split())))
 X_temp["uniq_desc_words"] = X_text["desc"].map(lambda x: len(set(str(x ).split())))
 X_temp["uniq_entity_words"] = X_text["entity"].map(lambda x: len(set(str(x ).split())))
 X_temp["num_description_stopwords"] = X_text["Description"].map(lambda x: len([
w for w in str(x ).lower().split() if w in eng_stopwords]))
X_temp["num_desc_stopwords"] = X_text["desc"].map(lambda x: len([
w for w in str(x ).lower().split() if w in eng_stopwords]))
X_temp["num_entity_stopwords"] = X_text["entity"].map(lambda x: len([
w for w in str(x ).lower().split() if w in eng_stopwords]))
X_temp["num_description_punctuation"] = X_text["Description"].map(lambda x: len([
c for c in str(x)if c in string.punctuation]))<define_variables>
target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()
state_gdp = {
41336: 116.679,
41325: 40.596,
41367: 23.02,
41401: 190.075,
41415: 5.984,
41324: 37.274,
41332: 42.389,
41335: 52.452,
41330: 67.629,
41380: 5.642,
41327: 81.284,
41345: 80.167,
41342: 121.414,
41326: 280.698,
41361: 32.270
}
state_population = {
41336: 33.48283,
41325: 19.47651,
41367: 15.39601,
41401: 16.74621,
41415: 0.86908,
41324: 8.21110,
41332: 10.21064,
41335: 15.00817,
41330: 23.52743,
41380: 2.31541,
41327: 15.61383,
41345: 32.06742,
41342: 24.71140,
41326: 54.62141,
41361: 10.35977
}
state_area ={
41336:19102,
41325:9500,
41367:15099,
41401:243,
41415:91,
41324:1664,
41332:6686,
41335:36137,
41330:21035,
41380:821,
41327:1048,
41345:73631,
41342:124450,
41326:8104,
41361:13035
}
X_temp["state_gdp"] = X_temp.State.map(state_gdp)
 X_temp["state_population"] = X_temp.State.map(state_population)
 X_temp["state_area"] = X_temp.State.map(state_area)
 X_temp["state_gdp_per_person"] = X_temp["state_gdp"] / X_temp["state_population"] * 1e4
X_temp["fee_per_gdp_per_person"] = X_temp.Fee / X_temp["state_gdp_per_person"]<categorify>
train = pd.get_dummies(train, columns=['cp_time','cp_dose'])
 test_ = pd.get_dummies(test, columns=['cp_time','cp_dose'] )
def has_name(x):
if isinstance(x, float):
return 0
if "no name" in x.lower() :
return 0
return 1
def num_name_words(x):
if isinstance(x, float):
return 0
name_words = x.split(" ")
 return len(name_words)
 def contains_amp(x):
if isinstance(x, float):
return 0
if "&" in x:
return 1
if "and" in x.lower() :
return 1
if "+" in x.lower() :
return 1
return 0
def contains_comma(x):
if isinstance(x, float):
return 0
if "," in x:
return 1
return 0
def start_with_number(x):
if isinstance(x, float):
return 0
match = re.match(f"\d", x)
 if match:
return int(match.group())
 return 0
def contains_paren(x):
if isinstance(x, float):
return 0
if "(" in x:
return 1
if ")" in x:
return 1
return 0
def contains_number(x):
if isinstance(x, float):
return 0
if re.match(r".*\d", x):
return 1
return 0
def safe_calc_len(x):
if isinstance(x, float):
return 1
return len(x)
 def num_unlike_letters(x):
if isinstance(x, float):
return 0
letters = {
',', '.', '"', ':', ')', '(', '-', '!', '?', '|', ';', "'", '$',
'&', '/', '[', ']', '>', '%', '=', '
'•', '~', '@', '£', '·', '_', '{', '}', '©', '^', '®', '`',
'<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′',
'Â', '█', '½', 'à', '…', '“', '★', '”', '–', '●', 'â', '►',
'−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦',
'║', '―', '¥', '▓', '—', '‹', '─', '▒', '：', '¼', '⊕', '▼',
'▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦',
'¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', '∙', '）', '↓', '、',
'│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗',
'▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', "ã", "ç", "å", "ä",
"¶", "ð"}
letter_in = set(x)
 intersection = letter_in.intersection(letters)
 if len(intersection)== 0:
return 0
else:
unlike_num = 0
for l in intersection:
unlike_num += len(re.findall(re.escape(l), x))
return unlike_num
X_temp["num_name_words"] = X_temp.Name.map(lambda x: num_name_words(x))
X_temp["contains_amp"] = X_temp.Name.map(lambda x: contains_amp(x))
X_temp["contains_comma"] = X_temp.Name.map(lambda x: contains_comma(x))
X_temp["start_with_number"] = X_temp.Name.map(lambda x: start_with_number(x))
X_temp["contains_paren"] = X_temp.Name.map(lambda x: contains_paren(x))
X_temp["contains_number"] = X_temp.Name.map(lambda x: contains_number(x))
X_temp["name_length"] = X_temp.Name.map(lambda x: safe_calc_len(x))
X_temp["num_unlike_letters"] = X_temp.Name.map(lambda x: num_unlike_letters(x))
X_temp["rate_unlike_letters"] = X_temp.num_unlike_letters / X_temp.name_length
X_temp.head()<feature_engineering>
feature_cols = [c for c in train.columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['sig_id']]
n_components = 16
text_features = []
for i in text_columns:
print(f"generating features from: {i}")
 tfv = TfidfVectorizer(
 min_df=2,
strip_accents="unicode",
analyzer="word",
token_pattern=r"(?u)\b\w+\b",
ngram_range=(1, 3),
use_idf=1,
smooth_idf=1,
sublinear_tf=1)
 svd = TruncatedSVD(
 n_components=n_components,
random_state=1337)
 tfidf_col = tfv.fit_transform(X_text.loc[:, i].values)
 svd_col = svd.fit_transform(tfidf_col)
 svd_col = pd.DataFrame(svd_col)
 svd_col = svd_col.add_prefix("Tfidf_{}_".format(i))
text_features.append(svd_col)
 text_features = pd.concat(text_features, axis=1)
 X_temp = pd.concat([X_temp, text_features], axis=1)
 for i in text_columns:
X_temp.drop(i, axis=1, inplace=True )<define_variables>
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct
train_image_files = sorted(
 glob.glob(".. /input/petfinder-adoption-prediction/train_images/*.jpg"))
test_image_files = sorted(
 glob.glob(".. /input/petfinder-adoption-prediction/test_images/*.jpg"))
train_df_imgs = pd.DataFrame(train_image_files)
 test_df_imgs = pd.DataFrame(test_image_files)
 train_df_imgs.columns = ["image_file_name"]
test_df_imgs.columns = ["image_file_name"]
train_imgs_pets = train_df_imgs["image_file_name"].apply(
 lambda x: x.split("/")[-1].split("-")[0])
 test_imgs_pets = test_df_imgs["image_file_name"].apply(
 lambda x: x.split("/")[-1].split("-")[0])
 train_df_imgs = train_df_imgs.assign(PetID=train_imgs_pets)
 test_df_imgs = test_df_imgs.assign(PetID=test_imgs_pets)
 def get_size(filename):
st = os.stat(filename)
 return st.st_size
def get_dimensions(filename):
img_size = Image.open(filename ).size
return img_size
train_df_imgs["image_size"] = train_df_imgs["image_file_name"].apply(get_size)
 test_df_imgs["image_size"] = test_df_imgs["image_file_name"].apply(get_size)
 train_df_imgs["temp_size"] = train_df_imgs["image_file_name"].apply(get_dimensions)
 test_df_imgs["temp_size"] = test_df_imgs["image_file_name"].apply(get_dimensions)
 train_df_imgs["width"] = train_df_imgs["temp_size"].apply(lambda x: x[0])
 test_df_imgs["width"] = test_df_imgs["temp_size"].apply(lambda x: x[0])
 train_df_imgs["height"] = train_df_imgs["temp_size"].apply(lambda x: x[1])
 test_df_imgs["height"] = test_df_imgs["temp_size"].apply(lambda x: x[1])
 train_df_imgs.drop(["temp_size"], axis=1, inplace=True)
 test_df_imgs.drop(["temp_size"], axis=1, inplace=True)
 aggs = {
"image_size": ["sum", "mean", "var"],
"width": ["sum", "mean", "var"],
"height": ["sum", "mean", "var"]
}
agg_train_imgs = train_df_imgs.groupby("PetID" ).agg(aggs)
 new_columns = [
k + "_" + agg for k in aggs.keys() for agg in aggs[k]
]
agg_test_imgs = test_df_imgs.groupby("PetID" ).agg(aggs)
 agg_train_imgs.columns = new_columns
agg_train_imgs = agg_train_imgs.reset_index()
agg_test_imgs.columns = new_columns
agg_test_imgs = agg_test_imgs.reset_index() 
<merge>
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
agg_imgs = pd.concat([agg_train_imgs, agg_test_imgs], axis=0 ).reset_index(drop=True)
 X_temp = X_temp.merge(agg_imgs, how="left", on="PetID")
 X_temp.head()<define_variables>
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss
hasnans = []
for c in X_temp.columns:
if X_temp[c].hasnans:
hasnans.append(c )<data_type_conversions>
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dropout1 = nn.Dropout(0.2)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(0.2)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(0.2)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = self.dropout1(x)
 x = F.leaky_relu(self.dense1(x), 1e-3)
 x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x
sanitize_col = list(
 set(hasnans)- {
"Name", "AdoptionSpeed"
})
 X_temp[sanitize_col] = X_temp[sanitize_col].fillna(0.0)
 X_temp.head()<define_variables>
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 26
BATCH_SIZE = 256
LEARNING_RATE = 6e-4
WEIGHT_DECAY = 1e-5
NFOLDS = 7
EARLY_STOPPING_STEPS = 10
EARLY_STOP = True
num_features=len(feature_cols)
 num_targets=len(target_cols)
 hidden_size=2048
big_num_cols = ["len_description", "len_meta_desc", "len_entity",
"num_description_words", "num_desc_words", "num_entity_words",
"uniq_description_words", "uniq_desc_words", "uniq_entity_words",
"num_description_stopwords", "num_desc_stopwords", "num_entity_stopwords",
"num_description_punctuation",
"state_gdp", "state_population", "state_area", "state_gdp_per_person",
"image_size_sum", "image_size_mean", "image_size_var", "width_sum",
"width_mean", "width_var", "height_sum", "height_mean", "height_var"]
for c in big_num_cols:
X_temp[c] = X_temp[c].map(lambda x: np.log1p(x))
X_temp.head()<find_best_params>
def run_training(fold, seed):
seed_everything(seed)
 mskf = MultilabelStratifiedKFold(n_splits=7,random_state=seed)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=target)) :
train.loc[v_idx, 'kfold'] = int(f)
 train['kfold'] = train['kfold'].astype(int)
 trn_idx = train[train['kfold'] != fold].index
val_idx = train[train['kfold'] == fold].index
train_df = train[train['kfold'] != fold].reset_index(drop=True)
 valid_df = train[train['kfold'] == fold].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1e-2, epochs=EPOCHS, steps_per_epoch=len(trainloader))
loss_fn = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing =0.001)
 early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
best_loss = np.inf
for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer,scheduler, loss_tr, trainloader, DEVICE)
 print(f"SEED: {seed}, FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}")
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"SEED: {seed} ,FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"SEED{seed}_FOLD{fold}_nonscored.pth")
 elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.load_state_dict(torch.load(f"SEED{seed}_FOLD{fold}_nonscored.pth"))
model.to(DEVICE)
 predictions = np.zeros(( len(test_), target.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions
train_resc = train.RescuerID
test_resc = test.RescuerID<drop_column>
def run_k_fold(NFOLDS, seed):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
oof_, pred_ = run_training(fold, seed)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions
X_temp.drop(drop_columns, axis=1, inplace=True)
 X_temp.head()<count_missing_values>
SEED = [0,1,2,3,4,5,6]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 train[target_cols] = oof
test_[target_cols] = predictions
X_train_non_null = X_train.fillna(-1)
 X_test_non_null = X_test.fillna(-1)
 X_train_non_null.isnull().any().any() , X_test_non_null.isnull().any().any()<define_variables>
train = train.merge(train_targets_scored, on='sig_id')
 target = train[train_targets_scored.columns]
target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()
cat_features = ["Type", "Breed1", "Breed2", "Gender", "Color1", "Color2", "Color3",
"MaturitySize", "FurLength", "Vaccinated", "Dewormed", "Sterilized",
"Health", "Quantity", "State", "language", "main_breed_BreedName",
"second_breed_BreedName"]
X_train_cat = X_train_non_null.loc[:, cat_features]
X_test_cat = X_test_non_null.loc[:, cat_features]<groupby>
feature_cols = [c for c in train.columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['sig_id','kfold']]
def stratified_group_k_fold(X, y, groups, k, seed=None):
labels_num = np.max(y)+ 1
y_counts_per_group = defaultdict(lambda: np.zeros(labels_num))
y_distr = Counter()
for label, g in zip(y, groups):
y_counts_per_group[g][label] += 1
y_distr[label] += 1
y_counts_per_fold = defaultdict(lambda: np.zeros(labels_num))
groups_per_fold = defaultdict(set)
 def eval_y_counts_per_fold(y_counts, fold):
y_counts_per_fold[fold] += y_counts
std_per_label = []
for label in range(labels_num):
label_std = np.std([y_counts_per_fold[i][label] / y_distr[label] for i in range(k)])
 std_per_label.append(label_std)
 y_counts_per_fold[fold] -= y_counts
return np.mean(std_per_label)
 groups_and_y_counts = list(y_counts_per_group.items())
 random.Random(seed ).shuffle(groups_and_y_counts)
 for g, y_counts in sorted(groups_and_y_counts, key=lambda x: -np.std(x[1])) :
best_fold = None
min_eval = None
for i in range(k):
fold_eval = eval_y_counts_per_fold(y_counts, i)
 if min_eval is None or fold_eval < min_eval:
min_eval = fold_eval
best_fold = i
y_counts_per_fold[best_fold] += y_counts
groups_per_fold[best_fold].add(g)
 all_groups = set(groups)
 for i in range(k):
train_groups = all_groups - groups_per_fold[i]
test_groups = groups_per_fold[i]
train_indices = [i for i, g in enumerate(groups)if g in train_groups]
test_indices = [i for i, g in enumerate(groups)if g in test_groups]
yield train_indices, test_indices<split>
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 26
BATCH_SIZE = 256
LEARNING_RATE = 6e-4
WEIGHT_DECAY = 1e-5
NFOLDS = 7
EARLY_STOPPING_STEPS = 10
EARLY_STOP = True
num_features=len(feature_cols)
 num_targets=len(target_cols)
 hidden_size=2048
y = X_train_non_null.AdoptionSpeed
X_train_cat_y = X_train_cat.copy()
X_train_cat_y["AdoptionSpeed"] = y
X_train_cat_encoded = np.zeros(( X_train_cat.shape[0], len(cat_features)* 2))
X_test_cat_encoded = np.zeros(( X_test_cat.shape[0], len(cat_features)* 2))
k = 10
fold = StratifiedKFold(n_splits=k, shuffle=True, random_state=1213)
 for trn_idx, val_idx in fold.split(X_train_cat, y.values.astype(int)) :
X_trn, X_val = X_train_cat_y.loc[trn_idx, :], X_train_cat_y.loc[val_idx, :]
for j, c in enumerate(cat_features):
X_trn_enc = X_trn.groupby(c ).agg({
"AdoptionSpeed": ["mean", "std"]
})
 cte_columns = [f"{c}_{x}" for x in ["mean", "std"]]
X_trn_enc.columns = cte_columns
X_temp = np.zeros(( X_test_cat.shape[0], 2))
X_temp_df = pd.DataFrame(data=X_temp, columns=cte_columns)
 for x in X_trn_enc.columns:
X_val[x] = X_val[c].map(X_trn_enc[x])
 X_temp_df[x] = X_test_cat[c].map(X_trn_enc[x] ).reset_index(drop=True)
 X_train_cat_encoded[val_idx, 2 * j:2 *(j + 1)] = X_val[X_trn_enc.columns].values
X_test_cat_encoded[:, 2 * j: 2 *(j + 1)] += X_temp_df.values / k<prepare_output>
def run_training(fold, seed):
seed_everything(seed)
 mskf = MultilabelStratifiedKFold(n_splits=7,random_state=seed)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=target)) :
train.loc[v_idx, 'kfold'] = int(f)
 train['kfold'] = train['kfold'].astype(int)
 trn_idx = train[train['kfold'] != fold].index
val_idx = train[train['kfold'] == fold].index
train_df = train[train['kfold'] != fold].reset_index(drop=True)
 valid_df = train[train['kfold'] == fold].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1e-2, epochs=EPOCHS, steps_per_epoch=len(trainloader))
loss_fn = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing =0.001)
 early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
best_loss = np.inf
for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer,scheduler, loss_tr, trainloader, DEVICE)
 print(f"SEED: {seed}, FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}")
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"SEED: {seed} ,FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"SEED{seed}_FOLD{fold}_scored.pth")
 elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.load_state_dict(torch.load(f"SEED{seed}_FOLD{fold}_scored.pth"))
model.to(DEVICE)
 predictions = np.zeros(( len(test_), target.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions
columns = [f"{c}_{x}" for c in cat_features for x in ["mean", "std"]]
X_train_cat_encoded_df = pd.DataFrame(data=X_train_cat_encoded, columns=columns)
 X_test_cat_encoded_df = pd.DataFrame(data=X_test_cat_encoded, columns=columns)
 X_test_cat_encoded_df.head()<prepare_x_and_y>
def run_k_fold(NFOLDS, seed):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
oof_, pred_ = run_training(fold, seed)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions
X_train_num = X_train_non_null.drop(cat_features + ["AdoptionSpeed"], axis=1)
 X_test_num = X_test_non_null.drop(cat_features, axis=1)
 target = X_train_non_null["AdoptionSpeed"]<concatenate>
SEED = [0,1,2,3,4,5,6]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 train[target_cols] = oof
test[target_cols] = predictions
X_train_num = pd.concat([X_train_num, X_train_cat_encoded_df], axis=1)
 X_test_cat_encoded_df.index = X_test_num.index
X_test_num = pd.concat([X_test_num, X_test_cat_encoded_df], axis=1 )<train_model>
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
cv_score = 0
roc_score = 0
for i in range(len(target_cols)) :
cv_score += log_loss(y_true[:, i], y_pred[:, i])
 roc_score +=roc_auc_score(y_true[:, i], y_pred[:, i], average='micro')
 print("CV log_loss: ", cv_score / y_pred.shape[1])
 print("Overall AUC: ", roc_score / y_pred.shape[1] )
X_train_num.fillna(0.0, inplace=True)
 X_test_num.fillna(0.0, inplace=True )<data_type_conversions>
oof_pretrain= valid_results[target_cols]
oof_pretrain.to_csv('off_pretrain_last.csv', index=False )
<filter><EOS> 
public_id = list(df['sig_id'].values)
 test_id = list(test_features['sig_id'].values)
 private_id = list(set(test_id)-set(public_id))
df_submit = pd.DataFrame(index = public_id+private_id, columns=target_cols)
 df_submit.index.name = 'sig_id'
df_submit[:] = 0
df_submit.loc[test.sig_id,:] = test[target_cols].values
df_submit.loc[test_features[test_features.cp_type=='ctl_vehicle'].sig_id]= 0
df_submit.to_csv('submission.csv',index=True )