Kaggle competition: Mechanisms of Action (MoA) Prediction
<SOS> metric: MeanColumnwiseLogLoss Kaggle data source: mechanisms-of-action-(moa)-prediction<train_on_grid>
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"
warnings.filterwarnings('ignore' )
if 1 == 0:
if "PetID" in train_df.columns.values:
train_pet_id = train_df["PetID"]
train_adoption_speed = train_df["AdoptionSpeed"]
train_df.drop(["PetID", "AdoptionSpeed"], axis=1, inplace=True)
 if "PetID" in test_df.columns.values:
test_pet_id = test_df["PetID"]
test_df.drop(["PetID"], axis=1, inplace=True)
 def filter_by_varth(train, test, threshold):
vt = VarianceThreshold(threshold=threshold)
 vt.fit(train)
 indices = vt.get_support(indices=True)
 return(train.iloc[:,indices], test.iloc[:,indices])
 for c in train_df.columns.values:
if train_df[c].isna().any() :
print("null", c)
 plog("lists similarity check before from removal by variance")
 print(train_df.columns.values == train_df.columns.values)
 plog("train - test diff")
 print(list(set(train_df.columns.values.tolist())- set(test_df.columns.values.tolist())))
 plog("test - train diff")
 list(set(test_df.columns.values.tolist())- set(train_df.columns.values.tolist()))
train_t, test_t = filter_by_varth(train_df, test_df, 0.20)
 plog("lists similarity check after from removal by variance")
 print(train_t.columns.values == test_t.columns.values)
 print(train_t.shape)
 print(test_t.shape)
 train_df = pd.concat([train_t, train_pet_id, train_adoption_speed], axis=1)
 test_df = pd.concat([test_t, test_pet_id], axis=1)
 print(train_df.shape)
 print(test_df.shape)
 plog("Done" )<normalization>
!pip install.. /input/iterative-stratification/iterative_stratification-0.1.6-py3-none-any.whl
rng = np.random.RandomState(304)
 qt = QuantileTransformer(output_distribution='normal', random_state=rng)
 if 1 == 0:
for c in Columns.scaling_cols.value:
if c in train_df.columns.values.tolist() :
p_val = normaltest(train_df[c])[1]
if p_val < 0.05:
plog("Transforming "+ c + " on train and test datasets with pval of normaltest:" + str(p_val))
qt.fit(x_train[c].reshape(-1, 1))
qt.transform(x_train[c].reshape(-1, 1))
qt.transform(x_test[c].reshape(-1, 1))
else:
plog("Not transforming "+ c + " on train and test datasets with pval of normaltest:" + str(p_val))
plog("Done")
<normalization>
import sys
from iterstrat.ml_stratifiers import MultilabelStratifiedKFold
if 1 == 0:
for c in Columns.scaling_cols.value:
if c in train_df.columns.values.tolist() :
p_val = normaltest(train_df[c])[1]
if p_val < 0.05:
plog("Transforming "+ c + " on train and test datasets with pval of normaltest:" + str(p_val))
train_df[c] = np.log(train_df[c]+1)
 test_df[c] = np.log(test_df[c]+1)
 else:
plog("Not transforming "+ c + " on train and test datasets with pval of normaltest:" + str(p_val))
plog("Done" )<categorify>
warnings.filterwarnings('ignore' )
def normalize(train, test, col):
mean = train[col].mean(axis=0)
 std = train[col].std(axis=0)
 train[col] =(train[col]-mean)/std
test[col] =(test[col]-mean)/std
return train, test
if 1 == 0:
for c in Columns.scaling_cols.value:
if c in train_df.columns.values.tolist() :
plog("Starting normalizing for " + c)
 train_df, test_df = scale_num_var(train_df, test_df, c)
 plog("Normalizing Ended for " + c )<normalization>
%%time
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_drug = pd.read_csv('.. /input/lish-moa/train_drug.csv')
 non_ctl_idx = train_features.loc[train_features['cp_type']!='ctl_vehicle'].index.to_list()
train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sub_rankgauss = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )
if 1 == 0:
scaler = RobustScaler()
plog("Starting RobustScaler for train")
 scaler.fit(train_df[Columns.scaling_cols.value])
 train_df[Columns.scaling_cols.value] = scaler.transform(train_df[Columns.scaling_cols.value])
 plog("Ended RobustScaler for train")
 scaler = RobustScaler()
plog("Starting RobustScaler for test")
 test_df[Columns.scaling_cols.value] = scaler.transform(test_df[Columns.scaling_cols.value])
 plog("Ended RobustScaler for test" )<init_hyperparams>
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
def detect_outliers(f_train):
n_samples = len(f_train)
 outliers_fraction = 0.10
n_outliers = int(outliers_fraction * n_samples)
 n_inliers = n_samples - n_outliers
anomaly_algorithms = [ 
("One-Class SVM", svm.OneClassSVM(nu=outliers_fraction, kernel="rbf",
gamma=0.1)) , 
("Isolation Forest", IsolationForest(behaviour='new',
contamination=outliers_fraction,
random_state=42)) , 
("Local Outlier Factor", LocalOutlierFactor(
 n_neighbors=35, contamination=outliers_fraction)) ]
df_pred = pd.DataFrame(columns=["One-Class SVM","Isolation Forest","Local Outlier Factor"])
 if 1 == 1 :
for name, algorithm in anomaly_algorithms:
plog(name)
 if name == "Local Outlier Factor":
y_pred = algorithm.fit_predict(f_train)
 else:
y_pred = algorithm.fit(f_train ).predict(f_train)
 df_pred[name] = y_pred
df_pred["outlier"] = df_pred.apply(lambda x: Counter([x['One-Class SVM'], x['Isolation Forest'], x["Local Outlier Factor"]] ).most_common(1)[0][0], axis=1)
 return df_pred["outlier"]
o_cols = train_df.columns.values.tolist().copy()
low_imp_features = []
for c in ["PetID", "RescuerID"]+low_imp_features:
if c in o_cols:
o_cols.remove(c)
 for c in o_cols:
if train_df[c].isna().any() :
plog("train null value exist for column " + c)
 train_df[c].fillna(0, inplace=True)
 if c != "AdoptionSpeed":
if test_df[c].isna().any() :
plog("test null value exist for column " + c)
 test_df[c].fillna(0, inplace=True)
 print(o_cols)
 plog("Outlier detection started")
 call=1
if call ==1:
df_o = detect_outliers(train_df[o_cols])
 print("df_o", df_o.shape, "train_df", train_df.shape)
 train_df = pd.concat([train_df, df_o], axis=1)
 else:
plog("For Dogs")
 train_df_dogs=train_df[train_df["Type"]==1].reset_index().drop(["index"] ,axis=1)
 df_od = detect_outliers(train_df_dogs[o_cols+["AdoptionSpeed"]])
 print("df_od", df_od.shape, "train_df_dogs", train_df_dogs.shape)
 train_df_dogs = pd.concat([train_df_dogs,df_od], axis=1)
 print("train_df_dogs", train_df_dogs.shape)
 plog("For Cats")
 train_df_cats=train_df[train_df["Type"]==2].reset_index().drop(["index"] ,axis=1)
 df_oc = detect_outliers(train_df_cats[o_cols+["AdoptionSpeed"]])
 print("df_oc", df_oc.shape, "train_df_cats", train_df_cats.shape)
 train_df_cats = pd.concat([train_df_cats, df_oc], axis=1)
 print("train_df_cats", train_df_cats.shape)
 plog("Concatenating dogs and cats datasets")
 train_df=pd.concat([train_df_dogs,train_df_cats])
 plog("Outlier detection ended")
 print("train_df", train_df.shape)
 print("test_df", test_df.shape)
 plog("Done" )<drop_column>
for col in(GENES + CELLS):
transformer = QuantileTransformer(n_quantiles=100,random_state=0, output_distribution="normal")
 vec_len = len(train_features[col].values)
 vec_len_test = len(test_features[col].values)
 raw_vec = train_features[col].values.reshape(vec_len, 1)
 transformer.fit(raw_vec)
 train_features[col] = transformer.transform(raw_vec ).reshape(1, vec_len)[0]
test_features[col] = transformer.transform(test_features[col].values.reshape(vec_len_test, 1)).reshape(1, vec_len_test)[0]
def prepare_data(train_df, test_df):
train_x = train_df.drop(["AdoptionSpeed", "outlier", "PetID"], axis=1)
 train_y = train_df[Columns.dep_columns.value]
test_x = test_df.drop(["PetID"], axis=1)
 test_id = test_df[Columns.iden_columns.value]
return train_x, train_y, test_x, test_id
outlier = 1
x_train, y_train, x_test, id_test = prepare_data(train_df[train_df["outlier"]>=outlier], test_df)
 x_train_dogs, y_train_dogs, x_test_dogs, id_test_dogs = prepare_data(train_df[(train_df["Type"]==1)&(train_df["outlier"]>=outlier)],
test_df[test_df["Type"]==1])
 x_train_cats, y_train_cats, x_test_cats, id_test_cats = prepare_data(train_df[(train_df["Type"]==2)&(train_df["outlier"]>=outlier)],
test_df[test_df["Type"]==2])
 plog("x_train information")
 print(x_train.shape)
 print(x_train.columns.values)
 plog("y_train information")
 print(y_train.shape)
 print(y_train.columns.values)
 plog("x_test information")
 print(x_test.shape)
 print(x_test.columns.values)
 plog("id_test information")
 print(id_test.shape)
 print(id_test.columns.values)
 pd.concat([x_train, y_train], axis=1 ).to_csv("train.csv", index=False)
 pd.concat([id_test, x_test], axis=1 ).to_csv("test.csv", index=False)
 plog("Done" )<compute_test_metric>
def seed_everything(seed=42):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=42 )
def conditional_entropy(x,y):
y_counter = Counter(y)
 xy_counter = Counter(list(zip(x,y)))
 total_occurrences = sum(y_counter.values())
 entropy = 0
for xy in xy_counter.keys() :
p_xy = xy_counter[xy] / total_occurrences
p_y = y_counter[xy[1]] / total_occurrences
entropy += p_xy * math.log(p_y/p_xy)
 return entropy
def theils_u(x, y):
s_xy = conditional_entropy(x,y)
 x_counter = Counter(x)
 total_occurrences = sum(x_counter.values())
 p_x = list(map(lambda n: n/total_occurrences, x_counter.values()))
s_x = entropy(p_x)
 if s_x == 0:
return 1
else:
return(s_x - s_xy)/ s_x
def by_theilsu(train,cols):
theilu = pd.DataFrame(index=cols, columns=cols)
 i = 0
for c1 in cols:
for c2 in cols:
u = theils_u(train[c1], train[c2])
 theilu.loc[c1, c2] = u
i = i+1
theilu.fillna(value=np.nan, inplace=True)
 return theilu
cols = []
for c in Columns.ind_num_cat_columns.value:
if c in x_train.columns.values.tolist() :
cols.append(c)
 df_a = pd.concat([x_train[cols],y_train],axis=1)
 print("Done")
<create_dataframe>
n_comp = 600
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[GENES]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )
if 1 == 0:
print("Effect of x on y by theils, ex: Effect of Breed1 on Type is 0.99")
 theils_a = by_theilsu(df_a,cols+ Columns.dep_columns.value)
 theilu_d = by_theilsu(df_a[df_a["Type"]==1],cols+Columns.dep_columns.value)
 theilu_c = by_theilsu(df_a[df_a["Type"]==2],cols+Columns.dep_columns.value)
 theils_all = pd.concat([theils_a.loc["AdoptionSpeed",:], theilu_d.loc["AdoptionSpeed",:], theilu_c.loc["AdoptionSpeed",:]], axis=1)
 theils_all = pd.DataFrame(index=(["All", "Dogs", "Cats"]), data=(theils_all.values.T), columns=theils_a.columns.values)
 plt.rcParams["figure.figsize"] = [20,5]
ax = plt.axes()
sns.heatmap(theils_all, annot=True, fmt='.4f', ax=ax)
 ax.set_title("Entrophy based categorical to categorical dependent correlation for all types")
 plt.show()<statistical_test>
n_comp = 50
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 =(PCA(n_components=n_comp, random_state=42 ).fit_transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )
def correlation_ratio(categories, measurements):
fcat, _ = pd.factorize(categories)
 cat_num = np.max(fcat)+1
y_avg_array = np.zeros(cat_num)
 n_array = np.zeros(cat_num)
 for i in range(0,cat_num):
cat_measures = measurements[np.argwhere(fcat == i ).flatten() ]
n_array[i] = len(cat_measures)
 y_avg_array[i] = np.average(cat_measures)
 y_total_avg = np.sum(np.multiply(y_avg_array,n_array)) /np.sum(n_array)
 numerator = np.sum(np.multiply(n_array,np.power(np.subtract(y_avg_array,y_total_avg),2)))
 denominator = np.sum(np.power(np.subtract(measurements,y_total_avg),2))
if numerator == 0:
eta = 0.0
else:
eta = numerator/denominator
return eta
def by_correlation_ratio(df, dep_col, cols):
cr = pd.DataFrame(columns=cols)
 i = 0
for c in cols:
eta = correlation_ratio(df[dep_col], df[c])
 cr.loc[i, c] = eta
cr.fillna(value=np.nan, inplace=True)
 return cr
print("Done" )<concatenate>
var_thresh = VarianceThreshold(0.8)
 data = train_features.append(test_features)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
 train_features.shape
if 1 == 0:
cols = []
for c in Columns.ind_num_cat_columns.value:
if c in x_train.columns.values.tolist() :
cols.append(c)
 cr = by_correlation_ratio(pd.concat([x_train, y_train] ,axis=1 ).reset_index() , Columns.dep_columns.value[0], cols)
 cr_dogs = by_correlation_ratio(pd.concat([x_train_dogs, y_train_dogs] ,axis=1 ).reset_index() , Columns.dep_columns.value[0], cols)
 cr_cats = by_correlation_ratio(pd.concat([x_train_cats, y_train_cats] ,axis=1 ).reset_index() , Columns.dep_columns.value[0], cols)
 cr_all = pd.concat([cr, cr_dogs, cr_cats])
 df_cr_all = pd.DataFrame(index=(["All", "Dogs", "Cats"]), data=(cr_all.values), columns=cr_all.columns.values)
 plt.rcParams["figure.figsize"] = [20, 5]
ax = plt.axes()
sns.heatmap(df_cr_all, annot=True, fmt='.4f', ax=ax)
 ax.set_title('Correlations between independent categorical and dependant ordinal/continous variable')
 plt.show() 
<concatenate>
%%time
def fe_cluster(train, test, n_clusters_g = 35, n_clusters_c = 5, SEED = 123):
features_g = list(train.columns[4:776])
 features_c = list(train.columns[776:876])
 def create_cluster(train, test, features, kind = 'g', n_clusters = n_clusters_g):
train_ = train[features].copy()
test_ = test[features].copy()
data = pd.concat([train_, test_], axis = 0)
 kmeans = KMeans(n_clusters = n_clusters, random_state = SEED ).fit(data)
 train[f'clusters_{kind}'] = kmeans.labels_[:train.shape[0]]
test[f'clusters_{kind}'] = kmeans.labels_[train.shape[0]:]
train = pd.get_dummies(train, columns = [f'clusters_{kind}'])
 test = pd.get_dummies(test, columns = [f'clusters_{kind}'])
 return train, test
train, test = create_cluster(train, test, features_g, kind = 'g', n_clusters = n_clusters_g)
 train, test = create_cluster(train, test, features_c, kind = 'c', n_clusters = n_clusters_c)
 return train, test
train_features ,test_features=fe_cluster(train_features,test_features )
cols = []
for c in Columns.boxplot_cols.value + Columns.item_type_cols.value + Columns.kbin_cols.value + Columns.item_cnt_mtype_cols.value:
if c in x_train.columns.values.tolist() :
cols.append(c)
 corr = pd.concat([x_train[cols], y_train], axis=1)
 corr_dogs = pd.concat([x_train_dogs[cols], y_train_dogs], axis=1)
 corr_cats = pd.concat([x_train_cats[cols], y_train_cats], axis=1)
 corr_all = pd.concat([corr.corr('spearman' ).loc["AdoptionSpeed",:], corr_dogs.corr('spearman' ).loc["AdoptionSpeed",:], corr_cats.corr('spearman' ).loc["AdoptionSpeed",:]], axis=1)
 df_corr_all = pd.DataFrame(columns=(["All", "Dogs", "Cats"]), data=(corr_all.values), index=corr_all.index.values)
 plt.rcParams["figure.figsize"] = [10, 40]
ax = plt.axes()
sns.heatmap(df_corr_all.sort_values(by=['All']), annot=True, fmt='.4f', ax=ax)
 ax.set_title('Dataset correlation between independent continous/ordinal and dependent ordinal variables')
 plt.show()
df_corr_all.sort_values(by=["All"], ascending=False ).to_csv("spearman_for_continous.csv" )<create_dataframe>
%%time
def fe_stats(train, test):
features_g = list(train.columns[4:776])
 features_c = list(train.columns[776:876])
 for df in train, test:
df['g_sum'] = df[features_g].sum(axis = 1)
 df['g_mean'] = df[features_g].mean(axis = 1)
 df['g_std'] = df[features_g].std(axis = 1)
 df['g_kurt'] = df[features_g].kurtosis(axis = 1)
 df['g_skew'] = df[features_g].skew(axis = 1)
 df['c_sum'] = df[features_c].sum(axis = 1)
 df['c_mean'] = df[features_c].mean(axis = 1)
 df['c_std'] = df[features_c].std(axis = 1)
 df['c_kurt'] = df[features_c].kurtosis(axis = 1)
 df['c_skew'] = df[features_c].skew(axis = 1)
 df['gc_sum'] = df[features_g + features_c].sum(axis = 1)
 df['gc_mean'] = df[features_g + features_c].mean(axis = 1)
 df['gc_std'] = df[features_g + features_c].std(axis = 1)
 df['gc_kurt'] = df[features_g + features_c].kurtosis(axis = 1)
 df['gc_skew'] = df[features_g + features_c].skew(axis = 1)
 return train, test
train_features,test_features=fe_stats(train_features,test_features )
def by_kruskal(arr, dep_col):
cols = arr.columns.values.tolist()
cols.remove(dep_col)
 df = pd.DataFrame(columns=cols)
 for c in cols:
arg = []
for v in arr[c].unique() :
if(len(arr[arr[c] == v])>= 5):
arg.append(arr[arr[c] == v][dep_col])
 if len(arg)>=2:
H, pval = kruskal(*arg)
 if pval <= 0.01:
df.loc[0, c] = pval
if pval > 0.01:
plog("Accept NULL hypothesis - No significant difference between groups for "+ c + " within 99% of significance level with p-value " + str(pval))
df.loc[0, c] = pval
else:
plog("Not Tested Because of lack of at least 2 groupes having minimum 5 measurements")
 df.loc[0, c] = 0
return df<prepare_x_and_y>
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target = train[train_targets_scored.columns]
cols = []
for c in Columns.boxplot_cols.value + Columns.item_type_cols.value + Columns.kbin_cols.value + Columns.item_cnt_mtype_cols.value:
if c in x_train.columns.values.tolist() :
cols.append(c)
 df_a = pd.DataFrame(columns=["AdoptionSpeed"])
 for c in cols :
print(c)
 df = by_kruskal(pd.concat([x_train, y_train], axis=1)[[c] + Columns.dep_columns.value], c)
 df["Variable"] = c
df_a = pd.concat([df_a, df])
 df_a = df_a.reset_index().drop("index", axis=1)
 df_a.set_index("Variable", inplace=True)
 print(df_a[df_a.AdoptionSpeed.isnull() ])
 plt.rcParams["figure.figsize"] = [15, 100]
sns.heatmap(df_a.astype(float ).sort_values(by=['AdoptionSpeed']), annot=True)
 plt.show()<drop_column>
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1 )
if 1 == 0:
df_a2 = df_a.reset_index()
d_cols=""
for c in df_a2[df_a2["AdoptionSpeed"]==0]["Variable"].values.tolist() :
x_train.drop(c, axis=1, inplace=True)
 x_test.drop(c, axis=1, inplace=True)
 d_cols = c + ", " + d_cols
plog("Dropped " + d_cols )<concatenate>
target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()
if 1 == 0:
cols = []
for c in Columns.boxplot_cols.value + Columns.item_type_cols.value + Columns.kbin_cols.value + Columns.item_cnt_mtype_cols.value:
if c in x_train.columns.values.tolist() :
cols.append(c)
 cr = by_correlation_ratio(pd.concat([x_train, y_train] ,axis=1 ).reset_index() , Columns.dep_columns.value[0], cols)
 cr = cr.T
cr_dogs = by_correlation_ratio(pd.concat([x_train_dogs, y_train_dogs] ,axis=1 ).reset_index() , Columns.dep_columns.value[0], cols)
 cr_dogs = cr_dogs.T
cr_cats = by_correlation_ratio(pd.concat([x_train_cats, y_train_cats] ,axis=1 ).reset_index() , Columns.dep_columns.value[0], cols)
 cr_cats = cr_cats.T
cr_all = pd.concat([cr, cr_dogs, cr_cats], axis=1)
 df_cr_all = pd.DataFrame(data=(cr_all.values), index=cr_all.index)
 df_cr_all.columns = ["All", "Dogs", "Cats"]
ax = plt.axes()
plt.rcParams["figure.figsize"] = [5, 100]
sns.heatmap(df_cr_all, annot=True, fmt='.4f', ax=ax)
 ax.set_title('Correlations between independent continous variables and dependant ordinal variable(supposed as continous)')
 plt.show()
if 1 == 0:
cols = []
for index, row in df_cr_all.iterrows() :
if row["All"] <= 0.0003:
if index in Columns.boxplot_cols.value + Columns.item_type_cols.value + Columns.kbin_cols.value + Columns.item_cnt_mtype_cols.value:
if index in x_train.columns.values.tolist() :
cols.append(index)
 print(cols)
 x_train.drop(cols, axis=1, inplace=True)
 x_test.drop(cols, axis=1, inplace=True)
<save_to_csv>
def assign_folds(seed,fold):
scored = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
 targets = scored.columns[1:]
scored = scored.merge(train_drug, on='sig_id', how='left')
 scored = scored.iloc[non_ctl_idx]
vc = scored.drug_id.value_counts()
vc1 = vc.loc[vc<=18].index.sort_values()
vc2 = vc.loc[vc>18].index.sort_values()
dct1 = {}; dct2 = {}
skf = MultilabelStratifiedKFold(n_splits=fold, shuffle=True,
random_state=seed)
 tmp = scored.groupby('drug_id')[targets].mean().loc[vc1]
for fold,(idxT,idxV)in enumerate(skf.split(tmp,tmp[targets])) :
dd = {k:fold for k in tmp.index[idxV].values}
dct1.update(dd)
 skf = MultilabelStratifiedKFold(n_splits=fold, shuffle=True,
random_state=seed)
 tmp = scored.loc[scored.drug_id.isin(vc2)].reset_index(drop=True)
 for fold,(idxT,idxV)in enumerate(skf.split(tmp,tmp[targets])) :
dd = {k:fold for k in tmp.sig_id[idxV].values}
dct2.update(dd)
 scored['fold'] = scored.drug_id.map(dct1)
 scored.loc[scored.fold.isna() ,'fold'] =\
scored.loc[scored.fold.isna() ,'sig_id'].map(dct2)
 scored.fold = scored.fold.astype('int8')
 return scored['fold'].values
result = chi2(x_train[Columns.ind_num_cat_columns.value], y_train["AdoptionSpeed"].values.reshape(-1, 1))
chi2_pvals = []
for x in result[1]:
chi2_pvals.append(x)
 df_chi2_pvals = pd.DataFrame(index=Columns.ind_num_cat_columns.value, columns=["AdoptionSpeed"], data = chi2_pvals ).sort_values(by=["AdoptionSpeed"], ascending=False)
 df_chi2_pvals.to_csv("chitest_for_categorical.csv")
 if 1 == 1:
x_train_fs = x_train.copy()
mic = mutual_info_classif(x_train_fs.astype("float"), y_train, random_state=42)
 df_mic = pd.DataFrame(data=mic, index=x_train.columns.values.tolist() , columns=["importance"])
 print(df_mic)
 print(df_mic.sort_values(by=["importance"]))
df_mic.sort_values(by=["importance"] ).to_csv("df_mic.csv")
 try:
x_fpr = SelectFpr(mutual_info_classif, alpha=0.03 ).fit_transform(x_train_fs.astype("float"), y_train)
 df_x_fpr = pd.DataFrame(data=x_fpr, index=x_train.columns.values.tolist() , columns=["importance"])
 print(df_x_fpr)
 print(df_x_fpr.sort_values(by=["importance"]))
df_x_fpr.sort_values(by=["importance"] ).to_csv("x_fpr.csv")
 except Exception as e:
print(e)
 traceback.print_exc()
try:
x_fdr = SelectFdr(mutual_info_classif, alpha=0.03 ).fit_transform(x_train_fs.astype("float"), y_train)
 df_x_fdr = pd.DataFrame(data=x_fdr, index=x_train.columns.values.tolist() , columns=["importance"])
 print(df_x_fdr)
 print(df_x_fdr.sort_values(by=["importance"]))
df_x_fdr.sort_values(by=["importance"] ).to_csv("x_fdr.csv")
 except Exception as e:
print(e)
 traceback.print_exc()
try:
x_fwe = SelectFwe(mutual_info_classif, alpha=0.03 ).fit_transform(x_train_fs.astype("float"), y_train)
 print(x_fwe)
 df_x_fwe = pd.DataFrame(data=x_fwe, index=x_train.columns.values.tolist() , columns=["importance"])
 print(df_x_fwe)
 print(df_x_fwe.sort_values(by=["importance"]))
df_x_fwe.sort_values(by=["importance"] ).to_csv("x_fwe.csv")
 except Exception as e:
print(e)
 traceback.print_exc()<import_modules>
folds = train.copy()
seed_folds = train.copy()
val_folds = assign_folds(34,5)
 folds['kfold'] = val_folds
folds
def report(df, alg, best_est, perf, est, results, n_top=3):
for i in range(1, n_top + 1):
candidates = np.flatnonzero(results['rank_test_score'] == i)
 for candidate in candidates:
df.loc[len(df)] = [alg, best_est, perf, est, format(i), results['mean_test_score'][candidate],results['std_test_score'][candidate],results['params'][candidate]]
return df
layer_sizes = []
for i in range(0,100):
t =(np.random.random_integers(25,300), np.random.random_integers(25,300), np.random.random_integers(25,300))
layer_sizes.append(t)
 def iterate_by_randomsearch(train_x, train_y):
classifiers = [ 
(AdaBoostClassifier() , {"n_estimators": sp.stats.randint(25, 100),
'learning_rate': sp.stats.uniform(0.0001, 1)}), 
(lgb.LGBMClassifier() , {'num_leaves': sp.stats.randint(25, 330),
'n_estimators': sp.stats.randint(25, 150),
'learning_rate': sp.stats.uniform(0.001, 0.5),
'boosting_type' : ['gbdt', 'dart'],
'max_depth': sp.stats.randint(3,30),
'min_split_gain': sp.stats.uniform(0.001, 0.5),
'objective': 'multiclass',
"n_jobs":[4]})
 ]
print(len(train_x), len(train_y))
df = pd.DataFrame(columns=['alg', 'best_estimator', 'perf', 'est','rank','mean','std', 'parameters'])
 for clf in classifiers:
n_iter=10
kappa_scorer = make_scorer(cohen_kappa_score, weights="quadratic")
 kfold = RepeatedStratifiedKFold(n_splits=5, n_repeats=2, random_state=42)
 cvs = kfold.split(train_x, train_y)
 print(cvs)
 random_search = RandomizedSearchCV(clf[0], param_distributions=clf[1], verbose=8,
cv=cvs, n_iter=10, scoring=kappa_scorer, n_jobs = 4)
 print(type(clf[0] ).__name__, "started at", datetime.now())
 start = time()
random_search.fit(train_x, train_y)
 df = report(df, type(clf[0] ).__name__, random_search.best_estimator_ , time() - start, n_iter, random_search.cv_results_)
 print(type(clf[0] ).__name__, "ended at", datetime.now())
 best = df['mean'].idxmax()
est = df.loc[best, ["best_estimator", "mean"]]
i = 1
clfs = []
means = []
clf_p=""
dt=np.dtype('str,float')
 for idx, clf in df.sort_values(by="mean", ascending=False ).iterrows() :
if i<6:
clfs.append(( str(i)+type(clf["best_estimator"] ).__name__, clf["best_estimator"]))
means.append(clf["mean"])
 clf_p = type(clf["best_estimator"] ).__name__
i = i+1
return clfs, means
print("Done" )<find_best_model_class>
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct
def voting_predict(clfs, means, train_x, train_y, test_x, test_id):
clf = VotingClassifier(estimators=clfs, voting = 'soft')
 clf.fit(train_x, train_y.values.ravel())
 pred = clf.predict(test_x)
 prediction_df = pd.DataFrame({'PetID': test_id.values.ravel() ,
'AdoptionSpeed': pred})
 return prediction_df
def voting_predict_with_weights(clfs, means, train_x, train_y, test_x, test_id):
clf = VotingClassifier(estimators=clfs, weights=means, voting = 'soft')
 clf.fit(train_x, train_y.values.ravel())
 pred = clf.predict(test_x)
 prediction_df = pd.DataFrame({'PetID': test_id.values.ravel() ,
'AdoptionSpeed': pred})
 return prediction_df
print("Done" )<find_best_model_class>
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
if 1 == 0:
clfs, means = iterate_by_randomsearch(x_train, y_train.values.ravel())
 print("--------clfs--------")
 print(clfs)
 print("--------means--------")
 print(means)
 pred = voting_predict(clfs, means, x_train, y_train, x_test, id_test)
 class_sub = pred
print("Done" )<save_to_csv>
class SmoothBCEwLogits(_WeightedLoss):
def __init__(self, weight=None, reduction='mean', smoothing=0.0):
super().__init__(weight=weight, reduction=reduction)
 self.smoothing = smoothing
self.weight = weight
self.reduction = reduction
@staticmethod
def _smooth(targets:torch.Tensor, n_labels:int, smoothing=0.0):
assert 0 <= smoothing < 1
with torch.no_grad() :
targets = targets *(1.0 - smoothing)+ 0.5 * smoothing
return targets
def forward(self, inputs, targets):
targets = SmoothBCEwLogits._smooth(targets, inputs.size(-1),
self.smoothing)
 loss = F.binary_cross_entropy_with_logits(inputs, targets,self.weight)
 if self.reduction == 'sum':
loss = loss.sum()
elif self.reduction == 'mean':
loss = loss.mean()
return loss
if 1 == 0:
x_train_a = x_train
x_test_a = x_test
clfs, means = iterate_by_randomsearch(x_train_a, y_train.values.ravel())
 print(clfs)
 print(means)
 pred = voting_predict(clfs, means, x_train_a, y_train, x_test_a, id_test)
 pred.to_csv("submission.csv",index=False)
 print("Done" )<compute_train_metric>
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(0.4)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(0.4)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = F.relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x
def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):

 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_ratings = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_ratings)]
for j in range(num_ratings)]
for a, b in zip(rater_a, rater_b):
x = a - min_rating
y = b - min_rating
if not np.isscalar(x):
x=x[0]
if not np.isscalar(y):
y=y[0]
conf_mat[x][y] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):

 if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
x = r - min_rating
if not np.isscalar(x):
x = x[0]
hist_ratings[x] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):

 rater_a = y
rater_b = y_pred
min_rating = None
max_rating = None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(min(rater_a), min(rater_b))
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b,
min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(hist_rater_a[i] * hist_rater_b[j]
/ num_scored_items)
 d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator)
 class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = quadratic_weighted_kappa(y, X_p)
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 print(self.coef_)
 def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']
def rmse(actual, predicted):
return sqrt(mean_squared_error(actual, predicted))
def run_cv_model(train, test, target, model_fn, cat_features, params={}, eval_fn=None, label='model', n_splits=5, n_repeats=2):
kf = RepeatedStratifiedKFold(n_splits=n_splits, random_state=42, n_repeats = n_repeats)
 fold_splits = kf.split(train, target)
 cv_scores = []
qwk_scores = []
pred_full_test = 0
pred_train = np.zeros(( train.shape[0], n_splits*n_repeats))
all_coefficients = np.zeros(( n_splits*n_repeats, 4))
feature_importance_df = pd.DataFrame()
i = 1
for dev_index, val_index in fold_splits:
if isinstance(train, pd.DataFrame):
dev_X, val_X = train.iloc[dev_index], train.iloc[val_index]
else:
dev_X, val_X = train[dev_index], train[val_index]
if isinstance(target, pd.DataFrame):
dev_y, val_y = target.iloc[dev_index], target.iloc[val_index]
else:
dev_y, val_y = target[dev_index], target[val_index]
params2 = params.copy()
pred_val_y, pred_test_y, importances, coefficients, qwk = model_fn(dev_X, dev_y, val_X, val_y, test, params2, cat_features)
 pred_full_test = pred_full_test + pred_test_y
pred_train[val_index] = pred_val_y
all_coefficients[i-1, :] = coefficients
if eval_fn is not None:
cv_score = eval_fn(val_y, pred_val_y)
 cv_scores.append(cv_score)
 qwk_scores.append(qwk)
 fold_importance_df = pd.DataFrame()
if isinstance(train, pd.DataFrame):
fold_importance_df['feature'] = train.columns.values
else:
fold_importance_df['feature'] = ["pca_"+ str(i)for i in range(train.shape[1])]
fold_importance_df['importance'] = importances
fold_importance_df['fold'] = i
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 i += 1
pred_full_test = pred_full_test /(n_splits*n_repeats)
 results = {'label': label,
'train': pred_train, 'test': pred_full_test,
'cv': cv_scores, 'qwk': qwk_scores,
'importance': feature_importance_df,
'coefficients': all_coefficients}
return results
def runLGB(train_X, train_y, test_X, test_y, test_X2, params, cat_features):
d_train = lgb.Dataset(train_X, label=train_y)
 d_valid = lgb.Dataset(test_X, label=test_y)
 watchlist = [d_train, d_valid]
num_rounds = params.pop('num_rounds')
 verbose_eval = params.pop('verbose_eval')
 early_stop = None
if params.get('early_stop'):
early_stop = params.pop('early_stop')
 if len(cat_features)> 0:
model = lgb.train(params,
categorical_feature=list(cat_features),
train_set=d_train,
num_boost_round=num_rounds,
valid_sets=watchlist,
verbose_eval=verbose_eval,
early_stopping_rounds=early_stop)
 else:
model = lgb.train(params,
train_set=d_train,
num_boost_round=num_rounds,
valid_sets=watchlist,
verbose_eval=verbose_eval,
early_stopping_rounds=early_stop)
 pred_test_y = model.predict(test_X, num_iteration=model.best_iteration)
 optR = OptimizedRounder()
optR.fit(pred_test_y, test_y)
 coefficients = optR.coefficients()
pred_test_y_k = optR.predict(pred_test_y, coefficients)
 qwk = quadratic_weighted_kappa(test_y, pred_test_y_k)
 pred_test_y2 = model.predict(test_X2, num_iteration=model.best_iteration)
 return pred_test_y.reshape(-1, 1), pred_test_y2.reshape(-1, 1), model.feature_importance(importance_type="gain"), coefficients, qwk
plog("Done" )<compute_train_metric>
def process_data(data):
data = pd.get_dummies(data, columns=['cp_time','cp_dose'])
 return data
def by_regressor_rs(train, test, y_train, runALG, metric, name, cv, i, id_test):
mqwk = 0
j = 0
for j in range(10):
params = {'application': 'regression',
'boosting': 'gbdt',
'metric': 'rmse',
'num_leaves': random.randint(50, 300),
'max_depth': random.randint(5, 25),
'learning_rate': random.uniform(0.001, 0.05),
'bagging_fraction': random.uniform(0.5, 1),
'feature_fraction': random.uniform(0.5, 1),
'min_split_gain': random.uniform(0.001, 0.05),
'min_child_samples': random.randint(75, 200),
'min_child_weight': random.uniform(0.001, 0.05),
'verbosity': -1,
'data_random_seed': 3,
'early_stop': 100,
'verbose_eval': False,
'n_jobs':4,
'lambda_l2': random.uniform(0.001, 0.1),
'num_rounds': 10000}
print("RS prms", params)
 results_t = run_cv_model(train, test, y_train, runALG, params, metric, name, cv, i)
 print("RS QWK Scores", results_t["qwk"], "rs mean qwk scores", np.mean(results_t["qwk"]), "rs mean rmse scores" , np.mean(results_t["cv"]))
if np.mean(results_t["qwk"])> mqwk:
results = results_t
mqwk = np.mean(results_t["qwk"])
 optR = OptimizedRounder()
coefficients_ = np.mean(results['coefficients'], axis=0)
 print(coefficients_)
 train_predictions = [r[0] for r in results['train']]
train_predictions = optR.predict(train_predictions, coefficients_ ).astype(int)
 Counter(train_predictions)
 optR = OptimizedRounder()
test_predictions = [r[0] for r in results['test']]
test_predictions = optR.predict(test_predictions, coefficients_ ).astype(int)
 Counter(test_predictions)
 pd.DataFrame(sk_cmatrix(y_train, train_predictions), index=list(range(5)) , columns=list(range(5)))
 submission = pd.DataFrame({'PetID': id_test.PetID.values, 'AdoptionSpeed': test_predictions})
 return submission<save_to_csv>
feature_cols = [c for c in process_data(folds ).columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold','sig_id']]
len(feature_cols )
def by_regressor(train, test, y_train, runALG, prms, metric, name, cv, i, id_test, pca, c_features):
results = run_cv_model(train, test, y_train, runALG, c_features, prms, metric, name, cv, i)
 print("RS QWK Scores", results["qwk"], "rs mean qwk scores", np.mean(results["qwk"]), "rs mean rmse scores" , np.mean(results["cv"]))
grp = results["importance"].groupby('feature',as_index=False)['importance'].mean()
print(grp.head())
 grp2 = grp.sort_values(by=["importance"])
 grp2.plot(kind="barh", y="importance", x="feature")
 grp2.to_csv("feature_importances.csv")
 optR = OptimizedRounder()
coefficients_ = np.mean(results['coefficients'], axis=0)
 print(coefficients_)
 train_predictions = [r[0] for r in results['train']]
train_predictions = optR.predict(train_predictions, coefficients_ ).astype(int)
 Counter(train_predictions)
 optR = OptimizedRounder()
test_predictions = [r[0] for r in results['test']]
test_predictions = optR.predict(test_predictions, coefficients_ ).astype(int)
 Counter(test_predictions)
 pd.DataFrame(sk_cmatrix(y_train, train_predictions), index=list(range(5)) , columns=list(range(5)))
 submission = pd.DataFrame({'PetID': id_test.PetID.values, 'AdoptionSpeed': test_predictions})
 return submission
plog("Done" )<init_hyperparams>
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 25
BATCH_SIZE = 128
LEARNING_RATE = 1e-3
WEIGHT_DECAY = 1e-5
NFOLDS = 5
EARLY_STOPPING_STEPS = 10
EARLY_STOP = False
num_features=len(feature_cols)
 num_targets=len(target_cols)
 hidden_size=2048
if 1 == 1:
warnings.filterwarnings("ignore")
 params = {'application': 'regression',
'boosting': 'gbdt',
'metric': 'rmse',
'max_depth': 11,
'num_leaves': 350,
'learning_rate': 0.01,
'bagging_fraction': 0.85,
'feature_fraction': 0.8,
'min_split_gain': 0.01,
'min_child_samples': 75,
'min_child_weight': 0.1,
'verbosity': -1,
'data_random_seed': 3,
'early_stop': 100,
'verbose_eval': False,
'n_jobs':4,
'num_rounds': 10000, }
pca = 0
if pca == 1:
kPCA = KernelPCA(n_components=150, kernel='rbf')
 kPCA.fit(x_train)
 x_train = kPCA.transform(x_train)
 x_test = kPCA.transform(x_test)
 x_train_a = x_train.drop(low_imp_features, axis=1)
 x_test_a = x_test.drop(low_imp_features, axis=1)
 if pca == 0:
cat_features = [x_train_a.columns.get_loc(c)for c in x_train_a.columns if c in Columns.ind_num_cat_columns.value]
else:
cat_features = []
plog("listing categorical features")
 print(cat_features)
 if call==1:
submission = by_regressor(x_train_a, x_test_a, y_train, runLGB, params, rmse, 'lgb', 5, 2, id_test, pca ,cat_features)
 reg_sub = submission
submission.to_csv('submission.csv', index=False)
 else:
x_train_dogs.drop(["Type"], axis=1, inplace=True)
 x_test_dogs.drop(["Type"], axis=1, inplace=True)
 submission_d = by_regressor(x_train_dogs, x_test_dogs, y_train_dogs, runLGB, params, rmse, 'lgb', 5, 2, id_test_dogs)
 x_train_cats.drop(["Type"], axis=1, inplace=True)
 x_test_cats.drop(["Type"], axis=1, inplace=True)
 submission_c = by_regressor(x_train_cats, x_test_cats, y_train_cats, runLGB, params, rmse, 'lgb', 5, 2, id_test_cats)
 submission = pd.concat([submission_d, submission_c])
 submission.to_csv('submission.csv', index=False )<save_to_csv>
def run_training(fold, seed):
seed_everything(seed)
 train = process_data(folds)
 test_ = process_data(test)
 asn_folds = assign_folds(seed,5)
 seed_folds['kfold'] = asn_folds
trn_idx = train[seed_folds['kfold'] != fold].index
val_idx = train[seed_folds['kfold'] == fold].index
train_df = train[seed_folds['kfold'] != fold].reset_index(drop=True)
 valid_df = train[seed_folds['kfold'] == fold].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1e-2, epochs=EPOCHS, steps_per_epoch=len(trainloader))
loss_fn = nn.BCEWithLogitsLoss()
loss_tr = SmoothBCEwLogits(smoothing =0.001)
 early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
best_loss = np.inf
for epoch in range(EPOCHS):
train_loss = train_fn(model, optimizer,scheduler, loss_tr, trainloader, DEVICE)
 print(f"SEED: {seed}, FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}")
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"SEED: {seed} ,FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"RankGauss_{fold}_.pth")
 elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.load_state_dict(torch.load(f"RankGauss_{fold}_.pth"))
model.to(DEVICE)
 predictions = np.zeros(( len(test_), target.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions
if 1 == 0:
x_train_a = x_train.drop(low_imp_features, axis=1)
 x_test_a = x_test.drop(low_imp_features, axis=1)
 submission = by_regressor_rs(x_train_a, x_test_a, y_train, runLGB, rmse, 'lgb', 5, 2, id_test)
<choose_model_class>
def run_k_fold(NFOLDS, seed):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
oof_, pred_ = run_training(fold, seed)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions
class RatioOrdinalClassfier(BaseEstimator, ClassifierMixin):

 def __init__(self, estimator=lgb.LGBMClassifier()):

 self.estimator = estimator
self.estimators_ = []
def encode_classes(self, y, yi):
if y[0] > yi:
return 1
else:
return 0
def fit(self, X, y):

 self.sorted_classes_ = np.unique(y)
 self.num_classes_ = len(self.sorted_classes_)
 self.num_instances_ = len(X)
 self.probas_ = np.zeros(( self.num_instances_, 1))
for yi in self.sorted_classes_[:-1]:
yt = y.copy()
yt[yt <= yi] = 0
yt[yt > yi] = 1
est = self.estimator
est.fit(X, yt.ravel())
 filename = "ownestimatormodel_"+str(yi)+".sav"
with open(filename, 'wb')as file:
pickle.dump(est, file)
 with open(filename ,'rb')as f:
est = pickle.load(f)
 self.estimators_.append(est)
 return self
def predict_proba(self, X):
i = 0
for yi in self.sorted_classes_[:-1]:
filename = "ownestimatormodel_"+str(yi)+".sav"
with open(filename ,'rb')as f:
est = pickle.load(f)
 yt_proba = est.predict_proba(X)[:, 1:2]
if i == 0:
self.probas_ = yt_proba
else:
self.probas_ = np.concatenate(( self.probas_, yt_proba), axis=1)
 i += 1
def predict(self, X):
self.predict_proba(X)
 ypf = np.zeros(( self.num_instances_,1))
try:
getattr(self, "probas_")
 for i in range(self.num_classes_):
if i == 0:
ypi = 1 - self.probas_[:, 0:1]
elif 0 < i < self.num_classes_-1:
ypi = self.probas_[:, i-1:i] - self.probas_[:, i:i+1]
elif i == self.num_classes_-1:
ypi = self.probas_[:, i-1:i]
if i == 0:
ypf = ypi
else:
ypf = np.concatenate(( ypf,ypi), axis=1)
 return np.argmax(ypf, axis=1 ).reshape(len(ypf), 1)
 except AttributeError:
raise RuntimeError("You must train classifer before predicting data!")
 if 1 == 0:
params = {
'boosting_type': 'gbdt',
'metric': 'cohen_kappa_score',
'max_depth': 9,
'num_leaves': 70,
'learning_rate': 0.01,
'bagging_fraction': 0.85,
'feature_fraction': 0.8,
'min_split_gain': 0.01,
'min_child_samples': 50,
'min_child_weight': 0.1,
'verbosity': -1,
'data_random_seed': 3,
'n_jobs': 4,
'objective':'binary',
}
est = lgb.LGBMClassifier(**params)
 clf = RatioOrdinalClassfier(estimator=est)
 clf.fit(x_train.values,y_train.values)
 pred_y = clf.predict(x_train.values)
 print(cohen_kappa_score(y_train, pred_y))
submission_p = clf.predict(x_test.values)
 print(type(id_test), type(id_test.values), type(submission_p))
submission_df = pd.DataFrame(data=submission_p, columns=["AdoptionSpeed"])
 submission = pd.concat([id_test, submission_df], axis=1)
 submission.to_csv('submission.csv', index=False )<save_to_csv>
%%time
SEED = [34, 14, 75]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for seed in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 train[target_cols] = oof
test[target_cols] = predictions
if 1==0:
merge_sub =(class_sub["AdoptionSpeed"].astype('int32')+ reg_sub["AdoptionSpeed"].astype('int32')) /2
submission_df = pd.DataFrame(data=merge_sub, columns=["AdoptionSpeed"])
 submission_df["AdoptionSpeed"] = submission_df.apply(lambda x: math.floor(x['AdoptionSpeed']), axis=1)
 submission = pd.concat([id_test, submission_df], axis=1)
 submission.to_csv('submission.csv', index=False )<categorify>
torch.cuda.empty_cache()
if 1 == 0 :
x_train_k = x_train.copy()
x_test_k = x_test.copy()
id_test_k = id_test.copy()
y_train_k = np.zeros(( len(y_train), 5))
i = 0
for x in y_train["AdoptionSpeed"].values:
if x == 1:
y_train_k[i,2] = 1
elif x == 2:
y_train_k[i, 1:3] = 1
elif x == 3:
y_train_k[i, 1:4] = 1
elif x == 4:
y_train_k[i, 1:5] = 1
i += 1
print(y_train_k)
 train_cat = np.array([])
 test_cat = np.array([])
 for c in Columns.ind_num_cat_columns.value:
val_cats = np.unique(x_train_k[c].values.tolist() + x_test_k[c].values.tolist())
 ohe = OneHotEncoder(categories=[val_cats], sparse=False)
 print(datetime.now() , c)
 print(x_train_k[c].values.shape)
 x_train_k[c] = x_train_k[c].astype('int32')
 t_cat = ohe.fit_transform(x_train_k[c].values.reshape(-1, 1))
if train_cat.size == 0:
train_cat = t_cat
else:
train_cat = np.concatenate(( train_cat, t_cat), axis=1)
 x_train_k.drop(c, inplace=True, axis=1)
 x_test_k[c] = x_test_k[c].astype('int32')
 te_cat = ohe.fit_transform(x_test_k[c].values.reshape(-1, 1))
if test_cat.size == 0:
test_cat = te_cat
else:
test_cat = np.concatenate(( test_cat, te_cat), axis=1)
 x_test_k.drop(c, inplace=True, axis=1)
 print(train_cat.shape, test_cat.shape)
 print(datetime.now() , "x_train and x_test to ndarray")
 x_train_k = np.array(x_train_k.values)
 x_test_k = np.array(x_test_k.values)
 print(datetime.now() , "concatenate x_train/x_test and train_cat/test_cat")
 x_train_k = np.concatenate(( x_train_k, train_cat), axis=1)
 x_test_k = np.concatenate(( x_test_k, test_cat), axis=1)
 print(datetime.now() , x_train_k.shape, y_train_k.shape)
 partial_x_train, x_val, partial_y_train, y_val = train_test_split(x_train_k, y_train_k, test_size=0.20, random_state=42, stratify=y_train_k)
 model = models.Sequential()
model.add(layers.Dense(int(partial_x_train), kernel_regularizer=regularizers.l2(0.001), activation='relu', input_shape=(partial_x_train.shape[1],)))
 model.add(layers.Dense(int(85), kernel_regularizer=regularizers.l2(0.001), activation='relu'))
model.add(layers.Dense(5, activation='softmax'))
model.compile(optimizer='rmsprop',
loss='categorical_crossentropy',
metrics=['accuracy'])
 history = model.fit(partial_x_train, partial_y_train, epochs=100,
batch_size=256, validation_data=(x_val, y_val))
loss = history.history['loss']
val_loss = history.history['val_loss']
epochs = range(1, len(loss)+1)
 plt.rcParams["figure.figsize"] = [20, 10]
plt.plot(epochs, loss, 'bo', label='Training loss')
 plt.plot(epochs, val_loss, 'b', label='Validation loss')
 plt.title('Training and validation loss')
 plt.xlabel('Epochs')
 plt.ylabel('Loss')
 plt.legend()
plt.show()
acc = history.history['acc']
val_acc = history.history['val_acc']
plt.plot(epochs, acc, 'bo', label='Training acc')
 plt.plot(epochs, val_acc, 'b', label='Validation acc')
 plt.title('Training and validation acc')
 plt.xlabel('Epochs')
 plt.ylabel('Acc')
 plt.legend()
plt.show()
pred_labels = model.predict_classes(x_val)
 true_labels = np.argmax(y_val, axis=1)
 print(cohen_kappa_score(pred_labels,true_labels))
pred = model.predict_classes(x_test_k)
 id_test_k = np.array(id_test_k.values)
 print(id_test_k.shape, pred.shape)
 submission = np.concatenate(( id_test_k, pred.reshape(-1,1)) , axis=1)
 print(submission.shape)
 print(submission)
 submission_df = pd.DataFrame(columns=["PetID, AdoptionSpeed"], data=submission)
 print(submission_df.groupby(['AdoptionSpeed'] ).size())
<compute_test_metric>
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score_ = log_loss(y_true[:, i], y_pred[:, i])
 score += score_ / target.shape[1]
print("CV log_loss: ", score )
@contextmanager
def timer(name):

 t0 = time.time()
yield
print(f'[{name}] done in {time.time() - t0:.0f} s')
 def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):

 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(rater_a + rater_b)
 if max_rating is None:
max_rating = max(rater_a + rater_b)
 num_ratings = int(max_rating - min_rating + 1)
 conf_mat = [[0 for i in range(num_ratings)]
for j in range(num_ratings)]
for a, b in zip(rater_a, rater_b):
conf_mat[a - min_rating][b - min_rating] += 1
return conf_mat
def histogram(ratings, min_rating=None, max_rating=None):

 if min_rating is None:
min_rating = min(ratings)
 if max_rating is None:
max_rating = max(ratings)
 num_ratings = int(max_rating - min_rating + 1)
 hist_ratings = [0 for x in range(num_ratings)]
for r in ratings:
hist_ratings[r - min_rating] += 1
return hist_ratings
def quadratic_weighted_kappa(y, y_pred):

 rater_a = y
rater_b = y_pred
min_rating=None
max_rating=None
rater_a = np.array(rater_a, dtype=int)
 rater_b = np.array(rater_b, dtype=int)
 assert(len(rater_a)== len(rater_b))
if min_rating is None:
min_rating = min(min(rater_a), min(rater_b))
if max_rating is None:
max_rating = max(max(rater_a), max(rater_b))
conf_mat = confusion_matrix(rater_a, rater_b,
min_rating, max_rating)
 num_ratings = len(conf_mat)
 num_scored_items = float(len(rater_a))
hist_rater_a = histogram(rater_a, min_rating, max_rating)
 hist_rater_b = histogram(rater_b, min_rating, max_rating)
 numerator = 0.0
denominator = 0.0
for i in range(num_ratings):
for j in range(num_ratings):
expected_count =(hist_rater_a[i] * hist_rater_b[j]
/ num_scored_items)
 d = pow(i - j, 2.0)/ pow(num_ratings - 1, 2.0)
 numerator += d * conf_mat[i][j] / num_scored_items
denominator += d * expected_count / num_scored_items
return(1.0 - numerator / denominator)
 class OptimizedRounder(object):
def __init__(self):
self.coef_ = 0
def _kappa_loss(self, coef, X, y):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
ll = quadratic_weighted_kappa(y, X_p)
 return -ll
def fit(self, X, y):
loss_partial = partial(self._kappa_loss, X=X, y=y)
 initial_coef = [0.5, 1.5, 2.5, 3.5]
self.coef_ = sp.optimize.minimize(loss_partial, initial_coef, method='nelder-mead')
 def predict(self, X, coef):
X_p = np.copy(X)
 for i, pred in enumerate(X_p):
if pred < coef[0]:
X_p[i] = 0
elif pred >= coef[0] and pred < coef[1]:
X_p[i] = 1
elif pred >= coef[1] and pred < coef[2]:
X_p[i] = 2
elif pred >= coef[2] and pred < coef[3]:
X_p[i] = 3
else:
X_p[i] = 4
return X_p
def coefficients(self):
return self.coef_['x']
def cloud(text, title, size =(10,7)) :
wordcloud = WordCloud(width=800, height=400,
collocations=True
).generate(" ".join(text))
fig = plt.figure(figsize=size, dpi=80, facecolor='k',edgecolor='k')
 plt.imshow(wordcloud,interpolation='bilinear')
 plt.axis('off')
 plt.title(title, fontsize=25,color='w')
 plt.tight_layout(pad=0)
 plt.show()<load_from_csv>
sub_gauss = sub_rankgauss.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub_gauss
print(os.listdir(".. /input/train"))
train = pd.read_csv(".. /input/train/train.csv")
 test = pd.read_csv(".. /input/test/test.csv")
 breed_label = pd.read_csv('.. /input/breed_labels.csv')
 train.Name = train.Name.astype(str)
 test.Name = test.Name.astype(str)
 train.Description = train.Description.astype(str)
 test.Description = test.Description.astype(str)
 display(train.sample(4))<load_from_disk>
!pip install --no-index --find-links /kaggle/input/pytorchtabnet/pytorch_tabnet-2.0.0-py3-none-any.whl pytorch-tabnet
samplesentiment = pd.read_json('.. /input/train_sentiment/{}'.format("4fdebca57.json"), orient='index', typ='series')
 print("Document Sentiment")
 print(samplesentiment["documentSentiment"])
 print("
Entities")
 print(samplesentiment["entities"][0])
 print("
Sentences")
 print(samplesentiment['sentences'][0] )<create_dataframe>
os.environ["CUDA_LAUNCH_BLOCKING"] = '1'
with timer("Process Sentiment Train Data"):
sentiment_list = os.listdir('.. /input/train_sentiment')
 train_sentiment_df = pd.DataFrame()
for i,x in enumerate(sentiment_list):
samplesentiment = pd.read_json('.. /input/train_sentiment/{}'.format(x), orient='index', typ='series')
 sentences = json_normalize(samplesentiment.sentences ).loc[:,['sentiment.magnitude', 'sentiment.score']].agg(
 {
'sentiment.magnitude' : ['count','mean','std'],
'sentiment.score' : ['mean','std', 'sum']
} ).unstack().to_frame().sort_index(level=1 ).T
sentences.columns = sentences.columns.map('_'.join)
 sentiment = pd.concat([json_normalize(samplesentiment["documentSentiment"]),
sentences,
], axis =1)
 sentiment.index = [x[:9]]
train_sentiment_df = pd.concat([train_sentiment_df, sentiment], axis =0)
 display(train_sentiment_df.sample(5))
train = train.join(train_sentiment_df, on = "PetID")
 del train_sentiment_df<create_dataframe>
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 df = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )
with timer("Process Sentiment Test Data"):
sentiment_list = os.listdir('.. /input/test_sentiment/')
 test_sentiment_df = pd.DataFrame()
for i,x in enumerate(sentiment_list):
samplesentiment = pd.read_json('.. /input/test_sentiment/{}'.format(x), orient='index', typ='series')
 sentences = json_normalize(samplesentiment.sentences ).loc[:,['sentiment.magnitude', 'sentiment.score']].agg(
 {
'sentiment.magnitude' : ['count','mean','std'],
'sentiment.score' : ['mean','std', 'sum']
} ).unstack().to_frame().sort_index(level=1 ).T
sentences.columns = sentences.columns.map('_'.join)
 sentiment = pd.concat([json_normalize(samplesentiment["documentSentiment"]),
sentences,
], axis =1)
 sentiment.index = [x[:9]]
test_sentiment_df = pd.concat([test_sentiment_df, sentiment], axis =0)
 display(test_sentiment_df.sample(5))
test = test.join(test_sentiment_df, on = "PetID")
 del test_sentiment_df<load_from_disk>
train_features2=train_features.copy()
test_features2=test_features.copy()
sample_image_metadata = pd.read_json('.. /input/train_metadata/{}'.format("c161afd26-3.json"), orient='index', typ='series')
 print("Label Annotations")
 print(sample_image_metadata['labelAnnotations'])
 print("
Image Properties Annotations")
 print(sample_image_metadata['imagePropertiesAnnotation'])
 print("
Crop Hints Annotation")
 print(sample_image_metadata['cropHintsAnnotation'] )<prepare_x_and_y>
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
train.set_index("PetID", inplace= True)
 test.set_index("PetID", inplace= True)
 traindex = train.index
testdex = test.index
y = train.AdoptionSpeed
train.drop('AdoptionSpeed', axis =1 , inplace=True)
 df = pd.concat([train,test], axis = 0)
 df = df.merge(breed_label, left_on = 'Breed1', right_on = 'BreedID', how = 'left', suffixes = ['','b1'] ).drop('BreedID',axis=1)
 df.rename(columns = {'BreedName':'b1name'}, inplace=True)
 df = df.merge(breed_label, left_on = 'Breed2', right_on = 'BreedID', how = 'left', suffixes = ['','b2'] ).drop('BreedID',axis=1)
 df.rename(columns = {'BreedName':'b2name'}, inplace=True )<categorify>
qt = QuantileTransformer(n_quantiles=100,random_state=42,output_distribution='normal')
 train_features[GENES+CELLS] = qt.fit_transform(train_features[GENES+CELLS])
 test_features[GENES+CELLS] = qt.transform(test_features[GENES+CELLS] )
print("Encode Variables")
 categorical = ["Name"]
print("Encoding :",categorical)
 df.Name = df.Name.str.lower()
lbl = preprocessing.LabelEncoder()
for col in categorical:
df["encoded_" + col] = lbl.fit_transform(df[col].astype(str))
df.drop('RescuerID', axis =1 ,inplace=True )<feature_engineering>
seed = 42
def set_seed(seed):
random.seed(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 os.environ["PYTHONHASHSEED"] = str(seed)
 if torch.cuda.is_available() :
torch.cuda.manual_seed(seed)
 torch.cuda.manual_seed_all(seed)
 torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
set_seed(seed )
textfeats = ["Description"]
for cols in textfeats:
df[cols] = df[cols].astype(str)
 df[cols] = df[cols].astype(str ).fillna('missing')
 df[cols] = df[cols].str.lower()
df[cols + '_num_words'] = df[cols].apply(lambda comment: len(comment.split()))
df[cols + '_num_unique_words'] = df[cols].apply(lambda comment: len(set(w for w in comment.split())))
 df[cols + '_words_vs_unique'] = df[cols+'_num_unique_words'] / df[cols+'_num_words'] * 100<data_type_conversions>
n_comp = 600
gpca= load(open('.. /input/moa-tabnet-train-inference/gpca.pkl', 'rb'))
train2=(gpca.transform(train_features[GENES]))
test2 =(gpca.transform(test_features[GENES]))
train_gpca = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test_gpca = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train_gpca), axis=1)
 test_features = pd.concat(( test_features, test_gpca), axis=1 )
df['name_and_breeds'] = df['Name'].astype(str)+ ' ' + df['b1name'].astype(str)+ ' ' + df['b2name'].astype(str)
 df['name_and_breeds'] = df['name_and_breeds'].str.lower()
df.drop(['Name', 'b1name','b2name'], axis =1, inplace= True )<split>
n_comp = 50
cpca= load(open('.. /input/moa-tabnet-train-inference/cpca.pkl', 'rb'))
train2=(cpca.transform(train_features[CELLS]))
test2 =(cpca.transform(test_features[CELLS]))
train_cpca = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test_cpca = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train_cpca), axis=1)
 test_features = pd.concat(( test_features, test_cpca), axis=1 )
train = df.iloc[0:len(traindex),:]
test = df.iloc[len(traindex):,:]
del df; gc.collect() ;<choose_model_class>
c_n = [f for f in list(train_features.columns)if f not in ['sig_id', 'cp_type', 'cp_time', 'cp_dose']]
mask =(train_features[c_n].var() >= 0.85 ).values
tmp = train_features[c_n].loc[:, mask]
train_features = pd.concat([train_features[['sig_id', 'cp_type', 'cp_time', 'cp_dose']], tmp], axis=1)
 tmp = test_features[c_n].loc[:, mask]
test_features = pd.concat([test_features[['sig_id', 'cp_type', 'cp_time', 'cp_dose']], tmp], axis=1 )
word_vect = TfidfVectorizer(
 sublinear_tf=True,
strip_accents='unicode',
analyzer='word',
token_pattern=r'\w{1,}',
stop_words='english',
ngram_range=(1, 2),
max_features=20000)
 char_vectorizer = TfidfVectorizer(
 sublinear_tf=True,
strip_accents='unicode',
analyzer='word',
ngram_range=(2,5),
max_features=3000)
 with timer("Word Grams TFIDF"):
word_vect.fit(train['Description'])
 train_word_features = word_vect.transform(train['Description'])
 test_word_features = word_vect.transform(test['Description'])
 with timer("Character Grams TFIDF"):
char_vectorizer.fit(train['name_and_breeds'])
 train_char_features = char_vectorizer.transform(train['name_and_breeds'])
 test_char_features = char_vectorizer.transform(test['name_and_breeds'])
 num_features = [f_ for f_ in train.columns
if f_ not in ["Description", "name_and_breeds"]]
feature_names =(word_vect.get_feature_names() +
char_vectorizer.get_feature_names() +
num_features)
 with timer("Sparse Combine"):
X = hstack(
 [
train_char_features,
train_word_features,
train[num_features]
]
).tocsr()
del train_char_features
gc.collect()
testing = hstack(
 [
test_char_features,
test_word_features,
test[num_features]
]
).tocsr()
del test_char_features; gc.collect()<create_dataframe>
def fe_cluster_genes(train, test, n_clusters_g = 22, SEED = 42):
features_g = GENES
def create_cluster(train, test, features, kind = 'g', n_clusters = n_clusters_g):
train_ = train[features].copy()
test_ = test[features].copy()
kmeans_genes = load(open('.. /input/moa-tabnet-train-inference/kmeans_genes.pkl', 'rb'))
train[f'clusters_{kind}'] = kmeans_genes.predict(train_.values)
 test[f'clusters_{kind}'] = kmeans_genes.predict(test_.values)
 train = pd.get_dummies(train, columns = [f'clusters_{kind}'])
 test = pd.get_dummies(test, columns = [f'clusters_{kind}'])
 return train, test
train, test = create_cluster(train, test, features_g, kind = 'g', n_clusters = n_clusters_g)
 return train, test
train_features2 ,test_features2=fe_cluster_genes(train_features2,test_features2 )
lgtrain = lgb.Dataset(X, label=y, feature_name= feature_names,
free_raw_data=False, categorical_feature = categorical_vars)
 feature_importance_df = pd.DataFrame()
n_fold = 0
print("Light Gradient Boosting Classifier: ")
 lgbm_params = {
'application': 'regression',
'boosting': 'gbdt',
'metric': 'rmse',
'num_leaves': 150,
'max_depth': -1,
'learning_rate': 0.01,
'bagging_fraction': 0.85,
'feature_fraction': 0.8,
'min_split_gain': 0.01,
'min_child_samples': 150,
'min_child_weight': 0.1,
'verbosity': -1,
'data_random_seed': 3,
'early_stop': 100,
'verbose_eval': 100,
'num_rounds': 3000}
folds = StratifiedKFold(n_splits=5, shuffle=True, random_state=1)
 fold_preds = np.zeros(( testing.shape[0],))
oof_preds = np.zeros(( X.shape[0],))
all_coefficients = np.zeros(( 5, 4))
lgtrain.construct()
modelstart = time.time()
for trn_idx, val_idx in folds.split(X, y):
clf = lgb.train(
 params=lgbm_params,
train_set=lgtrain.subset(trn_idx),
valid_sets=lgtrain.subset(val_idx),
categorical_feature = categorical_vars,
num_boost_round=1500,
early_stopping_rounds=150,
verbose_eval=300 
)
 fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = feature_names
fold_importance_df["importance"] = clf.feature_importance(importance_type='gain')
 fold_importance_df["fold"] = n_fold + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 pred_test_y = clf.predict(X[val_idx], num_iteration=clf.best_iteration)
 optR = OptimizedRounder()
optR.fit(pred_test_y, y.iloc[val_idx].values)
 coefficients = optR.coefficients()
pred_test_y_k = optR.predict(pred_test_y, coefficients)
 all_coefficients[n_fold, :] = coefficients
print("Valid Counts = ", Counter(y.iloc[val_idx].values))
print("Predicted Counts = ", Counter(pred_test_y_k))
print("Coefficients = ", coefficients)
 qwk = quadratic_weighted_kappa(y.iloc[val_idx].values, pred_test_y_k)
 print("QWK = ", qwk)
 print('Predict 2/2')
 fold_preds += clf.predict(testing, num_iteration=clf.best_iteration)/ folds.n_splits
n_fold += 1
print("Model Runtime: %0.2f Minutes"%(( time.time() - modelstart)/60))<save_to_csv>
def fe_cluster_cells(train, test, n_clusters_c = 4, SEED = 42):
features_c = CELLS
def create_cluster(train, test, features, kind = 'c', n_clusters = n_clusters_c):
train_ = train[features].copy()
test_ = test[features].copy()
kmeans_cells = load(open('.. /input/moa-tabnet-train-inference/kmeans_cells.pkl', 'rb'))
train[f'clusters_{kind}'] = kmeans_cells.predict(train_.values)
 test[f'clusters_{kind}'] = kmeans_cells.predict(test_.values)
 train = pd.get_dummies(train, columns = [f'clusters_{kind}'])
 test = pd.get_dummies(test, columns = [f'clusters_{kind}'])
 return train, test
train, test = create_cluster(train, test, features_c, kind = 'c', n_clusters = n_clusters_c)
 return train, test
train_features2 ,test_features2=fe_cluster_cells(train_features2,test_features2 )
optR = OptimizedRounder()
coefficients_ = np.mean(all_coefficients, axis=0)
 print(coefficients_)
 optR = OptimizedRounder()
test_predictions = optR.predict(fold_preds, coefficients_ ).astype(int)
 Counter(test_predictions)
 submission = pd.DataFrame({'PetID': testdex, 'AdoptionSpeed': test_predictions})
 submission.to_csv('submission.csv', index=False)
 submission.head()<count_values>
train_pca=pd.concat(( train_gpca,train_cpca),axis=1)
 test_pca=pd.concat(( test_gpca,test_cpca),axis=1 )
print("Train Dependent Variable Class Distribution:
", y.value_counts())
 print("Prediction Dependent Variable Class Distribution:
", submission['AdoptionSpeed'].value_counts())
 print("Notebook Runtime: %0.2f Minutes"%(( time.time() - notebookstart)/60))<set_options>
def fe_cluster_pca(train, test,n_clusters=5,SEED = 42):
kmeans_pca = load(open('.. /input/moa-tabnet-train-inference/kmeans_pca.pkl', 'rb'))
train[f'clusters_pca'] = kmeans_pca.predict(train.values)
 test[f'clusters_pca'] = kmeans_pca.predict(test.values)
 train = pd.get_dummies(train, columns = [f'clusters_pca'])
 test = pd.get_dummies(test, columns = [f'clusters_pca'])
 return train, test
train_cluster_pca ,test_cluster_pca = fe_cluster_pca(train_pca,test_pca )
%matplotlib inline<load_from_csv>
train_cluster_pca = train_cluster_pca.iloc[:,650:]
test_cluster_pca = test_cluster_pca.iloc[:,650:]
train = pd.read_csv('.. /input/clicks_train.csv',usecols=['ad_id', 'clicked'] )<groupby>
train_features_cluster=train_features2.iloc[:,876:]
test_features_cluster=test_features2.iloc[:,876:]
ad_train_likelehood = train.groupby('ad_id')['clicked'].agg(['count', 'sum', 'mean'] ).reset_index()<load_from_csv>
gsquarecols=['g-574','g-211','g-216','g-0','g-255','g-577','g-153','g-389','g-60','g-370','g-248','g-167','g-203','g-177','g-301','g-332','g-517','g-6','g-744','g-224','g-162','g-3','g-736','g-486','g-283','g-22','g-359','g-361','g-440','g-335','g-106','g-307','g-745','g-146','g-416','g-298','g-666','g-91','g-17','g-549','g-145','g-157','g-768','g-568','g-396']
ad_train_likelehood['likelihood'] =(ad_train_likelehood['sum'] + 12*M)/(12 + ad_train_likelehood['count'])
 test = pd.read_csv(".. /input/clicks_test.csv")
 test = test.merge(ad_train_likelehood, how='left')
 test.likelihood.fillna(M, inplace=True)
<save_to_csv>
def fe_stats(train, test):
features_g = GENES
features_c = CELLS
for df in train, test:
df['g_sum'] = df[features_g].sum(axis = 1)
 df['g_mean'] = df[features_g].mean(axis = 1)
 df['g_std'] = df[features_g].std(axis = 1)
 df['g_kurt'] = df[features_g].kurtosis(axis = 1)
 df['g_skew'] = df[features_g].skew(axis = 1)
 df['c_sum'] = df[features_c].sum(axis = 1)
 df['c_mean'] = df[features_c].mean(axis = 1)
 df['c_std'] = df[features_c].std(axis = 1)
 df['c_kurt'] = df[features_c].kurtosis(axis = 1)
 df['c_skew'] = df[features_c].skew(axis = 1)
 df['gc_sum'] = df[features_g + features_c].sum(axis = 1)
 df['gc_mean'] = df[features_g + features_c].mean(axis = 1)
 df['gc_std'] = df[features_g + features_c].std(axis = 1)
 df['gc_kurt'] = df[features_g + features_c].kurtosis(axis = 1)
 df['gc_skew'] = df[features_g + features_c].skew(axis = 1)
 df['c52_c42'] = df['c-52'] * df['c-42']
df['c13_c73'] = df['c-13'] * df['c-73']
df['c26_c13'] = df['c-23'] * df['c-13']
df['c33_c6'] = df['c-33'] * df['c-6']
df['c11_c55'] = df['c-11'] * df['c-55']
df['c38_c63'] = df['c-38'] * df['c-63']
df['c38_c94'] = df['c-38'] * df['c-94']
df['c13_c94'] = df['c-13'] * df['c-94']
df['c4_c52'] = df['c-4'] * df['c-52']
df['c4_c42'] = df['c-4'] * df['c-42']
df['c13_c38'] = df['c-13'] * df['c-38']
df['c55_c2'] = df['c-55'] * df['c-2']
df['c55_c4'] = df['c-55'] * df['c-4']
df['c4_c13'] = df['c-4'] * df['c-13']
df['c82_c42'] = df['c-82'] * df['c-42']
df['c66_c42'] = df['c-66'] * df['c-42']
df['c6_c38'] = df['c-6'] * df['c-38']
df['c2_c13'] = df['c-2'] * df['c-13']
df['c62_c42'] = df['c-62'] * df['c-42']
df['c90_c55'] = df['c-90'] * df['c-55']
for feature in features_c:
df[f'{feature}_squared'] = df[feature] ** 2
for feature in gsquarecols:
df[f'{feature}_squared'] = df[feature] ** 2
return train, test
train_features2,test_features2=fe_stats(train_features2,test_features2 )
test.sort_values(['display_id','likelihood'], inplace=True, ascending=False)
 subm = test.groupby('display_id')['ad_id'].apply(lambda x: " ".join(map(str,x)) ).reset_index()
subm.to_csv("subm.csv", index=False )<import_modules>
train_features_stats=train_features2.iloc[:,902:]
test_features_stats=test_features2.iloc[:,902:]
startTime = datetime.datetime.now()<install_modules>
train_features = pd.concat(( train_features, train_features_cluster,train_cluster_pca,train_features_stats), axis=1)
 test_features = pd.concat(( test_features, test_features_cluster,test_cluster_pca,test_features_stats), axis=1 )
!cp.. /input/gdcm-conda-install/gdcm.tar.
 !tar -xvzf gdcm.tar
!conda install --offline./gdcm/gdcm-2.8.9-py37h71b2a6d_0.tar.bz2
print("GDCM installed.")
<load_from_csv>
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target = train[train_targets_scored.columns]
dataDirPath = '.. /input/rsna-str-pulmonary-embolism-detection/'
modelDirPath = '.. /input/firstbaselinemodel/'
testDataDF = pd.read_csv(dataDirPath+'test.csv', dtype={'StudyInstanceUID':'string', 'SeriesInstanceUID':'string', 'SOPInstanceUID':'string'})
 testDataDF = testDataDF.set_index('SOPInstanceUID')
 print(testDataDF.head())
 DEBUG =(testDataDF.shape[0]==146853)
 DEBUG
if DEBUG:
testDataDF = testDataDF.head(1000 )<count_unique_values>
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1 )
listOfStudyID = testDataDF['StudyInstanceUID'].unique()
print(len(listOfStudyID))<categorify>
target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()
def window(img, WL=50, WW=350):
upper, lower = WL+WW//2, WL-WW//2
X = np.clip(img.copy() , lower, upper)
 X = X - np.min(X)
 X = X / np.max(X)
 X =(X*255.0 ).astype('uint8')
 return X
data_transform = transforms.Compose([
transforms.ToTensor() ,
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
 ])
 def dcmDataToImage(dcmData):
image = dcmData.pixel_array * int(dcmData.RescaleSlope)+ int(dcmData.RescaleIntercept)
 image = np.stack([window(image, WL=-600, WW=1500),
window(image, WL=40, WW=400),
window(image, WL=100, WW=700)], 2)
 if image.shape[0] != 512 or image.shape[1] != 512:
image = cv2.resize(image,(512,512), interpolation = cv2.INTER_AREA)
 return image
def getVolByVTK(PathDicom):
reader = vtk.vtkDICOMImageReader()
reader.SetDirectoryName(PathDicom)
 reader.Update()
_extent = reader.GetDataExtent()
ConstPixelDims = [_extent[1]-_extent[0]+1, _extent[3]-_extent[2]+1, _extent[5]-_extent[4]+1]
ConstPixelSpacing = reader.GetPixelSpacing()
imageData = reader.GetOutput()
pointData = imageData.GetPointData()
assert(pointData.GetNumberOfArrays() ==1)
 arrayData = pointData.GetArray(0)
 ArrayDicom = numpy_support.vtk_to_numpy(arrayData)
 ArrayDicom = ArrayDicom.reshape(ConstPixelDims, order='F')
 return ArrayDicom
def VTKsliceToImage(VTKvol, index):
image = np.flipud(VTKvol[:,:,index].T.copy())
 image = np.stack([window(image, WL=-600, WW=1500),
window(image, WL=40, WW=400),
window(image, WL=100, WW=700)], 2)
 if image.shape[0] != 512 or image.shape[1] != 512:
image = cv2.resize(image,(512,512), interpolation = cv2.INTER_AREA)
 return image
def chunks(lst, n):

 for i in range(0, len(lst), n):
yield lst[i:i + n]
def sortByImgPosHelper(dcmDataDict):
return dcmDataDict['img_pos']<categorify>
target=target[target_cols]
def negativeOrIndeterminate(imgLvlLabels, stdLvlLabels):
workingArray = stdLvlLabels[:2]
if workingArray[0]>0.5 and workingArray[1]>0.5:
minInd = np.argmin(workingArray)
 workingArray[minInd] = min(workingArray[minInd],0.49)
 if max(workingArray)>0.5:
imgLvlLabels[:] = np.minimum(imgLvlLabels,0.49)[:]
if max(workingArray)<0.5 and max(imgLvlLabels)<0.5:
maxInd = np.argmax(imgLvlLabels)
 imgLvlLabels[maxInd] = max(imgLvlLabels[maxInd],0.51)
 stdLvlLabels[:2] = workingArray
return
def rightLeftCentral(imgLvlLabels, stdLvlLabels):
workingArray = stdLvlLabels[4:7]
if max(imgLvlLabels)>0.5:
maxInd = np.argmax(workingArray)
 workingArray[maxInd] = max(workingArray[maxInd],0.51)
 else:
workingArray[:] = np.minimum(workingArray,0.49)[:]
stdLvlLabels[4:7] = workingArray
return
def rv_lv_ratio(imgLvlLabels, stdLvlLabels):
workingArray = stdLvlLabels[7:9]
if max(imgLvlLabels)>0.5:
maxInd = np.argmax(workingArray)
 workingArray[maxInd] = max(workingArray[maxInd],0.51)
 stdLvlLabels[7:9] = workingArray
else:
workingArray[:] = np.minimum(workingArray,0.49)[:]
if min(workingArray)>0.5:
minInd = np.argmin(workingArray)
 workingArray[minInd] = min(workingArray[minInd],0.49)
 stdLvlLabels[7:9] = workingArray
return
def chronicOrAcuteChronic(imgLvlLabels, stdLvlLabels):
workingArray = stdLvlLabels[2:4]
if min(workingArray)>0.5:
minInd = np.argmin(workingArray)
 workingArray[minInd] = min(workingArray[minInd],0.49)
 stdLvlLabels[2:4] = workingArray
return
def modifyAll(imgLvlLabels, stdLvlLabels):
negativeOrIndeterminate(imgLvlLabels, stdLvlLabels)
 rightLeftCentral(imgLvlLabels, stdLvlLabels)
 rv_lv_ratio(imgLvlLabels, stdLvlLabels)
 chronicOrAcuteChronic(imgLvlLabels, stdLvlLabels)
 return<choose_model_class>
train = pd.get_dummies(train, columns=['cp_time','cp_dose'])
 test_ = pd.get_dummies(test, columns=['cp_time','cp_dose'] )
INPUT_SIZE = 64
HIDDEN_SIZE = 32
NUM_LAYERS = 1
NUM_CLASSES = 1
class BiGRU(nn.Module):
def __init__(self, input_size, hidden_size, num_layers, num_classes):
super(BiGRU, self ).__init__()
self.hidden_size = hidden_size
self.num_layers = num_layers
self.GRU = nn.GRU(
 input_size, hidden_size, num_layers, batch_first=True, bidirectional=True 
)
 self.linear1 = nn.Linear(hidden_size*2, hidden_size)
 self.linear2 = nn.Linear(hidden_size, num_classes)
 self.linear3 = nn.Linear(hidden_size*2, hidden_size)
 self.linear4 = nn.Linear(hidden_size, 9)
 def forward(self, x):
imageLevelOutputs = []
h0 = torch.zeros(self.num_layers * 2, x.size(0), self.hidden_size ).cuda()
out, h_n = self.GRU(x, h0)
 for i, out_t in enumerate(out.chunk(out.size(1), dim=1)) :
out_t = out_t.squeeze(1)
 out_t = F.relu(self.linear1(out_t))
out_t = self.linear2(out_t)
 imageLevelOutputs += [out_t]
imageLevelOutputs = torch.stack(imageLevelOutputs, 1 ).squeeze(2)
 h_n = h_n.view(1,-1)
 studyLevelOutputs = F.relu(self.linear3(h_n))
studyLevelOutputs = self.linear4(studyLevelOutputs)
 return(imageLevelOutputs, studyLevelOutputs )<find_best_params>
feature_cols = [c for c in train.columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['sig_id']]
CNNmodel1_Path = modelDirPath+'CNNmodel_01_epoch1_CV4_20201008_2252_embedder.pth'
CNNembedderModel1 = torch.load(CNNmodel1_Path)
 CNNembedderModel1.eval() ;
RNNmodel11_Path = modelDirPath+'CNNmodel_01_epoch1_CV4_20201008_2252_sequence4_20201010_1528.pth'
RNNmodel11 = torch.load(RNNmodel11_Path)
 RNNmodel11.eval() ;
RNNmodel12_Path = modelDirPath+'CNNmodel_01_epoch1_CV4_20201008_2252_sequence7_20201022_0539.pth'
RNNmodel12 = torch.load(RNNmodel12_Path)
 RNNmodel12.eval() ;
RNNmodel13_Path = modelDirPath+'CNNmodel_01_epoch1_CV4_20201008_2252_sequence5_20201022_0415.pth'
RNNmodel13 = torch.load(RNNmodel13_Path)
 RNNmodel13.eval() ;
RNNmodel14_Path = modelDirPath+'CNNmodel_01_epoch1_CV4_20201008_2252_sequence7_20201022_1749.pth'
RNNmodel14 = torch.load(RNNmodel14_Path)
 RNNmodel14.eval() ;
CNNmodel2_Path = modelDirPath+'CNNmodel_01_cv0_epoch1_20201014_0012_embedder.pth'
CNNembedderModel2 = torch.load(CNNmodel2_Path)
 CNNembedderModel2.eval() ;
RNNmodel21_Path = modelDirPath+'CNNmodel_01_cv0_epoch1_20201014_0012_sequence5_20201020_1653.pth'
RNNmodel21 = torch.load(RNNmodel21_Path)
 RNNmodel21.eval() ;
RNNmodel22_Path = modelDirPath+'CNNmodel_01_cv0_epoch1_20201014_0012_sequence8_20201020_2122.pth'
RNNmodel22 = torch.load(RNNmodel22_Path)
 RNNmodel22.eval() ;
CNNmodel3_Path = modelDirPath+'CNNmodel_01_cv3_epoch2_20201018_1153_embedder.pth'
CNNembedderModel3 = torch.load(CNNmodel3_Path)
 CNNembedderModel3.eval() ;
RNNmodel31_Path = modelDirPath+'CNNmodel_01_cv3_epoch2_20201018_1153_sequence2_20201019_0622.pth'
RNNmodel31 = torch.load(RNNmodel31_Path)
 RNNmodel31.eval() ;
RNNmodel32_Path = modelDirPath+'CNNmodel_01_cv3_epoch2_20201018_1153_sequence8_20201023_0529.pth'
RNNmodel32 = torch.load(RNNmodel32_Path)
 RNNmodel32.eval() ;
logRegModel_indeterminate = pickle.load(open('.. /input/firstbaselinemodel/logRegModel_indeterminate.sav', 'rb'))
logRegModel_chronic_pe = pickle.load(open('.. /input/firstbaselinemodel/logRegModel_chronic_pe.sav', 'rb'))
logRegModel_acute_and_chronic_pe = pickle.load(open('.. /input/firstbaselinemodel/logRegModel_acute_and_chronic_pe.sav', 'rb'))
logRegModel_rv_lv_ratio_gte_1 = pickle.load(open('.. /input/firstbaselinemodel/logRegModel_rv_lv_ratio_gte_1.sav', 'rb'))
logRegModel_rv_lv_ratio_lt_1 = pickle.load(open('.. /input/firstbaselinemodel/logRegModel_rv_lv_ratio_lt_1.sav', 'rb'))<define_variables>
train = train[feature_cols]
test = test_[feature_cols]
listOfMetricLabels = ['negative_exam_for_pe', 'indeterminate',
'chronic_pe', 'acute_and_chronic_pe',
'central_pe', 'leftsided_pe', 'rightsided_pe',
'rv_lv_ratio_gte_1', 'rv_lv_ratio_lt_1']
defaultScore = {'pe_present_on_image': 0.053915069524414806,
'negative_exam_for_pe': 0.6763928618101033,
'rv_lv_ratio_gte_1': 0.12875001256566257,
'rv_lv_ratio_lt_1': 0.17437230326919448,
'leftsided_pe': 0.21089872969528548,
'chronic_pe': 0.040139752506710064,
'rightsided_pe': 0.2575653665766779,
'acute_and_chronic_pe': 0.019458347341720122,
'central_pe': 0.054468517151291695,
'indeterminate': 0.020484822355039723}
q_weighted_means = [0.00326324, 0.05970682, 0.32645303, 0.67452216, 0.71344817, 0.4734337, 0.0740926, 0.00369781]
sliceBins = [0/8, 1/8, 2/8, 3/8, 4/8, 5/8, 6/8, 7/8]
def getBinnedProb(sliceLoc):
thisProb = 0
for eachIndex, eachSlice in enumerate(sliceBins):
if sliceLoc>=eachSlice:
thisProb = q_weighted_means[eachIndex]
return(thisProb )<categorify>
X_test = test.values
def RNNinferenceHelper(RNNmodel, embeddingVol):
imageLevelLabels, studyLevelLabels = RNNmodel(embeddingVol)
 imageLevelLabels = torch.sigmoid(imageLevelLabels ).squeeze(0 ).cpu().detach().numpy()
studyLevelLabels = torch.sigmoid(studyLevelLabels ).squeeze(0 ).cpu().detach().numpy()
return imageLevelLabels,studyLevelLabels<define_variables>
class LogitsLogLoss(Metric):
def __init__(self):
self._name = "logits_ll"
self._maximize = False
def __call__(self, y_true, y_pred):
logits = 1 /(1 + np.exp(-y_pred))
aux =(1 - y_true)* np.log(1 - logits + 5e-5)+ y_true * np.log(logits + 5e-5)
 return np.mean(-aux )
numExceptions = 0
submissionList = []
pred = 0.5<define_variables>
MAX_EPOCH = 200
tabnet_params = dict(
 n_d = 32,
n_a = 32,
n_steps = 1,
gamma = 1.3,
lambda_sparse = 0,
optimizer_fn = optim.Adam,
optimizer_params = dict(lr = 2e-2, weight_decay = 1e-5),
mask_type = "entmax",
scheduler_params = dict(mode = "min", patience = 5, min_lr = 1e-5, factor = 0.9),
scheduler_fn = ReduceLROnPlateau,
seed = seed,
verbose = 10
)
def inferenceBlock(eachStudyID):
thisStudyDF = testDataDF[testDataDF['StudyInstanceUID']==eachStudyID]
listOfImageIDs = thisStudyDF.index
listOfDcm_dataDict = []
for eachImageID in listOfImageIDs:
eachImagePath = dataDirPath+'/test/'+thisStudyDF.loc[eachImageID, 'StudyInstanceUID']+'/'+thisStudyDF.loc[eachImageID, 'SeriesInstanceUID']+'/'+eachImageID+'.dcm'
dcm_data = dcmread(eachImagePath)
 img_pos = dcm_data[0x20, 0x32].value[2]
listOfDcm_dataDict.append({'imageID':eachImageID, 'dcm_data':dcm_data, 'img_pos':img_pos})
 listOfDcm_dataDict.sort(key=sortByImgPosHelper, reverse=True)
 tensorChunkIterator = chunks(listOfDcm_dataDict,48)
 embeddingList1 = []
embeddingList2 = []
embeddingList3 = []
for eachChunk in tensorChunkIterator:
images = [dcmDataToImage(eachImageID['dcm_data'])for eachImageID in eachChunk]
images = [eachImage.astype(np.float32)for eachImage in images]
listOfTensors = [data_transform(eachImage)for eachImage in images]
stackedImagesTensor = torch.stack(listOfTensors, dim=0, out=None)
 stackedImagesTensor = stackedImagesTensor.cuda()
embedding1 = CNNembedderModel1(stackedImagesTensor ).detach()
embeddingList1.append(embedding1)
 embedding2 = CNNembedderModel2(stackedImagesTensor ).detach()
embeddingList2.append(embedding2)
 embedding3 = CNNembedderModel3(stackedImagesTensor ).detach()
embeddingList3.append(embedding3)
 embeddingVol1 = torch.cat(embeddingList1, dim=0)
 embeddingVol1 = embeddingVol1.unsqueeze(0)
 embeddingVol2 = torch.cat(embeddingList2, dim=0)
 embeddingVol2 = embeddingVol2.unsqueeze(0)
 embeddingVol3 = torch.cat(embeddingList3, dim=0)
 embeddingVol3 = embeddingVol3.unsqueeze(0)
 imageLevelLabels11, studyLevelLabels11 = RNNinferenceHelper(RNNmodel11, embeddingVol1)
 imageLevelLabels12, studyLevelLabels12 = RNNinferenceHelper(RNNmodel12, embeddingVol1)
 imageLevelLabels13, studyLevelLabels13 = RNNinferenceHelper(RNNmodel13, embeddingVol1)
 imageLevelLabels14, studyLevelLabels14 = RNNinferenceHelper(RNNmodel14, embeddingVol1)
 imageLevelLabels21, studyLevelLabels21 = RNNinferenceHelper(RNNmodel21, embeddingVol2)
 imageLevelLabels22, studyLevelLabels22 = RNNinferenceHelper(RNNmodel22, embeddingVol2)
 imageLevelLabels31, studyLevelLabels31 = RNNinferenceHelper(RNNmodel31, embeddingVol3)
 imageLevelLabels32, studyLevelLabels32 = RNNinferenceHelper(RNNmodel32, embeddingVol3)
 imageLevelLabels = 0.125*imageLevelLabels11 + 0.125*imageLevelLabels12 + 0.125*imageLevelLabels13 + 0.125*imageLevelLabels14 + 0.125*imageLevelLabels21 + 0.125*imageLevelLabels22 + 0.125*imageLevelLabels31 + 0.125*imageLevelLabels32
studyLevelLabels = 0.125*studyLevelLabels11 + 0.125*studyLevelLabels12 + 0.125*studyLevelLabels13 + 0.125*studyLevelLabels14 + 0.125*studyLevelLabels21 + 0.125*studyLevelLabels22 + 0.125*studyLevelLabels31 + 0.125*studyLevelLabels32
studyLevelLabels_copy = np.copy(studyLevelLabels)
 studyLevelLabels_copy = studyLevelLabels_copy[np.newaxis,:]
studyLevelLabels[1] = logRegModel_indeterminate.predict_proba(studyLevelLabels_copy)[0,1]
studyLevelLabels[2] = logRegModel_chronic_pe.predict_proba(studyLevelLabels_copy)[0,1]
studyLevelLabels[3] = logRegModel_acute_and_chronic_pe.predict_proba(studyLevelLabels_copy)[0,1]
studyLevelLabels[7] = logRegModel_rv_lv_ratio_gte_1.predict_proba(studyLevelLabels_copy)[0,1]
studyLevelLabels[8] = logRegModel_rv_lv_ratio_lt_1.predict_proba(studyLevelLabels_copy)[0,1]
modifyAll(imageLevelLabels, studyLevelLabels)
 for eachIndex in range(len(listOfDcm_dataDict)) :
submissionList.append([listOfDcm_dataDict[eachIndex]['imageID'], imageLevelLabels[eachIndex]])
 for eachIndex, eachMetric in enumerate(listOfMetricLabels):
submissionList.append([eachStudyID+'_'+eachMetric, studyLevelLabels[eachIndex]])
 return
def contingencyPlanB(eachStudyID):
thisStudyDF = testDataDF[testDataDF['StudyInstanceUID']==eachStudyID]
listOfImageIDs = thisStudyDF.index
listOfDcm_dataDict = []
for eachImageID in listOfImageIDs:
eachImagePath = dataDirPath+'/test/'+thisStudyDF.loc[eachImageID, 'StudyInstanceUID']+'/'+thisStudyDF.loc[eachImageID, 'SeriesInstanceUID']+'/'+eachImageID+'.dcm'
dcm_data = dcmread(eachImagePath)
 img_pos = dcm_data[0x20, 0x32].value[2]
listOfDcm_dataDict.append({'imageID':eachImageID, 'dcm_data':dcm_data, 'img_pos':img_pos})
 listOfDcm_dataDict.sort(key=sortByImgPosHelper, reverse=True)
 imageLevelLabels = np.full(len(listOfImageIDs), defaultScore['pe_present_on_image'])
 studyLevelLabels = np.zeros(len(listOfMetricLabels))
for eachIndex in range(len(listOfDcm_dataDict)) :
thisSliceLoc = eachIndex/(len(listOfDcm_dataDict)-1)
 imageLevelLabels[eachIndex] = getBinnedProb(thisSliceLoc)
 for eachIndex, eachMetric in enumerate(listOfMetricLabels):
studyLevelLabels[eachIndex] = defaultScore[eachMetric]
modifyAll(imageLevelLabels, studyLevelLabels)
 for eachIndex in range(len(listOfDcm_dataDict)) :
submissionList.append([listOfDcm_dataDict[eachIndex]['imageID'], imageLevelLabels[eachIndex]])
 for eachIndex, eachMetric in enumerate(listOfMetricLabels):
submissionList.append([eachStudyID+'_'+eachMetric, studyLevelLabels[eachIndex]])
 return
def contingencyPlanC(eachStudyID):
global numExceptions
numExceptions += 1
thisStudyDF = testDataDF[testDataDF['StudyInstanceUID']==eachStudyID]
listOfImageIDs = thisStudyDF.index
imageLevelLabels = np.full(len(listOfImageIDs), defaultScore['pe_present_on_image'])
 studyLevelLabels = np.zeros(len(listOfMetricLabels))
for eachIndex, eachMetric in enumerate(listOfMetricLabels):
studyLevelLabels[eachIndex] = defaultScore[eachMetric]
modifyAll(imageLevelLabels, studyLevelLabels)
 for eachImageIDs in listOfImageIDs:
submissionList.append([eachImageIDs, imageLevelLabels[eachIndex]])
 for eachIndex, eachMetric in enumerate(listOfMetricLabels):
submissionList.append([eachStudyID+'_'+eachMetric, studyLevelLabels[eachIndex]])
 return
with torch.no_grad() :
for eachStudyID in listOfStudyID:
gc.collect()
try:
try:
inferenceBlock(eachStudyID)
 except:
contingencyPlanB(eachStudyID)
 except:
contingencyPlanC(eachStudyID)
 submissionDF = pd.DataFrame(submissionList, columns = ['id','label'])
 submissionDF.fillna(0.5)
 print('finish' )<feature_engineering>
%%time
test_cv_preds = []
NB_SPLITS = 10
mskf = MultilabelStratifiedKFold(n_splits = NB_SPLITS, random_state = 0, shuffle = True)
 SEED = [20,21,22]
for s in SEED:
tabnet_params['seed'] = s
for fold_nb,(train_idx, val_idx)in enumerate(mskf.split(train, target)) :
model = TabNetRegressor()
model.load_model(f".. /input/moa-tabnet-train-inference/TabNet_seed_{s}_fold_{fold_nb+1}.zip")
 preds_test = model.predict(X_test)
 test_cv_preds.append(1 /(1 + np.exp(-preds_test)))
 test_preds_all = np.stack(test_cv_preds )
def check_consistency(sub, test):

 for i in test['StudyInstanceUID'].unique() :
df_tmp = sub.loc[sub.id.str.contains(i, regex = False)].reset_index(drop = True)
 df_tmp['StudyInstanceUID'] = df_tmp['id'].str.split('_' ).str[0]
df_tmp['label_type'] = df_tmp['id'].str.split('_' ).str[1:].apply(lambda x: '_'.join(x))
del df_tmp['id']
if i == test['StudyInstanceUID'].unique() [0]:
df = df_tmp.copy()
else:
df = pd.concat([df, df_tmp], axis = 0)
 df_exam = df.pivot(index = 'StudyInstanceUID', columns = 'label_type', values = 'label')
 df_image = sub.loc[sub.id.isin(test.SOPInstanceUID)].reset_index(drop = True)
 df_image = df_image.merge(test, how = 'left', left_on = 'id', right_on = 'SOPInstanceUID')
 df_image.rename(columns = {"label": "pe_present_on_image"}, inplace = True)
 del df_image['id']
df = df_exam.merge(df_image, how = 'left', on = 'StudyInstanceUID')
 ids = ['StudyInstanceUID', 'SeriesInstanceUID', 'SOPInstanceUID']
labels = [c for c in df.columns if c not in ids]
df = df[ids + labels]
df['positive_images_in_exam'] = df['StudyInstanceUID'].map(df.groupby(['StudyInstanceUID'] ).pe_present_on_image.max())
 df_pos = df.loc[df.positive_images_in_exam > 0.5]
df_neg = df.loc[df.positive_images_in_exam <= 0.5]
rule1a = df_pos.loc[(( df_pos.rv_lv_ratio_lt_1 > 0.5)& 
(df_pos.rv_lv_ratio_gte_1 > 0.5)) | 
(( df_pos.rv_lv_ratio_lt_1 <= 0.5)& 
(df_pos.rv_lv_ratio_gte_1 <= 0.5)) ].reset_index(drop = True)
 rule1a['broken_rule'] = '1a'
rule1b = df_pos.loc[(df_pos.central_pe <= 0.5)& 
(df_pos.rightsided_pe <= 0.5)& 
(df_pos.leftsided_pe <= 0.5)].reset_index(drop = True)
 rule1b['broken_rule'] = '1b'
rule1c = df_pos.loc[(df_pos.acute_and_chronic_pe > 0.5)& 
(df_pos.chronic_pe > 0.5)].reset_index(drop = True)
 rule1c['broken_rule'] = '1c'
rule1d = df_pos.loc[(df_pos.indeterminate > 0.5)| 
(df_pos.negative_exam_for_pe > 0.5)].reset_index(drop = True)
 rule1d['broken_rule'] = '1d'
rule2a = df_neg.loc[(( df_neg.indeterminate > 0.5)& 
(df_neg.negative_exam_for_pe > 0.5)) | 
(( df_neg.indeterminate <= 0.5)& 
(df_neg.negative_exam_for_pe <= 0.5)) ].reset_index(drop = True)
 rule2a['broken_rule'] = '2a'
rule2b = df_neg.loc[(df_neg.rv_lv_ratio_lt_1 > 0.5)| 
(df_neg.rv_lv_ratio_gte_1 > 0.5)| 
(df_neg.central_pe > 0.5)| 
(df_neg.rightsided_pe > 0.5)| 
(df_neg.leftsided_pe > 0.5)| 
(df_neg.acute_and_chronic_pe > 0.5)| 
(df_neg.chronic_pe > 0.5)].reset_index(drop = True)
 rule2b['broken_rule'] = '2b'
errors = pd.concat([rule1a, rule1b, rule1c, rule1d, rule2a, rule2b], axis = 0)
 print('Found', len(errors), 'inconsistent predictions')
 return errors<save_to_csv>
all_feat = [col for col in df.columns if col not in ["sig_id"]]
test = pd.read_csv(".. /input/lish-moa/test_features.csv")
 sig_id = test[test["cp_type"] != "ctl_vehicle"].sig_id.reset_index(drop = True)
 tmp = pd.DataFrame(test_preds_all.mean(axis = 0), columns = all_feat)
 tmp["sig_id"] = sig_id
sub_tabnet = pd.merge(test[["sig_id"]], tmp, on = "sig_id", how = "left")
 sub_tabnet.fillna(0, inplace = True)
 sub_tabnet
print('numExceptions:', numExceptions)
 test = pd.read_csv(dataDirPath+'test.csv')
 consistencyErrors = check_consistency(submissionDF, test)
 submissionDF.to_csv('submission.csv', index=False )<set_options>
from sklearn.model_selection import KFold
from sklearn import preprocessing
from sklearn.metrics import confusion_matrix
from sklearn.feature_selection import VarianceThreshold
from sklearn.decomposition import PCA
from tensorflow.keras import layers,regularizers,Sequential,Model,backend,callbacks,optimizers,metrics,losses
pd.set_option('display.max_rows', 500)
 pd.set_option('display.max_columns', 500)
 pd.set_option('display.width', 1000)
 warnings.filterwarnings('ignore')
 SEED = 10<init_hyperparams>
train_features = pd.read_csv('/kaggle/input/lish-moa/train_features.csv')
 non_ctl_idx = train_features.loc[train_features['cp_type']!='ctl_vehicle'].index.to_list()
train_features = train_features.drop(['sig_id','cp_type','cp_dose','cp_time'],axis=1)
 train_targets_scored = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
 train_targets_scored = train_targets_scored.drop('sig_id',axis=1)
 labels_train = train_targets_scored.values
train_features = train_features.iloc[non_ctl_idx]
labels_train = labels_train[non_ctl_idx]
test_features = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 test_features = test_features.drop(['sig_id','cp_dose','cp_time'],axis=1)
 json_file_path = '.. /input/t-test-pca-rfe-logistic-regression/main_predictors.json'
with open(json_file_path, 'r')as j:
predictors = json.loads(j.read())
 predictors = predictors['start_predictors']
def seed_everything(seed=0):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 def reduce_mem_usage(df, verbose=True):
numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
start_mem = df.memory_usage(deep=True ).sum() / 1024**2
for col in df.columns:
col_type = df[col].dtypes
if col_type in numerics:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8 ).min and c_max < np.iinfo(np.int8 ).max:
df[col] = df[col].astype(np.int8)
 elif c_min > np.iinfo(np.int16 ).min and c_max < np.iinfo(np.int16 ).max:
df[col] = df[col].astype(np.int16)
 elif c_min > np.iinfo(np.int32 ).min and c_max < np.iinfo(np.int32 ).max:
df[col] = df[col].astype(np.int32)
 elif c_min > np.iinfo(np.int64 ).min and c_max < np.iinfo(np.int64 ).max:
df[col] = df[col].astype(np.int64)
 else:
c_prec = df[col].apply(lambda x: np.finfo(x ).precision ).max()
if c_min > np.finfo(np.float32 ).min and c_max < np.finfo(np.float32 ).max and c_prec == np.finfo(np.float32 ).precision:
df[col] = df[col].astype(np.float32)
 else:
df[col] = df[col].astype(np.float64)
 end_mem = df.memory_usage().sum() / 1024**2
if verbose: print('Mem.usage decreased to {:5.2f} Mb({:.1f}% reduction)'.format(end_mem, 100 *(start_mem - end_mem)/ start_mem))
return df<load_pretrained>
cs = train_features.columns.str.contains('c-')
 gs = train_features.columns.str.contains('g-')
 def preprocessor(train,test):
n_gs = 2
n_cs = 100
pca_cs = PCA(n_components = n_cs)
 pca_gs = PCA(n_components = n_gs)
 train_pca_gs = pca_gs.fit_transform(train[:,gs])
 train_pca_cs = pca_cs.fit_transform(train[:,cs])
 test_pca_gs = pca_gs.transform(test[:,gs])
 test_pca_cs = pca_cs.transform(test[:,cs])
 train_c_mean = train[:,cs].mean(axis=1)
 test_c_mean = test[:,cs].mean(axis=1)
 train_g_mean = train[:,gs].mean(axis=1)
 test_g_mean = test[:,gs].mean(axis=1)
 train = np.concatenate(( train,train_pca_gs,train_pca_cs,train_c_mean[:,np.newaxis]
,train_g_mean[:,np.newaxis]),axis=1)
 test = np.concatenate(( test,test_pca_gs,test_pca_cs,test_c_mean[:,np.newaxis],
test_g_mean[:,np.newaxis]),axis=1)
 scaler = preprocessing.StandardScaler()
train = scaler.fit_transform(train)
 test = scaler.transform(test)
 return train, test
seed_everything(SEED)
 print('Load Data')
 train_df = pd.read_pickle('.. /input/ieee-data-minification/train_transaction.pkl')
 test_df = pd.read_pickle('.. /input/ieee-data-minification/test_transaction.pkl')
 train_identity = pd.read_pickle('.. /input/ieee-data-minification/train_identity.pkl')
 test_identity = pd.read_pickle('.. /input/ieee-data-minification/test_identity.pkl')
 base_columns = list(train_df)+ list(train_identity )<merge>
n_labels = train_targets_scored.shape[1]
n_train = train_features.shape[0]
n_test = test_features.shape[0]
p_min = 0.0005
p_max = 0.9995
def logloss(y_true, y_pred):
y_pred = tf.clip_by_value(y_pred,p_min,p_max)
 return -backend.mean(y_true*backend.log(y_pred)+(1-y_true)*backend.log(1-y_pred))
train_df = pd.merge(train_df,train_identity, how = 'left', on = 'TransactionID',validate = "many_to_one")
 test_df = pd.merge(test_df,test_identity, how = 'left', on = 'TransactionID',validate = "many_to_one" )<drop_column>
def build_model(n_features, n_features_2, n_labels, label_smoothing = 0.0005):
input_1 = layers.Input(shape =(n_features,), name = 'Input1')
 input_2 = layers.Input(shape =(n_features_2,), name = 'Input2')
 head_1 = Sequential([
layers.BatchNormalization() ,
layers.Dropout(0.2),
layers.Dense(512, activation="elu"),
layers.BatchNormalization() ,
layers.Dense(256, activation = "elu")
 ],name='Head1')
 input_3 = head_1(input_1)
 input_3_concat = layers.Concatenate()([input_2, input_3])
 head_2 = Sequential([
layers.BatchNormalization() ,
layers.Dropout(0.3),
layers.Dense(512, "relu"),
layers.BatchNormalization() ,
layers.Dense(512, "elu"),
layers.BatchNormalization() ,
layers.Dense(256, "relu"),
layers.BatchNormalization() ,
layers.Dense(256, "elu")
 ],name='Head2')
 input_4 = head_2(input_3_concat)
 input_4_avg = layers.Average()([input_3, input_4])
 head_3 = Sequential([
layers.BatchNormalization() ,
layers.Dense(256, kernel_initializer='lecun_normal', activation='selu'),
layers.BatchNormalization() ,
layers.Dense(n_labels, kernel_initializer='lecun_normal', activation='selu'),
layers.BatchNormalization() ,
layers.Dense(n_labels, activation="sigmoid")
 ],name='Head3')
 output = head_3(input_4_avg)
 model = Model(inputs = [input_1, input_2], outputs = output)
 model.compile(optimizer='adam', loss=losses.BinaryCrossentropy(label_smoothing=label_smoothing), metrics=logloss)
 return model
train_df.drop(["TransactionID", "TransactionDT"],axis=1, inplace=True)
 test_df.drop(["TransactionDT"],axis=1, inplace=True )<prepare_x_and_y>
%%time
n_seeds = 5
np.random.seed(1)
 seeds = np.random.randint(0,100,size=n_seeds)
 n_folds = 10
y_pred = np.zeros(( n_test,n_labels))
oof = tf.constant(0.0)
 hists = []
for seed in seeds:
fold = 0
kf = KFold(n_splits=n_folds,shuffle=True,random_state=seed)
 for train, test in kf.split(train_features):
print(f'FOLD: {fold}, SEED: {seed}')
 X_train, X_test = preprocessor(train_features.iloc[train].values,
train_features.iloc[test].values)
 _,data_test = preprocessor(train_features.iloc[train].values,
test_features.drop('cp_type',axis=1 ).values)
 X_train_2 = train_features.iloc[train][predictors].values
X_test_2 = train_features.iloc[test][predictors].values
data_test_2 = test_features[predictors].values
y_train = labels_train[train]
y_test = labels_train[test]
n_features = X_train.shape[1]
n_features_2 = X_train_2.shape[1]
model = build_model(n_features, n_features_2, n_labels)
 reduce_lr = callbacks.ReduceLROnPlateau(monitor='val_logloss', factor=0.1, patience=2, mode='min', min_lr=1E-5)
 early_stopping = callbacks.EarlyStopping(monitor='val_logloss', min_delta=1E-5, patience=10, mode='min',restore_best_weights=True)
 hist = model.fit([X_train,X_train_2],y_train, batch_size=128, epochs=182,verbose=0,validation_data =([X_test,X_test_2],y_test),
callbacks=[reduce_lr, early_stopping])
 hists.append(hist)
 model.save('ResNet_seed_'+str(seed)+'_fold_'+str(fold))
y_val = model.predict([X_test,X_test_2])
 oof += logloss(tf.constant(y_test,dtype=tf.float32),tf.constant(y_val,dtype=tf.float32)) /(n_folds*n_seeds)
 y_pred += model.predict([data_test,data_test_2])/(n_folds*n_seeds)
 fold += 1
tf.keras.backend.clear_session()
del model, X_train, X_test, X_train_2, X_test_2
X = train_df.drop(["isFraud"],axis=1)
 y= train_df["isFraud"]
X_Test = test_df.copy()
X_Test.drop(['TransactionID', 'isFraud'],axis=1,inplace=True)
 del train_df, test_df, train_identity, test_identity
gc.collect()<count_missing_values>
tf.print('
OOF score is ',oof )
print(f"Before dropna, top missing columns:
{X.isna().sum().sort_values(ascending = False ).head(5)}
")
 thresh = 0.80
X_less_nas = X.dropna(thresh=X.shape[0]*(1-thresh), axis='columns')
 cols_dropped = list(set(X.columns)-set(X_less_nas.columns))
X_Test.drop(cols_dropped, axis=1, inplace=True)
 print(f"After dropna, top missing columns:
{X_less_nas.isna().sum().sort_values(ascending = False ).head(5)}")
 print(f"
No.of cols dropped = {len(set(X.columns)-set(X_less_nas.columns)) }, or {len(set(X.columns)-set(X_less_nas.columns)) /len(X.columns)*100:.2f}% of columns")
 del X ; gc.collect()<define_variables>
sub_resnet = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 sub_resnet.iloc[:,1:] = np.clip(y_pred,p_min,p_max)
 sub_resnet.iloc[test_features['cp_type'] == 'ctl_vehicle',1:] = 0
sub_resnet
Catfeats = ['ProductCD'] + \
["card"+f"{i+1}" for i in range(6)] + \
["addr"+f"{i+1}" for i in range(2)] + \
["P_emaildomain", "R_emaildomain"] + \
["M"+f"{i+1}" for i in range(9)] + \
["DeviceType", "DeviceInfo"] + \
["id_"+f"{i}" for i in range(12, 39)]
Catfeats = list(set(Catfeats)- set(cols_dropped))<train_model>
%%time
train_features = pd.read_csv('/kaggle/input/lish-moa/train_features.csv')
 non_ctl_idx = train_features.loc[train_features['cp_type']!='ctl_vehicle'].index.to_list()
train_features = train_features.drop(['sig_id','cp_type'],axis=1)
 train_targets_scored = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
 train_targets_scored = train_targets_scored.drop('sig_id',axis=1)
 labels_train = train_targets_scored.values
train_features = train_features.iloc[non_ctl_idx]
labels_train = labels_train[non_ctl_idx]
test_features = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 test_features = test_features.drop('sig_id',axis=1 )
X_less_nas.fillna(-10000, inplace=True)
 X_Test.fillna(-10000, inplace=True )<train_model>
cat = 'cp_dose'
le = preprocessing.LabelEncoder()
le.fit(train_features[cat])
 train_features[cat] = le.transform(train_features[cat])
 test_features[cat] = le.transform(test_features[cat] )
X_tr, X_val, y_tr, y_val = train_test_split(X_less_nas, y, test_size=0.2, random_state=SEED,stratify = y)
 cat_params = {
'loss_function': 'Logloss',
'custom_loss':['AUC'],
'logging_level':'Silent',
'task_type' : 'GPU',
'early_stopping_rounds' : 100
}
simple_model = CatBoostClassifier(**cat_params)
 simple_model.fit(
 X_tr, y_tr,
cat_features=Catfeats,
eval_set=(X_val, y_val),plot=True 
); 
<train_model>
scaler = preprocessing.MinMaxScaler()
scaler.fit(train_features.append(test_features.drop('cp_type',axis=1)))
 data_train = scaler.transform(train_features)
 data_test = scaler.transform(test_features.drop('cp_type',axis=1))
simple_model.fit(
 X_less_nas, y,
cat_features=Catfeats,logging_level = 'Silent' 
);<save_to_csv>
%%time
n_labels = train_targets_scored.shape[1]
n_features = data_train.shape[1]
n_train = data_train.shape[0]
n_test = data_test.shape[0]
p_min = 0.001
p_max = 0.999
@tf.autograph.experimental.do_not_convert
def logloss(y_true, y_pred):
y_pred = tf.clip_by_value(y_pred,p_min,p_max)
 return -backend.mean(y_true*backend.log(y_pred)+(1-y_true)*backend.log(1-y_pred))
n_seeds = 6
np.random.seed(1)
 seeds = np.random.randint(0,100,size=n_seeds)
 n_folds = 5
y_pred = np.zeros(( n_test,n_labels))
oof = tf.constant(0.0)
 hists = []
for seed in seeds:
fold = 0
mskf = MultilabelStratifiedKFold(n_splits=n_folds,shuffle=True,random_state=seed)
 for train, test in mskf.split(data_train,labels_train):
X_train = data_train[train]
X_test = data_train[test]
y_train = labels_train[train]
y_test = labels_train[test]
model = Sequential()
model.add(layers.Dense(2048, input_shape=(n_features,)))
 model.add(layers.Activation('relu'))
model.add(layers.BatchNormalization())
 model.add(layers.Dropout(0.3))
model.add(layers.Dense(1024))
model.add(layers.Activation('relu'))
model.add(layers.BatchNormalization())
 model.add(layers.Dropout(0.3))
model.add(layers.Dense(512))
model.add(layers.Activation('relu'))
model.add(layers.BatchNormalization())
 model.add(layers.Dropout(0.3))
model.add(layers.Dense(n_labels))
model.add(layers.Activation('sigmoid'))
model.compile(optimizer='adam', loss=losses.BinaryCrossentropy(label_smoothing=0.001), metrics=logloss)
 reduce_lr = callbacks.ReduceLROnPlateau(monitor='val_logloss', factor=0.3, patience=5, mode='min', min_lr=1E-5)
 early_stopping = callbacks.EarlyStopping(monitor='val_logloss', min_delta=1E-5, patience=15, mode='min',restore_best_weights=True)
 hist = model.fit(X_train,y_train, batch_size=128, epochs=182,verbose=0,validation_data =(X_test,y_test),callbacks=[reduce_lr, early_stopping])
 hists.append(hist)
 model.save('LabelSmooth_seed_'+str(seed)+'_fold_'+str(fold))
y_val = model.predict(X_test)
 oof += logloss(tf.constant(y_test,dtype=tf.float32),tf.constant(y_val,dtype=tf.float32)) /(n_folds*n_seeds)
 y_pred += model.predict(data_test)/(n_folds*n_seeds)
 fold += 1
tf.print('OOF score is ',oof )
submission = pd.read_csv('.. /input/ieee-fraud-detection/sample_submission.csv')
 submission['isFraud'] = simple_model.predict_proba(X_Test)[:,1]
submission.to_csv('simple_model_Catboost.csv', index=False )<import_modules>
sub_smooth = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 sub_smooth.iloc[:,1:] = np.clip(y_pred,p_min,p_max)
 sub_smooth.iloc[test_features['cp_type'] == 'ctl_vehicle',1:] = 0
sub_smooth
import gc
import numpy as np
import pandas as pd<install_modules>
SEED = 34
NFOLDS = 5
DATA_DIR = '/kaggle/input/lish-moa/'
np.random.seed(SEED)
 train = pd.read_csv(DATA_DIR + 'train_features.csv')
 non_ctl_idx = train.loc[train['cp_type']!='ctl_vehicle'].index.to_list()
train_drug = pd.read_csv(DATA_DIR + 'train_drug.csv')
 targets = pd.read_csv(DATA_DIR + 'train_targets_scored.csv')
 test = pd.read_csv(DATA_DIR + 'test_features.csv')
 sub_xgb = pd.read_csv(DATA_DIR + 'sample_submission.csv')
 ctl_mask = train.cp_type=='ctl_vehicle'
train = train[~ctl_mask]
targets = targets[~ctl_mask]
X = train.iloc[:,1:].to_numpy()
X_test = test.iloc[:,1:].to_numpy()
y = targets.iloc[:,1:].to_numpy()
clf = Pipeline([('encode', CountEncoder(cols=[0, 2])) , 
('classify', MultiOutputClassifier(
 XGBClassifier(tree_method='gpu_hist')))
 ])
 params = {'classify__estimator__colsample_bytree': 0.6522,
'classify__estimator__gamma': 3.6975,
'classify__estimator__learning_rate': 0.0503,
'classify__estimator__max_delta_step': 2.0706,
'classify__estimator__max_depth': 10,
'classify__estimator__min_child_weight': 31.5800,
'classify__estimator__n_estimators': 166,
'classify__estimator__subsample': 0.8639
}
_ = clf.set_params(**params )
<import_modules>
def assign_folds(seed,fold):
scored = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
 targets = scored.columns[1:]
scored = scored.merge(train_drug, on='sig_id', how='left')
 scored = scored.iloc[non_ctl_idx]
vc = scored.drug_id.value_counts()
vc1 = vc.loc[vc<=18].index.sort_values()
vc2 = vc.loc[vc>18].index.sort_values()
dct1 = {}; dct2 = {}
skf = MultilabelStratifiedKFold(n_splits=fold, shuffle=True,
random_state=seed)
 tmp = scored.groupby('drug_id')[targets].mean().loc[vc1]
for fold,(idxT,idxV)in enumerate(skf.split(tmp,tmp[targets])) :
dd = {k:fold for k in tmp.index[idxV].values}
dct1.update(dd)
 skf = MultilabelStratifiedKFold(n_splits=fold, shuffle=True,
random_state=seed)
 tmp = scored.loc[scored.drug_id.isin(vc2)].reset_index(drop=True)
 for fold,(idxT,idxV)in enumerate(skf.split(tmp,tmp[targets])) :
dd = {k:fold for k in tmp.sig_id[idxV].values}
dct2.update(dd)
 scored['fold'] = scored.drug_id.map(dct1)
 scored.loc[scored.fold.isna() ,'fold'] =\
scored.loc[scored.fold.isna() ,'sig_id'].map(dct2)
 scored.fold = scored.fold.astype('int8')
 return scored['fold'].values
<install_modules>
use_xgb = True
<load_from_csv>
%%time
if use_xgb:
oof_preds = np.zeros(y.shape)
 test_preds = np.zeros(( test.shape[0], y.shape[1]))
oof_losses = []
x = assign_folds(SEED,NFOLDS)
 for n in range(NFOLDS):
print(f'Inferencing FOLD: {n}')
 trn_idx = np.where(x != n)[0]
val_idx = np.where(x == n)[0]
X_train, X_val = X[trn_idx], X[val_idx]
y_train, y_val = y[trn_idx], y[val_idx]
ctl_mask = X_train[:,0]=='ctl_vehicle'
X_train = X_train[~ctl_mask,:]
y_train = y_train[~ctl_mask]
clf = pickle.load(open(f'.. /input/moa-xgb-weights/xgb_cv_fold_{n}.dat', "rb"))
val_preds = clf.predict_proba(X_val)
 val_preds = np.array(val_preds)[:,:,1].T
oof_preds[val_idx] = val_preds
loss = log_loss(np.ravel(y_val), np.ravel(val_preds))
oof_losses.append(loss)
 preds = clf.predict_proba(X_test)
 preds = np.array(preds)[:,:,1].T
test_preds += preds / NFOLDS
print(oof_losses)
 print('Mean OOF loss across folds', np.mean(oof_losses))
print('STD OOF loss across folds', np.std(oof_losses))
train_transaction = pd.read_csv('.. /input/ieee-fraud-detection/train_transaction.csv', index_col='TransactionID')
 test_transaction = pd.read_csv('.. /input/ieee-fraud-detection/test_transaction.csv', index_col='TransactionID')
 train_identity = pd.read_csv('.. /input/ieee-fraud-detection/train_identity.csv', index_col='TransactionID')
 test_identity = pd.read_csv('.. /input/ieee-fraud-detection/test_identity.csv', index_col='TransactionID')
 sample_submission = pd.read_csv('.. /input/ieee-fraud-detection/sample_submission.csv', index_col='TransactionID')
 train = train_transaction.merge(train_identity, how='left', left_index=True, right_index=True)
 del train_transaction, train_identity
gc.collect()
test = test_transaction.merge(test_identity, how='left', left_index=True, right_index=True)
 del test_transaction, test_identity
gc.collect
train=reduce_mem_usage(train)
 test=reduce_mem_usage(test)
 print('training set shape:', train.shape)
 print('test set shape:', test.shape )<drop_column>
control_mask = train['cp_type']=='ctl_vehicle'
oof_preds[control_mask] = 0
print('OOF log loss: ', log_loss(np.ravel(y), np.ravel(oof_preds)) )
cols_to_drop=['V300','V309','V111','C3','V124','V106','V125','V315','V134','V102','V123','V316','V113',
'V136','V305','V110','V299','V289','V286','V318','V103','V304','V116','V29','V284','V293',
'V137','V295','V301','V104','V311','V115','V109','V119','V321','V114','V133','V122','V319',
'V105','V112','V118','V117','V121','V108','V135','V320','V303','V297','V120']
print('{} features are going to be dropped for being useless'.format(len(cols_to_drop)))
 train = train.drop(cols_to_drop, axis=1)
 test = test.drop(cols_to_drop, axis=1)
 del cols_to_drop
gc.collect<filter>
control_mask = test['cp_type']=='ctl_vehicle'
test_preds[control_mask] = 0
sub_xgb.iloc[:,1:] = test_preds
sub_xgb
<count_missing_values><EOS> 
%%time
ids = sub_tabnet[['sig_id']]
sub_tabnet.drop('sig_id',inplace=True,axis=1)
 sub_resnet.drop('sig_id',inplace=True,axis=1)
 sub_smooth.drop('sig_id',inplace=True,axis=1)
 sub_xgb.drop('sig_id',inplace=True,axis=1)
 sub_gauss.drop('sig_id',inplace=True,axis=1)
 sub_en = 0.3*sub_gauss + 0.3*sub_tabnet + 0.25*sub_resnet + 0.1*sub_smooth +0.05*sub_xgb
sub_en['sig_id'] = ids[['sig_id']]
cols = sub_en.columns.tolist()
cols.insert(0, cols.pop(cols.index('sig_id')))
 sub_en = sub_en.reindex(columns=cols)
 sub_en.to_csv('submission.csv',index=False)
 sub_en