Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<load_from_csv>
from fastai import *
from fastai.text import *
import pandas as pd
train_targets = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 ss_svc = pd.read_csv('.. /input/lish-moa/sample_submission.csv')
 ss_lr = ss_svc.copy()
ss_rf = ss_svc.copy()
ss = ss_svc.copy()
cols = [c for c in ss_svc.columns.values if c != 'sig_id']<compute_test_metric>
train_df= pd.read_csv('.. /input/nlp-getting-started/train.csv')
 test_df= pd.read_csv('.. /input/nlp-getting-started/test.csv')
 train_df.head()
def preprocess(df):
df.loc[:, 'cp_type'] = df.loc[:, 'cp_type'].map({'trt_cp': 0, 'ctl_vehicle': 1})
 df.loc[:, 'cp_dose'] = df.loc[:, 'cp_dose'].map({'D1': 0, 'D2': 1})
 del df['sig_id']
return df
def log_loss_metric(y_true, y_pred):
metrics = []
for _target in train_targets.columns:
metrics.append(log_loss(y_true.loc[:, _target], y_pred.loc[:, _target].astype(float), labels = [0,1]))
return np.mean(metrics)
 train = preprocess(train_features)
 test = preprocess(test_features)
 del train_targets['sig_id']<choose_model_class>
def clean_data(df, text_col, new_col='cleaned_text'):

 tweets_data = df.copy()
tweets_data[new_col] = tweets_data[text_col]
tweets_data[new_col] = tweets_data[new_col].apply(lambda x : re.sub(
 '@[A-Za-z0-9]+', '', x))
tweets_data[new_col] = tweets_data[new_col].str.strip()
tweets_data[new_col] = tweets_data[new_col].map(lambda x: re.sub('\
',' ',str(x)))
 tweets_data[new_col] = tweets_data[new_col].map(lambda x: re.sub(r'\W',' ',str(x)))
 tweets_data[new_col] = tweets_data[new_col].map(lambda x: re.sub(r'https\s+|www.\s+',r'', str(x)))
 tweets_data[new_col] = tweets_data[new_col].map(lambda x: re.sub(r'http\s+|www.\s+',r'', str(x)))
 tweets_data[new_col] = tweets_data[new_col].map(lambda x: re.sub(r'\s+[a-zA-Z]\s+',' ',str(x)))
 tweets_data[new_col] = tweets_data[new_col].map(lambda x: re.sub(r'\^[a-zA-Z]\s+',' ',str(x)))
 tweets_data[new_col] = tweets_data[new_col].map(lambda x: re.sub(r'\s+',' ',str(x)))
 tweets_data[new_col] = tweets_data[new_col].str.lower()
tweets_data[new_col] = tweets_data[new_col].map(lambda x: re.sub(r"\’", "'", str(x)))
 tweets_data[new_col] = tweets_data[new_col].map(lambda x: re.sub(r"won't", "will not", str(x)))
 tweets_data[new_col] = tweets_data[new_col].map(lambda x: re.sub(r"can't", "can not", str(x)))
 tweets_data[new_col] = tweets_data[new_col].map(lambda x: re.sub(r"don't", "do not", str(x)))
 tweets_data[new_col] = tweets_data[new_col].map(lambda x: re.sub(r"dont", "do not", str(x)))
 tweets_data[new_col] = tweets_data[new_col].map(lambda x: re.sub(r"n\’t", " not", str(x)))
 tweets_data[new_col] = tweets_data[new_col].map(lambda x: re.sub(r"n't", " not", str(x)))
 tweets_data[new_col] = tweets_data[new_col].map(lambda x: re.sub(r"'re", " are", str(x)))
 tweets_data[new_col] = tweets_data[new_col].map(lambda x: re.sub(r"'s", " is", str(x)))
 tweets_data[new_col] = tweets_data[new_col].map(lambda x: re.sub(r"\’d", " would", str(x)))
 tweets_data[new_col] = tweets_data[new_col].map(lambda x: re.sub(r"\d", " would", str(x)))
 tweets_data[new_col] = tweets_data[new_col].map(lambda x: re.sub(r"'ll", " will", str(x)))
 tweets_data[new_col] = tweets_data[new_col].map(lambda x: re.sub(r"'t", " not", str(x)))
 tweets_data[new_col] = tweets_data[new_col].map(lambda x: re.sub(r"'ve", " have", str(x)))
 tweets_data[new_col] = tweets_data[new_col].map(lambda x: re.sub(r"'m", " am", str(x)))
 tweets_data[new_col] = tweets_data[new_col].map(lambda x: re.sub(r"
", "", str(x)))
 tweets_data[new_col] = tweets_data[new_col].map(lambda x: re.sub(r"\r", "", str(x)))
 tweets_data[new_col] = tweets_data[new_col].map(lambda x: re.sub(r"[0-9]", "digit", str(x)))
 tweets_data[new_col] = tweets_data[new_col].map(lambda x: re.sub(r"'", "", str(x)))
 tweets_data[new_col] = tweets_data[new_col].map(lambda x: re.sub(r
def create_model1(num_columns):
model = tf.keras.Sequential([
tf.keras.layers.Input(num_columns),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(512, activation="elu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(1024, activation="elu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(512, activation="elu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.4),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(2048, activation="elu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(206, activation="sigmoid"))
])
 model.compile(optimizer=tfa.optimizers.Lookahead(tf.optimizers.Adam() , sync_period=10),
loss='binary_crossentropy', 
)
 return model<choose_model_class>
train_df = clean_data(train_df, "text", 'cleaned_text')
 stemming = PorterStemmer()
stemming.stem('runs')
 temp = train_df['cleaned_text'].apply(lambda sentence : [stemming.stem(x)for x in sentence.split(" ")])
 sentences = []
for i in temp:
sentences.append(" ".join(i))
sentences[0]
train_df['stemmed_text'] = sentences
del sentences, temp
def create_model2(num_columns):
model = tf.keras.Sequential([
tf.keras.layers.Input(num_columns),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(2048, activation="relu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(1500, activation="relu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(1048, activation="relu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.5),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(500, activation="relu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(206, activation="sigmoid"))
])
 model.compile(optimizer=tfa.optimizers.Lookahead(tf.optimizers.Adam() , sync_period=10),
loss='binary_crossentropy', 
)
 return model<define_variables>
test_df = clean_data(test_df, "text", 'cleaned_text')
 stemming = PorterStemmer()
stemming.stem('runs')
 temp = test_df['cleaned_text'].apply(lambda sentence : [stemming.stem(x)for x in sentence.split(" ")])
 sentences = []
for i in temp:
sentences.append(" ".join(i))
sentences[0]
test_df['stemmed_text'] = sentences
del sentences, temp
top_feats = range(905)
 print(len(top_feats))<data_type_conversions>
tweets_data = pd.concat([train_df, test_df], axis=0, sort=False, ignore_index=True)
 tweets_data.head()
train=train.astype(float)
 test =test.astype(float )<init_hyperparams>
path = Path('/kaggle/working/' )
N_STARTS = 2
tf.random.set_seed(42)
 ss_NN = ss_svc.copy()
res_NN = train_targets.copy()
ss_NN.loc[:, train_targets.columns] = 0
res_NN.loc[:, train_targets.columns] = 0
for seed in range(N_STARTS):
for n,(tr, te)in enumerate(MultilabelStratifiedKFold(n_splits=4, random_state=seed, shuffle=True ).split(train_targets, train_targets)) :
print(f'Fold {n}')
 if seed%2 == 0:
model = create_model1(len(top_feats))
else :
model = create_model2(len(top_feats))
checkpoint_path = f'repeat:{seed}_Fold:{n}.hdf5'
reduce_lr_loss = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=3, verbose=1, epsilon=1e-4, mode='min')
 cb_checkpt = ModelCheckpoint(checkpoint_path, monitor = 'val_loss', verbose = 0, save_best_only = True,
save_weights_only = True, mode = 'min')
 model.fit(train.values[tr][:, top_feats],
train_targets.values[tr],
validation_data=(train.values[te][:, top_feats], train_targets.values[te]),
epochs=35, batch_size=128,
callbacks=[reduce_lr_loss, cb_checkpt], verbose=2 
)
 model.load_weights(checkpoint_path)
 test_predict = model.predict(test.values[:, top_feats])
 val_predict = model.predict(train.values[te][:, top_feats])
 ss_NN.loc[:, train_targets.columns] += test_predict
res_NN.loc[te, train_targets.columns] += val_predict
print('')
 ss_NN.loc[:, train_targets.columns] /=(( n+1)* N_STARTS)
 res_NN.loc[:, train_targets.columns] /= N_STARTS<compute_test_metric>
data_lm =(TextList.from_df(tweets_data, path, cols = "cleaned_text" ).split_by_rand_pct(0.1 ).label_for_lm().databunch(bs=32))
print(f'NN OOF Metric: {log_loss_metric(train_targets, res_NN)}')
 res_NN.loc[train['cp_type'] == 1, train_targets.columns] = 0
ss_NN.loc[test['cp_type'] == 1, train_targets.columns] = 0
print(f'NN OOF Metric with postprocessing: {log_loss_metric(train_targets, res_NN)}' )<feature_engineering>
data_lm.show_batch()
old_list = []
avg_list = []
new_list = []
for tar in range(206):
score1 = log_loss(train_targets.loc[:, train_targets.columns[tar]], valid_results.loc[:, train_targets.columns[tar]])
 score2 = log_loss(train_targets.loc[:, train_targets.columns[tar]], res_NN.loc[:, train_targets.columns[tar]])
 if score2 >= score1:
ss_NN.loc[:, train_targets.columns[tar]] = sub.loc[:, train_targets.columns[tar]]
res_NN.loc[:, train_targets.columns[tar]] = valid_results.loc[:, train_targets.columns[tar]]
else :
old_list.append(tar)
<compute_test_metric>
learn = language_model_learner(data_lm, AWD_LSTM, drop_mult=0.3 )
score1 = 1.- log_loss_metric(train_targets, res_NN)
 score2 = 1.- 0.014921110834529303
scores = [score1,score2]
sum_scores = sum(scores)
 weights = [x / sum_scores for x in scores]<feature_engineering>
learn.lr_find()
ss_NN.loc[:, train_targets.columns[:]] = ss_NN.loc[:, train_targets.columns[:]] * weights[0] + sub.loc[:, train_targets.columns[:]] * weights[1]
res_NN.loc[:, train_targets.columns[:]] = res_NN.loc[:, train_targets.columns[:]] * weights[0] + valid_results.loc[:, train_targets.columns[:]] * weights[1]<compute_test_metric>
learn.fit_one_cycle(5, 1e-2, moms=(0.8,0.7))
print(f'pytorch OOF Metric with postprocessing: {log_loss_metric(train_targets, valid_results)}' )<compute_test_metric>
learn.fit_one_cycle(5, 1e-4, moms=(0.8,0.7))
print(f'Final OOF Metric with postprocessing: {log_loss_metric(train_targets, res_NN)}' )<save_to_csv>
learn.save_encoder('fine_tuned_enc' )
ss_NN.to_csv('submission.csv', index=False )<load_from_csv>
X_train, X_val = train_test_split(train_df, test_size = 0.3 )
df = pd.read_csv('/kaggle/input/lish-moa/train_features.csv')
 non_ctl_idx = df.loc[df['cp_type']!='ctl_vehicle'].index.to_list()
df = df.drop(['sig_id','cp_type','cp_dose','cp_time'],axis=1)
 data_X_test = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 data_X_test = data_X_test.drop(['sig_id','cp_dose','cp_time'],axis=1)
 dada = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
 dada = dada.drop('sig_id',axis=1)
 df_label = dada.values
df = df.iloc[non_ctl_idx]
df_label = df_label[non_ctl_idx]<categorify>
data_classifier =(TextDataBunch.from_df('.', X_train, X_val, test_df, text_cols = "cleaned_text", label_cols = "target", vocab = data_lm.vocab))
cs = df.columns.str.contains('c-')
 gs = df.columns.str.contains('g-')
 def preprocessor(train,test):
n_gs = 2
n_cs = 100
pca_cs = PCA(n_components = n_cs)
 pca_gs = PCA(n_components = n_gs)
 train_pca_gs = pca_gs.fit_transform(train[:,gs])
 train_pca_cs = pca_cs.fit_transform(train[:,cs])
 test_pca_gs = pca_gs.transform(test[:,gs])
 test_pca_cs = pca_cs.transform(test[:,cs])
 train_c_mean = train[:,cs].mean(axis=1)
 test_c_mean = test[:,cs].mean(axis=1)
 train_g_mean = train[:,gs].mean(axis=1)
 test_g_mean = test[:,gs].mean(axis=1)
 train = np.concatenate(( train,train_pca_gs,train_pca_cs,train_c_mean[:,np.newaxis]
,train_g_mean[:,np.newaxis]),axis=1)
 test = np.concatenate(( test,test_pca_gs,test_pca_cs,test_c_mean[:,np.newaxis],
test_g_mean[:,np.newaxis]),axis=1)
 scaler = preprocessing.StandardScaler()
train = scaler.fit_transform(train)
 test = scaler.transform(test)
 return train, test<compute_test_metric>
data_classifier.show_batch()
p_min = 0.0005
p_max = 0.9995
def logloss(y_true, y_pred):
y_pred = tf.clip_by_value(y_pred,p_min,p_max)
 return -backend.mean(y_true*backend.log(y_pred)+(1-y_true)*backend.log(1-y_pred))<choose_model_class>
learn = text_classifier_learner(data_classifier, AWD_LSTM, drop_mult=0.7)
 learn.load_encoder('fine_tuned_enc' )
reduce_lr = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_logloss', factor=0.1, verbose=0,mode='min',
patience=3, min_lr=1E-7)
 early_st = tf.keras.callbacks.EarlyStopping(monitor='val_logloss', min_delta=1E-5, patience=7, verbose=0, mode='min',
baseline=None, restore_best_weights=True )<choose_model_class>
learn.fit_one_cycle(5, 5e-04, moms=(0.8,0.7))
def create_model1(num_columns):
model = tf.keras.Sequential([
tf.keras.layers.Input(num_columns),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(512, activation="elu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(1024, activation="elu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(512, activation="elu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.4),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(2048, activation="elu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.3),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(206, activation="sigmoid"))
])
 model.compile(optimizer=tfa.optimizers.Lookahead(tf.optimizers.Adam() , sync_period=10),
loss=losses.BinaryCrossentropy(label_smoothing=0.0015),metrics=logloss 
)
 return model<choose_model_class>
learn.freeze_to(-2)
 learn.fit_one_cycle(5, slice(1e-2/(2.6**4),1e-2), moms=(0.8,0.7))
def create_model(num_columns):
model = tf.keras.Sequential([
tf.keras.layers.Input(num_columns),
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(2048, activation="relu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(1048, activation="relu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.5),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(512, activation="relu")) ,
tf.keras.layers.BatchNormalization() ,
tf.keras.layers.Dropout(0.2),
tfa.layers.WeightNormalization(tf.keras.layers.Dense(206, activation="sigmoid"))
])
 model.compile(optimizer=tfa.optimizers.Lookahead(tf.optimizers.Adam() , sync_period=10),
loss=losses.BinaryCrossentropy(label_smoothing=0.001),metrics=logloss 
)
 return model<prepare_x_and_y>
learn.freeze_to(-3)
 learn.fit_one_cycle(5, slice(5e-3/(2.6**4),5e-3), moms=(0.8,0.7))
X = 976
n_seeds = 6
np.random.seed(1)
 seeds = np.random.randint(0,100,size=n_seeds)
 n_labels = df_label.shape[1]
n_train = df.shape[0]
n_test = data_X_test.shape[0]
n_folds = 5
y_pred = np.zeros(( n_test,n_labels))
oof = tf.constant(0.0)
 hists = []
for seed in seeds:
fold = 0
print(fold)
 mlkf = MultilabelStratifiedKFold(n_splits=n_folds,shuffle=True,random_state=seed)
 for n,(train, test)in enumerate(mlkf.split(df,df_label)) :
tf.print('OOF score is ',oof)
 print(f'Fold {n}')
 train_x, validation_x = preprocessor(df.iloc[train].values,
df.iloc[test].values)
 _,data_test = preprocessor(df.iloc[train].values,
data_X_test.drop('cp_type',axis=1 ).values)
 train_y = df_label[train]
validation_y = df_label[test]
if seed%2 == 0:
model = create_model1(X)
 else:
model = create_model(X)
 model.compile(optimizer='adam',loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=0.001), metrics=logloss)
 hist = model.fit(train_x,train_y, batch_size=128, epochs=200,verbose=0,validation_data =(validation_x,validation_y),callbacks =[reduce_lr, early_st])
 hists.append(hist)
 model.save('_seed_'+str(seed)+'_fold_'+str(fold))
y_val = model.predict(validation_x)
 oof += logloss(tf.constant(validation_y,dtype=tf.float32),tf.constant(y_val,dtype=tf.float32)) /(n_folds*n_seeds)
 y_pred += model.predict(data_test)/(n_folds*n_seeds)
 fold += 1 
<save_to_csv>
learn.unfreeze()
learn.fit_one_cycle(5, slice(1e-3/(2.6**4),1e-3), moms=(0.8,0.7))
final = pd.read_csv('/kaggle/input/lish-moa/sample_submission.csv')
 final.iloc[:,1:] = np.clip(y_pred,p_min,p_max)
 final.iloc[data_X_test['cp_type'] == 'ctl_vehicle',1:] = 0
final.to_csv('submission.csv', index=False)
 final.head()<load_from_csv>
learn.save('final_learner' )
train_features = pd.read_csv('/kaggle/input/lish-moa/train_features.csv')
 non_ctl_idx = train_features.loc[train_features['cp_type']!='ctl_vehicle'].index.to_list()
train_features = train_features.drop(['sig_id','cp_type','cp_dose','cp_time'],axis=1)
 train_targets_scored = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
 train_targets_scored = train_targets_scored.drop('sig_id',axis=1)
 labels_train = train_targets_scored.values
train_features = train_features.iloc[non_ctl_idx]
labels_train = labels_train[non_ctl_idx]
test_features = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 test_features = test_features.drop(['sig_id','cp_dose','cp_time'],axis=1 )<load_from_csv>
def get_preds_as_nparray(ds_type)-> np.ndarray:

 preds = learn.get_preds(ds_type)[0].detach().cpu().numpy()
sampler = [i for i in data_classifier.dl(ds_type ).sampler]
reverse_sampler = np.argsort(sampler)
 return preds[reverse_sampler, :]
def preprocessor_labelsmooth() :
train_features = pd.read_csv('/kaggle/input/lish-moa/train_features.csv')
 train_features = train_features.drop(['sig_id'],axis=1)
 non_ctl_idx = train_features.loc[train_features['cp_type']!='ctl_vehicle'].index.to_list()
train_features = train_features.drop(['cp_type'],axis=1)
 train_features = train_features.iloc[non_ctl_idx]
public_test_features = pd.read_csv('/kaggle/input/moapublictest/test_features.csv')
 public_test_features = public_test_features.drop('sig_id',axis=1)
 test_features = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 test_features = test_features.drop('sig_id',axis=1)
 cat = 'cp_dose'
le = preprocessing.LabelEncoder()
le.fit(train_features[cat])
 train_features[cat] = le.transform(train_features[cat])
 public_test_features[cat] = le.transform(public_test_features[cat])
 test_features[cat] = le.transform(test_features[cat])
 scaler = preprocessing.MinMaxScaler()
scaler.fit(train_features.append(public_test_features.drop('cp_type',axis=1)))
 data_train = scaler.transform(train_features)
 data_test = scaler.transform(test_features.drop('cp_type',axis=1))
cs = train_features.columns.str.contains('c-')
 gs = train_features.columns.str.contains('g-')
 return data_train, data_test, cs, gs
def preprocessor_autoencoder(data_test,cs,gs):
cells_test = data_test[:,cs]
genes_test = data_test[:,gs]
cells_autoencoder = tf.keras.models.load_model(
 '.. /input/moaaemodels/CellsAE')
 genes_autoencoder = tf.keras.models.load_model(
 '.. /input/moaaemodels/GenesAE')
 ae_cells_test = cells_autoencoder.encoder(cells_test ).numpy()
ae_genes_test = genes_autoencoder.encoder(genes_test ).numpy()
data_test = np.concatenate(( data_test[:,~(cs+gs)],ae_genes_test,ae_cells_test),axis=1)
 return data_test
def preprocessor_resnet() :
train_features = pd.read_csv('/kaggle/input/lish-moa/train_features.csv')
 non_ctl_idx = train_features.loc[train_features['cp_type']!='ctl_vehicle'].index.to_list()
train_features = train_features.drop(['sig_id','cp_type','cp_dose','cp_time'],axis=1)
 train_targets_scored = pd.read_csv('/kaggle/input/lish-moa/train_targets_scored.csv')
 train_targets_scored = train_targets_scored.drop('sig_id',axis=1)
 labels_train = train_targets_scored.values
train_features = train_features.iloc[non_ctl_idx]
labels_train = labels_train[non_ctl_idx]
test_features = pd.read_csv('/kaggle/input/lish-moa/test_features.csv')
 test_features = test_features.drop(['sig_id','cp_dose','cp_time'],axis=1)
 json_file_path = '.. /input/t-test-pca-rfe-logistic-regression/main_predictors.json'
with open(json_file_path, 'r')as j:
predictors = json.loads(j.read())
 predictors = predictors['start_predictors']
cs = train_features.columns.str.contains('c-')
 gs = train_features.columns.str.contains('g-')
 return train_features,test_features,cs,gs,predictors
def preprocessor_resnet_fold(train,test,cs,gs):
n_gs = 2
n_cs = 100
pca_cs = PCA(n_components = n_cs)
 pca_gs = PCA(n_components = n_gs)
 train_pca_gs = pca_gs.fit_transform(train[:,gs])
 train_pca_cs = pca_cs.fit_transform(train[:,cs])
 test_pca_gs = pca_gs.transform(test[:,gs])
 test_pca_cs = pca_cs.transform(test[:,cs])
 train_c_mean = train[:,cs].mean(axis=1)
 test_c_mean = test[:,cs].mean(axis=1)
 train_g_mean = train[:,gs].mean(axis=1)
 test_g_mean = test[:,gs].mean(axis=1)
 train = np.concatenate(( train,train_pca_gs,train_pca_cs,train_c_mean[:,np.newaxis]
,train_g_mean[:,np.newaxis]),axis=1)
 test = np.concatenate(( test,test_pca_gs,test_pca_cs,test_c_mean[:,np.newaxis],
test_g_mean[:,np.newaxis]),axis=1)
 scaler = preprocessing.StandardScaler()
train = scaler.fit_transform(train)
 test = scaler.transform(test)
 return train, test<define_variables>
test_preds = get_preds_as_nparray(DatasetType.Test)
 preds = []
model_paths = ['.. /input/moalabelsmoothmodels/LabelSmoothed','.. /input/moaaemodels/AutoEncoded','.. /input/moa-multi-input-resnet-model/TwoHeads']
n_models = 3
n_labels = labels_train.shape[1]
n_test = test_features.shape[0]
n_train = train_features.shape[0]
y_val = np.zeros(( n_models, n_train, n_labels))
y_pred = np.zeros(( n_models, n_test, n_labels))<compute_test_metric>
for i in test_preds:
preds.append(np.argmax(i))
p_min = 0.001
p_max = 0.999
def logloss(y_true, y_pred):
y_pred = tf.clip_by_value(y_pred,p_min,p_max)
 return -backend.mean(y_true*backend.log(y_pred)+(1-y_true)*backend.log(1-y_pred))<categorify>
sub = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 sub.head(3 )
<split><EOS> 
sub['target'] = preds
sub.to_csv('submission.csv', index=False)
 sub.head(3 )