Kaggle competition: Mechanisms of Action (MoA) Prediction
train_df3 = train_df2[train_df2['damageDealt'] - train_df2['damageDealt'].mean() <=(3 * train_df2['damageDealt'].std())]
train_df3 = train_df2[train_df2['walkDistance'] - train_df2['walkDistance'].mean() <=(3 * train_df2['walkDistance'].std())]
train_df3 = train_df2[train_df2['rideDistance'] - train_df2['rideDistance'].mean() <=(3 * train_df2['rideDistance'].std())]
train_df3 = train_df2[train_df2['swimDistance'] - train_df2['swimDistance'].mean() <=(3 * train_df2['swimDistance'].std())]<correct_missing_values>
warnings.filterwarnings('ignore' )
train_df3 = train_df3.dropna()<split>
train_features = pd.read_csv('.. /input/lish-moa/train_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv')
 test_features = pd.read_csv('.. /input/lish-moa/test_features.csv')
 sample_submission = pd.read_csv('.. /input/lish-moa/sample_submission.csv' )
id_cols = train_df3.loc[:, ["Id", "groupId", "matchId"]]
train_df4 = train_df3.drop(columns = id_cols)
 train_df4.dropna(inplace=True)
 X = train_df4.iloc[:, 0:-1]; y = train_df4.iloc[:, -1]
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.3, random_state=2015195017 )<create_dataframe>
GENES = [col for col in train_features.columns if col.startswith('g-')]
CELLS = [col for col in train_features.columns if col.startswith('c-')]
lgb_train = lgb.Dataset(X_train, label=y_train)
 lgb_val = lgb.Dataset(X_val, label=y_val)
 params = {'objective': 'regression',
'metric': 'mae',
'n_estimators': 20000,
'early_stopping_rounds': 5,
'num_leaves': 31,
'learning_rate': 0.05,
'bagging_fraction': 0.7,
'bagging_seed': 20181219,
'num_threads': 5,
'colsample_bytree': 0.7}
lgr = lgb.train(params, lgb_train, valid_sets=[lgb_train, lgb_val], early_stopping_rounds=200, verbose_eval=1000 )<drop_column>
def seed_everything(seed=1903):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything(seed=1903 )
test_df2 = test_df2.drop(columns=['Id', 'groupId', 'matchId'] )<predict_on_test>
n_comp = 28
data = pd.concat([pd.DataFrame(train_features[GENES]), pd.DataFrame(test_features[GENES])])
 data2 =(PCA(n_components=n_comp, random_state=1903 ).fit_transform(data[GENES]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_G-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1)
 n_comp = 5
data = pd.concat([pd.DataFrame(train_features[CELLS]), pd.DataFrame(test_features[CELLS])])
 data2 =(PCA(n_components=n_comp, random_state=1903 ).fit_transform(data[CELLS]))
train2 = data2[:train_features.shape[0]]; test2 = data2[-test_features.shape[0]:]
train2 = pd.DataFrame(train2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 test2 = pd.DataFrame(test2, columns=[f'pca_C-{i}' for i in range(n_comp)])
 train_features = pd.concat(( train_features, train2), axis=1)
 test_features = pd.concat(( test_features, test2), axis=1 )
y_pred_lgb = lgr.predict(test_df2, num_iteration=lgr.best_iteration )<prepare_output>
var_thresh = VarianceThreshold(threshold=0.4)
 data = train_features.append(test_features)
 data_transformed = var_thresh.fit_transform(data.iloc[:, 4:])
 train_features_transformed = data_transformed[ : train_features.shape[0]]
test_features_transformed = data_transformed[-test_features.shape[0] : ]
train_features = pd.DataFrame(train_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 train_features = pd.concat([train_features, pd.DataFrame(train_features_transformed)], axis=1)
 test_features = pd.DataFrame(test_features[['sig_id','cp_type','cp_time','cp_dose']].values.reshape(-1, 4),\
columns=['sig_id','cp_type','cp_time','cp_dose'])
 test_features = pd.concat([test_features, pd.DataFrame(test_features_transformed)], axis=1)
 train_features
submit_lgb = pd.DataFrame({'Id': test_df['Id'], "winPlacePerc": y_pred_lgb} , columns=['Id', 'winPlacePerc'])
 submit_lgb.head()<save_to_csv>
train = train_features.merge(train_targets_scored, on='sig_id')
 train = train[train['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target = train[train_targets_scored.columns]
train = train.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1)
submit_lgb.to_csv("submission_lgb.csv", index=False )<load_from_csv>
train_non_scored = train_features.merge(train_targets_nonscored, on='sig_id')
 train_non_scored = train_non_scored[train_non_scored['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 test = test_features[test_features['cp_type']!='ctl_vehicle'].reset_index(drop=True)
 target_non_scored = train_non_scored[train_targets_nonscored.columns]
train_non_scored = train_non_scored.drop('cp_type', axis=1)
 test = test.drop('cp_type', axis=1)
ROOT_FOLDER = '.. /input/whale-categorization-playground'
TRAIN_FOLDER = ROOT_FOLDER + '/train/train/'
TEST_FOLDER = ROOT_FOLDER + '/test/test/'
WIDTH = 150
HEIGHT = 150
df_train = pd.read_csv(ROOT_FOLDER + '/train.csv')
 df_test = pd.read_csv(ROOT_FOLDER + '/sample_submission.csv')
 label_to_index = {}
index_to_label = {}
TIMES_TO_TRAIN = len(df_train)* 50
BATCH_SIZE = 100
for index,(image, label)in df_train.iterrows() :
index_to_label[index] = label
if not label in label_to_index:
label_to_index[label] = len(label_to_index)
 model = Sequential()
model.add(Conv2D(filters=16, kernel_size=5, padding="same", activation="relu", input_shape=(150, 150, 3)))
 model.add(MaxPooling2D(pool_size=(5, 5), strides=(5, 5), padding="valid"))
model.add(Dropout(0.1))
model.add(BatchNormalization())
 model.add(Conv2D(32,(15, 15), activation="relu"))
model.add(MaxPooling2D(pool_size=(5, 5), strides=(5, 5), padding="valid"))
model.add(Dropout(0.2))
model.add(BatchNormalization())
 model.add(Flatten())
 model.add(Dense(4251, activation="relu"))
model.compile(optimizer='adam',
loss='binary_crossentropy', metrics=['binary_crossentropy', 'acc'])
 datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 for t in range(int(TIMES_TO_TRAIN / BATCH_SIZE)+ 1):
print(str(t)+ "/" + str(int(TIMES_TO_TRAIN / BATCH_SIZE)))
 x_train = []
y_train = []
for i in range(BATCH_SIZE):
idx = randrange(len(df_train))
image = df_train.loc[idx]['Image']
x_train.append(img_to_array(load_img(TRAIN_FOLDER + image, target_size=(WIDTH, HEIGHT))))
y_train.append(np.zeros(4251))
y_train[i][label_to_index[df_train.loc[idx]['Id']]] = 1
x_train = np.array(x_train)
 y_train = np.array(y_train)
 datagen.fit(x_train)
 model.fit_generator(datagen.flow(x_train, y_train, batch_size=BATCH_SIZE),
steps_per_epoch=x_train.shape[0] // BATCH_SIZE, verbose=False)
 model.fit(x_train, y_train, batch_size=BATCH_SIZE, verbose=False)
 def sort_predictions(array):
pairs = []
for idx, v in enumerate(array):
pairs.append(( idx, v))
pairs.sort(key=lambda p: p[1], reverse=True)
 top_sorted = []
for p in pairs[:5]:
top_sorted.append(index_to_label[p[0]])
 return top_sorted
print('----------------- PREDICTING -----------------')
 predictions = []
for index,(image, labels)in df_test.iterrows() :
image = df_test.loc[index]['Image']
x_test = np.array([img_to_array(load_img(TEST_FOLDER + image, target_size=(WIDTH, HEIGHT)))])
 preds = model.predict(x_test)[0]
top_preds = sort_predictions(preds)
 predictions.append(top_preds)
 print(str(index)+ '/' + str(len(df_test)))
 submission = pd.DataFrame(data=[[df_test.loc[i]['Image'], "{} {} {} {} {}".format(*predictions[i])] for i in range(len(predictions)) ],
columns=['Image', 'Id'])
 print(submission)
 submission.to_csv('submission.csv', index=False )<import_modules>
target_cols = target.drop('sig_id', axis=1 ).columns.values.tolist()
target_non_scored_cols = train_targets_nonscored.drop('sig_id', axis=1 ).columns.values.tolist()
trainData = pd.read_csv(".. /input/whale-categorization-playground/train.csv")
 trainData.sample(5)
<define_variables>
folds = train.copy()
mskf = MultilabelStratifiedKFold(n_splits=7)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=target)) :
folds.loc[v_idx, 'kfold'] = int(f)
 folds['kfold'] = folds['kfold'].astype(int )
Image(filename=".. /input/whale-categorization-playground/train/"+random.choice(trainData['Image']))<normalization>
folds_non_scored = train_non_scored.copy()
mskf = MultilabelStratifiedKFold(n_splits=7)
 for f,(t_idx, v_idx)in enumerate(mskf.split(X=train, y=target)) :
folds_non_scored.loc[v_idx, 'kfold'] = int(f)
 folds_non_scored['kfold'] = folds_non_scored['kfold'].astype(int )
def prepareImages(data, m, dataset):
print("Preparing images")
 X_train = np.zeros(( m, 100, 100, 3))
count = 0
for fig in data['Image']:
img = image.load_img(".. /input/whale-categorization-playground/"+dataset+"/"+fig, target_size=(100, 100, 3))
x = image.img_to_array(img)
 x = preprocess_input(x)
 X_train[count] = x
if(count%500 == 0):
print("Processing image: ", count+1, ", ", fig)
 count += 1
count = 0
print("Finished!")
 return X_train<categorify>
class MoADataset:
def __init__(self, features, targets):
self.features = features
self.targets = targets
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float),
'y' : torch.tensor(self.targets[idx, :], dtype=torch.float)
 }
return dct
class TestDataset:
def __init__(self, features):
self.features = features
def __len__(self):
return(self.features.shape[0])
 def __getitem__(self, idx):
dct = {
'x' : torch.tensor(self.features[idx, :], dtype=torch.float)
 }
return dct
def prepareY(Y):
values = array(Y)
 print(values.shape)
 label_encoder = LabelEncoder()
integer_encoded = label_encoder.fit_transform(values)
 print(integer_encoded)
 onehot_encoder = OneHotEncoder(sparse=False)
 integer_encoded = integer_encoded.reshape(len(integer_encoded), 1)
 onehot_encoded = onehot_encoder.fit_transform(integer_encoded)
 print(onehot_encoded)
 y = onehot_encoded
print(y.shape)
 return y, label_encoder 
<choose_model_class>
def train_fn(model, optimizer, scheduler, loss_fn, dataloader, device):
model.train()
final_loss = 0
for data in dataloader:
optimizer.zero_grad()
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 loss.backward()
optimizer.step()
scheduler.step()
final_loss += loss.item()
final_loss /= len(dataloader)
 return final_loss
def valid_fn(model, loss_fn, dataloader, device):
model.eval()
final_loss = 0
valid_preds = []
for data in dataloader:
inputs, targets = data['x'].to(device), data['y'].to(device)
 outputs = model(inputs)
 loss = loss_fn(outputs, targets)
 final_loss += loss.item()
valid_preds.append(outputs.sigmoid().detach().cpu().numpy())
 final_loss /= len(dataloader)
 valid_preds = np.concatenate(valid_preds)
 return final_loss, valid_preds
def inference_fn(model, dataloader, device):
model.eval()
preds = []
for data in dataloader:
inputs = data['x'].to(device)
 with torch.no_grad() :
outputs = model(inputs)
 preds.append(outputs.sigmoid().detach().cpu().numpy())
 preds = np.concatenate(preds)
 return preds
mod = Sequential()
mod.add(Conv2D(32,(7, 7), strides =(1, 1), name = 'conv0', input_shape =(100, 100, 3)))
 mod.add(BatchNormalization(axis = 3, name = 'bn0'))
mod.add(Activation('relu'))
mod.add(MaxPooling2D(( 2, 2), name='max_pool'))
mod.add(Conv2D(64,(3, 3), strides =(1,1), name="conv1"))
mod.add(Activation('relu'))
mod.add(AveragePooling2D(( 3, 3), name='avg_pool'))
mod.add(Flatten())
 mod.add(Dense(500, activation="relu", name='rl'))
mod.add(Dropout(0.8))
mod.add(Dense(4251, activation='softmax', name='sm'))
print(mod.output_shape)
 mod.compile(loss='categorical_crossentropy', optimizer="adam", metrics=['accuracy'] )<train_model>
class Model(nn.Module):
def __init__(self, num_features, num_targets, hidden_size):
super(Model, self ).__init__()
self.batch_norm1 = nn.BatchNorm1d(num_features)
 self.dropout1 = nn.Dropout(0.2)
 self.dense1 = nn.utils.weight_norm(nn.Linear(num_features, hidden_size))
self.batch_norm2 = nn.BatchNorm1d(hidden_size)
 self.dropout2 = nn.Dropout(0.3)
 self.dense2 = nn.utils.weight_norm(nn.Linear(hidden_size, hidden_size))
self.batch_norm3 = nn.BatchNorm1d(hidden_size)
 self.dropout3 = nn.Dropout(0.25)
 self.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
def forward(self, x):
x = self.batch_norm1(x)
 x = self.dropout1(x)
 x = F.relu(self.dense1(x))
x = self.batch_norm2(x)
 x = self.dropout2(x)
 x = F.relu(self.dense2(x))
x = self.batch_norm3(x)
 x = self.dropout3(x)
 x = self.dense3(x)
 return x
X = prepareImages(trainData, 9850, "train")
 X /= 255
print("Shape X-train: ", X.shape )<categorify>
def process_data(data):
data = pd.get_dummies(data, columns=['cp_time','cp_dose'])
 return data
Y = trainData['Id']
print("Shape Y-train: ", Y.shape)
 y, label_encoder = prepareY(Y )<train_model>
feature_cols = [c for c in process_data(folds ).columns if c not in target_cols]
feature_cols = [c for c in feature_cols if c not in ['kfold','sig_id']]
len(feature_cols )
history = mod.fit(X, y, epochs=100, batch_size=100, verbose=1)
 gc.collect()<categorify>
DEVICE =('cuda' if torch.cuda.is_available() else 'cpu')
 EPOCHS = 100
BATCH_SIZE = 128
LEARNING_RATE = 1e-3
WEIGHT_DECAY = 1e-5
NFOLDS = 7
EARLY_STOPPING_STEPS = 10
EARLY_STOP = True
num_features=len(feature_cols)
 num_targets=len(target_cols)
 num_non_scored_targets=len(target_non_scored_cols)
 hidden_size=2048
<categorify>
import copy
<categorify>
def run_training(fold, seed, pretrain=False):
seed_everything(seed)
 train = process_data(folds)
 test_ = process_data(test)
 trn_idx = train[train['kfold'] != fold].index
val_idx = train[train['kfold'] == fold].index
train_df = train[train['kfold'] != fold].reset_index(drop=True)
 valid_df = train[train['kfold'] == fold].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_cols].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 if pretrain:
model = Model(
 num_features=num_features,
num_targets=num_non_scored_targets,
hidden_size=hidden_size, 
)
 model.load_state_dict(torch.load(f"FOLD{fold}_non_scored.pth"))
model.batch_norm3 = nn.BatchNorm1d(hidden_size)
 model.dropout3 = nn.Dropout(0.5)
 model.dense3 = nn.utils.weight_norm(nn.Linear(hidden_size, num_targets))
else:
model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 if pretrain:
scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1e-3, epochs=EPOCHS, steps_per_epoch=len(trainloader))
epochs = 100
else:
scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1e-2, epochs=EPOCHS, steps_per_epoch=len(trainloader))
epochs = copy.copy(EPOCHS)
 loss_fn = nn.BCEWithLogitsLoss()
early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
oof = np.zeros(( len(train), target.iloc[:, 1:].shape[1]))
best_loss = np.inf
print(epochs)
 for epoch in range(epochs):
train_loss = train_fn(model, optimizer,scheduler, loss_fn, trainloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}")
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"FOLD{fold}_.pth")
 elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_targets,
hidden_size=hidden_size, 
)
 model.load_state_dict(torch.load(f"FOLD{fold}_.pth"))
model.to(DEVICE)
 predictions = np.zeros(( len(test_), target.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions
<train_model>
def run_training_non_scored(fold, seed):
seed_everything(seed)
 train = process_data(folds_non_scored)
 test_ = process_data(test)
 trn_idx = train[train['kfold'] != fold].index
val_idx = train[train['kfold'] == fold].index
train_df = train[train['kfold'] != fold].reset_index(drop=True)
 valid_df = train[train['kfold'] == fold].reset_index(drop=True)
 x_train, y_train = train_df[feature_cols].values, train_df[target_non_scored_cols].values
x_valid, y_valid = valid_df[feature_cols].values, valid_df[target_non_scored_cols].values
train_dataset = MoADataset(x_train, y_train)
 valid_dataset = MoADataset(x_valid, y_valid)
 trainloader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
 validloader = torch.utils.data.DataLoader(valid_dataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_non_scored_targets,
hidden_size=hidden_size, 
)
 model.to(DEVICE)
 optimizer = torch.optim.Adam(model.parameters() , lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
 scheduler = optim.lr_scheduler.OneCycleLR(optimizer=optimizer, pct_start=0.1, div_factor=1e3,
max_lr=1e-4, epochs=EPOCHS, steps_per_epoch=len(trainloader))
loss_fn = nn.BCEWithLogitsLoss()
early_stopping_steps = EARLY_STOPPING_STEPS
early_step = 0
oof = np.zeros(( len(train), target_non_scored.iloc[:, 1:].shape[1]))
best_loss = np.inf
for epoch in range(10):
train_loss = train_fn(model, optimizer,scheduler, loss_fn, trainloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, train_loss: {train_loss}")
 valid_loss, valid_preds = valid_fn(model, loss_fn, validloader, DEVICE)
 print(f"FOLD: {fold}, EPOCH: {epoch}, valid_loss: {valid_loss}")
 if valid_loss < best_loss:
best_loss = valid_loss
oof[val_idx] = valid_preds
torch.save(model.state_dict() , f"FOLD{fold}_non_scored.pth")
 elif(EARLY_STOP == True):
early_step += 1
if(early_step >= early_stopping_steps):
break
x_test = test_[feature_cols].values
testdataset = TestDataset(x_test)
 testloader = torch.utils.data.DataLoader(testdataset, batch_size=BATCH_SIZE, shuffle=False)
 model = Model(
 num_features=num_features,
num_targets=num_non_scored_targets,
hidden_size=hidden_size, 
)
 model.load_state_dict(torch.load(f"FOLD{fold}_non_scored.pth"))
model.to(DEVICE)
 predictions = np.zeros(( len(test_), target_non_scored.iloc[:, 1:].shape[1]))
predictions = inference_fn(model, testloader, DEVICE)
 return oof, predictions
<create_dataframe>
test = os.listdir(".. /input/whale-categorization-playground/test/")
 print(len(test))
col = ['Image']
testData1 = pd.DataFrame(test[0:3899], columns=col)
 testData2 = pd.DataFrame(test[3900:7799], columns=col)
 testData3 = pd.DataFrame(test[7800:11699], columns=col)
 testData4 = pd.DataFrame(test[11700:15609], columns=col)
 testData = pd.DataFrame(test, columns=col )<normalization>
def run_k_fold(NFOLDS, seed, pretrain=False):
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 for fold in range(NFOLDS):
oof_, pred_ = run_training(fold, seed, pretrain)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions
gc.collect()
X = prepareImages(testData1, 3900, "test")
 X /= 255<predict_on_test>
def run_k_fold_none_scored(NFOLDS, seed):
oof = np.zeros(( len(train), len(target_non_scored_cols)))
 predictions = np.zeros(( len(test), len(target_non_scored_cols)))
 for fold in range(NFOLDS):
oof_, pred_ = run_training_non_scored(fold, seed)
 predictions += pred_ / NFOLDS
oof += oof_
return oof, predictions
predictions1 = mod.predict(np.array(X), verbose=1)
 gc.collect()<predict_on_test>
SEED = [1903, 1881]
oof = np.zeros(( len(train), len(target_cols)))
 predictions = np.zeros(( len(test), len(target_cols)))
 run_k_fold_none_scored(NFOLDS, SEED[0])
 for seed in SEED:
oof_, predictions_ = run_k_fold(NFOLDS, seed, pretrain=True)
 oof += oof_ / len(SEED)
 predictions += predictions_ / len(SEED)
 train[target_cols] = oof
test[target_cols] = predictions
X = prepareImages(testData2, 3900, "test")
 X /= 255
predictions2 = mod.predict(np.array(X), verbose=1)
 gc.collect()<predict_on_test>
valid_results = train_targets_scored.drop(columns=target_cols ).merge(train[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 y_true = train_targets_scored[target_cols].values
y_pred = valid_results[target_cols].values
score = 0
for i in range(len(target_cols)) :
score_ = log_loss(y_true[:, i], y_pred[:, i])
 score += score_ / target.shape[1]
print("CV log_loss: ", score )
<predict_on_test><EOS> 
sub = sample_submission.drop(columns=target_cols ).merge(test[['sig_id']+target_cols], on='sig_id', how='left' ).fillna(0)
 sub.to_csv('submission.csv', index=False )