Kaggle competition: Natural Language Processing with Disaster Tweets
<SOS> metric: meanfscore Kaggle data source: natural-language-processing-with-disaster-tweets<categorify>
start_time = time.time()
def create_autoencoder(input_dim,output_dim,noise=0.05):
i = Input(input_dim)
 encoded = BatchNormalization()(i)
 encoded = GaussianNoise(noise )(encoded)
 encoded = Dense(64,activation='relu' )(encoded)
 decoded = Dropout(0.2 )(encoded)
 decoded = Dense(input_dim,name='decoded' )(decoded)
 x = Dense(32,activation='relu' )(decoded)
 x = BatchNormalization()(x)
 x = Dropout(0.2 )(x)
 x = Dense(32,activation='relu' )(x)
 x = BatchNormalization()(x)
 x = Dropout(0.2 )(x)
 x = Dense(output_dim,activation='sigmoid',name='label_output' )(x)
 encoder = Model(inputs=i,outputs=encoded)
 autoencoder = Model(inputs=i,outputs=[decoded,x])
 autoencoder.compile(optimizer=Adam(0.005),loss={'decoded':'mse','label_output':'binary_crossentropy'})
 return autoencoder, encoder<categorify>
!pip install tensorflow-text==2.0.0 --user
def create_model(input_dim,output_dim,encoder):
inputs = Input(input_dim)
 x = encoder(inputs)
 x = Concatenate()([x,inputs])
 x = BatchNormalization()(x)
 x = Dropout(0.13 )(x)
 hidden_units = [384, 896, 896, 394]
for idx, hidden_unit in enumerate(hidden_units):
x = Dense(hidden_unit )(x)
 x = BatchNormalization()(x)
 x = Lambda(tf.keras.activations.relu )(x)
 x = Dropout(0.25 )(x)
 x = Dense(output_dim,activation='sigmoid' )(x)
 model = Model(inputs=inputs,outputs=x)
 model.compile(optimizer=Adam(0.0005),loss=BinaryCrossentropy(label_smoothing=0.05),metrics=[tf.keras.metrics.AUC(name = 'auc')])
 return model<categorify>
import tensorflow as tf
import tensorflow_hub as hub
import tensorflow_text as textb
autoencoder, encoder = create_autoencoder(X.shape[-1],y.shape[-1],noise=0.1)
 if TRAINING:
autoencoder.fit(X,(X,y),
epochs=1000,
batch_size=4096,
validation_split=0.1,
callbacks=[EarlyStopping('val_loss',patience=10,restore_best_weights=True)])
 encoder.save_weights('./encoder.hdf5')
 else:
encoder.load_weights('.. /input/janestreetmodels/encoder.hdf5')
 encoder.trainable = False<train_model>
pd.set_option('display.max_colwidth', -1)
 pd.set_option('display.max_rows', 310 )
FOLDS = 4
SEED = 42
if TRAINING:
gkf = PurgedGroupTimeSeriesSplit(n_splits = FOLDS, group_gap=20)
 splits = list(gkf.split(y, groups=train['date'].values))
for fold,(train_indices, test_indices)in enumerate(splits):
model = create_model(130, 5, encoder)
 X_train, X_test = X[train_indices], X[test_indices]
y_train, y_test = y[train_indices], y[test_indices]
model.fit(X_train,y_train,validation_data=(X_test,y_test),epochs=100,batch_size=4096,callbacks=[EarlyStopping('val_auc',mode='max',patience=10,restore_best_weights=True)])
 model.save_weights(f'./model_{SEED}_{fold}.hdf5')
 model.compile(Adam(0.00001),loss='binary_crossentropy')
 model.fit(X_test,y_test,epochs=3,batch_size=4096)
 model.save_weights(f'./model_{SEED}_{fold}_finetune.hdf5')
 else:
models = []
for f in range(FOLDS):
model = create_model(130, 5, encoder)
 if USE_FINETUNE:
model.load_weights(f'.. /input/janestreetmodels/model_{SEED}_{f}_finetune.hdf5')
 else:
model.load_weights(f'.. /input/janestreetmodels/model_{SEED}_{f}.hdf5')
 models.append(model )<feature_engineering>
train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 submission = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 train_copy = train.copy()
length_train = len(train.index)
 length_train
if not TRAINING:
f_median = np.median
models = models[-2:]
env = janestreet.make_env()
th = 0.50195
for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
if np.isnan(x_tt[:, 1:].sum()):
x_tt[:, 1:] = np.nan_to_num(x_tt[:, 1:])+ np.isnan(x_tt[:, 1:])* f_mean
pred = np.mean([model(x_tt, training = False ).numpy() for model in models],axis=1)
 pred = f(pred)
 pred_df.action = np.where(pred >= th, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )<import_modules>
index_hot = [4415, 4400, 4399,4403,4397,4396, 4394,4414, 4393,4392,4404,4407,4420,4412,4408,4391,4405]
train_example = train.copy()
train_example.loc[index_hot,:]
import numpy as np
import pandas as pd
import os
import tensorflow as tf
import tensorflow_addons as tfa
import numpy as np
import pandas as pd
import pathlib
from sklearn.model_selection import GroupKFold
from tqdm import tqdm
from random import choices 
<set_options>
index_hot = [6840,6834,6837,6841,6816]
train_example = train.copy()
train_example.loc[index_hot,:]
<define_variables>
index_hot = [6828,6831]
train_example = train.copy()
train_example.loc[index_hot,:]
CV_STRATEGY = 'StratifiedGroupKFold'
SEED = 2021
START_DATE = 86
FOLDS = 5
DEBUG = False<import_modules>
index_hot = [591, 587]
train_example = train.copy()
train_example.loc[index_hot,:]
class PurgedGroupTimeSeriesSplit(_BaseKFold):

 @_deprecate_positional_args
def __init__(self,
n_splits=5,
*,
max_train_group_size=np.inf,
max_test_group_size=np.inf,
group_gap=None,
verbose=False 
):
super().__init__(n_splits, shuffle=False, random_state=None)
 self.max_train_group_size = max_train_group_size
self.group_gap = group_gap
self.max_test_group_size = max_test_group_size
self.verbose = verbose
def split(self, X, y=None, groups=None):

 if groups is None:
raise ValueError(
 "The 'groups' parameter should not be None")
 X, y, groups = indexable(X, y, groups)
 n_samples = _num_samples(X)
 n_splits = self.n_splits
group_gap = self.group_gap
max_test_group_size = self.max_test_group_size
max_train_group_size = self.max_train_group_size
n_folds = n_splits + 1
group_dict = {}
u, ind = np.unique(groups, return_index=True)
 unique_groups = u[np.argsort(ind)]
n_samples = _num_samples(X)
 n_groups = _num_samples(unique_groups)
 for idx in np.arange(n_samples):
if(groups[idx] in group_dict):
group_dict[groups[idx]].append(idx)
 else:
group_dict[groups[idx]] = [idx]
if n_folds > n_groups:
raise ValueError(
("Cannot have number of folds={0} greater than"
" the number of groups={1}" ).format(n_folds,
n_groups))
group_test_size = min(n_groups // n_folds, max_test_group_size)
 group_test_starts = range(n_groups - n_splits * group_test_size,
n_groups, group_test_size)
 for group_test_start in group_test_starts:
train_array = []
test_array = []
group_st = max(0, group_test_start - group_gap - max_train_group_size)
 for train_group_idx in unique_groups[group_st:(group_test_start - group_gap)]:
train_array_tmp = group_dict[train_group_idx]
train_array = np.sort(np.unique(
 np.concatenate(( train_array,
train_array_tmp)) ,
axis=None), axis=None)
 train_end = train_array.size
for test_group_idx in unique_groups[group_test_start:
group_test_start +
group_test_size]:
test_array_tmp = group_dict[test_group_idx]
test_array = np.sort(np.unique(
 np.concatenate(( test_array,
test_array_tmp)) ,
axis=None), axis=None)
 test_array = test_array[group_gap:]
if self.verbose > 0:
pass
yield [int(i)for i in train_array], [int(i)for i in test_array]<data_type_conversions>
index_hot = [601,576,584,608,606,603,592,604]
train_example = train.copy()
train_example.loc[index_hot,:]
train = pd.read_feather('.. /input/janestreet-save-as-feather/train.feather')
 train = train.query(f'date >= {START_DATE}' ).reset_index(drop = True)
 train = train.astype({c: np.float32 for c in train.select_dtypes(include='float64' ).columns})
 train.fillna(train.mean() ,inplace=True)
 train = train.query('weight > 0' ).reset_index(drop = True)
 train['action'] =(( train['resp_1'] > 0)&(train['resp_2'] > 0)&(train['resp_3'] > 0)&(train['resp_4'] > 0)&(train['resp'] > 0)).astype('int')
 features = [c for c in train.columns if 'feature' in c]
resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4']
X = train[features].values
y = np.stack([(train[c] > 0 ).astype('int')for c in resp_cols] ).T
f_mean = np.mean(train[features[1:]].values,axis=0 )<categorify>
index_hot = [3667,3674,3688,3696]
train_example = train.copy()
train_example.loc[index_hot,:]
def create_autoencoder(input_dim,output_dim,noise=0.05):
i = tf.keras.layers.Input(input_dim)
 encoded = tf.keras.layers.BatchNormalization()(i)
 encoded = tf.keras.layers.GaussianNoise(noise )(encoded)
 encoded = tf.keras.layers.Dense(64,activation='relu' )(encoded)
 decoded = tf.keras.layers.Dropout(0.2 )(encoded)
 decoded = tf.keras.layers.Dense(input_dim,name='decoded' )(decoded)
 x = tf.keras.layers.Dense(32,activation='relu' )(decoded)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dropout(0.2 )(x)
 x = tf.keras.layers.Dense(output_dim,activation='sigmoid',name='label_output' )(x)
 encoder = tf.keras.models.Model(inputs=i,outputs=encoded)
 autoencoder = tf.keras.models.Model(inputs=i,outputs=[decoded,x])
 autoencoder.compile(optimizer=tf.keras.optimizers.Adam(0.001),
loss={'decoded':'mse','label_output':'binary_crossentropy'})
 return autoencoder, encoder<categorify>
index_hot = [3913,3914,3936,3921,3941,3937,3938]
train_example = train.copy()
train_example.loc[index_hot,:]
autoencoder, encoder = create_autoencoder(X.shape[-1],y.shape[-1],noise=0.1)
 encoder.load_weights('.. /input/janestreet-1dcnn-for-feature-extraction-train/encoder.hdf5')
 encoder.trainable = False<categorify>
index_hot = [3136, 3133]
train_example = train.copy()
train_example.loc[index_hot,:]
def create_model(input_dim, output_dim, encoder):
inputs = tf.keras.layers.Input(input_dim)
 x = encoder(inputs)
 x = tf.keras.layers.Concatenate()([x,inputs])
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.Dense(4096, activation='relu' )(x)
 x = tf.keras.layers.Reshape(( 256, 16))(x)
 x = tf.keras.layers.Conv1D(filters=16,
kernel_size=7,
strides=1,
activation='relu' )(x)
 x = tf.keras.layers.MaxPooling1D(pool_size=2 )(x)
 x = tf.keras.layers.Flatten()(x)
 for i in range(2):
x = tf.keras.layers.Dense(256 //(2 ** i), activation='relu' )(x)
 x = tf.keras.layers.BatchNormalization()(x)
 x = tf.keras.layers.GaussianNoise(0.01 )(x)
 x = tf.keras.layers.Dropout(0.2 )(x)
 x = tf.keras.layers.Dense(output_dim, activation='sigmoid' )(x)
 model = tf.keras.models.Model(inputs=inputs,outputs=x)
 opt = tfa.optimizers.RectifiedAdam(learning_rate=1e-03)
 opt = tfa.optimizers.SWA(opt)
 loss = tf.keras.losses.BinaryCrossentropy(label_smoothing=1e-02)
 model.compile(optimizer=opt,
loss=loss,
metrics=[tf.keras.metrics.AUC(name = 'auc')])
 return model<load_pretrained>
index_hot = [3930,3933,3924,3917]
train_example = train.copy()
train_example.loc[index_hot,:]
%%time
models = []
for fold in range(FOLDS):
tf.keras.backend.clear_session()
model = create_model(X.shape[-1], y.shape[-1], encoder)
 model.load_weights(pathlib.Path(f'.. /input/janestreet-1dcnn-for-feature-extraction-train/model_{SEED}_{fold}.hdf5'))
models.append(model)
<feature_engineering>
index_hot = [246,270,266,259,253,251,250,271]
train_example = train.copy()
train_example.loc[index_hot,:]
f = np.median
th = 0.5
models = models[-2:]
env = janestreet.make_env()
for(test_df, pred_df)in tqdm(env.iter_test()):
if test_df['weight'].item() > 0:
x_tt = test_df.loc[:, features].values
if np.isnan(x_tt[:, 1:].sum()):
x_tt[:, 1:] = np.nan_to_num(x_tt[:, 1:])+ np.isnan(x_tt[:, 1:])* f_mean
pred = np.mean([model(x_tt, training = False ).numpy() for model in models],axis=0)
 pred = f(pred)
 pred_df.action = np.where(pred >= th, 1, 0 ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )<import_modules>
def url(text):
text = re.sub(r' \w{1,3}\.{3,3} http\S{0,}', ' ', text)
 text = re.sub(r' \w{1,3}Û_ http\S{0,}', ' ', text)
 text = re.sub(r"mp3 http\S{0,}", r" ", text)
 text = re.sub(r"rar http\S{0,}", r" ", text)
 pattern = re.compile(r'(pin\:\d+ | via)http\S{0,}')
 text = pattern.sub(r' ', text)
 pattern = re.compile(r'Full read by|Full read b|Full read|Full rea|Full re|Full r')
 text = pattern.sub(r' ', text)
 pattern = re.compile(r'Full story at|Full story a|Full story|Full stor|Full sto|Full st|Full s')
 text = pattern.sub(r' ', text)
 return text
import torch
import torch.nn as nn
import torch.nn.functional as F
import pytorch_lightning as pl
from torch.optim.lr_scheduler import ReduceLROnPlateau
from pytorch_lightning.callbacks.early_stopping import EarlyStopping<import_modules>
def clean(text):
text = new_line(text)
 text = re.sub(r'(&amp;|&gt;|&lt;)', " ", text)
 text = re.sub(r"\s+", " ", text)
 text = url(text)
 text = re.sub(r'@\S{0,}', ' USER ', text)
 text = re.sub(r"\s+", " ", text)
 text = re.sub(r'\b(USER )(\1\b)+', r'\1', text)
 text = re.sub(r'([a-zA-Z])\1{1,}', r'\1\1', text)
 text = re.sub(r"htt\S{0,}", " ", text)
 text = re.sub(r"[^a-zA-Z\d\s]", " ", text)
 text = re.sub(r'^\d\S{0,}| \d\S{0,}| \d\S{0,}$', ' NUMBER ', text)
 text = re.sub(r"\s+", " ", text)
 text = re.sub(r'\b(NUMBER )(\1\b)+', r'\1', text)
 text = re.sub(r"[0-9]", " ", text)
 text = text.strip()
text = re.sub(r' via\s{1,}USER$', ' ', text)
 text = re.sub(r"\s+", " ", text)
 text = text.strip()
return text
class PurgedGroupTimeSeriesSplit(_BaseKFold):

 @_deprecate_positional_args
def __init__(self,
n_splits=5,
*,
max_train_group_size=np.inf,
max_test_group_size=np.inf,
group_gap=None,
verbose=False 
):
super().__init__(n_splits, shuffle=False, random_state=None)
 self.max_train_group_size = max_train_group_size
self.group_gap = group_gap
self.max_test_group_size = max_test_group_size
self.verbose = verbose
def split(self, X, y=None, groups=None):

 if groups is None:
raise ValueError(
 "The 'groups' parameter should not be None")
 X, y, groups = indexable(X, y, groups)
 n_samples = _num_samples(X)
 n_splits = self.n_splits
group_gap = self.group_gap
max_test_group_size = self.max_test_group_size
max_train_group_size = self.max_train_group_size
n_folds = n_splits + 1
group_dict = {}
u, ind = np.unique(groups, return_index=True)
 unique_groups = u[np.argsort(ind)]
n_samples = _num_samples(X)
 n_groups = _num_samples(unique_groups)
 for idx in np.arange(n_samples):
if(groups[idx] in group_dict):
group_dict[groups[idx]].append(idx)
 else:
group_dict[groups[idx]] = [idx]
if n_folds > n_groups:
raise ValueError(
("Cannot have number of folds={0} greater than"
" the number of groups={1}" ).format(n_folds,
n_groups))
group_test_size = min(n_groups // n_folds, max_test_group_size)
 group_test_starts = range(n_groups - n_splits * group_test_size,
n_groups, group_test_size)
 for group_test_start in group_test_starts:
train_array = []
test_array = []
group_st = max(0, group_test_start - group_gap - max_train_group_size)
 for train_group_idx in unique_groups[group_st:(group_test_start - group_gap)]:
train_array_tmp = group_dict[train_group_idx]
train_array = np.sort(np.unique(
 np.concatenate(( train_array,
train_array_tmp)) ,
axis=None), axis=None)
 train_end = train_array.size
for test_group_idx in unique_groups[group_test_start:
group_test_start +
group_test_size]:
test_array_tmp = group_dict[test_group_idx]
test_array = np.sort(np.unique(
 np.concatenate(( test_array,
test_array_tmp)) ,
axis=None), axis=None)
 test_array = test_array[group_gap:]
if self.verbose > 0:
pass
yield [int(i)for i in train_array], [int(i)for i in test_array]<prepare_x_and_y>
train.text = train.text.apply(clean)
 test.text = test.text.apply(clean )
train = pd.read_csv('.. /input/jane-street-market-prediction/train.csv')
 train = train.query('date > 85' ).reset_index(drop = True)
 train = train.astype({c: np.float32 for c in train.select_dtypes(include='float64' ).columns})
 train.fillna(train.mean() ,inplace=True)
 train = train.query('weight > 0' ).reset_index(drop = True)
 train['action'] =(( train['resp_1'] > 0)&(train['resp_2'] > 0)&(train['resp_3'] > 0)&(train['resp_4'] > 0)&(train['resp'] > 0)).astype('int')
 features = [c for c in train.columns if 'feature' in c]
resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4']
X = train[features].values
y = np.stack([(train[c] > 0 ).astype('int')for c in resp_cols] ).T
f_mean = np.mean(train[features[1:]].values,axis=0 )<categorify>
max_length_tr = train.text.map(len ).max()
max_length_te = test.text.map(len ).max()
max_length = max(max_length_tr, max_length_te)
 print("At the stage of text processing:")
 print(f"...the size of longest text string in train set is {max_length_tr}")
 print(f"...the size of longest text string in test set is {max_length_te}")
class AE_Dataset:
def __init__(self, dataset, targets):
self.dataset = dataset
self.targets = targets
def __len__(self):
return self.dataset.shape[0]
def __getitem__(self, item):
return {
'x': torch.tensor(self.dataset[item, :], dtype=torch.float),
'y': torch.tensor(self.targets[item], dtype=torch.float)
 }
class AE_DataModule(pl.LightningDataModule):
def __init__(self, data, targets, BATCH_SIZE, fold = None):
super().__init__()
self.BATCH_SIZE = BATCH_SIZE
self.data = data
self.targets = targets
self.fold = fold
def preapre_data(self):
pass
def setup(self, stage=None):
pass
def train_dataloader(self):
self.train_dataset = AE_Dataset(dataset = self.data,targets = self.targets)
 return torch.utils.data.DataLoader(
 self.train_dataset, batch_size=self.BATCH_SIZE)
 def valid_dataloader(self):
return None
def test_dataloader(self):
return None
class LitAutoEncoder(pl.LightningModule):
def __init__(self, input_shape):
super().__init__()
self.encoder = nn.Sequential(
 nn.BatchNorm1d(input_shape),
nn.Linear(input_shape, 64),
nn.ReLU() ,
nn.Linear(64, 32))
self.decoder = nn.Sequential(
 nn.Linear(32, 64),
nn.ReLU() ,
nn.Linear(64, input_shape))
def forward(self, x):
embedding = self.encoder(x)
 return embedding
def training_step(self, batch, batch_idx):
x, y = batch['x'], batch['y']
z = self.encoder(x)
 x_hat = self.decoder(z)
 loss = F.mse_loss(x_hat, x)
 return loss
def configure_optimizers(self):
optimizer = torch.optim.Adam(self.parameters() , lr=1e-3)
 return optimizer<categorify>
def cut(max_len, delta, x):
new_max = max_len - delta
length = len(x)
 if length <= new_max:
return x
else:
return x[:(new_max-length)]
delta = 25
train.text = train.text.map(lambda x: cut(max_length, delta, x))
test.text = test.text.map(lambda x: cut(max_length, delta, x))
new_max_length_tr = train.text.map(len ).max()
new_max_length_te = test.text.map(len ).max()
print("After we cut tails of the longest tweets:")
 print(f"...the size of longest text string in train set is {new_max_length_tr}")
 print(f"...the size of longest text string in test set is {new_max_length_te}" )
train_mode = False
EPOCHS = 10
GPU = 0
BATCH_SIZE = 4096
NUM_FEATURES = X.shape[1]
if train_mode:
early_stop_callback = EarlyStopping(
 monitor='train_loss', min_delta=0.00, patience=3, verbose=True, mode='min')
 DataLoader = AE_DataModule(data=X, targets=y, BATCH_SIZE=BATCH_SIZE)
 trainer = pl.Trainer(gpus=GPU, max_epochs=EPOCHS, weights_summary='full', callbacks=[early_stop_callback])
 AEncoder = LitAutoEncoder(input_shape=NUM_FEATURES)
 trainer.fit(AEncoder, DataLoader)
 torch.save(AEncoder.state_dict() , 'encoder.pkl')
 else:
AEncoder = LitAutoEncoder(input_shape=NUM_FEATURES)
 AEncoder.load_state_dict(torch.load('.. /input/pytorch-jsprediction-freezed-at-v20/encoder.pkl',map_location=torch.device('cpu')) )<categorify>
df_mislabeled = train.groupby(['text'] ).nunique().sort_values(by='target', ascending=False)
 df_mislabeled = df_mislabeled[df_mislabeled['target'] > 1]['target']
index_misl = df_mislabeled.index.tolist()
lenght = len(index_misl)
 print(f"There are {lenght} equivalence classes with mislabelling" )
class MLP_Dataset:
def __init__(self, dataset, targets):
self.dataset = dataset
self.targets = targets
def __len__(self):
return self.dataset.shape[0]
def __getitem__(self, item):
return {
'x': torch.tensor(self.dataset[item, :], dtype=torch.float),
'y': torch.tensor(self.targets[item], dtype=torch.float)
 }
class MLP_DataModule(pl.LightningDataModule):
def __init__(self, data, targets, BATCH_SIZE, fold = None):
super().__init__()
self.BATCH_SIZE = BATCH_SIZE
self.data = data
self.targets = targets
self.fold = fold
def preapre_data(self):
pass
def setup(self, stage=None):
pass
def train_dataloader(self):
dataset = MLP_Dataset(dataset = self.data, targets = self.targets)
 train_loader = torch.utils.data.DataLoader(dataset, batch_size=self.BATCH_SIZE)
 return train_loader
def valid_dataloader(self):
dataset = MLP_Dataset(dataset = self.data, targets = self.targets)
 valid_loader = torch.utils.data.DataLoader(dataset,batch_size=self.BATCH_SIZE)
 return valid_loader
def test_dataloader(self):
return None 
<categorify>
index_misl
class MLP(nn.Module):
def __init__(self, config, AEncoder):
super(MLP, self ).__init__()
self.criterion = nn.BCELoss()
self.AEncoder = AEncoder
self.lr = config["lr"]
input_shape = 260
drop_out = [config[key] for key, v in config.items() if 'dropout' in key]
hidden_size = [config[key] for key,v in config.items() if 'layer' in key]
layers = []
for i in range(len(hidden_size)) :
out_shape = hidden_size[i]
layers.append(nn.Dropout(drop_out[i]))
layers.append(nn.Linear(input_shape, out_shape))
layers.append(nn.BatchNorm1d(out_shape))
layers.append(nn.SiLU())
 input_shape = out_shape
layers.append(nn.Dropout(drop_out[-1]))
layers.append(nn.Linear(input_shape, 5))
layers.append(nn.Sigmoid())
 self.model = torch.nn.Sequential(*layers)
 def encoder_decoder(self, x):
self.AEncoder.eval()
encoded = self.AEncoder(x)
 decoded = self.AEncoder.decoder(encoded)
 return decoded
def forward(self, x):
decoded = self.encoder_decoder(x)
 x = torch.cat(( x, decoded), dim=1)
 x = self.model(x)
 return x<train_model>
train_nu_target = train[train['text'].isin(index_misl)].sort_values(by = 'text')
 train_nu_target[0:309]
def Train(params, num_epochs=30, batch_size=4096, patience=3):
loss_fn = nn.BCELoss().to(device)
 config = {**params}
Val_Loss = 0
N_Samples = 0
for fold,(train_idx, valid_idx)in enumerate(splits[2:]):
print('Fold : {}'.format(fold))
tr_x, tr_y = X[train_idx], y[train_idx]
train_loader = MLP_DataModule(data=tr_x, targets=tr_y, BATCH_SIZE=batch_size ).train_dataloader()
val_x, val_y = X[valid_idx], y[valid_idx]
val_loader = MLP_DataModule(data=val_x, targets=val_y, BATCH_SIZE=batch_size ).valid_dataloader()
model = MLP(config, AEncoder ).to(device)
 optimizer = torch.optim.Adam(model.parameters() , lr=config['lr'])
 the_last_loss = 100
trigger_times=0
for epoch in range(num_epochs):
running_loss = 0.0
model.train()
for batch in train_loader:
inputs, labels = batch['x'].to(device), batch['y'].to(device)
 optimizer.zero_grad()
with torch.set_grad_enabled(True):
outputs = model(inputs)
 loss = loss_fn(outputs, labels)
 loss.backward()
optimizer.step()
running_loss += loss.item() * inputs.size(0)
 epoch_loss = running_loss / len(train_loader.dataset)
 print(' Training: Epoch({})- Loss: {:.4f}'.format(epoch, epoch_loss))
model.eval()
vrunning_loss = 0.0
num_samples = 0
for batch in val_loader:
data, labels = batch['x'].to(device), batch['y'].to(device)
 data = data.to(device)
 labels = labels.to(device)
 optimizer.zero_grad()
with torch.no_grad() :
outputs = model(data)
 loss = loss_fn(outputs, labels)
 vrunning_loss += loss.item() * data.size(0)
 num_samples += labels.size(0)
 vepoch_loss = vrunning_loss/num_samples
print(' Validation({})- Loss: {:.4f}'.format(epoch, vepoch_loss))
if vepoch_loss > the_last_loss:
trigger_times += 1
if trigger_times >= patience:
print('Meet Early stopping!')
 break
else:
trigger_times = 0
the_last_loss = vepoch_loss
torch.save(model.state_dict() , f'model_{fold}.pkl')
 Val_Loss += vepoch_loss * num_samples
N_Samples += num_samples
torch.save(model.state_dict() , f'model_{fold}.pkl')
 return Val_Loss/N_Samples<split>
num_records = train_nu_target.shape[0]
length = len(index_misl)
 print(f"There are {num_records} records in train set which are split in {lenght} equivalence classes(with mislabelling)" )
FOLDS=5
gkf = PurgedGroupTimeSeriesSplit(n_splits = FOLDS, group_gap=20)
 splits = list(gkf.split(y, groups=train['date'].values))<set_options>
copy = train_nu_target.copy()
classes = copy.groupby('text' ).agg({'keyword':np.size, 'target':np.mean} ).rename(columns={'keyword':'Number of records in train set', 'target':'Target mean'})
 classes.sort_values('Number of records in train set', ascending=False ).head(100 )
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 device<choose_model_class>
majority_df = train_nu_target.groupby(['text'])['target'].mean()
len(majority_df.index )
config = {
"layer_1_size": 260,
"layer_2_size": 730,
"layer_3_size": 800,
"layer_4_size": 800,
"layer_5_size": 260,
"dropout_1":0.1,
"dropout_2":0.7,
"dropout_3":0.8,
"dropout_4":0.45,
"dropout_5":0.60,
"dropout_output":0.5,
"lr": 1e-3
}
train_mode = False
if train_mode:
model_loss = Train(config)
 print(model_loss)
 else:
models = []
for i in range(len(splits)) :
mlp = MLP(config, AEncoder)
 mlp.load_state_dict(torch.load(f'.. /input/pytorch-jsprediction-freezed-at-v20/model_{i}.pkl',map_location=torch.device('cpu')))
 models.append(mlp )<train_model>
def relabel(r, majority_index):
ind = ''
if r['text'] in majority_index:
ind = r['text']
if majority_df[ind] < 0.5:
return 0
else:
return 1
else:
return r['target']
def fit(model, x):
x = torch.tensor(x, dtype=torch.float)
 model.eval()
pred = model(x)
 return pred.detach().numpy()<train_model>
train['target'] = train.apply(lambda row: relabel(row, majority_df.index), axis = 1 )
mlp.eval()
fit(mlp, X[0:1,:] )<split>
new_df = train[train['text'].isin(majority_df.index)].sort_values(['target', 'text'], ascending = [False, True])
 new_df.head(310 )
env = janestreet.make_env()
env_iter = env.iter_test()<feature_engineering>
df_mislabeled = train.groupby(['text'] ).nunique().sort_values(by='target', ascending=False)
 df_mislabeled = df_mislabeled[df_mislabeled['target'] > 1]['target']
index_misl = df_mislabeled.index.tolist()
len(index_misl )
opt_th = 0.5
if not train_mode:
for(test_df, pred_df)in env_iter:
if test_df['weight'].item() > 0:
test_df = test_df.loc[:, features].values
if np.isnan(test_df[:, 1:].sum()):
test_df[:, 1:] = np.nan_to_num(test_df[:, 1:])+ np.isnan(test_df[:, 1:])* f_mean
pred_vector = np.mean([fit(model, test_df)for model in models],axis=0)
 pred = np.mean(pred_vector)
 pred_df.action =(pred_vector > opt_th ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )<prepare_x_and_y>
use = hub.load("https://tfhub.dev/google/universal-sentence-encoder-multilingual-large/3" )
def make_df(train_path, test_path, img_size):
train_ids = next(os.walk(train_path)) [1]
test_ids = next(os.walk(test_path)) [1]
X_train = np.zeros(( len(train_ids), img_size, img_size, 3), dtype=np.uint8)
 Y_train = np.zeros(( len(train_ids), img_size, img_size, 1), dtype=np.bool)
 for i, id_ in enumerate(train_ids):
path = train_path + id_
img = cv2.imread(path + '/images/' + id_ + '.png')
 img = cv2.resize(img,(img_size, img_size))
X_train[i] = img
mask = np.zeros(( img_size, img_size, 1), dtype=np.bool)
 for mask_file in next(os.walk(path + '/masks/')) [2]:
mask_ = cv2.imread(path + '/masks/' + mask_file, 0)
 mask_ = cv2.resize(mask_,(img_size, img_size))
mask_ = mask_[:, :, np.newaxis]
mask = np.maximum(mask, mask_)
 Y_train[i] = mask
X_test = np.zeros(( len(test_ids), img_size, img_size, 3), dtype=np.uint8)
 sizes_test = []
for i, id_ in enumerate(test_ids):
path = test_path + id_
img = cv2.imread(path + '/images/' + id_ + '.png')
 sizes_test.append([img.shape[0], img.shape[1]])
 img = cv2.resize(img,(img_size, img_size))
X_test[i] = img
return X_train, Y_train, X_test, sizes_test<choose_model_class>
X_train = []
for r in tqdm(train.text.values):
emb = use(r)
 review_emb = tf.reshape(emb, [-1] ).numpy()
X_train.append(review_emb)
 X_train = np.array(X_train)
 y_train = train.target.values
X_test = []
for r in tqdm(test.text.values):
emb = use(r)
 review_emb = tf.reshape(emb, [-1] ).numpy()
X_test.append(review_emb)
 X_test = np.array(X_test )
def Unet(img_size):
inputs = Input(( img_size, img_size, 3))
s = Lambda(lambda x: x / 255 )(inputs)
 c1 = Conv2D(16,(3, 3), activation='selu', kernel_initializer='he_normal', padding='same' )(s)
 c1 = Dropout(0.1 )(c1)
 c1 = Conv2D(16,(3, 3), activation='selu', kernel_initializer='he_normal', padding='same' )(c1)
 p1 = MaxPooling2D(( 2, 2))(c1)
 print('c1')
 c2 = Conv2D(32,(3, 3), activation='selu', kernel_initializer='he_normal', padding='same' )(p1)
 c2 = Dropout(0.1 )(c2)
 c2 = Conv2D(32,(3, 3), activation='selu', kernel_initializer='he_normal', padding='same' )(c2)
 p2 = MaxPooling2D(( 2, 2))(c2)
 print('c2')
 c3 = Conv2D(64,(3, 3), activation='selu', kernel_initializer='he_normal', padding='same' )(p2)
 c3 = Dropout(0.2 )(c3)
 c3 = Conv2D(64,(3, 3), activation='selu', kernel_initializer='he_normal', padding='same' )(c3)
 p3 = MaxPooling2D(( 2, 2))(c3)
 print('c3')
 c4 = Conv2D(128,(3, 3), activation='selu', kernel_initializer='he_normal', padding='same' )(p3)
 c4 = Dropout(0.2 )(c4)
 c4 = Conv2D(128,(3, 3), activation='selu', kernel_initializer='he_normal', padding='same' )(c4)
 p4 = MaxPooling2D(pool_size=(2, 2))(c4)
 print('c4')
 c5 = Conv2D(256,(3, 3), activation='selu', kernel_initializer='he_normal', padding='same' )(p4)
 c5 = Dropout(0.3 )(c5)
 c5 = Conv2D(256,(3, 3), activation='selu', kernel_initializer='he_normal', padding='same' )(c5)
 print('c5')
 u6 = Conv2DTranspose(128,(2, 2), strides=(2, 2), padding='same' )(c5)
 u6 = concatenate([u6, c4])
 c6 = Conv2D(128,(3, 3), activation='selu', kernel_initializer='he_normal', padding='same' )(u6)
 c6 = Dropout(0.2 )(c6)
 c6 = Conv2D(128,(3, 3), activation='selu', kernel_initializer='he_normal', padding='same' )(c6)
 print('c6')
 u7 = Conv2DTranspose(64,(2, 2), strides=(2, 2), padding='same' )(c6)
 u7 = concatenate([u7, c3])
 c7 = Conv2D(64,(3, 3), activation='selu', kernel_initializer='he_normal', padding='same' )(u7)
 c7 = Dropout(0.2 )(c7)
 c7 = Conv2D(64,(3, 3), activation='selu', kernel_initializer='he_normal', padding='same' )(c7)
 print('c7')
 u8 = Conv2DTranspose(32,(2, 2), strides=(2, 2), padding='same' )(c7)
 u8 = concatenate([u8, c2])
 c8 = Conv2D(32,(3, 3), activation='selu', kernel_initializer='he_normal', padding='same' )(u8)
 c8 = Dropout(0.1 )(c8)
 c8 = Conv2D(32,(3, 3), activation='selu', kernel_initializer='he_normal', padding='same' )(c8)
 print('c8')
 u9 = Conv2DTranspose(16,(2, 2), strides=(2, 2), padding='same' )(c8)
 u9 = concatenate([u9, c1], axis=3)
 c9 = Conv2D(16,(3, 3), activation='selu', kernel_initializer='he_normal', padding='same' )(u9)
 c9 = Dropout(0.1 )(c9)
 c9 = Conv2D(16,(3, 3), activation='selu', kernel_initializer='he_normal', padding='same' )(c9)
 print('c9')
 outputs = Conv2D(1,(1, 1), activation='sigmoid' )(c9)
 model = Model(inputs=[inputs], outputs=[outputs])
 return model<compute_test_metric>
train_arrays, test_arrays, train_labels, test_labels = train_test_split(X_train,
y_train,
random_state =42,
test_size=0.20 )
def mean_iou(y_true, y_pred):
prec = []
for t in np.arange(0.5, 1.0, 0.05):
y_pred_ = tf.to_int32(y_pred > t)
 score, up_opt = tf.metrics.mean_iou(y_true, y_pred_, 2, y_true)
 K.get_session().run(tf.local_variables_initializer())
 with tf.control_dependencies([up_opt]):
score = tf.identity(score)
 prec.append(score)
 return K.mean(K.stack(prec), axis=0)
 def dice_coef(y_true, y_pred):
smooth = 1.
 y_true_f = K.flatten(y_true)
 y_pred_f = K.flatten(y_pred)
 intersection = K.sum(y_true_f * y_pred_f)
 return(2.* intersection + smooth)/(K.sum(y_true_f)+ K.sum(y_pred_f)+ smooth)
 def bce_dice_loss(y_true, y_pred):
return 0.5 * binary_crossentropy(y_true, y_pred)- dice_coef(y_true, y_pred )<categorify>
def svc_param_selection(X, y, nfolds):
Cs = [1.40]
gammas = [2.25]
param_grid = {'C': Cs, 'gamma' : gammas}
grid_search = GridSearchCV(SVC(kernel='rbf'), param_grid, cv=nfolds, n_jobs=8)
 grid_search.fit(X, y)
 grid_search.best_params_
return grid_search
model = svc_param_selection(train_arrays,train_labels, 10 )
def rle_encoding(x):
dots = np.where(x.T.flatten() == 1)[0]
run_lengths = []
prev = -2
for b in dots:
if(b>prev+1): run_lengths.extend(( b + 1, 0))
run_lengths[-1] += 1
prev = b
return run_lengths
def prob_to_rles(x, cutoff=0.5):
lab_img = label(x > cutoff)
 for i in range(1, lab_img.max() + 1):
yield rle_encoding(lab_img == i )<split>
model.best_params_
if __name__ == "__main__":
img_size = 256
batch_size = 32
train_path = '.. /input/stage1_train/'
test_path = '.. /input/stage2_test_final/'
X_train, Y_train, X_test, sizes_test = make_df(train_path, test_path, img_size)
 xtr, xval, ytr, yval = train_test_split(X_train, Y_train, test_size=0.1, random_state=7)
 train_generator, val_generator = generator(xtr, xval, ytr, yval, batch_size)
 print('imported')
 print(xtr)
 model = Unet(img_size)
 model.compile(optimizer='adam', loss=bce_dice_loss, metrics=[mean_iou])
 print('going to model fit generator now')
 model.fit_generator(train_generator, steps_per_epoch=25, epochs=20,
validation_data=val_generator, validation_steps=len(xval)/batch_size)
 preds_test = model.predict(X_test, verbose=1)
 preds_test_upsampled = []
for i in range(len(preds_test)) :
preds_test_upsampled.append(cv2.resize(preds_test[i], 
(sizes_test[i][1], sizes_test[i][0])))
 test_ids = next(os.walk(test_path)) [1]
new_test_ids = []
rles = []
for n, id_ in enumerate(test_ids):
rle = list(prob_to_rles(preds_test_upsampled[n]))
rles.extend(rle)
 new_test_ids.extend([id_] * len(rle))
sub = pd.DataFrame()
sub['ImageId'] = new_test_ids
sub['EncodedPixels'] = pd.Series(rles ).apply(lambda x: ' '.join(str(y)for y in x))
sub.to_csv('submission_relu.csv', index=False )<load_from_csv>
pred = model.predict(test_arrays )
types = {'Semana':np.uint8, 'Agencia_ID':np.uint16, 'Canal_ID':np.uint8,
'Ruta_SAK':np.uint16, 'Cliente_ID':np.uint32, 'Producto_ID':np.uint16,
'Demanda_uni_equil':np.uint32}
train=pd.read_csv('.. /input/train.csv',usecols=types.keys() ,dtype=types,nrows=1000)
 print(train.dtype)
 print(train.info(memery_usage=True)) 
<load_from_csv>
cm = confusion_matrix(test_labels,pred)
 cm
types = {'Semana':np.uint8, 'Agencia_ID':np.uint16, 'Canal_ID':np.uint8,
'Ruta_SAK':np.uint16, 'Cliente_ID':np.uint32, 'Producto_ID':np.uint16,
'Demanda_uni_equil':np.uint32}
train=pd.read_csv('.. /input/train.csv',usecols=types.keys() ,dtypes=types,nrows=1000)
 print(train.dtype)
 print(train.info(memery_usage=True))<load_from_csv>
accuracy = accuracy_score(test_labels,pred)
 accuracy
print(check_output(['ls','.'] ).decode('utf8'))
submission=pd.read_csv('.. /input/sample_submission.csv')
 print(submission.shape)
 print(submission.columns)
 print(submission.head(20))<load_from_csv>
test_pred = model.predict(X_test)
 submission['target'] = test_pred.round().astype(int)
def evalerror(preds, dtrain):
labels = dtrain.get_label()
assert len(preds)== len(labels)
 labels = labels.tolist()
preds = preds.tolist()
terms_to_sum = [(math.log(labels[i] + 1)- math.log(max(0,preds[i])+ 1)) ** 2.0 for i,pred in enumerate(labels)]
return 'error',(sum(terms_to_sum)*(1.0/len(preds)))** 0.5
nrows=10000
train=pd.read_csv('.. /input/train.csv',nrows=nrows)
 test=pd.read_csv('.. /input/test.csv',nrows=nrows)
 print(train.columns)
 print(test.columns)
 print(test.columns.values)
 ids=test['id']
test=test.drop(['id'],axis=1)
 y=train['Demanda_uni_equil']
X=train[test.columns.values]
X_train,X_test,y_train,y_test=train_test_split(X,y,test_size=0.2,random_state=1000)
 params = {}
params['objective'] = "reg:linear"
params['eta'] = 0.05
params['max_depth'] = 5
params['subsample'] = 0.8
params['colsample_bytree'] = 0.6
params['silent'] = True
print('')
 test_preds = np.zeros(test.shape[0])
 xg_train = xgb.DMatrix(X_train, label=y_train)
 xg_test = xgb.DMatrix(X_test)
 watchlist = [(xg_train, 'train')]
num_rounds = 100
xgclassifier = xgb.train(params, xg_train, num_rounds, watchlist, feval = evalerror, early_stopping_rounds= 20, verbose_eval = 10)
 preds = xgclassifier.predict(xg_test, ntree_limit=xgclassifier.best_iteration)
 print('RMSLE Score:', rmsle(y_test, preds))
fxg_test = xgb.DMatrix(test)
 fold_preds = np.around(xgclassifier.predict(fxg_test, ntree_limit=xgclassifier.best_iteration), decimals = 1)
 test_preds += fold_preds
submission = pd.DataFrame({'id':ids, 'Demanda_uni_equil': test_preds})
 submission.to_csv('submission.csv', index=False )<prepare_x_and_y>
count = 2
prob_disaster = 0.9
keyword_list_disaster = list(ag[(ag['Count']>count)&(ag['Disaster Probability']>=prob_disaster)].index)
 keyword_list_disaster
nrows=100000
train=pd.read_csv('.. /input/train.csv',nrows=nrows)
 print(train.shape)
 print(train.columns)
 data=train.copy()
data['target']=data['Demanda_uni_equil']
data.drop(['Demanda_uni_equil'],axis=1,inplace=True)
 nCliente_ID = pd.DataFrame(pd.groupby(data,['Cliente_ID','Semana'])['target'].count())
 print(nCliente_ID.shape)
 print(nCliente_ID.columns)
 print(nCliente_ID.head(2))
nCliente_ID = nCliente_ID.reset_index()
print(nCliente_ID.shape)
 print(nCliente_ID.columns)
 print(nCliente_ID.head(2))
nCliente_ID.rename(columns={'target': 'nCliente_ID'}, inplace=True)
 print(nCliente_ID.shape)
 print(nCliente_ID.columns)
 print(nCliente_ID.head(2))
nCliente_ID = pd.DataFrame(pd.groupby(nCliente_ID,['Cliente_ID'])['nCliente_ID'].mean())
 print(nCliente_ID.shape)
 print(nCliente_ID.columns)
 print(nCliente_ID.head(2))
nCliente_ID = nCliente_ID.reset_index()
print(nCliente_ID.shape)
 print(nCliente_ID.columns)
 print(nCliente_ID.head(2))
data = pd.merge(data, nCliente_ID,
how='left',
left_on=['Cliente_ID'],
right_on=['Cliente_ID'],
left_index=False, right_index=False, sort=True,
suffixes=('_x', '_y'), copy=False)
 print(data.columns)
 print(data.head(50))
del nCliente_ID
gc.collect()
print('merge completo nCliente_ID')
 print(data.shape[0] )<import_modules>
ids_disaster = test['id'][test.keyword.isin(keyword_list_disaster)].values
submission['target'][submission['id'].isin(ids_disaster)] = 1
import pandas as pd
import numpy as np
import tensorflow as tf
import tensorflow_addons as tfa
from tensorflow.keras.models import Sequential
import tensorflow.keras.layers as L
from tensorflow.keras.callbacks import ReduceLROnPlateau, ModelCheckpoint, EarlyStopping
from tensorflow.keras.models import load_model
from sklearn.metrics import log_loss
from sklearn.decomposition import PCA
import random<load_from_csv>
submission.to_csv("submission.csv", index=False)
 submission.head(10 )
train_data = pd.read_csv('.. /input/lishmy/train_data_exc.csv')
 test_data = pd.read_csv('.. /input/lish-moa/test_features.csv')
 train_targets_scored = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 train_targets_nonscored = pd.read_csv('.. /input/lish-moa/train_targets_nonscored.csv' )<define_variables>
index_hot = [2700, 2695, 2713, 2698, 2692, 2686, 2685,2684]
train_example = train_copy.copy()
train_example.loc[index_hot,:]
genes = [col for col in train_data if col[0:2]=='g-']
cells = [col for col in train_data if col[0:2]=='c-']<load_from_csv>
index_hot = [6842,6821,6824,6828,6831,6843]
train_example = train_copy.copy()
train_example.loc[index_hot,:]
temp = pd.read_csv('.. /input/lish-moa/train_features.csv')
 features = temp.drop(['sig_id'],axis=1 ).columns
temp = pd.read_csv('.. /input/lish-moa/train_targets_scored.csv')
 targets = temp.drop('sig_id',axis=1 ).columns
del temp<import_modules>
index_hot = [6113,6103,6097,6094,6091,6119,6123]
train_example = train_copy.copy()
train_example.loc[index_hot,:]
<categorify><EOS> 
index_hot = [3670,3674,3688,3667,3696]
train_example = train_copy.copy()
train_example.loc[index_hot,:]