Kaggle competition: Jane Street Market Prediction
y_pred = label_encoder.inverse_transform(classifier.predict(x_test))
test['cuisine'] = y_pred
test[['id', 'cuisine']].to_csv('submission.csv', index=False)
 test[['id', 'cuisine']].head()<load_pretrained>
import torch
import torch.nn as nn
import torch.nn.functional as F
import pytorch_lightning as pl
from torch.optim.lr_scheduler import ReduceLROnPlateau
from pytorch_lightning.callbacks.early_stopping import EarlyStopping
nltk.download('wordnet' )<string_transform>
class PurgedGroupTimeSeriesSplit(_BaseKFold):

 @_deprecate_positional_args
def __init__(self,
n_splits=5,
*,
max_train_group_size=np.inf,
max_test_group_size=np.inf,
group_gap=None,
verbose=False 
):
super().__init__(n_splits, shuffle=False, random_state=None)
 self.max_train_group_size = max_train_group_size
self.group_gap = group_gap
self.max_test_group_size = max_test_group_size
self.verbose = verbose
def split(self, X, y=None, groups=None):

 if groups is None:
raise ValueError(
 "The 'groups' parameter should not be None")
 X, y, groups = indexable(X, y, groups)
 n_samples = _num_samples(X)
 n_splits = self.n_splits
group_gap = self.group_gap
max_test_group_size = self.max_test_group_size
max_train_group_size = self.max_train_group_size
n_folds = n_splits + 1
group_dict = {}
u, ind = np.unique(groups, return_index=True)
 unique_groups = u[np.argsort(ind)]
n_samples = _num_samples(X)
 n_groups = _num_samples(unique_groups)
 for idx in np.arange(n_samples):
if(groups[idx] in group_dict):
group_dict[groups[idx]].append(idx)
 else:
group_dict[groups[idx]] = [idx]
if n_folds > n_groups:
raise ValueError(
("Cannot have number of folds={0} greater than"
" the number of groups={1}" ).format(n_folds,
n_groups))
group_test_size = min(n_groups // n_folds, max_test_group_size)
 group_test_starts = range(n_groups - n_splits * group_test_size,
n_groups, group_test_size)
 for group_test_start in group_test_starts:
train_array = []
test_array = []
group_st = max(0, group_test_start - group_gap - max_train_group_size)
 for train_group_idx in unique_groups[group_st:(group_test_start - group_gap)]:
train_array_tmp = group_dict[train_group_idx]
train_array = np.sort(np.unique(
 np.concatenate(( train_array,
train_array_tmp)) ,
axis=None), axis=None)
 train_end = train_array.size
for test_group_idx in unique_groups[group_test_start:
group_test_start +
group_test_size]:
test_array_tmp = group_dict[test_group_idx]
test_array = np.sort(np.unique(
 np.concatenate(( test_array,
test_array_tmp)) ,
axis=None), axis=None)
 test_array = test_array[group_gap:]
if self.verbose > 0:
pass
yield [int(i)for i in train_array], [int(i)for i in test_array]
def porter_stem_ingredients(ingredient):
porter_stemmer = PorterStemmer()
tokens = ingredient.split()
stemmed_tokens = [porter_stemmer.stem(token)for token in tokens]
ingredient = ' '.join(stemmed_tokens)
 return ingredient<string_transform>
train = pd.read_csv('.. /input/jane-street-market-prediction/train.csv')
 train = train.query('date > 85' ).reset_index(drop = True)
 train = train.astype({c: np.float32 for c in train.select_dtypes(include='float64' ).columns})
 train.fillna(train.mean() ,inplace=True)
 train = train.query('weight > 0' ).reset_index(drop = True)
 train['action'] =(( train['resp_1'] > 0)&(train['resp_2'] > 0)&(train['resp_3'] > 0)&(train['resp_4'] > 0)&(train['resp'] > 0)).astype('int')
 features = [c for c in train.columns if 'feature' in c]
resp_cols = ['resp_1', 'resp_2', 'resp_3', 'resp', 'resp_4']
X = train[features].values
y = np.stack([(train[c] > 0 ).astype('int')for c in resp_cols] ).T
f_mean = np.mean(train[features[1:]].values,axis=0 )
def snowball_stem_ingredients(ingredient):
snowball_stemmer = SnowballStemmer('english')
 tokens = ingredient.split()
stemmed_tokens = [snowball_stemmer.stem(token)for token in tokens]
ingredient = ' '.join(stemmed_tokens)
 return ingredient<string_transform>
class AE_Dataset:
def __init__(self, dataset, targets):
self.dataset = dataset
self.targets = targets
def __len__(self):
return self.dataset.shape[0]
def __getitem__(self, item):
return {
'x': torch.tensor(self.dataset[item, :], dtype=torch.float),
'y': torch.tensor(self.targets[item], dtype=torch.float)
 }
class AE_DataModule(pl.LightningDataModule):
def __init__(self, data, targets, BATCH_SIZE, fold = None):
super().__init__()
self.BATCH_SIZE = BATCH_SIZE
self.data = data
self.targets = targets
self.fold = fold
def preapre_data(self):
pass
def setup(self, stage=None):
pass
def train_dataloader(self):
self.train_dataset = AE_Dataset(dataset = self.data,targets = self.targets)
 return torch.utils.data.DataLoader(
 self.train_dataset, batch_size=self.BATCH_SIZE)
 def valid_dataloader(self):
return None
def test_dataloader(self):
return None
class LitAutoEncoder(pl.LightningModule):
def __init__(self, input_shape):
super().__init__()
self.encoder = nn.Sequential(
 nn.BatchNorm1d(input_shape),
nn.Linear(input_shape, 64),
nn.ReLU() ,
nn.Linear(64, 32))
self.decoder = nn.Sequential(
 nn.Linear(32, 64),
nn.ReLU() ,
nn.Linear(64, input_shape))
def forward(self, x):
embedding = self.encoder(x)
 return embedding
def training_step(self, batch, batch_idx):
x, y = batch['x'], batch['y']
z = self.encoder(x)
 x_hat = self.decoder(z)
 loss = F.mse_loss(x_hat, x)
 return loss
def configure_optimizers(self):
optimizer = torch.optim.Adam(self.parameters() , lr=1e-3)
 return optimizer
def wordnet_lemmatizer_ingredients(ingredient):
lemmatizer = WordNetLemmatizer()
tokens = ingredient.split()
lemmatized_tokens = [lemmatizer.lemmatize(token)for token in tokens]
ingredient = ' '.join(lemmatized_tokens)
 return ingredient<categorify>
train_mode = False
EPOCHS = 10
GPU = 0
BATCH_SIZE = 4096
NUM_FEATURES = X.shape[1]
if train_mode:
early_stop_callback = EarlyStopping(
 monitor='train_loss', min_delta=0.00, patience=3, verbose=True, mode='min')
 DataLoader = AE_DataModule(data=X, targets=y, BATCH_SIZE=BATCH_SIZE)
 trainer = pl.Trainer(gpus=GPU, max_epochs=EPOCHS, weights_summary='full', callbacks=[early_stop_callback])
 AEncoder = LitAutoEncoder(input_shape=NUM_FEATURES)
 trainer.fit(AEncoder, DataLoader)
 torch.save(AEncoder.state_dict() , 'encoder.pkl')
 else:
AEncoder = LitAutoEncoder(input_shape=NUM_FEATURES)
 AEncoder.load_state_dict(torch.load('.. /input/pytorch-jsprediction-freezed-at-v20/encoder.pkl',map_location=torch.device('cpu')) )
def main_preprocessing(ingredient):
ingredient = char_preprocess_ingredients(ingredient)
 ingredient = wordnet_lemmatizer_ingredients(ingredient)
 ingredient = snowball_stem_ingredients(ingredient)
 ingredient = porter_stem_ingredients(ingredient)
 return ingredient<feature_engineering>
class MLP_Dataset:
def __init__(self, dataset, targets):
self.dataset = dataset
self.targets = targets
def __len__(self):
return self.dataset.shape[0]
def __getitem__(self, item):
return {
'x': torch.tensor(self.dataset[item, :], dtype=torch.float),
'y': torch.tensor(self.targets[item], dtype=torch.float)
 }
class MLP_DataModule(pl.LightningDataModule):
def __init__(self, data, targets, BATCH_SIZE, fold = None):
super().__init__()
self.BATCH_SIZE = BATCH_SIZE
self.data = data
self.targets = targets
self.fold = fold
def preapre_data(self):
pass
def setup(self, stage=None):
pass
def train_dataloader(self):
dataset = MLP_Dataset(dataset = self.data, targets = self.targets)
 train_loader = torch.utils.data.DataLoader(dataset, batch_size=self.BATCH_SIZE)
 return train_loader
def valid_dataloader(self):
dataset = MLP_Dataset(dataset = self.data, targets = self.targets)
 valid_loader = torch.utils.data.DataLoader(dataset,batch_size=self.BATCH_SIZE)
 return valid_loader
def test_dataloader(self):
return None
warnings.filterwarnings('ignore')
 df_train = pd.read_json('.. /input/train.json')
 df_train['ingredients'] = df_train['ingredients'].apply(lambda x : [main_preprocessing(y)for y in x])
 df_train['all_ingredients'] = df_train['ingredients'].map(";".join )<normalization>
class MLP(nn.Module):
def __init__(self, config, AEncoder):
super(MLP, self ).__init__()
self.criterion = nn.BCELoss()
self.AEncoder = AEncoder
self.lr = config["lr"]
input_shape = 260
drop_out = [config[key] for key, v in config.items() if 'dropout' in key]
hidden_size = [config[key] for key,v in config.items() if 'layer' in key]
layers = []
for i in range(len(hidden_size)) :
out_shape = hidden_size[i]
layers.append(nn.Dropout(drop_out[i]))
layers.append(nn.Linear(input_shape, out_shape))
layers.append(nn.BatchNorm1d(out_shape))
layers.append(nn.SiLU())
 input_shape = out_shape
layers.append(nn.Dropout(drop_out[-1]))
layers.append(nn.Linear(input_shape, 5))
layers.append(nn.Sigmoid())
 self.model = torch.nn.Sequential(*layers)
 def encoder_decoder(self, x):
self.AEncoder.eval()
encoded = self.AEncoder(x)
 decoded = self.AEncoder.decoder(encoded)
 return decoded
def forward(self, x):
decoded = self.encoder_decoder(x)
 x = torch.cat(( x, decoded), dim=1)
 x = self.model(x)
 return x
vectorizer = CountVectorizer(analyzer = "word",
ngram_range =(1,1),
binary = True,
tokenizer = None,
preprocessor = None,
stop_words = None,
max_df = 0.99)
 X = vectorizer.fit_transform(df_train['all_ingredients'].values )<categorify>
def Train(params, num_epochs=30, batch_size=4096, patience=3):
loss_fn = nn.BCELoss().to(device)
 config = {**params}
Val_Loss = 0
N_Samples = 0
for fold,(train_idx, valid_idx)in enumerate(splits[2:]):
print('Fold : {}'.format(fold))
tr_x, tr_y = X[train_idx], y[train_idx]
train_loader = MLP_DataModule(data=tr_x, targets=tr_y, BATCH_SIZE=batch_size ).train_dataloader()
val_x, val_y = X[valid_idx], y[valid_idx]
val_loader = MLP_DataModule(data=val_x, targets=val_y, BATCH_SIZE=batch_size ).valid_dataloader()
model = MLP(config, AEncoder ).to(device)
 optimizer = torch.optim.Adam(model.parameters() , lr=config['lr'])
 the_last_loss = 100
trigger_times=0
for epoch in range(num_epochs):
running_loss = 0.0
model.train()
for batch in train_loader:
inputs, labels = batch['x'].to(device), batch['y'].to(device)
 optimizer.zero_grad()
with torch.set_grad_enabled(True):
outputs = model(inputs)
 loss = loss_fn(outputs, labels)
 loss.backward()
optimizer.step()
running_loss += loss.item() * inputs.size(0)
 epoch_loss = running_loss / len(train_loader.dataset)
 print(' Training: Epoch({})- Loss: {:.4f}'.format(epoch, epoch_loss))
model.eval()
vrunning_loss = 0.0
num_samples = 0
for batch in val_loader:
data, labels = batch['x'].to(device), batch['y'].to(device)
 data = data.to(device)
 labels = labels.to(device)
 optimizer.zero_grad()
with torch.no_grad() :
outputs = model(data)
 loss = loss_fn(outputs, labels)
 vrunning_loss += loss.item() * data.size(0)
 num_samples += labels.size(0)
 vepoch_loss = vrunning_loss/num_samples
print(' Validation({})- Loss: {:.4f}'.format(epoch, vepoch_loss))
if vepoch_loss > the_last_loss:
trigger_times += 1
if trigger_times >= patience:
print('Meet Early stopping!')
 break
else:
trigger_times = 0
the_last_loss = vepoch_loss
torch.save(model.state_dict() , f'model_{fold}.pkl')
 Val_Loss += vepoch_loss * num_samples
N_Samples += num_samples
torch.save(model.state_dict() , f'model_{fold}.pkl')
 return Val_Loss/N_Samples
enc = LabelEncoder()
y = enc.fit_transform(df_train.cuisine )<split>
FOLDS=5
gkf = PurgedGroupTimeSeriesSplit(n_splits = FOLDS, group_gap=20)
 splits = list(gkf.split(y, groups=train['date'].values))
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2 )<train_model>
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 device
logistic = LogisticRegression(C=1, class_weight=None, dual=False, fit_intercept=True,
intercept_scaling=1, max_iter=100, multi_class='warn',
n_jobs=None, penalty='l2', random_state=1, solver='warn',
tol=0.1, verbose=0, warm_start=False)
 logistic.fit(X_train, y_train)
 print('Accuracy: %.5f' % logistic.score(X_test, y_test))<train_model>
config = {
"layer_1_size": 260,
"layer_2_size": 730,
"layer_3_size": 800,
"layer_4_size": 800,
"layer_5_size": 260,
"dropout_1":0.1,
"dropout_2":0.7,
"dropout_3":0.8,
"dropout_4":0.45,
"dropout_5":0.60,
"dropout_output":0.5,
"lr": 1e-3
}
train_mode = False
if train_mode:
model_loss = Train(config)
 print(model_loss)
 else:
models = []
for i in range(len(splits)) :
mlp = MLP(config, AEncoder)
 mlp.load_state_dict(torch.load(f'.. /input/pytorch-jsprediction-freezed-at-v20/model_{i}.pkl',map_location=torch.device('cpu')))
 models.append(mlp )
ovrLR = OneVsRestClassifier(estimator=logistic, n_jobs=1)
 ovrLR.fit(X_train, y_train)
 print('Accuracy: %.5f' % ovrLR.score(X_test, y_test))<train_model>
def fit(model, x):
x = torch.tensor(x, dtype=torch.float)
 model.eval()
pred = model(x)
 return pred.detach().numpy()
lsvc = LinearSVC(C=0.2, class_weight=None, dual=True, fit_intercept=True,
intercept_scaling=1, loss='squared_hinge', max_iter=100,
multi_class='ovr', penalty='l2', random_state=1, tol=0.0001,
verbose=0)
 lsvc.fit(X_train, y_train)
 print('Accuracy: %.5f' % lsvc.score(X_test, y_test))<train_model>
mlp.eval()
fit(mlp, X[0:1,:] )
ovrLSVC = OneVsRestClassifier(estimator=lsvc, n_jobs=1)
 ovrLSVC.fit(X_train, y_train)
 print('Accuracy: %.5f' % ovrLSVC.score(X_test, y_test))<train_model>
env = janestreet.make_env()
env_iter = env.iter_test()
etc = ExtraTreesClassifier(bootstrap=False, class_weight=None, criterion='gini',
max_depth=None, max_features=0.25, max_leaf_nodes=None,
min_impurity_decrease=0.0, min_impurity_split=None,
min_samples_leaf=1, min_samples_split=2,
min_weight_fraction_leaf=0.0, n_estimators=400, n_jobs=None,
oob_score=False, random_state=1, verbose=0, warm_start=False)
 etc.fit(X_train, y_train)
 print('Accuracy: %.5f' % etc.score(X_test, y_test))<train_model>
opt_th = 0.5
if not train_mode:
for(test_df, pred_df)in env_iter:
if test_df['weight'].item() > 0:
test_df = test_df.loc[:, features].values
if np.isnan(test_df[:, 1:].sum()):
test_df[:, 1:] = np.nan_to_num(test_df[:, 1:])+ np.isnan(test_df[:, 1:])* f_mean
pred_vector = np.mean([fit(model, test_df)for model in models],axis=0)
 pred = np.mean(pred_vector)
 pred_df.action =(pred_vector > opt_th ).astype(int)
 else:
pred_df.action = 0
env.predict(pred_df )