Kaggle competition: CareerCon 2019 - Help Navigate Robots
<SOS> metric: categorizationaccuracy Kaggle data source: career-con-2019<compute_test_metric>
warnings.simplefilter('ignore' )
<predict_on_test>
train_df = pd.read_csv('.. /input/career-con-2019/X_train.csv')
 test_df = pd.read_csv('.. /input/career-con-2019/X_test.csv')
 target = pd.read_csv('.. /input/career-con-2019/y_train.csv')
 sub = pd.read_csv('.. /input/career-con-2019/sample_submission.csv')
predAns = model3.predict(Xtest )<prepare_output>
def quaternion_to_euler(x, y, z, w):
t0 = +2.0 *(w * x + y * z)
 t1 = +1.0 - 2.0 *(x * x + y * y)
 X = math.atan2(t0, t1)
 t2 = +2.0 *(w * y - z * x)
 t2 = +1.0 if t2 > +1.0 else t2
t2 = -1.0 if t2 < -1.0 else t2
Y = math.asin(t2)
 t3 = +2.0 *(w * z + x * y)
 t4 = +1.0 - 2.0 *(y * y + z * z)
 Z = math.atan2(t3, t4)
 return X, Y, Z
def feature_eng(data):
data['total_angular_velocity'] =(data['angular_velocity_X'] ** 2 + data['angular_velocity_Y'] ** 2 + data['angular_velocity_Z'] ** 2)** 0.5
data['total_linear_acceleration'] =(data['linear_acceleration_X'] ** 2 + data['linear_acceleration_Y'] ** 2 + data['linear_acceleration_Z'] ** 2)** 0.5
data['acc_vs_vel'] = data['total_linear_acceleration'] / data['total_angular_velocity']
x, y, z, w = data['orientation_X'].tolist() , data['orientation_Y'].tolist() , data['orientation_Z'].tolist() , data['orientation_W'].tolist()
nx, ny, nz = [], [], []
for i in range(len(x)) :
xx, yy, zz = quaternion_to_euler(x[i], y[i], z[i], w[i])
 nx.append(xx)
 ny.append(yy)
 nz.append(zz)
 data['euler_x'] = nx
data['euler_y'] = ny
data['euler_z'] = nz
data['total_angle'] =(data['euler_x'] ** 2 + data['euler_y'] ** 2 + data['euler_z'] ** 2)** 0.5
data['angle_vs_acc'] = data['total_angle'] / data['total_linear_acceleration']
data['angle_vs_vel'] = data['total_angle'] / data['total_angular_velocity']
df = pd.DataFrame()
for col in data.columns:
if col in ['row_id', 'series_id', 'measurement_number']:
continue
df[col + '_mean'] = data.groupby(['series_id'])[col].mean()
df[col + '_min'] = data.groupby(['series_id'])[col].min()
df[col + '_max'] = data.groupby(['series_id'])[col].max()
df[col + '_std'] = data.groupby(['series_id'])[col].std()
df[col + '_max_to_min'] = df[col + '_max'] / df[col + '_min']
df[col + '_abs_max'] = data.groupby('series_id')[col].apply(lambda x: np.max(np.abs(x)))
 df[col + '_abs_min'] = data.groupby('series_id')[col].apply(lambda x: np.min(np.abs(x)))
 return df
predAns = np.expm1(predAns )<create_dataframe>
new_training_data = feature_eng(train_df)
 new_test_data = feature_eng(test_df )
output = pandas.DataFrame({"id":ID,"loss":predAns} )<save_to_csv>
new_training_data.fillna(0, inplace = True)
 new_test_data.fillna(0, inplace = True)
 new_training_data.replace(-np.inf, 0, inplace = True)
 new_training_data.replace(np.inf, 0, inplace = True)
 new_test_data.replace(-np.inf, 0, inplace = True)
 new_test_data.replace(np.inf, 0, inplace = True )
output.to_csv("output.csv",index=False )<load_from_csv>
le = LabelEncoder()
target = le.fit_transform(target['surface'] )
random_state = 42
np.random.seed(random_state)
 df_train = pd.read_csv('.. /input/train.csv')
 df_test = pd.read_csv('.. /input/test.csv' )<categorify>
rf_param_grid = {
'max_depth' : list(range(8,100,10)) ,
'n_estimators': list(range(50,2000,100)) ,
'min_samples_split': [2,6,8],
'min_samples_leaf': [2,6,8],
'bootstrap': [True, False]
}
def augment(x,y,t=2):
xs,xn = [],[]
for i in range(t):
mask = y>0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 for c in range(x1.shape[1]):
np.random.shuffle(ids)
 x1[:,c] = x1[ids][:,c]
xs.append(x1)
 for i in range(t//2):
mask = y==0
x1 = x[mask].copy()
ids = np.arange(x1.shape[0])
 for c in range(x1.shape[1]):
np.random.shuffle(ids)
 x1[:,c] = x1[ids][:,c]
xn.append(x1)
 xs = np.vstack(xs)
 xn = np.vstack(xn)
 ys = np.ones(xs.shape[0])
 yn = np.zeros(xn.shape[0])
 x = np.vstack([x,xs,xn])
 y = np.concatenate([y,ys,yn])
 return x,y<init_hyperparams>
m = RandomForestClassifier(n_jobs=-1 )
lgb_params = {
"objective" : "binary",
"metric" : "auc",
"boosting": 'gbdt',
"max_depth" : -1,
"num_leaves" : 13,
"learning_rate" : 0.01,
"bagging_freq": 5,
"bagging_fraction" : 0.4,
"feature_fraction" : 0.05,
"min_data_in_leaf": 80,
"min_sum_heassian_in_leaf": 10,
"tree_learner": "serial",
"boost_from_average": "false",
"bagging_seed" : random_state,
"verbosity" : 1,
"seed": random_state
}<create_dataframe>
m_r = GridSearchCV(param_grid=rf_param_grid, estimator = m, scoring = "accuracy", cv = 4 )
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=random_state)
 oof = df_train[['ID_code', 'target']]
oof['predict'] = 0
predictions = df_test[['ID_code']]
val_aucs = []
feature_importance_df = pd.DataFrame()<prepare_x_and_y>
features = [col for col in df_train.columns if col not in ['target', 'ID_code']]
X_test = df_test[features].values<split>
m_r = RandomizedSearchCV(param_distributions=rf_param_grid, estimator = m, scoring = "accuracy", cv = 4,n_jobs=-1,n_iter=10 )
for fold,(trn_idx, val_idx)in enumerate(skf.split(df_train, df_train['target'])) :
X_train, y_train = df_train.iloc[trn_idx][features], df_train.iloc[trn_idx]['target']
X_valid, y_valid = df_train.iloc[val_idx][features], df_train.iloc[val_idx]['target']
N = 5
p_valid,yp = 0,0
for i in range(N):
X_t, y_t = augment(X_train.values, y_train.values)
 X_t = pd.DataFrame(X_t)
 X_t = X_t.add_prefix('var_')
 trn_data = lgb.Dataset(X_t, label=y_t)
 val_data = lgb.Dataset(X_valid, label=y_valid)
 evals_result = {}
lgb_clf = lgb.train(lgb_params,
trn_data,
100000,
valid_sets = [trn_data, val_data],
early_stopping_rounds=3000,
verbose_eval = 1000,
evals_result=evals_result 
)
 p_valid += lgb_clf.predict(X_valid)
 yp += lgb_clf.predict(X_test)
 fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = features
fold_importance_df["importance"] = lgb_clf.feature_importance()
fold_importance_df["fold"] = fold + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
 oof['predict'][val_idx] = p_valid/N
val_score = roc_auc_score(y_valid, p_valid)
 val_aucs.append(val_score)
 predictions['fold{}'.format(fold+1)] = yp/N<compute_test_metric>
%time m_r.fit(new_training_data,target )
mean_auc = np.mean(val_aucs)
 std_auc = np.std(val_aucs)
 all_auc = roc_auc_score(oof['target'], oof['predict'])
 print("Mean auc: %.9f, std: %.9f.All auc: %.9f." %(mean_auc, std_auc, all_auc))<save_to_csv>
m_r.best_params_
predictions['target'] = np.mean(predictions[[col for col in predictions.columns if col not in ['ID_code', 'target']]].values, axis=1)
 predictions.to_csv('lgb_all_predictions.csv', index=None)
 sub_df = pd.DataFrame({"ID_code":df_test["ID_code"].values})
 sub_df["target"] = predictions['target']
sub_df.to_csv("lgb_submission.csv", index=False)
 oof.to_csv('lgb_oof.csv', index=False )<import_modules>
for param, score in zip(m_r.cv_results_['params'], m_r.cv_results_['mean_test_score']):
print(param, score )
import numpy as np
import pandas as pd
import os
import time
import gc
import random
from tqdm._tqdm_notebook import tqdm_notebook as tqdm
from keras.preprocessing import text, sequence
import torch
from torch import nn
from torch.utils import data
from torch.nn import functional as F
from sklearn.model_selection import train_test_split<set_options>
rf_bp = m_r.best_params_
def seed_everything(seed=1234):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()<define_variables>
rf_classifier=RandomForestClassifier(n_estimators=rf_bp["n_estimators"],
min_samples_split=rf_bp['min_samples_split'],
min_samples_leaf=rf_bp['min_samples_leaf'],
max_depth=rf_bp['max_depth'],
bootstrap=rf_bp['bootstrap'] )
CRAWL_EMBEDDING_PATH = '.. /input/fasttext-crawl-300d-2m/crawl-300d-2M.vec'
GLOVE_EMBEDDING_PATH = '.. /input/glove840b300dtxt/glove.840B.300d.txt'
NUM_MODELS = 1
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
MAX_LEN = 224<feature_engineering>
rf_classifier.fit(new_training_data,target )
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
 def load_embeddings(path):
with open(path)as f:
return dict(get_coefs(*line.strip().split(' ')) for line in tqdm(f))
def build_matrix(word_index, path):
embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( len(word_index)+ 1, 300))
unknown_words = []
for word, i in word_index.items() :
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
unknown_words.append(word)
 return embedding_matrix, unknown_words<train_model>
y_pred = rf_classifier.predict(new_test_data )
def sigmoid(x):
return 1 /(1 + np.exp(-x))
def train_model(model, train,test, loss_fn, output_dim, lr=0.001,
batch_size=512, n_epochs=1,
enable_checkpoint_ensemble=True):
param_lrs = [{'params': param, 'lr': lr} for param in model.parameters() ]
optimizer = torch.optim.Adam(param_lrs, lr=lr)
 scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lambda epoch: 1 ** epoch)
 train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)
 test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)
 all_test_preds = []
checkpoint_weights = [2 ** epoch for epoch in range(n_epochs)]
for epoch in range(n_epochs):
start_time = time.time()
scheduler.step()
model.train()
avg_loss = 0.
 for data in tqdm(train_loader, disable=False):
x_batch = data[:-1]
y_batch = data[-1]
y_pred = model(*x_batch)
 loss = loss_fn(y_pred, y_batch)
 optimizer.zero_grad()
loss.backward()
optimizer.step()
avg_loss += loss.item() / len(train_loader)
 model.eval()
test_preds = np.zeros(( len(test), output_dim))
for i, x_batch in enumerate(test_loader):
y_pred = sigmoid(model(*x_batch ).detach().cpu().numpy())
 test_preds[i * batch_size:(i+1)* batch_size, :] = y_pred
all_test_preds.append(test_preds)
 elapsed_time = time.time() - start_time
print('Epoch {}/{} \t triain_loss={:.4f} \t time={:.2f}s'.format(
 epoch + 1, n_epochs, avg_loss, elapsed_time))
if enable_checkpoint_ensemble:
test_preds = np.average(all_test_preds, weights=checkpoint_weights, axis=0)
 else:
test_preds = all_test_preds[-1]
return test_preds<normalization>
sub['surface'] = le.inverse_transform(y_pred)
 sub.to_csv('random_f_rs.csv', index=False )
class SpatialDropout(nn.Dropout2d):
def forward(self, x):
x = x.unsqueeze(2)
 x = x.permute(0, 3, 2, 1)
 x = super(SpatialDropout, self ).forward(x)
 x = x.permute(0, 3, 2, 1)
 x = x.squeeze(2)
 return x
class NeuralNet(nn.Module):
def __init__(self, embedding_matrix, num_aux_targets):
super(NeuralNet, self ).__init__()
embed_size = embedding_matrix.shape[1]
self.embedding = nn.Embedding(max_features, embed_size)
 self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))
self.embedding.weight.requires_grad = False
self.embedding_dropout = SpatialDropout(0.5)
 self.lstm = nn.LSTM(embed_size,LSTM_UNITS,num_layers=2,dropout=0.2,bidirectional=True,batch_first=True)
 for name, param in self.lstm.named_parameters() :
if 'bias' in name:
torch.nn.init.constant_(param, 0.0)
 elif 'weight' in name:
torch.nn.init.orthogonal_(param)
 self.conv1=nn.Conv2d(1,3, kernel_size=3,stride=1,padding=1)
 self.maxpool1=nn.MaxPool2d(kernel_size=(56,4))
self.batch1=torch.nn.BatchNorm2d(3)
 self.droupout_out=torch.nn.Dropout(0.05)
 self.droupout_aux_out=torch.nn.Dropout(0.05)
 self.linear_out = nn.Linear(3*int(MAX_LEN/56)*int(LSTM_UNITS/2), 1)
 self.linear_aux_out = nn.Linear(3*int(MAX_LEN/56)*int(LSTM_UNITS/2), num_aux_targets)
 def forward(self, x):
h_embedding = self.embedding(x)
 h_embedding = self.embedding_dropout(h_embedding)
 h_lstm, _ = self.lstm(h_embedding)
 addaxis_lstm=h_lstm.unsqueeze(1)
 conv_1=self.conv1(addaxis_lstm)
 max_pool1=F.relu(self.maxpool1(conv_1))
max_pool1_batch=self.batch1(max_pool1)
 flatten=torch.flatten(max_pool1_batch, start_dim=1)
 result = self.linear_out(flatten)
 aux_result = self.linear_aux_out(flatten)
 result = self.droupout_out(result)
 aux_result = self.droupout_aux_out(aux_result)
 out = torch.cat([result, aux_result], 1)
 return out<categorify>
n_fold = 4
folds = StratifiedKFold(n_splits=n_fold, shuffle=True )
def preprocess(data):

 punct = "/-'?!.,
def clean_special_chars(text, punct):
for p in punct:
text = text.replace(p, ' ')
 return text
data = data.astype(str ).apply(lambda x: clean_special_chars(x, punct))
return data<load_from_csv>
def randomforest_evaluate(**params):
params['n_estimators'] = int(round(params['n_estimators'],0))
params['min_samples_split'] = int(round(params['min_samples_split'],0))
params['min_samples_leaf'] = int(round(params['min_samples_leaf'],0))
params['bootstrap'] = int(round(params['bootstrap'],0))
test_pred_proba = np.zeros(( new_training_data.shape[0],9))
for n_fold,(train_idx, valid_idx)in enumerate(folds.split(new_training_data, target)) :
X_train, X_valid = new_training_data.iloc[train_idx], new_training_data.iloc[valid_idx]
y_train, y_valid = target[train_idx], target[valid_idx]
model = RandomForestClassifier(**params,n_jobs=-1)
 model.fit(X_train, y_train)
 y_pred_valid = model.predict_proba(new_training_data)
 test_pred_proba += y_pred_valid / folds.n_splits
return accuracy_score(target, test_pred_proba.argmax(1))
train = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv')
 test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv' )<prepare_x_and_y>
rf_param_grid = {
'max_depth' :(8,100),
'n_estimators':(50,2000),
'min_samples_split':(2,10),
'min_samples_leaf':(2, 10),
'bootstrap':(True, False),
}
rf_b_o = BayesianOptimization(randomforest_evaluate, rf_param_grid)
 %time rf_b_o.maximize(init_points=5, n_iter=20 )
x_train = preprocess(train['comment_text'])
 y_train = np.where(train['target'] >= 0.5, 1, 0)
 y_aux_train = train[['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']]
x_test = preprocess(test['comment_text'] )<define_variables>
for i,n in rf_b_o.max["params"].items() :
print(i,int(round(n)) )
max_features = None<string_transform>
model=RandomForestClassifier(n_estimators=int(round(rf_b_o.max["params"]["n_estimators"],0)) ,max_depth=int(round(rf_b_o.max["params"]["max_depth"])) ,
min_samples_leaf=int(round(rf_b_o.max["params"]["min_samples_leaf"])) ,min_samples_split=int(round(rf_b_o.max["params"]["min_samples_split"])))
 model.fit(new_training_data,target)
 y_pred = model.predict(new_test_data)
 sub['surface'] = le.inverse_transform(y_pred)
 sub.to_csv('random_f_bo.csv', index=False )
tokenizer = text.Tokenizer()
tokenizer.fit_on_texts(list(x_train)+ list(x_test))
x_train = tokenizer.texts_to_sequences(x_train)
 x_test = tokenizer.texts_to_sequences(x_test)
 x_train = sequence.pad_sequences(x_train, maxlen=MAX_LEN)
 x_test = sequence.pad_sequences(x_test, maxlen=MAX_LEN )<define_variables>
def randomforest_evaluate(params):
print(params)
 params['n_estimators'] = int(round(params['n_estimators'],0))
params['min_samples_split'] = int(round(params['min_samples_split'],0))
params['min_samples_leaf'] = int(round(params['min_samples_leaf'],0))
params['bootstrap'] = int(round(params['bootstrap'],0))
test_pred_proba = np.zeros(( new_training_data.shape[0],9))
for n_fold,(train_idx, valid_idx)in enumerate(folds.split(new_training_data, target)) :
X_train, X_valid = new_training_data.iloc[train_idx], new_training_data.iloc[valid_idx]
y_train, y_valid = target[train_idx], target[valid_idx]
model = RandomForestClassifier(**params,n_jobs=-1)
 model.fit(X_train, y_train)
 y_pred_valid = model.predict_proba(new_training_data)
 test_pred_proba += y_pred_valid / folds.n_splits
return accuracy_score(target, test_pred_proba.argmax(1))
space = {
'max_depth': hp.uniform('max_depth', 8,100),
'n_estimators': hp.uniform('n_estimators', 50, 2000),
'min_samples_split' : hp.choice('min_samples_split',[2,3,6,9]),
'min_samples_leaf' : hp.choice('min_samples_leaf', [2,3,6,9]),
'bootstrap': hp.choice('bootstrap', [False,True]),
}
%time best = fmin(randomforest_evaluate, space, algo=tpe.suggest, max_evals=25)
 if best["min_samples_split"] <=1:
best["min_samples_split"] = 2
if best["min_samples_split"] <=1:
best["min_samples_leaf"] = 2
print("Best Params:")
 print(best )
max_features = max_features or len(tokenizer.word_index)+ 1
max_features<choose_model_class>
model=RandomForestClassifier(n_estimators=int(round(best["n_estimators"],0)) ,max_depth=int(round(best["max_depth"],0)) ,
min_samples_leaf=int(round(best["min_samples_leaf"])) ,min_samples_split=int(round(best["min_samples_split"])))
 model.fit(new_training_data,target)
 y_pred = model.predict(new_test_data)
 sub['surface'] = le.inverse_transform(y_pred)
 sub.to_csv('random_f_tpe.csv', index=False )
crawl_matrix, unknown_words_crawl = build_matrix(tokenizer.word_index, CRAWL_EMBEDDING_PATH)
 print('n unknown words(crawl): ', len(unknown_words_crawl))<choose_model_class>
!git clone https://github.com/thuijskens/scikit-hyperband.git 2>/dev/null 1>/dev/null
glove_matrix, unknown_words_glove = build_matrix(tokenizer.word_index, GLOVE_EMBEDDING_PATH)
 print('n unknown words(glove): ', len(unknown_words_glove))<concatenate>
!cp -r scikit-hyperband/* .
embedding_matrix = np.concatenate([crawl_matrix, glove_matrix], axis=-1)
 embedding_matrix.shape
del crawl_matrix
del glove_matrix
gc.collect()<data_type_conversions>
!python setup.py install 2>/dev/null 1>/dev/null
x_train_torch = torch.tensor(x_train, dtype=torch.long ).cuda()
x_test_torch = torch.tensor(x_test, dtype=torch.long ).cuda()
y_train_torch = torch.tensor(np.hstack([y_train[:, np.newaxis], y_aux_train]), dtype=torch.float32 ).cuda()<train_model>
model = RandomForestClassifier()
param_dist = {
'max_depth': sp_randint(8,100),
'n_estimators': sp_randint(50,2000),
'min_samples_split' : [2,3,6,9,10],
'min_samples_leaf' : [2,3,6,9,10],
'bootstrap': [True, False]
}
y = LabelBinarizer().fit_transform(target)
 search = HyperbandSearchCV(model, param_dist,
resource_param='n_estimators',
scoring='accuracy')
 %time search.fit(new_training_data, y)
 print(search.best_params_ )
train_dataset = data.TensorDataset(x_train_torch, y_train_torch)
 test_dataset = data.TensorDataset(x_test_torch)
 all_test_preds = []
for model_idx in range(NUM_MODELS):
print('Model ', model_idx)
 seed_everything(1234 + model_idx)
 model = NeuralNet(embedding_matrix, y_aux_train.shape[-1])
 model.cuda()
test_preds = train_model(model, train_dataset,test_dataset, output_dim=y_train_torch.shape[-1], batch_size=512,
n_epochs=4,loss_fn=nn.BCEWithLogitsLoss(reduction='mean'))
all_test_preds.append(test_preds)
 print()<save_to_csv>
best=search.best_params_
submission = pd.DataFrame.from_dict({
'id': test['id'],
'prediction': np.mean(all_test_preds, axis=0)[:, 0]
})
 submission.to_csv('submission.csv', index=False )<set_options>
model=RandomForestClassifier(n_estimators=int(round(best["n_estimators"],0)) ,max_depth=int(round(best["max_depth"],0)) ,
min_samples_leaf=best["min_samples_leaf"],min_samples_split=best["min_samples_split"])
 model.fit(new_training_data,target)
 y_pred = model.predict(new_test_data)
 sub['surface'] = le.inverse_transform(y_pred)
 sub.to_csv('random_f_hb.csv', index=False)
%reload_ext autoreload
%autoreload 2
%matplotlib inline 
<set_options>
def initilialize_poplulation(numberOfParents):
max_depth = np.empty([numberOfParents, 1], dtype = np.uint8)
 n_estimators = np.empty([numberOfParents, 1], dtype = np.uint8)
 min_samples_split = np.empty([numberOfParents, 1])
 min_samples_leaf = np.empty([numberOfParents, 1])
 bootstrap = np.empty([numberOfParents, 1])
 for i in range(numberOfParents):
max_depth[i] = int(random.randrange(8, 100,step=1))
n_estimators[i] = int(random.randrange(50, 2000, step = 100))
min_samples_split[i] = int(random.randrange(2, 10, step= 1))
min_samples_leaf[i] = int(random.randrange(2, 10))
bootstrap[i] = int(random.randint(0, 1))
population = np.concatenate(( max_depth, n_estimators, min_samples_split, min_samples_leaf, bootstrap), axis= 1)
 return population
def seed_everything(seed=123):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 torch.manual_seed(seed)
 torch.cuda.manual_seed(seed)
 torch.backends.cudnn.deterministic = True
seed_everything()<define_variables>
def fitness_accscore(y_true, y_pred):
fitness = round(accuracy_score(y_true, y_pred), 4)
 return fitness
def train_population(population, train,y_train, test, y_test):
fScore = []
for i in range(population.shape[0]):
param_dist = {
'max_depth': int(population[i][0]),
'n_estimators': int(population[i][1]),
'min_samples_split' : int(population[i][2]),
'min_samples_leaf' : int(population[i][3]),
'bootstrap': int(population[i][4])
 }
model = RandomForestClassifier(**param_dist)
 model.fit(train,y_train)
 preds = model.predict(test)
 fScore.append(fitness_accscore(y_test, preds))
return fScore
CRAWL_EMBEDDING_PATH = '.. /input/pickled-crawl300d2m-for-kernel-competitions/crawl-300d-2M.pkl'
GLOVE_EMBEDDING_PATH = '.. /input/pickled-glove840b300d-for-10sec-loading/glove.840B.300d.pkl' 
<load_pretrained>
def new_parents_selection(population, fitness, numParents):
selectedParents = np.empty(( numParents, population.shape[1]))
for parentId in range(numParents):
bestFitnessId = np.where(fitness == np.max(fitness))
bestFitnessId = bestFitnessId[0][0]
selectedParents[parentId, :] = population[bestFitnessId, :]
fitness[bestFitnessId] = -1
return selectedParents
NUM_MODELS = 2
LSTM_UNITS = 128
DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS
MAX_LEN = 220
def get_coefs(word, *arr):
return word, np.asarray(arr, dtype='float32')
 def load_embeddings(path):
with open(path,'rb')as f:
emb_arr = pickle.load(f)
 return emb_arr<categorify>
def mutation(crossover, numberOfParameters):
minMaxValue = np.zeros(( numberOfParameters, 2))
minMaxValue[0:] = [8, 100]
minMaxValue[1, :] = [50, 2000]
minMaxValue[2, :] = [2, 10]
minMaxValue[3, :] = [2, 10]
minMaxValue[4, :] = [0, 1]
mutationValue = 0
parameterSelect = np.random.randint(0,5, 1)
 if parameterSelect == 0:
mutationValue = np.random.randint(-50, 50, 1)
 if parameterSelect == 1:
mutationValue = np.random.randint(-200, 200, 1)
 if parameterSelect == 2:
mutationValue = np.random.randint(-10, 10, 1)
 if parameterSelect == 3:
mutationValue = np.random.randint(-10, 10, 1)
 if parameterSelect == 4:
mutationValue = 0
for idx in range(crossover.shape[0]):
crossover[idx, parameterSelect] = crossover[idx, parameterSelect] + mutationValue
if(crossover[idx, parameterSelect] > minMaxValue[parameterSelect, 1]):
crossover[idx, parameterSelect] = minMaxValue[parameterSelect, 1]
if(crossover[idx, parameterSelect] < minMaxValue[parameterSelect, 0]):
crossover[idx, parameterSelect] = minMaxValue[parameterSelect, 0]
return crossover
def build_matrix(word_index, path):
embedding_index = load_embeddings(path)
 embedding_matrix = np.zeros(( max_features + 1, 300))
unknown_words = []
for word, i in word_index.items() :
if i <= max_features:
try:
embedding_matrix[i] = embedding_index[word]
except KeyError:
try:
embedding_matrix[i] = embedding_index[word.lower() ]
except KeyError:
try:
embedding_matrix[i] = embedding_index[word.title() ]
except KeyError:
unknown_words.append(word)
 return embedding_matrix, unknown_words<normalization>
X_train, X_test, y_train, y_test = train_test_split(new_training_data, target, test_size = 0.20, random_state = 97)
def sigmoid(x):
return 1 /(1 + np.exp(-x))
class SpatialDropout(nn.Dropout2d):
def forward(self, x):
x = x.unsqueeze(2)
 x = x.permute(0, 3, 2, 1)
 x = super(SpatialDropout, self ).forward(x)
 x = x.permute(0, 3, 2, 1)
 x = x.squeeze(2)
 return x
def train_model(learn,test,output_dim,lr=0.001,
batch_size=512, n_epochs=5,
enable_checkpoint_ensemble=True):
all_test_preds = []
checkpoint_weights = [2 ** epoch for epoch in range(n_epochs)]
test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)
 n = len(learn.data.train_dl)
 phases = [(TrainingPhase(n ).schedule_hp('lr', lr *(0.6**(i)))) for i in range(n_epochs)]
sched = GeneralScheduler(learn, phases)
 learn.callbacks.append(sched)
 for epoch in range(n_epochs):
learn.fit(1)
 test_preds = np.zeros(( len(test), output_dim))
for i, x_batch in enumerate(test_loader):
X = x_batch[0].cuda()
y_pred = sigmoid(learn.model(X ).detach().cpu().numpy())
 test_preds[i * batch_size:(i+1)* batch_size, :] = y_pred
all_test_preds.append(test_preds)
 if enable_checkpoint_ensemble:
test_preds = np.average(all_test_preds, weights=checkpoint_weights, axis=0)
 else:
test_preds = all_test_preds[-1]
return test_preds<load_from_csv>
start = timeit.default_timer()
numberOfParents = 8
numberOfParentsMating = 4
numberOfParameters = 5
numberOfGenerations = 4
populationSize =(numberOfParents, numberOfParameters)
 population = initilialize_poplulation(numberOfParents)
 fitnessHistory = np.empty([numberOfGenerations+1, numberOfParents])
 populationHistory = np.empty([(numberOfGenerations+1)*numberOfParents, numberOfParameters])
 populationHistory[0:numberOfParents, :] = population
for generation in range(numberOfGenerations):
print("This is number %s generation" %(generation))
fitnessValue = train_population(population=population, train=X_train,y_train=y_train, test=X_test, y_test=y_test)
 fitnessHistory[generation, :] = fitnessValue
print('Best accuracy score in the this iteration = {}'.format(np.max(fitnessHistory[generation, :])))
 parents = new_parents_selection(population=population, fitness=fitnessValue, numParents=numberOfParentsMating)
 children = crossover_uniform(parents=parents, childrenSize=(populationSize[0] - parents.shape[0], numberOfParameters))
children_mutated = mutation(children, numberOfParameters)

population[0:parents.shape[0], :] = parents
population[parents.shape[0]:, :] = children_mutated
populationHistory[(generation+1)*numberOfParents :(generation+1)*numberOfParents+ numberOfParents , :] = population
stop = timeit.default_timer()
print('Execution time: ',(stop - start)/60 )
train = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv')
 test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv' )<define_variables>
fitness = train_population(population=population, train=X_train,y_train=y_train, test=X_test, y_test=y_test)
 fitnessHistory[generation+1, :] = fitness
bestFitnessIndex = np.where(fitness == np.max(fitness)) [0][0]
print("Best fitness is =", fitness[bestFitnessIndex])
 print("Best parameters are:")
 print('Max_depth', int(population[bestFitnessIndex][0]))
print('n_estimators', int(population[bestFitnessIndex][1]))
print('min_samples_split', int(population[bestFitnessIndex][2]))
print('min_samples_leaf', int(population[bestFitnessIndex][3]))
print('bootstrap', int(population[bestFitnessIndex][4]))
<define_variables><EOS> 
model=RandomForestClassifier(n_estimators=int(population[bestFitnessIndex][1]),max_depth=int(population[bestFitnessIndex][0]),
min_samples_leaf=int(population[bestFitnessIndex][3]),min_samples_split=int(population[bestFitnessIndex][2]))
model.fit(new_training_data,target)
 y_pred = model.predict(new_test_data)
 sub['surface'] = le.inverse_transform(y_pred)
 sub.to_csv('random_f_ga.csv', index=False)