Kaggle competition: Tabular Playground Series - Jan 2021
<SOS> metric: RMSE Kaggle data source: tabular-playground-series-jan-2021<feature_engineering>
warnings.filterwarnings("ignore" )
X_train = X_train / 255.0
test = test / 255.0<categorify>
train = pd.read_csv('.. /input/tabular-playground-series-jan-2021/train.csv')
 test = pd.read_csv('.. /input/tabular-playground-series-jan-2021/test.csv')
 sub = pd.read_csv('.. /input/tabular-playground-series-jan-2021/sample_submission.csv' )
Y_train=to_categorical(Y_train, num_classes=10 )<define_variables>
features = ['cont1', 'cont2', 'cont3', 'cont4', 'cont5', 'cont6', 'cont7',
'cont8', 'cont9', 'cont10', 'cont11', 'cont12', 'cont13', 'cont14']
random_seed=2<split>
X = train_01[features]
y = train_01['target']
X_train,X_val,Y_train,Y_val = train_test_split(X_train,Y_train,test_size=0.1,random_state=random_seed )<choose_model_class>
params_xgb = {'lambda': 1,
'alpha': 0,
'colsample_bytree': 1,
'subsample': 1,
'learning_rate': 0.05,
'max_depth': 6,
'min_child_weight': 3,
'random_state': 48}
model = Sequential()
model.add(Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same',
activation ='relu', input_shape =(28,28,1)))
 model.add(Conv2D(filters = 32, kernel_size =(5,5),padding = 'Same',
activation ='relu'))
model.add(MaxPool2D(pool_size=(2,2)))
 model.add(Dropout(0.25))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
model.add(Conv2D(filters = 64, kernel_size =(3,3),padding = 'Same',
activation ='relu'))
model.add(MaxPool2D(pool_size=(2,2), strides=(2,2)))
 model.add(Dropout(0.25))
model.add(Flatten())
 model.add(Dense(256, activation = "relu"))
model.add(Dropout(0.5))
model.add(Dense(10, activation = "softmax"))<choose_model_class>
model_xgb = xgb.XGBRegressor(**params_xgb )
optimizer = RMSprop(lr=0.001,rho=0.9,epsilon=1e-08,decay=0.0 )<choose_model_class>
param_range = np.linspace(0, 1, 10)
 param_range
model.compile(optimizer=optimizer,loss="categorical_crossentropy",metrics=["accuracy"] )<choose_model_class>
param_name = "alpha"
learning_rate_reduction = ReduceLROnPlateau(monitor='val_loss',patience=3,verbose=1,factor=0.5,min_lr=0.00001 )<define_variables>
param_name = "lambda"
epochs=15
batch_size=86<init_hyperparams>
param_range = np.linspace(0.1, 1, 10)
 param_range
datagen = ImageDataGenerator(
 featurewise_center=False,
samplewise_center=False,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=10,
zoom_range = 0.1,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=False,
vertical_flip=False)
 datagen.fit(X_train )<train_model>
param_name = 'colsample_bytree'
history = model.fit(datagen.flow(X_train,Y_train, batch_size=batch_size),
epochs = epochs, validation_data =(X_val,Y_val),
verbose = 2, steps_per_epoch=X_train.shape[0] // batch_size
, callbacks=[learning_rate_reduction] )<predict_on_test>
param_name = 'subsample'
results = model.predict(test)
 results = np.argmax(results,axis = 1)
 results = pd.Series(results,name="Label" )<save_to_csv>
param_name = 'n_estimators'
submission = pd.concat([pd.Series(range(1,28001),name = "ImageId"),results],axis = 1)
 submission.to_csv("cnn_mnist_datagen.csv",index=False )<install_modules>
param_range = [100, 200, 300, 400, 500, 600, 700, 800, 900, 1000]
pip install -U lightautoml<install_modules>
X = train[features]
y = train['target']
pip install -U transformers<import_modules>
def objective(trial,data=X,target=y):
train_x, test_x, train_y, test_y = train_test_split(data, target, test_size=0.15,random_state=42)
 param = {
'tree_method':'gpu_hist',
'lambda': trial.suggest_loguniform('lambda', 1e-3, 1),
'alpha': trial.suggest_loguniform('alpha', 1e-3, 1),
'colsample_bytree': trial.suggest_categorical('colsample_bytree', [0.1, 0.2, 0.3,0.5,0.7,0.9]),
'subsample': trial.suggest_categorical('subsample', [0.1, 0.2,0.3,0.4,0.5,0.8,1.0]),
'learning_rate': trial.suggest_categorical('learning_rate', [0.0008, 0.01, 0.015, 0.02,0.03, 0.05,0.08,0.1]),
'n_estimators': 4000,
'max_depth': trial.suggest_categorical('max_depth', [5,7,9,11,13,15,17,20,23,25]),
'random_state': 48,
'min_child_weight': trial.suggest_int('min_child_weight', 1, 400),
}
model = xgb.XGBRegressor(**param)
 model.fit(train_x,train_y,eval_set=[(test_x,test_y)],early_stopping_rounds=100,verbose=False)
 preds = model.predict(test_x)
 rmse = mean_squared_error(test_y, preds,squared=False)
 return rmse
import os
import time
import numpy as np
import pandas as pd
from sklearn.metrics import f1_score
from sklearn.model_selection import train_test_split
import torch
import matplotlib.pyplot as plt
from lightautoml.automl.presets.text_presets import TabularNLPAutoML
from lightautoml.dataset.roles import DatetimeRole
from lightautoml.tasks import Task 
<define_variables>
study = optuna.create_study(direction='minimize')
 study.optimize(objective, n_trials=50)
 print('Number of finished trials:', len(study.trials))
print('Best trial:', study.best_trial.params )
N_THREADS = 4
RANDOM_STATE = 42
TEST_SIZE = 0.2
TIMEOUT = 6 * 3600
TARGET_NAME = 'target'<set_options>
study.best_trial.params
np.random.seed(RANDOM_STATE)
 torch.set_num_threads(N_THREADS )<load_from_csv>
Best_params_xgb = {'lambda': 0.001951466677835018,
'alpha': 0.7843235982110978,
'colsample_bytree': 0.5,
'subsample': 0.8,
'learning_rate': 0.01,
'max_depth': 11,
'min_child_weight': 205,
'n_estimators': 3000,
'random_state': 48,
'tree_method':'gpu_hist'}
%%time
train_data = pd.read_csv('.. /input/nlp-getting-started/train.csv')
 train_data.head()<load_from_csv>
X = train[features]
y = train['target']
test_data = pd.read_csv('.. /input/nlp-getting-started/test.csv')
 test_data.head()<load_from_csv>
train_x, test_x, train_y, test_y = train_test_split(X, y, test_size=0.15,random_state=42)
 model_xgb = xgb.XGBRegressor(**Best_params_xgb)
 model_xgb.fit(train_x,train_y,eval_set=[(test_x,test_y)],
early_stopping_rounds=100,verbose=False )
submission = pd.read_csv('.. /input/nlp-getting-started/sample_submission.csv')
 submission.head()<count_values>
preds = model_xgb.predict(test_x)
 rmse = mean_squared_error(test_y, preds,squared=False)
 rmse
train_data.target.value_counts()<count_values>
test_X = test[features]
train_data['keyword'].value_counts(dropna = False )<count_values>
preds = model_xgb.predict(test_X )
train_data['location'].value_counts(dropna = False )<drop_column>
sub['target']=preds
sub.to_csv('submission_xgb.csv', index=False )
def clean_text(text):
return text<feature_engineering>
X = train_01[features]
y = train_01['target']
all_data = pd.concat([
train_data.drop(TARGET_NAME, axis = 1),
test_data
] ).reset_index(drop = True)
 all_data['location'] = all_data['location'].astype(str)
 all_data.loc[all_data['location'].value_counts() [all_data['location']].values < 5, 'location'] = "RARE_VALUE"
all_data.loc[all_data['location'] == 'nan', 'location'] = np.nan
all_data['text'] = all_data['text'].map(clean_text)
 all_data<prepare_x_and_y>
params_lgb = {'num_leaves': 31,
'min_data_in_leaf': 20,
'min_child_weight': 0.001,
'max_depth': -1,
'learning_rate': 0.005,
'bagging_fraction': 1,
'feature_fraction': 1,
'lambda_l1': 0,
'lambda_l2': 0,
'random_state': 48}
y_train = train_data.target.values
train_data = all_data[:len(train_data)]
train_data[TARGET_NAME] = y_train
test_data = all_data[len(train_data):]<define_variables>
model_lgb = lgb.LGBMRegressor(**params_lgb )
%%time
roles = {'target': TARGET_NAME,
'text': ['text'],
'drop': ['id']}<choose_model_class>
param_range = np.linspace(0, 1, 10)
 param_range
%%time
automl = TabularNLPAutoML(task = task,
timeout = TIMEOUT,
cpu_limit = N_THREADS,
reader_params = {'cv': 5},
general_params = {'nested_cv': False, 'use_algos': [['linear_l2', 'lgb', 'nn']]},
text_params = {'lang': 'en'},
nn_params = {'lang': 'en',
'bert_name': 'vinai/bertweet-base',
'opt_params': { 'lr': 1e-5},
'max_length': 300, 'bs': 13,
'n_epoch': 5
}, 
)
 oof_pred = automl.fit_predict(train_data, roles = roles)
 print('oof_pred:
{}
Shape = {}'.format(oof_pred, oof_pred.shape))<set_options>
param_name = 'lambda_l1'
automl.collect_used_feats()<predict_on_test>
param_name = 'lambda_l2'
test_pred = automl.predict(test_data)
 print('Prediction for test data:
{}
Shape = {}'.format(test_pred, test_pred.shape))<compute_train_metric>
param_name = 'feature_fraction'
def select_threshold_f1(y_true, y_pred):
best_score = -1
best_thr = None
for thr in np.arange(0, 1.01, 0.01):
score = f1_score(y_true,(y_pred > thr ).astype(int))
if score > best_score:
best_score = score
best_thr = thr
print('Best score: {}
Best selected threshold: {:.2f}'.format(best_score, best_thr))
return best_thr
best_thr = select_threshold_f1(train_data[TARGET_NAME], oof_pred.data[:, 0] )<data_type_conversions>
param_name = 'bagging_fraction'
submission['target'] =(test_pred.data[:, 0] > best_thr ).astype(int)
 submission<count_values>
param_name = 'n_estimators'
submission['target'].value_counts()<save_to_csv>
param_range = [100, 200, 300, 400, 500, 600, 700, 800, 900, 1000]
submission.to_csv('LightAutoML_preds_without_id.csv', index = False )<set_options>
X = train[features]
y = train['target']
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 print("Using %s" %(device))
seed_val = 42
random.seed(seed_val)
 np.random.seed(seed_val)
 torch.manual_seed(seed_val)
 torch.cuda.manual_seed_all(seed_val )<categorify>
def objective_lgb(trial,data=X,target=y):
train_x, test_x, train_y, test_y = train_test_split(data, target, test_size=0.15,random_state=42)
 param = {
'tree_method':'gpu_hist',
'lambda_l2': trial.suggest_loguniform('lambda_l2', 1e-3, 1),
'lambda_l1': trial.suggest_loguniform('lambda_l1', 1e-3, 1),
'feature_framcion': trial.suggest_categorical('feature_framcion', [0.1, 0.2, 0.3,0.5,0.7,0.9]),
'bagging_fraction': trial.suggest_categorical('bagging_framcion', [0.1, 0.2,0.3,0.4,0.5,0.8,1.0]),
'learning_rate': trial.suggest_categorical('learning_rate', [0.0008, 0.01, 0.015, 0.02,0.03, 0.05,0.08,0.1]),
'n_estimators': 4000,
'num_leaves': trial.suggest_categorical('num_leaves', [31,50,150,200,250,300,350]),
'max_depth': trial.suggest_categorical('max_depth', [5,7,9,11,13,15,17,20,23,25]),
'min_data_in_leaf': trial.suggest_categorical('min_data_in_leaf', [10,20,30]),
'min_child_weight': trial.suggest_categorical('min_child_weight', [0.001,0.005, 0.01, 0.05, 0.1,0.5]),
'random_state': 48
}
model = lgb.LGBMRegressor(**param)
 model.fit(train_x,train_y,eval_set=[(test_x,test_y)],early_stopping_rounds=100,verbose=False)
 preds = model.predict(test_x)
 rmse = mean_squared_error(test_y, preds,squared=False)
 return rmse
def clean_text(text):
text = text.lower()
text = re.sub(r'[!]+', '!', text)
 text = re.sub(r'[?]+', '?', text)
 text = re.sub(r'[.]+', '.', text)
 text = re.sub(r"'", "", text)
 text = re.sub('\s+', ' ', text ).strip()
text = re.sub(r'&amp;?', r'and', text)
 text = re.sub(r"https?:\/\/t.co\/[A-Za-z0-9]+", "", text)
 text = re.sub(r'[:"$%&\*+,-/:;<=>@\\^_`{|}~]+', '', text)
 emoji_pattern = re.compile("["
u"\U0001F600-\U0001F64F"
u"\U0001F300-\U0001F5FF"
u"\U0001F680-\U0001F6FF"
u"\U0001F1E0-\U0001F1FF"
u"\U00002702-\U000027B0"
u"\U000024C2-\U0001F251"
"]+", flags=re.UNICODE)
 text = emoji_pattern.sub(r'EMOJI', text)
 return text<load_from_csv>
study = optuna.create_study(direction='minimize')
 study.optimize(objective_lgb, n_trials=50)
 print('Number of finished trials:', len(study.trials))
print('Best trial:', study.best_trial.params )
train = pd.read_csv("/kaggle/input/nlp-getting-started/train.csv")
 test = pd.read_csv("/kaggle/input/nlp-getting-started/test.csv")
 train['text_new'] = train['text'].apply(clean_text)
 test['text'] = test['text'].apply(clean_text)
 train_texts = list(train["text"])
 train_labels = list(train["target"])
 res_texts = list(test["text"])
 x_train, x_test, train_label, test_label = train_test_split(train_texts, train_labels, test_size=0.2)
 print(len(res_texts))<categorify>
study.best_trial.params
tokenizer = DistilBertTokenizerFast.from_pretrained('distilbert-base-uncased')
 train_encoding = tokenizer(x_train, truncation=True, padding=True)
 test_encoding = tokenizer(x_test, truncation=True, padding=True)
 res_encoding = tokenizer(res_texts, truncation=True, padding=True )<categorify>
Best_params_lgb = {'lambda_l2': 0.013616569506899653,
'lambda_l1': 0.006495842188985166,
'feature_framcion': 0.3,
'bagging_framcion': 0.3,
'learning_rate': 0.015,
'num_leaves': 200,
'max_depth': 25,
'min_data_in_leaf': 30,
'min_child_weight': 0.001,
'n_estimators': 3000,
'random_state': 48,
'tree_method':'gpu_hist'}
class TwitterDataset(Dataset):
def __init__(self, encodings, labels):
self.encodings = encodings
self.labels = labels
def __getitem__(self, idx):
item = {key: torch.tensor(val[idx])for key, val in self.encodings.items() }
item['labels'] = torch.tensor(self.labels[idx])
 return item
def __len__(self):
return len(self.labels)
 train_dataset = TwitterDataset(train_encoding, train_label)
 test_dataset = TwitterDataset(test_encoding, test_label )<compute_test_metric>
train_x, test_x, train_y, test_y = train_test_split(X, y, test_size=0.15,random_state=42)
 model_lgb = lgb.LGBMRegressor(**Best_params_lgb)
 model_lgb.fit(train_x,train_y,eval_set=[(test_x,test_y)],
early_stopping_rounds=100,verbose=False )
def flat_accuracy(preds, labels):
pred_flat = np.argmax(preds, axis=1 ).flatten()
labels_flat = labels.flatten()
return np.sum(pred_flat == labels_flat)/ len(labels_flat )<load_pretrained>
preds = model_lgb.predict(test_x)
 rmse = mean_squared_error(test_y, preds,squared=False)
 rmse
step_nums = 30
model = DistilBertForSequenceClassification.from_pretrained('distilbert-base-uncased')
 model.to(device)
 train_loader = DataLoader(train_dataset, batch_size=16, shuffle=True)
 test_dataloader = DataLoader(test_dataset, batch_size=16, shuffle=True)
 optim = AdamW(model.parameters() , lr=2e-5)
 scheduler = get_linear_schedule_with_warmup(optim,
num_warmup_steps = 0,
num_training_steps = step_nums)
 first_batch = next(iter(train_loader))
for batch_idx, batch in enumerate([first_batch] * step_nums):
model.train()
optim.zero_grad()
input_ids = batch['input_ids'].to(device)
 attention_mask = batch['attention_mask'].to(device)
 labels = batch['labels'].to(device)
 outputs = model(input_ids, attention_mask=attention_mask, labels=labels)
 loss = outputs[0]
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , 1.0)
 optim.step()
scheduler.step()
model.eval()
with torch.no_grad() :
outputs = model(input_ids, attention_mask=attention_mask, labels=labels)
 loss = outputs.loss
logits = outputs.logits
logits = logits.detach().cpu().numpy()
label_ids = labels.to('cpu' ).numpy()
print(f"step: {batch_idx:d}, loss: {loss.item() :.2f}, accuracy: {flat_accuracy(logits, label_ids)*100:.2f}%" )<load_pretrained>
test_X = test[features]
epoth_num=5
model = DistilBertForSequenceClassification.from_pretrained('distilbert-base-uncased')
 model.to(device)
 train_loader = DataLoader(train_dataset, batch_size=16, shuffle=True)
 test_dataloader = DataLoader(test_dataset, batch_size=16, shuffle=True)
 optim = AdamW(model.parameters() , lr=2e-5)
 total_steps = len(train_loader)* epoth_num
scheduler = get_linear_schedule_with_warmup(optim,
num_warmup_steps = 0,
num_training_steps = total_steps)
 best_accuracy=0
for epoch in range(epoth_num):
print("------------Epoch: %d ----------------" % epoch)
 model.train()
total_train_loss = 0
iter_num = 0
total_iter = len(train_loader)
 for batch in train_loader:
optim.zero_grad()
input_ids = batch['input_ids'].to(device)
 attention_mask = batch['attention_mask'].to(device)
 labels = batch['labels'].to(device)
 outputs = model(input_ids, attention_mask=attention_mask, labels=labels)
 loss = outputs[0]
total_train_loss += loss.item()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters() , 1.0)
 optim.step()
scheduler.step()
iter_num += 1
if(iter_num % 100==0):
print(f"epoth: {epoch}, iter_num: {iter_num}, loss: {loss.item() :.4f}, {iter_num/total_iter*100:.2f}%")
 print(f"Epoch: {epoch}, Average training loss: {total_train_loss/len(train_loader):.4f}")
 print("")
 print("Running Validation...")
 model.eval()
total_eval_accuracy = 0
total_eval_loss = 0
for batch in test_dataloader:
with torch.no_grad() :
input_ids = batch['input_ids'].to(device)
 attention_mask = batch['attention_mask'].to(device)
 labels = batch['labels'].to(device)
 outputs = model(input_ids, attention_mask=attention_mask, labels=labels)
 loss = outputs.loss
logits = outputs.logits
total_eval_loss += loss.item()
logits = logits.detach().cpu().numpy()
label_ids = labels.to('cpu' ).numpy()
total_eval_accuracy += flat_accuracy(logits, label_ids)
 avg_val_accuracy = total_eval_accuracy / len(test_dataloader)
 if avg_val_accuracy > best_accuracy:
best_accuracy = avg_val_accuracy
print("Best model till now.")
 torch.save(model.state_dict() , './model.weights')
 else:
model.load_state_dict(torch.load("./model.weights"))
print(f"Accuracy: {avg_val_accuracy:.4f}")
 print(f"Average testing loss: {total_eval_loss/len(test_dataloader):.4f}")
 print("-------------------------------")
<categorify>
preds = model_lgb.predict(test_X )
print("Restoring the best model weights.")
 model.load_state_dict(torch.load("./model.weights"))
model.eval()
class TwitterValDataset(Dataset):
def __init__(self, encodings):
self.encodings = encodings
def __getitem__(self, idx):
item = {key: torch.tensor(val[idx])for key, val in self.encodings.items() }
return item
def __len__(self):
return len(self.encodings['input_ids'])
 val_dataset = TwitterValDataset(res_encoding)
 val_loader = DataLoader(val_dataset, batch_size=16, shuffle=False)
 result = []
for batch in val_loader:
with torch.no_grad() :
input_ids = batch['input_ids'].to(device)
 attention_mask = batch['attention_mask'].to(device)
 outputs = model(input_ids, attention_mask=attention_mask)
 logits = outputs.logits
logits = logits.detach().cpu().numpy()
pred_flat = np.argmax(logits, axis=1 ).flatten()
result.extend(pred_flat)
 df = pd.read_csv("/kaggle/input/nlp-getting-started/sample_submission.csv")
 df['target'] = result
df.to_csv("submission.csv", index=False )<load_from_csv>
sub['target']=preds
sub.to_csv('submission_lgb.csv', index=False )