Kaggle competition: Tabular Playground Series - Jan 2021
<SOS> metric: RMSE Kaggle data source: tabular-playground-series-jan-2021<categorify>
import numpy as np
import pandas as pd
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import KFold
from tqdm import tqdm
def get_transforms(*, aug_list):
return Compose(
 _get_augmentations(aug_list)
 )<choose_model_class>
train_data = pd.read_csv('.. /input/tabular-playground-series-jan-2021/train.csv')
 test_data = pd.read_csv('.. /input/tabular-playground-series-jan-2021/test.csv')
 features = ['cont1', 'cont2', 'cont3', 'cont4', 'cont5', 'cont6', 'cont7',
'cont8', 'cont9', 'cont10', 'cont11', 'cont12', 'cont13', 'cont14']
X_train = train_data[features]
y_train = train_data["target"]
final_X_test = test_data[features]
X_train,X_test,y_train,y_test = train_test_split(X_train, y_train, test_size=0.1, random_state=0)
 regressor = xgb.XGBRegressor(
 colsample_bytree=0.5,
alpha=0.01563,
learning_rate=0.01,
max_depth=15,
min_child_weight=257,
n_estimators=4000,
reg_lambda=0.003,
subsample=0.7,
random_state=2020,
metric_period=100,
silent=1)
 regressor.fit(X_train, y_train, early_stopping_rounds=6, eval_set=[(X_test, y_test)], verbose=1 )
class CustomModel(nn.Module):
def __init__(self, model_name, pretrained=False):
super().__init__()
self.model = timm.create_model(model_name, pretrained=pretrained)
 if hasattr(self.model, 'classifier'):
n_features = self.model.classifier.in_features
self.model.classifier = nn.Linear(n_features, CFG['target_size'])
 elif hasattr(self.model, 'fc'):
n_features = self.model.fc.in_features
self.model.fc = nn.Linear(n_features, CFG['target_size'])
 def forward(self, x):
x = self.model(x)
 return x<prepare_output>
predictions = regressor.predict(final_X_test )
model = CustomModel(TAG['model_name'], pretrained=False)
 model_paths = glob.glob(f'{MODEL_DIR}/*.pth')
 model_paths.sort()
states = [torch.load(path)for path in model_paths]
test_dataset = TTADataset(test, TEST_PATH, ttas=ttas)
 test_loader = DataLoader(test_dataset, batch_size=inference_batch_size, shuffle=False,
num_workers=2, pin_memory=True)
 predictions = inference(model, states, test_loader, device)
 prediction = predictions.mean(1)
 test['label'] = prediction.argmax(1)
 test[['image_id', 'label']].to_csv(OUTPUT_DIR+'submission.csv', index=False)
 test.head()<find_best_params>
X = train_data.drop(['id','target'], axis=1)
 Xtest = test_data.drop(['id'], axis=1)
 y = train_data['target']
train = int(len(X)*0.9)
 Xtrain, Xval = X.iloc[:train], X.iloc[train:]
ytrain, yval = y.iloc[:train], y.iloc[train:]
def valid_inference(model, state, test_loader, device):
model.to(device)
 tk0 = tqdm(enumerate(test_loader), total=len(test_loader))
probs = []
for i,(images, labels)in tk0:
images = images.to(device)
 labels = labels.to(device)
 batch_size, n_crops, c, h, w = images.size()
images = images.view(-1, c, h, w)
 model.load_state_dict(state['model'])
 model.eval()
with torch.no_grad() :
y_preds = model(images ).softmax(1)
 y_preds = y_preds.view(batch_size, n_crops,-1)
 avg_preds = y_preds.to('cpu' ).numpy()
probs.append(avg_preds)
 del images, labels, y_preds, avg_preds
torch.cuda.empty_cache()
probs = np.concatenate(probs)
 return probs<load_from_csv>
params={'random_state': 33,'n_estimators':5000,
'min_data_per_group': 5,
'boosting_type': 'gbdt',
'num_leaves': 256,
'max_dept': -1,
'learning_rate': 0.02,
'subsample_for_bin': 200000,
'lambda_l1': 1.074622455507616e-05,
'lambda_l2': 2.0521330798729704e-06,
'n_jobs': -1,
'cat_smooth': 1.0,
'silent': True,
'importance_type': 'split',
'metric': 'rmse',
'feature_pre_filter': False,
'bagging_fraction': 0.8206341150202605,
'min_data_in_leaf': 100,
'min_sum_hessian_in_leaf': 0.001,
'bagging_freq': 6,
'feature_fraction': 0.5,
'min_gain_to_split': 0.0,
'min_child_samples': 20}
if debug:
train = pd.read_csv('.. /input/cassava-leaf-disease-classification/train.csv')
 folds = train.copy()
Fold = StratifiedKFold(n_splits=CFG['n_fold'], shuffle=True, random_state=CFG['seed'])
 for n,(train_index, val_index)in enumerate(Fold.split(folds, folds[CFG['target_col']])) :
folds.loc[val_index, 'fold'] = int(n)
 folds['fold'] = folds['fold'].astype(int)
 model_paths = glob.glob(f'{MODEL_DIR}/*.pth')
 model_paths.sort()
states = [torch.load(path)for path in model_paths]
oof_df = pd.DataFrame()
oof_aug_preds = []
for fold, state in enumerate(states):
val_idx = folds[folds['fold'] == fold].index
valid_folds = folds.loc[val_idx].reset_index(drop=True)
 valid_dataset = TTADataset(valid_folds, TRAIN_PATH, ttas=ttas)
 valid_loader = DataLoader(valid_dataset,
batch_size=inference_batch_size,
shuffle=False,
num_workers=CFG['num_workers'], pin_memory=True)
 valid_preds = valid_inference(model, state, valid_loader, device)
 valid_pred = valid_preds.mean(1)
 valid_folds[[str(c)for c in range(5)]] = valid_pred
valid_folds['preds'] = valid_pred.argmax(1)
 oof_df = pd.concat([oof_df, valid_folds])
 oof_aug_preds.append(valid_preds)
 LOGGER.info(f"========== fold: {fold} result ==========")
 _ = get_result(valid_folds)
 LOGGER.info(f"========== CV result ==========")
 score = get_result(oof_df)
 score_rem3 = get_result(oof_df.query('fold!=3'))
oof_aug_preds = np.concatenate(oof_aug_preds )<compute_train_metric>
N_FOLDS = 5
kf = KFold(n_splits = N_FOLDS)
 oof = np.zeros(len(y))
oof_vanilla = np.zeros(len(y))
preds = np.zeros(len(Xtest))
params['learning_rate'] = 0.005
params['num_iterations'] = 5000
for train_ind, test_ind in tqdm(kf.split(X)) :
Xtrain = X.iloc[train_ind]
Xval = X.iloc[test_ind]
ytrain = y.iloc[train_ind]
yval = y.iloc[test_ind]
model = LGBMRegressor(**params)
 vanilla_model = LGBMRegressor()
model.fit(Xtrain, ytrain, eval_set =(( Xval,yval)) , early_stopping_rounds = 50, verbose = 0)
 vanilla_model.fit(Xtrain, ytrain)
 p = model.predict(Xval)
 p_vanilla = vanilla_model.predict(Xval)
 oof[test_ind] = p
oof_vanilla[test_ind] = p_vanilla
preds += model.predict(Xtest)/N_FOLDS
print(f'mean square error on training data(vanilla model): {np.round(mean_squared_error(y, oof_vanilla, squared=False),5)}')
 print(f'mean square error on training data(with optuna tuning): {np.round(mean_squared_error(y, oof, squared=False),5)}' )
if debug:
LOGGER.info(f"========== augmentation result ==========")
 get_aug_score(oof_aug_preds, oof_df['label'], ttas)
 get_aug_csv(oof_aug_preds, oof_df, ttas )<import_modules>
sub1 = pd.read_csv('.. /input/resultsdriven-tabular-playground-series-201/submission - 2021-01-15T023916.124.csv')
 predictions1 = sub1['target'].tolist()
sub2 = pd.read_csv('.. /input/resultsdriventabularplaygroundseries2011/submission - 2021-01-16T012125.132.csv')
 predictions2 = sub2['target'].tolist()
import numpy as np
import pandas as pd
import os 
<set_options>
results = [y*1.0002 for x, y in zip(predictions1, predictions2)]
<import_modules><EOS> 
output = pd.DataFrame({"id":test_data.id, "target":results})
 output.to_csv('submission.csv', index=False )