Kaggle competition: Contradictory, My Dear Watson
<SOS> metric: categorizationaccuracy Kaggle data source: contradictory,-my-dear-watson<sort_values>
os.environ["WANDB_API_KEY"] = "0"
for key, value in sorted(temp_rss.items() , key=lambda item: item[1]):
print("%s: %s" %(key, value))<find_best_model_class>
SEED = 42
EPOCHS = 10
MAX_LEN = 96
NUM_SPLITS = 3
LR = 3e-5
BATCH_SIZE = 16
temp_rss = {}
temp_mse = {}
for i in alpha_ridge:
lasso_reg = ElasticNet(alpha= i, normalize=True)
 lasso_reg.fit(X_train, y_train)
 y_pred = lasso_reg.predict(X_test)
 mse = mean_squared_error(y_test, y_pred)
 rss = sum(( y_pred-y_test)**2)
 temp_mse[i] = mse
temp_rss[i] = rss<sort_values>
os.environ['PYTHONHASHSEED']=str(SEED)
 random.seed(SEED)
 np.random.seed(SEED)
 tf.random.set_seed(SEED )
for key, value in sorted(temp_mse.items() , key=lambda item: item[1]):
print("%s: %s" %(key, value))<sort_values>
train = pd.read_csv(".. /input/contradictory-my-dear-watson-translated-dataset/train_augmented.csv", index_col=["id"])
 test = pd.read_csv(".. /input/contradictory-my-dear-watson/test.csv", index_col=["id"])
 test["label"] = -1
for key, value in sorted(temp_rss.items() , key=lambda item: item[1]):
print("%s: %s" %(key, value))<compute_train_metric>
df = pd.concat([train, test])
 df.loc[df["label"]!=-1, "type"] = "train"
df.loc[df["label"]==-1, "type"] = "test"
kfolds = KFold(n_splits=10, shuffle=True, random_state=42)
 def rmsle(y, y_pred):
return np.sqrt(mean_squared_error(y, y_pred))
def cv_rmse(model, X=X):
rmse = np.sqrt(-cross_val_score(model, X, y, scoring="neg_mean_squared_error", cv=kfolds))
return(rmse )<define_search_space>
del df
gc.collect()
alphas_alt = [14.5, 14.6, 14.7, 14.8, 14.9, 15, 15.1, 15.2, 15.3, 15.4, 15.5]
alphas2 = [5e-05, 0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007, 0.0008]
e_alphas = [0.0001, 0.0002, 0.0003, 0.0004, 0.0005, 0.0006, 0.0007]
e_l1ratio = [0.8, 0.85, 0.9, 0.95, 0.99, 1]<define_search_model>
strategy = tf.distribute.experimental.TPUStrategy(tpu )
ridge = make_pipeline(RobustScaler() , RidgeCV(alphas=alphas_alt, cv=kfolds))
lasso = make_pipeline(RobustScaler() , LassoCV(max_iter=1e7,
alphas=alphas2,
random_state=42,
cv=kfolds))
elasticnet = make_pipeline(RobustScaler() , ElasticNetCV(max_iter=1e7, alphas=e_alphas, cv=kfolds, l1_ratio=e_l1ratio))
svr = make_pipeline(RobustScaler() , SVR(C= 20, epsilon= 0.008, gamma=0.0003,))<choose_model_class>
BATCH_SIZE = BATCH_SIZE*strategy.num_replicas_in_sync
gbr = GradientBoostingRegressor(n_estimators=3000, learning_rate=0.05, max_depth=4, max_features='sqrt', min_samples_leaf=15, min_samples_split=10, loss='huber', random_state =42 )<choose_model_class>
MODEL = 'jplu/tf-xlm-roberta-large'
TOKENIZER = AutoTokenizer.from_pretrained(MODEL )
lightgbm = LGBMRegressor(objective='regression',
num_leaves=4,
learning_rate=0.01,
n_estimators=5000,
max_bin=200,
bagging_fraction=0.75,
bagging_freq=5,
bagging_seed=7,
feature_fraction=0.2,
feature_fraction_seed=7,
verbose=-1,
)<choose_model_class>
def fast_encode(df):
text = df[['premise', 'hypothesis']].values.tolist()
encoded = TOKENIZER.batch_encode_plus(
 text,
pad_to_max_length=True,
max_length=MAX_LEN 
)
 return np.array(encoded["input_ids"] )
xgboost = XGBRegressor(learning_rate=0.01,n_estimators=3460,
max_depth=3, min_child_weight=0,
gamma=0, subsample=0.7,
colsample_bytree=0.7,
objective='reg:linear', nthread=-1,
scale_pos_weight=1, seed=27,
reg_alpha=0.00006 )<choose_model_class>
test_encoded = fast_encode(test)
 test_dataset =(
 tf.data.Dataset 
.from_tensor_slices(test_encoded)
.batch(BATCH_SIZE)
 )
stack_gen = StackingCVRegressor(regressors=(ridge, lasso, elasticnet, xgboost, lightgbm),
meta_regressor=xgboost,
use_features_in_secondary=True )<compute_test_metric>
def mish(x):
return x*tanh(softplus(x))
get_custom_objects() ["mish"] = Activation(mish )
score = cv_rmse(ridge)
 print("Ridge: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(lasso)
 print("LASSO: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(elasticnet)
 print("elastic net: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(svr)
 print("SVR: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(lightgbm)
 print("lightgbm: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() ,)
 ".format(score.mean() , score.std()), datetime.now() ,)
 score = cv_rmse(xgboost)
 print("xgboost: {:.4f}({:.4f})
 ".format(score.mean() , score.std()), datetime.now() , )<train_model>
def create_model(transformer):

 input_ids = Input(shape=(MAX_LEN,), dtype=tf.int32)
 sequence_output = transformer(input_ids)[0]
cls_token = sequence_output[:, 0, :]
cls_token = Dropout(0.3 )(cls_token)
 cls_token = Dense(32, activation='mish' )(cls_token)
 cls_token = Dense(16, activation='mish' )(cls_token)
 out = Dense(3, activation='softmax' )(cls_token)
 optimizer = RectifiedAdam(lr=LR)
 model = Model(inputs=input_ids, outputs=out)
 model.compile(
 optimizer,
loss='sparse_categorical_crossentropy',
metrics=['accuracy'] 
)
 return model
print('START Fit')
 print('stack_gen')
 stack_gen_model = stack_gen.fit(np.array(X), np.array(y))
print('elasticnet')
 elastic_model_full_data = elasticnet.fit(X, y)
 print('Lasso')
 lasso_model_full_data = lasso.fit(X, y)
 print('Ridge')
 ridge_model_full_data = ridge.fit(X, y)
 print('Svr')
 svr_model_full_data = svr.fit(X, y)
 print('xgboost')
 xgb_model_full_data = xgboost.fit(X, y)
 print('lightgbm')
 lgb_model_full_data = lightgbm.fit(X, y )<predict_on_test>
kfold = StratifiedKFold(n_splits=NUM_SPLITS, shuffle=True, random_state=SEED )
1.0 * elastic_model_full_data.predict(X )<predict_on_test>
eas = EarlyStopping(monitor='val_loss', min_delta=0.001, patience=3,
verbose=1, mode='min', baseline=None, restore_best_weights=True )
def blend_models_predict(X):
return(( 0.1 * elastic_model_full_data.predict(X)) + \ 
(0.05 * lasso_model_full_data.predict(X)) + \ 
(0.2 * ridge_model_full_data.predict(X)) + \ 
(0.1 * svr_model_full_data.predict(X)) + \ 
(0.15 * xgb_model_full_data.predict(X)) + \ 
(0.1 * lgb_model_full_data.predict(X)) + \ 
(0.3 * stack_gen_model.predict(np.array(X))))<compute_test_metric>
oof_preds = np.zeros(( len(train)))
 test_preds = np.zeros(( len(test), 3))
print('RMSLE score on train data:')
 print(rmsle(y, blend_models_predict(X)) )<load_from_csv>
for fold,(train_index, valid_index)in enumerate(kfold.split(train, train['label'])) :
tf.tpu.experimental.initialize_tpu_system(tpu)
 print("*"*60)
 print("*"+" "*26+f"FOLD {fold+1}"+" "*26+"*")
 print("*"*60, end="
")
 X_train = train.iloc[train_index, :].reset_index(drop=True)
 X_valid = train.iloc[valid_index, :].reset_index(drop=True)
 y_train = X_train['label'].values
y_valid = X_valid['label'].values
train_encoded = fast_encode(X_train)
 valid_encoded = fast_encode(X_valid)
 train_dataset = tf.data.Dataset.from_tensor_slices(( train_encoded, y_train))
train_dataset = train_dataset.repeat()
train_dataset = train_dataset.batch(BATCH_SIZE, drop_remainder=True)
 train_dataset = train_dataset.prefetch(tf.data.experimental.AUTOTUNE)
 valid_dataset = tf.data.Dataset.from_tensor_slices(( valid_encoded, y_valid))
valid_dataset = valid_dataset.batch(BATCH_SIZE)
 valid_dataset = valid_dataset.prefetch(tf.data.experimental.AUTOTUNE)
 num_steps = len(X_train)//BATCH_SIZE
with strategy.scope() :
transformer_layer = TFXLMRobertaModel.from_pretrained(MODEL)
 model = create_model(transformer_layer)
 history = model.fit(
 train_dataset,
steps_per_epoch=num_steps,
validation_data=valid_dataset,
epochs=EPOCHS,
callbacks=[eas] 
)
 valid_preds = model.predict(valid_dataset)
 oof_preds[valid_index] = valid_preds.argmax(axis=1)
 preds = model.predict(test_dataset)
 test_preds += preds/NUM_SPLITS
print('Predict submission')
 submission = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/sample_submission.csv")
 submission.iloc[:,1] = np.floor(np.expm1(blend_models_predict(X_sub)) )<load_from_csv>
print(f"Accuracy: {accuracy_score(train['label'], oof_preds)}" )
<save_to_csv><EOS> 
submission = pd.read_csv(".. /input/contradictory-my-dear-watson/sample_submission.csv")
 submission['prediction'] = np.argmax(test_preds, axis=1)
 submission.to_csv("submission.csv", index=False )