Kaggle competition: TReNDS Neuroimaging
<SOS> metric: WMAE Kaggle data source: trends-neuroimaging<choose_model_class>
import numpy as np
import pandas as pd
from os.path import join as opj
from tqdm.notebook import tqdm
def initialize_accelerator(ACCELERATOR):
if ACCELERATOR == "TPU":
print("Connecting to TPU")
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print(f"Running on TPU {tpu.master() }")
 except ValueError:
print("Could not connect to TPU")
 tpu = None
if tpu:
try:
print("Initializing TPU")
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print("TPU initialized")
 except :
print("Failed to initialize TPU")
 strategy = tf.distribute.get_strategy()
else:
print("Unable to initialize TPU")
 ACCELERATOR = "GPU"
if ACCELERATOR != "TPU":
print("Using default strategy for CPU and single GPU")
 strategy = tf.distribute.get_strategy()
if ACCELERATOR == "GPU":
print(f"GPUs Available: {len(tf.config.experimental.list_physical_devices('GPU')) }")
 AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print(f"REPLICAS: {REPLICAS}")
 return strategy, AUTO, REPLICAS
STRATEGY, AUTO, REPLICAS = initialize_accelerator('TPU' )<train_model>
%matplotlib inline
sns.set_style('darkgrid')
 sns.set_context('notebook' )
def run_experiments(df, var_stratified, encode_text):
skf = StratifiedKFold(n_splits = CV_SPLIT, shuffle = True, random_state = SEED)
 acc_oof = []
for(fold,(train_index, valid_index)) in enumerate(skf.split(df, df[var_stratified])) :
STRATEGY, AUTO, REPLICAS = initialize_accelerator('TPU')
 K.clear_session()
with STRATEGY.scope() :
model = build_model(MODEL_NAME, MAX_LENGTH, METRICS)
 if fold == 0:
print(model.summary())
 print("
")
 print("
print(f"
print("
X_train = df.iloc[train_index].sample(frac=1)
 X_valid = df.iloc[valid_index]
y_train = to_categorical(X_train['label'].values)
 y_valid = to_categorical(X_valid['label'].values)
 print("
Tokenizing")
 X_train_encoded = encode_text(texts = X_train, tokenizer = TOKENIZER, maxlen = MAX_LENGTH, padding = PADDING)
 X_valid_encoded = encode_text(texts = X_valid, tokenizer = TOKENIZER, maxlen = MAX_LENGTH, padding = PADDING)
 ds_train =(
 tf.data.Dataset 
.from_tensor_slices(( X_train_encoded, y_train)) 
.repeat() 
.shuffle(SEED)
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 ds_valid =(
 tf.data.Dataset 
.from_tensor_slices(( X_valid_encoded, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 n_train = X_train.shape[0]
sv = tf.keras.callbacks.ModelCheckpoint(
 "model.h5",
monitor = 'val_'+METRICS[0],
verbose = 0,
save_best_only = True,
save_weights_only = True,
mode = "max",
save_freq = "epoch" 
)
 print("
Training")
 history = model.fit(
 ds_train,
epochs = EPOCHS,
callbacks = [sv],
steps_per_epoch = n_train // BATCH_SIZE,
validation_data = ds_valid,
verbose = VERBOSE 
)
 model.load_weights("model.h5")
 pred = model.predict(ds_valid)
 acc = accuracy_score(X_valid['label'].values, np.argmax(pred, axis=1))
acc_oof.append(acc)
 print(f"
Fold {fold + 1} Accuracy: {round(acc, 4)}
")
 g = gc.collect()
print(f"
CV Mean Accuracy: {round(np.mean(acc_oof), 4)}")
 print(f"CV StdDev Accuracy: {round(np.std(acc_oof), 4)}
" )<choose_model_class>
from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler, RobustScaler
from sklearn.model_selection import GridSearchCV
from sklearn.pipeline import Pipeline
from sklearn.linear_model import Ridge
from sklearn.svm import SVR
def build_model(model_name, max_len, metrics):
input_ids = Input(shape =(max_len,), dtype = tf.int32, name = "input_ids")
 transformer_model = TFAutoModel.from_pretrained(model_name)
 transformer_embeddings = transformer_model(input_ids)[0]
transformer_token = transformer_embeddings[:, 0, :]
output_values = Dense(3, activation = "softmax" )(transformer_token)
 model = Model(inputs = input_ids, outputs = output_values)
 model.compile(optimizer = Adam(learning_rate = 1e-5),
loss = tf.keras.losses.CategoricalCrossentropy(label_smoothing=0.2),
metrics = metrics)
 return model<feature_engineering>
path = '/kaggle/input/trends-feature-exploration-engineering/datasets'
df_train['predictor'] = " [CLS] " + df_train['premise'] + " [SEP] " + df_train['hypothesis']
df_test['predictor'] = " [CLS] " + df_test['premise'] + " [SEP] " + df_test['hypothesis']<define_variables>
scaler_targets = joblib.load(opj(path, 'targets_scaler.pkl'))
targets = pd.read_hdf(opj(path, 'targets.h5'))
targets.head()
MAX_LENGTH = 75
MODEL_NAME = "distilbert-base-multilingual-cased"
PADDING = True
BATCH_SIZE = 16 * REPLICAS
EPOCHS = 10
CV_SPLIT = 5
SEED = 2020
VERBOSE = 1
METRICS = ["categorical_accuracy"]<load_pretrained>
def load_dataset_and_scale(target, dataset_id='merge', scale_values=[1, 1, 1, 1]):
X_tr = pd.read_hdf(opj(path, '%s_train.h5' % dataset_id))
X_te = pd.read_hdf(opj(path, '%s_test.h5' % dataset_id))
y = pd.read_hdf(opj(path, 'targets.h5'))
y_tr = y.loc[X_tr.index, target]
missval = y_tr.isnull().values
idx = ~missval
X_tr = X_tr[idx]
X_te = X_te
y_tr = y_tr[idx]
print('Removing %s missing values from target dataset.' % missval.sum())
 median_offset = X_tr.iloc[:, X_tr.columns.str.contains('corr_coef')].median().mean()
X_tr.iloc[:, X_tr.columns.str.contains('corr_coef')] -= median_offset
X_te.iloc[:, X_te.columns.str.contains('corr_coef')] -= median_offset
mask_ids = []
for m in ['IC_', '_vs_', 'corr_coef', '^c[0-9]+_c[0-9]+']:
mask_ids.append(X_tr.columns.str.contains(m))
mask_ids = np.array(mask_ids)
 for i, m in enumerate(mask_ids):
if m.sum() ==0:
continue
scale_value = scale_values[i]
unify_mask_scale = np.percentile(X_tr.iloc[:, m].abs() , 90)
 X_te.iloc[:, m] /= unify_mask_scale
X_tr.iloc[:, m] /= unify_mask_scale
X_te.iloc[:, m] *= scale_value
X_tr.iloc[:, m] *= scale_value
X_tr.dropna(axis=1, inplace=True)
 X_te.dropna(axis=1, inplace=True)
 X_tr = X_tr.T.drop_duplicates().T
X_te = X_te.T.drop_duplicates().T
print('Size of dataset(train/test): ', X_tr.shape, X_te.shape)
 X_tr = X_tr.values
X_te = X_te.values
y_tr = y_tr.values
return X_tr, X_te, y_tr
TOKENIZER = AutoTokenizer.from_pretrained(MODEL_NAME )<categorify>
target = 'age'
tidx = 0
scale_values = [1,
1,
np.nan,
np.nan]
X_tr, X_te, y_tr = load_dataset_and_scale(target, scale_values=scale_values )
def original_encode(texts, tokenizer, maxlen, padding):
enc_di = tokenizer.batch_encode_plus(
 texts['predictor'],
return_attention_masks=False,
return_token_type_ids=False,
pad_to_max_length=padding,
max_length=maxlen)
 return enc_di["input_ids"]<train_model>
estimator = Ridge(tol=1e-3)
 alphas = np.logspace(0, 4, 21)
 grid = create_grid(model_metric, alphas=alphas, estimator=estimator,
cv=5, scaler_targets=scaler_targets, tidx=tidx)
 _ = grid.fit(X_tr, y_tr)
 print('Dataset scales used: ', scale_values)
 print("Best score at: %f using %s" %(grid.best_score_, grid.best_params_))
start = time.time()
run_experiments(df_train, 'language_label', original_encode)
 end = time.time()
print('Time used :',(end-start)/60 )<feature_engineering>
def extract_predictions(X_tr, X_te, grid, y_tr):
res = {}
res['tr'] = grid.predict(X_tr)
 res['te'] = grid.predict(X_te)
 unique_values = np.unique(y_tr)
 for t in ['tr', 'te']:
for i, a in enumerate(res[t]):
res[t][i] = unique_values[np.argmin(np.abs(a-unique_values)) ]
return res['tr'], res['te']
df_train_en['predictor'] = " [CLS] " + df_train_en['premise'] + " [SEP] " + df_train_en['hypothesis']
df_test_en['predictor'] = " [CLS] " + df_test_en['premise'] + " [SEP] " + df_test_en['hypothesis']<define_variables>
pred_tr, pred_te = extract_predictions(X_tr, X_te, grid, y_tr )
MAX_LENGTH = 75
MODEL_NAME = "distilbert-base-multilingual-cased"
PADDING = True
BATCH_SIZE = 16 * REPLICAS
EPOCHS = 10
CV_SPLIT = 5
SEED = 2020
VERBOSE = 1
METRICS = ["categorical_accuracy"]<load_pretrained>
def create_df_pred(grid):
df_pred = pd.DataFrame(grid.cv_results_)
 columns = [c for c in df_pred.columns if 'time' not in c
and 'split' not in c
and 'rank' not in c
and c!='params']
df_pred = df_pred[columns].sort_values('mean_test_score', ascending=False)
 df_pred['param_estimator__alpha'] = df_pred['param_estimator__alpha'].astype('float')
 df_pred['param_scaler'] = df_pred['param_scaler'].astype('str')
 return df_pred
TOKENIZER = AutoTokenizer.from_pretrained(MODEL_NAME )<categorify>
df_pred = create_df_pred(grid ).sort_values('mean_test_score')
 df_pred.head()
def original_encode(texts, tokenizer, maxlen, padding):
enc_di = tokenizer.batch_encode_plus(
 texts['predictor'],
return_attention_masks=False,
return_token_type_ids=False,
pad_to_max_length=padding,
max_length=maxlen)
 return enc_di["input_ids"]<train_model>
target = 'age'
tidx = 0
scale_values = [1,
1,
1,
1]
X_tr, X_te, y_tr = load_dataset_and_scale(target, scale_values=scale_values )
start = time.time()
run_experiments(df_train_en, 'label', original_encode)
 end = time.time()
print('Time used :',(end-start)/60 )<feature_engineering>
estimator = Ridge(tol=1e-3)
 alphas = np.logspace(1, 5, 21)
 grid = create_grid(model_metric, alphas=alphas, estimator=estimator,
cv=5, scaler_targets=scaler_targets, tidx=tidx)
 _ = grid.fit(X_tr, y_tr)
 print('Dataset scales used: ', scale_values)
 print("Best score at: %f using %s" %(grid.best_score_, grid.best_params_))
df_train_no_stop['predictor'] = " [CLS] " + df_train_no_stop['premise'] + " [SEP] " + df_train_no_stop['hypothesis']
df_test_no_stop['predictor'] = " [CLS] " + df_test_no_stop['premise'] + " [SEP] " + df_test_no_stop['hypothesis']<define_variables>
pred_tr, pred_te = extract_predictions(X_tr, X_te, grid, y_tr )
MAX_LENGTH = 50
MODEL_NAME = "distilbert-base-multilingual-cased"
PADDING = True
BATCH_SIZE = 16 * REPLICAS
EPOCHS = 10
CV_SPLIT = 5
SEED = 2020
VERBOSE = 1
METRICS = ["categorical_accuracy"]<categorify>
df_pred = create_df_pred(grid ).sort_values('mean_test_score', ascending=False)
 df_pred.head()
def original_encode(texts, tokenizer, maxlen, padding):
enc_di = tokenizer.batch_encode_plus(
 texts['predictor'],
return_attention_masks=False,
return_token_type_ids=False,
pad_to_max_length=padding,
max_length=maxlen)
 return enc_di["input_ids"]<train_model>
def run_prediction(model_metric, estimator=None, alphas=[0.1, 1, 10], cv=5,
scaler_targets=None, target='age', tidx=0, scale_values=None):
X_tr, X_te, y_tr = load_dataset_and_scale(target, scale_values=scale_values)
 grid = create_grid(model_metric, alphas=alphas, estimator=estimator,
cv=cv, scaler_targets=scaler_targets, tidx=tidx)
 _ = grid.fit(X_tr, y_tr)
 print('Dataset scales used: ', scale_values)
 print("Best score at: %f using %s" %(grid.best_score_, grid.best_params_))
pred_tr, pred_te = extract_predictions(X_tr, X_te, grid, y_tr)
 plot_predictions(pred_tr, pred_te, y_tr)
 df_pred = create_df_pred(grid)
 display(df_pred.sort_values('mean_test_score', ascending=False ).head())
 plot_hyperparam_fitting(df_pred)
 return df_pred, pred_tr, pred_te, grid, y_tr
start = time.time()
run_experiments(df_train_no_stop, 'label', original_encode)
 end = time.time()
print('Time used :',(end-start)/60 )<load_pretrained>
target = 'age'
tidx = 0
def build_model_multimodal(model_name, max_len, metrics):
input_ids_prem = Input(shape =(MAX_LENGTH_PREM,), dtype = tf.int32, name = "input_ids_prem")
 input_ids_hyp = Input(shape =(MAX_LENGTH_HYP,), dtype = tf.int32, name = "input_ids_hyp")
 transformer_model_prem = TFAutoModel.from_pretrained(model_name)
 transformer_embeddings_prem = transformer_model_prem(input_ids_prem)[0]
transformer_token_prem = transformer_embeddings_prem[:, 0, :]
transformer_model_hyp = TFAutoModel.from_pretrained(model_name)
 transformer_embeddings_hyp = transformer_model_hyp(input_ids_hyp)[0]
transformer_token_hyp = transformer_embeddings_hyp[:, 0, :]
transformer_concat = Concatenate()([transformer_token_prem, transformer_token_hyp])
 output_values = Dense(3, activation = "softmax" )(transformer_concat)
 model = Model(inputs = [input_ids_prem, input_ids_hyp], outputs = output_values)
 model.compile(optimizer = Adam(learning_rate = 1e-5),
loss = tf.keras.losses.CategoricalCrossentropy(from_logits=True, label_smoothing=0.2),
metrics = metrics)
 return model<categorify>
scale_values = [0.25,
0.04,
np.nan,
np.nan,
]
def original_encode_multimodal(texts, tokenizer, maxlen, padding):
enc_di = tokenizer.batch_encode_plus(
 texts,
return_attention_masks=False,
return_token_type_ids=False,
pad_to_max_length=padding,
max_length=maxlen)
 return enc_di["input_ids"]<categorify>
estimator = Ridge(tol=1e-3)
 alphas = np.logspace(-2, 4, 31)
 cv = 5
def run_experiments_multimodal(df, var_stratified, encode_text):
skf = StratifiedKFold(n_splits = CV_SPLIT, shuffle = True, random_state = SEED)
 acc_oof = []
for(fold,(train_index, valid_index)) in enumerate(skf.split(df, df[var_stratified])) :
STRATEGY, AUTO, REPLICAS = initialize_accelerator('TPU')
 K.clear_session()
with STRATEGY.scope() :
model = build_model_multimodal(MODEL_NAME, MAX_LENGTH_PREM, METRICS)
 if fold == 0:
print(model.summary())
 print("
")
 print("
print(f"
print("
X_train = df.iloc[train_index]
X_valid = df.iloc[valid_index]
y_train = to_categorical(X_train['label'].values)
 y_valid = to_categorical(X_valid['label'].values)
 print("
Tokenizing")
 X_train_encoded_prem = encode_text(texts = X_train['premise'], tokenizer = TOKENIZER, maxlen = MAX_LENGTH_PREM, padding = PADDING)
 X_train_encoded_hyp = encode_text(texts = X_train['hypothesis'], tokenizer = TOKENIZER, maxlen = MAX_LENGTH_HYP, padding = PADDING)
 X_valid_encoded_prem = encode_text(texts = X_valid['premise'], tokenizer = TOKENIZER, maxlen = MAX_LENGTH_PREM, padding = PADDING)
 X_valid_encoded_hyp = encode_text(texts = X_valid['hypothesis'], tokenizer = TOKENIZER, maxlen = MAX_LENGTH_HYP, padding = PADDING)
 ds_train =(
 tf.data.Dataset 
.from_tensor_slices(( {"input_ids_prem": X_train_encoded_prem, "input_ids_hyp": X_train_encoded_hyp}, y_train)) 
.repeat() 
.shuffle(SEED)
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 ds_valid =(
 tf.data.Dataset 
.from_tensor_slices(( {"input_ids_prem": X_valid_encoded_prem, "input_ids_hyp": X_valid_encoded_hyp}, y_valid)) 
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 n_train = X_train.shape[0]
sv = tf.keras.callbacks.ModelCheckpoint(
 "model.h5",
monitor = 'val_'+METRICS[0],
verbose = 0,
save_best_only = True,
save_weights_only = True,
mode = "max",
save_freq = "epoch" 
)
 print("
Training")
 history = model.fit(
 ds_train,
epochs = EPOCHS,
callbacks = [sv],
steps_per_epoch = n_train // BATCH_SIZE,
validation_data = ds_valid,
verbose = VERBOSE 
)
 model.load_weights("model.h5")
 pred = model.predict(ds_valid)
 acc = accuracy_score(X_valid['label'].values, np.argmax(pred, axis=1))
acc_oof.append(acc)
 print(f"
Fold {fold + 1} Accuracy: {round(acc, 4)}
")
 g = gc.collect()
print(f"
CV Mean Accuracy: {round(np.mean(acc_oof), 4)}")
 print(f"CV StdDev Accuracy: {round(np.std(acc_oof), 4)}
" )<define_variables>
df_pred, pred_tr, pred_te, grid, y_tr = run_prediction(
 model_metric, estimator=estimator, alphas=alphas, cv=cv,
scaler_targets=scaler_targets, target=target, tidx=tidx,
scale_values=scale_values )
MAX_LENGTH_PREM = 50
MAX_LENGTH_HYP = 25
MODEL_NAME = "distilbert-base-multilingual-cased"
PADDING = True
BATCH_SIZE = 16 * REPLICAS
EPOCHS = 10
CV_SPLIT = 5
SEED = 2020
VERBOSE = 1
METRICS = ["categorical_accuracy"]<load_pretrained>
target = 'age'
tidx = 0
TOKENIZER = AutoTokenizer.from_pretrained(MODEL_NAME )<categorify>
scale_values = [0.25,
0.04,
0.087,
np.nan,
]
start = time.time()
run_experiments_multimodal(df_train_en, 'label', original_encode_multimodal)
 end = time.time()
print('Time used :',(end-start)/60 )<define_variables>
estimator = Ridge(tol=1e-3)
 alphas = np.logspace(-2, 4, 31)
 cv = 5
MAX_LENGTH = 75
MODEL_NAME = "jplu/tf-xlm-roberta-large"
PADDING = True
BATCH_SIZE = 16 * REPLICAS
EPOCHS = 10
CV_SPLIT = 5
SEED = 2020
VERBOSE = 1
METRICS = ["categorical_accuracy"]<load_pretrained>
df_pred, pred_tr, pred_te, grid, y_tr = run_prediction(
 model_metric, estimator=estimator, alphas=alphas, cv=cv,
scaler_targets=scaler_targets, target=target, tidx=tidx,
scale_values=scale_values )
TOKENIZER = AutoTokenizer.from_pretrained(MODEL_NAME )<categorify>
target = 'age'
tidx = 0
y_train = to_categorical(df_train_en['label'].values)
 X_train_encoded = original_encode(texts = df_train_en, tokenizer = TOKENIZER, maxlen = MAX_LENGTH, padding = PADDING)
 X_test_encoded = original_encode(texts = df_test_en, tokenizer = TOKENIZER, maxlen = MAX_LENGTH, padding = PADDING )<prepare_x_and_y>
scale_values = [0.25,
0.04,
0.087,
0.025,
]
ds_train =(
 tf.data.Dataset 
.from_tensor_slices(( X_train_encoded, y_train)) 
.repeat() 
.shuffle(SEED)
.batch(BATCH_SIZE)
.cache() 
.prefetch(AUTO)
)
 ds_test =(
 tf.data.Dataset 
.from_tensor_slices(( X_test_encoded)) 
.batch(BATCH_SIZE)
.prefetch(AUTO)
 )<choose_model_class>
estimator = Ridge(tol=1e-3)
 alphas = np.logspace(-2, 4, 31)
 cv = 5
K.clear_session()
with STRATEGY.scope() :
model = build_model(MODEL_NAME, MAX_LENGTH, METRICS )<train_model>
df_pred, pred_tr, pred_te, grid, y_tr = run_prediction(
 model_metric, estimator=estimator, alphas=alphas, cv=cv,
scaler_targets=scaler_targets, target=target, tidx=tidx,
scale_values=scale_values )
n_train = df_train_en.shape[0]
history = model.fit(
 ds_train,
epochs = EPOCHS,
steps_per_epoch = n_train // BATCH_SIZE,
verbose = VERBOSE
)<predict_on_test>
predictions_ridge = {}
predictions_ridge[target] = pred_te
pred = model.predict(ds_test )<save_to_csv>
target = 'domain1_var1'
tidx = 1
sub = pd.read_csv('.. /input/contradictory-my-dear-watson/sample_submission.csv')
 sub['prediction'] = np.argmax(pred, axis=1)
 sub.to_csv('submission.csv', index=False )<find_best_params>
scale_values = [0.25,
0.01,
0.032,
0.019,
]
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.cluster_spec().as_dict() ['worker'])
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()<install_modules>
estimator = Ridge(tol=1e-3)
 alphas = np.logspace(-1, 5, 31)
 cv = 5
!pip install nlp
!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py
!python pytorch-xla-env-setup.py --version nightly --apt-packages libomp5 libopenblas-dev<set_options>
df_pred, pred_tr, pred_te, grid, y_tr = run_prediction(
 model_metric, estimator=estimator, alphas=alphas, cv=cv,
scaler_targets=scaler_targets, target=target, tidx=tidx,
scale_values=scale_values )
%%time
%autosave 60
os.environ['XLA_USE_BF16'] = "1"
os.environ['XLA_TENSOR_ALLOCATOR_MAXSIZE'] = '100000000'
gc.enable()
XLMRobertaTokenizer,
XLMRobertaModel,
get_cosine_schedule_with_warmup)
 warnings.filterwarnings("ignore")
 print('PYTORCH:', xv.__torch_gitrev__)
 print('XLA:', xv.__xla_gitrev__ )<load_from_csv>
predictions_ridge[target] = pred_te
test = pd.read_csv('.. /input/contradictory-my-dear-watson/test.csv')
 sample_submission = pd.read_csv('.. /input/contradictory-my-dear-watson/sample_submission.csv' )<load_pretrained>
target = 'domain1_var2'
tidx = 2
TRAIN_BATCH_SIZE = 16
VALID_BATCH_SIZE = 16
EPOCHS = 3
MAX_LEN = 80
LR = 2e-5 * xm.xrt_world_size()
METRICS_DEBUG = True
tokenizer = XLMRobertaTokenizer.from_pretrained('xlm-roberta-large' )<categorify>
scale_values = [0.121,
0.019,
0.032,
0.025,
]
def convert_to_features(batch):
input_pairs = list(zip(batch['premise'], batch['hypothesis']))
encodings = tokenizer.batch_encode_plus(input_pairs,
add_special_tokens=True,
pad_to_max_length=True,
max_length=MAX_LEN,
truncation=True,
return_attention_mask=True,
return_token_type_ids=True)
 return encodings<load_from_csv>
estimator = Ridge(tol=1e-3)
 alphas = np.logspace(-1, 5, 31)
 cv = 5
test_dataset = nlp.load_dataset('csv', data_files=['.. /input/contradictory-my-dear-watson/test.csv'])['train']
drop_columns = test_dataset.column_names
encoded_test_dataset = test_dataset.map(convert_to_features, batched=True, remove_columns=drop_columns)
 encoded_test_dataset.set_format("torch", columns=['attention_mask', 'input_ids', 'token_type_ids'] )<load_pretrained>
df_pred, pred_tr, pred_te, grid, y_tr = run_prediction(
 model_metric, estimator=estimator, alphas=alphas, cv=cv,
scaler_targets=scaler_targets, target=target, tidx=tidx,
scale_values=scale_values )
class XLMRoberta(nn.Module):
def __init__(self, num_labels, multisample):
super(XLMRoberta, self ).__init__()
output_hidden_states = False
self.num_labels = num_labels
self.multisample= multisample
self.roberta = XLMRobertaModel.from_pretrained("xlm-roberta-large",
output_hidden_states=output_hidden_states,
num_labels=1)
 self.layer_norm = nn.LayerNorm(1024*2)
 self.dropout = nn.Dropout(p=0.2)
 self.high_dropout = nn.Dropout(p=0.5)
 self.classifier = nn.Linear(1024*2, self.num_labels)
 def forward(self,
input_ids=None,
attention_mask=None,
token_type_ids=None,
position_ids=None,
head_mask=None,
inputs_embeds=None):
outputs = self.roberta(input_ids,
attention_mask=attention_mask,
token_type_ids=token_type_ids,
position_ids=position_ids,
head_mask=head_mask,
inputs_embeds=inputs_embeds)
 average_pool = torch.mean(outputs[0], 1)
 max_pool, _ = torch.max(outputs[0], 1)
 concatenate_layer = torch.cat(( average_pool, max_pool), 1)
 normalization = self.layer_norm(concatenate_layer)
 if self.multisample:
logits = torch.mean(
 torch.stack(
 [self.classifier(self.dropout(normalization)) for _ in range(5)],
dim=0, 
),
dim=0, 
)
 else:
logits = self.dropout(normalization)
 logits = self.classifier(logits)
 outputs = logits
return outputs<load_pretrained>
predictions_ridge[target] = pred_te
TEST_BATCH_SIZE = 32
test_data_loader = DataLoader(
 encoded_test_dataset,
batch_size=TEST_BATCH_SIZE,
drop_last=False,
num_workers=4,
shuffle=False 
)
 WRAPPED_MODEL = xmp.MpModelWrapper(XLMRoberta(num_labels=3, multisample=False))
device = xm.xla_device()
model = WRAPPED_MODEL.to(device ).eval()
model.load_state_dict(torch.load(".. /input/contradictorywatsonpublicxlmroberta/model.bin"))<categorify>
target = 'domain2_var1'
tidx = 3
test_preds = []
for i, data in tqdm(enumerate(test_data_loader), total=len(test_data_loader)) :
ids = data["input_ids"]
mask = data["attention_mask"]
type_ids = data["token_type_ids"]
ids = ids.to(device, dtype=torch.long)
 mask = mask.to(device, dtype=torch.long)
 type_ids = type_ids.to(device, dtype=torch.long)
 outputs = model(
 input_ids = ids,
attention_mask = mask,
token_type_ids = type_ids 
)
 outputs_np = outputs.cpu().detach().numpy().tolist()
test_preds.extend(outputs_np )<save_to_csv>
scale_values = [0.25,
0.008,
0.012,
0.012,
]
test_preds = torch.FloatTensor(test_preds)
 top1_prob, top1_label = torch.topk(test_preds, 1)
 y = top1_label.cpu().detach().numpy()
sample_submission.prediction = y
sample_submission.to_csv('submission.csv', index=False )<set_options>
estimator = Ridge(tol=1e-3)
 alphas = np.logspace(-1, 5, 31)
 cv = 5
os.environ['WANDB_API_KEY'] = '0'
warnings.filterwarnings("ignore" )<define_variables>
df_pred, pred_tr, pred_te, grid, y_tr = run_prediction(
 model_metric, estimator=estimator, alphas=alphas, cv=cv,
scaler_targets=scaler_targets, target=target, tidx=tidx,
scale_values=scale_values )
DATA_PATH = '/kaggle/input/contradictory-my-dear-watson-eng/'
MODEL_PATH = '/kaggle/input/tf-roberta/'<set_options>
predictions_ridge[target] = pred_te
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 print('Running on TPU ', tpu.master())
 except ValueError:
strategy = tf.distribute.get_strategy()
print('Number of replicas:', strategy.num_replicas_in_sync )<load_from_csv>
target = 'domain2_var2'
tidx = 4
train = pd.read_csv(DATA_PATH + '/trans_train.csv', index_col=False)
 test = pd.read_csv(DATA_PATH + '/trans_test.csv', index_col=False )<feature_engineering>
scale_values = [0.261,
0.025,
0.052,
0.022,
]
def basic_clean(row):
row['hypothesis'] = re.sub(' +', ' ', row['hypothesis'] ).strip().lower()
row['premise'] = re.sub(' +', ' ', row['premise'] ).strip().lower()
return row
train = train[train['hypothesis'].notna() ]
train = train[train['premise'].notna() ]
test = test[test['hypothesis'].notna() ]
test = test[test['premise'].notna() ]
train = train.apply(basic_clean, axis=1 ).reset_index()
test = test.apply(basic_clean, axis=1 ).reset_index()<choose_model_class>
estimator = Ridge(tol=1e-3)
 alphas = np.logspace(-1, 5, 31)
 cv = 5
MAX_LEN = 256
tokenizer = tokenizers.ByteLevelBPETokenizer(
 vocab_file=MODEL_PATH + 'vocab-roberta-base.json',
merges_file=MODEL_PATH + 'merges-roberta-base.txt',
lowercase=True,
add_prefix_space=True
)<categorify>
df_pred, pred_tr, pred_te, grid, y_tr = run_prediction(
 model_metric, estimator=estimator, alphas=alphas, cv=cv,
scaler_targets=scaler_targets, target=target, tidx=tidx,
scale_values=scale_values )
def encode_sentence(s):
return tokenizer.encode(s )<categorify>
predictions_ridge[target] = pred_te
encode_sentence('I love machine learning' )<categorify>
def create_df_pred(grid):
df_pred = pd.DataFrame(grid.cv_results_)
 columns = [c for c in df_pred.columns if 'time' not in c
and 'split' not in c
and 'rank' not in c
and c!='params']
df_pred = df_pred[columns].sort_values('mean_test_score', ascending=False)
 df_pred['param_estimator__C'] = df_pred['param_estimator__C'].astype('float')
 return df_pred
def roberta_encode(df, tokenizer):
ct = df.shape[0]
input_ids = np.ones(( ct, MAX_LEN), dtype='int32')
 attention_mask = np.zeros(( ct, MAX_LEN), dtype='int32')
 token_type_ids = np.zeros(( ct, MAX_LEN), dtype='int32')
 for k, row in df.iterrows() :
enc_hypothesis = tokenizer.encode(row['hypothesis'])
 enc_premise = tokenizer.encode(row['premise'])
 input_length = len(enc_hypothesis.ids)+ len(enc_premise.ids)+ 4
if input_length > MAX_LEN:
continue
input_ids[k,:input_length] = [0] + enc_hypothesis.ids + [2,2] + enc_premise.ids + [2]
attention_mask[k,:input_length] = 1
type_sep = np.zeros_like([0])
 type_s1 = np.zeros_like(enc_hypothesis.ids)
 type_s2 = np.ones_like(enc_premise.ids)
 z = [type_sep, type_s1, type_sep, type_sep, type_s2, type_sep]
token_type_ids[k,:input_length] = np.concatenate(z)
 return {
'input_word_ids': input_ids,
'input_mask': attention_mask,
'input_type_ids': token_type_ids
}<categorify>
target = 'age'
tidx = 0
train_input = roberta_encode(train, tokenizer)
 test_input = roberta_encode(test, tokenizer )<load_pretrained>
scale_values = [0.25,
0.015,
0.042,
0.014,
]
def build_model() :
input_word_ids = tf.keras.Input(shape=(MAX_LEN,), dtype=tf.int32, name='input_word_ids')
 input_mask = tf.keras.Input(shape=(MAX_LEN,), dtype=tf.int32, name='input_mask')
 input_type_ids = tf.keras.Input(shape=(MAX_LEN,), dtype=tf.int32, name='input_type_ids')
 config = RobertaConfig.from_pretrained(MODEL_PATH + 'config-roberta-base.json')
 bert_model = TFRobertaModel.from_pretrained(MODEL_PATH + 'pretrained-roberta-base.h5', config=config)
 x = bert_model(input_word_ids, attention_mask=input_mask, token_type_ids=input_type_ids)
 x = x[0]
x = tf.keras.layers.Dropout(0.1 )(x)
 x = tf.keras.layers.Conv1D(1, 1 )(x)
 x = tf.keras.layers.Flatten()(x)
 x = tf.keras.layers.Dense(3, activation='softmax' )(x)
 model = tf.keras.Model(inputs=[input_word_ids, input_mask, input_type_ids], outputs=x)
 model.compile(
 optimizer=tf.keras.optimizers.Adam(lr=1e-5),
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
 return model<choose_model_class>
estimator = SVR(gamma='scale', epsilon=0.2, tol=1e-3, max_iter=5000)
 Cs = np.logspace(-1, 1, 11)
 cv = 5
accuracy = []
history = []
VER = 'v0'
EPOCHS = 4
KFOLDS = 5
BATCH_SIZE= 16 * strategy.num_replicas_in_sync
pred_test = np.zeros(( test.shape[0], 3))
skf = StratifiedKFold(n_splits=KFOLDS, shuffle=True, random_state=42)
 for fold,(idxT, idxV)in enumerate(skf.split(train_input['input_word_ids'], train.label.values)) :
print('
print('
print('
K.clear_session()
with strategy.scope() :
print('Building model...')
 model = build_model()
sv = tf.keras.callbacks.ModelCheckpoint(
 '%s-roberta-%i.h5' %(VER, fold),
monitor='val_loss',
verbose=1,
save_best_only=True,
save_weights_only=True,
mode='auto',
save_freq='epoch')
 kfold_train_input = {
'input_word_ids': train_input['input_word_ids'][idxT,],
'input_mask': train_input['input_mask'][idxT,],
'input_type_ids': train_input['input_type_ids'][idxT,]}
kfold_train_output = train.label.values[idxT,]
kfold_val_input = {
'input_word_ids': train_input['input_word_ids'][idxV,],
'input_mask': train_input['input_mask'][idxV,],
'input_type_ids': train_input['input_type_ids'][idxV,]}
kfold_val_output = train.label.values[idxV,]
print('Training...')
 kfold_history = model.fit(kfold_train_input,
kfold_train_output,
epochs=EPOCHS,
batch_size=BATCH_SIZE,
verbose=1,
callbacks=[sv],
validation_data=(kfold_val_input, kfold_val_output))
history.append(kfold_history)
 print('Loading model...')
 model.load_weights('%s-roberta-%i.h5' %(VER, fold))
print('Predicting Test...')
 pred_test += model.predict(test_input)/ skf.n_splits
print('Predicting OOF...')
 oof = [np.argmax(i)for i in model.predict(kfold_val_input)]
kfold_accuracy = accuracy_score(oof, kfold_val_output)
 accuracy.append(kfold_accuracy)
 print('> FOLD %i - Accuracy: %.4f' %(fold + 1, kfold_accuracy))
print()<compute_test_metric>
df_pred, pred_tr, pred_te, grid, y_tr = run_prediction(
 model_metric, estimator=estimator, alphas=Cs, cv=cv,
scaler_targets=scaler_targets, target=target, tidx=tidx,
scale_values=scale_values )
print('> OVERALL KFold CV Accuracy: %.4f' % np.mean(accuracy))<create_dataframe>
predictions_svr = {}
predictions_svr[target] = pred_te
submission = test.id.copy().to_frame()
submission['prediction'] = [np.argmax(i)for i in pred_test]<save_to_csv>
target = 'domain1_var1'
tidx = 1
submission.to_csv('submission.csv', index=False )<save_to_csv>
scale_values = [0.25,
0.012,
0.032,
0.018,
]
submission.to_csv('submission.csv', index=False )<import_modules>
estimator = SVR(gamma='scale', epsilon=0.2, tol=1e-3, max_iter=5000)
 Cs = np.logspace(-1, 1, 11)
 cv = 5
print(tf.__version__ )<set_options>
df_pred, pred_tr, pred_te, grid, y_tr = run_prediction(
 model_metric, estimator=estimator, alphas=Cs, cv=cv,
scaler_targets=scaler_targets, target=target, tidx=tidx,
scale_values=scale_values )
def seed_everything(seed=0):
np.random.seed(seed)
 tf.random.set_seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 os.environ['TF_DETERMINISTIC_OPS'] = '1'
sns.set()
SEED = 17
seed_everything(SEED)
 warnings.filterwarnings("ignore" )<find_best_params>
predictions_svr[target] = pred_te
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print('Running on TPU ', tpu.master())
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
print("REPLICAS: ", strategy.num_replicas_in_sync )<define_variables>
target = 'domain1_var2'
tidx = 2
HEIGHT, WIDTH =(512, 512)
 CHANNELS = 3
N_CLASSES = 104
GCS_PATH = KaggleDatasets().get_gcs_path('tpu-getting-started')+ '/tfrecords-jpeg-%sx%s' %(HEIGHT, WIDTH)
 TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train/*.tfrec')
 VALIDATION_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')
 TEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test/*.tfrec')
 CLASSES = [
'pink primrose', 'hard-leaved pocket orchid', 'canterbury bells', 'sweet pea',
'wild geranium', 'tiger lily', 'moon orchid', 'bird of paradise', 'monkshood',
'globe thistle', 'snapdragon', "colt's foot", 'king protea', 'spear thistle',
'yellow iris', 'globe-flower', 'purple coneflower', 'peruvian lily',
'balloon flower', 'giant white arum lily', 'fire lily', 'pincushion flower',
'fritillary', 'red ginger', 'grape hyacinth', 'corn poppy',
'prince of wales feathers', 'stemless gentian', 'artichoke', 'sweet william',
'carnation', 'garden phlox', 'love in the mist', 'cosmos', 'alpine sea holly',
'ruby-lipped cattleya', 'cape flower', 'great masterwort', 'siam tulip',
'lenten rose', 'barberton daisy', 'daffodil', 'sword lily', 'poinsettia',
'bolero deep blue', 'wallflower', 'marigold', 'buttercup', 'daisy',
'common dandelion', 'petunia', 'wild pansy', 'primula', 'sunflower',
'lilac hibiscus', 'bishop of llandaff', 'gaura', 'geranium', 'orange dahlia',
'pink-yellow dahlia', 'cautleya spicata', 'japanese anemone', 'black-eyed susan',
'silverbush', 'californian poppy', 'osteospermum', 'spring crocus', 'iris',
'windflower', 'tree poppy', 'gazania', 'azalea', 'water lily', 'rose',
'thorn apple', 'morning glory', 'passion flower', 'lotus', 'toad lily',
'anthurium', 'frangipani', 'clematis', 'hibiscus', 'columbine', 'desert-rose',
'tree mallow', 'magnolia', 'cyclamen ', 'watercress', 'canna lily',
'hippeastrum ', 'bee balm', 'pink quill', 'foxglove', 'bougainvillea',
'camellia', 'mallow', 'mexican petunia', 'bromelia', 'blanket flower',
'trumpet creeper', 'blackberry lily', 'common tulip', 'wild rose']<categorify>
scale_values = [0.18,
0.01,
np.nan,
0.025,
]
AUTO = tf.data.experimental.AUTOTUNE
def decode_image(image_data):
image = tf.image.decode_jpeg(image_data, channels=3)
 image = tf.cast(image, tf.float32)/ 255.0
image = tf.reshape(image, [HEIGHT, WIDTH, 3])
 return image
def read_labeled_tfrecord(example):
LABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"class": tf.io.FixedLenFeature([], tf.int64),
}
example = tf.io.parse_single_example(example, LABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 label = tf.cast(example['class'], tf.int32)
 return image, label
def read_unlabeled_tfrecord(example):
UNLABELED_TFREC_FORMAT = {
"image": tf.io.FixedLenFeature([], tf.string),
"id": tf.io.FixedLenFeature([], tf.string),
}
example = tf.io.parse_single_example(example, UNLABELED_TFREC_FORMAT)
 image = decode_image(example['image'])
 idnum = example['id']
return image, idnum
def load_dataset(filenames, labeled=True, ordered=False):
ignore_order = tf.data.Options()
if not ordered:
ignore_order.experimental_deterministic = False
dataset = tf.data.TFRecordDataset(filenames, num_parallel_reads=AUTO)
 dataset = dataset.with_options(ignore_order)
 dataset = dataset.map(read_labeled_tfrecord if labeled else read_unlabeled_tfrecord, num_parallel_calls=AUTO)
 return dataset
def data_augment(image, label):
image = tf.image.random_crop(image, size=[480, 480, CHANNELS])
 image = tf.image.random_flip_left_right(image, seed=SEED)
 return image, label
def get_training_dataset() :
dataset = load_dataset(TRAINING_FILENAMES, labeled=True)
 dataset = dataset.map(data_augment, num_parallel_calls=AUTO)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_validation_dataset(ordered=False):
dataset = load_dataset(VALIDATION_FILENAMES, labeled=True, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.cache()
dataset = dataset.prefetch(AUTO)
 return dataset
def get_train_valid_datasets() :
dataset = load_dataset(TRAINING_FILENAMES + VALIDATION_FILENAMES, labeled=True)
 dataset = dataset.map(data_augment, num_parallel_calls=AUTO)
 dataset = dataset.repeat()
dataset = dataset.shuffle(2048)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def get_test_dataset(ordered=False):
dataset = load_dataset(TEST_FILENAMES, labeled=False, ordered=ordered)
 dataset = dataset.batch(BATCH_SIZE)
 dataset = dataset.prefetch(AUTO)
 return dataset
def count_data_items(filenames):
n = [int(re.compile(r"-([0-9]*)\." ).search(filename ).group(1)) for filename in filenames]
return np.sum(n )<categorify>
estimator = SVR(gamma='scale', epsilon=0.2, tol=1e-3, max_iter=5000)
 Cs = np.logspace(-1.5, 0.5, 11)
 cv = 5
np.set_printoptions(threshold=15, linewidth=80)
 def batch_to_numpy_images_and_labels(data):
images, labels = data
numpy_images = images.numpy()
numpy_labels = labels.numpy()
if numpy_labels.dtype == object:
numpy_labels = [None for _ in enumerate(numpy_images)]
return numpy_images, numpy_labels
def title_from_label_and_target(label, correct_label):
if correct_label is None:
return CLASSES[label], True
correct =(label == correct_label)
 return "{} [{}{}{}]".format(CLASSES[label], 'OK' if correct else 'NO', u"\u2192" if not correct else '',
CLASSES[correct_label] if not correct else ''), correct
def display_one_flower(image, title, subplot, red=False, titlesize=16):
plt.subplot(*subplot)
 plt.axis('off')
 plt.imshow(image)
 if len(title)> 0:
plt.title(title, fontsize=int(titlesize)if not red else int(titlesize/1.2), color='red' if red else 'black', fontdict={'verticalalignment':'center'}, pad=int(titlesize/1.5))
return(subplot[0], subplot[1], subplot[2]+1)
 def display_batch_of_images(databatch, predictions=None):

 images, labels = batch_to_numpy_images_and_labels(databatch)
 if labels is None:
labels = [None for _ in enumerate(images)]
rows = int(math.sqrt(len(images)))
 cols = len(images)//rows
FIGSIZE = 13.0
SPACING = 0.1
subplot=(rows,cols,1)
 if rows < cols:
plt.figure(figsize=(FIGSIZE,FIGSIZE/cols*rows))
else:
plt.figure(figsize=(FIGSIZE/rows*cols,FIGSIZE))
for i,(image, label)in enumerate(zip(images[:rows*cols], labels[:rows*cols])) :
title = '' if label is None else CLASSES[label]
correct = True
if predictions is not None:
title, correct = title_from_label_and_target(predictions[i], label)
 dynamic_titlesize = FIGSIZE*SPACING/max(rows,cols)*40+3
subplot = display_one_flower(image, title, subplot, not correct, titlesize=dynamic_titlesize)
 plt.tight_layout()
if label is None and predictions is None:
plt.subplots_adjust(wspace=0, hspace=0)
 else:
plt.subplots_adjust(wspace=SPACING, hspace=SPACING)
 plt.show()<count_values>
df_pred, pred_tr, pred_te, grid, y_tr = run_prediction(
 model_metric, estimator=estimator, alphas=Cs, cv=cv,
scaler_targets=scaler_targets, target=target, tidx=tidx,
scale_values=scale_values )
NUM_TRAINING_IMAGES = count_data_items(TRAINING_FILENAMES)
 NUM_VALIDATION_IMAGES = count_data_items(VALIDATION_FILENAMES)
 NUM_TEST_IMAGES = count_data_items(TEST_FILENAMES)
 print('Number of training images %d' % NUM_TRAINING_IMAGES)
 print('Number of validation images %d' % NUM_VALIDATION_IMAGES)
 print('Number of test images %d' % NUM_TEST_IMAGES )<define_variables>
predictions_svr[target] = pred_te
dataset = load_dataset(TRAINING_FILENAMES, labeled=True)
 one_element = tf.data.Dataset.from_tensors(next(iter(dataset)))
 augmented_element = one_element.repeat().map(data_augment ).batch(25)
 display_batch_of_images(next(iter(augmented_element)) )<choose_model_class>
target = 'domain2_var1'
tidx = 3
class BiTModel(tf.keras.Model):
def __init__(self, num_classes, module):
super().__init__()
self.num_classes = num_classes
self.head = tf.keras.layers.Dense(num_classes, kernel_initializer='zeros')
 self.bit_model = module
def call(self, images):
bit_embedding = self.bit_model(images)
 return self.head(bit_embedding )<define_variables>
scale_values = [0.25,
0.025,
0.036,
0.023,
]
LR = 3e-3
SCHEDULE_LENGTH = 500
STEPS_PER_EPOCH = 50
BATCH_SIZE = 32 * strategy.num_replicas_in_sync
LR = 0.003 * BATCH_SIZE / 512
SCHEDULE_LENGTH = SCHEDULE_LENGTH * 512 / BATCH_SIZE
EPOCHS = int(SCHEDULE_LENGTH / STEPS_PER_EPOCH )<load_pretrained>
estimator = SVR(gamma='scale', epsilon=0.2, tol=1e-3, max_iter=5000)
 Cs = np.logspace(-1, 1, 11)
 cv = 5
with strategy.scope() :
GCS_PATH_TO_SAVEDMODEL = KaggleDatasets().get_gcs_path('bit-tf-hub-models')+ '/BiT TF Hub Models/bit_m-r101x1_1'
loaded_model = tf.saved_model.load(GCS_PATH_TO_SAVEDMODEL)
 model = BiTModel(num_classes=N_CLASSES, module=loaded_model )<choose_model_class>
df_pred, pred_tr, pred_te, grid, y_tr = run_prediction(
 model_metric, estimator=estimator, alphas=Cs, cv=cv,
scaler_targets=scaler_targets, target=target, tidx=tidx,
scale_values=scale_values )
SCHEDULE_BOUNDARIES = [int(SCHEDULE_LENGTH * 0.3), int(SCHEDULE_LENGTH * 0.6), int(SCHEDULE_LENGTH * 0.9)]
lr_schedule = tf.keras.optimizers.schedules \ 
.PiecewiseConstantDecay(
 boundaries=SCHEDULE_BOUNDARIES, values=[LR, LR*0.1, LR*0.001, LR*0.0001] 
)
 optimizer = tf.keras.optimizers.SGD(learning_rate=lr_schedule, momentum=0.9)
 loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
 model.compile(
 optimizer=optimizer,
loss = loss_fn,
metrics=['sparse_categorical_accuracy']
)<train_model>
predictions_svr[target] = pred_te
history = model.fit(x=get_training_dataset() ,
steps_per_epoch=STEPS_PER_EPOCH,
validation_data=get_validation_dataset() ,
epochs=EPOCHS, verbose=1 )<define_variables>
target = 'domain2_var2'
tidx = 4
val_ds = get_validation_dataset(ordered=True)
 val_images_ds = val_ds.map(lambda image, label: image)
 val_labels_ds = val_ds.map(lambda image, label: label ).unbatch()
val_correct_labels = next(iter(val_labels_ds.batch(NUM_VALIDATION_IMAGES)) ).numpy()
test_ds = get_test_dataset(ordered=True)
 test_images_ds = test_ds.map(lambda image, idnum: image )<compute_train_metric>
scale_values = [0.189,
0.025,
0.050,
0.022,
]
val_probabilities = model.predict(val_images_ds)
 val_predictions = np.argmax(val_probabilities, axis=-1)
 score_sch = f1_score(val_correct_labels, val_predictions, labels=range(len(CLASSES)) , average='macro')
 precision_sch = precision_score(val_correct_labels, val_predictions, labels=range(len(CLASSES)) , average='macro')
 recall_sch = recall_score(val_correct_labels, val_predictions, labels=range(len(CLASSES)) , average='macro')
 test_probabilities = model.predict(test_images_ds)
 print('f1 score: {:.3f}, precision: {:.3f}, recall: {:.3f}'.format(score_sch, precision_sch, recall_sch))<save_to_csv>
estimator = SVR(gamma='scale', epsilon=0.2, tol=1e-3, max_iter=5000)
 Cs = np.logspace(-1, 1, 11)
 cv = 5
print('[INFO] Computing predictions...')
 probabilities = test_probabilities
predictions = np.argmax(probabilities, axis=-1)
 print('[INFO] Generating submission.csv file...')
 test_ids_ds = test_ds.map(lambda image, idnum: idnum ).unbatch()
test_ids = next(iter(test_ids_ds.batch(NUM_TEST_IMAGES)) ).numpy().astype('U')
 np.savetxt('submission.csv', np.rec.fromarrays([test_ids, predictions]), fmt=['%s', '%d'], delimiter=',', header='id,label', comments='' )<set_options>
df_pred, pred_tr, pred_te, grid, y_tr = run_prediction(
 model_metric, estimator=estimator, alphas=Cs, cv=cv,
scaler_targets=scaler_targets, target=target, tidx=tidx,
scale_values=scale_values )
!pip install --quiet efficientnet
def seed_everything(seed=0):
random.seed(seed)
 np.random.seed(seed)
 tf.random.set_seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 os.environ['TF_DETERMINISTIC_OPS'] = '1'
seed = 0
seed_everything(seed)
 warnings.filterwarnings("ignore" )<set_options>
predictions_svr[target] = pred_te
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print(f'Running on TPU {tpu.master() }')
 except ValueError:
tpu = None
if tpu:
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 else:
strategy = tf.distribute.get_strategy()
AUTO = tf.data.experimental.AUTOTUNE
REPLICAS = strategy.num_replicas_in_sync
print(f'REPLICAS: {REPLICAS}' )<define_variables>
submission = pd.read_csv(opj('/kaggle', 'input', 'trends-assessment-prediction', 'sample_submission.csv')).set_index('Id')
 submission.head()
BATCH_SIZE = 16 * REPLICAS
WARMUP_EPOCHS = 3
WARMUP_LEARNING_RATE = 1e-4 * REPLICAS
EPOCHS = 20
LEARNING_RATE = 3e-5 * REPLICAS
HEIGHT = 512
WIDTH = 512
CHANNELS = 3
N_CLASSES = 104
ES_PATIENCE = 5<define_variables>
def back_transform(y_test, unique_values, scaler=None, tidx=0):
pow_age = 1.0
pow_d1v1 = 1.5
pow_d1v2 = 1.5
pow_d2v1 = 1.5
pow_d2v2 = 1.5
pow_d21 = 1.5
pow_d22 = 1.0
powers = [pow_age, pow_d1v1, pow_d1v2, pow_d2v1, pow_d2v2, pow_d21, pow_d22]
for i, a in enumerate(y_test):
y_test[i] = unique_values[np.argmin(np.abs(a-unique_values)) ]
y_test = scaler.inverse_transform(np.transpose([y_test] * 7)) [:, tidx]
y_test = np.power(y_test, 1./powers[tidx])
 return y_test
model_path = f'model_{HEIGHT}x{WIDTH}.h5'
GCS_PATH = KaggleDatasets().get_gcs_path('tpu-getting-started')+ f'/tfrecords-jpeg-{HEIGHT}x{WIDTH}'
TRAINING_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/train/*.tfrec')
 VALIDATION_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/val/*.tfrec')
 TEST_FILENAMES = tf.io.gfile.glob(GCS_PATH + '/test/*.tfrec')
 CLASSES = [
'pink primrose', 'hard-leaved pocket orchid', 'canterbury bells', 'sweet pea',
'wild geranium', 'tiger lily', 'moon orchid', 'bird of paradise', 'monkshood',
'globe thistle', 'snapdragon', "colt's foot", 'king protea', 'spear thistle',
'yellow iris', 'globe-flower', 'purple coneflower', 'peruvian lily',
'balloon flower', 'giant white arum lily', 'fire lily', 'pincushion flower',
'fritillary', 'red ginger', 'grape hyacinth', 'corn poppy',
'prince of wales feathers', 'stemless gentian', 'artichoke', 'sweet william',
'carnation', 'garden phlox', 'love in the mist', 'cosmos', 'alpine sea holly',
'ruby-lipped cattleya', 'cape flower', 'great masterwort', 'siam tulip',
'lenten rose', 'barberton daisy', 'daffodil', 'sword lily', 'poinsettia',
'bolero deep blue', 'wallflower', 'marigold', 'buttercup', 'daisy',
'common dandelion', 'petunia', 'wild pansy', 'primula', 'sunflower',
'lilac hibiscus', 'bishop of llandaff', 'gaura', 'geranium', 'orange dahlia',
'pink-yellow dahlia', 'cautleya spicata', 'japanese anemone',
'black-eyed susan', 'silverbush', 'californian poppy', 'osteospermum',
'spring crocus', 'iris', 'windflower', 'tree poppy', 'gazania', 'azalea',
'water lily', 'rose', 'thorn apple', 'morning glory', 'passion flower',
'lotus', 'toad lily', 'anthurium', 'frangipani', 'clematis', 'hibiscus',
'columbine', 'desert-rose', 'tree mallow', 'magnolia', 'cyclamen ',
'watercress', 'canna lily', 'hippeastrum ', 'bee balm', 'pink quill',
'foxglove', 'bougainvillea', 'camellia', 'mallow', 'mexican petunia',
'bromelia', 'blanket flower', 'trumpet creeper', 'blackberry lily',
'common tulip', 'wild rose']<categorify>
prediction_dict = predictions_ridge
for i, t in enumerate(['age', 'domain1_var1', 'domain1_var2', 'domain2_var1', 'domain2_var2']):
unique_values = sorted(targets[t].dropna().unique())
 pred_values = back_transform(prediction_dict[t], unique_values, scaler=scaler_targets, tidx=i)
 submission.iloc[submission.index.str.contains(t), 0] = pred_values
display(submission.head(10))
submission.to_csv('submission_ridge.csv' )
<categorify><EOS> 
prediction_dict = predictions_svr
for i, t in enumerate(['age', 'domain1_var1', 'domain1_var2', 'domain2_var1', 'domain2_var2']):
unique_values = sorted(targets[t].dropna().unique())
 pred_values = back_transform(prediction_dict[t], unique_values, scaler=scaler_targets, tidx=i)
 submission.iloc[submission.index.str.contains(t), 0] = pred_values
display(submission.head(10))
submission.to_csv('submission_svr.csv' )