Kaggle competition: House Prices - Advanced Regression Techniques
snli = nlp.load_dataset(path='snli')
 print('The number of training examples in snli dataset:', snli['train'].num_rows)
 print('The number of validation examples in snli dataset:', snli['validation'].num_rows, '
')
 print('The class names in snli dataset:', snli['train'].features['label'].names)
 print('The feature names in snli dataset:', list(snli['train'].features.keys()), '
')
 for idx, elt in enumerate(snli['train']):
print('premise:', elt['premise'])
 print('hypothesis:', elt['hypothesis'])
 print('label:', elt['label'])
 print('label name:', snli['train'].features['label'].names[elt['label']])
 print('-' * 80)
 if idx >= 10:
break<create_dataframe>
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestRegressor
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.linear_model import Lasso,Ridge,ElasticNet
from xgboost import XGBRegressor
from lightgbm import LGBMRegressor
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import GridSearchCV
xnli = nlp.load_dataset(path='xnli')
 print('The number of validation examples in xnli dataset:', xnli['validation'].num_rows, '
')
 print('The class names in xnli dataset:', xnli['validation'].features['label'].names)
 print('The feature names in xnli dataset:', list(xnli['validation'].features.keys()), '
')
 for idx, elt in enumerate(xnli['validation']):
print('premise:', elt['premise'])
 print('hypothesis:', elt['hypothesis'])
 print('label:', elt['label'])
 print('label name:', xnli['validation'].features['label'].names[elt['label']])
 print('-' * 80)
 if idx >= 3:
break<create_dataframe>
train = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/train.csv')
 test = pd.read_csv('.. /input/house-prices-advanced-regression-techniques/test.csv' )
for k in raw_ds_mapping:
for idx, x in enumerate(get_raw_dataset(k)) :
print(x)
 if idx >= 3:
break<create_dataframe>
data = pd.concat([train,test] )
def get_unbatched_dataset(ds_names, model_name, max_len=64):

 if type(ds_names)== list:
ds_names = {k: None for k in ds_names}
ds_names = {k: v for k, v in ds_names.items() if k in raw_ds_mapping}
tokenizer = transformers.AutoTokenizer.from_pretrained(model_name, use_fast=True)
 raw_datasets = [get_raw_dataset(x)for x in ds_names]
nb_examples = 0
sentence_pairs = []
labels = []
for name in ds_names:
raw_ds = get_raw_dataset(name)
 nb_examples_to_use = raw_ds_mapping[name][2]
if ds_names[name]:
nb_examples_to_use = min(ds_names[name], nb_examples_to_use)
 nb_examples += nb_examples_to_use
n = 0
for x in raw_ds:
sentence_pairs.append(( x['premise'], x['hypothesis']))
labels.append(x['label'])
 n += 1
if n >= nb_examples_to_use:
break
r = dict(tokenizer.batch_encode_plus(batch_text_or_text_pairs=sentence_pairs, max_length=max_len, padding='max_length', truncation=True))
dataset = tf.data.Dataset.from_tensor_slices(( r, labels))
return dataset, nb_examples
def get_batched_training_dataset(dataset, nb_examples, batch_size=16, shuffle_buffer_size=1, repeat=False):
if repeat:
dataset = dataset.repeat()
if not shuffle_buffer_size:
shuffle_buffer_size = nb_examples
dataset = dataset.shuffle(shuffle_buffer_size)
 dataset = dataset.batch(batch_size, drop_remainder=True)
 dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE)
 return dataset
def get_prediction_dataset(dataset, batch_size=16):
dataset = dataset.batch(batch_size, drop_remainder=False)
 dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE)
 return dataset<train_model>
data.index = range(2919)
 data.drop(['Id','Utilities'],axis=1,inplace=True)
 data['MoSold'] = data['MoSold'].astype(str)
 data['YrSold'] = data['YrSold'].astype(str )
for k in raw_ds_mapping.keys() :
ds, nb_examples = get_unbatched_dataset({k: 100}, model_name='distilbert-base-uncased')
 ds_batched = get_batched_training_dataset(ds, nb_examples, batch_size=16, shuffle_buffer_size=1, repeat=False)
 print('{} - select {} examples'.format(k, nb_examples))
for x in ds_batched:
break<choose_model_class>
l2 = ['LotFrontage','MasVnrArea']
l3 = ['BsmtFinSF1','BsmtFinSF2','BsmtUnfSF','TotalBsmtSF','BsmtFullBath','BsmtHalfBath','GarageYrBlt','GarageCars','GarageArea']
for item in l2 :
data[item]= data[item].fillna(data[item].mean())
 for item in l3 :
data[item]= data[item].fillna(0 )
class Classifier(tf.keras.Model):
def __init__(self, model_name):
super(Classifier, self ).__init__()
self.transformer = transformers.TFAutoModel.from_pretrained(model_name)
 self.dropout = tf.keras.layers.Dropout(rate=0.05)
 self.global_pool = tf.keras.layers.GlobalAveragePooling1D()
self.classifier = tf.keras.layers.Dense(3)
 def call(self, inputs, training=False):
x = self.transformer(inputs, training=training)[0]
x = self.dropout(x, training=training)
 x = self.global_pool(x)
 return self.classifier(x)
 class Trainer:
def __init__(
 self, ds_names, model_name, max_len=64,
batch_size_per_replica=16, prediction_batch_size_per_replica=64,
shuffle_buffer_size=1 
):
global strategy
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 except ValueError:
strategy = tf.distribute.get_strategy()
print('Number of replicas:', strategy.num_replicas_in_sync)
 self.ds_names = ds_names
self.model_name = model_name
self.max_len = max_len
self.batch_size_per_replica = batch_size_per_replica
self.prediction_batch_size_per_replica = prediction_batch_size_per_replica
self.batch_size = batch_size_per_replica * strategy.num_replicas_in_sync
self.prediction_batch_size = prediction_batch_size_per_replica * strategy.num_replicas_in_sync
self.shuffle_buffer_size = shuffle_buffer_size
train_ds, self.nb_examples = get_unbatched_dataset(
 ds_names=ds_names, model_name=model_name, max_len=max_len 
)
 self.train_ds = get_batched_training_dataset(
 train_ds, self.nb_examples, batch_size=self.batch_size,
shuffle_buffer_size=self.shuffle_buffer_size, repeat=True 
)
 valid_ds, self.nb_valid_examples = get_unbatched_dataset(
 ds_names=['original valid'], model_name=model_name, max_len=max_len 
)
 self.valid_ds = get_prediction_dataset(valid_ds, self.prediction_batch_size)
 self.valid_labels = next(iter(self.valid_ds.map(lambda inputs, label: label ).unbatch().batch(len(original_valid))))
test_ds, self.nb_test_examples = get_unbatched_dataset(
 ds_names=['original test'], model_name=model_name, max_len=max_len 
)
 self.test_ds = get_prediction_dataset(test_ds, self.prediction_batch_size)
 self.steps_per_epoch = self.nb_examples // self.batch_size
def get_model(self, model_name, lr, verbose=False):
with strategy.scope() :
model = Classifier(model_name)
 model.trainable = True
dummy = model(tf.constant(1, shape=[1, 64]))
if verbose:
model.summary()
optimizer = tf.keras.optimizers.Adam(lr=lr)
 loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True, reduction=tf.keras.losses.Reduction.SUM)
 metrics = {
'train loss': tf.keras.metrics.Sum() ,
'train acc': tf.keras.metrics.SparseCategoricalAccuracy()
}
return model, loss_fn, optimizer, metrics
def get_routines(self, model, loss_fn, optimizer, metrics):
def train_1_step(batch):
inputs, labels = batch
with tf.GradientTape() as tape:
logits = model(inputs, training=True)
 loss = loss_fn(labels, logits)/ self.batch_size
grads = tape.gradient(loss, model.trainable_variables)
 optimizer.apply_gradients(zip(grads, model.trainable_variables))
metrics['train loss'].update_state(loss)
 metrics['train acc'].update_state(labels, logits)
 @tf.function
def dist_train_1_epoch(data_iter):

 for _ in tf.range(self.steps_per_epoch):
strategy.run(train_1_step, args=(next(data_iter),))
@tf.function
def predict_step(batch):
inputs, _ = batch
logits = model(inputs, training=False)
 return logits
def predict_fn(dist_test_ds):
all_logits = []
for batch in dist_test_ds:
logits = strategy.run(predict_step, args=(batch,))
logits = strategy.experimental_local_results(logits)
 logits = tf.concat(logits, axis=0)
 all_logits.append(logits)
 logits = tf.concat(all_logits, axis=0)
 return logits
return dist_train_1_epoch, predict_fn
def train(self, train_name, model_name, epochs, verbose=False):
global strategy
try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 strategy = tf.distribute.experimental.TPUStrategy(tpu)
 except ValueError:
strategy = tf.distribute.get_strategy()
print('Number of replicas:', strategy.num_replicas_in_sync)
 model, loss_fn, optimizer, metrics = self.get_model(model_name, 1e-5, verbose=verbose)
 dist_train_1_epoch, predict_fn = self.get_routines(model, loss_fn, optimizer, metrics)
 train_dist_ds = strategy.experimental_distribute_dataset(self.train_ds)
 train_dist_iter = iter(train_dist_ds)
 dist_valid_ds = strategy.experimental_distribute_dataset(self.valid_ds)
 dist_test_ds = strategy.experimental_distribute_dataset(self.test_ds)
 history = {}
best_acc=0.5
for epoch in range(epochs):
s = datetime.datetime.now()
dist_train_1_epoch(train_dist_iter)
 train_loss = metrics['train loss'].result() / self.steps_per_epoch
train_acc = metrics['train acc'].result()
metrics['train loss'].reset_states()
metrics['train acc'].reset_states()
print('epoch: {}
'.format(epoch + 1))
print('train loss: {}'.format(train_loss))
print('train acc: {}
'.format(train_acc))
e = datetime.datetime.now()
elapsed =(e - s ).total_seconds()
logits = predict_fn(dist_valid_ds)
 valid_loss = tf.reduce_mean(tf.keras.losses.sparse_categorical_crossentropy(self.valid_labels, logits, from_logits=True, axis=-1))
valid_acc = tf.reduce_mean(tf.keras.metrics.sparse_categorical_accuracy(self.valid_labels, logits))
if valid_acc>best_acc:
best_acc=valid_acc
model.save_weights('best.h5')
 print('valid loss: {}'.format(valid_loss))
print('valid acc: {}
'.format(valid_acc))
print('train timing: {}
'.format(elapsed))
history[epoch] = {
'train loss': float(train_loss),
'train acc': float(train_acc),
'valid loss': float(valid_loss),
'valid acc': float(valid_acc),
'train timing': elapsed
}
print('-' * 40)
 print('best acc:{}'.format(best_acc))
model.load_weights('best.h5')
 logits = predict_fn(dist_test_ds)
 preds = tf.math.argmax(logits, axis=-1)
 submission = pd.read_csv('/kaggle/input/contradictory-my-dear-watson/sample_submission.csv')
 submission['prediction'] = preds.numpy()
submission.to_csv(f'submission-{train_name}.csv', index=False)
 return history, submission,logits<define_variables>
data['FireplaceQu'] = data['FireplaceQu'].fillna('None')
 data['MiscFeature'] = data['MiscFeature'].fillna('None')
 data['Alley']=data['Alley'].fillna('None' )
def print_config(trainer):
print('nb.of training examples used: {}'.format(trainer.nb_examples))
print('nb.of valid examples used: {}'.format(trainer.nb_valid_examples))
print('nb.of test examples used: {}'.format(trainer.nb_test_examples))
print('per replica batch size for training: {}'.format(trainer.batch_size_per_replica))
print('batch size for training: {}'.format(trainer.batch_size))
print('per replica batch size for prediction: {}'.format(trainer.prediction_batch_size_per_replica))
print('batch size for prediction: {}'.format(trainer.prediction_batch_size))
print('steps per epoch: {}'.format(trainer.steps_per_epoch))<train_model>
l1 = data['MSZoning'].unique().tolist()
for i in range(len(l1)-1):
j = np.random.randint(0,4)
 data['MSZoning'] = data['MSZoning'].fillna(l1[j] )
trainer = Trainer(
 ds_names={'original train': None, 'xnli valid': None, 'mnli train': 60000, 'mnli valid 1': None, 'mnli valid 2': None}, model_name=model_name,
max_len=208, batch_size_per_replica=16, prediction_batch_size_per_replica=64,
shuffle_buffer_size=None 
)
 print_config(trainer)
 train_name = f'{model_name} + extra-xnli-mnli'.replace('/', '-')
 history_3, submission_3,preds = trainer.train(train_name=train_name, model_name=model_name, epochs=epochs, verbose=True )<concatenate>
data['Electrical'] = data['Electrical'].fillna(data['Electrical'].mode() [0])
 data['KitchenQual'] = data['KitchenQual'].fillna(data['KitchenQual'].mode() [0])
 data['SaleType'] = data['SaleType'].fillna(data['SaleType'].mode() [0] )
np.savez_compressed('preds',a=preds )<save_to_csv>
data['Exterior1st'] = data['Exterior1st'].fillna('None')
 data['Exterior2nd'] = data['Exterior2nd'].fillna('None')
 data['MasVnrType'] = data['MasVnrType'].fillna('None')
 data['BsmtQual'] = data['BsmtQual'].fillna('None')
 data['BsmtCond'] = data['BsmtCond'].fillna('None')
 data['BsmtExposure'] = data['BsmtExposure'].fillna('None')
 data['BsmtFinType1'] = data['BsmtFinType1'].fillna('None')
 data['Fence'] = data['Fence'].fillna('None')
 data['Functional'] = data['Functional'].fillna('Typical')
 data['BsmtFinType2'] = data['BsmtFinType2'].fillna('None')
 data['GarageType'] = data['GarageType'].fillna('None')
 data['GarageFinish'] = data['GarageFinish'].fillna('None')
 data['GarageQual'] = data['GarageQual'].fillna('None')
 data['GarageCond'] = data['GarageCond'].fillna('None')
 data['PoolQC'] = data['PoolQC'].fillna('None' )
s = pd.read_csv(f'submission-{train_name}.csv')
 s.to_csv(f'submission.csv', index=False)
 s.head(20 )<install_modules>
cols =('FireplaceQu', 'BsmtQual', 'BsmtCond', 'GarageQual', 'GarageCond', 'HouseStyle',
'ExterQual', 'ExterCond','HeatingQC', 'PoolQC', 'KitchenQual', 'BsmtFinType1','Neighborhood', 'SaleCondition',
'BsmtFinType2', 'Functional', 'Fence', 'BsmtExposure', 'GarageFinish', 'LandSlope','Condition1','Condition2',
'LotShape', 'PavedDrive', 'Street', 'Alley', 'CentralAir', 'MSSubClass', 'OverallCond','GarageType','SaleType','BldgType')
 for c in cols:
lb = LabelEncoder()
lb.fit(list(data[c].values))
data[c] = lb.transform(list(data[c].values))
!pip install -q --upgrade pip
!pip install --use-feature=2020-resolver -q transformers==3.0.2
!pip install -q googletrans 
<set_options>
data = pd.get_dummies(data )
print(f"TensorFlow version: {tf.__version__}")
 print(f"Transformers version: {transformers.__version__}")
 warnings.filterwarnings("ignore")
<init_hyperparams>
data['Total_Bathrooms'] = data['FullBath'] + 0.5 * data['HalfBath'] + data['BsmtFullBath'] + 0.5 * data['BsmtHalfBath']
data['GarageScale'] = data['GarageCars'] * data['GarageArea']
data['GarageOrdinal'] = data['GarageQual'] + data['GarageCond']+data['GarageType']
data['GarageState'] = data['GarageFinish'] + data['GarageYrBlt'] + data['PavedDrive']
data['Porchtotal'] = data['OpenPorchSF'] + data['EnclosedPorch'] + data['3SsnPorch'] + data['ScreenPorch']
data['Extertotal'] = data['ExterQual'] + data['ExterCond']
data['KitchenCombined'] = data['KitchenQual'] * data['KitchenAbvGr']
data['FireplaceCombined'] = data['FireplaceQu'] * data['Fireplaces']
data['BsmtOrdinal'] = data['BsmtQual'] + data['BsmtCond']
data['BsmtFinishedAll'] = data['BsmtFinSF1'] + data['BsmtFinSF2']
data['AllFlrSF'] = data['1stFlrSF'] + data['2ndFlrSF']
data['OverallCombined'] = data['OverallQual'] + data['OverallCond']
data['TotalSF'] = data['AllFlrSF'] + data['TotalBsmtSF']
data['YrBltAndRemod'] = data["YearRemodAdd"] + data['YearBuilt']
data['roomtotal'] = data['BedroomAbvGr'] + data['KitchenAbvGr']+ data['TotRmsAbvGrd']
data['SaleState'] = data['SaleType'] + data['SaleCondition']
data['HouseComment'] = data['BldgType'] + data['HouseStyle'] + data['OverallCond']
data['Condition'] = data['Condition1'] + data['Condition2'] + data['Neighborhood']
class Configuration() :

 def __init__(
 self,
model_name,
translation = True,
max_length = 64,
padding = True,
batch_size = 128,
epochs = 5,
learning_rate = 1e-5,
metrics = ["sparse_categorical_accuracy"],
verbose = 1,
train_splits = 5,
accelerator = "TPU",
myluckynumber = 13 
):
self.SEED = myluckynumber
self.ACCELERATOR = accelerator
self.PATH_TRAIN = Path("/kaggle/input/contradictory-my-dear-watson/train.csv")
 self.PATH_TEST = Path("/kaggle/input/contradictory-my-dear-watson/test.csv")
 self.TRAIN_SPLITS = train_splits
self.LANGUAGE_MAP = {
"English" : 0,
"Chinese" : 1,
"Arabic" : 2,
"French" : 3,
"Swahili" : 4,
"Urdu" : 5,
"Vietnamese": 6,
"Russian" : 7,
"Hindi" : 8,
"Greek" : 9,
"Thai" : 10,
"Spanish" : 11,
"German" : 12,
"Turkish" : 13,
"Bulgarian" : 14
}
self.INVERSE_LANGUAGE_MAP = {v: k for k, v in self.LANGUAGE_MAP.items() }
self.MODEL_NAME = model_name
self.TRANSLATION = translation
self.TOKENIZER = AutoTokenizer.from_pretrained(self.MODEL_NAME)
 self.MAX_LENGTH = max_length
self.PAD_TO_MAX_LENGTH = padding
self.BATCH_SIZE = batch_size
self.EPOCHS = epochs
self.LEARNING_RATE = learning_rate
self.METRICS = metrics
self.VERBOSE = verbose
self.initialize_accelerator()
def initialize_accelerator(self):

 if self.ACCELERATOR == "TPU":
print("Connecting to TPU")
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print(f"Running on TPU {tpu.master() }")
 except ValueError:
print("Could not connect to TPU")
 tpu = None
if tpu:
try:
print("Initializing TPU")
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 self.strategy = tf.distribute.experimental.TPUStrategy(tpu)
 self.tpu = tpu
print("TPU initialized")
 except _:
print("Failed to initialize TPU")
 else:
print("Unable to initialize TPU")
 self.ACCELERATOR = "GPU"
if self.ACCELERATOR != "TPU":
print("Using default strategy for CPU and single GPU")
 self.strategy = tf.distribute.get_strategy()
if self.ACCELERATOR == "GPU":
print(f"GPUs Available: {len(tf.config.experimental.list_physical_devices('GPU')) }")
 self.AUTO = tf.data.experimental.AUTOTUNE
self.REPLICAS = self.strategy.num_replicas_in_sync
print(f"REPLICAS: {self.REPLICAS}")
<categorify>
def logs(res, ls):
m = res.shape[1]
for l in ls:
res = res.assign(newcol=pd.Series(np.log(1.01+res[l])).values)
 res.columns.values[m] = l + '_log'
m += 1
return res
log_features = ['LotFrontage','LotArea','MasVnrArea','BsmtFinSF1','BsmtFinSF2','BsmtUnfSF',
'TotalBsmtSF','1stFlrSF','2ndFlrSF','LowQualFinSF','GrLivArea',
'BsmtFullBath','BsmtHalfBath','FullBath','HalfBath','BedroomAbvGr','KitchenAbvGr',
'TotRmsAbvGrd','Fireplaces','GarageCars','GarageArea','WoodDeckSF','OpenPorchSF',
'EnclosedPorch','3SsnPorch','ScreenPorch','PoolArea','MiscVal','YearRemodAdd','Total_Bathrooms'
,'GarageScale','GarageOrdinal','Porchtotal','Extertotal','KitchenCombined','FireplaceCombined','BsmtOrdinal',
'BsmtFinishedAll','AllFlrSF','OverallCombined','TotalSF','YrBltAndRemod','roomtotal','SaleState','HouseComment','Condition']
data = logs(data, log_features )
def translate_text_to_english(text):

 translator = Translator()
return translator.translate(text, dest = "en" ).text
def encode_text(df, tokenizer, max_len, padding):

 text = df[["premise", "hypothesis"]].values.tolist()
text_encoded = tokenizer.batch_encode_plus(
 text,
pad_to_max_length = padding,
max_length = max_len 
)
 return text_encoded
def get_tf_dataset(X, y, auto, labelled = True, repeat = False, shuffle = False, batch_size = 128):

 if labelled:
ds =(tf.data.Dataset.from_tensor_slices(( X["input_ids"], y)))
 else:
ds =(tf.data.Dataset.from_tensor_slices(X["input_ids"]))
if repeat:
ds = ds.repeat()
if shuffle:
ds = ds.shuffle(2048)
 ds = ds.batch(batch_size)
 ds = ds.prefetch(auto)
 return ds 
<choose_model_class>
def build_model(model_name, max_len, learning_rate, metrics):

 input_ids = Input(shape =(max_len,), dtype = tf.int32, name = "input_ids")
 transformer_model = TFAutoModel.from_pretrained(model_name)
 transformer_embeddings = transformer_model(input_ids)[0]
output_values = Dense(3, activation = "softmax" )(transformer_embeddings[:, 0, :])
 model = Model(inputs = input_ids, outputs = output_values)
 opt = Adam(learning_rate = learning_rate)
 loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits = True)
 metrics = metrics
model.compile(optimizer = opt, loss = loss, metrics = metrics)
 return model 
<load_from_csv>
train_data = data[:1460]
test_data = data[1460:].drop(['SalePrice'],axis = 1 )
def run_model(config):

 df_train = pd.read_csv(config.PATH_TRAIN)
 df_test = pd.read_csv(config.PATH_TEST)
 if config.TRANSLATION:
df_train.loc[df_train.language != "English", "premise"] = df_train[df_train.language != "English"].premise.apply(lambda x: translate_text_to_english(x))
df_test.loc[df_test.language != "English", "premise"] = df_test[df_test.language != "English"].premise.apply(lambda x: translate_text_to_english(x))
df_train.loc[df_train.language != "English", "hypothesis"] = df_train[df_train.language != "English"].hypothesis.apply(lambda x: translate_text_to_english(x))
df_test.loc[df_test.language != "English", "hypothesis"] = df_test[df_test.language != "English"].hypothesis.apply(lambda x: translate_text_to_english(x))
df_train["language_label"] = df_train.language.astype(str)+ "_" + df_train.label.astype(str)
 skf = StratifiedKFold(n_splits = config.TRAIN_SPLITS, shuffle = True, random_state = config.SEED)
 preds_oof = np.zeros(( df_train.shape[0], 3))
preds_test = np.zeros(( df_test.shape[0], 3))
acc_oof = []
for(fold,(train_index, valid_index)) in enumerate(skf.split(df_train, df_train.language_label)) :
if config.ACCELERATOR == "TPU":
if config.tpu:
config.initialize_accelerator()
K.clear_session()
with config.strategy.scope() :
model = build_model(config.MODEL_NAME, config.MAX_LENGTH, config.LEARNING_RATE, config.METRICS)
 if fold == 0:
print(model.summary())
 print("
")
 print("
print(f"
print("
X_train = df_train.iloc[train_index]
X_valid = df_train.iloc[valid_index]
y_train = X_train.label.values
y_valid = X_valid.label.values
print("
Tokenizing")
 X_train_encoded = encode_text(df = X_train, tokenizer = config.TOKENIZER, max_len = config.MAX_LENGTH, padding = config.PAD_TO_MAX_LENGTH)
 X_valid_encoded = encode_text(df = X_valid, tokenizer = config.TOKENIZER, max_len = config.MAX_LENGTH, padding = config.PAD_TO_MAX_LENGTH)
 ds_train = get_tf_dataset(X_train_encoded, y_train, config.AUTO, repeat = True, shuffle = True, batch_size = config.BATCH_SIZE * config.REPLICAS)
 ds_valid = get_tf_dataset(X_valid_encoded, y_valid, config.AUTO, batch_size = config.BATCH_SIZE * config.REPLICAS * 4)
 n_train = X_train.shape[0]
if fold == 0:
X_test_encoded = encode_text(df = df_test, tokenizer = config.TOKENIZER, max_len = config.MAX_LENGTH, padding = config.PAD_TO_MAX_LENGTH)
 sv = tf.keras.callbacks.ModelCheckpoint(
 "model.h5",
monitor = "val_sparse_categorical_accuracy",
verbose = 0,
save_best_only = True,
save_weights_only = True,
mode = "max",
save_freq = "epoch" 
)
 print("
Training")
 model_history = model.fit(
 ds_train,
epochs = config.EPOCHS,
callbacks = [sv],
steps_per_epoch = n_train / config.BATCH_SIZE // config.REPLICAS,
validation_data = ds_valid,
verbose = config.VERBOSE 
)
 print("
Validating")
 model.load_weights("model.h5")
 ds_valid = get_tf_dataset(X_valid_encoded, -1, config.AUTO, labelled = False, batch_size = config.BATCH_SIZE * config.REPLICAS * 4)
 preds_valid = model.predict(ds_valid, verbose = config.VERBOSE)
 acc = accuracy_score(y_valid, np.argmax(preds_valid, axis = 1))
preds_oof[valid_index] = preds_valid
acc_oof.append(acc)
 print("
Inferencing")
 ds_test = get_tf_dataset(X_test_encoded, -1, config.AUTO, labelled = False, batch_size = config.BATCH_SIZE * config.REPLICAS * 4)
 preds_test += model.predict(ds_test, verbose = config.VERBOSE)/ config.TRAIN_SPLITS
print(f"
Fold {fold + 1} Accuracy: {round(acc, 4)}
")
 g = gc.collect()
print(f"
CV Mean Accuracy: {round(np.mean(acc_oof), 4)}")
 print(f"CV StdDev Accuracy: {round(np.std(acc_oof), 4)}
")
 return preds_oof, preds_test 
<train_model>
y = np.log1p(train_data['SalePrice'])
 x = train_data.drop(['SalePrice'],axis=1 )
config_1 = Configuration("jplu/tf-xlm-roberta-large", translation = False, max_length = 84, batch_size = 64, epochs = 16, train_splits = 4)
 preds_train_1, preds_test_1 = run_model(config_1)
<save_to_csv>
xtrain ,xtest ,ytrain ,ytest = train_test_split(x,y,test_size = 0.3, random_state = 4200 )
df_test = pd.read_csv(config_1.PATH_TEST)
 df_submission = pd.DataFrame({"id": df_test.id.values, "prediction": np.argmax(preds_test_1, axis = 1)})
 df_submission.to_csv("submission.csv", index = False)
 df_submission.prediction.value_counts() 
<import_modules>
la = Lasso(alpha=0.1,max_iter=500)
 la.fit(x,y)
 pred1 = np.expm1(la.predict(test_data))
from transformers import BertTokenizer, TFBertModel
from transformers import RobertaTokenizer, TFRobertaModel
from transformers import XLMRobertaTokenizer, TFXLMRobertaModel
from transformers import AutoTokenizer
import matplotlib.pyplot as plt
import tensorflow as tf
from dask import bag, diagnostics
from sklearn.utils import shuffle<install_modules>
R = Ridge(alpha=1.0,max_iter=500)
 R.fit(x,y)
 pred2 = np.expm1(R.predict(test_data))
!pip install --quiet googletrans 
<load_from_csv>
xgbr = XGBRegressor(booster='gbtree'
, colsample_bylevel=1
, colsample_bynode=1
, colsample_bytree=0.6
, gamma=0
, importance_type='gain'
, learning_rate=0.01
, max_delta_step=0
, max_depth= 3
, min_child_weight=1.5
, n_estimators=5400
, n_jobs=-1
, nthread=None
, objective='reg:squarederror'
, reg_alpha=0.3
, reg_lambda=0.7
, scale_pos_weight=1
, silent=None
, subsample=0.6
, verbosity=1)
 xgbr.fit(x,y)
 pred3 = np.expm1(xgbr.predict(test_data))
submission = pd.read_csv("/kaggle/input/output/submission(2 ).csv")
 submission.head()<save_to_csv>
lgmb = LGBMRegressor(objective='regression',
boosting_type='gbdt',
num_leaves= 5,
max_depth = 3,
learning_rate=0.01,
n_estimators=6300,
subsample_for_bin=100,
class_weight=None,
min_split_gain=0.0,
min_child_weight=1.3,
min_child_samples=3,
subsample=0.1,
subsample_freq=0,
colsample_bytree=0.6,
reg_alpha=0.1,
reg_lambda=0.6,
random_state=None,
n_jobs= -1,
silent=True,
importance_type='gain')
 lgmb.fit(x,y)
 pred4 = np.expm1(lgmb.predict(test_data))
submission.to_csv("submission1.csv", index = False)
<load_from_csv>
gbrt = GradientBoostingRegressor(alpha=0.5, ccp_alpha=0.0,
criterion='friedman_mse',
init=None, learning_rate=0.01,
loss='huber', max_depth=3,
max_leaf_nodes=5,
min_impurity_decrease=0.0,
min_impurity_split=None,
min_samples_leaf=10,
min_samples_split=3,
min_weight_fraction_leaf=0.0,
n_estimators=3350,
n_iter_no_change=None,
random_state=5,subsample=0.6,
tol=0.0001,
validation_fraction=0.1,
verbose=0)
 gbrt.fit(x,y)
 pred5 = np.expm1(gbrt.predict(test_data))
train = pd.read_csv("/kaggle/input/contradictory-my-dear-watson/train.csv")
 train.head()<load_pretrained>
E = ElasticNet(alpha=0.1,l1_ratio=0.1)
 E.fit(x,y)
 pred6 = np.expm1(E.predict(test_data))
tokenizer = AutoTokenizer.from_pretrained('jplu/tf-xlm-roberta-large' )<categorify>
pred = 0.0* pred1 + 0.2* pred2 + 0.37*pred3 + 0.38*pred4 +0.00* pred5+0.05* pred6
<categorify><EOS> 
pred = pd.DataFrame({"id": test['Id'], "SalePrice": pred})
 pred.to_csv('sample_submission.csv',index=False )