Kaggle competition: Contradictory, My Dear Watson
<SOS> metric: categorizationaccuracy Kaggle data source: contradictory,-my-dear-watson<split>
!pip install -q --upgrade pip
!pip install --use-feature=2020-resolver -q transformers==3.0.2
!pip install -q googletrans
X_tr, X_val, y_tr, y_val = train_test_split(X_train, y, random_state = 3 )<choose_model_class>
print(f"TensorFlow version: {tf.__version__}")
 print(f"Transformers version: {transformers.__version__}")
 warnings.filterwarnings("ignore")
model = Sequential()
model.add(Dense(1, input_dim = X_train.shape[1], W_regularizer=l1(0.001)))
 model.compile(loss = "mse", optimizer = "adam" )<train_model>
class Configuration() :

 def __init__(
 self,
model_name,
translation = True,
max_length = 64,
padding = True,
batch_size = 128,
epochs = 5,
learning_rate = 1e-5,
metrics = ["sparse_categorical_accuracy"],
verbose = 1,
train_splits = 5,
accelerator = "TPU",
myluckynumber = 13 
):
self.SEED = myluckynumber
self.ACCELERATOR = accelerator
self.PATH_TRAIN = Path("/kaggle/input/contradictory-my-dear-watson/train.csv")
 self.PATH_TEST = Path("/kaggle/input/contradictory-my-dear-watson/test.csv")
 self.TRAIN_SPLITS = train_splits
self.LANGUAGE_MAP = {
"English" : 0,
"Chinese" : 1,
"Arabic" : 2,
"French" : 3,
"Swahili" : 4,
"Urdu" : 5,
"Vietnamese": 6,
"Russian" : 7,
"Hindi" : 8,
"Greek" : 9,
"Thai" : 10,
"Spanish" : 11,
"German" : 12,
"Turkish" : 13,
"Bulgarian" : 14
}
self.INVERSE_LANGUAGE_MAP = {v: k for k, v in self.LANGUAGE_MAP.items() }
self.MODEL_NAME = model_name
self.TRANSLATION = translation
self.TOKENIZER = AutoTokenizer.from_pretrained(self.MODEL_NAME)
 self.MAX_LENGTH = max_length
self.PAD_TO_MAX_LENGTH = padding
self.BATCH_SIZE = batch_size
self.EPOCHS = epochs
self.LEARNING_RATE = learning_rate
self.METRICS = metrics
self.VERBOSE = verbose
self.initialize_accelerator()
def initialize_accelerator(self):

 if self.ACCELERATOR == "TPU":
print("Connecting to TPU")
 try:
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
print(f"Running on TPU {tpu.master() }")
 except ValueError:
print("Could not connect to TPU")
 tpu = None
if tpu:
try:
print("Initializing TPU")
 tf.config.experimental_connect_to_cluster(tpu)
 tf.tpu.experimental.initialize_tpu_system(tpu)
 self.strategy = tf.distribute.experimental.TPUStrategy(tpu)
 self.tpu = tpu
print("TPU initialized")
 except _:
print("Failed to initialize TPU")
 else:
print("Unable to initialize TPU")
 self.ACCELERATOR = "GPU"
if self.ACCELERATOR != "TPU":
print("Using default strategy for CPU and single GPU")
 self.strategy = tf.distribute.get_strategy()
if self.ACCELERATOR == "GPU":
print(f"GPUs Available: {len(tf.config.experimental.list_physical_devices('GPU')) }")
 self.AUTO = tf.data.experimental.AUTOTUNE
self.REPLICAS = self.strategy.num_replicas_in_sync
print(f"REPLICAS: {self.REPLICAS}")
hist = model.fit(X_tr, y_tr, validation_data =(X_val, y_val))<import_modules>
def translate_text_to_english(text):

 translator = Translator()
return translator.translate(text, dest = "en" ).text
def encode_text(df, tokenizer, max_len, padding):

 text = df[["premise", "hypothesis"]].values.tolist()
text_encoded = tokenizer.batch_encode_plus(
 text,
pad_to_max_length = padding,
max_length = max_len 
)
 return text_encoded
def get_tf_dataset(X, y, auto, labelled = True, repeat = False, shuffle = False, batch_size = 128):

 if labelled:
ds =(tf.data.Dataset.from_tensor_slices(( X["input_ids"], y)))
 else:
ds =(tf.data.Dataset.from_tensor_slices(X["input_ids"]))
if repeat:
ds = ds.repeat()
if shuffle:
ds = ds.shuffle(2048)
 ds = ds.batch(batch_size)
 ds = ds.prefetch(auto)
 return ds
print(os.listdir(".. /input"))
warnings.filterwarnings('ignore')
<load_from_csv>
def build_model(model_name, max_len, learning_rate, metrics):

 input_ids = Input(shape =(max_len,), dtype = tf.int32, name = "input_ids")
 transformer_model = TFAutoModel.from_pretrained(model_name)
 transformer_embeddings = transformer_model(input_ids)[0]
output_values = Dense(3, activation = "softmax" )(transformer_embeddings[:, 0, :])
 model = Model(inputs = input_ids, outputs = output_values)
 opt = Adam(learning_rate = learning_rate)
 loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits = True)
 metrics = metrics
model.compile(optimizer = opt, loss = loss, metrics = metrics)
 return model
train = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/train.csv")
 train.head()<load_from_csv>
def run_model(config):

 df_train = pd.read_csv(config.PATH_TRAIN)
 df_test = pd.read_csv(config.PATH_TEST)
 if config.TRANSLATION:
df_train.loc[df_train.language != "English", "premise"] = df_train[df_train.language != "English"].premise.apply(lambda x: translate_text_to_english(x))
df_test.loc[df_test.language != "English", "premise"] = df_test[df_test.language != "English"].premise.apply(lambda x: translate_text_to_english(x))
df_train.loc[df_train.language != "English", "hypothesis"] = df_train[df_train.language != "English"].hypothesis.apply(lambda x: translate_text_to_english(x))
df_test.loc[df_test.language != "English", "hypothesis"] = df_test[df_test.language != "English"].hypothesis.apply(lambda x: translate_text_to_english(x))
df_train["language_label"] = df_train.language.astype(str)+ "_" + df_train.label.astype(str)
 skf = StratifiedKFold(n_splits = config.TRAIN_SPLITS, shuffle = True, random_state = config.SEED)
 preds_oof = np.zeros(( df_train.shape[0], 3))
preds_test = np.zeros(( df_test.shape[0], 3))
acc_oof = []
for(fold,(train_index, valid_index)) in enumerate(skf.split(df_train, df_train.language_label)) :
if config.ACCELERATOR == "TPU":
if config.tpu:
config.initialize_accelerator()
K.clear_session()
with config.strategy.scope() :
model = build_model(config.MODEL_NAME, config.MAX_LENGTH, config.LEARNING_RATE, config.METRICS)
 if fold == 0:
print(model.summary())
 print("
")
 print("
print(f"
print("
X_train = df_train.iloc[train_index]
X_valid = df_train.iloc[valid_index]
y_train = X_train.label.values
y_valid = X_valid.label.values
print("
Tokenizing")
 X_train_encoded = encode_text(df = X_train, tokenizer = config.TOKENIZER, max_len = config.MAX_LENGTH, padding = config.PAD_TO_MAX_LENGTH)
 X_valid_encoded = encode_text(df = X_valid, tokenizer = config.TOKENIZER, max_len = config.MAX_LENGTH, padding = config.PAD_TO_MAX_LENGTH)
 ds_train = get_tf_dataset(X_train_encoded, y_train, config.AUTO, repeat = True, shuffle = True, batch_size = config.BATCH_SIZE * config.REPLICAS)
 ds_valid = get_tf_dataset(X_valid_encoded, y_valid, config.AUTO, batch_size = config.BATCH_SIZE * config.REPLICAS * 4)
 n_train = X_train.shape[0]
if fold == 0:
X_test_encoded = encode_text(df = df_test, tokenizer = config.TOKENIZER, max_len = config.MAX_LENGTH, padding = config.PAD_TO_MAX_LENGTH)
 sv = tf.keras.callbacks.ModelCheckpoint(
 "model.h5",
monitor = "val_sparse_categorical_accuracy",
verbose = 0,
save_best_only = True,
save_weights_only = True,
mode = "max",
save_freq = "epoch" 
)
 print("
Training")
 model_history = model.fit(
 ds_train,
epochs = config.EPOCHS,
callbacks = [sv],
steps_per_epoch = n_train / config.BATCH_SIZE // config.REPLICAS,
validation_data = ds_valid,
verbose = config.VERBOSE 
)
 print("
Validating")
 model.load_weights("model.h5")
 ds_valid = get_tf_dataset(X_valid_encoded, -1, config.AUTO, labelled = False, batch_size = config.BATCH_SIZE * config.REPLICAS * 4)
 preds_valid = model.predict(ds_valid, verbose = config.VERBOSE)
 acc = accuracy_score(y_valid, np.argmax(preds_valid, axis = 1))
preds_oof[valid_index] = preds_valid
acc_oof.append(acc)
 print("
Inferencing")
 ds_test = get_tf_dataset(X_test_encoded, -1, config.AUTO, labelled = False, batch_size = config.BATCH_SIZE * config.REPLICAS * 4)
 preds_test += model.predict(ds_test, verbose = config.VERBOSE)/ config.TRAIN_SPLITS
print(f"
Fold {fold + 1} Accuracy: {round(acc, 4)}
")
 g = gc.collect()
print(f"
CV Mean Accuracy: {round(np.mean(acc_oof), 4)}")
 print(f"CV StdDev Accuracy: {round(np.std(acc_oof), 4)}
")
 return preds_oof, preds_test
test = pd.read_csv(".. /input/house-prices-advanced-regression-techniques/test.csv")
 test.head()<count_missing_values>
config_1 = Configuration("jplu/tf-xlm-roberta-large", translation = False, max_length = 84, batch_size = 64, epochs = 16, train_splits = 4)
 preds_train_1, preds_test_1 = run_model(config_1)
missing_percentage(test )<sort_values>
df_test = pd.read_csv(config_1.PATH_TEST)
 df_submission = pd.DataFrame({"id": df_test.id.values, "prediction": np.argmax(preds_test_1, axis = 1)})
 df_submission.to_csv("submission.csv", index = False)
 df_submission.prediction.value_counts()