Move the py files to a directory

train.py

@@ -1,106 +0,0 @@
-from transformers import (
-    AutoTokenizer,
-    AutoModel,
-    BertModel,
-    GPT2Model,
-    EncoderDecoderModel,
-    DataCollatorForSeq2Seq,
-    Seq2SeqTrainer,
-    Seq2SeqTrainingArguments,
-)
-from datasets import load_dataset
-from laonlp import word_tokenize
-from functools import partial
-import random
-
-
-def group_texts(tokenizer, examples):
-    tokenized_inputs = [" ".join(word_tokenize(x)) for x in examples["text"]]
-
-    tokenized_inputs = tokenizer(
-        examples["text"],
-        # return_special_tokens_mask=True,
-        # padding="max_length",
-        # truncation=True,
-        # max_length=tokenizer.model_max_length,
-        # return_tensors="pt",
-    )
-
-    return tokenized_inputs
-
-
-if __name__ == "__main__":
-    encoder_src = "BERT\\models\\bert-culturaX-data"
-    decoder_src = "NlpHUST/gpt2-vietnamese"
-
-    encoder_tokenizer = AutoTokenizer.from_pretrained(encoder_src)
-    decoder_tokenizer = AutoTokenizer.from_pretrained(decoder_src)
-    decoder_tokenizer.model_max_length = encoder_tokenizer.model_max_length
-    decoder_tokenizer.pad_token = decoder_tokenizer.eos_token
-    print(f"The max length for the tokenizer is: {encoder_tokenizer.model_max_length}")
-
-    encoder = AutoModel.from_pretrained(encoder_src)
-    decoder = AutoModel.from_pretrained(decoder_src)
-    decoder.config.max_length = decoder_tokenizer.model_max_length
-
-    model = EncoderDecoderModel(encoder=encoder, decoder=decoder)
-    model.config.decoder_start_token_id = decoder_tokenizer.bos_token_id
-    model.config.pad_token_id = decoder_tokenizer.pad_token_id
-    model.config.vocab_size = decoder_tokenizer.vocab_size
-
-    data_collator = DataCollatorForSeq2Seq(decoder_tokenizer, model=model)
-
-    raw_lo_dataset = load_dataset("bert/dataset/original/lo")
-    raw_vi_dataset = load_dataset("bert/dataset/original/vi")
-
-    train_dataset = raw_lo_dataset["train"].map(
-        partial(group_texts, encoder_tokenizer),
-        remove_columns=["text"],
-        batched=True,
-        num_proc=12,
-    )
-    eval_dataset = raw_lo_dataset["validation"].map(
-        partial(group_texts, encoder_tokenizer),
-        batched=True,
-        remove_columns=["text"],
-    )
-    train_labels = raw_vi_dataset["train"].map(
-        partial(group_texts, decoder_tokenizer),
-        remove_columns=["text"],
-        batched=True,
-        num_proc=12,
-    )
-    eval_labels = raw_vi_dataset["validation"].map(
-        partial(group_texts, decoder_tokenizer),
-        batched=True,
-        remove_columns=["text"],
-    )
-    train_dataset = train_dataset.add_column("labels", train_labels["input_ids"])
-    eval_dataset = eval_dataset.add_column("labels", eval_labels["input_ids"])
-
-    print(
-        f"the dataset contains in total {len(train_dataset)*encoder_tokenizer.model_max_length} tokens"
-    )
-
-    model_name = "transformer-bert-gpt"
-
-    training_args = Seq2SeqTrainingArguments(
-        output_dir=f"bert/models/{model_name}",
-        evaluation_strategy="epoch",
-        per_device_train_batch_size=16,
-        per_device_eval_batch_size=16,
-        weight_decay=0.01,
-        save_total_limit=3,
-        num_train_epochs=2,
-        push_to_hub=True,
-    )
-
-    trainer = Seq2SeqTrainer(
-        model=model,
-        args=training_args,
-        data_collator=data_collator,
-        train_dataset=train_dataset,
-        eval_dataset=eval_dataset,
-    )
-
-    trainer.train()