Upload 3 files

BERT.py

@@ -0,0 +1,84 @@
+from transformers import (
+    AutoTokenizer,
+    BertConfig,
+    BertForMaskedLM,
+    Trainer,
+    TrainingArguments,
+    DataCollatorForLanguageModeling,
+)
+from datasets import load_dataset
+from laonlp import word_tokenize
+import random
+
+tokenizer = AutoTokenizer.from_pretrained("bert/models/tokenizer")
+print(f"The max length for the tokenizer is: {tokenizer.model_max_length}")
+
+
+def group_texts(examples):
+    tokenized_inputs = [" ".join(word_tokenize(x)) for x in examples["text"]]
+
+    tokenized_inputs = tokenizer(
+        tokenized_inputs,
+        return_special_tokens_mask=True,
+        padding=True,
+        truncation=True,
+        max_length=tokenizer.model_max_length,
+        return_tensors="pt",
+    )
+
+    return tokenized_inputs
+
+
+if __name__ == "__main__":
+    train_dataset = load_dataset(path="bert/dataset/CulturaX", split="train")
+    eval_dataset = load_dataset(path="bert/dataset/laonlp", split="validation")
+
+    data_collator = DataCollatorForLanguageModeling(
+        tokenizer=tokenizer, mlm=True, mlm_probability=0.15
+    )
+
+    config_encoder = BertConfig(vocab_size=tokenizer.vocab_size)
+
+    model = BertForMaskedLM.from_pretrained("BERT\\models\\bert-culturaX-data")
+
+    train_dataset = train_dataset.map(
+        group_texts,
+        batched=True,
+        remove_columns=[
+            "text",
+            "timestamp",
+            "url",
+            "source",
+        ],
+        num_proc=12,
+    ).shuffle(seed=random.randint(0, 1000))
+    eval_dataset = eval_dataset.map(
+        group_texts, batched=True, remove_columns=["text"]
+    ).shuffle(seed=random.randint(0, 1000))
+
+    print(
+        f"the dataset contains in total {len(train_dataset)*tokenizer.model_max_length} tokens"
+    )
+
+    model_name = "bert-culturaX-data"
+
+    training_args = TrainingArguments(
+        output_dir=f"bert/models/{model_name}",
+        evaluation_strategy="epoch",
+        per_device_train_batch_size=16,
+        per_device_eval_batch_size=16,
+        weight_decay=0.01,
+        save_total_limit=3,
+        num_train_epochs=2,
+        push_to_hub=True,
+    )
+
+    trainer = Trainer(
+        model=model,
+        args=training_args,
+        data_collator=data_collator,
+        train_dataset=train_dataset,
+        eval_dataset=eval_dataset,
+    )
+
+    trainer.train()

tokenizer.py

@@ -0,0 +1,30 @@
+from datasets import load_dataset
+from transformers import BertTokenizerFast
+from laonlp import word_tokenize
+from tqdm import tqdm
+
+
+def tokenize(examples):
+    examples["text"] = [" ".join(word_tokenize(x)) for x in examples["text"]]
+    return examples
+
+
+# create a python generator to dynamically load the data
+def batch_iterator(batch_size=10000):
+    for i in tqdm(range(0, len(raw_datasets), batch_size)):
+        yield raw_datasets[i : i + batch_size]["text"]
+
+
+if __name__ == "__main__":
+    raw_datasets = load_dataset(path="bert/dataset/culturaX", split="train")
+
+    raw_datasets = raw_datasets.map(tokenize, batched=True, num_proc=12)
+
+    # create a tokenizer from existing one to re-use special tokens
+    tokenizer = BertTokenizerFast.from_pretrained("bert-base-uncased")
+
+    bert_tokenizer = tokenizer.train_new_from_iterator(
+        text_iterator=batch_iterator(),
+        vocab_size=32_000,
+    )
+    bert_tokenizer.save_pretrained("bert/models/tokenizer")

train.py

@@ -0,0 +1,106 @@
+from transformers import (
+    AutoTokenizer,
+    AutoModel,
+    BertModel,
+    GPT2Model,
+    EncoderDecoderModel,
+    DataCollatorForSeq2Seq,
+    Seq2SeqTrainer,
+    Seq2SeqTrainingArguments,
+)
+from datasets import load_dataset
+from laonlp import word_tokenize
+from functools import partial
+import random
+
+
+def group_texts(tokenizer, examples):
+    tokenized_inputs = [" ".join(word_tokenize(x)) for x in examples["text"]]
+
+    tokenized_inputs = tokenizer(
+        examples["text"],
+        # return_special_tokens_mask=True,
+        # padding="max_length",
+        # truncation=True,
+        # max_length=tokenizer.model_max_length,
+        # return_tensors="pt",
+    )
+
+    return tokenized_inputs
+
+
+if __name__ == "__main__":
+    encoder_src = "BERT\\models\\bert-culturaX-data"
+    decoder_src = "NlpHUST/gpt2-vietnamese"
+
+    encoder_tokenizer = AutoTokenizer.from_pretrained(encoder_src)
+    decoder_tokenizer = AutoTokenizer.from_pretrained(decoder_src)
+    decoder_tokenizer.model_max_length = encoder_tokenizer.model_max_length
+    decoder_tokenizer.pad_token = decoder_tokenizer.eos_token
+    print(f"The max length for the tokenizer is: {encoder_tokenizer.model_max_length}")
+
+    encoder = AutoModel.from_pretrained(encoder_src)
+    decoder = AutoModel.from_pretrained(decoder_src)
+    decoder.config.max_length = decoder_tokenizer.model_max_length
+
+    model = EncoderDecoderModel(encoder=encoder, decoder=decoder)
+    model.config.decoder_start_token_id = decoder_tokenizer.bos_token_id
+    model.config.pad_token_id = decoder_tokenizer.pad_token_id
+    model.config.vocab_size = decoder_tokenizer.vocab_size
+
+    data_collator = DataCollatorForSeq2Seq(decoder_tokenizer, model=model)
+
+    raw_lo_dataset = load_dataset("bert/dataset/original/lo")
+    raw_vi_dataset = load_dataset("bert/dataset/original/vi")
+
+    train_dataset = raw_lo_dataset["train"].map(
+        partial(group_texts, encoder_tokenizer),
+        remove_columns=["text"],
+        batched=True,
+        num_proc=12,
+    )
+    eval_dataset = raw_lo_dataset["validation"].map(
+        partial(group_texts, encoder_tokenizer),
+        batched=True,
+        remove_columns=["text"],
+    )
+    train_labels = raw_vi_dataset["train"].map(
+        partial(group_texts, decoder_tokenizer),
+        remove_columns=["text"],
+        batched=True,
+        num_proc=12,
+    )
+    eval_labels = raw_vi_dataset["validation"].map(
+        partial(group_texts, decoder_tokenizer),
+        batched=True,
+        remove_columns=["text"],
+    )
+    train_dataset = train_dataset.add_column("labels", train_labels["input_ids"])
+    eval_dataset = eval_dataset.add_column("labels", eval_labels["input_ids"])
+
+    print(
+        f"the dataset contains in total {len(train_dataset)*encoder_tokenizer.model_max_length} tokens"
+    )
+
+    model_name = "transformer-bert-gpt"
+
+    training_args = Seq2SeqTrainingArguments(
+        output_dir=f"bert/models/{model_name}",
+        evaluation_strategy="epoch",
+        per_device_train_batch_size=16,
+        per_device_eval_batch_size=16,
+        weight_decay=0.01,
+        save_total_limit=3,
+        num_train_epochs=2,
+        push_to_hub=True,
+    )
+
+    trainer = Seq2SeqTrainer(
+        model=model,
+        args=training_args,
+        data_collator=data_collator,
+        train_dataset=train_dataset,
+        eval_dataset=eval_dataset,
+    )
+
+    trainer.train()