from pathlib import Path import shutil from datasets import load_dataset from transformers import TrainingArguments from span_marker import SpanMarkerModel, Trainer from span_marker.model_card import SpanMarkerModelCardData from huggingface_hub import upload_folder, upload_file def main() -> None: # Load the dataset, ensure "tokens" and "ner_tags" columns, and get a list of labels dataset = load_dataset("ljvmiranda921/tlunified-ner") labels = dataset["train"].features["ner_tags"].feature.names # Initialize a SpanMarker model using a pretrained BERT-style encoder encoder_id = "bert-base-multilingual-cased" model_id = f"tomaarsen/span-marker-mbert-base-tlunified" model = SpanMarkerModel.from_pretrained( encoder_id, labels=labels, # SpanMarker hyperparameters: model_max_length=256, marker_max_length=128, entity_max_length=8, # Model card variables model_card_data=SpanMarkerModelCardData( model_id=model_id, encoder_id=encoder_id, dataset_name="TLUnified", license="gpl-3.0", language=["tl"], ), ) # Prepare the 🤗 transformers training arguments output_dir = Path("models") / model_id args = TrainingArguments( output_dir=output_dir, run_name=model_id, # Training Hyperparameters: learning_rate=5e-5, per_device_train_batch_size=16, per_device_eval_batch_size=16, num_train_epochs=3, weight_decay=0.01, warmup_ratio=0.1, bf16=True, # Replace `bf16` with `fp16` if your hardware can't use bf16. # Other Training parameters logging_first_step=True, logging_steps=50, evaluation_strategy="steps", save_strategy="steps", eval_steps=400, save_total_limit=1, dataloader_num_workers=4, ) # Initialize the trainer using our model, training args & dataset, and train trainer = Trainer( model=model, args=args, train_dataset=dataset["train"], eval_dataset=dataset["validation"], ) trainer.train() # Compute & save the metrics on the test set metrics = trainer.evaluate(dataset["test"], metric_key_prefix="test") trainer.save_metrics("test", metrics) # Save the model & training script locally trainer.save_model(output_dir / "checkpoint-final") shutil.copy2(__file__, output_dir / "checkpoint-final" / "train.py") # Upload everything to the Hub breakpoint() model.push_to_hub(model_id, private=True) upload_folder(folder_path=output_dir / "runs", path_in_repo="runs", repo_id=model_id) upload_file(path_or_fileobj=__file__, path_in_repo="train.py", repo_id=model_id) upload_file(path_or_fileobj=output_dir / "all_results.json", path_in_repo="all_results.json", repo_id=model_id) upload_file(path_or_fileobj=output_dir / "emissions.csv", path_in_repo="emissions.csv", repo_id=model_id) if __name__ == "__main__": main()