from pathlib import Path import random import shutil from datasets import load_dataset from transformers import TrainingArguments from span_marker import SpanMarkerModel, Trainer from span_marker.model_card import SpanMarkerModelCardData from huggingface_hub import upload_folder, upload_file def main() -> None: # Load the dataset, ensure "tokens" and "ner_tags" columns, and get a list of labels labels = ["O", "B-ORG", "I-ORG"] dataset_id = "tomaarsen/ner-orgs" dataset = load_dataset(dataset_id) train_dataset = dataset["train"] eval_dataset = dataset["validation"] eval_dataset = eval_dataset.select(random.sample(range(len(eval_dataset)), k=3000)) test_dataset = dataset["test"] # Initialize a SpanMarker model using a pretrained BERT-style encoder encoder_id = "BAAI/bge-base-en-v1.5" model_id = "nbroad/span-marker-bge-base-orgs-v1" model = SpanMarkerModel.from_pretrained( encoder_id, labels=labels, # SpanMarker hyperparameters: model_max_length=256, marker_max_length=128, entity_max_length=8, # Model card variables model_card_data=SpanMarkerModelCardData( model_id=model_id, dataset_id=dataset_id, encoder_id=encoder_id, dataset_name="FewNERD, CoNLL2003, and OntoNotes v5", license="cc-by-sa-4.0", language=["en"], ), ) # Prepare the 🤗 transformers training arguments output_dir = Path("models") / model_id args = TrainingArguments( output_dir=output_dir, run_name=model_id, # Training Hyperparameters: learning_rate=5e-5, per_device_train_batch_size=128, per_device_eval_batch_size=128, num_train_epochs=3, weight_decay=0.01, warmup_ratio=0.05, fp16=True, # Replace `bf16` with `fp16` if your hardware can't use bf16. # Other Training parameters logging_first_step=True, logging_steps=100, evaluation_strategy="steps", save_strategy="steps", eval_steps=600, save_total_limit=1, dataloader_num_workers=4, metric_for_best_model="overall_f1", greater_is_better=True, ) # Initialize the trainer using our model, training args & dataset, and train trainer = Trainer( model=model, args=args, train_dataset=train_dataset, eval_dataset=eval_dataset, ) trainer.train() # Compute & save the metrics on the test set metrics = trainer.evaluate(test_dataset, metric_key_prefix="test") trainer.save_metrics("test", metrics) # Save the model & training script locally trainer.save_model(output_dir / "checkpoint-final") shutil.copy2(__file__, output_dir / "checkpoint-final" / "train.py") # Upload everything to the Hub breakpoint() model.push_to_hub(model_id, private=True) upload_folder(folder_path=output_dir / "runs", path_in_repo="runs", repo_id=model_id) upload_file(path_or_fileobj=__file__, path_in_repo="train.py", repo_id=model_id) upload_file(path_or_fileobj=output_dir / "all_results.json", path_in_repo="all_results.json", repo_id=model_id) upload_file(path_or_fileobj=output_dir / "emissions.csv", path_in_repo="emissions.csv", repo_id=model_id) if __name__ == "__main__": main()