shaurya-06/adapter

README.md

@@ -0,0 +1,61 @@
+---
+license: apache-2.0
+library_name: peft
+tags:
+- trl
+- sft
+- generated_from_trainer
+base_model: Helsinki-NLP/opus-mt-ROMANCE-en
+datasets:
+- generator
+model-index:
+- name: adapter
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# adapter
+
+This model is a fine-tuned version of [Helsinki-NLP/opus-mt-ROMANCE-en](https://huggingface.co/Helsinki-NLP/opus-mt-ROMANCE-en) on the generator dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0002
+- train_batch_size: 4
+- eval_batch_size: 8
+- seed: 42
+- gradient_accumulation_steps: 2
+- total_train_batch_size: 8
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.03
+- num_epochs: 10
+
+### Training results
+
+
+
+### Framework versions
+
+- PEFT 0.10.1.dev0
+- Transformers 4.40.0
+- Pytorch 2.1.0+cu121
+- Datasets 2.19.0
+- Tokenizers 0.19.1

adapter_config.json

@@ -0,0 +1,51 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "Helsinki-NLP/opus-mt-ROMANCE-en",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 64,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "model.decoder.layers.1.fc1",
+    "model.encoder.layers.4.fc1",
+    "model.encoder.layers.0.fc1",
+    "model.decoder.layers.4.fc1",
+    "model.encoder.layers.2.fc2",
+    "model.encoder.layers.5.fc2",
+    "model.decoder.layers.0.fc2",
+    "model.encoder.layers.0.fc2",
+    "model.encoder.layers.4.fc2",
+    "model.encoder.layers.1.fc2",
+    "model.encoder.layers.2.fc1",
+    "model.encoder.layers.3.fc1",
+    "model.decoder.layers.5.fc2",
+    "model.decoder.layers.2.fc1",
+    "model.decoder.layers.1.fc2",
+    "model.encoder.layers.5.fc1",
+    "model.decoder.layers.0.fc1",
+    "model.decoder.layers.4.fc2",
+    "model.decoder.layers.3.fc2",
+    "model.encoder.layers.3.fc2",
+    "model.decoder.layers.3.fc1",
+    "model.decoder.layers.2.fc2",
+    "model.encoder.layers.1.fc1",
+    "model.decoder.layers.5.fc1"
+  ],
+  "task_type": "SEQ_2_SEQ_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b89ebdcdd1526c540859ee6d1c964e45224b8e532ab7c36722c70ab6c9dcd18c
+size 15734784

runs/Apr28_21-20-07_757a20c23d9b/events.out.tfevents.1714339233.757a20c23d9b.200681.12

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:076250396e2235755edafaccbd6ab53a0cd5cba54917d2f978ef93efd9b8a5b3
+size 10877

runs/Apr28_21-20-07_757a20c23d9b/events.out.tfevents.1714339295.757a20c23d9b.200681.13

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b909f0505cbc1155ba452655fa2bfd6dff2666c5279a0ee282adeb1ce1b39f2d
+size 10877

runs/Apr28_21-20-07_757a20c23d9b/events.out.tfevents.1714339358.757a20c23d9b.200681.14

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e8da711db1bb5d8485cdd859564e4cd3b2dc7061ac606d366eb1dd0a2490cfd
+size 10877

special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "eos_token": "</s>",
+  "pad_token": "<pad>",
+  "unk_token": "<unk>"
+}

tokenizer_config.json

@@ -0,0 +1,38 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "65000": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "</s>",
+  "model_max_length": 512,
+  "pad_token": "<pad>",
+  "separate_vocabs": false,
+  "source_lang": "fr+fr_BE+fr_CA+fr_FR+wa+frp+oc+ca+rm+lld+fur+lij+lmo+es+es_AR+es_CL+es_CO+es_CR+es_DO+es_EC+es_ES+es_GT+es_HN+es_MX+es_NI+es_PA+es_PE+es_PR+es_SV+es_UY+es_VE+pt+pt_br+pt_BR+pt_PT+gl+lad+an+mwl+it+it_IT+co+nap+scn+vec+sc+ro+la",
+  "sp_model_kwargs": {},
+  "target_lang": "en",
+  "tokenizer_class": "MarianTokenizer",
+  "unk_token": "<unk>"
+}

trainer_state.json

@@ -0,0 +1,198 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 9.6,
+  "eval_steps": 500,
+  "global_step": 120,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.4,
+      "grad_norm": 0.46419957280158997,
+      "learning_rate": 0.0001999633286223284,
+      "loss": 0.5697,
+      "step": 5
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 0.47279733419418335,
+      "learning_rate": 0.00019868265225415265,
+      "loss": 0.581,
+      "step": 10
+    },
+    {
+      "epoch": 1.2,
+      "grad_norm": 0.34838807582855225,
+      "learning_rate": 0.00019559521426716118,
+      "loss": 0.5531,
+      "step": 15
+    },
+    {
+      "epoch": 1.6,
+      "grad_norm": 0.4025287628173828,
+      "learning_rate": 0.00019075754196709572,
+      "loss": 0.528,
+      "step": 20
+    },
+    {
+      "epoch": 2.0,
+      "grad_norm": 0.5065444707870483,
+      "learning_rate": 0.0001842582073616649,
+      "loss": 0.5522,
+      "step": 25
+    },
+    {
+      "epoch": 2.4,
+      "grad_norm": 0.47670507431030273,
+      "learning_rate": 0.00017621620551276366,
+      "loss": 0.5206,
+      "step": 30
+    },
+    {
+      "epoch": 2.8,
+      "grad_norm": 0.39348119497299194,
+      "learning_rate": 0.00016677877587886956,
+      "loss": 0.5305,
+      "step": 35
+    },
+    {
+      "epoch": 3.2,
+      "grad_norm": 0.5051579475402832,
+      "learning_rate": 0.00015611870653623825,
+      "loss": 0.5061,
+      "step": 40
+    },
+    {
+      "epoch": 3.6,
+      "grad_norm": 0.43032407760620117,
+      "learning_rate": 0.00014443117063539038,
+      "loss": 0.4894,
+      "step": 45
+    },
+    {
+      "epoch": 4.0,
+      "grad_norm": 0.42734888195991516,
+      "learning_rate": 0.000131930153013598,
+      "loss": 0.4779,
+      "step": 50
+    },
+    {
+      "epoch": 4.4,
+      "grad_norm": 0.3881720006465912,
+      "learning_rate": 0.00011884453238783185,
+      "loss": 0.4904,
+      "step": 55
+    },
+    {
+      "epoch": 4.8,
+      "grad_norm": 0.4254741072654724,
+      "learning_rate": 0.00010541389085854176,
+      "loss": 0.4678,
+      "step": 60
+    },
+    {
+      "epoch": 5.2,
+      "grad_norm": 0.3979746997356415,
+      "learning_rate": 9.18841274472569e-05,
+      "loss": 0.4605,
+      "step": 65
+    },
+    {
+      "epoch": 5.6,
+      "grad_norm": 0.5344672799110413,
+      "learning_rate": 7.85029559788976e-05,
+      "loss": 0.4667,
+      "step": 70
+    },
+    {
+      "epoch": 6.0,
+      "grad_norm": 0.44632646441459656,
+      "learning_rate": 6.551536973720298e-05,
+      "loss": 0.4588,
+      "step": 75
+    },
+    {
+      "epoch": 6.4,
+      "grad_norm": 0.45458871126174927,
+      "learning_rate": 5.3159155930021e-05,
+      "loss": 0.4698,
+      "step": 80
+    },
+    {
+      "epoch": 6.8,
+      "grad_norm": 0.46564164757728577,
+      "learning_rate": 4.16605420892506e-05,
+      "loss": 0.4429,
+      "step": 85
+    },
+    {
+      "epoch": 7.2,
+      "grad_norm": 0.4069627523422241,
+      "learning_rate": 3.123005411465766e-05,
+      "loss": 0.451,
+      "step": 90
+    },
+    {
+      "epoch": 7.6,
+      "grad_norm": 0.3835998475551605,
+      "learning_rate": 2.205866179584084e-05,
+      "loss": 0.4238,
+      "step": 95
+    },
+    {
+      "epoch": 8.0,
+      "grad_norm": 0.4063926935195923,
+      "learning_rate": 1.4314282383241096e-05,
+      "loss": 0.4493,
+      "step": 100
+    },
+    {
+      "epoch": 8.4,
+      "grad_norm": 0.3951762318611145,
+      "learning_rate": 8.138706223637827e-06,
+      "loss": 0.4394,
+      "step": 105
+    },
+    {
+      "epoch": 8.8,
+      "grad_norm": 0.408200204372406,
+      "learning_rate": 3.6450007480777093e-06,
+      "loss": 0.4367,
+      "step": 110
+    },
+    {
+      "epoch": 9.2,
+      "grad_norm": 0.5846555233001709,
+      "learning_rate": 9.154403421193225e-07,
+      "loss": 0.429,
+      "step": 115
+    },
+    {
+      "epoch": 9.6,
+      "grad_norm": 0.3439011573791504,
+      "learning_rate": 0.0,
+      "loss": 0.4372,
+      "step": 120
+    },
+    {
+      "epoch": 9.6,
+      "step": 120,
+      "total_flos": 77528353996800.0,
+      "train_loss": 0.4846629718939463,
+      "train_runtime": 54.9394,
+      "train_samples_per_second": 18.202,
+      "train_steps_per_second": 2.184
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 120,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 10,
+  "save_steps": 500,
+  "total_flos": 77528353996800.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:99d9de74d1a3b9b5e5e92d75fa38b4585c6562b7ce7aaa44691e6d0f1d9a8f5b
+size 5176