huggingartists

README.md

@@ -0,0 +1,97 @@
+---
+language: en
+datasets:
+- huggingartists/oxxxymiron
+tags:
+- huggingartists
+- lyrics
+- lm-head
+- causal-lm
+widget:
+- text: "I am"
+---
+
+<div class="inline-flex flex-col" style="line-height: 1.5;">
+    <div class="flex">
+        <div
+			style="display:DISPLAY_1; margin-left: auto; margin-right: auto; width: 92px; height:92px; border-radius: 50%; background-size: cover; background-image: url(&#39;https://images.genius.com/e525200b65911e414a9d38c348eb1c6b.667x667x1.jpg&#39;)">
+        </div>
+    </div>
+    <div style="text-align: center; margin-top: 3px; font-size: 16px; font-weight: 800">🤖 HuggingArtists Model 🤖</div>
+    <div style="text-align: center; font-size: 16px; font-weight: 800">Oxxxymiron</div>
+    <a href="https://genius.com/artists/oxxxymiron">
+    	<div style="text-align: center; font-size: 14px;">@oxxxymiron</div>
+    </a>
+</div>
+
+I was made with [huggingartists](https://github.com/AlekseyKorshuk/huggingartists).
+
+Create your own bot based on your favorite artist with [the demo](https://colab.research.google.com/github/AlekseyKorshuk/huggingartists/blob/master/huggingartists-demo.ipynb)!
+
+## How does it work?
+
+To understand how the model was developed, check the [W&B report](https://wandb.ai/huggingartists/huggingartists/reportlist).
+
+## Training data
+
+The model was trained on lyrics from Oxxxymiron.
+
+Dataset is available [here](https://huggingface.co/datasets/huggingartists/oxxxymiron).
+And can be used with:
+
+```python
+from datasets import load_dataset
+
+dataset = load_dataset("huggingartists/oxxxymiron")
+```
+
+[Explore the data](https://wandb.ai/huggingartists/huggingartists/runs/1hspe90t/artifacts), which is tracked with [W&B artifacts](https://docs.wandb.com/artifacts) at every step of the pipeline.
+
+## Training procedure
+
+The model is based on a pre-trained [GPT-2](https://huggingface.co/gpt2) which is fine-tuned on Oxxxymiron's lyrics.
+
+Hyperparameters and metrics are recorded in the [W&B training run](https://wandb.ai/huggingartists/huggingartists/runs/128ekp1a) for full transparency and reproducibility.
+
+At the end of training, [the final model](https://wandb.ai/huggingartists/huggingartists/runs/128ekp1a/artifacts) is logged and versioned.
+
+## How to use
+
+You can use this model directly with a pipeline for text generation:
+
+```python
+from transformers import pipeline
+generator = pipeline('text-generation',
+                     model='huggingartists/oxxxymiron')
+generator("I am", num_return_sequences=5)
+```
+
+Or with Transformers library:
+
+```python
+from transformers import AutoTokenizer, AutoModelWithLMHead
+  
+tokenizer = AutoTokenizer.from_pretrained("huggingartists/oxxxymiron")
+
+model = AutoModelWithLMHead.from_pretrained("huggingartists/oxxxymiron")
+```
+
+## Limitations and bias
+
+The model suffers from [the same limitations and bias as GPT-2](https://huggingface.co/gpt2#limitations-and-bias).
+
+In addition, the data present in the user's tweets further affects the text generated by the model.
+
+## About
+
+*Built by Aleksey Korshuk*
+
+[![Follow](https://img.shields.io/github/followers/AlekseyKorshuk?style=social)](https://github.com/AlekseyKorshuk)
+
+[![Follow](https://img.shields.io/twitter/follow/alekseykorshuk?style=social)](https://twitter.com/intent/follow?screen_name=alekseykorshuk)
+
+[![Follow](https://img.shields.io/badge/dynamic/json?color=blue&label=Telegram%20Channel&query=%24.result&url=https%3A%2F%2Fapi.telegram.org%2Fbot1929545866%3AAAFGhV-KKnegEcLiyYJxsc4zV6C-bdPEBtQ%2FgetChatMemberCount%3Fchat_id%3D-1001253621662&style=social&logo=telegram)](https://t.me/joinchat/_CQ04KjcJ-4yZTky)
+
+For more details, visit the project repository.
+
+[![GitHub stars](https://img.shields.io/github/stars/AlekseyKorshuk/huggingartists?style=social)](https://github.com/AlekseyKorshuk/huggingartists)

config.json

@@ -0,0 +1,41 @@
+{
+  "_name_or_path": "gpt2",
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 50256,
+  "embd_pdrop": 0.1,
+  "eos_token_id": 50256,
+  "gradient_checkpointing": false,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_ctx": 1024,
+  "n_embd": 768,
+  "n_head": 12,
+  "n_inner": null,
+  "n_layer": 12,
+  "n_positions": 1024,
+  "resid_pdrop": 0.1,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "task_specific_params": {
+    "text-generation": {
+      "do_sample": true,
+      "max_length": 200,
+      "min_length": 100,
+      "temperature": 1.0,
+      "top_p": 0.95
+    }
+  },
+  "torch_dtype": "float32",
+  "transformers_version": "4.9.2",
+  "use_cache": true,
+  "vocab_size": 50257
+}

flax_model.msgpack

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ac4cf6a4bf61083f345dafb51f48b425222e4be63498c60332f05a6f2e0e22b
+size 497764120

optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e93f0da009d8b801c5faae8d9843a872ebfab4e3039cb0bffa0fa0875fad771
+size 995604017

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b291f749d6354df814b9ba0c5b259b0e94ea9e3d97b132031e674298a1ad7e1
+size 510403817

rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d74db31a0d8982efe9dc40546a3693dc5af9aacf52ce2f4a7551ad7e3e1e232
+size 14503

scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:24e3a97a9e192b59907cc3ef0039e0bc2d4534437003eccea3487bcaea3049db
+size 623

special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": "<|endoftext|>", "eos_token": "<|endoftext|>", "unk_token": "<|endoftext|>"}

tokenizer_config.json

@@ -0,0 +1 @@
+{"unk_token": "<|endoftext|>", "bos_token": "<|endoftext|>", "eos_token": "<|endoftext|>", "add_prefix_space": false, "model_max_length": 1024, "special_tokens_map_file": null, "name_or_path": "gpt2", "tokenizer_class": "GPT2Tokenizer"}

trainer_state.json

@@ -0,0 +1,946 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "global_step": 777,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.02,
+      "learning_rate": 0.00013707387498762246,
+      "loss": 2.6224,
+      "step": 5
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 0.00013669596372654658,
+      "loss": 2.5403,
+      "step": 10
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 0.00013606765583958522,
+      "loss": 2.4692,
+      "step": 15
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 0.00013519126168650572,
+      "loss": 2.3553,
+      "step": 20
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 0.00013407000386857356,
+      "loss": 2.2478,
+      "step": 25
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 0.0001327080053786798,
+      "loss": 2.4781,
+      "step": 30
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 0.0001311102744406261,
+      "loss": 2.2512,
+      "step": 35
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 0.00012928268609331444,
+      "loss": 2.2522,
+      "step": 40
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 0.00012723196058755901,
+      "loss": 2.4658,
+      "step": 45
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 0.00012496563867495734,
+      "loss": 2.2714,
+      "step": 50
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 0.0001224920538796866,
+      "loss": 2.2569,
+      "step": 55
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 0.00011982030185518478,
+      "loss": 2.2897,
+      "step": 60
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00011696020693839531,
+      "loss": 2.1927,
+      "step": 65
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00011392228602455959,
+      "loss": 2.1867,
+      "step": 70
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00011071770989539364,
+      "loss": 2.1953,
+      "step": 75
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00010735826214284955,
+      "loss": 2.2485,
+      "step": 80
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00010385629583950372,
+      "loss": 2.1775,
+      "step": 85
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00010022468811490008,
+      "loss": 2.2653,
+      "step": 90
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 9.647679280487579e-05,
+      "loss": 2.2253,
+      "step": 95
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 9.262639134798382e-05,
+      "loss": 2.2196,
+      "step": 100
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 8.868764210957135e-05,
+      "loss": 2.1454,
+      "step": 105
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 8.467502831985555e-05,
+      "loss": 2.1683,
+      "step": 110
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 8.060330481743385e-05,
+      "loss": 2.1135,
+      "step": 115
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 7.648744379405968e-05,
+      "loss": 2.2169,
+      "step": 120
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 7.234257974018543e-05,
+      "loss": 2.0504,
+      "step": 125
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 6.818395379371482e-05,
+      "loss": 2.1956,
+      "step": 130
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 6.40268576966004e-05,
+      "loss": 2.1556,
+      "step": 135
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 5.9886577565364115e-05,
+      "loss": 2.1258,
+      "step": 140
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 5.577833768230335e-05,
+      "loss": 2.0906,
+      "step": 145
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 5.171724451406823e-05,
+      "loss": 2.0648,
+      "step": 150
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 4.7718231163460816e-05,
+      "loss": 2.1031,
+      "step": 155
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 4.379600245871268e-05,
+      "loss": 2.456,
+      "step": 160
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 3.99649808821543e-05,
+      "loss": 2.1486,
+      "step": 165
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 3.6239253537102286e-05,
+      "loss": 2.1327,
+      "step": 170
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 3.263252034797391e-05,
+      "loss": 2.1147,
+      "step": 175
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 2.9158043684102103e-05,
+      "loss": 2.0989,
+      "step": 180
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 2.5828599592490882e-05,
+      "loss": 2.082,
+      "step": 185
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 2.265643081883308e-05,
+      "loss": 2.0394,
+      "step": 190
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 1.9653201789538038e-05,
+      "loss": 2.0346,
+      "step": 195
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 1.682995572030529e-05,
+      "loss": 2.1082,
+      "step": 200
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 1.419707400896067e-05,
+      "loss": 2.1204,
+      "step": 205
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.1764238061872434e-05,
+      "loss": 1.976,
+      "step": 210
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 9.540393694315812e-06,
+      "loss": 1.9811,
+      "step": 215
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 7.533718235689727e-06,
+      "loss": 2.0259,
+      "step": 220
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 5.751590460543944e-06,
+      "loss": 2.0717,
+      "step": 225
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 4.2005634559834046e-06,
+      "loss": 2.0042,
+      "step": 230
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 2.8863405252193965e-06,
+      "loss": 2.0301,
+      "step": 235
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 1.8137542158731914e-06,
+      "loss": 2.0554,
+      "step": 240
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 9.867485501471999e-07,
+      "loss": 2.0167,
+      "step": 245
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 4.083645222054329e-07,
+      "loss": 2.0252,
+      "step": 250
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 8.072891609113784e-08,
+      "loss": 2.0128,
+      "step": 255
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 5.046485299251069e-09,
+      "loss": 2.1545,
+      "step": 260
+    },
+    {
+      "epoch": 1.02,
+      "learning_rate": 1.815955227603999e-07,
+      "loss": 2.1436,
+      "step": 265
+    },
+    {
+      "epoch": 1.04,
+      "learning_rate": 6.097268375260298e-07,
+      "loss": 2.055,
+      "step": 270
+    },
+    {
+      "epoch": 1.06,
+      "learning_rate": 1.2878661419176275e-06,
+      "loss": 2.0446,
+      "step": 275
+    },
+    {
+      "epoch": 1.08,
+      "learning_rate": 2.2135198403619775e-06,
+      "loss": 2.0962,
+      "step": 280
+    },
+    {
+      "epoch": 1.1,
+      "learning_rate": 3.3832841986266328e-06,
+      "loss": 1.9858,
+      "step": 285
+    },
+    {
+      "epoch": 1.12,
+      "learning_rate": 4.7928578597388414e-06,
+      "loss": 2.0331,
+      "step": 290
+    },
+    {
+      "epoch": 1.14,
+      "learning_rate": 6.437057660565834e-06,
+      "loss": 2.0861,
+      "step": 295
+    },
+    {
+      "epoch": 1.16,
+      "learning_rate": 8.309837690896773e-06,
+      "loss": 2.0373,
+      "step": 300
+    },
+    {
+      "epoch": 1.18,
+      "learning_rate": 1.0404311524944352e-05,
+      "loss": 2.0409,
+      "step": 305
+    },
+    {
+      "epoch": 1.2,
+      "learning_rate": 1.271277754351776e-05,
+      "loss": 2.1014,
+      "step": 310
+    },
+    {
+      "epoch": 1.22,
+      "learning_rate": 1.5226747253755048e-05,
+      "loss": 2.0414,
+      "step": 315
+    },
+    {
+      "epoch": 1.24,
+      "learning_rate": 1.79369765022795e-05,
+      "loss": 2.1258,
+      "step": 320
+    },
+    {
+      "epoch": 1.25,
+      "learning_rate": 2.083349946700612e-05,
+      "loss": 2.0382,
+      "step": 325
+    },
+    {
+      "epoch": 1.27,
+      "learning_rate": 2.3905665302606086e-05,
+      "loss": 1.9499,
+      "step": 330
+    },
+    {
+      "epoch": 1.29,
+      "learning_rate": 2.7142177304879985e-05,
+      "loss": 2.1053,
+      "step": 335
+    },
+    {
+      "epoch": 1.31,
+      "learning_rate": 3.05311344500276e-05,
+      "loss": 2.0373,
+      "step": 340
+    },
+    {
+      "epoch": 1.33,
+      "learning_rate": 3.4060075156069894e-05,
+      "loss": 1.9227,
+      "step": 345
+    },
+    {
+      "epoch": 1.35,
+      "learning_rate": 3.771602310550759e-05,
+      "loss": 2.0504,
+      "step": 350
+    },
+    {
+      "epoch": 1.37,
+      "learning_rate": 4.148553496072039e-05,
+      "loss": 2.0583,
+      "step": 355
+    },
+    {
+      "epoch": 1.39,
+      "learning_rate": 4.5354749796652995e-05,
+      "loss": 2.0166,
+      "step": 360
+    },
+    {
+      "epoch": 1.41,
+      "learning_rate": 4.930944006901758e-05,
+      "loss": 2.0524,
+      "step": 365
+    },
+    {
+      "epoch": 1.43,
+      "learning_rate": 5.333506393059682e-05,
+      "loss": 2.0846,
+      "step": 370
+    },
+    {
+      "epoch": 1.45,
+      "learning_rate": 5.741681870327513e-05,
+      "loss": 2.0408,
+      "step": 375
+    },
+    {
+      "epoch": 1.47,
+      "learning_rate": 6.153969530917408e-05,
+      "loss": 1.9538,
+      "step": 380
+    },
+    {
+      "epoch": 1.49,
+      "learning_rate": 6.56885334607442e-05,
+      "loss": 2.1089,
+      "step": 385
+    },
+    {
+      "epoch": 1.51,
+      "learning_rate": 6.984807740687121e-05,
+      "loss": 2.0182,
+      "step": 390
+    },
+    {
+      "epoch": 1.53,
+      "learning_rate": 7.400303203001308e-05,
+      "loss": 1.9797,
+      "step": 395
+    },
+    {
+      "epoch": 1.54,
+      "learning_rate": 7.813811908809188e-05,
+      "loss": 2.0459,
+      "step": 400
+    },
+    {
+      "epoch": 1.56,
+      "learning_rate": 8.22381333943327e-05,
+      "loss": 2.0528,
+      "step": 405
+    },
+    {
+      "epoch": 1.58,
+      "learning_rate": 8.628799872846947e-05,
+      "loss": 2.0164,
+      "step": 410
+    },
+    {
+      "epoch": 1.6,
+      "learning_rate": 9.027282327372695e-05,
+      "loss": 1.973,
+      "step": 415
+    },
+    {
+      "epoch": 1.62,
+      "learning_rate": 9.417795437572915e-05,
+      "loss": 2.1012,
+      "step": 420
+    },
+    {
+      "epoch": 1.64,
+      "learning_rate": 9.798903242198118e-05,
+      "loss": 1.9342,
+      "step": 425
+    },
+    {
+      "epoch": 1.66,
+      "learning_rate": 0.00010169204364380207,
+      "loss": 1.9812,
+      "step": 430
+    },
+    {
+      "epoch": 1.68,
+      "learning_rate": 0.0001052733716465509,
+      "loss": 2.1037,
+      "step": 435
+    },
+    {
+      "epoch": 1.7,
+      "learning_rate": 0.0001087198474786628,
+      "loss": 2.0833,
+      "step": 440
+    },
+    {
+      "epoch": 1.72,
+      "learning_rate": 0.00011201879805538593,
+      "loss": 2.088,
+      "step": 445
+    },
+    {
+      "epoch": 1.74,
+      "learning_rate": 0.00011515809275916009,
+      "loss": 2.0211,
+      "step": 450
+    },
+    {
+      "epoch": 1.76,
+      "learning_rate": 0.00011812618804528036,
+      "loss": 1.9749,
+      "step": 455
+    },
+    {
+      "epoch": 1.78,
+      "learning_rate": 0.00012091216988882859,
+      "loss": 2.1269,
+      "step": 460
+    },
+    {
+      "epoch": 1.8,
+      "learning_rate": 0.0001235057939167871,
+      "loss": 2.1407,
+      "step": 465
+    },
+    {
+      "epoch": 1.81,
+      "learning_rate": 0.0001258975230777674,
+      "loss": 1.9911,
+      "step": 470
+    },
+    {
+      "epoch": 1.83,
+      "learning_rate": 0.00012807856271083559,
+      "loss": 1.8852,
+      "step": 475
+    },
+    {
+      "epoch": 1.85,
+      "learning_rate": 0.00013004089288448385,
+      "loss": 2.084,
+      "step": 480
+    },
+    {
+      "epoch": 1.87,
+      "learning_rate": 0.00013177729788683344,
+      "loss": 2.0223,
+      "step": 485
+    },
+    {
+      "epoch": 1.89,
+      "learning_rate": 0.00013328139275863037,
+      "loss": 1.8615,
+      "step": 490
+    },
+    {
+      "epoch": 1.91,
+      "learning_rate": 0.00013454764677146868,
+      "loss": 1.9669,
+      "step": 495
+    },
+    {
+      "epoch": 1.93,
+      "learning_rate": 0.00013557140376490993,
+      "loss": 1.997,
+      "step": 500
+    },
+    {
+      "epoch": 1.95,
+      "learning_rate": 0.00013634889926771582,
+      "loss": 2.0132,
+      "step": 505
+    },
+    {
+      "epoch": 1.97,
+      "learning_rate": 0.00013687727434023874,
+      "loss": 2.1001,
+      "step": 510
+    },
+    {
+      "epoch": 1.99,
+      "learning_rate": 0.00013715458608706872,
+      "loss": 1.9605,
+      "step": 515
+    },
+    {
+      "epoch": 2.01,
+      "learning_rate": 0.00013717981480128154,
+      "loss": 1.9686,
+      "step": 520
+    },
+    {
+      "epoch": 2.03,
+      "learning_rate": 0.0001369528677140173,
+      "loss": 1.9054,
+      "step": 525
+    },
+    {
+      "epoch": 2.05,
+      "learning_rate": 0.00013647457933560234,
+      "loss": 1.9621,
+      "step": 530
+    },
+    {
+      "epoch": 2.07,
+      "learning_rate": 0.00013574670838695924,
+      "loss": 1.9001,
+      "step": 535
+    },
+    {
+      "epoch": 2.08,
+      "learning_rate": 0.00013477193133258972,
+      "loss": 1.9553,
+      "step": 540
+    },
+    {
+      "epoch": 2.1,
+      "learning_rate": 0.0001335538325389091,
+      "loss": 1.9963,
+      "step": 545
+    },
+    {
+      "epoch": 2.12,
+      "learning_rate": 0.00013209689109412243,
+      "loss": 1.9554,
+      "step": 550
+    },
+    {
+      "epoch": 2.14,
+      "learning_rate": 0.00013040646433810595,
+      "loss": 2.0022,
+      "step": 555
+    },
+    {
+      "epoch": 2.16,
+      "learning_rate": 0.00012848876816285752,
+      "loss": 1.8702,
+      "step": 560
+    },
+    {
+      "epoch": 2.18,
+      "learning_rate": 0.00012635085415595263,
+      "loss": 1.9121,
+      "step": 565
+    },
+    {
+      "epoch": 2.2,
+      "learning_rate": 0.00012400058367105258,
+      "loss": 2.0007,
+      "step": 570
+    },
+    {
+      "epoch": 2.22,
+      "learning_rate": 0.0001214465989208104,
+      "loss": 2.0254,
+      "step": 575
+    },
+    {
+      "epoch": 2.24,
+      "learning_rate": 0.00011869829119846914,
+      "loss": 1.9571,
+      "step": 580
+    },
+    {
+      "epoch": 2.26,
+      "learning_rate": 0.00011576576634500534,
+      "loss": 1.9448,
+      "step": 585
+    },
+    {
+      "epoch": 2.28,
+      "learning_rate": 0.00011265980758879933,
+      "loss": 2.0852,
+      "step": 590
+    },
+    {
+      "epoch": 2.3,
+      "learning_rate": 0.00010939183589447406,
+      "loss": 1.9757,
+      "step": 595
+    },
+    {
+      "epoch": 2.32,
+      "learning_rate": 0.00010597386796670587,
+      "loss": 2.0051,
+      "step": 600
+    },
+    {
+      "epoch": 2.34,
+      "learning_rate": 0.0001024184720634304,
+      "loss": 2.015,
+      "step": 605
+    },
+    {
+      "epoch": 2.36,
+      "learning_rate": 9.873872178092492e-05,
+      "loss": 1.9478,
+      "step": 610
+    },
+    {
+      "epoch": 2.37,
+      "learning_rate": 9.494814798070336e-05,
+      "loss": 1.8666,
+      "step": 615
+    },
+    {
+      "epoch": 2.39,
+      "learning_rate": 9.106068903499513e-05,
+      "loss": 1.9268,
+      "step": 620
+    },
+    {
+      "epoch": 2.41,
+      "learning_rate": 8.709063957376078e-05,
+      "loss": 1.8729,
+      "step": 625
+    },
+    {
+      "epoch": 2.43,
+      "learning_rate": 8.305259792170682e-05,
+      "loss": 1.9046,
+      "step": 630
+    },
+    {
+      "epoch": 2.45,
+      "learning_rate": 7.89614124185811e-05,
+      "loss": 1.8255,
+      "step": 635
+    },
+    {
+      "epoch": 2.47,
+      "learning_rate": 7.48321268201335e-05,
+      "loss": 1.9063,
+      "step": 640
+    },
+    {
+      "epoch": 2.49,
+      "learning_rate": 7.06799249805101e-05,
+      "loss": 1.9487,
+      "step": 645
+    },
+    {
+      "epoch": 2.51,
+      "learning_rate": 6.652007501948994e-05,
+      "loss": 1.9612,
+      "step": 650
+    },
+    {
+      "epoch": 2.53,
+      "learning_rate": 6.236787317986654e-05,
+      "loss": 1.829,
+      "step": 655
+    },
+    {
+      "epoch": 2.55,
+      "learning_rate": 5.8238587581418946e-05,
+      "loss": 1.8833,
+      "step": 660
+    },
+    {
+      "epoch": 2.57,
+      "learning_rate": 5.414740207829316e-05,
+      "loss": 1.9743,
+      "step": 665
+    },
+    {
+      "epoch": 2.59,
+      "learning_rate": 5.010936042623931e-05,
+      "loss": 1.8592,
+      "step": 670
+    },
+    {
+      "epoch": 2.61,
+      "learning_rate": 4.6139310965004967e-05,
+      "loss": 1.8143,
+      "step": 675
+    },
+    {
+      "epoch": 2.63,
+      "learning_rate": 4.225185201929667e-05,
+      "loss": 1.9597,
+      "step": 680
+    },
+    {
+      "epoch": 2.64,
+      "learning_rate": 3.8461278219075114e-05,
+      "loss": 1.9673,
+      "step": 685
+    },
+    {
+      "epoch": 2.66,
+      "learning_rate": 3.478152793656968e-05,
+      "loss": 1.9703,
+      "step": 690
+    },
+    {
+      "epoch": 2.68,
+      "learning_rate": 3.1226132033294165e-05,
+      "loss": 1.8479,
+      "step": 695
+    },
+    {
+      "epoch": 2.7,
+      "learning_rate": 2.7808164105525978e-05,
+      "loss": 1.8916,
+      "step": 700
+    },
+    {
+      "epoch": 2.72,
+      "learning_rate": 2.454019241120065e-05,
+      "loss": 1.9233,
+      "step": 705
+    },
+    {
+      "epoch": 2.74,
+      "learning_rate": 2.1434233654994646e-05,
+      "loss": 1.8525,
+      "step": 710
+    },
+    {
+      "epoch": 2.76,
+      "learning_rate": 1.850170880153093e-05,
+      "loss": 1.9783,
+      "step": 715
+    },
+    {
+      "epoch": 2.78,
+      "learning_rate": 1.5753401079189615e-05,
+      "loss": 1.8898,
+      "step": 720
+    },
+    {
+      "epoch": 2.8,
+      "learning_rate": 1.3199416328947464e-05,
+      "loss": 1.9018,
+      "step": 725
+    },
+    {
+      "epoch": 2.82,
+      "learning_rate": 1.0849145844047363e-05,
+      "loss": 1.9754,
+      "step": 730
+    },
+    {
+      "epoch": 2.84,
+      "learning_rate": 8.711231837142545e-06,
+      "loss": 1.9263,
+      "step": 735
+    },
+    {
+      "epoch": 2.86,
+      "learning_rate": 6.793535661894062e-06,
+      "loss": 1.836,
+      "step": 740
+    },
+    {
+      "epoch": 2.88,
+      "learning_rate": 5.103108905877591e-06,
+      "loss": 1.9056,
+      "step": 745
+    },
+    {
+      "epoch": 2.9,
+      "learning_rate": 3.6461674610908713e-06,
+      "loss": 1.9471,
+      "step": 750
+    },
+    {
+      "epoch": 2.92,
+      "learning_rate": 2.4280686674102973e-06,
+      "loss": 1.7961,
+      "step": 755
+    },
+    {
+      "epoch": 2.93,
+      "learning_rate": 1.453291613040777e-06,
+      "loss": 1.9143,
+      "step": 760
+    },
+    {
+      "epoch": 2.95,
+      "learning_rate": 7.254206643976737e-07,
+      "loss": 1.9633,
+      "step": 765
+    },
+    {
+      "epoch": 2.97,
+      "learning_rate": 2.4713228598268823e-07,
+      "loss": 1.9096,
+      "step": 770
+    },
+    {
+      "epoch": 2.99,
+      "learning_rate": 2.018519871846962e-08,
+      "loss": 1.7972,
+      "step": 775
+    }
+  ],
+  "max_steps": 777,
+  "num_train_epochs": 3,
+  "total_flos": 811703697408000.0,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:100c85c50ebb4281d2e3e83d1d7e4182a972554b3437d8264b44adb0e8333b79
+size 2671