Upload 11 files

adapter_config.json

@@ -0,0 +1,28 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "Qwen/Qwen-7B-Chat",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "c_attn"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ccb0d95949741c36479d6bf4467bccf75c1ef0b0201a2bd35f030ff99504b58
+size 33562824

all_results.json

@@ -0,0 +1,7 @@
+{
+    "epoch": 5.0,
+    "train_loss": 0.886494568245396,
+    "train_runtime": 33369.3902,
+    "train_samples_per_second": 1.148,
+    "train_steps_per_second": 0.287
+}

special_tokens_map.json

@@ -0,0 +1,10 @@
+{
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|im_end|>"
+}

tokenization_qwen.py

@@ -0,0 +1,276 @@
+# Copyright (c) Alibaba Cloud.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""Tokenization classes for QWen."""
+
+import base64
+import logging
+import os
+import unicodedata
+from typing import Collection, Dict, List, Set, Tuple, Union
+
+import tiktoken
+from transformers import PreTrainedTokenizer, AddedToken
+
+logger = logging.getLogger(__name__)
+
+
+VOCAB_FILES_NAMES = {"vocab_file": "qwen.tiktoken"}
+
+PAT_STR = r"""(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+"""
+ENDOFTEXT = "<|endoftext|>"
+IMSTART = "<|im_start|>"
+IMEND = "<|im_end|>"
+# as the default behavior is changed to allow special tokens in
+# regular texts, the surface forms of special tokens need to be
+# as different as possible to minimize the impact
+EXTRAS = tuple((f"<|extra_{i}|>" for i in range(205)))
+# changed to use actual index to avoid misconfiguration with vocabulary expansion
+SPECIAL_START_ID = 151643
+SPECIAL_TOKENS = tuple(
+    enumerate(
+        (
+            (
+                ENDOFTEXT,
+                IMSTART,
+                IMEND,
+            )
+            + EXTRAS
+        ),
+        start=SPECIAL_START_ID,
+    )
+)
+SPECIAL_TOKENS_SET = set(t for i, t in SPECIAL_TOKENS)
+
+
+def _load_tiktoken_bpe(tiktoken_bpe_file: str) -> Dict[bytes, int]:
+    with open(tiktoken_bpe_file, "rb") as f:
+        contents = f.read()
+    return {
+        base64.b64decode(token): int(rank)
+        for token, rank in (line.split() for line in contents.splitlines() if line)
+    }
+
+
+class QWenTokenizer(PreTrainedTokenizer):
+    """QWen tokenizer."""
+
+    vocab_files_names = VOCAB_FILES_NAMES
+
+    def __init__(
+        self,
+        vocab_file,
+        errors="replace",
+        extra_vocab_file=None,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+
+        # how to handle errors in decoding UTF-8 byte sequences
+        # use ignore if you are in streaming inference
+        self.errors = errors  
+
+        self.mergeable_ranks = _load_tiktoken_bpe(vocab_file)  # type: Dict[bytes, int]
+        self.special_tokens = {
+            token: index
+            for index, token in SPECIAL_TOKENS
+        }
+
+        # try load extra vocab from file
+        if extra_vocab_file is not None:
+            used_ids = set(self.mergeable_ranks.values()) | set(self.special_tokens.values())
+            extra_mergeable_ranks = _load_tiktoken_bpe(extra_vocab_file)
+            for token, index in extra_mergeable_ranks.items():
+                if token in self.mergeable_ranks:
+                    logger.info(f"extra token {token} exists, skipping")
+                    continue
+                if index in used_ids:
+                    logger.info(f'the index {index} for extra token {token} exists, skipping')
+                    continue
+                self.mergeable_ranks[token] = index
+            # the index may be sparse after this, but don't worry tiktoken.Encoding will handle this
+
+        enc = tiktoken.Encoding(
+            "Qwen",
+            pat_str=PAT_STR,
+            mergeable_ranks=self.mergeable_ranks,
+            special_tokens=self.special_tokens,
+        )
+        assert (
+            len(self.mergeable_ranks) + len(self.special_tokens) == enc.n_vocab
+        ), f"{len(self.mergeable_ranks) + len(self.special_tokens)} != {enc.n_vocab} in encoding"
+
+        self.decoder = {
+            v: k for k, v in self.mergeable_ranks.items()
+        }  # type: dict[int, bytes|str]
+        self.decoder.update({v: k for k, v in self.special_tokens.items()})
+
+        self.tokenizer = enc  # type: tiktoken.Encoding
+
+        self.eod_id = self.tokenizer.eot_token
+        self.im_start_id = self.special_tokens[IMSTART]
+        self.im_end_id = self.special_tokens[IMEND]
+
+    def __getstate__(self):
+        # for pickle lovers
+        state = self.__dict__.copy()
+        del state["tokenizer"]
+        return state
+
+    def __setstate__(self, state):
+        # tokenizer is not python native; don't pass it; rebuild it
+        self.__dict__.update(state)
+        enc = tiktoken.Encoding(
+            "Qwen",
+            pat_str=PAT_STR,
+            mergeable_ranks=self.mergeable_ranks,
+            special_tokens=self.special_tokens,
+        )
+        self.tokenizer = enc
+
+    def __len__(self) -> int:
+        return self.tokenizer.n_vocab
+
+    def get_vocab(self) -> Dict[bytes, int]:
+        return self.mergeable_ranks
+
+    def convert_tokens_to_ids(
+        self, tokens: Union[bytes, str, List[Union[bytes, str]]]
+    ) -> List[int]:
+        ids = []
+        if isinstance(tokens, (str, bytes)):
+            if tokens in self.special_tokens:
+                return self.special_tokens[tokens]
+            else:
+                return self.mergeable_ranks.get(tokens)
+        for token in tokens:
+            if token in self.special_tokens:
+                ids.append(self.special_tokens[token])
+            else:
+                ids.append(self.mergeable_ranks.get(token))
+        return ids
+
+    def _add_tokens(
+        self,
+        new_tokens: Union[List[str], List[AddedToken]],
+        special_tokens: bool = False,
+    ) -> int:
+        if not special_tokens and new_tokens:
+            raise ValueError("Adding regular tokens is not supported")
+        for token in new_tokens:
+            surface_form = token.content if isinstance(token, AddedToken) else token
+            if surface_form not in SPECIAL_TOKENS_SET:
+                raise ValueError("Adding unknown special tokens is not supported")
+        return 0
+
+    def save_vocabulary(self, save_directory: str, **kwargs) -> Tuple[str]:
+        """
+        Save only the vocabulary of the tokenizer (vocabulary).
+
+        Returns:
+            `Tuple(str)`: Paths to the files saved.
+        """
+        file_path = os.path.join(save_directory, "qwen.tiktoken")
+        with open(file_path, "w", encoding="utf8") as w:
+            for k, v in self.mergeable_ranks.items():
+                line = base64.b64encode(k).decode("utf8") + " " + str(v) + "\n"
+                w.write(line)
+        return (file_path,)
+
+    def tokenize(
+        self,
+        text: str,
+        allowed_special: Union[Set, str] = "all",
+        disallowed_special: Union[Collection, str] = (),
+        **kwargs,
+    ) -> List[Union[bytes, str]]:
+        """
+        Converts a string in a sequence of tokens.
+
+        Args:
+            text (`str`):
+                The sequence to be encoded.
+            allowed_special (`Literal["all"]` or `set`):
+                The surface forms of the tokens to be encoded as special tokens in regular texts.
+                Default to "all".
+            disallowed_special (`Literal["all"]` or `Collection`):
+                The surface forms of the tokens that should not be in regular texts and trigger errors.
+                Default to an empty tuple.
+
+            kwargs (additional keyword arguments, *optional*):
+                Will be passed to the underlying model specific encode method.
+
+        Returns:
+            `List[bytes|str]`: The list of tokens.
+        """
+        tokens = []
+        text = unicodedata.normalize("NFC", text)
+
+        # this implementation takes a detour: text -> token id -> token surface forms
+        for t in self.tokenizer.encode(
+            text, allowed_special=allowed_special, disallowed_special=disallowed_special
+        ):
+            tokens.append(self.decoder[t])
+        return tokens
+
+    def convert_tokens_to_string(self, tokens: List[Union[bytes, str]]) -> str:
+        """
+        Converts a sequence of tokens in a single string.
+        """
+        text = ""
+        temp = b""
+        for t in tokens:
+            if isinstance(t, str):
+                if temp:
+                    text += temp.decode("utf-8", errors=self.errors)
+                    temp = b""
+                text += t
+            elif isinstance(t, bytes):
+                temp += t
+            else:
+                raise TypeError("token should only be of type types or str")
+        if temp:
+            text += temp.decode("utf-8", errors=self.errors)
+        return text
+
+    @property
+    def vocab_size(self):
+        return self.tokenizer.n_vocab
+
+    def _convert_id_to_token(self, index: int) -> Union[bytes, str]:
+        """Converts an id to a token, special tokens included"""
+        if index in self.decoder:
+            return self.decoder[index]
+        raise ValueError("unknown ids")
+
+    def _convert_token_to_id(self, token: Union[bytes, str]) -> int:
+        """Converts a token to an id using the vocab, special tokens included"""
+        if token in self.special_tokens:
+            return self.special_tokens[token]
+        if token in self.mergeable_ranks:
+            return self.mergeable_ranks[token]
+        raise ValueError("unknown token")
+
+    def _tokenize(self, text: str, **kwargs):
+        """
+        Converts a string in a sequence of tokens (string), using the tokenizer. Split in words for word-based
+        vocabulary or sub-words for sub-word-based vocabularies (BPE/SentencePieces/WordPieces).
+
+        Do NOT take care of added tokens.
+        """
+        raise NotImplementedError
+
+    def _decode(
+        self,
+        token_ids: Union[int, List[int]],
+        skip_special_tokens: bool = False,
+        errors: str = None,
+        **kwargs,
+    ) -> str:
+        if isinstance(token_ids, int):
+            token_ids = [token_ids]
+        if skip_special_tokens:
+            token_ids = [i for i in token_ids if i < self.eod_id]
+        return self.tokenizer.decode(token_ids, errors=errors or self.errors)

tokenizer_config.json

@@ -0,0 +1,17 @@
+{
+  "added_tokens_decoder": {},
+  "auto_map": {
+    "AutoTokenizer": [
+      "tokenization_qwen.QWenTokenizer",
+      null
+    ]
+  },
+  "chat_template": "{% set system_message = 'You are a helpful assistant.' %}{% if messages[0]['role'] == 'system' %}{% set system_message = messages[0]['content'] %}{% endif %}{% if system_message is defined %}{{ '<|im_start|>system\\n' + system_message + '<|im_end|>\\n' }}{% endif %}{% for message in messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ '<|im_start|>user\\n' + content + '<|im_end|>\\n<|im_start|>assistant\\n' }}{% elif message['role'] == 'assistant' %}{{ content + '<|im_end|>' + '\\n' }}{% endif %}{% endfor %}",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|im_end|>",
+  "model_max_length": 32768,
+  "pad_token": "<|im_end|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "QWenTokenizer"
+}

train_results.json

@@ -0,0 +1,7 @@
+{
+    "epoch": 5.0,
+    "train_loss": 0.886494568245396,
+    "train_runtime": 33369.3902,
+    "train_samples_per_second": 1.148,
+    "train_steps_per_second": 0.287
+}

trainer_log.jsonl

@@ -0,0 +1,96 @@
+{"current_steps": 100, "total_steps": 9580, "loss": 1.8222, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 0.00019994623498004714, "epoch": 0.05, "percentage": 1.04, "elapsed_time": "0:05:45", "remaining_time": "9:05:27"}
+{"current_steps": 200, "total_steps": 9580, "loss": 1.8565, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 0.00019978499773373596, "epoch": 0.1, "percentage": 2.09, "elapsed_time": "0:11:32", "remaining_time": "9:01:24"}
+{"current_steps": 300, "total_steps": 9580, "loss": 1.7994, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 0.00019951646163954176, "epoch": 0.16, "percentage": 3.13, "elapsed_time": "0:17:22", "remaining_time": "8:57:18"}
+{"current_steps": 400, "total_steps": 9580, "loss": 1.6984, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 0.0001991409154544338, "epoch": 0.21, "percentage": 4.18, "elapsed_time": "0:23:03", "remaining_time": "8:49:00"}
+{"current_steps": 500, "total_steps": 9580, "loss": 1.6973, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 0.00019865876300337478, "epoch": 0.26, "percentage": 5.22, "elapsed_time": "0:28:50", "remaining_time": "8:43:38"}
+{"current_steps": 600, "total_steps": 9580, "loss": 1.7661, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 0.00019807052274508773, "epoch": 0.31, "percentage": 6.26, "elapsed_time": "0:34:35", "remaining_time": "8:37:45"}
+{"current_steps": 700, "total_steps": 9580, "loss": 1.6948, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 0.00019737682721455714, "epoch": 0.37, "percentage": 7.31, "elapsed_time": "0:40:22", "remaining_time": "8:32:05"}
+{"current_steps": 800, "total_steps": 9580, "loss": 1.8664, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 0.0001965784223428638, "epoch": 0.42, "percentage": 8.35, "elapsed_time": "0:46:07", "remaining_time": "8:26:11"}
+{"current_steps": 900, "total_steps": 9580, "loss": 1.7087, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 0.00019567616665508485, "epoch": 0.47, "percentage": 9.39, "elapsed_time": "0:51:54", "remaining_time": "8:20:35"}
+{"current_steps": 1000, "total_steps": 9580, "loss": 1.6826, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 0.0001946710303471214, "epoch": 0.52, "percentage": 10.44, "elapsed_time": "0:57:38", "remaining_time": "8:14:34"}
+{"current_steps": 1100, "total_steps": 9580, "loss": 1.5399, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 0.00019356409424244655, "epoch": 0.57, "percentage": 11.48, "elapsed_time": "1:03:33", "remaining_time": "8:09:55"}
+{"current_steps": 1200, "total_steps": 9580, "loss": 1.6394, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 0.00019235654862989537, "epoch": 0.63, "percentage": 12.53, "elapsed_time": "1:09:19", "remaining_time": "8:04:10"}
+{"current_steps": 1300, "total_steps": 9580, "loss": 1.645, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 0.00019104969198374688, "epoch": 0.68, "percentage": 13.57, "elapsed_time": "1:15:06", "remaining_time": "7:58:25"}
+{"current_steps": 1400, "total_steps": 9580, "loss": 1.5853, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 0.00018964492956747425, "epoch": 0.73, "percentage": 14.61, "elapsed_time": "1:20:53", "remaining_time": "7:52:37"}
+{"current_steps": 1500, "total_steps": 9580, "loss": 1.5778, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 0.00018814377192266423, "epoch": 0.78, "percentage": 15.66, "elapsed_time": "1:26:39", "remaining_time": "7:46:48"}
+{"current_steps": 1600, "total_steps": 9580, "loss": 1.6368, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 0.00018654783324473137, "epoch": 0.84, "percentage": 16.7, "elapsed_time": "1:32:22", "remaining_time": "7:40:40"}
+{"current_steps": 1700, "total_steps": 9580, "loss": 1.6269, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 0.00018487617447307124, "epoch": 0.89, "percentage": 17.75, "elapsed_time": "1:38:04", "remaining_time": "7:34:38"}
+{"current_steps": 1800, "total_steps": 9580, "loss": 1.7073, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 0.00018309682531549338, "epoch": 0.94, "percentage": 18.79, "elapsed_time": "1:43:50", "remaining_time": "7:28:50"}
+{"current_steps": 1900, "total_steps": 9580, "loss": 1.5544, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 0.00018122812210849337, "epoch": 0.99, "percentage": 19.83, "elapsed_time": "1:49:36", "remaining_time": "7:23:02"}
+{"current_steps": 2000, "total_steps": 9580, "loss": 1.3285, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 0.00017927207426937544, "epoch": 1.04, "percentage": 20.88, "elapsed_time": "1:55:21", "remaining_time": "7:17:13"}
+{"current_steps": 2100, "total_steps": 9580, "loss": 1.2529, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 0.00017723078513716157, "epoch": 1.1, "percentage": 21.92, "elapsed_time": "2:01:14", "remaining_time": "7:11:51"}
+{"current_steps": 2200, "total_steps": 9580, "loss": 1.3708, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 0.00017510644971087015, "epoch": 1.15, "percentage": 22.96, "elapsed_time": "2:07:00", "remaining_time": "7:06:04"}
+{"current_steps": 2300, "total_steps": 9580, "loss": 1.3309, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 0.0001729013522892329, "epoch": 1.2, "percentage": 24.01, "elapsed_time": "2:12:47", "remaining_time": "7:00:17"}
+{"current_steps": 2400, "total_steps": 9580, "loss": 1.4201, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 0.0001706178640143872, "epoch": 1.25, "percentage": 25.05, "elapsed_time": "2:18:33", "remaining_time": "6:54:30"}
+{"current_steps": 2500, "total_steps": 9580, "loss": 1.2956, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 0.00016825844032218625, "epoch": 1.3, "percentage": 26.1, "elapsed_time": "2:24:20", "remaining_time": "6:48:46"}
+{"current_steps": 2600, "total_steps": 9580, "loss": 1.3375, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 0.00016582561830186785, "epoch": 1.36, "percentage": 27.14, "elapsed_time": "2:30:11", "remaining_time": "6:43:13"}
+{"current_steps": 2700, "total_steps": 9580, "loss": 1.2897, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 0.00016332201396792123, "epoch": 1.41, "percentage": 28.18, "elapsed_time": "2:35:57", "remaining_time": "6:37:24"}
+{"current_steps": 2800, "total_steps": 9580, "loss": 1.3323, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 0.00016075031944708584, "epoch": 1.46, "percentage": 29.23, "elapsed_time": "2:41:43", "remaining_time": "6:31:36"}
+{"current_steps": 2900, "total_steps": 9580, "loss": 1.2841, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 0.0001581133000835061, "epoch": 1.51, "percentage": 30.27, "elapsed_time": "2:47:30", "remaining_time": "6:25:50"}
+{"current_steps": 3000, "total_steps": 9580, "loss": 1.2666, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 0.00015541379146515603, "epoch": 1.57, "percentage": 31.32, "elapsed_time": "2:53:19", "remaining_time": "6:20:09"}
+{"current_steps": 3100, "total_steps": 9580, "loss": 1.2336, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 0.0001526546963747302, "epoch": 1.62, "percentage": 32.36, "elapsed_time": "2:59:09", "remaining_time": "6:14:30"}
+{"current_steps": 3200, "total_steps": 9580, "loss": 1.2627, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 0.00014986740918973633, "epoch": 1.67, "percentage": 33.4, "elapsed_time": "3:04:55", "remaining_time": "6:08:42"}
+{"current_steps": 3300, "total_steps": 9580, "loss": 1.2683, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 0.00014699862334591993, "epoch": 1.72, "percentage": 34.45, "elapsed_time": "3:10:38", "remaining_time": "6:02:48"}
+{"current_steps": 3400, "total_steps": 9580, "loss": 1.3076, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 0.00014407929986366458, "epoch": 1.77, "percentage": 35.49, "elapsed_time": "3:16:26", "remaining_time": "5:57:03"}
+{"current_steps": 3500, "total_steps": 9580, "loss": 1.3296, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 0.0001411125778926756, "epoch": 1.83, "percentage": 36.53, "elapsed_time": "3:22:09", "remaining_time": "5:51:11"}
+{"current_steps": 3600, "total_steps": 9580, "loss": 1.2321, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 0.0001381016475502724, "epoch": 1.88, "percentage": 37.58, "elapsed_time": "3:27:54", "remaining_time": "5:45:22"}
+{"current_steps": 3700, "total_steps": 9580, "loss": 1.1754, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 0.00013504974649105364, "epoch": 1.93, "percentage": 38.62, "elapsed_time": "3:33:44", "remaining_time": "5:39:39"}
+{"current_steps": 3800, "total_steps": 9580, "loss": 1.2665, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 0.0001319601564254462, "epoch": 1.98, "percentage": 39.67, "elapsed_time": "3:39:26", "remaining_time": "5:33:46"}
+{"current_steps": 3900, "total_steps": 9580, "loss": 1.0379, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 0.00012883619959088054, "epoch": 2.04, "percentage": 40.71, "elapsed_time": "3:45:13", "remaining_time": "5:28:00"}
+{"current_steps": 4000, "total_steps": 9580, "loss": 0.8172, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 0.0001256812351793875, "epoch": 2.09, "percentage": 41.75, "elapsed_time": "3:50:58", "remaining_time": "5:22:13"}
+{"current_steps": 4100, "total_steps": 9580, "loss": 0.8519, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 0.0001224986557254578, "epoch": 2.14, "percentage": 42.8, "elapsed_time": "3:56:48", "remaining_time": "5:16:31"}
+{"current_steps": 4200, "total_steps": 9580, "loss": 0.8636, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 0.00011929188345804825, "epoch": 2.19, "percentage": 43.84, "elapsed_time": "4:02:42", "remaining_time": "5:10:53"}
+{"current_steps": 4300, "total_steps": 9580, "loss": 0.8727, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 0.00011606436662065767, "epoch": 2.24, "percentage": 44.89, "elapsed_time": "4:08:31", "remaining_time": "5:05:09"}
+{"current_steps": 4400, "total_steps": 9580, "loss": 0.8618, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 0.00011281957576342934, "epoch": 2.3, "percentage": 45.93, "elapsed_time": "4:14:18", "remaining_time": "4:59:23"}
+{"current_steps": 4500, "total_steps": 9580, "loss": 0.7976, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 0.00010956100001126682, "epoch": 2.35, "percentage": 46.97, "elapsed_time": "4:20:10", "remaining_time": "4:53:42"}
+{"current_steps": 4600, "total_steps": 9580, "loss": 0.7871, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 0.00010629214331197683, "epoch": 2.4, "percentage": 48.02, "elapsed_time": "4:26:02", "remaining_time": "4:48:00"}
+{"current_steps": 4700, "total_steps": 9580, "loss": 0.8557, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 0.00010301652066847249, "epoch": 2.45, "percentage": 49.06, "elapsed_time": "4:31:52", "remaining_time": "4:42:16"}
+{"current_steps": 4800, "total_steps": 9580, "loss": 0.8898, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 9.973765435908962e-05, "epoch": 2.51, "percentage": 50.1, "elapsed_time": "4:37:40", "remaining_time": "4:36:30"}
+{"current_steps": 4900, "total_steps": 9580, "loss": 0.8269, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 9.64590701500791e-05, "epoch": 2.56, "percentage": 51.15, "elapsed_time": "4:43:31", "remaining_time": "4:30:47"}
+{"current_steps": 5000, "total_steps": 9580, "loss": 0.7502, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 9.318429350434922e-05, "epoch": 2.61, "percentage": 52.19, "elapsed_time": "4:49:18", "remaining_time": "4:25:00"}
+{"current_steps": 5100, "total_steps": 9580, "loss": 0.8201, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 8.991684579053403e-05, "epoch": 2.66, "percentage": 53.24, "elapsed_time": "4:55:12", "remaining_time": "4:19:18"}
+{"current_steps": 5200, "total_steps": 9580, "loss": 0.8618, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 8.666024049646397e-05, "epoch": 2.71, "percentage": 54.28, "elapsed_time": "5:00:57", "remaining_time": "4:13:30"}
+{"current_steps": 5300, "total_steps": 9580, "loss": 0.8255, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 8.341797945111142e-05, "epoch": 2.77, "percentage": 55.32, "elapsed_time": "5:06:44", "remaining_time": "4:07:42"}
+{"current_steps": 5400, "total_steps": 9580, "loss": 0.759, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 8.019354905907224e-05, "epoch": 2.82, "percentage": 56.37, "elapsed_time": "5:12:31", "remaining_time": "4:01:55"}
+{"current_steps": 5500, "total_steps": 9580, "loss": 0.8287, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 7.699041655163378e-05, "epoch": 2.87, "percentage": 57.41, "elapsed_time": "5:18:21", "remaining_time": "3:56:09"}
+{"current_steps": 5600, "total_steps": 9580, "loss": 0.8356, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 7.381202625845948e-05, "epoch": 2.92, "percentage": 58.46, "elapsed_time": "5:24:03", "remaining_time": "3:50:18"}
+{"current_steps": 5700, "total_steps": 9580, "loss": 0.9181, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 7.066179590389994e-05, "epoch": 2.97, "percentage": 59.5, "elapsed_time": "5:29:47", "remaining_time": "3:44:29"}
+{"current_steps": 5800, "total_steps": 9580, "loss": 0.5811, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 6.754311293191257e-05, "epoch": 3.03, "percentage": 60.54, "elapsed_time": "5:35:33", "remaining_time": "3:38:41"}
+{"current_steps": 5900, "total_steps": 9580, "loss": 0.4604, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 6.44593308635417e-05, "epoch": 3.08, "percentage": 61.59, "elapsed_time": "5:41:22", "remaining_time": "3:32:55"}
+{"current_steps": 6000, "total_steps": 9580, "loss": 0.412, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 6.1413765690876e-05, "epoch": 3.13, "percentage": 62.63, "elapsed_time": "5:47:10", "remaining_time": "3:27:08"}
+{"current_steps": 6100, "total_steps": 9580, "loss": 0.4478, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 5.840969231136102e-05, "epoch": 3.18, "percentage": 63.67, "elapsed_time": "5:53:05", "remaining_time": "3:21:26"}
+{"current_steps": 6200, "total_steps": 9580, "loss": 0.3905, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 5.5450341006300535e-05, "epoch": 3.24, "percentage": 64.72, "elapsed_time": "5:58:53", "remaining_time": "3:15:39"}
+{"current_steps": 6300, "total_steps": 9580, "loss": 0.401, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 5.2538893967333866e-05, "epoch": 3.29, "percentage": 65.76, "elapsed_time": "6:04:41", "remaining_time": "3:09:52"}
+{"current_steps": 6400, "total_steps": 9580, "loss": 0.417, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.9678481874623836e-05, "epoch": 3.34, "percentage": 66.81, "elapsed_time": "6:10:36", "remaining_time": "3:04:08"}
+{"current_steps": 6500, "total_steps": 9580, "loss": 0.4213, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.687218053043516e-05, "epoch": 3.39, "percentage": 67.85, "elapsed_time": "6:16:27", "remaining_time": "2:58:22"}
+{"current_steps": 6600, "total_steps": 9580, "loss": 0.4371, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.412300755172314e-05, "epoch": 3.44, "percentage": 68.89, "elapsed_time": "6:22:19", "remaining_time": "2:52:37"}
+{"current_steps": 6700, "total_steps": 9580, "loss": 0.4686, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.1433919125288914e-05, "epoch": 3.5, "percentage": 69.94, "elapsed_time": "6:28:07", "remaining_time": "2:46:50"}
+{"current_steps": 6800, "total_steps": 9580, "loss": 0.4017, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.8807806828990455e-05, "epoch": 3.55, "percentage": 70.98, "elapsed_time": "6:33:56", "remaining_time": "2:41:03"}
+{"current_steps": 6900, "total_steps": 9580, "loss": 0.4227, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.624749452242799e-05, "epoch": 3.6, "percentage": 72.03, "elapsed_time": "6:39:45", "remaining_time": "2:35:16"}
+{"current_steps": 7000, "total_steps": 9580, "loss": 0.4035, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.375573531044645e-05, "epoch": 3.65, "percentage": 73.07, "elapsed_time": "6:45:34", "remaining_time": "2:29:28"}
+{"current_steps": 7100, "total_steps": 9580, "loss": 0.4271, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.1335208582720856e-05, "epoch": 3.71, "percentage": 74.11, "elapsed_time": "6:51:27", "remaining_time": "2:23:43"}
+{"current_steps": 7200, "total_steps": 9580, "loss": 0.4546, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.8988517132607428e-05, "epoch": 3.76, "percentage": 75.16, "elapsed_time": "6:57:16", "remaining_time": "2:17:56"}
+{"current_steps": 7300, "total_steps": 9580, "loss": 0.3988, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.6718184358358956e-05, "epoch": 3.81, "percentage": 76.2, "elapsed_time": "7:03:10", "remaining_time": "2:12:10"}
+{"current_steps": 7400, "total_steps": 9580, "loss": 0.3839, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.4526651549713608e-05, "epoch": 3.86, "percentage": 77.24, "elapsed_time": "7:09:06", "remaining_time": "2:06:24"}
+{"current_steps": 7500, "total_steps": 9580, "loss": 0.3956, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.2436969776075456e-05, "epoch": 3.91, "percentage": 78.29, "elapsed_time": "7:14:52", "remaining_time": "2:00:36"}
+{"current_steps": 7600, "total_steps": 9580, "loss": 0.4, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.040917410080746e-05, "epoch": 3.97, "percentage": 79.33, "elapsed_time": "7:20:45", "remaining_time": "1:54:49"}
+{"current_steps": 7700, "total_steps": 9580, "loss": 0.3287, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.8466962472390136e-05, "epoch": 4.02, "percentage": 80.38, "elapsed_time": "7:26:32", "remaining_time": "1:49:01"}
+{"current_steps": 7800, "total_steps": 9580, "loss": 0.2069, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.661242335176261e-05, "epoch": 4.07, "percentage": 81.42, "elapsed_time": "7:32:22", "remaining_time": "1:43:13"}
+{"current_steps": 7900, "total_steps": 9580, "loss": 0.1645, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.4847550925581377e-05, "epoch": 4.12, "percentage": 82.46, "elapsed_time": "7:38:07", "remaining_time": "1:37:25"}
+{"current_steps": 8000, "total_steps": 9580, "loss": 0.1888, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.3174242961870542e-05, "epoch": 4.18, "percentage": 83.51, "elapsed_time": "7:44:02", "remaining_time": "1:31:38"}
+{"current_steps": 8100, "total_steps": 9580, "loss": 0.1644, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.1594298769351286e-05, "epoch": 4.23, "percentage": 84.55, "elapsed_time": "7:49:54", "remaining_time": "1:25:51"}
+{"current_steps": 8200, "total_steps": 9580, "loss": 0.2169, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.0109417262644261e-05, "epoch": 4.28, "percentage": 85.59, "elapsed_time": "7:55:43", "remaining_time": "1:20:03"}
+{"current_steps": 8300, "total_steps": 9580, "loss": 0.1992, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 8.72119513542623e-06, "epoch": 4.33, "percentage": 86.64, "elapsed_time": "8:01:31", "remaining_time": "1:14:15"}
+{"current_steps": 8400, "total_steps": 9580, "loss": 0.2011, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 7.431125143504525e-06, "epoch": 4.38, "percentage": 87.68, "elapsed_time": "8:07:19", "remaining_time": "1:08:27"}
+{"current_steps": 8500, "total_steps": 9580, "loss": 0.1783, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 6.240594499656316e-06, "epoch": 4.44, "percentage": 88.73, "elapsed_time": "8:13:07", "remaining_time": "1:02:39"}
+{"current_steps": 8600, "total_steps": 9580, "loss": 0.1813, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 5.150883381957983e-06, "epoch": 4.49, "percentage": 89.77, "elapsed_time": "8:18:54", "remaining_time": "0:56:51"}
+{"current_steps": 8700, "total_steps": 9580, "loss": 0.2027, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.1631635572092706e-06, "epoch": 4.54, "percentage": 90.81, "elapsed_time": "8:24:39", "remaining_time": "0:51:02"}
+{"current_steps": 8800, "total_steps": 9580, "loss": 0.216, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.2784971209318673e-06, "epoch": 4.59, "percentage": 91.86, "elapsed_time": "8:30:38", "remaining_time": "0:45:15"}
+{"current_steps": 8900, "total_steps": 9580, "loss": 0.202, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.4978353552977398e-06, "epoch": 4.65, "percentage": 92.9, "elapsed_time": "8:36:28", "remaining_time": "0:39:27"}
+{"current_steps": 9000, "total_steps": 9580, "loss": 0.1762, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.822017706215029e-06, "epoch": 4.7, "percentage": 93.95, "elapsed_time": "8:42:21", "remaining_time": "0:33:39"}
+{"current_steps": 9100, "total_steps": 9580, "loss": 0.217, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.2517708806714657e-06, "epoch": 4.75, "percentage": 94.99, "elapsed_time": "8:48:12", "remaining_time": "0:27:51"}
+{"current_steps": 9200, "total_steps": 9580, "loss": 0.2045, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 7.877080653061031e-07, "epoch": 4.8, "percentage": 96.03, "elapsed_time": "8:54:06", "remaining_time": "0:22:03"}
+{"current_steps": 9300, "total_steps": 9580, "loss": 0.2263, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.303282670495068e-07, "epoch": 4.85, "percentage": 97.08, "elapsed_time": "8:59:54", "remaining_time": "0:16:15"}
+{"current_steps": 9400, "total_steps": 9580, "loss": 0.192, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.800157765413535e-07, "epoch": 4.91, "percentage": 98.12, "elapsed_time": "9:05:44", "remaining_time": "0:10:27"}
+{"current_steps": 9500, "total_steps": 9580, "loss": 0.2436, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.703975490257916e-08, "epoch": 4.96, "percentage": 99.16, "elapsed_time": "9:11:33", "remaining_time": "0:04:38"}
+{"current_steps": 9580, "total_steps": 9580, "loss": null, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": null, "epoch": 5.0, "percentage": 100.0, "elapsed_time": "9:16:09", "remaining_time": "0:00:00"}

trainer_state.json

@@ -0,0 +1,695 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 5.0,
+  "eval_steps": 500,
+  "global_step": 9580,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.05,
+      "grad_norm": 1.0843530893325806,
+      "learning_rate": 0.00019994623498004714,
+      "loss": 1.8222,
+      "step": 100
+    },
+    {
+      "epoch": 0.1,
+      "grad_norm": 0.9015785455703735,
+      "learning_rate": 0.00019978499773373596,
+      "loss": 1.8565,
+      "step": 200
+    },
+    {
+      "epoch": 0.16,
+      "grad_norm": 1.2422339916229248,
+      "learning_rate": 0.00019951646163954176,
+      "loss": 1.7994,
+      "step": 300
+    },
+    {
+      "epoch": 0.21,
+      "grad_norm": 1.0499353408813477,
+      "learning_rate": 0.0001991409154544338,
+      "loss": 1.6984,
+      "step": 400
+    },
+    {
+      "epoch": 0.26,
+      "grad_norm": 1.122926950454712,
+      "learning_rate": 0.00019865876300337478,
+      "loss": 1.6973,
+      "step": 500
+    },
+    {
+      "epoch": 0.31,
+      "grad_norm": 1.3406814336776733,
+      "learning_rate": 0.00019807052274508773,
+      "loss": 1.7661,
+      "step": 600
+    },
+    {
+      "epoch": 0.37,
+      "grad_norm": 0.7354043126106262,
+      "learning_rate": 0.00019737682721455714,
+      "loss": 1.6948,
+      "step": 700
+    },
+    {
+      "epoch": 0.42,
+      "grad_norm": 2.118002414703369,
+      "learning_rate": 0.0001965784223428638,
+      "loss": 1.8664,
+      "step": 800
+    },
+    {
+      "epoch": 0.47,
+      "grad_norm": 1.6859912872314453,
+      "learning_rate": 0.00019567616665508485,
+      "loss": 1.7087,
+      "step": 900
+    },
+    {
+      "epoch": 0.52,
+      "grad_norm": 1.2144606113433838,
+      "learning_rate": 0.0001946710303471214,
+      "loss": 1.6826,
+      "step": 1000
+    },
+    {
+      "epoch": 0.57,
+      "grad_norm": 1.680190920829773,
+      "learning_rate": 0.00019356409424244655,
+      "loss": 1.5399,
+      "step": 1100
+    },
+    {
+      "epoch": 0.63,
+      "grad_norm": 1.834701418876648,
+      "learning_rate": 0.00019235654862989537,
+      "loss": 1.6394,
+      "step": 1200
+    },
+    {
+      "epoch": 0.68,
+      "grad_norm": 1.9838495254516602,
+      "learning_rate": 0.00019104969198374688,
+      "loss": 1.645,
+      "step": 1300
+    },
+    {
+      "epoch": 0.73,
+      "grad_norm": 2.141674280166626,
+      "learning_rate": 0.00018964492956747425,
+      "loss": 1.5853,
+      "step": 1400
+    },
+    {
+      "epoch": 0.78,
+      "grad_norm": 2.6233127117156982,
+      "learning_rate": 0.00018814377192266423,
+      "loss": 1.5778,
+      "step": 1500
+    },
+    {
+      "epoch": 0.84,
+      "grad_norm": 1.6887329816818237,
+      "learning_rate": 0.00018654783324473137,
+      "loss": 1.6368,
+      "step": 1600
+    },
+    {
+      "epoch": 0.89,
+      "grad_norm": 3.113426446914673,
+      "learning_rate": 0.00018487617447307124,
+      "loss": 1.6269,
+      "step": 1700
+    },
+    {
+      "epoch": 0.94,
+      "grad_norm": 2.1365339756011963,
+      "learning_rate": 0.00018309682531549338,
+      "loss": 1.7073,
+      "step": 1800
+    },
+    {
+      "epoch": 0.99,
+      "grad_norm": 2.3787343502044678,
+      "learning_rate": 0.00018122812210849337,
+      "loss": 1.5544,
+      "step": 1900
+    },
+    {
+      "epoch": 1.04,
+      "grad_norm": 2.1635191440582275,
+      "learning_rate": 0.00017927207426937544,
+      "loss": 1.3285,
+      "step": 2000
+    },
+    {
+      "epoch": 1.1,
+      "grad_norm": 2.331688642501831,
+      "learning_rate": 0.00017723078513716157,
+      "loss": 1.2529,
+      "step": 2100
+    },
+    {
+      "epoch": 1.15,
+      "grad_norm": 5.185802936553955,
+      "learning_rate": 0.00017510644971087015,
+      "loss": 1.3708,
+      "step": 2200
+    },
+    {
+      "epoch": 1.2,
+      "grad_norm": 3.1339027881622314,
+      "learning_rate": 0.0001729013522892329,
+      "loss": 1.3309,
+      "step": 2300
+    },
+    {
+      "epoch": 1.25,
+      "grad_norm": 3.4236795902252197,
+      "learning_rate": 0.0001706178640143872,
+      "loss": 1.4201,
+      "step": 2400
+    },
+    {
+      "epoch": 1.3,
+      "grad_norm": 3.849432945251465,
+      "learning_rate": 0.00016825844032218625,
+      "loss": 1.2956,
+      "step": 2500
+    },
+    {
+      "epoch": 1.36,
+      "grad_norm": 3.8217601776123047,
+      "learning_rate": 0.00016582561830186785,
+      "loss": 1.3375,
+      "step": 2600
+    },
+    {
+      "epoch": 1.41,
+      "grad_norm": 1.5418843030929565,
+      "learning_rate": 0.00016332201396792123,
+      "loss": 1.2897,
+      "step": 2700
+    },
+    {
+      "epoch": 1.46,
+      "grad_norm": 2.2709996700286865,
+      "learning_rate": 0.00016075031944708584,
+      "loss": 1.3323,
+      "step": 2800
+    },
+    {
+      "epoch": 1.51,
+      "grad_norm": 4.869096279144287,
+      "learning_rate": 0.0001581133000835061,
+      "loss": 1.2841,
+      "step": 2900
+    },
+    {
+      "epoch": 1.57,
+      "grad_norm": 3.3247766494750977,
+      "learning_rate": 0.00015541379146515603,
+      "loss": 1.2666,
+      "step": 3000
+    },
+    {
+      "epoch": 1.62,
+      "grad_norm": 4.0471110343933105,
+      "learning_rate": 0.0001526546963747302,
+      "loss": 1.2336,
+      "step": 3100
+    },
+    {
+      "epoch": 1.67,
+      "grad_norm": 2.8835229873657227,
+      "learning_rate": 0.00014986740918973633,
+      "loss": 1.2627,
+      "step": 3200
+    },
+    {
+      "epoch": 1.72,
+      "grad_norm": 4.162999629974365,
+      "learning_rate": 0.00014699862334591993,
+      "loss": 1.2683,
+      "step": 3300
+    },
+    {
+      "epoch": 1.77,
+      "grad_norm": 2.161064624786377,
+      "learning_rate": 0.00014407929986366458,
+      "loss": 1.3076,
+      "step": 3400
+    },
+    {
+      "epoch": 1.83,
+      "grad_norm": 2.2110209465026855,
+      "learning_rate": 0.0001411125778926756,
+      "loss": 1.3296,
+      "step": 3500
+    },
+    {
+      "epoch": 1.88,
+      "grad_norm": 3.9512176513671875,
+      "learning_rate": 0.0001381016475502724,
+      "loss": 1.2321,
+      "step": 3600
+    },
+    {
+      "epoch": 1.93,
+      "grad_norm": 1.0244474411010742,
+      "learning_rate": 0.00013504974649105364,
+      "loss": 1.1754,
+      "step": 3700
+    },
+    {
+      "epoch": 1.98,
+      "grad_norm": 4.0455217361450195,
+      "learning_rate": 0.0001319601564254462,
+      "loss": 1.2665,
+      "step": 3800
+    },
+    {
+      "epoch": 2.04,
+      "grad_norm": 4.162383556365967,
+      "learning_rate": 0.00012883619959088054,
+      "loss": 1.0379,
+      "step": 3900
+    },
+    {
+      "epoch": 2.09,
+      "grad_norm": 2.3285574913024902,
+      "learning_rate": 0.0001256812351793875,
+      "loss": 0.8172,
+      "step": 4000
+    },
+    {
+      "epoch": 2.14,
+      "grad_norm": 5.439870834350586,
+      "learning_rate": 0.0001224986557254578,
+      "loss": 0.8519,
+      "step": 4100
+    },
+    {
+      "epoch": 2.19,
+      "grad_norm": 2.932978391647339,
+      "learning_rate": 0.00011929188345804825,
+      "loss": 0.8636,
+      "step": 4200
+    },
+    {
+      "epoch": 2.24,
+      "grad_norm": 5.512314319610596,
+      "learning_rate": 0.00011606436662065767,
+      "loss": 0.8727,
+      "step": 4300
+    },
+    {
+      "epoch": 2.3,
+      "grad_norm": 4.129833698272705,
+      "learning_rate": 0.00011281957576342934,
+      "loss": 0.8618,
+      "step": 4400
+    },
+    {
+      "epoch": 2.35,
+      "grad_norm": 2.5295193195343018,
+      "learning_rate": 0.00010956100001126682,
+      "loss": 0.7976,
+      "step": 4500
+    },
+    {
+      "epoch": 2.4,
+      "grad_norm": 4.148761749267578,
+      "learning_rate": 0.00010629214331197683,
+      "loss": 0.7871,
+      "step": 4600
+    },
+    {
+      "epoch": 2.45,
+      "grad_norm": 5.00084924697876,
+      "learning_rate": 0.00010301652066847249,
+      "loss": 0.8557,
+      "step": 4700
+    },
+    {
+      "epoch": 2.51,
+      "grad_norm": 5.721644878387451,
+      "learning_rate": 9.973765435908962e-05,
+      "loss": 0.8898,
+      "step": 4800
+    },
+    {
+      "epoch": 2.56,
+      "grad_norm": 5.872501850128174,
+      "learning_rate": 9.64590701500791e-05,
+      "loss": 0.8269,
+      "step": 4900
+    },
+    {
+      "epoch": 2.61,
+      "grad_norm": 5.41200065612793,
+      "learning_rate": 9.318429350434922e-05,
+      "loss": 0.7502,
+      "step": 5000
+    },
+    {
+      "epoch": 2.66,
+      "grad_norm": 3.3696563243865967,
+      "learning_rate": 8.991684579053403e-05,
+      "loss": 0.8201,
+      "step": 5100
+    },
+    {
+      "epoch": 2.71,
+      "grad_norm": 3.6817996501922607,
+      "learning_rate": 8.666024049646397e-05,
+      "loss": 0.8618,
+      "step": 5200
+    },
+    {
+      "epoch": 2.77,
+      "grad_norm": 4.630326747894287,
+      "learning_rate": 8.341797945111142e-05,
+      "loss": 0.8255,
+      "step": 5300
+    },
+    {
+      "epoch": 2.82,
+      "grad_norm": 2.672938823699951,
+      "learning_rate": 8.019354905907224e-05,
+      "loss": 0.759,
+      "step": 5400
+    },
+    {
+      "epoch": 2.87,
+      "grad_norm": 3.0967512130737305,
+      "learning_rate": 7.699041655163378e-05,
+      "loss": 0.8287,
+      "step": 5500
+    },
+    {
+      "epoch": 2.92,
+      "grad_norm": 4.052151679992676,
+      "learning_rate": 7.381202625845948e-05,
+      "loss": 0.8356,
+      "step": 5600
+    },
+    {
+      "epoch": 2.97,
+      "grad_norm": 3.5286691188812256,
+      "learning_rate": 7.066179590389994e-05,
+      "loss": 0.9181,
+      "step": 5700
+    },
+    {
+      "epoch": 3.03,
+      "grad_norm": 4.84535026550293,
+      "learning_rate": 6.754311293191257e-05,
+      "loss": 0.5811,
+      "step": 5800
+    },
+    {
+      "epoch": 3.08,
+      "grad_norm": 2.9033737182617188,
+      "learning_rate": 6.44593308635417e-05,
+      "loss": 0.4604,
+      "step": 5900
+    },
+    {
+      "epoch": 3.13,
+      "grad_norm": 1.6759638786315918,
+      "learning_rate": 6.1413765690876e-05,
+      "loss": 0.412,
+      "step": 6000
+    },
+    {
+      "epoch": 3.18,
+      "grad_norm": 3.905642032623291,
+      "learning_rate": 5.840969231136102e-05,
+      "loss": 0.4478,
+      "step": 6100
+    },
+    {
+      "epoch": 3.24,
+      "grad_norm": 7.030938148498535,
+      "learning_rate": 5.5450341006300535e-05,
+      "loss": 0.3905,
+      "step": 6200
+    },
+    {
+      "epoch": 3.29,
+      "grad_norm": 5.1767988204956055,
+      "learning_rate": 5.2538893967333866e-05,
+      "loss": 0.401,
+      "step": 6300
+    },
+    {
+      "epoch": 3.34,
+      "grad_norm": 2.0529091358184814,
+      "learning_rate": 4.9678481874623836e-05,
+      "loss": 0.417,
+      "step": 6400
+    },
+    {
+      "epoch": 3.39,
+      "grad_norm": 4.730635643005371,
+      "learning_rate": 4.687218053043516e-05,
+      "loss": 0.4213,
+      "step": 6500
+    },
+    {
+      "epoch": 3.44,
+      "grad_norm": 5.270570278167725,
+      "learning_rate": 4.412300755172314e-05,
+      "loss": 0.4371,
+      "step": 6600
+    },
+    {
+      "epoch": 3.5,
+      "grad_norm": 5.049856185913086,
+      "learning_rate": 4.1433919125288914e-05,
+      "loss": 0.4686,
+      "step": 6700
+    },
+    {
+      "epoch": 3.55,
+      "grad_norm": 5.543980121612549,
+      "learning_rate": 3.8807806828990455e-05,
+      "loss": 0.4017,
+      "step": 6800
+    },
+    {
+      "epoch": 3.6,
+      "grad_norm": 5.048495292663574,
+      "learning_rate": 3.624749452242799e-05,
+      "loss": 0.4227,
+      "step": 6900
+    },
+    {
+      "epoch": 3.65,
+      "grad_norm": 6.684240818023682,
+      "learning_rate": 3.375573531044645e-05,
+      "loss": 0.4035,
+      "step": 7000
+    },
+    {
+      "epoch": 3.71,
+      "grad_norm": 6.066633701324463,
+      "learning_rate": 3.1335208582720856e-05,
+      "loss": 0.4271,
+      "step": 7100
+    },
+    {
+      "epoch": 3.76,
+      "grad_norm": 4.037696361541748,
+      "learning_rate": 2.8988517132607428e-05,
+      "loss": 0.4546,
+      "step": 7200
+    },
+    {
+      "epoch": 3.81,
+      "grad_norm": 4.5075602531433105,
+      "learning_rate": 2.6718184358358956e-05,
+      "loss": 0.3988,
+      "step": 7300
+    },
+    {
+      "epoch": 3.86,
+      "grad_norm": 2.9646005630493164,
+      "learning_rate": 2.4526651549713608e-05,
+      "loss": 0.3839,
+      "step": 7400
+    },
+    {
+      "epoch": 3.91,
+      "grad_norm": 6.707378387451172,
+      "learning_rate": 2.2436969776075456e-05,
+      "loss": 0.3956,
+      "step": 7500
+    },
+    {
+      "epoch": 3.97,
+      "grad_norm": 6.770478248596191,
+      "learning_rate": 2.040917410080746e-05,
+      "loss": 0.4,
+      "step": 7600
+    },
+    {
+      "epoch": 4.02,
+      "grad_norm": 3.7659125328063965,
+      "learning_rate": 1.8466962472390136e-05,
+      "loss": 0.3287,
+      "step": 7700
+    },
+    {
+      "epoch": 4.07,
+      "grad_norm": 2.905404567718506,
+      "learning_rate": 1.661242335176261e-05,
+      "loss": 0.2069,
+      "step": 7800
+    },
+    {
+      "epoch": 4.12,
+      "grad_norm": 5.363249778747559,
+      "learning_rate": 1.4847550925581377e-05,
+      "loss": 0.1645,
+      "step": 7900
+    },
+    {
+      "epoch": 4.18,
+      "grad_norm": 5.363257884979248,
+      "learning_rate": 1.3174242961870542e-05,
+      "loss": 0.1888,
+      "step": 8000
+    },
+    {
+      "epoch": 4.23,
+      "grad_norm": 2.431536912918091,
+      "learning_rate": 1.1594298769351286e-05,
+      "loss": 0.1644,
+      "step": 8100
+    },
+    {
+      "epoch": 4.28,
+      "grad_norm": 2.4400837421417236,
+      "learning_rate": 1.0109417262644261e-05,
+      "loss": 0.2169,
+      "step": 8200
+    },
+    {
+      "epoch": 4.33,
+      "grad_norm": 8.288840293884277,
+      "learning_rate": 8.72119513542623e-06,
+      "loss": 0.1992,
+      "step": 8300
+    },
+    {
+      "epoch": 4.38,
+      "grad_norm": 3.36771297454834,
+      "learning_rate": 7.431125143504525e-06,
+      "loss": 0.2011,
+      "step": 8400
+    },
+    {
+      "epoch": 4.44,
+      "grad_norm": 0.056750617921352386,
+      "learning_rate": 6.240594499656316e-06,
+      "loss": 0.1783,
+      "step": 8500
+    },
+    {
+      "epoch": 4.49,
+      "grad_norm": 4.8992743492126465,
+      "learning_rate": 5.150883381957983e-06,
+      "loss": 0.1813,
+      "step": 8600
+    },
+    {
+      "epoch": 4.54,
+      "grad_norm": 2.5150370597839355,
+      "learning_rate": 4.1631635572092706e-06,
+      "loss": 0.2027,
+      "step": 8700
+    },
+    {
+      "epoch": 4.59,
+      "grad_norm": 4.545584201812744,
+      "learning_rate": 3.2784971209318673e-06,
+      "loss": 0.216,
+      "step": 8800
+    },
+    {
+      "epoch": 4.65,
+      "grad_norm": 6.020129680633545,
+      "learning_rate": 2.4978353552977398e-06,
+      "loss": 0.202,
+      "step": 8900
+    },
+    {
+      "epoch": 4.7,
+      "grad_norm": 5.093991279602051,
+      "learning_rate": 1.822017706215029e-06,
+      "loss": 0.1762,
+      "step": 9000
+    },
+    {
+      "epoch": 4.75,
+      "grad_norm": 2.0878984928131104,
+      "learning_rate": 1.2517708806714657e-06,
+      "loss": 0.217,
+      "step": 9100
+    },
+    {
+      "epoch": 4.8,
+      "grad_norm": 2.145759105682373,
+      "learning_rate": 7.877080653061031e-07,
+      "loss": 0.2045,
+      "step": 9200
+    },
+    {
+      "epoch": 4.85,
+      "grad_norm": 8.151817321777344,
+      "learning_rate": 4.303282670495068e-07,
+      "loss": 0.2263,
+      "step": 9300
+    },
+    {
+      "epoch": 4.91,
+      "grad_norm": 6.520354270935059,
+      "learning_rate": 1.800157765413535e-07,
+      "loss": 0.192,
+      "step": 9400
+    },
+    {
+      "epoch": 4.96,
+      "grad_norm": 7.271907806396484,
+      "learning_rate": 3.703975490257916e-08,
+      "loss": 0.2436,
+      "step": 9500
+    },
+    {
+      "epoch": 5.0,
+      "step": 9580,
+      "total_flos": 1.5336527740560998e+18,
+      "train_loss": 0.886494568245396,
+      "train_runtime": 33369.3902,
+      "train_samples_per_second": 1.148,
+      "train_steps_per_second": 0.287
+    }
+  ],
+  "logging_steps": 100,
+  "max_steps": 9580,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 5,
+  "save_steps": 1000,
+  "total_flos": 1.5336527740560998e+18,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:347246aece17461af2bf7ca00e633da815261b88a5734925bcb6f918b0b7dd08
+size 5048