etri-xainlp commited on
Commit
c6be107
1 Parent(s): 85620b1

Upload 26 files

Browse files
config.json ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "/disk3/leehj/qwen/pre-trained/Qwen-14B",
3
+ "architectures": [
4
+ "QWenLMHeadModel"
5
+ ],
6
+ "attn_dropout_prob": 0.0,
7
+ "auto_map": {
8
+ "AutoConfig": "configuration_qwen.QWenConfig",
9
+ "AutoModelForCausalLM": "modeling_qwen.QWenLMHeadModel"
10
+ },
11
+ "bf16": false,
12
+ "emb_dropout_prob": 0.0,
13
+ "fp16": true,
14
+ "fp32": false,
15
+ "hidden_size": 5120,
16
+ "initializer_range": 0.02,
17
+ "intermediate_size": 27392,
18
+ "kv_channels": 128,
19
+ "layer_norm_epsilon": 1e-06,
20
+ "max_position_embeddings": 8192,
21
+ "model_type": "qwen",
22
+ "no_bias": true,
23
+ "num_attention_heads": 40,
24
+ "num_hidden_layers": 40,
25
+ "onnx_safe": null,
26
+ "rotary_emb_base": 10000,
27
+ "rotary_pct": 1.0,
28
+ "scale_attn_weights": true,
29
+ "seq_length": 2048,
30
+ "softmax_in_fp32": false,
31
+ "tie_word_embeddings": false,
32
+ "tokenizer_class": "QWenTokenizer",
33
+ "torch_dtype": "bfloat16",
34
+ "transformers_version": "4.32.0",
35
+ "use_cache": false,
36
+ "use_cache_kernel": false,
37
+ "use_cache_quantization": false,
38
+ "use_dynamic_ntk": true,
39
+ "use_flash_attn": true,
40
+ "use_logn_attn": true,
41
+ "vocab_size": 152064
42
+ }
configuration_qwen.py ADDED
@@ -0,0 +1,71 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba Cloud.
2
+ #
3
+ # This source code is licensed under the license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ from transformers import PretrainedConfig
7
+
8
+
9
+ class QWenConfig(PretrainedConfig):
10
+ model_type = "qwen"
11
+ keys_to_ignore_at_inference = ["past_key_values"]
12
+
13
+ def __init__(
14
+ self,
15
+ vocab_size=151936,
16
+ hidden_size=4096,
17
+ num_hidden_layers=32,
18
+ num_attention_heads=32,
19
+ emb_dropout_prob=0.0,
20
+ attn_dropout_prob=0.0,
21
+ layer_norm_epsilon=1e-6,
22
+ initializer_range=0.02,
23
+ max_position_embeddings=8192,
24
+ scale_attn_weights=True,
25
+ use_cache=True,
26
+ bf16=False,
27
+ fp16=False,
28
+ fp32=False,
29
+ kv_channels=128,
30
+ rotary_pct=1.0,
31
+ rotary_emb_base=10000,
32
+ use_dynamic_ntk=True,
33
+ use_logn_attn=True,
34
+ use_flash_attn="auto",
35
+ intermediate_size=22016,
36
+ no_bias=True,
37
+ tie_word_embeddings=False,
38
+ use_cache_quantization=False,
39
+ use_cache_kernel=False,
40
+ softmax_in_fp32=False,
41
+ **kwargs,
42
+ ):
43
+ self.vocab_size = vocab_size
44
+ self.hidden_size = hidden_size
45
+ self.intermediate_size = intermediate_size
46
+ self.num_hidden_layers = num_hidden_layers
47
+ self.num_attention_heads = num_attention_heads
48
+ self.emb_dropout_prob = emb_dropout_prob
49
+ self.attn_dropout_prob = attn_dropout_prob
50
+ self.layer_norm_epsilon = layer_norm_epsilon
51
+ self.initializer_range = initializer_range
52
+ self.scale_attn_weights = scale_attn_weights
53
+ self.use_cache = use_cache
54
+ self.max_position_embeddings = max_position_embeddings
55
+ self.bf16 = bf16
56
+ self.fp16 = fp16
57
+ self.fp32 = fp32
58
+ self.kv_channels = kv_channels
59
+ self.rotary_pct = rotary_pct
60
+ self.rotary_emb_base = rotary_emb_base
61
+ self.use_dynamic_ntk = use_dynamic_ntk
62
+ self.use_logn_attn = use_logn_attn
63
+ self.use_flash_attn = use_flash_attn
64
+ self.no_bias = no_bias
65
+ self.use_cache_quantization = use_cache_quantization
66
+ self.use_cache_kernel = use_cache_kernel
67
+ self.softmax_in_fp32 = softmax_in_fp32
68
+ super().__init__(
69
+ tie_word_embeddings=tie_word_embeddings,
70
+ **kwargs
71
+ )
cpp_kernels.py ADDED
@@ -0,0 +1,55 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from torch.utils import cpp_extension
2
+ import pathlib
3
+ import os
4
+ import subprocess
5
+
6
+ def _get_cuda_bare_metal_version(cuda_dir):
7
+ raw_output = subprocess.check_output([cuda_dir + "/bin/nvcc", "-V"],
8
+ universal_newlines=True)
9
+ output = raw_output.split()
10
+ release_idx = output.index("release") + 1
11
+ release = output[release_idx].split(".")
12
+ bare_metal_major = release[0]
13
+ bare_metal_minor = release[1][0]
14
+
15
+ return raw_output, bare_metal_major, bare_metal_minor
16
+
17
+ def _create_build_dir(buildpath):
18
+ try:
19
+ os.mkdir(buildpath)
20
+ except OSError:
21
+ if not os.path.isdir(buildpath):
22
+ print(f"Creation of the build directory {buildpath} failed")
23
+
24
+ # Check if cuda 11 is installed for compute capability 8.0
25
+ cc_flag = []
26
+ _, bare_metal_major, bare_metal_minor = _get_cuda_bare_metal_version(cpp_extension.CUDA_HOME)
27
+ if int(bare_metal_major) >= 11:
28
+ cc_flag.append('-gencode')
29
+ cc_flag.append('arch=compute_80,code=sm_80')
30
+ if int(bare_metal_minor) >= 7:
31
+ cc_flag.append('-gencode')
32
+ cc_flag.append('arch=compute_90,code=sm_90')
33
+
34
+ # Build path
35
+ srcpath = pathlib.Path(__file__).parent.absolute()
36
+ buildpath = srcpath / 'build'
37
+ _create_build_dir(buildpath)
38
+
39
+ def _cpp_extention_load_helper(name, sources, extra_cuda_flags):
40
+ return cpp_extension.load(
41
+ name=name,
42
+ sources=sources,
43
+ build_directory=buildpath,
44
+ extra_cflags=['-O3', ],
45
+ extra_cuda_cflags=['-O3',
46
+ '-gencode', 'arch=compute_70,code=sm_70',
47
+ '--use_fast_math'] + extra_cuda_flags + cc_flag,
48
+ verbose=1
49
+ )
50
+
51
+ extra_flags = []
52
+
53
+ cache_autogptq_cuda_256_sources = ["./cache_autogptq_cuda_256.cpp",
54
+ "./cache_autogptq_cuda_kernel_256.cu"]
55
+ cache_autogptq_cuda_256 = _cpp_extention_load_helper("cache_autogptq_cuda_256", cache_autogptq_cuda_256_sources, extra_flags)
generation_config.json ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "chat_format": "raw",
3
+ "do_sample": true,
4
+ "eos_token_id": 151643,
5
+ "max_new_tokens": 512,
6
+ "pad_token_id": 151643,
7
+ "stop_words_ids": [
8
+ [
9
+ 151643
10
+ ]
11
+ ],
12
+ "top_k": 0,
13
+ "top_p": 0.8,
14
+ "transformers_version": "4.32.0"
15
+ }
latest ADDED
@@ -0,0 +1 @@
 
 
1
+ global_step756
modeling_qwen.py ADDED
@@ -0,0 +1,1416 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba Cloud.
2
+ #
3
+ # This source code is licensed under the license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ import importlib
7
+ import math
8
+ from typing import TYPE_CHECKING, Optional, Tuple, Union, Callable, List, Any, Generator
9
+
10
+ import torch
11
+ import torch.nn.functional as F
12
+ import torch.utils.checkpoint
13
+ from torch.cuda.amp import autocast
14
+
15
+ from torch.nn import CrossEntropyLoss
16
+ from transformers import PreTrainedTokenizer, GenerationConfig, StoppingCriteriaList
17
+ from transformers.generation.logits_process import LogitsProcessorList
18
+
19
+ if TYPE_CHECKING:
20
+ from transformers.generation.streamers import BaseStreamer
21
+ from transformers.generation.utils import GenerateOutput
22
+ from transformers.modeling_outputs import (
23
+ BaseModelOutputWithPast,
24
+ CausalLMOutputWithPast,
25
+ )
26
+ from transformers.modeling_utils import PreTrainedModel
27
+ from transformers.utils import logging
28
+
29
+ try:
30
+ from einops import rearrange
31
+ except ImportError:
32
+ rearrange = None
33
+ from torch import nn
34
+
35
+ SUPPORT_CUDA = torch.cuda.is_available()
36
+ SUPPORT_BF16 = SUPPORT_CUDA and torch.cuda.is_bf16_supported()
37
+ SUPPORT_FP16 = SUPPORT_CUDA and torch.cuda.get_device_capability(0)[0] >= 7
38
+
39
+ from .configuration_qwen import QWenConfig
40
+ from .qwen_generation_utils import (
41
+ HistoryType,
42
+ make_context,
43
+ decode_tokens,
44
+ get_stop_words_ids,
45
+ StopWordsLogitsProcessor,
46
+ )
47
+
48
+
49
+ logger = logging.get_logger(__name__)
50
+
51
+ _CHECKPOINT_FOR_DOC = "qwen"
52
+ _CONFIG_FOR_DOC = "QWenConfig"
53
+
54
+ QWen_PRETRAINED_MODEL_ARCHIVE_LIST = ["qwen-7b"]
55
+
56
+ _ERROR_BAD_CHAT_FORMAT = """\
57
+ We detect you are probably using the pretrained model (rather than chat model) for chatting, since the chat_format in generation_config is not "chatml".
58
+ If you are directly using the model downloaded from Huggingface, please make sure you are using our "Qwen/Qwen-7B-Chat" Huggingface model (rather than "Qwen/Qwen-7B") when you call model.chat().
59
+ 我们检测到您可能在使用预训练模型(而非chat模型)进行多轮chat,因为您当前在generation_config指定的chat_format,并未设置为我们在对话中所支持的"chatml"格式。
60
+ 如果您在直接使用我们从Huggingface提供的模型,请确保您在调用model.chat()时,使用的是"Qwen/Qwen-7B-Chat"模型(而非"Qwen/Qwen-7B"预训练模型)。
61
+ """
62
+
63
+ _SENTINEL = object()
64
+ _ERROR_STREAM_IN_CHAT = """\
65
+ Pass argument `stream` to model.chat() is buggy, deprecated, and marked for removal. Please use model.chat_stream(...) instead of model.chat(..., stream=True).
66
+ 向model.chat()传入参数stream的用法可能存在Bug,该用法已被废弃,将在未来被移除。请使用model.chat_stream(...)代替model.chat(..., stream=True)。
67
+ """
68
+
69
+ _ERROR_INPUT_CPU_QUERY_WITH_FLASH_ATTN_ACTIVATED = """\
70
+ We detect you have activated flash attention support, but running model computation on CPU. Please make sure that your input data has been placed on GPU. If you actually want to run CPU computation, please following the readme and set device_map="cpu" to disable flash attention when loading the model (calling AutoModelForCausalLM.from_pretrained).
71
+ 检测到您的模型已激活了flash attention支持,但正在执行CPU运算任务。如使用flash attention,请您确认模型输入已经传到GPU上。如果您确认要执行CPU运算,请您在载入模型(调用AutoModelForCausalLM.from_pretrained)时,按照readme说法,指定device_map="cpu"以禁用flash attention。
72
+ """
73
+
74
+ apply_rotary_emb_func = None
75
+ rms_norm = None
76
+ flash_attn_unpadded_func = None
77
+
78
+ def _import_flash_attn():
79
+ global apply_rotary_emb_func, rms_norm, flash_attn_unpadded_func
80
+ try:
81
+ from flash_attn.layers.rotary import apply_rotary_emb_func as __apply_rotary_emb_func
82
+ apply_rotary_emb_func = __apply_rotary_emb_func
83
+ except ImportError:
84
+ logger.warn(
85
+ "Warning: import flash_attn rotary fail, please install FlashAttention rotary to get higher efficiency "
86
+ "https://github.com/Dao-AILab/flash-attention/tree/main/csrc/rotary"
87
+ )
88
+
89
+ try:
90
+ from flash_attn.ops.rms_norm import rms_norm as __rms_norm
91
+ rms_norm = __rms_norm
92
+ except ImportError:
93
+ logger.warn(
94
+ "Warning: import flash_attn rms_norm fail, please install FlashAttention layer_norm to get higher efficiency "
95
+ "https://github.com/Dao-AILab/flash-attention/tree/main/csrc/layer_norm"
96
+ )
97
+
98
+ try:
99
+ import flash_attn
100
+ if not hasattr(flash_attn, '__version__'):
101
+ from flash_attn.flash_attn_interface import flash_attn_unpadded_func as __flash_attn_unpadded_func
102
+ else:
103
+ if int(flash_attn.__version__.split(".")[0]) >= 2:
104
+ from flash_attn.flash_attn_interface import flash_attn_varlen_func as __flash_attn_unpadded_func
105
+ else:
106
+ from flash_attn.flash_attn_interface import flash_attn_unpadded_func as __flash_attn_unpadded_func
107
+ flash_attn_unpadded_func = __flash_attn_unpadded_func
108
+ except ImportError:
109
+ logger.warn(
110
+ "Warning: import flash_attn fail, please install FlashAttention to get higher efficiency "
111
+ "https://github.com/Dao-AILab/flash-attention"
112
+ )
113
+
114
+ def quantize_cache_v(fdata, bits, qmax, qmin):
115
+ # b, s, head, h-dim->b, head, s, h-dim
116
+ qtype = torch.uint8
117
+ device = fdata.device
118
+ shape = fdata.shape
119
+
120
+ fdata_cal = torch.flatten(fdata, 2)
121
+ fmax = torch.amax(fdata_cal, dim=-1, keepdim=True)
122
+ fmin = torch.amin(fdata_cal, dim=-1, keepdim=True)
123
+ # Compute params
124
+ if qmax.device != fmax.device:
125
+ qmax = qmax.to(device)
126
+ qmin = qmin.to(device)
127
+ scale = (fmax - fmin) / (qmax - qmin)
128
+ zero = qmin - fmin / scale
129
+ scale = scale.unsqueeze(-1).repeat(1,1,shape[2],1).contiguous()
130
+ zero = zero.unsqueeze(-1).repeat(1,1,shape[2],1).contiguous()
131
+ # Quantize
132
+ res_data = fdata / scale + zero
133
+ qdata = torch.clamp(res_data, qmin, qmax).to(qtype)
134
+ return qdata.contiguous(), scale, zero
135
+
136
+ def dequantize_cache_torch(qdata, scale, zero):
137
+ data = scale * (qdata - zero)
138
+ return data
139
+
140
+ class FlashSelfAttention(torch.nn.Module):
141
+ def __init__(
142
+ self,
143
+ causal=False,
144
+ softmax_scale=None,
145
+ attention_dropout=0.0,
146
+ ):
147
+ super().__init__()
148
+ assert flash_attn_unpadded_func is not None, (
149
+ "Please install FlashAttention first, " "e.g., with pip install flash-attn"
150
+ )
151
+ assert (
152
+ rearrange is not None
153
+ ), "Please install einops first, e.g., with pip install einops"
154
+ self.causal = causal
155
+ self.softmax_scale = softmax_scale
156
+ self.dropout_p = attention_dropout
157
+
158
+ def unpad_input(self, hidden_states, attention_mask):
159
+ valid_mask = attention_mask.squeeze(1).squeeze(1).eq(0)
160
+ seqlens_in_batch = valid_mask.sum(dim=-1, dtype=torch.int32)
161
+ indices = torch.nonzero(valid_mask.flatten(), as_tuple=False).flatten()
162
+ max_seqlen_in_batch = seqlens_in_batch.max().item()
163
+ cu_seqlens = F.pad(torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.torch.int32), (1, 0))
164
+ hidden_states = hidden_states[indices]
165
+ return hidden_states, indices, cu_seqlens, max_seqlen_in_batch
166
+
167
+ def pad_input(self, hidden_states, indices, batch, seqlen):
168
+ output = torch.zeros(batch * seqlen, *hidden_states.shape[1:], device=hidden_states.device,
169
+ dtype=hidden_states.dtype)
170
+ output[indices] = hidden_states
171
+ return rearrange(output, '(b s) ... -> b s ...', b=batch)
172
+
173
+ def forward(self, q, k, v, attention_mask=None):
174
+ assert all((i.dtype in [torch.float16, torch.bfloat16] for i in (q, k, v)))
175
+ assert all((i.is_cuda for i in (q, k, v)))
176
+ batch_size, seqlen_q = q.shape[0], q.shape[1]
177
+ seqlen_k = k.shape[1]
178
+ seqlen_out = seqlen_q
179
+
180
+ q, k, v = [rearrange(x, "b s ... -> (b s) ...") for x in [q, k, v]]
181
+ cu_seqlens_q = torch.arange(
182
+ 0,
183
+ (batch_size + 1) * seqlen_q,
184
+ step=seqlen_q,
185
+ dtype=torch.int32,
186
+ device=q.device,
187
+ )
188
+
189
+ if attention_mask is not None:
190
+ k, indices_k, cu_seqlens_k, seqlen_k = self.unpad_input(k, attention_mask)
191
+ if q.size(0) == v.size(0):
192
+ q = q[indices_k]
193
+ cu_seqlens_q = cu_seqlens_k
194
+ seqlen_q = seqlen_k
195
+ v = v[indices_k]
196
+ else:
197
+ cu_seqlens_k = torch.arange(
198
+ 0,
199
+ (batch_size + 1) * seqlen_k,
200
+ step=seqlen_k,
201
+ dtype=torch.int32,
202
+ device=q.device,
203
+ )
204
+
205
+ if self.training:
206
+ assert seqlen_k == seqlen_q
207
+ is_causal = self.causal
208
+ dropout_p = self.dropout_p
209
+ else:
210
+ is_causal = seqlen_q == seqlen_k
211
+ dropout_p = 0
212
+
213
+ output = flash_attn_unpadded_func(
214
+ q,
215
+ k,
216
+ v,
217
+ cu_seqlens_q,
218
+ cu_seqlens_k,
219
+ seqlen_q,
220
+ seqlen_k,
221
+ dropout_p,
222
+ softmax_scale=self.softmax_scale,
223
+ causal=is_causal,
224
+ )
225
+ if attention_mask is not None and seqlen_q == seqlen_k:
226
+ output = self.pad_input(output, indices_k, batch_size, seqlen_out)
227
+ else:
228
+ new_shape = (batch_size, output.shape[0] // batch_size) + output.shape[1:]
229
+ output = output.view(new_shape)
230
+ return output
231
+
232
+
233
+ class QWenAttention(nn.Module):
234
+ def __init__(self, config):
235
+ super().__init__()
236
+
237
+ self.register_buffer("masked_bias", torch.tensor(-1e4), persistent=False)
238
+ self.seq_length = config.seq_length
239
+
240
+ self.hidden_size = config.hidden_size
241
+ self.split_size = config.hidden_size
242
+ self.num_heads = config.num_attention_heads
243
+ self.head_dim = self.hidden_size // self.num_heads
244
+
245
+ self.use_flash_attn = config.use_flash_attn
246
+ self.scale_attn_weights = True
247
+
248
+ self.projection_size = config.kv_channels * config.num_attention_heads
249
+
250
+ assert self.projection_size % config.num_attention_heads == 0
251
+ self.hidden_size_per_attention_head = (
252
+ self.projection_size // config.num_attention_heads
253
+ )
254
+
255
+ self.c_attn = nn.Linear(config.hidden_size, 3 * self.projection_size)
256
+
257
+ self.c_proj = nn.Linear(
258
+ config.hidden_size, self.projection_size, bias=not config.no_bias
259
+ )
260
+
261
+ self.is_fp32 = not (config.bf16 or config.fp16)
262
+ if (
263
+ self.use_flash_attn
264
+ and flash_attn_unpadded_func is not None
265
+ and not self.is_fp32
266
+ ):
267
+ self.core_attention_flash = FlashSelfAttention(
268
+ causal=True, attention_dropout=config.attn_dropout_prob
269
+ )
270
+ self.bf16 = config.bf16
271
+
272
+ self.use_dynamic_ntk = config.use_dynamic_ntk
273
+ self.use_logn_attn = config.use_logn_attn
274
+
275
+ logn_list = [
276
+ math.log(i, self.seq_length) if i > self.seq_length else 1
277
+ for i in range(1, 32768)
278
+ ]
279
+ logn_tensor = torch.tensor(logn_list)[None, :, None, None]
280
+ self.register_buffer("logn_tensor", logn_tensor, persistent=False)
281
+
282
+ self.attn_dropout = nn.Dropout(config.attn_dropout_prob)
283
+ self.softmax_in_fp32 = config.softmax_in_fp32 if hasattr(config, 'softmax_in_fp32') else False
284
+ self.use_cache_quantization = config.use_cache_quantization if hasattr(config, 'use_cache_quantization') else False
285
+ self.use_cache_kernel = config.use_cache_kernel if hasattr(config,'use_cache_kernel') else False
286
+ cache_dtype = torch.float
287
+ if self.bf16:
288
+ cache_dtype=torch.bfloat16
289
+ elif config.fp16:
290
+ cache_dtype = torch.float16
291
+ self.cache_qmax = torch.tensor(torch.iinfo(torch.uint8).max, dtype=cache_dtype)
292
+ self.cache_qmin = torch.tensor(torch.iinfo(torch.uint8).min, dtype=cache_dtype)
293
+
294
+ if config.use_cache_quantization and config.use_cache_kernel:
295
+ from .cpp_kernels import cache_autogptq_cuda_256
296
+ try:
297
+ self.cache_kernels = cache_autogptq_cuda_256
298
+ except ImportError:
299
+ self.cache_kernels = None
300
+
301
+ def _attn(self, query, key, value, registered_causal_mask, attention_mask=None, head_mask=None):
302
+ device = query.device
303
+ if self.use_cache_quantization:
304
+ qk, qk_scale, qk_zero = key
305
+ if self.use_cache_kernel and self.cache_kernels is not None:
306
+ shape = query.shape[:-1] + (qk.shape[-2],)
307
+ attn_weights = torch.zeros(shape, dtype=torch.float16, device=device)
308
+ self.cache_kernels.vecquant8matmul_batched_faster_old(
309
+ query.contiguous() if query.dtype == torch.float16 else query.to(torch.float16).contiguous(),
310
+ qk.transpose(-1, -2).contiguous(),
311
+ attn_weights,
312
+ qk_scale.contiguous() if qk_scale.dtype == torch.float16 else qk_scale.to(torch.float16).contiguous(),
313
+ qk_zero.contiguous()if qk_zero.dtype == torch.float16 else qk_zero.to(torch.float16).contiguous())
314
+ # attn_weights = attn_weights.to(query.dtype).contiguous()
315
+ else:
316
+ key = dequantize_cache_torch(qk, qk_scale, qk_zero)
317
+ attn_weights = torch.matmul(query, key.transpose(-1, -2))
318
+ else:
319
+ attn_weights = torch.matmul(query, key.transpose(-1, -2))
320
+
321
+ if self.scale_attn_weights:
322
+ if self.use_cache_quantization:
323
+ size_temp = value[0].size(-1)
324
+ else:
325
+ size_temp = value.size(-1)
326
+ attn_weights = attn_weights / torch.full(
327
+ [],
328
+ size_temp ** 0.5,
329
+ dtype=attn_weights.dtype,
330
+ device=attn_weights.device,
331
+ )
332
+ if self.use_cache_quantization:
333
+ query_length, key_length = query.size(-2), key[0].size(-2)
334
+ else:
335
+ query_length, key_length = query.size(-2), key.size(-2)
336
+ causal_mask = registered_causal_mask[
337
+ :, :, key_length - query_length : key_length, :key_length
338
+ ]
339
+ mask_value = torch.finfo(attn_weights.dtype).min
340
+ mask_value = torch.full([], mask_value, dtype=attn_weights.dtype).to(
341
+ attn_weights.device
342
+ )
343
+ attn_weights = torch.where(
344
+ causal_mask, attn_weights.to(attn_weights.dtype), mask_value
345
+ )
346
+
347
+ if attention_mask is not None:
348
+ attn_weights = attn_weights + attention_mask
349
+
350
+ if self.softmax_in_fp32:
351
+ attn_weights = nn.functional.softmax(attn_weights.float(), dim=-1)
352
+ else:
353
+ attn_weights = nn.functional.softmax(attn_weights, dim=-1)
354
+
355
+ attn_weights = attn_weights.type(query.dtype)
356
+ attn_weights = self.attn_dropout(attn_weights)
357
+
358
+ if head_mask is not None:
359
+ attn_weights = attn_weights * head_mask
360
+
361
+ if self.use_cache_quantization:
362
+ qv, qv_scale, qv_zero = value
363
+ if self.use_cache_kernel and self.cache_kernels is not None:
364
+ shape = attn_weights.shape[:-1] + (query.shape[-1],)
365
+ attn_output = torch.zeros(shape, dtype=torch.float16, device=device)
366
+ self.cache_kernels.vecquant8matmul_batched_column_compression_faster_old(
367
+ attn_weights.contiguous() if attn_weights.dtype == torch.float16 else attn_weights.to(torch.float16).contiguous(),
368
+ qv.contiguous(), # dtype: int32
369
+ attn_output,
370
+ qv_scale.contiguous() if qv_scale.dtype == torch.float16 else qv_scale.to(torch.float16).contiguous(),
371
+ qv_zero.contiguous() if qv_zero.dtype == torch.float16 else qv_zero.to(torch.float16).contiguous())
372
+ if attn_output.dtype != query.dtype:
373
+ attn_output = attn_output.to(query.dtype)
374
+ attn_weights = attn_weights.to(query.dtype)
375
+ else:
376
+ value = dequantize_cache_torch(qv, qv_scale, qv_zero)
377
+ attn_output = torch.matmul(attn_weights, value)
378
+ else:
379
+ attn_output = torch.matmul(attn_weights, value)
380
+
381
+ attn_output = attn_output.transpose(1, 2)
382
+
383
+ return attn_output, attn_weights
384
+
385
+ def _upcast_and_reordered_attn(
386
+ self, query, key, value, registered_causal_mask, attention_mask=None, head_mask=None
387
+ ):
388
+ bsz, num_heads, q_seq_len, dk = query.size()
389
+ _, _, k_seq_len, _ = key.size()
390
+
391
+ attn_weights = torch.empty(
392
+ bsz * num_heads,
393
+ q_seq_len,
394
+ k_seq_len,
395
+ dtype=torch.float32,
396
+ device=query.device,
397
+ )
398
+
399
+ scale_factor = 1.0
400
+ if self.scale_attn_weights:
401
+ scale_factor /= float(value.size(-1)) ** 0.5
402
+
403
+ with autocast(enabled=False):
404
+ q, k = query.reshape(-1, q_seq_len, dk), key.transpose(-1, -2).reshape(
405
+ -1, dk, k_seq_len
406
+ )
407
+ attn_weights = torch.baddbmm(
408
+ attn_weights, q.float(), k.float(), beta=0, alpha=scale_factor
409
+ )
410
+ attn_weights = attn_weights.reshape(bsz, num_heads, q_seq_len, k_seq_len)
411
+
412
+ query_length, key_length = query.size(-2), key.size(-2)
413
+ causal_mask = registered_causal_mask[
414
+ :, :, key_length - query_length : key_length, :key_length
415
+ ]
416
+ mask_value = torch.finfo(attn_weights.dtype).min
417
+ mask_value = torch.tensor(mask_value, dtype=attn_weights.dtype).to(
418
+ attn_weights.device
419
+ )
420
+ attn_weights = torch.where(causal_mask, attn_weights, mask_value)
421
+
422
+ if attention_mask is not None:
423
+ attn_weights = attn_weights + attention_mask
424
+
425
+ attn_weights = nn.functional.softmax(attn_weights, dim=-1)
426
+
427
+ if attn_weights.dtype != torch.float32:
428
+ raise RuntimeError(
429
+ "Error with upcasting, attn_weights does not have dtype torch.float32"
430
+ )
431
+ attn_weights = attn_weights.type(value.dtype)
432
+ attn_weights = self.attn_dropout(attn_weights)
433
+
434
+ if head_mask is not None:
435
+ attn_weights = attn_weights * head_mask
436
+
437
+ attn_output = torch.matmul(attn_weights, value)
438
+
439
+ return attn_output, attn_weights
440
+
441
+ def _split_heads(self, tensor, num_heads, attn_head_size):
442
+ new_shape = tensor.size()[:-1] + (num_heads, attn_head_size)
443
+ tensor = tensor.view(new_shape)
444
+ return tensor
445
+
446
+ def _merge_heads(self, tensor, num_heads, attn_head_size):
447
+ tensor = tensor.contiguous()
448
+ new_shape = tensor.size()[:-2] + (num_heads * attn_head_size,)
449
+ return tensor.view(new_shape)
450
+
451
+ def forward(
452
+ self,
453
+ hidden_states: Optional[Tuple[torch.FloatTensor]],
454
+ rotary_pos_emb_list: Optional[List[torch.Tensor]] = None,
455
+ registered_causal_mask: Optional[torch.Tensor] = None,
456
+ layer_past: Optional[Tuple[torch.Tensor]] = None,
457
+ attention_mask: Optional[torch.FloatTensor] = None,
458
+ head_mask: Optional[torch.FloatTensor] = None,
459
+ encoder_hidden_states: Optional[torch.Tensor] = None,
460
+ encoder_attention_mask: Optional[torch.FloatTensor] = None,
461
+ output_attentions: Optional[bool] = False,
462
+ use_cache: Optional[bool] = False,
463
+ ):
464
+ mixed_x_layer = self.c_attn(hidden_states)
465
+
466
+ query, key, value = mixed_x_layer.split(self.split_size, dim=2)
467
+
468
+ query = self._split_heads(query, self.num_heads, self.head_dim)
469
+ key = self._split_heads(key, self.num_heads, self.head_dim)
470
+ value = self._split_heads(value, self.num_heads, self.head_dim)
471
+
472
+ if rotary_pos_emb_list is not None:
473
+ cur_len = query.shape[1]
474
+ if len(rotary_pos_emb_list) == 1:
475
+ rotary_pos_emb = rotary_pos_emb_list[0]
476
+ rotary_pos_emb = [i[:, -cur_len:, :, :] for i in rotary_pos_emb]
477
+ rotary_pos_emb = (rotary_pos_emb,) * 2
478
+ q_pos_emb, k_pos_emb = rotary_pos_emb
479
+ # Slice the pos emb for current inference
480
+ query = apply_rotary_pos_emb(query, q_pos_emb)
481
+ key = apply_rotary_pos_emb(key, k_pos_emb)
482
+ else:
483
+ query_list = []
484
+ key_list = []
485
+ for i, rotary_pos_emb in enumerate(rotary_pos_emb_list):
486
+ rotary_pos_emb = [i[:, -cur_len:, :, :] for i in rotary_pos_emb]
487
+ rotary_pos_emb = (rotary_pos_emb,) * 2
488
+ q_pos_emb, k_pos_emb = rotary_pos_emb
489
+ # Slice the pos emb for current inference
490
+ query_list += [apply_rotary_pos_emb(query[i:i+1, :, :], q_pos_emb)]
491
+ key_list += [apply_rotary_pos_emb(key[i:i+1, :, :], k_pos_emb)]
492
+ query = torch.cat(query_list, dim=0)
493
+ key = torch.cat(key_list, dim=0)
494
+
495
+ if self.use_cache_quantization:
496
+ key = quantize_cache_v(key.permute(0, 2, 1, 3),
497
+ bits=8,
498
+ qmin=self.cache_qmin,
499
+ qmax=self.cache_qmax)
500
+ value = quantize_cache_v(value.permute(0, 2, 1, 3),
501
+ bits=8,
502
+ qmin=self.cache_qmin,
503
+ qmax=self.cache_qmax)
504
+
505
+
506
+ if layer_past is not None:
507
+ past_key, past_value = layer_past[0], layer_past[1]
508
+ if self.use_cache_quantization:
509
+ # use_cache_quantization:
510
+ # present=((q_key,key_scale,key_zero_point),
511
+ # (q_value,value_scale,value_zero_point))
512
+ key = (torch.cat((past_key[0], key[0]), dim=2),
513
+ torch.cat((past_key[1], key[1]), dim=2),
514
+ torch.cat((past_key[2], key[2]), dim=2))
515
+ value = (torch.cat((past_value[0], value[0]), dim=2),
516
+ torch.cat((past_value[1], value[1]), dim=2),
517
+ torch.cat((past_value[2], value[2]), dim=2))
518
+ else:
519
+ # not use_cache_quantization:
520
+ # present=(key,value)
521
+ key = torch.cat((past_key, key), dim=1)
522
+ value = torch.cat((past_value, value), dim=1)
523
+
524
+ if use_cache:
525
+ present = (key, value)
526
+ else:
527
+ present = None
528
+
529
+ if self.use_logn_attn and not self.training:
530
+ if self.use_cache_quantization:
531
+ seq_start = key[0].size(2) - query.size(1)
532
+ seq_end = key[0].size(2)
533
+ else:
534
+ seq_start = key.size(1) - query.size(1)
535
+ seq_end = key.size(1)
536
+ logn_tensor = self.logn_tensor[:, seq_start:seq_end, :, :]
537
+ query = query * logn_tensor.expand_as(query)
538
+
539
+ if (
540
+ self.use_flash_attn
541
+ and flash_attn_unpadded_func is not None
542
+ and not self.is_fp32
543
+ and query.is_cuda
544
+ ):
545
+ q, k, v = query, key, value
546
+ context_layer = self.core_attention_flash(q, k, v, attention_mask=attention_mask)
547
+
548
+ # b s h d -> b s (h d)
549
+ context_layer = context_layer.flatten(2,3).contiguous()
550
+
551
+ else:
552
+ query = query.permute(0, 2, 1, 3)
553
+ if not self.use_cache_quantization:
554
+ key = key.permute(0, 2, 1, 3)
555
+ value = value.permute(0, 2, 1, 3)
556
+ if (
557
+ registered_causal_mask is None
558
+ and self.use_flash_attn
559
+ and flash_attn_unpadded_func is not None
560
+ and not self.is_fp32
561
+ and not query.is_cuda
562
+ ):
563
+ raise Exception(_ERROR_INPUT_CPU_QUERY_WITH_FLASH_ATTN_ACTIVATED)
564
+ attn_output, attn_weight = self._attn(
565
+ query, key, value, registered_causal_mask, attention_mask, head_mask
566
+ )
567
+ context_layer = self._merge_heads(
568
+ attn_output, self.num_heads, self.head_dim
569
+ )
570
+
571
+ attn_output = self.c_proj(context_layer)
572
+
573
+ outputs = (attn_output, present)
574
+ if output_attentions:
575
+ if (
576
+ self.use_flash_attn
577
+ and flash_attn_unpadded_func is not None
578
+ and not self.is_fp32
579
+ ):
580
+ raise ValueError("Cannot output attentions while using flash-attn")
581
+ else:
582
+ outputs += (attn_weight,)
583
+
584
+ return outputs
585
+
586
+
587
+ class QWenMLP(nn.Module):
588
+ def __init__(self, config):
589
+ super().__init__()
590
+ self.w1 = nn.Linear(
591
+ config.hidden_size, config.intermediate_size // 2, bias=not config.no_bias
592
+ )
593
+ self.w2 = nn.Linear(
594
+ config.hidden_size, config.intermediate_size // 2, bias=not config.no_bias
595
+ )
596
+ ff_dim_in = config.intermediate_size // 2
597
+ self.c_proj = nn.Linear(ff_dim_in, config.hidden_size, bias=not config.no_bias)
598
+
599
+ def forward(self, hidden_states):
600
+ a1 = self.w1(hidden_states)
601
+ a2 = self.w2(hidden_states)
602
+ intermediate_parallel = a1 * F.silu(a2)
603
+ output = self.c_proj(intermediate_parallel)
604
+ return output
605
+
606
+ class QWenBlock(nn.Module):
607
+ def __init__(self, config):
608
+ super().__init__()
609
+ hidden_size = config.hidden_size
610
+ self.bf16 = config.bf16
611
+
612
+ self.ln_1 = RMSNorm(
613
+ hidden_size,
614
+ eps=config.layer_norm_epsilon,
615
+ )
616
+ self.attn = QWenAttention(config)
617
+ self.ln_2 = RMSNorm(
618
+ hidden_size,
619
+ eps=config.layer_norm_epsilon,
620
+ )
621
+
622
+ self.mlp = QWenMLP(config)
623
+
624
+ def forward(
625
+ self,
626
+ hidden_states: Optional[Tuple[torch.FloatTensor]],
627
+ rotary_pos_emb_list: Optional[List[torch.Tensor]] = None,
628
+ registered_causal_mask: Optional[torch.Tensor] = None,
629
+ layer_past: Optional[Tuple[torch.Tensor]] = None,
630
+ attention_mask: Optional[torch.FloatTensor] = None,
631
+ head_mask: Optional[torch.FloatTensor] = None,
632
+ encoder_hidden_states: Optional[torch.Tensor] = None,
633
+ encoder_attention_mask: Optional[torch.FloatTensor] = None,
634
+ use_cache: Optional[bool] = False,
635
+ output_attentions: Optional[bool] = False,
636
+ ):
637
+ layernorm_output = self.ln_1(hidden_states)
638
+
639
+ attn_outputs = self.attn(
640
+ layernorm_output,
641
+ rotary_pos_emb_list,
642
+ registered_causal_mask=registered_causal_mask,
643
+ layer_past=layer_past,
644
+ attention_mask=attention_mask,
645
+ head_mask=head_mask,
646
+ use_cache=use_cache,
647
+ output_attentions=output_attentions,
648
+ )
649
+ attn_output = attn_outputs[0]
650
+
651
+ outputs = attn_outputs[1:]
652
+
653
+ residual = hidden_states
654
+ layernorm_input = attn_output + residual
655
+
656
+ layernorm_output = self.ln_2(layernorm_input)
657
+
658
+ residual = layernorm_input
659
+ mlp_output = self.mlp(layernorm_output)
660
+ hidden_states = residual + mlp_output
661
+
662
+ if use_cache:
663
+ outputs = (hidden_states,) + outputs
664
+ else:
665
+ outputs = (hidden_states,) + outputs[1:]
666
+
667
+ return outputs
668
+
669
+
670
+ class QWenPreTrainedModel(PreTrainedModel):
671
+ config_class = QWenConfig
672
+ base_model_prefix = "transformer"
673
+ is_parallelizable = False
674
+ supports_gradient_checkpointing = True
675
+ _no_split_modules = ["QWenBlock"]
676
+
677
+ def __init__(self, *inputs, **kwargs):
678
+ super().__init__(*inputs, **kwargs)
679
+
680
+ def _init_weights(self, module):
681
+ """Initialize the weights."""
682
+ if isinstance(module, nn.Linear):
683
+ module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
684
+ if module.bias is not None:
685
+ module.bias.data.zero_()
686
+ elif isinstance(module, nn.Embedding):
687
+ module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
688
+ if module.padding_idx is not None:
689
+ module.weight.data[module.padding_idx].zero_()
690
+ elif isinstance(module, RMSNorm):
691
+ module.weight.data.fill_(1.0)
692
+
693
+ for name, p in module.named_parameters():
694
+ if name == "c_proj.weight":
695
+ p.data.normal_(
696
+ mean=0.0,
697
+ std=(
698
+ self.config.initializer_range
699
+ / math.sqrt(2 * self.config.num_hidden_layers)
700
+ ),
701
+ )
702
+
703
+ def _set_gradient_checkpointing(self, module, value=False):
704
+ if isinstance(module, QWenModel):
705
+ module.gradient_checkpointing = value
706
+
707
+
708
+ class QWenModel(QWenPreTrainedModel):
709
+ _keys_to_ignore_on_load_missing = ["attn.masked_bias"]
710
+
711
+ def __init__(self, config):
712
+ super().__init__(config)
713
+ self.vocab_size = config.vocab_size
714
+ self.num_hidden_layers = config.num_hidden_layers
715
+ self.embed_dim = config.hidden_size
716
+ self.use_cache_quantization = self.config.use_cache_quantization if hasattr(self.config, 'use_cache_quantization') else False
717
+
718
+ self.gradient_checkpointing = False
719
+ self.use_dynamic_ntk = config.use_dynamic_ntk
720
+ self.seq_length = config.seq_length
721
+
722
+ self.wte = nn.Embedding(self.vocab_size, self.embed_dim)
723
+
724
+ self.drop = nn.Dropout(config.emb_dropout_prob)
725
+
726
+ if config.rotary_pct == 1.0:
727
+ self.rotary_ndims = None
728
+ else:
729
+ assert config.rotary_pct < 1
730
+ self.rotary_ndims = int(
731
+ config.kv_channels * config.rotary_pct
732
+ )
733
+ dim = (
734
+ self.rotary_ndims
735
+ if self.rotary_ndims is not None
736
+ else config.kv_channels
737
+ )
738
+ self.rotary_emb = RotaryEmbedding(dim, base=config.rotary_emb_base)
739
+
740
+ self.use_flash_attn = config.use_flash_attn
741
+ self.is_fp32 = not (config.bf16 or config.fp16)
742
+ if (
743
+ self.use_flash_attn
744
+ and flash_attn_unpadded_func is not None
745
+ and not self.is_fp32
746
+ ):
747
+ self.registered_causal_mask = None
748
+ else:
749
+ max_positions = config.max_position_embeddings
750
+ self.register_buffer(
751
+ "registered_causal_mask",
752
+ torch.tril(
753
+ torch.ones((max_positions, max_positions), dtype=torch.bool)
754
+ ).view(1, 1, max_positions, max_positions),
755
+ persistent=False,
756
+ )
757
+
758
+ self.h = nn.ModuleList(
759
+ [
760
+ QWenBlock(
761
+ config
762
+ )
763
+ for i in range(config.num_hidden_layers)
764
+ ]
765
+ )
766
+ self.ln_f = RMSNorm(
767
+ self.embed_dim,
768
+ eps=config.layer_norm_epsilon,
769
+ )
770
+
771
+ self.post_init()
772
+
773
+ def get_input_embeddings(self):
774
+ return self.wte
775
+
776
+ def set_input_embeddings(self, new_embeddings):
777
+ self.wte = new_embeddings
778
+
779
+ def get_ntk_alpha(self, true_seq_len):
780
+ context_value = math.log(true_seq_len / self.seq_length, 2) + 1
781
+ ntk_alpha = 2 ** math.ceil(context_value) - 1
782
+ ntk_alpha = max(ntk_alpha, 1)
783
+ return ntk_alpha
784
+
785
+ def forward(
786
+ self,
787
+ input_ids: Optional[torch.LongTensor] = None,
788
+ past_key_values: Optional[Tuple[Tuple[torch.Tensor]]] = None,
789
+ attention_mask: Optional[torch.FloatTensor] = None,
790
+ token_type_ids: Optional[torch.LongTensor] = None,
791
+ position_ids: Optional[torch.LongTensor] = None,
792
+ head_mask: Optional[torch.FloatTensor] = None,
793
+ inputs_embeds: Optional[torch.FloatTensor] = None,
794
+ encoder_hidden_states: Optional[torch.Tensor] = None,
795
+ encoder_attention_mask: Optional[torch.FloatTensor] = None,
796
+ use_cache: Optional[bool] = None,
797
+ output_attentions: Optional[bool] = None,
798
+ output_hidden_states: Optional[bool] = None,
799
+ return_dict: Optional[bool] = None,
800
+ ):
801
+ output_attentions = (
802
+ output_attentions
803
+ if output_attentions is not None
804
+ else self.config.output_attentions
805
+ )
806
+ output_hidden_states = (
807
+ output_hidden_states
808
+ if output_hidden_states is not None
809
+ else self.config.output_hidden_states
810
+ )
811
+ use_cache = use_cache if use_cache is not None else self.config.use_cache
812
+ return_dict = (
813
+ return_dict if return_dict is not None else self.config.use_return_dict
814
+ )
815
+
816
+ if input_ids is not None and inputs_embeds is not None:
817
+ raise ValueError(
818
+ "You cannot specify both input_ids and inputs_embeds at the same time"
819
+ )
820
+ elif input_ids is not None:
821
+ input_shape = input_ids.size()
822
+ input_ids = input_ids.view(-1, input_shape[-1])
823
+ batch_size = input_ids.shape[0]
824
+ elif inputs_embeds is not None:
825
+ input_shape = inputs_embeds.size()[:-1]
826
+ batch_size = inputs_embeds.shape[0]
827
+ else:
828
+ raise ValueError("You have to specify either input_ids or inputs_embeds")
829
+
830
+ device = input_ids.device if input_ids is not None else inputs_embeds.device
831
+
832
+ if token_type_ids is not None:
833
+ token_type_ids = token_type_ids.view(-1, input_shape[-1])
834
+ if position_ids is not None:
835
+ position_ids = position_ids.view(-1, input_shape[-1])
836
+
837
+ if past_key_values is None:
838
+ past_length = 0
839
+ past_key_values = tuple([None] * len(self.h))
840
+ else:
841
+ if self.use_cache_quantization:
842
+ past_length = past_key_values[0][0][0].size(2)
843
+ else:
844
+ past_length = past_key_values[0][0].size(-2)
845
+ if position_ids is None:
846
+ position_ids = torch.arange(
847
+ past_length,
848
+ input_shape[-1] + past_length,
849
+ dtype=torch.long,
850
+ device=device,
851
+ )
852
+ position_ids = position_ids.unsqueeze(0).view(-1, input_shape[-1])
853
+
854
+ if attention_mask is not None:
855
+ if batch_size <= 0:
856
+ raise ValueError("batch_size has to be defined and > 0")
857
+ attention_mask = attention_mask.view(batch_size, -1)
858
+ attention_mask = attention_mask[:, None, None, :]
859
+ attention_mask = attention_mask.to(dtype=self.dtype)
860
+ attention_mask = (1.0 - attention_mask) * torch.finfo(self.dtype).min
861
+
862
+ encoder_attention_mask = None
863
+ head_mask = self.get_head_mask(head_mask, self.config.num_hidden_layers)
864
+
865
+ if inputs_embeds is None:
866
+ inputs_embeds = self.wte(input_ids)
867
+ hidden_states = inputs_embeds
868
+
869
+ kv_seq_len = hidden_states.size()[1]
870
+ if past_key_values[0] is not None:
871
+ # past key values[0][0] shape: bs * seq_len * head_num * dim
872
+ if self.use_cache_quantization:
873
+ kv_seq_len += past_key_values[0][0][0].shape[2]
874
+ else:
875
+ kv_seq_len += past_key_values[0][0].shape[1]
876
+
877
+ if self.training or not self.use_dynamic_ntk:
878
+ ntk_alpha_list = [1.0]
879
+ elif kv_seq_len != hidden_states.size()[1]:
880
+ ntk_alpha_list = self.rotary_emb._ntk_alpha_cached_list
881
+ else:
882
+ ntk_alpha_list = []
883
+ if attention_mask is not None and kv_seq_len > self.seq_length:
884
+ true_seq_lens = attention_mask.squeeze(1).squeeze(1).eq(0).sum(dim=-1, dtype=torch.int32)
885
+ for i in range(hidden_states.size()[0]):
886
+ true_seq_len = true_seq_lens[i].item()
887
+ ntk_alpha = self.get_ntk_alpha(true_seq_len)
888
+ ntk_alpha_list.append(ntk_alpha)
889
+ else:
890
+ ntk_alpha = self.get_ntk_alpha(kv_seq_len)
891
+ ntk_alpha_list.append(ntk_alpha)
892
+ self.rotary_emb._ntk_alpha_cached_list = ntk_alpha_list
893
+
894
+ rotary_pos_emb_list = []
895
+ for ntk_alpha in ntk_alpha_list:
896
+ rotary_pos_emb = self.rotary_emb(kv_seq_len, ntk_alpha=ntk_alpha)
897
+ rotary_pos_emb_list.append(rotary_pos_emb)
898
+
899
+ hidden_states = self.drop(hidden_states)
900
+ output_shape = input_shape + (hidden_states.size(-1),)
901
+
902
+ if self.gradient_checkpointing and self.training:
903
+ if use_cache:
904
+ logger.warning_once(
905
+ "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."
906
+ )
907
+ use_cache = False
908
+
909
+ presents = () if use_cache else None
910
+ all_self_attentions = () if output_attentions else None
911
+ all_hidden_states = () if output_hidden_states else None
912
+ for i, (block, layer_past) in enumerate(zip(self.h, past_key_values)):
913
+
914
+ if output_hidden_states:
915
+ all_hidden_states = all_hidden_states + (hidden_states,)
916
+
917
+ if self.gradient_checkpointing and self.training:
918
+
919
+ def create_custom_forward(module):
920
+ def custom_forward(*inputs):
921
+ # None for past_key_value
922
+ return module(*inputs, use_cache, output_attentions)
923
+
924
+ return custom_forward
925
+
926
+ outputs = torch.utils.checkpoint.checkpoint(
927
+ create_custom_forward(block),
928
+ hidden_states,
929
+ rotary_pos_emb_list,
930
+ self.registered_causal_mask,
931
+ None,
932
+ attention_mask,
933
+ head_mask[i],
934
+ encoder_hidden_states,
935
+ encoder_attention_mask,
936
+ )
937
+ else:
938
+ outputs = block(
939
+ hidden_states,
940
+ layer_past=layer_past,
941
+ rotary_pos_emb_list=rotary_pos_emb_list,
942
+ registered_causal_mask=self.registered_causal_mask,
943
+ attention_mask=attention_mask,
944
+ head_mask=head_mask[i],
945
+ encoder_hidden_states=encoder_hidden_states,
946
+ encoder_attention_mask=encoder_attention_mask,
947
+ use_cache=use_cache,
948
+ output_attentions=output_attentions,
949
+ )
950
+
951
+ hidden_states = outputs[0]
952
+ if use_cache is True:
953
+ presents = presents + (outputs[1],)
954
+
955
+ if output_attentions:
956
+ all_self_attentions = all_self_attentions + (outputs[2 if use_cache else 1],)
957
+
958
+ hidden_states = self.ln_f(hidden_states)
959
+ hidden_states = hidden_states.view(output_shape)
960
+ # Add last hidden state
961
+ if output_hidden_states:
962
+ all_hidden_states = all_hidden_states + (hidden_states,)
963
+
964
+ if not return_dict:
965
+ return tuple(
966
+ v for v in [hidden_states, presents, all_hidden_states] if v is not None
967
+ )
968
+
969
+ return BaseModelOutputWithPast(
970
+ last_hidden_state=hidden_states,
971
+ past_key_values=presents,
972
+ hidden_states=all_hidden_states,
973
+ attentions=all_self_attentions,
974
+ )
975
+
976
+
977
+ class QWenLMHeadModel(QWenPreTrainedModel):
978
+ _keys_to_ignore_on_load_missing = [r"h\.\d+\.attn\.rotary_emb\.inv_freq"]
979
+ _keys_to_ignore_on_load_unexpected = [r"h\.\d+\.attn\.masked_bias"]
980
+
981
+ def __init__(self, config):
982
+ super().__init__(config)
983
+ assert (
984
+ config.bf16 + config.fp16 + config.fp32 <= 1
985
+ ), "Only one of \"bf16\", \"fp16\", \"fp32\" can be true"
986
+ logger.warn(
987
+ "Warning: please make sure that you are using the latest codes and checkpoints, "
988
+ "especially if you used Qwen-7B before 09.25.2023."
989
+ "请使用最新模型和代码,尤其如果���在9月25日前已经开始使用Qwen-7B,千万注意不要使用错误代码和模型。"
990
+ )
991
+
992
+ autoset_precision = config.bf16 + config.fp16 + config.fp32 == 0
993
+
994
+ if autoset_precision:
995
+ if SUPPORT_BF16:
996
+ logger.warn(
997
+ "The model is automatically converting to bf16 for faster inference. "
998
+ "If you want to disable the automatic precision, please manually add bf16/fp16/fp32=True to \"AutoModelForCausalLM.from_pretrained\"."
999
+ )
1000
+ config.bf16 = True
1001
+ elif SUPPORT_FP16:
1002
+ logger.warn(
1003
+ "The model is automatically converting to fp16 for faster inference. "
1004
+ "If you want to disable the automatic precision, please manually add bf16/fp16/fp32=True to \"AutoModelForCausalLM.from_pretrained\"."
1005
+ )
1006
+ config.fp16 = True
1007
+ else:
1008
+ config.fp32 = True
1009
+
1010
+ if config.bf16 and SUPPORT_CUDA and not SUPPORT_BF16:
1011
+ logger.warn("Your device does NOT seem to support bf16, you can switch to fp16 or fp32 by by passing fp16/fp32=True in \"AutoModelForCausalLM.from_pretrained\".")
1012
+ if config.fp16 and SUPPORT_CUDA and not SUPPORT_FP16:
1013
+ logger.warn("Your device does NOT support faster inference with fp16, please switch to fp32 which is likely to be faster")
1014
+ if config.fp32:
1015
+ if SUPPORT_BF16:
1016
+ logger.warn("Your device support faster inference by passing bf16=True in \"AutoModelForCausalLM.from_pretrained\".")
1017
+ elif SUPPORT_FP16:
1018
+ logger.warn("Your device support faster inference by passing fp16=True in \"AutoModelForCausalLM.from_pretrained\".")
1019
+
1020
+ if config.use_flash_attn == "auto":
1021
+ if config.bf16 or config.fp16:
1022
+ logger.warn("Try importing flash-attention for faster inference...")
1023
+ config.use_flash_attn = True
1024
+ else:
1025
+ config.use_flash_attn = False
1026
+ if config.use_flash_attn and config.fp32:
1027
+ logger.warn("Flash attention will be disabled because it does NOT support fp32.")
1028
+
1029
+ if config.use_flash_attn:
1030
+ _import_flash_attn()
1031
+
1032
+ self.transformer = QWenModel(config)
1033
+ self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
1034
+
1035
+ if config.bf16:
1036
+ self.transformer.bfloat16()
1037
+ self.lm_head.bfloat16()
1038
+ if config.fp16:
1039
+ self.transformer.half()
1040
+ self.lm_head.half()
1041
+ self.post_init()
1042
+
1043
+
1044
+ def get_output_embeddings(self):
1045
+ return self.lm_head
1046
+
1047
+ def set_output_embeddings(self, new_embeddings):
1048
+ self.lm_head = new_embeddings
1049
+
1050
+ def prepare_inputs_for_generation(
1051
+ self, input_ids, past_key_values=None, inputs_embeds=None, **kwargs
1052
+ ):
1053
+ token_type_ids = kwargs.get("token_type_ids", None)
1054
+ if past_key_values:
1055
+ input_ids = input_ids[:, -1].unsqueeze(-1)
1056
+ if token_type_ids is not None:
1057
+ token_type_ids = token_type_ids[:, -1].unsqueeze(-1)
1058
+
1059
+ attention_mask = kwargs.get("attention_mask", None)
1060
+ position_ids = kwargs.get("position_ids", None)
1061
+
1062
+ if attention_mask is not None and position_ids is None:
1063
+ position_ids = attention_mask.long().cumsum(-1) - 1
1064
+ position_ids.masked_fill_(attention_mask == 0, 1)
1065
+ if past_key_values:
1066
+ position_ids = position_ids[:, -1].unsqueeze(-1)
1067
+ else:
1068
+ position_ids = None
1069
+
1070
+ if inputs_embeds is not None and past_key_values is None:
1071
+ model_inputs = {"inputs_embeds": inputs_embeds}
1072
+ else:
1073
+ model_inputs = {"input_ids": input_ids}
1074
+
1075
+ model_inputs.update(
1076
+ {
1077
+ "past_key_values": past_key_values,
1078
+ "use_cache": kwargs.get("use_cache"),
1079
+ "position_ids": position_ids,
1080
+ "attention_mask": attention_mask,
1081
+ "token_type_ids": token_type_ids,
1082
+ }
1083
+ )
1084
+ return model_inputs
1085
+
1086
+ def forward(
1087
+ self,
1088
+ input_ids: Optional[torch.LongTensor] = None,
1089
+ past_key_values: Optional[Tuple[Tuple[torch.Tensor]]] = None,
1090
+ attention_mask: Optional[torch.FloatTensor] = None,
1091
+ token_type_ids: Optional[torch.LongTensor] = None,
1092
+ position_ids: Optional[torch.LongTensor] = None,
1093
+ head_mask: Optional[torch.FloatTensor] = None,
1094
+ inputs_embeds: Optional[torch.FloatTensor] = None,
1095
+ encoder_hidden_states: Optional[torch.Tensor] = None,
1096
+ encoder_attention_mask: Optional[torch.FloatTensor] = None,
1097
+ labels: Optional[torch.LongTensor] = None,
1098
+ use_cache: Optional[bool] = None,
1099
+ output_attentions: Optional[bool] = None,
1100
+ output_hidden_states: Optional[bool] = None,
1101
+ return_dict: Optional[bool] = None,
1102
+ ) -> Union[Tuple, CausalLMOutputWithPast]:
1103
+
1104
+ return_dict = (
1105
+ return_dict if return_dict is not None else self.config.use_return_dict
1106
+ )
1107
+
1108
+ transformer_outputs = self.transformer(
1109
+ input_ids,
1110
+ past_key_values=past_key_values,
1111
+ attention_mask=attention_mask,
1112
+ token_type_ids=token_type_ids,
1113
+ position_ids=position_ids,
1114
+ head_mask=head_mask,
1115
+ inputs_embeds=inputs_embeds,
1116
+ encoder_hidden_states=encoder_hidden_states,
1117
+ encoder_attention_mask=encoder_attention_mask,
1118
+ use_cache=use_cache,
1119
+ output_attentions=output_attentions,
1120
+ output_hidden_states=output_hidden_states,
1121
+ return_dict=return_dict,
1122
+ )
1123
+ hidden_states = transformer_outputs[0]
1124
+
1125
+ lm_logits = self.lm_head(hidden_states)
1126
+
1127
+ loss = None
1128
+ if labels is not None:
1129
+ labels = labels.to(lm_logits.device)
1130
+ shift_logits = lm_logits[..., :-1, :].contiguous()
1131
+ shift_labels = labels[..., 1:].contiguous()
1132
+ loss_fct = CrossEntropyLoss()
1133
+ loss = loss_fct(
1134
+ shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1)
1135
+ )
1136
+
1137
+ if not return_dict:
1138
+ output = (lm_logits,) + transformer_outputs[1:]
1139
+ return ((loss,) + output) if loss is not None else output
1140
+
1141
+ return CausalLMOutputWithPast(
1142
+ loss=loss,
1143
+ logits=lm_logits,
1144
+ past_key_values=transformer_outputs.past_key_values,
1145
+ hidden_states=transformer_outputs.hidden_states,
1146
+ attentions=transformer_outputs.attentions,
1147
+ )
1148
+
1149
+ @staticmethod
1150
+ def _reorder_cache(
1151
+ past_key_values: Tuple[Tuple[torch.Tensor]], beam_idx: torch.Tensor
1152
+ ) -> Tuple[Tuple[torch.Tensor]]:
1153
+
1154
+ return tuple(
1155
+ tuple(
1156
+ past_state.index_select(0, beam_idx.to(past_state.device))
1157
+ for past_state in layer_past
1158
+ )
1159
+ for layer_past in past_key_values
1160
+ )
1161
+
1162
+ def chat(
1163
+ self,
1164
+ tokenizer: PreTrainedTokenizer,
1165
+ query: str,
1166
+ history: Optional[HistoryType],
1167
+ system: str = "You are a helpful assistant.",
1168
+ append_history: bool = True,
1169
+ stream: Optional[bool] = _SENTINEL,
1170
+ stop_words_ids: Optional[List[List[int]]] = None,
1171
+ generation_config: Optional[GenerationConfig] = None,
1172
+ **kwargs,
1173
+ ) -> Tuple[str, HistoryType]:
1174
+ generation_config = generation_config if generation_config is not None else self.generation_config
1175
+
1176
+ assert stream is _SENTINEL, _ERROR_STREAM_IN_CHAT
1177
+ assert generation_config.chat_format == 'chatml', _ERROR_BAD_CHAT_FORMAT
1178
+ if history is None:
1179
+ history = []
1180
+ if stop_words_ids is None:
1181
+ stop_words_ids = []
1182
+
1183
+ max_window_size = kwargs.get('max_window_size', None)
1184
+ if max_window_size is None:
1185
+ max_window_size = generation_config.max_window_size
1186
+ raw_text, context_tokens = make_context(
1187
+ tokenizer,
1188
+ query,
1189
+ history=history,
1190
+ system=system,
1191
+ max_window_size=max_window_size,
1192
+ chat_format=generation_config.chat_format,
1193
+ )
1194
+
1195
+ stop_words_ids.extend(get_stop_words_ids(
1196
+ generation_config.chat_format, tokenizer
1197
+ ))
1198
+ input_ids = torch.tensor([context_tokens]).to(self.device)
1199
+ outputs = self.generate(
1200
+ input_ids,
1201
+ stop_words_ids=stop_words_ids,
1202
+ return_dict_in_generate=False,
1203
+ generation_config=generation_config,
1204
+ **kwargs,
1205
+ )
1206
+
1207
+ response = decode_tokens(
1208
+ outputs[0],
1209
+ tokenizer,
1210
+ raw_text_len=len(raw_text),
1211
+ context_length=len(context_tokens),
1212
+ chat_format=generation_config.chat_format,
1213
+ verbose=False,
1214
+ errors='replace'
1215
+ )
1216
+
1217
+ if append_history:
1218
+ history.append((query, response))
1219
+
1220
+ return response, history
1221
+
1222
+ def chat_stream(
1223
+ self,
1224
+ tokenizer: PreTrainedTokenizer,
1225
+ query: str,
1226
+ history: Optional[HistoryType],
1227
+ system: str = "You are a helpful assistant.",
1228
+ stop_words_ids: Optional[List[List[int]]] = None,
1229
+ logits_processor: Optional[LogitsProcessorList] = None,
1230
+ generation_config: Optional[GenerationConfig] = None,
1231
+ **kwargs,
1232
+ ) -> Generator[str, Any, None]:
1233
+ generation_config = generation_config if generation_config is not None else self.generation_config
1234
+ assert generation_config.chat_format == 'chatml', _ERROR_BAD_CHAT_FORMAT
1235
+ if history is None:
1236
+ history = []
1237
+ if stop_words_ids is None:
1238
+ stop_words_ids = []
1239
+
1240
+ max_window_size = kwargs.get('max_window_size', None)
1241
+ if max_window_size is None:
1242
+ max_window_size = generation_config.max_window_size
1243
+ raw_text, context_tokens = make_context(
1244
+ tokenizer,
1245
+ query,
1246
+ history=history,
1247
+ system=system,
1248
+ max_window_size=max_window_size,
1249
+ chat_format=generation_config.chat_format,
1250
+ )
1251
+
1252
+ stop_words_ids.extend(get_stop_words_ids(
1253
+ generation_config.chat_format, tokenizer
1254
+ ))
1255
+ if stop_words_ids is not None:
1256
+ stop_words_logits_processor = StopWordsLogitsProcessor(
1257
+ stop_words_ids=stop_words_ids,
1258
+ eos_token_id=generation_config.eos_token_id,
1259
+ )
1260
+ if logits_processor is None:
1261
+ logits_processor = LogitsProcessorList([stop_words_logits_processor])
1262
+ else:
1263
+ logits_processor.append(stop_words_logits_processor)
1264
+ input_ids = torch.tensor([context_tokens]).to(self.device)
1265
+
1266
+ from transformers_stream_generator.main import NewGenerationMixin, StreamGenerationConfig
1267
+ self.__class__.generate_stream = NewGenerationMixin.generate
1268
+ self.__class__.sample_stream = NewGenerationMixin.sample_stream
1269
+ stream_config = StreamGenerationConfig(**generation_config.to_dict(), do_stream=True)
1270
+
1271
+ def stream_generator():
1272
+ outputs = []
1273
+ for token in self.generate_stream(
1274
+ input_ids,
1275
+ return_dict_in_generate=False,
1276
+ generation_config=stream_config,
1277
+ logits_processor=logits_processor,
1278
+ seed=-1,
1279
+ **kwargs):
1280
+ outputs.append(token.item())
1281
+ yield tokenizer.decode(outputs, skip_special_tokens=True, errors='ignore')
1282
+
1283
+ return stream_generator()
1284
+
1285
+ def generate(
1286
+ self,
1287
+ inputs: Optional[torch.Tensor] = None,
1288
+ generation_config: Optional[GenerationConfig] = None,
1289
+ logits_processor: Optional[LogitsProcessorList] = None,
1290
+ stopping_criteria: Optional[StoppingCriteriaList] = None,
1291
+ prefix_allowed_tokens_fn: Optional[
1292
+ Callable[[int, torch.Tensor], List[int]]
1293
+ ] = None,
1294
+ synced_gpus: Optional[bool] = None,
1295
+ assistant_model: Optional["PreTrainedModel"] = None,
1296
+ streamer: Optional["BaseStreamer"] = None,
1297
+ **kwargs,
1298
+ ) -> Union[GenerateOutput, torch.LongTensor]:
1299
+ generation_config = generation_config if generation_config is not None else self.generation_config
1300
+
1301
+ # Process stop_words_ids.
1302
+ stop_words_ids = kwargs.pop("stop_words_ids", None)
1303
+ if stop_words_ids is None and generation_config is not None:
1304
+ stop_words_ids = getattr(generation_config, "stop_words_ids", None)
1305
+ if stop_words_ids is None:
1306
+ stop_words_ids = getattr(generation_config, "stop_words_ids", None)
1307
+
1308
+ if stop_words_ids is not None:
1309
+ stop_words_logits_processor = StopWordsLogitsProcessor(
1310
+ stop_words_ids=stop_words_ids,
1311
+ eos_token_id=generation_config.eos_token_id,
1312
+ )
1313
+ if logits_processor is None:
1314
+ logits_processor = LogitsProcessorList([stop_words_logits_processor])
1315
+ else:
1316
+ logits_processor.append(stop_words_logits_processor)
1317
+
1318
+ return super().generate(
1319
+ inputs,
1320
+ generation_config=generation_config,
1321
+ logits_processor=logits_processor,
1322
+ stopping_criteria=stopping_criteria,
1323
+ prefix_allowed_tokens_fn=prefix_allowed_tokens_fn,
1324
+ synced_gpus=synced_gpus,
1325
+ assistant_model=assistant_model,
1326
+ streamer=streamer,
1327
+ **kwargs,
1328
+ )
1329
+
1330
+
1331
+ class RotaryEmbedding(torch.nn.Module):
1332
+ def __init__(self, dim, base=10000):
1333
+ super().__init__()
1334
+ self.dim = dim
1335
+ self.base = base
1336
+ inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2).float() / dim))
1337
+ self.register_buffer("inv_freq", inv_freq, persistent=False)
1338
+ if importlib.util.find_spec("einops") is None:
1339
+ raise RuntimeError("einops is required for Rotary Embedding")
1340
+
1341
+ self._rotary_pos_emb_cache = None
1342
+ self._seq_len_cached = 0
1343
+ self._ntk_alpha_cached = 1.0
1344
+ self._ntk_alpha_cached_list = [1.0]
1345
+
1346
+ def update_rotary_pos_emb_cache(self, max_seq_len, offset=0, ntk_alpha=1.0):
1347
+ seqlen = max_seq_len + offset
1348
+ if seqlen > self._seq_len_cached or ntk_alpha != self._ntk_alpha_cached:
1349
+ base = self.base * ntk_alpha ** (self.dim / (self.dim - 2))
1350
+ self.inv_freq = 1.0 / (
1351
+ base
1352
+ ** (
1353
+ torch.arange(0, self.dim, 2, device=self.inv_freq.device).float()
1354
+ / self.dim
1355
+ )
1356
+ )
1357
+ self._seq_len_cached = max(2 * seqlen, 16)
1358
+ self._ntk_alpha_cached = ntk_alpha
1359
+ seq = torch.arange(self._seq_len_cached, device=self.inv_freq.device)
1360
+ freqs = torch.outer(seq.type_as(self.inv_freq), self.inv_freq)
1361
+
1362
+ emb = torch.cat((freqs, freqs), dim=-1)
1363
+ from einops import rearrange
1364
+
1365
+ emb = rearrange(emb, "n d -> 1 n 1 d")
1366
+
1367
+ cos, sin = emb.cos(), emb.sin()
1368
+ self._rotary_pos_emb_cache = [cos, sin]
1369
+
1370
+ def forward(self, max_seq_len, offset=0, ntk_alpha=1.0):
1371
+ self.update_rotary_pos_emb_cache(max_seq_len, offset, ntk_alpha)
1372
+ cos, sin = self._rotary_pos_emb_cache
1373
+ return [cos[:, offset : offset + max_seq_len], sin[:, offset : offset + max_seq_len]]
1374
+
1375
+
1376
+ def _rotate_half(x):
1377
+ from einops import rearrange
1378
+
1379
+ x = rearrange(x, "... (j d) -> ... j d", j=2)
1380
+ x1, x2 = x.unbind(dim=-2)
1381
+ return torch.cat((-x2, x1), dim=-1)
1382
+
1383
+
1384
+ def apply_rotary_pos_emb(t, freqs):
1385
+ cos, sin = freqs
1386
+ if apply_rotary_emb_func is not None and t.is_cuda:
1387
+ t_ = t.float()
1388
+ cos = cos.squeeze(0).squeeze(1)[:, : cos.shape[-1] // 2]
1389
+ sin = sin.squeeze(0).squeeze(1)[:, : sin.shape[-1] // 2]
1390
+ output = apply_rotary_emb_func(t_, cos, sin).type_as(t)
1391
+ return output
1392
+ else:
1393
+ rot_dim = freqs[0].shape[-1]
1394
+ cos, sin = freqs
1395
+ t_, t_pass_ = t[..., :rot_dim], t[..., rot_dim:]
1396
+ t_ = t_.float()
1397
+ t_pass_ = t_pass_.float()
1398
+ t_ = (t_ * cos) + (_rotate_half(t_) * sin)
1399
+ return torch.cat((t_, t_pass_), dim=-1).type_as(t)
1400
+
1401
+
1402
+ class RMSNorm(torch.nn.Module):
1403
+ def __init__(self, dim: int, eps: float = 1e-6):
1404
+ super().__init__()
1405
+ self.eps = eps
1406
+ self.weight = nn.Parameter(torch.ones(dim))
1407
+
1408
+ def _norm(self, x):
1409
+ return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps)
1410
+
1411
+ def forward(self, x):
1412
+ if rms_norm is not None and x.is_cuda:
1413
+ return rms_norm(x, self.weight, self.eps)
1414
+ else:
1415
+ output = self._norm(x.float()).type_as(x)
1416
+ return output * self.weight
pytorch_model-00001-of-00003.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9030fdae1efd71718a4f0fa6e2aa0aa5e19bd83a4dbf14605800e512df71099c
3
+ size 9963536445
pytorch_model-00002-of-00003.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:778124078d20d501edfda59555ffc65a67e61ae664706f33ece14cee85af5190
3
+ size 9878405831
pytorch_model-00003-of-00003.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:21d15d7bb93b10e73221b5c2d894183f300861e98821fb2b69a043efbb41a8b5
3
+ size 8492747709
pytorch_model.bin.index.json ADDED
@@ -0,0 +1,330 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "metadata": {
3
+ "total_size": 28334581760
4
+ },
5
+ "weight_map": {
6
+ "lm_head.weight": "pytorch_model-00003-of-00003.bin",
7
+ "transformer.h.0.attn.c_attn.bias": "pytorch_model-00001-of-00003.bin",
8
+ "transformer.h.0.attn.c_attn.weight": "pytorch_model-00001-of-00003.bin",
9
+ "transformer.h.0.attn.c_proj.weight": "pytorch_model-00001-of-00003.bin",
10
+ "transformer.h.0.ln_1.weight": "pytorch_model-00001-of-00003.bin",
11
+ "transformer.h.0.ln_2.weight": "pytorch_model-00001-of-00003.bin",
12
+ "transformer.h.0.mlp.c_proj.weight": "pytorch_model-00001-of-00003.bin",
13
+ "transformer.h.0.mlp.w1.weight": "pytorch_model-00001-of-00003.bin",
14
+ "transformer.h.0.mlp.w2.weight": "pytorch_model-00001-of-00003.bin",
15
+ "transformer.h.1.attn.c_attn.bias": "pytorch_model-00001-of-00003.bin",
16
+ "transformer.h.1.attn.c_attn.weight": "pytorch_model-00001-of-00003.bin",
17
+ "transformer.h.1.attn.c_proj.weight": "pytorch_model-00001-of-00003.bin",
18
+ "transformer.h.1.ln_1.weight": "pytorch_model-00001-of-00003.bin",
19
+ "transformer.h.1.ln_2.weight": "pytorch_model-00001-of-00003.bin",
20
+ "transformer.h.1.mlp.c_proj.weight": "pytorch_model-00001-of-00003.bin",
21
+ "transformer.h.1.mlp.w1.weight": "pytorch_model-00001-of-00003.bin",
22
+ "transformer.h.1.mlp.w2.weight": "pytorch_model-00001-of-00003.bin",
23
+ "transformer.h.10.attn.c_attn.bias": "pytorch_model-00001-of-00003.bin",
24
+ "transformer.h.10.attn.c_attn.weight": "pytorch_model-00001-of-00003.bin",
25
+ "transformer.h.10.attn.c_proj.weight": "pytorch_model-00001-of-00003.bin",
26
+ "transformer.h.10.ln_1.weight": "pytorch_model-00001-of-00003.bin",
27
+ "transformer.h.10.ln_2.weight": "pytorch_model-00001-of-00003.bin",
28
+ "transformer.h.10.mlp.c_proj.weight": "pytorch_model-00001-of-00003.bin",
29
+ "transformer.h.10.mlp.w1.weight": "pytorch_model-00001-of-00003.bin",
30
+ "transformer.h.10.mlp.w2.weight": "pytorch_model-00001-of-00003.bin",
31
+ "transformer.h.11.attn.c_attn.bias": "pytorch_model-00001-of-00003.bin",
32
+ "transformer.h.11.attn.c_attn.weight": "pytorch_model-00001-of-00003.bin",
33
+ "transformer.h.11.attn.c_proj.weight": "pytorch_model-00001-of-00003.bin",
34
+ "transformer.h.11.ln_1.weight": "pytorch_model-00001-of-00003.bin",
35
+ "transformer.h.11.ln_2.weight": "pytorch_model-00001-of-00003.bin",
36
+ "transformer.h.11.mlp.c_proj.weight": "pytorch_model-00001-of-00003.bin",
37
+ "transformer.h.11.mlp.w1.weight": "pytorch_model-00001-of-00003.bin",
38
+ "transformer.h.11.mlp.w2.weight": "pytorch_model-00001-of-00003.bin",
39
+ "transformer.h.12.attn.c_attn.bias": "pytorch_model-00001-of-00003.bin",
40
+ "transformer.h.12.attn.c_attn.weight": "pytorch_model-00001-of-00003.bin",
41
+ "transformer.h.12.attn.c_proj.weight": "pytorch_model-00001-of-00003.bin",
42
+ "transformer.h.12.ln_1.weight": "pytorch_model-00001-of-00003.bin",
43
+ "transformer.h.12.ln_2.weight": "pytorch_model-00001-of-00003.bin",
44
+ "transformer.h.12.mlp.c_proj.weight": "pytorch_model-00001-of-00003.bin",
45
+ "transformer.h.12.mlp.w1.weight": "pytorch_model-00001-of-00003.bin",
46
+ "transformer.h.12.mlp.w2.weight": "pytorch_model-00001-of-00003.bin",
47
+ "transformer.h.13.attn.c_attn.bias": "pytorch_model-00001-of-00003.bin",
48
+ "transformer.h.13.attn.c_attn.weight": "pytorch_model-00001-of-00003.bin",
49
+ "transformer.h.13.attn.c_proj.weight": "pytorch_model-00001-of-00003.bin",
50
+ "transformer.h.13.ln_1.weight": "pytorch_model-00001-of-00003.bin",
51
+ "transformer.h.13.ln_2.weight": "pytorch_model-00001-of-00003.bin",
52
+ "transformer.h.13.mlp.c_proj.weight": "pytorch_model-00002-of-00003.bin",
53
+ "transformer.h.13.mlp.w1.weight": "pytorch_model-00002-of-00003.bin",
54
+ "transformer.h.13.mlp.w2.weight": "pytorch_model-00002-of-00003.bin",
55
+ "transformer.h.14.attn.c_attn.bias": "pytorch_model-00002-of-00003.bin",
56
+ "transformer.h.14.attn.c_attn.weight": "pytorch_model-00002-of-00003.bin",
57
+ "transformer.h.14.attn.c_proj.weight": "pytorch_model-00002-of-00003.bin",
58
+ "transformer.h.14.ln_1.weight": "pytorch_model-00002-of-00003.bin",
59
+ "transformer.h.14.ln_2.weight": "pytorch_model-00002-of-00003.bin",
60
+ "transformer.h.14.mlp.c_proj.weight": "pytorch_model-00002-of-00003.bin",
61
+ "transformer.h.14.mlp.w1.weight": "pytorch_model-00002-of-00003.bin",
62
+ "transformer.h.14.mlp.w2.weight": "pytorch_model-00002-of-00003.bin",
63
+ "transformer.h.15.attn.c_attn.bias": "pytorch_model-00002-of-00003.bin",
64
+ "transformer.h.15.attn.c_attn.weight": "pytorch_model-00002-of-00003.bin",
65
+ "transformer.h.15.attn.c_proj.weight": "pytorch_model-00002-of-00003.bin",
66
+ "transformer.h.15.ln_1.weight": "pytorch_model-00002-of-00003.bin",
67
+ "transformer.h.15.ln_2.weight": "pytorch_model-00002-of-00003.bin",
68
+ "transformer.h.15.mlp.c_proj.weight": "pytorch_model-00002-of-00003.bin",
69
+ "transformer.h.15.mlp.w1.weight": "pytorch_model-00002-of-00003.bin",
70
+ "transformer.h.15.mlp.w2.weight": "pytorch_model-00002-of-00003.bin",
71
+ "transformer.h.16.attn.c_attn.bias": "pytorch_model-00002-of-00003.bin",
72
+ "transformer.h.16.attn.c_attn.weight": "pytorch_model-00002-of-00003.bin",
73
+ "transformer.h.16.attn.c_proj.weight": "pytorch_model-00002-of-00003.bin",
74
+ "transformer.h.16.ln_1.weight": "pytorch_model-00002-of-00003.bin",
75
+ "transformer.h.16.ln_2.weight": "pytorch_model-00002-of-00003.bin",
76
+ "transformer.h.16.mlp.c_proj.weight": "pytorch_model-00002-of-00003.bin",
77
+ "transformer.h.16.mlp.w1.weight": "pytorch_model-00002-of-00003.bin",
78
+ "transformer.h.16.mlp.w2.weight": "pytorch_model-00002-of-00003.bin",
79
+ "transformer.h.17.attn.c_attn.bias": "pytorch_model-00002-of-00003.bin",
80
+ "transformer.h.17.attn.c_attn.weight": "pytorch_model-00002-of-00003.bin",
81
+ "transformer.h.17.attn.c_proj.weight": "pytorch_model-00002-of-00003.bin",
82
+ "transformer.h.17.ln_1.weight": "pytorch_model-00002-of-00003.bin",
83
+ "transformer.h.17.ln_2.weight": "pytorch_model-00002-of-00003.bin",
84
+ "transformer.h.17.mlp.c_proj.weight": "pytorch_model-00002-of-00003.bin",
85
+ "transformer.h.17.mlp.w1.weight": "pytorch_model-00002-of-00003.bin",
86
+ "transformer.h.17.mlp.w2.weight": "pytorch_model-00002-of-00003.bin",
87
+ "transformer.h.18.attn.c_attn.bias": "pytorch_model-00002-of-00003.bin",
88
+ "transformer.h.18.attn.c_attn.weight": "pytorch_model-00002-of-00003.bin",
89
+ "transformer.h.18.attn.c_proj.weight": "pytorch_model-00002-of-00003.bin",
90
+ "transformer.h.18.ln_1.weight": "pytorch_model-00002-of-00003.bin",
91
+ "transformer.h.18.ln_2.weight": "pytorch_model-00002-of-00003.bin",
92
+ "transformer.h.18.mlp.c_proj.weight": "pytorch_model-00002-of-00003.bin",
93
+ "transformer.h.18.mlp.w1.weight": "pytorch_model-00002-of-00003.bin",
94
+ "transformer.h.18.mlp.w2.weight": "pytorch_model-00002-of-00003.bin",
95
+ "transformer.h.19.attn.c_attn.bias": "pytorch_model-00002-of-00003.bin",
96
+ "transformer.h.19.attn.c_attn.weight": "pytorch_model-00002-of-00003.bin",
97
+ "transformer.h.19.attn.c_proj.weight": "pytorch_model-00002-of-00003.bin",
98
+ "transformer.h.19.ln_1.weight": "pytorch_model-00002-of-00003.bin",
99
+ "transformer.h.19.ln_2.weight": "pytorch_model-00002-of-00003.bin",
100
+ "transformer.h.19.mlp.c_proj.weight": "pytorch_model-00002-of-00003.bin",
101
+ "transformer.h.19.mlp.w1.weight": "pytorch_model-00002-of-00003.bin",
102
+ "transformer.h.19.mlp.w2.weight": "pytorch_model-00002-of-00003.bin",
103
+ "transformer.h.2.attn.c_attn.bias": "pytorch_model-00001-of-00003.bin",
104
+ "transformer.h.2.attn.c_attn.weight": "pytorch_model-00001-of-00003.bin",
105
+ "transformer.h.2.attn.c_proj.weight": "pytorch_model-00001-of-00003.bin",
106
+ "transformer.h.2.ln_1.weight": "pytorch_model-00001-of-00003.bin",
107
+ "transformer.h.2.ln_2.weight": "pytorch_model-00001-of-00003.bin",
108
+ "transformer.h.2.mlp.c_proj.weight": "pytorch_model-00001-of-00003.bin",
109
+ "transformer.h.2.mlp.w1.weight": "pytorch_model-00001-of-00003.bin",
110
+ "transformer.h.2.mlp.w2.weight": "pytorch_model-00001-of-00003.bin",
111
+ "transformer.h.20.attn.c_attn.bias": "pytorch_model-00002-of-00003.bin",
112
+ "transformer.h.20.attn.c_attn.weight": "pytorch_model-00002-of-00003.bin",
113
+ "transformer.h.20.attn.c_proj.weight": "pytorch_model-00002-of-00003.bin",
114
+ "transformer.h.20.ln_1.weight": "pytorch_model-00002-of-00003.bin",
115
+ "transformer.h.20.ln_2.weight": "pytorch_model-00002-of-00003.bin",
116
+ "transformer.h.20.mlp.c_proj.weight": "pytorch_model-00002-of-00003.bin",
117
+ "transformer.h.20.mlp.w1.weight": "pytorch_model-00002-of-00003.bin",
118
+ "transformer.h.20.mlp.w2.weight": "pytorch_model-00002-of-00003.bin",
119
+ "transformer.h.21.attn.c_attn.bias": "pytorch_model-00002-of-00003.bin",
120
+ "transformer.h.21.attn.c_attn.weight": "pytorch_model-00002-of-00003.bin",
121
+ "transformer.h.21.attn.c_proj.weight": "pytorch_model-00002-of-00003.bin",
122
+ "transformer.h.21.ln_1.weight": "pytorch_model-00002-of-00003.bin",
123
+ "transformer.h.21.ln_2.weight": "pytorch_model-00002-of-00003.bin",
124
+ "transformer.h.21.mlp.c_proj.weight": "pytorch_model-00002-of-00003.bin",
125
+ "transformer.h.21.mlp.w1.weight": "pytorch_model-00002-of-00003.bin",
126
+ "transformer.h.21.mlp.w2.weight": "pytorch_model-00002-of-00003.bin",
127
+ "transformer.h.22.attn.c_attn.bias": "pytorch_model-00002-of-00003.bin",
128
+ "transformer.h.22.attn.c_attn.weight": "pytorch_model-00002-of-00003.bin",
129
+ "transformer.h.22.attn.c_proj.weight": "pytorch_model-00002-of-00003.bin",
130
+ "transformer.h.22.ln_1.weight": "pytorch_model-00002-of-00003.bin",
131
+ "transformer.h.22.ln_2.weight": "pytorch_model-00002-of-00003.bin",
132
+ "transformer.h.22.mlp.c_proj.weight": "pytorch_model-00002-of-00003.bin",
133
+ "transformer.h.22.mlp.w1.weight": "pytorch_model-00002-of-00003.bin",
134
+ "transformer.h.22.mlp.w2.weight": "pytorch_model-00002-of-00003.bin",
135
+ "transformer.h.23.attn.c_attn.bias": "pytorch_model-00002-of-00003.bin",
136
+ "transformer.h.23.attn.c_attn.weight": "pytorch_model-00002-of-00003.bin",
137
+ "transformer.h.23.attn.c_proj.weight": "pytorch_model-00002-of-00003.bin",
138
+ "transformer.h.23.ln_1.weight": "pytorch_model-00002-of-00003.bin",
139
+ "transformer.h.23.ln_2.weight": "pytorch_model-00002-of-00003.bin",
140
+ "transformer.h.23.mlp.c_proj.weight": "pytorch_model-00002-of-00003.bin",
141
+ "transformer.h.23.mlp.w1.weight": "pytorch_model-00002-of-00003.bin",
142
+ "transformer.h.23.mlp.w2.weight": "pytorch_model-00002-of-00003.bin",
143
+ "transformer.h.24.attn.c_attn.bias": "pytorch_model-00002-of-00003.bin",
144
+ "transformer.h.24.attn.c_attn.weight": "pytorch_model-00002-of-00003.bin",
145
+ "transformer.h.24.attn.c_proj.weight": "pytorch_model-00002-of-00003.bin",
146
+ "transformer.h.24.ln_1.weight": "pytorch_model-00002-of-00003.bin",
147
+ "transformer.h.24.ln_2.weight": "pytorch_model-00002-of-00003.bin",
148
+ "transformer.h.24.mlp.c_proj.weight": "pytorch_model-00002-of-00003.bin",
149
+ "transformer.h.24.mlp.w1.weight": "pytorch_model-00002-of-00003.bin",
150
+ "transformer.h.24.mlp.w2.weight": "pytorch_model-00002-of-00003.bin",
151
+ "transformer.h.25.attn.c_attn.bias": "pytorch_model-00002-of-00003.bin",
152
+ "transformer.h.25.attn.c_attn.weight": "pytorch_model-00002-of-00003.bin",
153
+ "transformer.h.25.attn.c_proj.weight": "pytorch_model-00002-of-00003.bin",
154
+ "transformer.h.25.ln_1.weight": "pytorch_model-00002-of-00003.bin",
155
+ "transformer.h.25.ln_2.weight": "pytorch_model-00002-of-00003.bin",
156
+ "transformer.h.25.mlp.c_proj.weight": "pytorch_model-00002-of-00003.bin",
157
+ "transformer.h.25.mlp.w1.weight": "pytorch_model-00002-of-00003.bin",
158
+ "transformer.h.25.mlp.w2.weight": "pytorch_model-00002-of-00003.bin",
159
+ "transformer.h.26.attn.c_attn.bias": "pytorch_model-00002-of-00003.bin",
160
+ "transformer.h.26.attn.c_attn.weight": "pytorch_model-00002-of-00003.bin",
161
+ "transformer.h.26.attn.c_proj.weight": "pytorch_model-00002-of-00003.bin",
162
+ "transformer.h.26.ln_1.weight": "pytorch_model-00002-of-00003.bin",
163
+ "transformer.h.26.ln_2.weight": "pytorch_model-00002-of-00003.bin",
164
+ "transformer.h.26.mlp.c_proj.weight": "pytorch_model-00002-of-00003.bin",
165
+ "transformer.h.26.mlp.w1.weight": "pytorch_model-00002-of-00003.bin",
166
+ "transformer.h.26.mlp.w2.weight": "pytorch_model-00002-of-00003.bin",
167
+ "transformer.h.27.attn.c_attn.bias": "pytorch_model-00002-of-00003.bin",
168
+ "transformer.h.27.attn.c_attn.weight": "pytorch_model-00002-of-00003.bin",
169
+ "transformer.h.27.attn.c_proj.weight": "pytorch_model-00002-of-00003.bin",
170
+ "transformer.h.27.ln_1.weight": "pytorch_model-00002-of-00003.bin",
171
+ "transformer.h.27.ln_2.weight": "pytorch_model-00002-of-00003.bin",
172
+ "transformer.h.27.mlp.c_proj.weight": "pytorch_model-00002-of-00003.bin",
173
+ "transformer.h.27.mlp.w1.weight": "pytorch_model-00002-of-00003.bin",
174
+ "transformer.h.27.mlp.w2.weight": "pytorch_model-00002-of-00003.bin",
175
+ "transformer.h.28.attn.c_attn.bias": "pytorch_model-00002-of-00003.bin",
176
+ "transformer.h.28.attn.c_attn.weight": "pytorch_model-00002-of-00003.bin",
177
+ "transformer.h.28.attn.c_proj.weight": "pytorch_model-00002-of-00003.bin",
178
+ "transformer.h.28.ln_1.weight": "pytorch_model-00002-of-00003.bin",
179
+ "transformer.h.28.ln_2.weight": "pytorch_model-00002-of-00003.bin",
180
+ "transformer.h.28.mlp.c_proj.weight": "pytorch_model-00002-of-00003.bin",
181
+ "transformer.h.28.mlp.w1.weight": "pytorch_model-00002-of-00003.bin",
182
+ "transformer.h.28.mlp.w2.weight": "pytorch_model-00002-of-00003.bin",
183
+ "transformer.h.29.attn.c_attn.bias": "pytorch_model-00003-of-00003.bin",
184
+ "transformer.h.29.attn.c_attn.weight": "pytorch_model-00003-of-00003.bin",
185
+ "transformer.h.29.attn.c_proj.weight": "pytorch_model-00003-of-00003.bin",
186
+ "transformer.h.29.ln_1.weight": "pytorch_model-00002-of-00003.bin",
187
+ "transformer.h.29.ln_2.weight": "pytorch_model-00003-of-00003.bin",
188
+ "transformer.h.29.mlp.c_proj.weight": "pytorch_model-00003-of-00003.bin",
189
+ "transformer.h.29.mlp.w1.weight": "pytorch_model-00003-of-00003.bin",
190
+ "transformer.h.29.mlp.w2.weight": "pytorch_model-00003-of-00003.bin",
191
+ "transformer.h.3.attn.c_attn.bias": "pytorch_model-00001-of-00003.bin",
192
+ "transformer.h.3.attn.c_attn.weight": "pytorch_model-00001-of-00003.bin",
193
+ "transformer.h.3.attn.c_proj.weight": "pytorch_model-00001-of-00003.bin",
194
+ "transformer.h.3.ln_1.weight": "pytorch_model-00001-of-00003.bin",
195
+ "transformer.h.3.ln_2.weight": "pytorch_model-00001-of-00003.bin",
196
+ "transformer.h.3.mlp.c_proj.weight": "pytorch_model-00001-of-00003.bin",
197
+ "transformer.h.3.mlp.w1.weight": "pytorch_model-00001-of-00003.bin",
198
+ "transformer.h.3.mlp.w2.weight": "pytorch_model-00001-of-00003.bin",
199
+ "transformer.h.30.attn.c_attn.bias": "pytorch_model-00003-of-00003.bin",
200
+ "transformer.h.30.attn.c_attn.weight": "pytorch_model-00003-of-00003.bin",
201
+ "transformer.h.30.attn.c_proj.weight": "pytorch_model-00003-of-00003.bin",
202
+ "transformer.h.30.ln_1.weight": "pytorch_model-00003-of-00003.bin",
203
+ "transformer.h.30.ln_2.weight": "pytorch_model-00003-of-00003.bin",
204
+ "transformer.h.30.mlp.c_proj.weight": "pytorch_model-00003-of-00003.bin",
205
+ "transformer.h.30.mlp.w1.weight": "pytorch_model-00003-of-00003.bin",
206
+ "transformer.h.30.mlp.w2.weight": "pytorch_model-00003-of-00003.bin",
207
+ "transformer.h.31.attn.c_attn.bias": "pytorch_model-00003-of-00003.bin",
208
+ "transformer.h.31.attn.c_attn.weight": "pytorch_model-00003-of-00003.bin",
209
+ "transformer.h.31.attn.c_proj.weight": "pytorch_model-00003-of-00003.bin",
210
+ "transformer.h.31.ln_1.weight": "pytorch_model-00003-of-00003.bin",
211
+ "transformer.h.31.ln_2.weight": "pytorch_model-00003-of-00003.bin",
212
+ "transformer.h.31.mlp.c_proj.weight": "pytorch_model-00003-of-00003.bin",
213
+ "transformer.h.31.mlp.w1.weight": "pytorch_model-00003-of-00003.bin",
214
+ "transformer.h.31.mlp.w2.weight": "pytorch_model-00003-of-00003.bin",
215
+ "transformer.h.32.attn.c_attn.bias": "pytorch_model-00003-of-00003.bin",
216
+ "transformer.h.32.attn.c_attn.weight": "pytorch_model-00003-of-00003.bin",
217
+ "transformer.h.32.attn.c_proj.weight": "pytorch_model-00003-of-00003.bin",
218
+ "transformer.h.32.ln_1.weight": "pytorch_model-00003-of-00003.bin",
219
+ "transformer.h.32.ln_2.weight": "pytorch_model-00003-of-00003.bin",
220
+ "transformer.h.32.mlp.c_proj.weight": "pytorch_model-00003-of-00003.bin",
221
+ "transformer.h.32.mlp.w1.weight": "pytorch_model-00003-of-00003.bin",
222
+ "transformer.h.32.mlp.w2.weight": "pytorch_model-00003-of-00003.bin",
223
+ "transformer.h.33.attn.c_attn.bias": "pytorch_model-00003-of-00003.bin",
224
+ "transformer.h.33.attn.c_attn.weight": "pytorch_model-00003-of-00003.bin",
225
+ "transformer.h.33.attn.c_proj.weight": "pytorch_model-00003-of-00003.bin",
226
+ "transformer.h.33.ln_1.weight": "pytorch_model-00003-of-00003.bin",
227
+ "transformer.h.33.ln_2.weight": "pytorch_model-00003-of-00003.bin",
228
+ "transformer.h.33.mlp.c_proj.weight": "pytorch_model-00003-of-00003.bin",
229
+ "transformer.h.33.mlp.w1.weight": "pytorch_model-00003-of-00003.bin",
230
+ "transformer.h.33.mlp.w2.weight": "pytorch_model-00003-of-00003.bin",
231
+ "transformer.h.34.attn.c_attn.bias": "pytorch_model-00003-of-00003.bin",
232
+ "transformer.h.34.attn.c_attn.weight": "pytorch_model-00003-of-00003.bin",
233
+ "transformer.h.34.attn.c_proj.weight": "pytorch_model-00003-of-00003.bin",
234
+ "transformer.h.34.ln_1.weight": "pytorch_model-00003-of-00003.bin",
235
+ "transformer.h.34.ln_2.weight": "pytorch_model-00003-of-00003.bin",
236
+ "transformer.h.34.mlp.c_proj.weight": "pytorch_model-00003-of-00003.bin",
237
+ "transformer.h.34.mlp.w1.weight": "pytorch_model-00003-of-00003.bin",
238
+ "transformer.h.34.mlp.w2.weight": "pytorch_model-00003-of-00003.bin",
239
+ "transformer.h.35.attn.c_attn.bias": "pytorch_model-00003-of-00003.bin",
240
+ "transformer.h.35.attn.c_attn.weight": "pytorch_model-00003-of-00003.bin",
241
+ "transformer.h.35.attn.c_proj.weight": "pytorch_model-00003-of-00003.bin",
242
+ "transformer.h.35.ln_1.weight": "pytorch_model-00003-of-00003.bin",
243
+ "transformer.h.35.ln_2.weight": "pytorch_model-00003-of-00003.bin",
244
+ "transformer.h.35.mlp.c_proj.weight": "pytorch_model-00003-of-00003.bin",
245
+ "transformer.h.35.mlp.w1.weight": "pytorch_model-00003-of-00003.bin",
246
+ "transformer.h.35.mlp.w2.weight": "pytorch_model-00003-of-00003.bin",
247
+ "transformer.h.36.attn.c_attn.bias": "pytorch_model-00003-of-00003.bin",
248
+ "transformer.h.36.attn.c_attn.weight": "pytorch_model-00003-of-00003.bin",
249
+ "transformer.h.36.attn.c_proj.weight": "pytorch_model-00003-of-00003.bin",
250
+ "transformer.h.36.ln_1.weight": "pytorch_model-00003-of-00003.bin",
251
+ "transformer.h.36.ln_2.weight": "pytorch_model-00003-of-00003.bin",
252
+ "transformer.h.36.mlp.c_proj.weight": "pytorch_model-00003-of-00003.bin",
253
+ "transformer.h.36.mlp.w1.weight": "pytorch_model-00003-of-00003.bin",
254
+ "transformer.h.36.mlp.w2.weight": "pytorch_model-00003-of-00003.bin",
255
+ "transformer.h.37.attn.c_attn.bias": "pytorch_model-00003-of-00003.bin",
256
+ "transformer.h.37.attn.c_attn.weight": "pytorch_model-00003-of-00003.bin",
257
+ "transformer.h.37.attn.c_proj.weight": "pytorch_model-00003-of-00003.bin",
258
+ "transformer.h.37.ln_1.weight": "pytorch_model-00003-of-00003.bin",
259
+ "transformer.h.37.ln_2.weight": "pytorch_model-00003-of-00003.bin",
260
+ "transformer.h.37.mlp.c_proj.weight": "pytorch_model-00003-of-00003.bin",
261
+ "transformer.h.37.mlp.w1.weight": "pytorch_model-00003-of-00003.bin",
262
+ "transformer.h.37.mlp.w2.weight": "pytorch_model-00003-of-00003.bin",
263
+ "transformer.h.38.attn.c_attn.bias": "pytorch_model-00003-of-00003.bin",
264
+ "transformer.h.38.attn.c_attn.weight": "pytorch_model-00003-of-00003.bin",
265
+ "transformer.h.38.attn.c_proj.weight": "pytorch_model-00003-of-00003.bin",
266
+ "transformer.h.38.ln_1.weight": "pytorch_model-00003-of-00003.bin",
267
+ "transformer.h.38.ln_2.weight": "pytorch_model-00003-of-00003.bin",
268
+ "transformer.h.38.mlp.c_proj.weight": "pytorch_model-00003-of-00003.bin",
269
+ "transformer.h.38.mlp.w1.weight": "pytorch_model-00003-of-00003.bin",
270
+ "transformer.h.38.mlp.w2.weight": "pytorch_model-00003-of-00003.bin",
271
+ "transformer.h.39.attn.c_attn.bias": "pytorch_model-00003-of-00003.bin",
272
+ "transformer.h.39.attn.c_attn.weight": "pytorch_model-00003-of-00003.bin",
273
+ "transformer.h.39.attn.c_proj.weight": "pytorch_model-00003-of-00003.bin",
274
+ "transformer.h.39.ln_1.weight": "pytorch_model-00003-of-00003.bin",
275
+ "transformer.h.39.ln_2.weight": "pytorch_model-00003-of-00003.bin",
276
+ "transformer.h.39.mlp.c_proj.weight": "pytorch_model-00003-of-00003.bin",
277
+ "transformer.h.39.mlp.w1.weight": "pytorch_model-00003-of-00003.bin",
278
+ "transformer.h.39.mlp.w2.weight": "pytorch_model-00003-of-00003.bin",
279
+ "transformer.h.4.attn.c_attn.bias": "pytorch_model-00001-of-00003.bin",
280
+ "transformer.h.4.attn.c_attn.weight": "pytorch_model-00001-of-00003.bin",
281
+ "transformer.h.4.attn.c_proj.weight": "pytorch_model-00001-of-00003.bin",
282
+ "transformer.h.4.ln_1.weight": "pytorch_model-00001-of-00003.bin",
283
+ "transformer.h.4.ln_2.weight": "pytorch_model-00001-of-00003.bin",
284
+ "transformer.h.4.mlp.c_proj.weight": "pytorch_model-00001-of-00003.bin",
285
+ "transformer.h.4.mlp.w1.weight": "pytorch_model-00001-of-00003.bin",
286
+ "transformer.h.4.mlp.w2.weight": "pytorch_model-00001-of-00003.bin",
287
+ "transformer.h.5.attn.c_attn.bias": "pytorch_model-00001-of-00003.bin",
288
+ "transformer.h.5.attn.c_attn.weight": "pytorch_model-00001-of-00003.bin",
289
+ "transformer.h.5.attn.c_proj.weight": "pytorch_model-00001-of-00003.bin",
290
+ "transformer.h.5.ln_1.weight": "pytorch_model-00001-of-00003.bin",
291
+ "transformer.h.5.ln_2.weight": "pytorch_model-00001-of-00003.bin",
292
+ "transformer.h.5.mlp.c_proj.weight": "pytorch_model-00001-of-00003.bin",
293
+ "transformer.h.5.mlp.w1.weight": "pytorch_model-00001-of-00003.bin",
294
+ "transformer.h.5.mlp.w2.weight": "pytorch_model-00001-of-00003.bin",
295
+ "transformer.h.6.attn.c_attn.bias": "pytorch_model-00001-of-00003.bin",
296
+ "transformer.h.6.attn.c_attn.weight": "pytorch_model-00001-of-00003.bin",
297
+ "transformer.h.6.attn.c_proj.weight": "pytorch_model-00001-of-00003.bin",
298
+ "transformer.h.6.ln_1.weight": "pytorch_model-00001-of-00003.bin",
299
+ "transformer.h.6.ln_2.weight": "pytorch_model-00001-of-00003.bin",
300
+ "transformer.h.6.mlp.c_proj.weight": "pytorch_model-00001-of-00003.bin",
301
+ "transformer.h.6.mlp.w1.weight": "pytorch_model-00001-of-00003.bin",
302
+ "transformer.h.6.mlp.w2.weight": "pytorch_model-00001-of-00003.bin",
303
+ "transformer.h.7.attn.c_attn.bias": "pytorch_model-00001-of-00003.bin",
304
+ "transformer.h.7.attn.c_attn.weight": "pytorch_model-00001-of-00003.bin",
305
+ "transformer.h.7.attn.c_proj.weight": "pytorch_model-00001-of-00003.bin",
306
+ "transformer.h.7.ln_1.weight": "pytorch_model-00001-of-00003.bin",
307
+ "transformer.h.7.ln_2.weight": "pytorch_model-00001-of-00003.bin",
308
+ "transformer.h.7.mlp.c_proj.weight": "pytorch_model-00001-of-00003.bin",
309
+ "transformer.h.7.mlp.w1.weight": "pytorch_model-00001-of-00003.bin",
310
+ "transformer.h.7.mlp.w2.weight": "pytorch_model-00001-of-00003.bin",
311
+ "transformer.h.8.attn.c_attn.bias": "pytorch_model-00001-of-00003.bin",
312
+ "transformer.h.8.attn.c_attn.weight": "pytorch_model-00001-of-00003.bin",
313
+ "transformer.h.8.attn.c_proj.weight": "pytorch_model-00001-of-00003.bin",
314
+ "transformer.h.8.ln_1.weight": "pytorch_model-00001-of-00003.bin",
315
+ "transformer.h.8.ln_2.weight": "pytorch_model-00001-of-00003.bin",
316
+ "transformer.h.8.mlp.c_proj.weight": "pytorch_model-00001-of-00003.bin",
317
+ "transformer.h.8.mlp.w1.weight": "pytorch_model-00001-of-00003.bin",
318
+ "transformer.h.8.mlp.w2.weight": "pytorch_model-00001-of-00003.bin",
319
+ "transformer.h.9.attn.c_attn.bias": "pytorch_model-00001-of-00003.bin",
320
+ "transformer.h.9.attn.c_attn.weight": "pytorch_model-00001-of-00003.bin",
321
+ "transformer.h.9.attn.c_proj.weight": "pytorch_model-00001-of-00003.bin",
322
+ "transformer.h.9.ln_1.weight": "pytorch_model-00001-of-00003.bin",
323
+ "transformer.h.9.ln_2.weight": "pytorch_model-00001-of-00003.bin",
324
+ "transformer.h.9.mlp.c_proj.weight": "pytorch_model-00001-of-00003.bin",
325
+ "transformer.h.9.mlp.w1.weight": "pytorch_model-00001-of-00003.bin",
326
+ "transformer.h.9.mlp.w2.weight": "pytorch_model-00001-of-00003.bin",
327
+ "transformer.ln_f.weight": "pytorch_model-00003-of-00003.bin",
328
+ "transformer.wte.weight": "pytorch_model-00001-of-00003.bin"
329
+ }
330
+ }
qwen.tiktoken ADDED
The diff for this file is too large to render. See raw diff
 
qwen_generation_utils.py ADDED
@@ -0,0 +1,416 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba Cloud.
2
+ #
3
+ # This source code is licensed under the license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ """Generation support."""
7
+
8
+ from typing import Tuple, List, Union, Iterable
9
+
10
+ import numpy as np
11
+ import torch
12
+ import torch.nn.functional as F
13
+ from transformers import PreTrainedTokenizer
14
+ from transformers import logging
15
+ from transformers.generation import LogitsProcessor
16
+
17
+ logger = logging.get_logger(__name__)
18
+
19
+ # Types.
20
+ HistoryType = List[Tuple[str, str]]
21
+ TokensType = List[int]
22
+ BatchTokensType = List[List[int]]
23
+
24
+
25
+ def pad_batch(batch: BatchTokensType, pad_id: int, seq_length: int) -> BatchTokensType:
26
+ for tokens in batch:
27
+ context_length = len(tokens)
28
+ if context_length < seq_length:
29
+ tokens.extend([pad_id] * (seq_length - context_length))
30
+ return batch
31
+
32
+
33
+ def get_ltor_masks_and_position_ids(
34
+ data,
35
+ eod_token,
36
+ reset_position_ids,
37
+ reset_attention_mask,
38
+ eod_mask_loss,
39
+ ):
40
+ """Build masks and position id for left to right model."""
41
+
42
+ # Extract batch size and sequence length.
43
+ micro_batch_size, seq_length = data.size()
44
+
45
+ # Attention mask (lower triangular).
46
+ if reset_attention_mask:
47
+ att_mask_batch = micro_batch_size
48
+ else:
49
+ att_mask_batch = 1
50
+ attention_mask = torch.tril(
51
+ torch.ones((att_mask_batch, seq_length, seq_length), device=data.device)
52
+ ).view(att_mask_batch, 1, seq_length, seq_length)
53
+
54
+ # Loss mask.
55
+ loss_mask = torch.ones(data.size(), dtype=torch.float, device=data.device)
56
+ if eod_mask_loss:
57
+ loss_mask[data == eod_token] = 0.0
58
+
59
+ # Position ids.
60
+ position_ids = torch.arange(seq_length, dtype=torch.long, device=data.device)
61
+ position_ids = position_ids.unsqueeze(0).expand_as(data)
62
+ # We need to clone as the ids will be modifed based on batch index.
63
+ if reset_position_ids:
64
+ position_ids = position_ids.clone()
65
+
66
+ if reset_position_ids or reset_attention_mask:
67
+ # Loop through the batches:
68
+ for b in range(micro_batch_size):
69
+
70
+ # Find indecies where EOD token is.
71
+ eod_index = position_ids[b, data[b] == eod_token]
72
+ # Detach indecies from positions if going to modify positions.
73
+ if reset_position_ids:
74
+ eod_index = eod_index.clone()
75
+
76
+ # Loop through EOD indecies:
77
+ prev_index = 0
78
+ for j in range(eod_index.size()[0]):
79
+ i = eod_index[j]
80
+ # Mask attention loss.
81
+ if reset_attention_mask:
82
+ attention_mask[b, 0, (i + 1) :, : (i + 1)] = 0
83
+ # Reset positions.
84
+ if reset_position_ids:
85
+ position_ids[b, (i + 1) :] -= i + 1 - prev_index
86
+ prev_index = i + 1
87
+
88
+ # Convert attention mask to binary:
89
+ attention_mask = attention_mask < 0.5
90
+
91
+ return attention_mask, loss_mask, position_ids
92
+
93
+
94
+ def get_batch(context_tokens: torch.LongTensor, eod_id: int):
95
+ """Generate batch from context tokens."""
96
+ # Move to GPU.
97
+ tokens = context_tokens.contiguous().to(context_tokens.device)
98
+ # Get the attention mask and postition ids.
99
+ attention_mask, _, position_ids = get_ltor_masks_and_position_ids(
100
+ tokens,
101
+ eod_id,
102
+ reset_position_ids=False,
103
+ reset_attention_mask=False,
104
+ eod_mask_loss=False,
105
+ )
106
+ return tokens, attention_mask, position_ids
107
+
108
+
109
+ def get_stop_words_ids(chat_format, tokenizer):
110
+ if chat_format == "raw":
111
+ stop_words_ids = [tokenizer.encode("Human:"), [tokenizer.eod_id]]
112
+ elif chat_format == "chatml":
113
+ stop_words_ids = [[tokenizer.im_end_id], [tokenizer.im_start_id]]
114
+ else:
115
+ raise NotImplementedError(f"Unknown chat format {chat_format!r}")
116
+ return stop_words_ids
117
+
118
+
119
+ def make_context(
120
+ tokenizer: PreTrainedTokenizer,
121
+ query: str,
122
+ history: List[Tuple[str, str]] = None,
123
+ system: str = "",
124
+ max_window_size: int = 6144,
125
+ chat_format: str = "chatml",
126
+ ):
127
+ if history is None:
128
+ history = []
129
+
130
+ if chat_format == "chatml":
131
+ im_start, im_end = "<|im_start|>", "<|im_end|>"
132
+ im_start_tokens = [tokenizer.im_start_id]
133
+ im_end_tokens = [tokenizer.im_end_id]
134
+ nl_tokens = tokenizer.encode("\n")
135
+
136
+ def _tokenize_str(role, content):
137
+ return f"{role}\n{content}", tokenizer.encode(
138
+ role, allowed_special=set()
139
+ ) + nl_tokens + tokenizer.encode(content, allowed_special=set())
140
+
141
+ system_text, system_tokens_part = _tokenize_str("system", system)
142
+ system_tokens = im_start_tokens + system_tokens_part + im_end_tokens
143
+
144
+ raw_text = ""
145
+ context_tokens = []
146
+
147
+ for turn_query, turn_response in reversed(history):
148
+ query_text, query_tokens_part = _tokenize_str("user", turn_query)
149
+ query_tokens = im_start_tokens + query_tokens_part + im_end_tokens
150
+ response_text, response_tokens_part = _tokenize_str(
151
+ "assistant", turn_response
152
+ )
153
+ response_tokens = im_start_tokens + response_tokens_part + im_end_tokens
154
+
155
+ next_context_tokens = nl_tokens + query_tokens + nl_tokens + response_tokens
156
+ prev_chat = (
157
+ f"\n{im_start}{query_text}{im_end}\n{im_start}{response_text}{im_end}"
158
+ )
159
+
160
+ current_context_size = (
161
+ len(system_tokens) + len(next_context_tokens) + len(context_tokens)
162
+ )
163
+ if current_context_size < max_window_size:
164
+ context_tokens = next_context_tokens + context_tokens
165
+ raw_text = prev_chat + raw_text
166
+ else:
167
+ break
168
+
169
+ context_tokens = system_tokens + context_tokens
170
+ raw_text = f"{im_start}{system_text}{im_end}" + raw_text
171
+ context_tokens += (
172
+ nl_tokens
173
+ + im_start_tokens
174
+ + _tokenize_str("user", query)[1]
175
+ + im_end_tokens
176
+ + nl_tokens
177
+ + im_start_tokens
178
+ + tokenizer.encode("assistant")
179
+ + nl_tokens
180
+ )
181
+ raw_text += f"\n{im_start}user\n{query}{im_end}\n{im_start}assistant\n"
182
+
183
+ elif chat_format == "raw":
184
+ raw_text = query
185
+ context_tokens = tokenizer.encode(raw_text)
186
+ else:
187
+ raise NotImplementedError(f"Unknown chat format {chat_format!r}")
188
+
189
+ return raw_text, context_tokens
190
+
191
+
192
+ def _decode_default(
193
+ tokens: List[int],
194
+ *,
195
+ stop_words: List[str],
196
+ eod_words: List[str],
197
+ tokenizer: PreTrainedTokenizer,
198
+ raw_text_len: int,
199
+ verbose: bool = False,
200
+ return_end_reason: bool = False,
201
+ errors: str='replace',
202
+ ):
203
+ trim_decode_tokens = tokenizer.decode(tokens, errors=errors)[raw_text_len:]
204
+ if verbose:
205
+ print("\nRaw Generate: ", trim_decode_tokens)
206
+
207
+ end_reason = f"Gen length {len(tokens)}"
208
+ for stop_word in stop_words:
209
+ trim_decode_tokens = trim_decode_tokens.replace(stop_word, "").strip()
210
+ for eod_word in eod_words:
211
+ if eod_word in trim_decode_tokens:
212
+ end_reason = f"Gen {eod_word!r}"
213
+ trim_decode_tokens = trim_decode_tokens.split(eod_word)[0]
214
+ trim_decode_tokens = trim_decode_tokens.strip()
215
+ if verbose:
216
+ print("\nEnd Reason:", end_reason)
217
+ print("\nGenerate: ", trim_decode_tokens)
218
+
219
+ if return_end_reason:
220
+ return trim_decode_tokens, end_reason
221
+ else:
222
+ return trim_decode_tokens
223
+
224
+
225
+ def _decode_chatml(
226
+ tokens: List[int],
227
+ *,
228
+ stop_words: List[str],
229
+ eod_token_ids: List[int],
230
+ tokenizer: PreTrainedTokenizer,
231
+ raw_text_len: int,
232
+ context_length: int,
233
+ verbose: bool = False,
234
+ return_end_reason: bool = False,
235
+ errors: str='replace'
236
+ ):
237
+ end_reason = f"Gen length {len(tokens)}"
238
+ eod_token_idx = context_length
239
+ for eod_token_idx in range(context_length, len(tokens)):
240
+ if tokens[eod_token_idx] in eod_token_ids:
241
+ end_reason = f"Gen {tokenizer.decode([tokens[eod_token_idx]])!r}"
242
+ break
243
+
244
+ trim_decode_tokens = tokenizer.decode(tokens[:eod_token_idx], errors=errors)[raw_text_len:]
245
+ if verbose:
246
+ print("\nRaw Generate w/o EOD:", tokenizer.decode(tokens, errors=errors)[raw_text_len:])
247
+ print("\nRaw Generate:", trim_decode_tokens)
248
+ print("\nEnd Reason:", end_reason)
249
+ for stop_word in stop_words:
250
+ trim_decode_tokens = trim_decode_tokens.replace(stop_word, "").strip()
251
+ trim_decode_tokens = trim_decode_tokens.strip()
252
+ if verbose:
253
+ print("\nGenerate:", trim_decode_tokens)
254
+
255
+ if return_end_reason:
256
+ return trim_decode_tokens, end_reason
257
+ else:
258
+ return trim_decode_tokens
259
+
260
+
261
+ def decode_tokens(
262
+ tokens: Union[torch.LongTensor, TokensType],
263
+ tokenizer: PreTrainedTokenizer,
264
+ raw_text_len: int,
265
+ context_length: int,
266
+ chat_format: str,
267
+ verbose: bool = False,
268
+ return_end_reason: bool = False,
269
+ errors: str="replace",
270
+ ) -> str:
271
+ if torch.is_tensor(tokens):
272
+ tokens = tokens.cpu().numpy().tolist()
273
+
274
+ if chat_format == "chatml":
275
+ return _decode_chatml(
276
+ tokens,
277
+ stop_words=[],
278
+ eod_token_ids=[tokenizer.im_start_id, tokenizer.im_end_id],
279
+ tokenizer=tokenizer,
280
+ raw_text_len=raw_text_len,
281
+ context_length=context_length,
282
+ verbose=verbose,
283
+ return_end_reason=return_end_reason,
284
+ errors=errors,
285
+ )
286
+ elif chat_format == "raw":
287
+ return _decode_default(
288
+ tokens,
289
+ stop_words=["<|endoftext|>"],
290
+ eod_words=["<|endoftext|>"],
291
+ tokenizer=tokenizer,
292
+ raw_text_len=raw_text_len,
293
+ verbose=verbose,
294
+ return_end_reason=return_end_reason,
295
+ errors=errors,
296
+ )
297
+ else:
298
+ raise NotImplementedError(f"Unknown chat format {chat_format!r}")
299
+
300
+
301
+ class StopWordsLogitsProcessor(LogitsProcessor):
302
+ """
303
+ :class:`transformers.LogitsProcessor` that enforces that when specified sequences appear, stop geration.
304
+
305
+ Args:
306
+ stop_words_ids (:obj:`List[List[int]]`):
307
+ List of list of token ids of stop ids. In order to get the tokens of the words
308
+ that should not appear in the generated text, use :obj:`tokenizer(bad_word,
309
+ add_prefix_space=True).input_ids`.
310
+ eos_token_id (:obj:`int`):
311
+ The id of the `end-of-sequence` token.
312
+ """
313
+
314
+ def __init__(self, stop_words_ids: Iterable[Iterable[int]], eos_token_id: int):
315
+
316
+ if not isinstance(stop_words_ids, List) or len(stop_words_ids) == 0:
317
+ raise ValueError(
318
+ f"`stop_words_ids` has to be a non-emtpy list, but is {stop_words_ids}."
319
+ )
320
+ if any(not isinstance(bad_word_ids, list) for bad_word_ids in stop_words_ids):
321
+ raise ValueError(
322
+ f"`stop_words_ids` has to be a list of lists, but is {stop_words_ids}."
323
+ )
324
+ if any(
325
+ any(
326
+ (not isinstance(token_id, (int, np.integer)) or token_id < 0)
327
+ for token_id in stop_word_ids
328
+ )
329
+ for stop_word_ids in stop_words_ids
330
+ ):
331
+ raise ValueError(
332
+ f"Each list in `stop_words_ids` has to be a list of positive integers, but is {stop_words_ids}."
333
+ )
334
+
335
+ self.stop_words_ids = list(
336
+ filter(
337
+ lambda bad_token_seq: bad_token_seq != [eos_token_id], stop_words_ids
338
+ )
339
+ )
340
+ self.eos_token_id = eos_token_id
341
+ for stop_token_seq in self.stop_words_ids:
342
+ assert (
343
+ len(stop_token_seq) > 0
344
+ ), "Stop words token sequences {} cannot have an empty list".format(
345
+ stop_words_ids
346
+ )
347
+
348
+ def __call__(
349
+ self, input_ids: torch.LongTensor, scores: torch.FloatTensor
350
+ ) -> torch.FloatTensor:
351
+ stopped_samples = self._calc_stopped_samples(input_ids)
352
+ for i, should_stop in enumerate(stopped_samples):
353
+ if should_stop:
354
+ scores[i, self.eos_token_id] = float(2**15)
355
+ return scores
356
+
357
+ def _tokens_match(self, prev_tokens: torch.LongTensor, tokens: List[int]) -> bool:
358
+ if len(tokens) == 0:
359
+ # if bad word tokens is just one token always ban it
360
+ return True
361
+ elif len(tokens) > len(prev_tokens):
362
+ # if bad word tokens are longer then prev input_ids they can't be equal
363
+ return False
364
+ elif prev_tokens[-len(tokens) :].tolist() == tokens:
365
+ # if tokens match
366
+ return True
367
+ else:
368
+ return False
369
+
370
+ def _calc_stopped_samples(self, prev_input_ids: Iterable[int]) -> Iterable[int]:
371
+ stopped_samples = []
372
+ for prev_input_ids_slice in prev_input_ids:
373
+ match = False
374
+ for stop_token_seq in self.stop_words_ids:
375
+ if self._tokens_match(prev_input_ids_slice, stop_token_seq):
376
+ # if tokens do not match continue
377
+ match = True
378
+ break
379
+ stopped_samples.append(match)
380
+
381
+ return stopped_samples
382
+
383
+
384
+ def top_k_logits(logits, top_k=0, top_p=0.0, filter_value=-float("Inf")):
385
+ """This function has been mostly taken from huggingface conversational
386
+ ai code at
387
+ https://medium.com/huggingface/how-to-build-a-state-of-the-art-
388
+ conversational-ai-with-transfer-learning-2d818ac26313"""
389
+
390
+ if top_k > 0:
391
+ # Remove all tokens with a probability less than the
392
+ # last token of the top-k
393
+ indices_to_remove = logits < torch.topk(logits, top_k)[0][..., -1, None]
394
+ logits[indices_to_remove] = filter_value
395
+
396
+ if top_p > 0.0:
397
+ # Cconvert to 1D
398
+ sorted_logits, sorted_indices = torch.sort(logits, descending=True, dim=-1)
399
+ cumulative_probs = torch.cumsum(F.softmax(sorted_logits, dim=-1), dim=-1)
400
+
401
+ # Remove tokens with cumulative probability above the threshold
402
+ sorted_indices_to_remove = cumulative_probs > top_p
403
+ # Shift the indices to the right to keep also the first token
404
+ # above the threshold
405
+ sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone()
406
+ sorted_indices_to_remove[..., 0] = 0
407
+ for i in range(sorted_indices.size(0)):
408
+ indices_to_remove = sorted_indices[i][sorted_indices_to_remove[i]]
409
+ logits[i][indices_to_remove] = filter_value
410
+
411
+ return logits
412
+
413
+
414
+ def switch(val1, val2, boolean):
415
+ boolean = boolean.type_as(val1)
416
+ return (1 - boolean) * val1 + boolean * val2
rng_state_0.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4c1d4aab692d41c18c919a673e506a5264ad9de9ff132b0de5dab52510157c75
3
+ size 21687
rng_state_1.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:72575f2b4522908157bba855533d15354ced6598ecdc09ca8ce5417fa99c9b92
3
+ size 21687
rng_state_2.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:eabb43b1fac99cd8e34e58482a942b31a1855cd53120560e1428e8a2aae185e9
3
+ size 21687
rng_state_3.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cfd9568c2d39f428f86eb5a99beb6f19ac57f3f024ec4494b8cadd107d148298
3
+ size 21687
rng_state_4.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:21ab6cb42d703108752ec781af0a5227b721321c411bb7f7bca00b6ec4e15324
3
+ size 21687
rng_state_5.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c9fe828193c13a4a1adccaf24735d85428cb925bfa6204603f3a0a4a34c8cec9
3
+ size 21687
rng_state_6.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:36e15c4dcb706b97a74d0f8259d6466dd2e46e60fd71dcb2c6d99e1e9ff2f204
3
+ size 21687
rng_state_7.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7a467ddda52bc75625e56adc924121f89adb1e65a5733eec05f6284586a9a926
3
+ size 21687
special_tokens_map.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ {
2
+ "pad_token": "<|endoftext|>"
3
+ }
tokenization_qwen.py ADDED
@@ -0,0 +1,246 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) Alibaba Cloud.
2
+ #
3
+ # This source code is licensed under the license found in the
4
+ # LICENSE file in the root directory of this source tree.
5
+
6
+ """Tokenization classes for QWen."""
7
+
8
+ import base64
9
+ import logging
10
+ import os
11
+ import unicodedata
12
+ from typing import Collection, Dict, List, Set, Tuple, Union
13
+
14
+ import tiktoken
15
+ from transformers import PreTrainedTokenizer, AddedToken
16
+
17
+ logger = logging.getLogger(__name__)
18
+
19
+
20
+ VOCAB_FILES_NAMES = {"vocab_file": "qwen.tiktoken"}
21
+
22
+ PAT_STR = r"""(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+"""
23
+ ENDOFTEXT = "<|endoftext|>"
24
+ IMSTART = "<|im_start|>"
25
+ IMEND = "<|im_end|>"
26
+ # as the default behavior is changed to allow special tokens in
27
+ # regular texts, the surface forms of special tokens need to be
28
+ # as different as possible to minimize the impact
29
+ EXTRAS = tuple((f"<|extra_{i}|>" for i in range(205)))
30
+ SPECIAL_TOKENS = (
31
+ ENDOFTEXT,
32
+ IMSTART,
33
+ IMEND,
34
+ ) + EXTRAS
35
+
36
+
37
+ def _load_tiktoken_bpe(tiktoken_bpe_file: str) -> Dict[bytes, int]:
38
+ with open(tiktoken_bpe_file, "rb") as f:
39
+ contents = f.read()
40
+ return {
41
+ base64.b64decode(token): int(rank)
42
+ for token, rank in (line.split() for line in contents.splitlines() if line)
43
+ }
44
+
45
+ class QWenTokenizer(PreTrainedTokenizer):
46
+ """QWen tokenizer."""
47
+
48
+ vocab_files_names = VOCAB_FILES_NAMES
49
+
50
+ def __init__(
51
+ self,
52
+ vocab_file,
53
+ errors="replace",
54
+ **kwargs,
55
+ ):
56
+ super().__init__(**kwargs)
57
+
58
+ self.errors = errors # how to handle errors in decoding
59
+
60
+ self.mergeable_ranks = _load_tiktoken_bpe(vocab_file) # type: dict[bytes, int]
61
+ self.special_tokens = {
62
+ token: index
63
+ for index, token in enumerate(
64
+ SPECIAL_TOKENS, start=len(self.mergeable_ranks)
65
+ )
66
+ }
67
+
68
+ enc = tiktoken.Encoding(
69
+ "Qwen",
70
+ pat_str=PAT_STR,
71
+ mergeable_ranks=self.mergeable_ranks,
72
+ special_tokens=self.special_tokens,
73
+ )
74
+ assert (
75
+ len(self.mergeable_ranks) + len(self.special_tokens) == enc.n_vocab
76
+ ), f"{len(self.mergeable_ranks) + len(self.special_tokens)} != {enc.n_vocab} in encoding"
77
+
78
+ self.decoder = {
79
+ v: k for k, v in self.mergeable_ranks.items()
80
+ } # type: dict[int, bytes|str]
81
+ self.decoder.update({v: k for k, v in self.special_tokens.items()})
82
+
83
+ self.tokenizer = enc # type: tiktoken.Encoding
84
+
85
+ self.eod_id = self.tokenizer.eot_token
86
+ self.im_start_id = self.special_tokens[IMSTART]
87
+ self.im_end_id = self.special_tokens[IMEND]
88
+
89
+ def __getstate__(self):
90
+ # for pickle lovers
91
+ state = self.__dict__.copy()
92
+ del state['tokenizer']
93
+ return state
94
+
95
+ def __setstate__(self, state):
96
+ # tokenizer is not python native; don't pass it; rebuild it
97
+ self.__dict__.update(state)
98
+ enc = tiktoken.Encoding(
99
+ "Qwen",
100
+ pat_str=PAT_STR,
101
+ mergeable_ranks=self.mergeable_ranks,
102
+ special_tokens=self.special_tokens,
103
+ )
104
+ self.tokenizer = enc
105
+
106
+
107
+ def __len__(self) -> int:
108
+ return self.tokenizer.n_vocab
109
+
110
+ def get_vocab(self) -> Dict[bytes, int]:
111
+ return self.mergeable_ranks
112
+
113
+ def convert_tokens_to_ids(
114
+ self, tokens: Union[bytes, str, List[Union[bytes, str]]]
115
+ ) -> List[int]:
116
+ ids = []
117
+ if isinstance(tokens, (str, bytes)):
118
+ if tokens in self.special_tokens:
119
+ return self.special_tokens[tokens]
120
+ else:
121
+ return self.mergeable_ranks.get(tokens)
122
+ for token in tokens:
123
+ if token in self.special_tokens:
124
+ ids.append(self.special_tokens[token])
125
+ else:
126
+ ids.append(self.mergeable_ranks.get(token))
127
+ return ids
128
+
129
+ def _add_tokens(self, new_tokens: Union[List[str], List[AddedToken]], special_tokens: bool = False) -> int:
130
+ if not special_tokens and new_tokens:
131
+ raise ValueError('Adding regular tokens is not supported')
132
+ for token in new_tokens:
133
+ surface_form = token.content if isinstance(token, AddedToken) else token
134
+ if surface_form not in SPECIAL_TOKENS:
135
+ raise ValueError('Adding unknown special tokens is not supported')
136
+ return 0
137
+
138
+ def save_vocabulary(self, save_directory: str, **kwargs) -> Tuple[str]:
139
+ """
140
+ Save only the vocabulary of the tokenizer (vocabulary).
141
+
142
+ Returns:
143
+ `Tuple(str)`: Paths to the files saved.
144
+ """
145
+ file_path = os.path.join(save_directory, "qwen.tiktoken")
146
+ with open(file_path, "w", encoding="utf8") as w:
147
+ for k, v in self.mergeable_ranks.items():
148
+ line = base64.b64encode(k).decode("utf8") + " " + str(v) + "\n"
149
+ w.write(line)
150
+ return (file_path,)
151
+
152
+ def tokenize(
153
+ self,
154
+ text: str,
155
+ allowed_special: Union[Set, str] = "all",
156
+ disallowed_special: Union[Collection, str] = (),
157
+ **kwargs,
158
+ ) -> List[Union[bytes, str]]:
159
+ """
160
+ Converts a string in a sequence of tokens.
161
+
162
+ Args:
163
+ text (`str`):
164
+ The sequence to be encoded.
165
+ allowed_special (`Literal["all"]` or `set`):
166
+ The surface forms of the tokens to be encoded as special tokens in regular texts.
167
+ Default to "all".
168
+ disallowed_special (`Literal["all"]` or `Collection`):
169
+ The surface forms of the tokens that should not be in regular texts and trigger errors.
170
+ Default to an empty tuple.
171
+
172
+ kwargs (additional keyword arguments, *optional*):
173
+ Will be passed to the underlying model specific encode method.
174
+
175
+ Returns:
176
+ `List[bytes|str]`: The list of tokens.
177
+ """
178
+ tokens = []
179
+ text = unicodedata.normalize("NFC", text)
180
+
181
+ # this implementation takes a detour: text -> token id -> token surface forms
182
+ for t in self.tokenizer.encode(
183
+ text, allowed_special=allowed_special, disallowed_special=disallowed_special
184
+ ):
185
+ tokens.append(self.decoder[t])
186
+ return tokens
187
+
188
+ def convert_tokens_to_string(self, tokens: List[Union[bytes, str]]) -> str:
189
+ """
190
+ Converts a sequence of tokens in a single string.
191
+ """
192
+ text = ""
193
+ temp = b""
194
+ for t in tokens:
195
+ if isinstance(t, str):
196
+ if temp:
197
+ text += temp.decode("utf-8", errors=self.errors)
198
+ temp = b""
199
+ text += t
200
+ elif isinstance(t, bytes):
201
+ temp += t
202
+ else:
203
+ raise TypeError("token should only be of type types or str")
204
+ if temp:
205
+ text += temp.decode("utf-8", errors=self.errors)
206
+ return text
207
+
208
+ @property
209
+ def vocab_size(self):
210
+ return self.tokenizer.n_vocab
211
+
212
+ def _convert_id_to_token(self, index: int) -> Union[bytes, str]:
213
+ """Converts an id to a token, special tokens included"""
214
+ if index in self.decoder:
215
+ return self.decoder[index]
216
+ raise ValueError("unknown ids")
217
+
218
+ def _convert_token_to_id(self, token: Union[bytes, str]) -> int:
219
+ """Converts a token to an id using the vocab, special tokens included"""
220
+ if token in self.special_tokens:
221
+ return self.special_tokens[token]
222
+ if token in self.mergeable_ranks:
223
+ return self.mergeable_ranks[token]
224
+ raise ValueError("unknown token")
225
+
226
+ def _tokenize(self, text: str, **kwargs):
227
+ """
228
+ Converts a string in a sequence of tokens (string), using the tokenizer. Split in words for word-based
229
+ vocabulary or sub-words for sub-word-based vocabularies (BPE/SentencePieces/WordPieces).
230
+
231
+ Do NOT take care of added tokens.
232
+ """
233
+ raise NotImplementedError
234
+
235
+ def _decode(
236
+ self,
237
+ token_ids: Union[int, List[int]],
238
+ skip_special_tokens: bool = False,
239
+ errors: str = None,
240
+ **kwargs,
241
+ ) -> str:
242
+ if isinstance(token_ids, int):
243
+ token_ids = [token_ids]
244
+ if skip_special_tokens:
245
+ token_ids = [i for i in token_ids if i < self.eod_id]
246
+ return self.tokenizer.decode(token_ids, errors=errors or self.errors)
tokenizer_config.json ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "auto_map": {
3
+ "AutoTokenizer": [
4
+ "tokenization_qwen.QWenTokenizer",
5
+ null
6
+ ]
7
+ },
8
+ "clean_up_tokenization_spaces": true,
9
+ "model_max_length": 2048,
10
+ "padding_side": "right",
11
+ "tokenizer_class": "QWenTokenizer"
12
+ }
trainer_state.json ADDED
@@ -0,0 +1,4555 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 2.9977695167286247,
5
+ "eval_steps": 500,
6
+ "global_step": 756,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.0,
13
+ "learning_rate": 0.0,
14
+ "loss": 1.5239,
15
+ "step": 1
16
+ },
17
+ {
18
+ "epoch": 0.01,
19
+ "learning_rate": 3.333333333333334e-06,
20
+ "loss": 1.4583,
21
+ "step": 2
22
+ },
23
+ {
24
+ "epoch": 0.01,
25
+ "learning_rate": 5.283208335737189e-06,
26
+ "loss": 1.4205,
27
+ "step": 3
28
+ },
29
+ {
30
+ "epoch": 0.02,
31
+ "learning_rate": 6.666666666666668e-06,
32
+ "loss": 1.4054,
33
+ "step": 4
34
+ },
35
+ {
36
+ "epoch": 0.02,
37
+ "learning_rate": 7.739760316291209e-06,
38
+ "loss": 1.3651,
39
+ "step": 5
40
+ },
41
+ {
42
+ "epoch": 0.02,
43
+ "learning_rate": 8.616541669070522e-06,
44
+ "loss": 1.3724,
45
+ "step": 6
46
+ },
47
+ {
48
+ "epoch": 0.03,
49
+ "learning_rate": 9.357849740192014e-06,
50
+ "loss": 1.3489,
51
+ "step": 7
52
+ },
53
+ {
54
+ "epoch": 0.03,
55
+ "learning_rate": 1e-05,
56
+ "loss": 1.3221,
57
+ "step": 8
58
+ },
59
+ {
60
+ "epoch": 0.04,
61
+ "learning_rate": 1e-05,
62
+ "loss": 1.3005,
63
+ "step": 9
64
+ },
65
+ {
66
+ "epoch": 0.04,
67
+ "learning_rate": 1e-05,
68
+ "loss": 1.248,
69
+ "step": 10
70
+ },
71
+ {
72
+ "epoch": 0.04,
73
+ "learning_rate": 1e-05,
74
+ "loss": 1.2617,
75
+ "step": 11
76
+ },
77
+ {
78
+ "epoch": 0.05,
79
+ "learning_rate": 1e-05,
80
+ "loss": 1.2399,
81
+ "step": 12
82
+ },
83
+ {
84
+ "epoch": 0.05,
85
+ "learning_rate": 1e-05,
86
+ "loss": 1.2312,
87
+ "step": 13
88
+ },
89
+ {
90
+ "epoch": 0.06,
91
+ "learning_rate": 1e-05,
92
+ "loss": 1.2318,
93
+ "step": 14
94
+ },
95
+ {
96
+ "epoch": 0.06,
97
+ "learning_rate": 1e-05,
98
+ "loss": 1.2094,
99
+ "step": 15
100
+ },
101
+ {
102
+ "epoch": 0.06,
103
+ "learning_rate": 1e-05,
104
+ "loss": 1.2108,
105
+ "step": 16
106
+ },
107
+ {
108
+ "epoch": 0.07,
109
+ "learning_rate": 1e-05,
110
+ "loss": 1.218,
111
+ "step": 17
112
+ },
113
+ {
114
+ "epoch": 0.07,
115
+ "learning_rate": 1e-05,
116
+ "loss": 1.2062,
117
+ "step": 18
118
+ },
119
+ {
120
+ "epoch": 0.08,
121
+ "learning_rate": 1e-05,
122
+ "loss": 1.1602,
123
+ "step": 19
124
+ },
125
+ {
126
+ "epoch": 0.08,
127
+ "learning_rate": 1e-05,
128
+ "loss": 1.1571,
129
+ "step": 20
130
+ },
131
+ {
132
+ "epoch": 0.08,
133
+ "learning_rate": 1e-05,
134
+ "loss": 1.1534,
135
+ "step": 21
136
+ },
137
+ {
138
+ "epoch": 0.09,
139
+ "learning_rate": 1e-05,
140
+ "loss": 1.1374,
141
+ "step": 22
142
+ },
143
+ {
144
+ "epoch": 0.09,
145
+ "learning_rate": 1e-05,
146
+ "loss": 1.1084,
147
+ "step": 23
148
+ },
149
+ {
150
+ "epoch": 0.1,
151
+ "learning_rate": 1e-05,
152
+ "loss": 1.1097,
153
+ "step": 24
154
+ },
155
+ {
156
+ "epoch": 0.1,
157
+ "learning_rate": 1e-05,
158
+ "loss": 1.1086,
159
+ "step": 25
160
+ },
161
+ {
162
+ "epoch": 0.1,
163
+ "learning_rate": 1e-05,
164
+ "loss": 1.1136,
165
+ "step": 26
166
+ },
167
+ {
168
+ "epoch": 0.11,
169
+ "learning_rate": 1e-05,
170
+ "loss": 1.0891,
171
+ "step": 27
172
+ },
173
+ {
174
+ "epoch": 0.11,
175
+ "learning_rate": 1e-05,
176
+ "loss": 1.093,
177
+ "step": 28
178
+ },
179
+ {
180
+ "epoch": 0.11,
181
+ "learning_rate": 1e-05,
182
+ "loss": 1.0812,
183
+ "step": 29
184
+ },
185
+ {
186
+ "epoch": 0.12,
187
+ "learning_rate": 1e-05,
188
+ "loss": 1.0625,
189
+ "step": 30
190
+ },
191
+ {
192
+ "epoch": 0.12,
193
+ "learning_rate": 1e-05,
194
+ "loss": 1.0789,
195
+ "step": 31
196
+ },
197
+ {
198
+ "epoch": 0.13,
199
+ "learning_rate": 1e-05,
200
+ "loss": 1.0537,
201
+ "step": 32
202
+ },
203
+ {
204
+ "epoch": 0.13,
205
+ "learning_rate": 1e-05,
206
+ "loss": 1.0817,
207
+ "step": 33
208
+ },
209
+ {
210
+ "epoch": 0.13,
211
+ "learning_rate": 1e-05,
212
+ "loss": 1.0276,
213
+ "step": 34
214
+ },
215
+ {
216
+ "epoch": 0.14,
217
+ "learning_rate": 1e-05,
218
+ "loss": 1.019,
219
+ "step": 35
220
+ },
221
+ {
222
+ "epoch": 0.14,
223
+ "learning_rate": 1e-05,
224
+ "loss": 1.0213,
225
+ "step": 36
226
+ },
227
+ {
228
+ "epoch": 0.15,
229
+ "learning_rate": 1e-05,
230
+ "loss": 1.0408,
231
+ "step": 37
232
+ },
233
+ {
234
+ "epoch": 0.15,
235
+ "learning_rate": 1e-05,
236
+ "loss": 1.0321,
237
+ "step": 38
238
+ },
239
+ {
240
+ "epoch": 0.15,
241
+ "learning_rate": 1e-05,
242
+ "loss": 1.0372,
243
+ "step": 39
244
+ },
245
+ {
246
+ "epoch": 0.16,
247
+ "learning_rate": 1e-05,
248
+ "loss": 1.0137,
249
+ "step": 40
250
+ },
251
+ {
252
+ "epoch": 0.16,
253
+ "learning_rate": 1e-05,
254
+ "loss": 1.0127,
255
+ "step": 41
256
+ },
257
+ {
258
+ "epoch": 0.17,
259
+ "learning_rate": 1e-05,
260
+ "loss": 0.9953,
261
+ "step": 42
262
+ },
263
+ {
264
+ "epoch": 0.17,
265
+ "learning_rate": 1e-05,
266
+ "loss": 1.0146,
267
+ "step": 43
268
+ },
269
+ {
270
+ "epoch": 0.17,
271
+ "learning_rate": 1e-05,
272
+ "loss": 0.9715,
273
+ "step": 44
274
+ },
275
+ {
276
+ "epoch": 0.18,
277
+ "learning_rate": 1e-05,
278
+ "loss": 1.0089,
279
+ "step": 45
280
+ },
281
+ {
282
+ "epoch": 0.18,
283
+ "learning_rate": 1e-05,
284
+ "loss": 0.966,
285
+ "step": 46
286
+ },
287
+ {
288
+ "epoch": 0.19,
289
+ "learning_rate": 1e-05,
290
+ "loss": 0.9821,
291
+ "step": 47
292
+ },
293
+ {
294
+ "epoch": 0.19,
295
+ "learning_rate": 1e-05,
296
+ "loss": 0.9886,
297
+ "step": 48
298
+ },
299
+ {
300
+ "epoch": 0.19,
301
+ "learning_rate": 1e-05,
302
+ "loss": 0.9724,
303
+ "step": 49
304
+ },
305
+ {
306
+ "epoch": 0.2,
307
+ "learning_rate": 1e-05,
308
+ "loss": 0.9824,
309
+ "step": 50
310
+ },
311
+ {
312
+ "epoch": 0.2,
313
+ "learning_rate": 1e-05,
314
+ "loss": 0.9939,
315
+ "step": 51
316
+ },
317
+ {
318
+ "epoch": 0.21,
319
+ "learning_rate": 1e-05,
320
+ "loss": 0.9586,
321
+ "step": 52
322
+ },
323
+ {
324
+ "epoch": 0.21,
325
+ "learning_rate": 1e-05,
326
+ "loss": 0.9623,
327
+ "step": 53
328
+ },
329
+ {
330
+ "epoch": 0.21,
331
+ "learning_rate": 1e-05,
332
+ "loss": 0.9608,
333
+ "step": 54
334
+ },
335
+ {
336
+ "epoch": 0.22,
337
+ "learning_rate": 1e-05,
338
+ "loss": 0.9648,
339
+ "step": 55
340
+ },
341
+ {
342
+ "epoch": 0.22,
343
+ "learning_rate": 1e-05,
344
+ "loss": 0.9637,
345
+ "step": 56
346
+ },
347
+ {
348
+ "epoch": 0.23,
349
+ "learning_rate": 1e-05,
350
+ "loss": 0.9633,
351
+ "step": 57
352
+ },
353
+ {
354
+ "epoch": 0.23,
355
+ "learning_rate": 1e-05,
356
+ "loss": 0.9404,
357
+ "step": 58
358
+ },
359
+ {
360
+ "epoch": 0.23,
361
+ "learning_rate": 1e-05,
362
+ "loss": 0.9505,
363
+ "step": 59
364
+ },
365
+ {
366
+ "epoch": 0.24,
367
+ "learning_rate": 1e-05,
368
+ "loss": 0.952,
369
+ "step": 60
370
+ },
371
+ {
372
+ "epoch": 0.24,
373
+ "learning_rate": 1e-05,
374
+ "loss": 0.9445,
375
+ "step": 61
376
+ },
377
+ {
378
+ "epoch": 0.25,
379
+ "learning_rate": 1e-05,
380
+ "loss": 0.938,
381
+ "step": 62
382
+ },
383
+ {
384
+ "epoch": 0.25,
385
+ "learning_rate": 1e-05,
386
+ "loss": 0.9406,
387
+ "step": 63
388
+ },
389
+ {
390
+ "epoch": 0.25,
391
+ "learning_rate": 1e-05,
392
+ "loss": 0.9375,
393
+ "step": 64
394
+ },
395
+ {
396
+ "epoch": 0.26,
397
+ "learning_rate": 1e-05,
398
+ "loss": 0.9551,
399
+ "step": 65
400
+ },
401
+ {
402
+ "epoch": 0.26,
403
+ "learning_rate": 1e-05,
404
+ "loss": 0.9309,
405
+ "step": 66
406
+ },
407
+ {
408
+ "epoch": 0.27,
409
+ "learning_rate": 1e-05,
410
+ "loss": 0.9238,
411
+ "step": 67
412
+ },
413
+ {
414
+ "epoch": 0.27,
415
+ "learning_rate": 1e-05,
416
+ "loss": 0.9394,
417
+ "step": 68
418
+ },
419
+ {
420
+ "epoch": 0.27,
421
+ "learning_rate": 1e-05,
422
+ "loss": 0.9225,
423
+ "step": 69
424
+ },
425
+ {
426
+ "epoch": 0.28,
427
+ "learning_rate": 1e-05,
428
+ "loss": 0.9243,
429
+ "step": 70
430
+ },
431
+ {
432
+ "epoch": 0.28,
433
+ "learning_rate": 1e-05,
434
+ "loss": 0.9428,
435
+ "step": 71
436
+ },
437
+ {
438
+ "epoch": 0.29,
439
+ "learning_rate": 1e-05,
440
+ "loss": 0.9456,
441
+ "step": 72
442
+ },
443
+ {
444
+ "epoch": 0.29,
445
+ "learning_rate": 1e-05,
446
+ "loss": 0.9152,
447
+ "step": 73
448
+ },
449
+ {
450
+ "epoch": 0.29,
451
+ "learning_rate": 1e-05,
452
+ "loss": 0.9175,
453
+ "step": 74
454
+ },
455
+ {
456
+ "epoch": 0.3,
457
+ "learning_rate": 1e-05,
458
+ "loss": 0.9345,
459
+ "step": 75
460
+ },
461
+ {
462
+ "epoch": 0.3,
463
+ "learning_rate": 1e-05,
464
+ "loss": 0.9294,
465
+ "step": 76
466
+ },
467
+ {
468
+ "epoch": 0.31,
469
+ "learning_rate": 1e-05,
470
+ "loss": 0.9362,
471
+ "step": 77
472
+ },
473
+ {
474
+ "epoch": 0.31,
475
+ "learning_rate": 1e-05,
476
+ "loss": 0.926,
477
+ "step": 78
478
+ },
479
+ {
480
+ "epoch": 0.31,
481
+ "learning_rate": 1e-05,
482
+ "loss": 0.9268,
483
+ "step": 79
484
+ },
485
+ {
486
+ "epoch": 0.32,
487
+ "learning_rate": 1e-05,
488
+ "loss": 0.8947,
489
+ "step": 80
490
+ },
491
+ {
492
+ "epoch": 0.32,
493
+ "learning_rate": 1e-05,
494
+ "loss": 0.9137,
495
+ "step": 81
496
+ },
497
+ {
498
+ "epoch": 0.33,
499
+ "learning_rate": 1e-05,
500
+ "loss": 0.9229,
501
+ "step": 82
502
+ },
503
+ {
504
+ "epoch": 0.33,
505
+ "learning_rate": 1e-05,
506
+ "loss": 0.9264,
507
+ "step": 83
508
+ },
509
+ {
510
+ "epoch": 0.33,
511
+ "learning_rate": 1e-05,
512
+ "loss": 0.924,
513
+ "step": 84
514
+ },
515
+ {
516
+ "epoch": 0.34,
517
+ "learning_rate": 1e-05,
518
+ "loss": 0.9193,
519
+ "step": 85
520
+ },
521
+ {
522
+ "epoch": 0.34,
523
+ "learning_rate": 1e-05,
524
+ "loss": 0.8953,
525
+ "step": 86
526
+ },
527
+ {
528
+ "epoch": 0.34,
529
+ "learning_rate": 1e-05,
530
+ "loss": 0.9146,
531
+ "step": 87
532
+ },
533
+ {
534
+ "epoch": 0.35,
535
+ "learning_rate": 1e-05,
536
+ "loss": 0.8716,
537
+ "step": 88
538
+ },
539
+ {
540
+ "epoch": 0.35,
541
+ "learning_rate": 1e-05,
542
+ "loss": 0.9132,
543
+ "step": 89
544
+ },
545
+ {
546
+ "epoch": 0.36,
547
+ "learning_rate": 1e-05,
548
+ "loss": 0.9098,
549
+ "step": 90
550
+ },
551
+ {
552
+ "epoch": 0.36,
553
+ "learning_rate": 1e-05,
554
+ "loss": 0.9142,
555
+ "step": 91
556
+ },
557
+ {
558
+ "epoch": 0.36,
559
+ "learning_rate": 1e-05,
560
+ "loss": 0.8964,
561
+ "step": 92
562
+ },
563
+ {
564
+ "epoch": 0.37,
565
+ "learning_rate": 1e-05,
566
+ "loss": 0.9001,
567
+ "step": 93
568
+ },
569
+ {
570
+ "epoch": 0.37,
571
+ "learning_rate": 1e-05,
572
+ "loss": 0.9355,
573
+ "step": 94
574
+ },
575
+ {
576
+ "epoch": 0.38,
577
+ "learning_rate": 1e-05,
578
+ "loss": 0.9194,
579
+ "step": 95
580
+ },
581
+ {
582
+ "epoch": 0.38,
583
+ "learning_rate": 1e-05,
584
+ "loss": 0.8819,
585
+ "step": 96
586
+ },
587
+ {
588
+ "epoch": 0.38,
589
+ "learning_rate": 1e-05,
590
+ "loss": 0.8973,
591
+ "step": 97
592
+ },
593
+ {
594
+ "epoch": 0.39,
595
+ "learning_rate": 1e-05,
596
+ "loss": 0.8867,
597
+ "step": 98
598
+ },
599
+ {
600
+ "epoch": 0.39,
601
+ "learning_rate": 1e-05,
602
+ "loss": 0.9023,
603
+ "step": 99
604
+ },
605
+ {
606
+ "epoch": 0.4,
607
+ "learning_rate": 1e-05,
608
+ "loss": 0.8993,
609
+ "step": 100
610
+ },
611
+ {
612
+ "epoch": 0.4,
613
+ "learning_rate": 1e-05,
614
+ "loss": 0.8881,
615
+ "step": 101
616
+ },
617
+ {
618
+ "epoch": 0.4,
619
+ "learning_rate": 1e-05,
620
+ "loss": 0.8819,
621
+ "step": 102
622
+ },
623
+ {
624
+ "epoch": 0.41,
625
+ "learning_rate": 1e-05,
626
+ "loss": 0.8875,
627
+ "step": 103
628
+ },
629
+ {
630
+ "epoch": 0.41,
631
+ "learning_rate": 1e-05,
632
+ "loss": 0.8871,
633
+ "step": 104
634
+ },
635
+ {
636
+ "epoch": 0.42,
637
+ "learning_rate": 1e-05,
638
+ "loss": 0.8898,
639
+ "step": 105
640
+ },
641
+ {
642
+ "epoch": 0.42,
643
+ "learning_rate": 1e-05,
644
+ "loss": 0.8707,
645
+ "step": 106
646
+ },
647
+ {
648
+ "epoch": 0.42,
649
+ "learning_rate": 1e-05,
650
+ "loss": 0.8959,
651
+ "step": 107
652
+ },
653
+ {
654
+ "epoch": 0.43,
655
+ "learning_rate": 1e-05,
656
+ "loss": 0.8812,
657
+ "step": 108
658
+ },
659
+ {
660
+ "epoch": 0.43,
661
+ "learning_rate": 1e-05,
662
+ "loss": 0.8955,
663
+ "step": 109
664
+ },
665
+ {
666
+ "epoch": 0.44,
667
+ "learning_rate": 1e-05,
668
+ "loss": 0.8712,
669
+ "step": 110
670
+ },
671
+ {
672
+ "epoch": 0.44,
673
+ "learning_rate": 1e-05,
674
+ "loss": 0.9046,
675
+ "step": 111
676
+ },
677
+ {
678
+ "epoch": 0.44,
679
+ "learning_rate": 1e-05,
680
+ "loss": 0.8832,
681
+ "step": 112
682
+ },
683
+ {
684
+ "epoch": 0.45,
685
+ "learning_rate": 1e-05,
686
+ "loss": 0.9039,
687
+ "step": 113
688
+ },
689
+ {
690
+ "epoch": 0.45,
691
+ "learning_rate": 1e-05,
692
+ "loss": 0.8783,
693
+ "step": 114
694
+ },
695
+ {
696
+ "epoch": 0.46,
697
+ "learning_rate": 1e-05,
698
+ "loss": 0.8819,
699
+ "step": 115
700
+ },
701
+ {
702
+ "epoch": 0.46,
703
+ "learning_rate": 1e-05,
704
+ "loss": 0.873,
705
+ "step": 116
706
+ },
707
+ {
708
+ "epoch": 0.46,
709
+ "learning_rate": 1e-05,
710
+ "loss": 0.8628,
711
+ "step": 117
712
+ },
713
+ {
714
+ "epoch": 0.47,
715
+ "learning_rate": 1e-05,
716
+ "loss": 0.8649,
717
+ "step": 118
718
+ },
719
+ {
720
+ "epoch": 0.47,
721
+ "learning_rate": 1e-05,
722
+ "loss": 0.89,
723
+ "step": 119
724
+ },
725
+ {
726
+ "epoch": 0.48,
727
+ "learning_rate": 1e-05,
728
+ "loss": 0.8864,
729
+ "step": 120
730
+ },
731
+ {
732
+ "epoch": 0.48,
733
+ "learning_rate": 1e-05,
734
+ "loss": 0.8653,
735
+ "step": 121
736
+ },
737
+ {
738
+ "epoch": 0.48,
739
+ "learning_rate": 1e-05,
740
+ "loss": 0.865,
741
+ "step": 122
742
+ },
743
+ {
744
+ "epoch": 0.49,
745
+ "learning_rate": 1e-05,
746
+ "loss": 0.8448,
747
+ "step": 123
748
+ },
749
+ {
750
+ "epoch": 0.49,
751
+ "learning_rate": 1e-05,
752
+ "loss": 0.8363,
753
+ "step": 124
754
+ },
755
+ {
756
+ "epoch": 0.5,
757
+ "learning_rate": 1e-05,
758
+ "loss": 0.8586,
759
+ "step": 125
760
+ },
761
+ {
762
+ "epoch": 0.5,
763
+ "learning_rate": 1e-05,
764
+ "loss": 0.8603,
765
+ "step": 126
766
+ },
767
+ {
768
+ "epoch": 0.5,
769
+ "learning_rate": 1e-05,
770
+ "loss": 0.8784,
771
+ "step": 127
772
+ },
773
+ {
774
+ "epoch": 0.51,
775
+ "learning_rate": 1e-05,
776
+ "loss": 0.8822,
777
+ "step": 128
778
+ },
779
+ {
780
+ "epoch": 0.51,
781
+ "learning_rate": 1e-05,
782
+ "loss": 0.8592,
783
+ "step": 129
784
+ },
785
+ {
786
+ "epoch": 0.52,
787
+ "learning_rate": 1e-05,
788
+ "loss": 0.8629,
789
+ "step": 130
790
+ },
791
+ {
792
+ "epoch": 0.52,
793
+ "learning_rate": 1e-05,
794
+ "loss": 0.8691,
795
+ "step": 131
796
+ },
797
+ {
798
+ "epoch": 0.52,
799
+ "learning_rate": 1e-05,
800
+ "loss": 0.8684,
801
+ "step": 132
802
+ },
803
+ {
804
+ "epoch": 0.53,
805
+ "learning_rate": 1e-05,
806
+ "loss": 0.8767,
807
+ "step": 133
808
+ },
809
+ {
810
+ "epoch": 0.53,
811
+ "learning_rate": 1e-05,
812
+ "loss": 0.8809,
813
+ "step": 134
814
+ },
815
+ {
816
+ "epoch": 0.54,
817
+ "learning_rate": 1e-05,
818
+ "loss": 0.8504,
819
+ "step": 135
820
+ },
821
+ {
822
+ "epoch": 0.54,
823
+ "learning_rate": 1e-05,
824
+ "loss": 0.8705,
825
+ "step": 136
826
+ },
827
+ {
828
+ "epoch": 0.54,
829
+ "learning_rate": 1e-05,
830
+ "loss": 0.8721,
831
+ "step": 137
832
+ },
833
+ {
834
+ "epoch": 0.55,
835
+ "learning_rate": 1e-05,
836
+ "loss": 0.8588,
837
+ "step": 138
838
+ },
839
+ {
840
+ "epoch": 0.55,
841
+ "learning_rate": 1e-05,
842
+ "loss": 0.8842,
843
+ "step": 139
844
+ },
845
+ {
846
+ "epoch": 0.56,
847
+ "learning_rate": 1e-05,
848
+ "loss": 0.8553,
849
+ "step": 140
850
+ },
851
+ {
852
+ "epoch": 0.56,
853
+ "learning_rate": 1e-05,
854
+ "loss": 0.8546,
855
+ "step": 141
856
+ },
857
+ {
858
+ "epoch": 0.56,
859
+ "learning_rate": 1e-05,
860
+ "loss": 0.8588,
861
+ "step": 142
862
+ },
863
+ {
864
+ "epoch": 0.57,
865
+ "learning_rate": 1e-05,
866
+ "loss": 0.8672,
867
+ "step": 143
868
+ },
869
+ {
870
+ "epoch": 0.57,
871
+ "learning_rate": 1e-05,
872
+ "loss": 0.8745,
873
+ "step": 144
874
+ },
875
+ {
876
+ "epoch": 0.57,
877
+ "learning_rate": 1e-05,
878
+ "loss": 0.8842,
879
+ "step": 145
880
+ },
881
+ {
882
+ "epoch": 0.58,
883
+ "learning_rate": 1e-05,
884
+ "loss": 0.8594,
885
+ "step": 146
886
+ },
887
+ {
888
+ "epoch": 0.58,
889
+ "learning_rate": 1e-05,
890
+ "loss": 0.8463,
891
+ "step": 147
892
+ },
893
+ {
894
+ "epoch": 0.59,
895
+ "learning_rate": 1e-05,
896
+ "loss": 0.8399,
897
+ "step": 148
898
+ },
899
+ {
900
+ "epoch": 0.59,
901
+ "learning_rate": 1e-05,
902
+ "loss": 0.8521,
903
+ "step": 149
904
+ },
905
+ {
906
+ "epoch": 0.59,
907
+ "learning_rate": 1e-05,
908
+ "loss": 0.8492,
909
+ "step": 150
910
+ },
911
+ {
912
+ "epoch": 0.6,
913
+ "learning_rate": 1e-05,
914
+ "loss": 0.8535,
915
+ "step": 151
916
+ },
917
+ {
918
+ "epoch": 0.6,
919
+ "learning_rate": 1e-05,
920
+ "loss": 0.8552,
921
+ "step": 152
922
+ },
923
+ {
924
+ "epoch": 0.61,
925
+ "learning_rate": 1e-05,
926
+ "loss": 0.8602,
927
+ "step": 153
928
+ },
929
+ {
930
+ "epoch": 0.61,
931
+ "learning_rate": 1e-05,
932
+ "loss": 0.8559,
933
+ "step": 154
934
+ },
935
+ {
936
+ "epoch": 0.61,
937
+ "learning_rate": 1e-05,
938
+ "loss": 0.8498,
939
+ "step": 155
940
+ },
941
+ {
942
+ "epoch": 0.62,
943
+ "learning_rate": 1e-05,
944
+ "loss": 0.832,
945
+ "step": 156
946
+ },
947
+ {
948
+ "epoch": 0.62,
949
+ "learning_rate": 1e-05,
950
+ "loss": 0.8518,
951
+ "step": 157
952
+ },
953
+ {
954
+ "epoch": 0.63,
955
+ "learning_rate": 1e-05,
956
+ "loss": 0.8296,
957
+ "step": 158
958
+ },
959
+ {
960
+ "epoch": 0.63,
961
+ "learning_rate": 1e-05,
962
+ "loss": 0.8346,
963
+ "step": 159
964
+ },
965
+ {
966
+ "epoch": 0.63,
967
+ "learning_rate": 1e-05,
968
+ "loss": 0.8491,
969
+ "step": 160
970
+ },
971
+ {
972
+ "epoch": 0.64,
973
+ "learning_rate": 1e-05,
974
+ "loss": 0.8432,
975
+ "step": 161
976
+ },
977
+ {
978
+ "epoch": 0.64,
979
+ "learning_rate": 1e-05,
980
+ "loss": 0.842,
981
+ "step": 162
982
+ },
983
+ {
984
+ "epoch": 0.65,
985
+ "learning_rate": 1e-05,
986
+ "loss": 0.8618,
987
+ "step": 163
988
+ },
989
+ {
990
+ "epoch": 0.65,
991
+ "learning_rate": 1e-05,
992
+ "loss": 0.8284,
993
+ "step": 164
994
+ },
995
+ {
996
+ "epoch": 0.65,
997
+ "learning_rate": 1e-05,
998
+ "loss": 0.8648,
999
+ "step": 165
1000
+ },
1001
+ {
1002
+ "epoch": 0.66,
1003
+ "learning_rate": 1e-05,
1004
+ "loss": 0.8601,
1005
+ "step": 166
1006
+ },
1007
+ {
1008
+ "epoch": 0.66,
1009
+ "learning_rate": 1e-05,
1010
+ "loss": 0.8533,
1011
+ "step": 167
1012
+ },
1013
+ {
1014
+ "epoch": 0.67,
1015
+ "learning_rate": 1e-05,
1016
+ "loss": 0.8414,
1017
+ "step": 168
1018
+ },
1019
+ {
1020
+ "epoch": 0.67,
1021
+ "learning_rate": 1e-05,
1022
+ "loss": 0.8386,
1023
+ "step": 169
1024
+ },
1025
+ {
1026
+ "epoch": 0.67,
1027
+ "learning_rate": 1e-05,
1028
+ "loss": 0.8454,
1029
+ "step": 170
1030
+ },
1031
+ {
1032
+ "epoch": 0.68,
1033
+ "learning_rate": 1e-05,
1034
+ "loss": 0.8384,
1035
+ "step": 171
1036
+ },
1037
+ {
1038
+ "epoch": 0.68,
1039
+ "learning_rate": 1e-05,
1040
+ "loss": 0.8452,
1041
+ "step": 172
1042
+ },
1043
+ {
1044
+ "epoch": 0.69,
1045
+ "learning_rate": 1e-05,
1046
+ "loss": 0.8445,
1047
+ "step": 173
1048
+ },
1049
+ {
1050
+ "epoch": 0.69,
1051
+ "learning_rate": 1e-05,
1052
+ "loss": 0.8499,
1053
+ "step": 174
1054
+ },
1055
+ {
1056
+ "epoch": 0.69,
1057
+ "learning_rate": 1e-05,
1058
+ "loss": 0.8417,
1059
+ "step": 175
1060
+ },
1061
+ {
1062
+ "epoch": 0.7,
1063
+ "learning_rate": 1e-05,
1064
+ "loss": 0.8419,
1065
+ "step": 176
1066
+ },
1067
+ {
1068
+ "epoch": 0.7,
1069
+ "learning_rate": 1e-05,
1070
+ "loss": 0.834,
1071
+ "step": 177
1072
+ },
1073
+ {
1074
+ "epoch": 0.71,
1075
+ "learning_rate": 1e-05,
1076
+ "loss": 0.8281,
1077
+ "step": 178
1078
+ },
1079
+ {
1080
+ "epoch": 0.71,
1081
+ "learning_rate": 1e-05,
1082
+ "loss": 0.8398,
1083
+ "step": 179
1084
+ },
1085
+ {
1086
+ "epoch": 0.71,
1087
+ "learning_rate": 1e-05,
1088
+ "loss": 0.87,
1089
+ "step": 180
1090
+ },
1091
+ {
1092
+ "epoch": 0.72,
1093
+ "learning_rate": 1e-05,
1094
+ "loss": 0.8256,
1095
+ "step": 181
1096
+ },
1097
+ {
1098
+ "epoch": 0.72,
1099
+ "learning_rate": 1e-05,
1100
+ "loss": 0.8087,
1101
+ "step": 182
1102
+ },
1103
+ {
1104
+ "epoch": 0.73,
1105
+ "learning_rate": 1e-05,
1106
+ "loss": 0.8268,
1107
+ "step": 183
1108
+ },
1109
+ {
1110
+ "epoch": 0.73,
1111
+ "learning_rate": 1e-05,
1112
+ "loss": 0.8269,
1113
+ "step": 184
1114
+ },
1115
+ {
1116
+ "epoch": 0.73,
1117
+ "learning_rate": 1e-05,
1118
+ "loss": 0.8021,
1119
+ "step": 185
1120
+ },
1121
+ {
1122
+ "epoch": 0.74,
1123
+ "learning_rate": 1e-05,
1124
+ "loss": 0.8157,
1125
+ "step": 186
1126
+ },
1127
+ {
1128
+ "epoch": 0.74,
1129
+ "learning_rate": 1e-05,
1130
+ "loss": 0.8319,
1131
+ "step": 187
1132
+ },
1133
+ {
1134
+ "epoch": 0.75,
1135
+ "learning_rate": 1e-05,
1136
+ "loss": 0.7985,
1137
+ "step": 188
1138
+ },
1139
+ {
1140
+ "epoch": 0.75,
1141
+ "learning_rate": 1e-05,
1142
+ "loss": 0.8481,
1143
+ "step": 189
1144
+ },
1145
+ {
1146
+ "epoch": 0.75,
1147
+ "learning_rate": 1e-05,
1148
+ "loss": 0.832,
1149
+ "step": 190
1150
+ },
1151
+ {
1152
+ "epoch": 0.76,
1153
+ "learning_rate": 1e-05,
1154
+ "loss": 0.8245,
1155
+ "step": 191
1156
+ },
1157
+ {
1158
+ "epoch": 0.76,
1159
+ "learning_rate": 1e-05,
1160
+ "loss": 0.8116,
1161
+ "step": 192
1162
+ },
1163
+ {
1164
+ "epoch": 0.77,
1165
+ "learning_rate": 1e-05,
1166
+ "loss": 0.8171,
1167
+ "step": 193
1168
+ },
1169
+ {
1170
+ "epoch": 0.77,
1171
+ "learning_rate": 1e-05,
1172
+ "loss": 0.8091,
1173
+ "step": 194
1174
+ },
1175
+ {
1176
+ "epoch": 0.77,
1177
+ "learning_rate": 1e-05,
1178
+ "loss": 0.8032,
1179
+ "step": 195
1180
+ },
1181
+ {
1182
+ "epoch": 0.78,
1183
+ "learning_rate": 1e-05,
1184
+ "loss": 0.8082,
1185
+ "step": 196
1186
+ },
1187
+ {
1188
+ "epoch": 0.78,
1189
+ "learning_rate": 1e-05,
1190
+ "loss": 0.8279,
1191
+ "step": 197
1192
+ },
1193
+ {
1194
+ "epoch": 0.79,
1195
+ "learning_rate": 1e-05,
1196
+ "loss": 0.8574,
1197
+ "step": 198
1198
+ },
1199
+ {
1200
+ "epoch": 0.79,
1201
+ "learning_rate": 1e-05,
1202
+ "loss": 0.8457,
1203
+ "step": 199
1204
+ },
1205
+ {
1206
+ "epoch": 0.79,
1207
+ "learning_rate": 1e-05,
1208
+ "loss": 0.8208,
1209
+ "step": 200
1210
+ },
1211
+ {
1212
+ "epoch": 0.8,
1213
+ "learning_rate": 1e-05,
1214
+ "loss": 0.8238,
1215
+ "step": 201
1216
+ },
1217
+ {
1218
+ "epoch": 0.8,
1219
+ "learning_rate": 1e-05,
1220
+ "loss": 0.8417,
1221
+ "step": 202
1222
+ },
1223
+ {
1224
+ "epoch": 0.8,
1225
+ "learning_rate": 1e-05,
1226
+ "loss": 0.809,
1227
+ "step": 203
1228
+ },
1229
+ {
1230
+ "epoch": 0.81,
1231
+ "learning_rate": 1e-05,
1232
+ "loss": 0.825,
1233
+ "step": 204
1234
+ },
1235
+ {
1236
+ "epoch": 0.81,
1237
+ "learning_rate": 1e-05,
1238
+ "loss": 0.8171,
1239
+ "step": 205
1240
+ },
1241
+ {
1242
+ "epoch": 0.82,
1243
+ "learning_rate": 1e-05,
1244
+ "loss": 0.8174,
1245
+ "step": 206
1246
+ },
1247
+ {
1248
+ "epoch": 0.82,
1249
+ "learning_rate": 1e-05,
1250
+ "loss": 0.8167,
1251
+ "step": 207
1252
+ },
1253
+ {
1254
+ "epoch": 0.82,
1255
+ "learning_rate": 1e-05,
1256
+ "loss": 0.8316,
1257
+ "step": 208
1258
+ },
1259
+ {
1260
+ "epoch": 0.83,
1261
+ "learning_rate": 1e-05,
1262
+ "loss": 0.8039,
1263
+ "step": 209
1264
+ },
1265
+ {
1266
+ "epoch": 0.83,
1267
+ "learning_rate": 1e-05,
1268
+ "loss": 0.8137,
1269
+ "step": 210
1270
+ },
1271
+ {
1272
+ "epoch": 0.84,
1273
+ "learning_rate": 1e-05,
1274
+ "loss": 0.8183,
1275
+ "step": 211
1276
+ },
1277
+ {
1278
+ "epoch": 0.84,
1279
+ "learning_rate": 1e-05,
1280
+ "loss": 0.8278,
1281
+ "step": 212
1282
+ },
1283
+ {
1284
+ "epoch": 0.84,
1285
+ "learning_rate": 1e-05,
1286
+ "loss": 0.8055,
1287
+ "step": 213
1288
+ },
1289
+ {
1290
+ "epoch": 0.85,
1291
+ "learning_rate": 1e-05,
1292
+ "loss": 0.7998,
1293
+ "step": 214
1294
+ },
1295
+ {
1296
+ "epoch": 0.85,
1297
+ "learning_rate": 1e-05,
1298
+ "loss": 0.8213,
1299
+ "step": 215
1300
+ },
1301
+ {
1302
+ "epoch": 0.86,
1303
+ "learning_rate": 1e-05,
1304
+ "loss": 0.796,
1305
+ "step": 216
1306
+ },
1307
+ {
1308
+ "epoch": 0.86,
1309
+ "learning_rate": 1e-05,
1310
+ "loss": 0.8069,
1311
+ "step": 217
1312
+ },
1313
+ {
1314
+ "epoch": 0.86,
1315
+ "learning_rate": 1e-05,
1316
+ "loss": 0.8021,
1317
+ "step": 218
1318
+ },
1319
+ {
1320
+ "epoch": 0.87,
1321
+ "learning_rate": 1e-05,
1322
+ "loss": 0.8076,
1323
+ "step": 219
1324
+ },
1325
+ {
1326
+ "epoch": 0.87,
1327
+ "learning_rate": 1e-05,
1328
+ "loss": 0.8225,
1329
+ "step": 220
1330
+ },
1331
+ {
1332
+ "epoch": 0.88,
1333
+ "learning_rate": 1e-05,
1334
+ "loss": 0.8025,
1335
+ "step": 221
1336
+ },
1337
+ {
1338
+ "epoch": 0.88,
1339
+ "learning_rate": 1e-05,
1340
+ "loss": 0.8132,
1341
+ "step": 222
1342
+ },
1343
+ {
1344
+ "epoch": 0.88,
1345
+ "learning_rate": 1e-05,
1346
+ "loss": 0.8062,
1347
+ "step": 223
1348
+ },
1349
+ {
1350
+ "epoch": 0.89,
1351
+ "learning_rate": 1e-05,
1352
+ "loss": 0.8243,
1353
+ "step": 224
1354
+ },
1355
+ {
1356
+ "epoch": 0.89,
1357
+ "learning_rate": 1e-05,
1358
+ "loss": 0.8204,
1359
+ "step": 225
1360
+ },
1361
+ {
1362
+ "epoch": 0.9,
1363
+ "learning_rate": 1e-05,
1364
+ "loss": 0.8211,
1365
+ "step": 226
1366
+ },
1367
+ {
1368
+ "epoch": 0.9,
1369
+ "learning_rate": 1e-05,
1370
+ "loss": 0.8167,
1371
+ "step": 227
1372
+ },
1373
+ {
1374
+ "epoch": 0.9,
1375
+ "learning_rate": 1e-05,
1376
+ "loss": 0.7995,
1377
+ "step": 228
1378
+ },
1379
+ {
1380
+ "epoch": 0.91,
1381
+ "learning_rate": 1e-05,
1382
+ "loss": 0.7867,
1383
+ "step": 229
1384
+ },
1385
+ {
1386
+ "epoch": 0.91,
1387
+ "learning_rate": 1e-05,
1388
+ "loss": 0.7931,
1389
+ "step": 230
1390
+ },
1391
+ {
1392
+ "epoch": 0.92,
1393
+ "learning_rate": 1e-05,
1394
+ "loss": 0.7975,
1395
+ "step": 231
1396
+ },
1397
+ {
1398
+ "epoch": 0.92,
1399
+ "learning_rate": 1e-05,
1400
+ "loss": 0.8271,
1401
+ "step": 232
1402
+ },
1403
+ {
1404
+ "epoch": 0.92,
1405
+ "learning_rate": 1e-05,
1406
+ "loss": 0.7776,
1407
+ "step": 233
1408
+ },
1409
+ {
1410
+ "epoch": 0.93,
1411
+ "learning_rate": 1e-05,
1412
+ "loss": 0.7944,
1413
+ "step": 234
1414
+ },
1415
+ {
1416
+ "epoch": 0.93,
1417
+ "learning_rate": 1e-05,
1418
+ "loss": 0.8222,
1419
+ "step": 235
1420
+ },
1421
+ {
1422
+ "epoch": 0.94,
1423
+ "learning_rate": 1e-05,
1424
+ "loss": 0.797,
1425
+ "step": 236
1426
+ },
1427
+ {
1428
+ "epoch": 0.94,
1429
+ "learning_rate": 1e-05,
1430
+ "loss": 0.8176,
1431
+ "step": 237
1432
+ },
1433
+ {
1434
+ "epoch": 0.94,
1435
+ "learning_rate": 1e-05,
1436
+ "loss": 0.7953,
1437
+ "step": 238
1438
+ },
1439
+ {
1440
+ "epoch": 0.95,
1441
+ "learning_rate": 1e-05,
1442
+ "loss": 0.8076,
1443
+ "step": 239
1444
+ },
1445
+ {
1446
+ "epoch": 0.95,
1447
+ "learning_rate": 1e-05,
1448
+ "loss": 0.7911,
1449
+ "step": 240
1450
+ },
1451
+ {
1452
+ "epoch": 0.96,
1453
+ "learning_rate": 1e-05,
1454
+ "loss": 0.8091,
1455
+ "step": 241
1456
+ },
1457
+ {
1458
+ "epoch": 0.96,
1459
+ "learning_rate": 1e-05,
1460
+ "loss": 0.7754,
1461
+ "step": 242
1462
+ },
1463
+ {
1464
+ "epoch": 0.96,
1465
+ "learning_rate": 1e-05,
1466
+ "loss": 0.7923,
1467
+ "step": 243
1468
+ },
1469
+ {
1470
+ "epoch": 0.97,
1471
+ "learning_rate": 1e-05,
1472
+ "loss": 0.8013,
1473
+ "step": 244
1474
+ },
1475
+ {
1476
+ "epoch": 0.97,
1477
+ "learning_rate": 1e-05,
1478
+ "loss": 0.7873,
1479
+ "step": 245
1480
+ },
1481
+ {
1482
+ "epoch": 0.98,
1483
+ "learning_rate": 1e-05,
1484
+ "loss": 0.7904,
1485
+ "step": 246
1486
+ },
1487
+ {
1488
+ "epoch": 0.98,
1489
+ "learning_rate": 1e-05,
1490
+ "loss": 0.8298,
1491
+ "step": 247
1492
+ },
1493
+ {
1494
+ "epoch": 0.98,
1495
+ "learning_rate": 1e-05,
1496
+ "loss": 0.8002,
1497
+ "step": 248
1498
+ },
1499
+ {
1500
+ "epoch": 0.99,
1501
+ "learning_rate": 1e-05,
1502
+ "loss": 0.7924,
1503
+ "step": 249
1504
+ },
1505
+ {
1506
+ "epoch": 0.99,
1507
+ "learning_rate": 1e-05,
1508
+ "loss": 0.7824,
1509
+ "step": 250
1510
+ },
1511
+ {
1512
+ "epoch": 1.0,
1513
+ "learning_rate": 1e-05,
1514
+ "loss": 0.7918,
1515
+ "step": 251
1516
+ },
1517
+ {
1518
+ "epoch": 1.0,
1519
+ "learning_rate": 1e-05,
1520
+ "loss": 0.8046,
1521
+ "step": 252
1522
+ },
1523
+ {
1524
+ "epoch": 1.0,
1525
+ "learning_rate": 1e-05,
1526
+ "loss": 0.7592,
1527
+ "step": 253
1528
+ },
1529
+ {
1530
+ "epoch": 1.01,
1531
+ "learning_rate": 1e-05,
1532
+ "loss": 0.7253,
1533
+ "step": 254
1534
+ },
1535
+ {
1536
+ "epoch": 1.01,
1537
+ "learning_rate": 1e-05,
1538
+ "loss": 0.7204,
1539
+ "step": 255
1540
+ },
1541
+ {
1542
+ "epoch": 1.02,
1543
+ "learning_rate": 1e-05,
1544
+ "loss": 0.7307,
1545
+ "step": 256
1546
+ },
1547
+ {
1548
+ "epoch": 1.02,
1549
+ "learning_rate": 1e-05,
1550
+ "loss": 0.7084,
1551
+ "step": 257
1552
+ },
1553
+ {
1554
+ "epoch": 1.02,
1555
+ "learning_rate": 1e-05,
1556
+ "loss": 0.7474,
1557
+ "step": 258
1558
+ },
1559
+ {
1560
+ "epoch": 1.03,
1561
+ "learning_rate": 1e-05,
1562
+ "loss": 0.7248,
1563
+ "step": 259
1564
+ },
1565
+ {
1566
+ "epoch": 1.03,
1567
+ "learning_rate": 1e-05,
1568
+ "loss": 0.7243,
1569
+ "step": 260
1570
+ },
1571
+ {
1572
+ "epoch": 1.03,
1573
+ "learning_rate": 1e-05,
1574
+ "loss": 0.7369,
1575
+ "step": 261
1576
+ },
1577
+ {
1578
+ "epoch": 1.04,
1579
+ "learning_rate": 1e-05,
1580
+ "loss": 0.7254,
1581
+ "step": 262
1582
+ },
1583
+ {
1584
+ "epoch": 1.04,
1585
+ "learning_rate": 1e-05,
1586
+ "loss": 0.7264,
1587
+ "step": 263
1588
+ },
1589
+ {
1590
+ "epoch": 1.05,
1591
+ "learning_rate": 1e-05,
1592
+ "loss": 0.6911,
1593
+ "step": 264
1594
+ },
1595
+ {
1596
+ "epoch": 1.05,
1597
+ "learning_rate": 1e-05,
1598
+ "loss": 0.7191,
1599
+ "step": 265
1600
+ },
1601
+ {
1602
+ "epoch": 1.05,
1603
+ "learning_rate": 1e-05,
1604
+ "loss": 0.717,
1605
+ "step": 266
1606
+ },
1607
+ {
1608
+ "epoch": 1.06,
1609
+ "learning_rate": 1e-05,
1610
+ "loss": 0.737,
1611
+ "step": 267
1612
+ },
1613
+ {
1614
+ "epoch": 1.06,
1615
+ "learning_rate": 1e-05,
1616
+ "loss": 0.7278,
1617
+ "step": 268
1618
+ },
1619
+ {
1620
+ "epoch": 1.07,
1621
+ "learning_rate": 1e-05,
1622
+ "loss": 0.7381,
1623
+ "step": 269
1624
+ },
1625
+ {
1626
+ "epoch": 1.07,
1627
+ "learning_rate": 1e-05,
1628
+ "loss": 0.709,
1629
+ "step": 270
1630
+ },
1631
+ {
1632
+ "epoch": 1.07,
1633
+ "learning_rate": 1e-05,
1634
+ "loss": 0.7021,
1635
+ "step": 271
1636
+ },
1637
+ {
1638
+ "epoch": 1.08,
1639
+ "learning_rate": 1e-05,
1640
+ "loss": 0.7472,
1641
+ "step": 272
1642
+ },
1643
+ {
1644
+ "epoch": 1.08,
1645
+ "learning_rate": 1e-05,
1646
+ "loss": 0.7367,
1647
+ "step": 273
1648
+ },
1649
+ {
1650
+ "epoch": 1.09,
1651
+ "learning_rate": 1e-05,
1652
+ "loss": 0.7065,
1653
+ "step": 274
1654
+ },
1655
+ {
1656
+ "epoch": 1.09,
1657
+ "learning_rate": 1e-05,
1658
+ "loss": 0.7272,
1659
+ "step": 275
1660
+ },
1661
+ {
1662
+ "epoch": 1.09,
1663
+ "learning_rate": 1e-05,
1664
+ "loss": 0.7133,
1665
+ "step": 276
1666
+ },
1667
+ {
1668
+ "epoch": 1.1,
1669
+ "learning_rate": 1e-05,
1670
+ "loss": 0.7388,
1671
+ "step": 277
1672
+ },
1673
+ {
1674
+ "epoch": 1.1,
1675
+ "learning_rate": 1e-05,
1676
+ "loss": 0.6945,
1677
+ "step": 278
1678
+ },
1679
+ {
1680
+ "epoch": 1.11,
1681
+ "learning_rate": 1e-05,
1682
+ "loss": 0.718,
1683
+ "step": 279
1684
+ },
1685
+ {
1686
+ "epoch": 1.11,
1687
+ "learning_rate": 1e-05,
1688
+ "loss": 0.7049,
1689
+ "step": 280
1690
+ },
1691
+ {
1692
+ "epoch": 1.11,
1693
+ "learning_rate": 1e-05,
1694
+ "loss": 0.7176,
1695
+ "step": 281
1696
+ },
1697
+ {
1698
+ "epoch": 1.12,
1699
+ "learning_rate": 1e-05,
1700
+ "loss": 0.722,
1701
+ "step": 282
1702
+ },
1703
+ {
1704
+ "epoch": 1.12,
1705
+ "learning_rate": 1e-05,
1706
+ "loss": 0.7099,
1707
+ "step": 283
1708
+ },
1709
+ {
1710
+ "epoch": 1.13,
1711
+ "learning_rate": 1e-05,
1712
+ "loss": 0.6972,
1713
+ "step": 284
1714
+ },
1715
+ {
1716
+ "epoch": 1.13,
1717
+ "learning_rate": 1e-05,
1718
+ "loss": 0.7266,
1719
+ "step": 285
1720
+ },
1721
+ {
1722
+ "epoch": 1.13,
1723
+ "learning_rate": 1e-05,
1724
+ "loss": 0.7021,
1725
+ "step": 286
1726
+ },
1727
+ {
1728
+ "epoch": 1.14,
1729
+ "learning_rate": 1e-05,
1730
+ "loss": 0.7085,
1731
+ "step": 287
1732
+ },
1733
+ {
1734
+ "epoch": 1.14,
1735
+ "learning_rate": 1e-05,
1736
+ "loss": 0.7117,
1737
+ "step": 288
1738
+ },
1739
+ {
1740
+ "epoch": 1.15,
1741
+ "learning_rate": 1e-05,
1742
+ "loss": 0.7316,
1743
+ "step": 289
1744
+ },
1745
+ {
1746
+ "epoch": 1.15,
1747
+ "learning_rate": 1e-05,
1748
+ "loss": 0.705,
1749
+ "step": 290
1750
+ },
1751
+ {
1752
+ "epoch": 1.15,
1753
+ "learning_rate": 1e-05,
1754
+ "loss": 0.68,
1755
+ "step": 291
1756
+ },
1757
+ {
1758
+ "epoch": 1.16,
1759
+ "learning_rate": 1e-05,
1760
+ "loss": 0.711,
1761
+ "step": 292
1762
+ },
1763
+ {
1764
+ "epoch": 1.16,
1765
+ "learning_rate": 1e-05,
1766
+ "loss": 0.7088,
1767
+ "step": 293
1768
+ },
1769
+ {
1770
+ "epoch": 1.17,
1771
+ "learning_rate": 1e-05,
1772
+ "loss": 0.683,
1773
+ "step": 294
1774
+ },
1775
+ {
1776
+ "epoch": 1.17,
1777
+ "learning_rate": 1e-05,
1778
+ "loss": 0.6971,
1779
+ "step": 295
1780
+ },
1781
+ {
1782
+ "epoch": 1.17,
1783
+ "learning_rate": 1e-05,
1784
+ "loss": 0.6915,
1785
+ "step": 296
1786
+ },
1787
+ {
1788
+ "epoch": 1.18,
1789
+ "learning_rate": 1e-05,
1790
+ "loss": 0.7053,
1791
+ "step": 297
1792
+ },
1793
+ {
1794
+ "epoch": 1.18,
1795
+ "learning_rate": 1e-05,
1796
+ "loss": 0.7176,
1797
+ "step": 298
1798
+ },
1799
+ {
1800
+ "epoch": 1.19,
1801
+ "learning_rate": 1e-05,
1802
+ "loss": 0.7129,
1803
+ "step": 299
1804
+ },
1805
+ {
1806
+ "epoch": 1.19,
1807
+ "learning_rate": 1e-05,
1808
+ "loss": 0.7166,
1809
+ "step": 300
1810
+ },
1811
+ {
1812
+ "epoch": 1.19,
1813
+ "learning_rate": 1e-05,
1814
+ "loss": 0.7275,
1815
+ "step": 301
1816
+ },
1817
+ {
1818
+ "epoch": 1.2,
1819
+ "learning_rate": 1e-05,
1820
+ "loss": 0.6941,
1821
+ "step": 302
1822
+ },
1823
+ {
1824
+ "epoch": 1.2,
1825
+ "learning_rate": 1e-05,
1826
+ "loss": 0.7153,
1827
+ "step": 303
1828
+ },
1829
+ {
1830
+ "epoch": 1.21,
1831
+ "learning_rate": 1e-05,
1832
+ "loss": 0.7449,
1833
+ "step": 304
1834
+ },
1835
+ {
1836
+ "epoch": 1.21,
1837
+ "learning_rate": 1e-05,
1838
+ "loss": 0.7393,
1839
+ "step": 305
1840
+ },
1841
+ {
1842
+ "epoch": 1.21,
1843
+ "learning_rate": 1e-05,
1844
+ "loss": 0.7146,
1845
+ "step": 306
1846
+ },
1847
+ {
1848
+ "epoch": 1.22,
1849
+ "learning_rate": 1e-05,
1850
+ "loss": 0.6976,
1851
+ "step": 307
1852
+ },
1853
+ {
1854
+ "epoch": 1.22,
1855
+ "learning_rate": 1e-05,
1856
+ "loss": 0.7072,
1857
+ "step": 308
1858
+ },
1859
+ {
1860
+ "epoch": 1.23,
1861
+ "learning_rate": 1e-05,
1862
+ "loss": 0.677,
1863
+ "step": 309
1864
+ },
1865
+ {
1866
+ "epoch": 1.23,
1867
+ "learning_rate": 1e-05,
1868
+ "loss": 0.7155,
1869
+ "step": 310
1870
+ },
1871
+ {
1872
+ "epoch": 1.23,
1873
+ "learning_rate": 1e-05,
1874
+ "loss": 0.7057,
1875
+ "step": 311
1876
+ },
1877
+ {
1878
+ "epoch": 1.24,
1879
+ "learning_rate": 1e-05,
1880
+ "loss": 0.7008,
1881
+ "step": 312
1882
+ },
1883
+ {
1884
+ "epoch": 1.24,
1885
+ "learning_rate": 1e-05,
1886
+ "loss": 0.7123,
1887
+ "step": 313
1888
+ },
1889
+ {
1890
+ "epoch": 1.25,
1891
+ "learning_rate": 1e-05,
1892
+ "loss": 0.6998,
1893
+ "step": 314
1894
+ },
1895
+ {
1896
+ "epoch": 1.25,
1897
+ "learning_rate": 1e-05,
1898
+ "loss": 0.7242,
1899
+ "step": 315
1900
+ },
1901
+ {
1902
+ "epoch": 1.25,
1903
+ "learning_rate": 1e-05,
1904
+ "loss": 0.7009,
1905
+ "step": 316
1906
+ },
1907
+ {
1908
+ "epoch": 1.26,
1909
+ "learning_rate": 1e-05,
1910
+ "loss": 0.7019,
1911
+ "step": 317
1912
+ },
1913
+ {
1914
+ "epoch": 1.26,
1915
+ "learning_rate": 1e-05,
1916
+ "loss": 0.7173,
1917
+ "step": 318
1918
+ },
1919
+ {
1920
+ "epoch": 1.26,
1921
+ "learning_rate": 1e-05,
1922
+ "loss": 0.6784,
1923
+ "step": 319
1924
+ },
1925
+ {
1926
+ "epoch": 1.27,
1927
+ "learning_rate": 1e-05,
1928
+ "loss": 0.6833,
1929
+ "step": 320
1930
+ },
1931
+ {
1932
+ "epoch": 1.27,
1933
+ "learning_rate": 1e-05,
1934
+ "loss": 0.6952,
1935
+ "step": 321
1936
+ },
1937
+ {
1938
+ "epoch": 1.28,
1939
+ "learning_rate": 1e-05,
1940
+ "loss": 0.6893,
1941
+ "step": 322
1942
+ },
1943
+ {
1944
+ "epoch": 1.28,
1945
+ "learning_rate": 1e-05,
1946
+ "loss": 0.6864,
1947
+ "step": 323
1948
+ },
1949
+ {
1950
+ "epoch": 1.28,
1951
+ "learning_rate": 1e-05,
1952
+ "loss": 0.691,
1953
+ "step": 324
1954
+ },
1955
+ {
1956
+ "epoch": 1.29,
1957
+ "learning_rate": 1e-05,
1958
+ "loss": 0.701,
1959
+ "step": 325
1960
+ },
1961
+ {
1962
+ "epoch": 1.29,
1963
+ "learning_rate": 1e-05,
1964
+ "loss": 0.7076,
1965
+ "step": 326
1966
+ },
1967
+ {
1968
+ "epoch": 1.3,
1969
+ "learning_rate": 1e-05,
1970
+ "loss": 0.6974,
1971
+ "step": 327
1972
+ },
1973
+ {
1974
+ "epoch": 1.3,
1975
+ "learning_rate": 1e-05,
1976
+ "loss": 0.7064,
1977
+ "step": 328
1978
+ },
1979
+ {
1980
+ "epoch": 1.3,
1981
+ "learning_rate": 1e-05,
1982
+ "loss": 0.6979,
1983
+ "step": 329
1984
+ },
1985
+ {
1986
+ "epoch": 1.31,
1987
+ "learning_rate": 1e-05,
1988
+ "loss": 0.6983,
1989
+ "step": 330
1990
+ },
1991
+ {
1992
+ "epoch": 1.31,
1993
+ "learning_rate": 1e-05,
1994
+ "loss": 0.7082,
1995
+ "step": 331
1996
+ },
1997
+ {
1998
+ "epoch": 1.32,
1999
+ "learning_rate": 1e-05,
2000
+ "loss": 0.7141,
2001
+ "step": 332
2002
+ },
2003
+ {
2004
+ "epoch": 1.32,
2005
+ "learning_rate": 1e-05,
2006
+ "loss": 0.7016,
2007
+ "step": 333
2008
+ },
2009
+ {
2010
+ "epoch": 1.32,
2011
+ "learning_rate": 1e-05,
2012
+ "loss": 0.7184,
2013
+ "step": 334
2014
+ },
2015
+ {
2016
+ "epoch": 1.33,
2017
+ "learning_rate": 1e-05,
2018
+ "loss": 0.731,
2019
+ "step": 335
2020
+ },
2021
+ {
2022
+ "epoch": 1.33,
2023
+ "learning_rate": 1e-05,
2024
+ "loss": 0.6935,
2025
+ "step": 336
2026
+ },
2027
+ {
2028
+ "epoch": 1.34,
2029
+ "learning_rate": 1e-05,
2030
+ "loss": 0.6983,
2031
+ "step": 337
2032
+ },
2033
+ {
2034
+ "epoch": 1.34,
2035
+ "learning_rate": 1e-05,
2036
+ "loss": 0.6945,
2037
+ "step": 338
2038
+ },
2039
+ {
2040
+ "epoch": 1.34,
2041
+ "learning_rate": 1e-05,
2042
+ "loss": 0.6929,
2043
+ "step": 339
2044
+ },
2045
+ {
2046
+ "epoch": 1.35,
2047
+ "learning_rate": 1e-05,
2048
+ "loss": 0.7104,
2049
+ "step": 340
2050
+ },
2051
+ {
2052
+ "epoch": 1.35,
2053
+ "learning_rate": 1e-05,
2054
+ "loss": 0.7037,
2055
+ "step": 341
2056
+ },
2057
+ {
2058
+ "epoch": 1.36,
2059
+ "learning_rate": 1e-05,
2060
+ "loss": 0.7152,
2061
+ "step": 342
2062
+ },
2063
+ {
2064
+ "epoch": 1.36,
2065
+ "learning_rate": 1e-05,
2066
+ "loss": 0.6899,
2067
+ "step": 343
2068
+ },
2069
+ {
2070
+ "epoch": 1.36,
2071
+ "learning_rate": 1e-05,
2072
+ "loss": 0.6838,
2073
+ "step": 344
2074
+ },
2075
+ {
2076
+ "epoch": 1.37,
2077
+ "learning_rate": 1e-05,
2078
+ "loss": 0.6725,
2079
+ "step": 345
2080
+ },
2081
+ {
2082
+ "epoch": 1.37,
2083
+ "learning_rate": 1e-05,
2084
+ "loss": 0.7175,
2085
+ "step": 346
2086
+ },
2087
+ {
2088
+ "epoch": 1.38,
2089
+ "learning_rate": 1e-05,
2090
+ "loss": 0.6856,
2091
+ "step": 347
2092
+ },
2093
+ {
2094
+ "epoch": 1.38,
2095
+ "learning_rate": 1e-05,
2096
+ "loss": 0.6924,
2097
+ "step": 348
2098
+ },
2099
+ {
2100
+ "epoch": 1.38,
2101
+ "learning_rate": 1e-05,
2102
+ "loss": 0.6914,
2103
+ "step": 349
2104
+ },
2105
+ {
2106
+ "epoch": 1.39,
2107
+ "learning_rate": 1e-05,
2108
+ "loss": 0.6825,
2109
+ "step": 350
2110
+ },
2111
+ {
2112
+ "epoch": 1.39,
2113
+ "learning_rate": 1e-05,
2114
+ "loss": 0.6952,
2115
+ "step": 351
2116
+ },
2117
+ {
2118
+ "epoch": 1.4,
2119
+ "learning_rate": 1e-05,
2120
+ "loss": 0.7079,
2121
+ "step": 352
2122
+ },
2123
+ {
2124
+ "epoch": 1.4,
2125
+ "learning_rate": 1e-05,
2126
+ "loss": 0.6956,
2127
+ "step": 353
2128
+ },
2129
+ {
2130
+ "epoch": 1.4,
2131
+ "learning_rate": 1e-05,
2132
+ "loss": 0.7019,
2133
+ "step": 354
2134
+ },
2135
+ {
2136
+ "epoch": 1.41,
2137
+ "learning_rate": 1e-05,
2138
+ "loss": 0.6973,
2139
+ "step": 355
2140
+ },
2141
+ {
2142
+ "epoch": 1.41,
2143
+ "learning_rate": 1e-05,
2144
+ "loss": 0.6784,
2145
+ "step": 356
2146
+ },
2147
+ {
2148
+ "epoch": 1.42,
2149
+ "learning_rate": 1e-05,
2150
+ "loss": 0.6913,
2151
+ "step": 357
2152
+ },
2153
+ {
2154
+ "epoch": 1.42,
2155
+ "learning_rate": 1e-05,
2156
+ "loss": 0.6872,
2157
+ "step": 358
2158
+ },
2159
+ {
2160
+ "epoch": 1.42,
2161
+ "learning_rate": 1e-05,
2162
+ "loss": 0.7088,
2163
+ "step": 359
2164
+ },
2165
+ {
2166
+ "epoch": 1.43,
2167
+ "learning_rate": 1e-05,
2168
+ "loss": 0.6955,
2169
+ "step": 360
2170
+ },
2171
+ {
2172
+ "epoch": 1.43,
2173
+ "learning_rate": 1e-05,
2174
+ "loss": 0.6927,
2175
+ "step": 361
2176
+ },
2177
+ {
2178
+ "epoch": 1.44,
2179
+ "learning_rate": 1e-05,
2180
+ "loss": 0.6839,
2181
+ "step": 362
2182
+ },
2183
+ {
2184
+ "epoch": 1.44,
2185
+ "learning_rate": 1e-05,
2186
+ "loss": 0.6717,
2187
+ "step": 363
2188
+ },
2189
+ {
2190
+ "epoch": 1.44,
2191
+ "learning_rate": 1e-05,
2192
+ "loss": 0.7001,
2193
+ "step": 364
2194
+ },
2195
+ {
2196
+ "epoch": 1.45,
2197
+ "learning_rate": 1e-05,
2198
+ "loss": 0.6687,
2199
+ "step": 365
2200
+ },
2201
+ {
2202
+ "epoch": 1.45,
2203
+ "learning_rate": 1e-05,
2204
+ "loss": 0.7016,
2205
+ "step": 366
2206
+ },
2207
+ {
2208
+ "epoch": 1.46,
2209
+ "learning_rate": 1e-05,
2210
+ "loss": 0.6807,
2211
+ "step": 367
2212
+ },
2213
+ {
2214
+ "epoch": 1.46,
2215
+ "learning_rate": 1e-05,
2216
+ "loss": 0.7073,
2217
+ "step": 368
2218
+ },
2219
+ {
2220
+ "epoch": 1.46,
2221
+ "learning_rate": 1e-05,
2222
+ "loss": 0.6704,
2223
+ "step": 369
2224
+ },
2225
+ {
2226
+ "epoch": 1.47,
2227
+ "learning_rate": 1e-05,
2228
+ "loss": 0.6812,
2229
+ "step": 370
2230
+ },
2231
+ {
2232
+ "epoch": 1.47,
2233
+ "learning_rate": 1e-05,
2234
+ "loss": 0.6579,
2235
+ "step": 371
2236
+ },
2237
+ {
2238
+ "epoch": 1.48,
2239
+ "learning_rate": 1e-05,
2240
+ "loss": 0.6825,
2241
+ "step": 372
2242
+ },
2243
+ {
2244
+ "epoch": 1.48,
2245
+ "learning_rate": 1e-05,
2246
+ "loss": 0.7014,
2247
+ "step": 373
2248
+ },
2249
+ {
2250
+ "epoch": 1.48,
2251
+ "learning_rate": 1e-05,
2252
+ "loss": 0.6789,
2253
+ "step": 374
2254
+ },
2255
+ {
2256
+ "epoch": 1.49,
2257
+ "learning_rate": 1e-05,
2258
+ "loss": 0.6753,
2259
+ "step": 375
2260
+ },
2261
+ {
2262
+ "epoch": 1.49,
2263
+ "learning_rate": 1e-05,
2264
+ "loss": 0.678,
2265
+ "step": 376
2266
+ },
2267
+ {
2268
+ "epoch": 1.49,
2269
+ "learning_rate": 1e-05,
2270
+ "loss": 0.6874,
2271
+ "step": 377
2272
+ },
2273
+ {
2274
+ "epoch": 1.5,
2275
+ "learning_rate": 1e-05,
2276
+ "loss": 0.6871,
2277
+ "step": 378
2278
+ },
2279
+ {
2280
+ "epoch": 1.5,
2281
+ "learning_rate": 1e-05,
2282
+ "loss": 0.6877,
2283
+ "step": 379
2284
+ },
2285
+ {
2286
+ "epoch": 1.51,
2287
+ "learning_rate": 1e-05,
2288
+ "loss": 0.7023,
2289
+ "step": 380
2290
+ },
2291
+ {
2292
+ "epoch": 1.51,
2293
+ "learning_rate": 1e-05,
2294
+ "loss": 0.7043,
2295
+ "step": 381
2296
+ },
2297
+ {
2298
+ "epoch": 1.51,
2299
+ "learning_rate": 1e-05,
2300
+ "loss": 0.6916,
2301
+ "step": 382
2302
+ },
2303
+ {
2304
+ "epoch": 1.52,
2305
+ "learning_rate": 1e-05,
2306
+ "loss": 0.6834,
2307
+ "step": 383
2308
+ },
2309
+ {
2310
+ "epoch": 1.52,
2311
+ "learning_rate": 1e-05,
2312
+ "loss": 0.6896,
2313
+ "step": 384
2314
+ },
2315
+ {
2316
+ "epoch": 1.53,
2317
+ "learning_rate": 1e-05,
2318
+ "loss": 0.6738,
2319
+ "step": 385
2320
+ },
2321
+ {
2322
+ "epoch": 1.53,
2323
+ "learning_rate": 1e-05,
2324
+ "loss": 0.6912,
2325
+ "step": 386
2326
+ },
2327
+ {
2328
+ "epoch": 1.53,
2329
+ "learning_rate": 1e-05,
2330
+ "loss": 0.6988,
2331
+ "step": 387
2332
+ },
2333
+ {
2334
+ "epoch": 1.54,
2335
+ "learning_rate": 1e-05,
2336
+ "loss": 0.6908,
2337
+ "step": 388
2338
+ },
2339
+ {
2340
+ "epoch": 1.54,
2341
+ "learning_rate": 1e-05,
2342
+ "loss": 0.6944,
2343
+ "step": 389
2344
+ },
2345
+ {
2346
+ "epoch": 1.55,
2347
+ "learning_rate": 1e-05,
2348
+ "loss": 0.6953,
2349
+ "step": 390
2350
+ },
2351
+ {
2352
+ "epoch": 1.55,
2353
+ "learning_rate": 1e-05,
2354
+ "loss": 0.6648,
2355
+ "step": 391
2356
+ },
2357
+ {
2358
+ "epoch": 1.55,
2359
+ "learning_rate": 1e-05,
2360
+ "loss": 0.686,
2361
+ "step": 392
2362
+ },
2363
+ {
2364
+ "epoch": 1.56,
2365
+ "learning_rate": 1e-05,
2366
+ "loss": 0.6776,
2367
+ "step": 393
2368
+ },
2369
+ {
2370
+ "epoch": 1.56,
2371
+ "learning_rate": 1e-05,
2372
+ "loss": 0.6716,
2373
+ "step": 394
2374
+ },
2375
+ {
2376
+ "epoch": 1.57,
2377
+ "learning_rate": 1e-05,
2378
+ "loss": 0.6742,
2379
+ "step": 395
2380
+ },
2381
+ {
2382
+ "epoch": 1.57,
2383
+ "learning_rate": 1e-05,
2384
+ "loss": 0.663,
2385
+ "step": 396
2386
+ },
2387
+ {
2388
+ "epoch": 1.57,
2389
+ "learning_rate": 1e-05,
2390
+ "loss": 0.6796,
2391
+ "step": 397
2392
+ },
2393
+ {
2394
+ "epoch": 1.58,
2395
+ "learning_rate": 1e-05,
2396
+ "loss": 0.6758,
2397
+ "step": 398
2398
+ },
2399
+ {
2400
+ "epoch": 1.58,
2401
+ "learning_rate": 1e-05,
2402
+ "loss": 0.6671,
2403
+ "step": 399
2404
+ },
2405
+ {
2406
+ "epoch": 1.59,
2407
+ "learning_rate": 1e-05,
2408
+ "loss": 0.6747,
2409
+ "step": 400
2410
+ },
2411
+ {
2412
+ "epoch": 1.59,
2413
+ "learning_rate": 1e-05,
2414
+ "loss": 0.6851,
2415
+ "step": 401
2416
+ },
2417
+ {
2418
+ "epoch": 1.59,
2419
+ "learning_rate": 1e-05,
2420
+ "loss": 0.672,
2421
+ "step": 402
2422
+ },
2423
+ {
2424
+ "epoch": 1.6,
2425
+ "learning_rate": 1e-05,
2426
+ "loss": 0.6563,
2427
+ "step": 403
2428
+ },
2429
+ {
2430
+ "epoch": 1.6,
2431
+ "learning_rate": 1e-05,
2432
+ "loss": 0.6671,
2433
+ "step": 404
2434
+ },
2435
+ {
2436
+ "epoch": 1.61,
2437
+ "learning_rate": 1e-05,
2438
+ "loss": 0.6748,
2439
+ "step": 405
2440
+ },
2441
+ {
2442
+ "epoch": 1.61,
2443
+ "learning_rate": 1e-05,
2444
+ "loss": 0.6986,
2445
+ "step": 406
2446
+ },
2447
+ {
2448
+ "epoch": 1.61,
2449
+ "learning_rate": 1e-05,
2450
+ "loss": 0.6935,
2451
+ "step": 407
2452
+ },
2453
+ {
2454
+ "epoch": 1.62,
2455
+ "learning_rate": 1e-05,
2456
+ "loss": 0.6781,
2457
+ "step": 408
2458
+ },
2459
+ {
2460
+ "epoch": 1.62,
2461
+ "learning_rate": 1e-05,
2462
+ "loss": 0.6774,
2463
+ "step": 409
2464
+ },
2465
+ {
2466
+ "epoch": 1.63,
2467
+ "learning_rate": 1e-05,
2468
+ "loss": 0.689,
2469
+ "step": 410
2470
+ },
2471
+ {
2472
+ "epoch": 1.63,
2473
+ "learning_rate": 1e-05,
2474
+ "loss": 0.6866,
2475
+ "step": 411
2476
+ },
2477
+ {
2478
+ "epoch": 1.63,
2479
+ "learning_rate": 1e-05,
2480
+ "loss": 0.6865,
2481
+ "step": 412
2482
+ },
2483
+ {
2484
+ "epoch": 1.64,
2485
+ "learning_rate": 1e-05,
2486
+ "loss": 0.6829,
2487
+ "step": 413
2488
+ },
2489
+ {
2490
+ "epoch": 1.64,
2491
+ "learning_rate": 1e-05,
2492
+ "loss": 0.6897,
2493
+ "step": 414
2494
+ },
2495
+ {
2496
+ "epoch": 1.65,
2497
+ "learning_rate": 1e-05,
2498
+ "loss": 0.6824,
2499
+ "step": 415
2500
+ },
2501
+ {
2502
+ "epoch": 1.65,
2503
+ "learning_rate": 1e-05,
2504
+ "loss": 0.6772,
2505
+ "step": 416
2506
+ },
2507
+ {
2508
+ "epoch": 1.65,
2509
+ "learning_rate": 1e-05,
2510
+ "loss": 0.6895,
2511
+ "step": 417
2512
+ },
2513
+ {
2514
+ "epoch": 1.66,
2515
+ "learning_rate": 1e-05,
2516
+ "loss": 0.6678,
2517
+ "step": 418
2518
+ },
2519
+ {
2520
+ "epoch": 1.66,
2521
+ "learning_rate": 1e-05,
2522
+ "loss": 0.685,
2523
+ "step": 419
2524
+ },
2525
+ {
2526
+ "epoch": 1.67,
2527
+ "learning_rate": 1e-05,
2528
+ "loss": 0.6731,
2529
+ "step": 420
2530
+ },
2531
+ {
2532
+ "epoch": 1.67,
2533
+ "learning_rate": 1e-05,
2534
+ "loss": 0.6672,
2535
+ "step": 421
2536
+ },
2537
+ {
2538
+ "epoch": 1.67,
2539
+ "learning_rate": 1e-05,
2540
+ "loss": 0.6986,
2541
+ "step": 422
2542
+ },
2543
+ {
2544
+ "epoch": 1.68,
2545
+ "learning_rate": 1e-05,
2546
+ "loss": 0.6701,
2547
+ "step": 423
2548
+ },
2549
+ {
2550
+ "epoch": 1.68,
2551
+ "learning_rate": 1e-05,
2552
+ "loss": 0.6561,
2553
+ "step": 424
2554
+ },
2555
+ {
2556
+ "epoch": 1.69,
2557
+ "learning_rate": 1e-05,
2558
+ "loss": 0.6701,
2559
+ "step": 425
2560
+ },
2561
+ {
2562
+ "epoch": 1.69,
2563
+ "learning_rate": 1e-05,
2564
+ "loss": 0.7023,
2565
+ "step": 426
2566
+ },
2567
+ {
2568
+ "epoch": 1.69,
2569
+ "learning_rate": 1e-05,
2570
+ "loss": 0.6546,
2571
+ "step": 427
2572
+ },
2573
+ {
2574
+ "epoch": 1.7,
2575
+ "learning_rate": 1e-05,
2576
+ "loss": 0.6609,
2577
+ "step": 428
2578
+ },
2579
+ {
2580
+ "epoch": 1.7,
2581
+ "learning_rate": 1e-05,
2582
+ "loss": 0.6699,
2583
+ "step": 429
2584
+ },
2585
+ {
2586
+ "epoch": 1.71,
2587
+ "learning_rate": 1e-05,
2588
+ "loss": 0.6567,
2589
+ "step": 430
2590
+ },
2591
+ {
2592
+ "epoch": 1.71,
2593
+ "learning_rate": 1e-05,
2594
+ "loss": 0.6913,
2595
+ "step": 431
2596
+ },
2597
+ {
2598
+ "epoch": 1.71,
2599
+ "learning_rate": 1e-05,
2600
+ "loss": 0.6743,
2601
+ "step": 432
2602
+ },
2603
+ {
2604
+ "epoch": 1.72,
2605
+ "learning_rate": 1e-05,
2606
+ "loss": 0.6797,
2607
+ "step": 433
2608
+ },
2609
+ {
2610
+ "epoch": 1.72,
2611
+ "learning_rate": 1e-05,
2612
+ "loss": 0.6595,
2613
+ "step": 434
2614
+ },
2615
+ {
2616
+ "epoch": 1.72,
2617
+ "learning_rate": 1e-05,
2618
+ "loss": 0.6772,
2619
+ "step": 435
2620
+ },
2621
+ {
2622
+ "epoch": 1.73,
2623
+ "learning_rate": 1e-05,
2624
+ "loss": 0.6414,
2625
+ "step": 436
2626
+ },
2627
+ {
2628
+ "epoch": 1.73,
2629
+ "learning_rate": 1e-05,
2630
+ "loss": 0.6669,
2631
+ "step": 437
2632
+ },
2633
+ {
2634
+ "epoch": 1.74,
2635
+ "learning_rate": 1e-05,
2636
+ "loss": 0.6793,
2637
+ "step": 438
2638
+ },
2639
+ {
2640
+ "epoch": 1.74,
2641
+ "learning_rate": 1e-05,
2642
+ "loss": 0.6723,
2643
+ "step": 439
2644
+ },
2645
+ {
2646
+ "epoch": 1.74,
2647
+ "learning_rate": 1e-05,
2648
+ "loss": 0.6536,
2649
+ "step": 440
2650
+ },
2651
+ {
2652
+ "epoch": 1.75,
2653
+ "learning_rate": 1e-05,
2654
+ "loss": 0.6862,
2655
+ "step": 441
2656
+ },
2657
+ {
2658
+ "epoch": 1.75,
2659
+ "learning_rate": 1e-05,
2660
+ "loss": 0.6533,
2661
+ "step": 442
2662
+ },
2663
+ {
2664
+ "epoch": 1.76,
2665
+ "learning_rate": 1e-05,
2666
+ "loss": 0.6689,
2667
+ "step": 443
2668
+ },
2669
+ {
2670
+ "epoch": 1.76,
2671
+ "learning_rate": 1e-05,
2672
+ "loss": 0.6696,
2673
+ "step": 444
2674
+ },
2675
+ {
2676
+ "epoch": 1.76,
2677
+ "learning_rate": 1e-05,
2678
+ "loss": 0.6647,
2679
+ "step": 445
2680
+ },
2681
+ {
2682
+ "epoch": 1.77,
2683
+ "learning_rate": 1e-05,
2684
+ "loss": 0.6497,
2685
+ "step": 446
2686
+ },
2687
+ {
2688
+ "epoch": 1.77,
2689
+ "learning_rate": 1e-05,
2690
+ "loss": 0.6554,
2691
+ "step": 447
2692
+ },
2693
+ {
2694
+ "epoch": 1.78,
2695
+ "learning_rate": 1e-05,
2696
+ "loss": 0.6734,
2697
+ "step": 448
2698
+ },
2699
+ {
2700
+ "epoch": 1.78,
2701
+ "learning_rate": 1e-05,
2702
+ "loss": 0.6685,
2703
+ "step": 449
2704
+ },
2705
+ {
2706
+ "epoch": 1.78,
2707
+ "learning_rate": 1e-05,
2708
+ "loss": 0.6638,
2709
+ "step": 450
2710
+ },
2711
+ {
2712
+ "epoch": 1.79,
2713
+ "learning_rate": 1e-05,
2714
+ "loss": 0.6707,
2715
+ "step": 451
2716
+ },
2717
+ {
2718
+ "epoch": 1.79,
2719
+ "learning_rate": 1e-05,
2720
+ "loss": 0.6688,
2721
+ "step": 452
2722
+ },
2723
+ {
2724
+ "epoch": 1.8,
2725
+ "learning_rate": 1e-05,
2726
+ "loss": 0.6793,
2727
+ "step": 453
2728
+ },
2729
+ {
2730
+ "epoch": 1.8,
2731
+ "learning_rate": 1e-05,
2732
+ "loss": 0.6347,
2733
+ "step": 454
2734
+ },
2735
+ {
2736
+ "epoch": 1.8,
2737
+ "learning_rate": 1e-05,
2738
+ "loss": 0.6654,
2739
+ "step": 455
2740
+ },
2741
+ {
2742
+ "epoch": 1.81,
2743
+ "learning_rate": 1e-05,
2744
+ "loss": 0.6622,
2745
+ "step": 456
2746
+ },
2747
+ {
2748
+ "epoch": 1.81,
2749
+ "learning_rate": 1e-05,
2750
+ "loss": 0.6748,
2751
+ "step": 457
2752
+ },
2753
+ {
2754
+ "epoch": 1.82,
2755
+ "learning_rate": 1e-05,
2756
+ "loss": 0.6764,
2757
+ "step": 458
2758
+ },
2759
+ {
2760
+ "epoch": 1.82,
2761
+ "learning_rate": 1e-05,
2762
+ "loss": 0.6683,
2763
+ "step": 459
2764
+ },
2765
+ {
2766
+ "epoch": 1.82,
2767
+ "learning_rate": 1e-05,
2768
+ "loss": 0.6484,
2769
+ "step": 460
2770
+ },
2771
+ {
2772
+ "epoch": 1.83,
2773
+ "learning_rate": 1e-05,
2774
+ "loss": 0.6778,
2775
+ "step": 461
2776
+ },
2777
+ {
2778
+ "epoch": 1.83,
2779
+ "learning_rate": 1e-05,
2780
+ "loss": 0.6532,
2781
+ "step": 462
2782
+ },
2783
+ {
2784
+ "epoch": 1.84,
2785
+ "learning_rate": 1e-05,
2786
+ "loss": 0.6759,
2787
+ "step": 463
2788
+ },
2789
+ {
2790
+ "epoch": 1.84,
2791
+ "learning_rate": 1e-05,
2792
+ "loss": 0.6621,
2793
+ "step": 464
2794
+ },
2795
+ {
2796
+ "epoch": 1.84,
2797
+ "learning_rate": 1e-05,
2798
+ "loss": 0.6589,
2799
+ "step": 465
2800
+ },
2801
+ {
2802
+ "epoch": 1.85,
2803
+ "learning_rate": 1e-05,
2804
+ "loss": 0.6906,
2805
+ "step": 466
2806
+ },
2807
+ {
2808
+ "epoch": 1.85,
2809
+ "learning_rate": 1e-05,
2810
+ "loss": 0.6393,
2811
+ "step": 467
2812
+ },
2813
+ {
2814
+ "epoch": 1.86,
2815
+ "learning_rate": 1e-05,
2816
+ "loss": 0.6528,
2817
+ "step": 468
2818
+ },
2819
+ {
2820
+ "epoch": 1.86,
2821
+ "learning_rate": 1e-05,
2822
+ "loss": 0.6466,
2823
+ "step": 469
2824
+ },
2825
+ {
2826
+ "epoch": 1.86,
2827
+ "learning_rate": 1e-05,
2828
+ "loss": 0.6508,
2829
+ "step": 470
2830
+ },
2831
+ {
2832
+ "epoch": 1.87,
2833
+ "learning_rate": 1e-05,
2834
+ "loss": 0.6738,
2835
+ "step": 471
2836
+ },
2837
+ {
2838
+ "epoch": 1.87,
2839
+ "learning_rate": 1e-05,
2840
+ "loss": 0.6498,
2841
+ "step": 472
2842
+ },
2843
+ {
2844
+ "epoch": 1.88,
2845
+ "learning_rate": 1e-05,
2846
+ "loss": 0.655,
2847
+ "step": 473
2848
+ },
2849
+ {
2850
+ "epoch": 1.88,
2851
+ "learning_rate": 1e-05,
2852
+ "loss": 0.656,
2853
+ "step": 474
2854
+ },
2855
+ {
2856
+ "epoch": 1.88,
2857
+ "learning_rate": 1e-05,
2858
+ "loss": 0.6491,
2859
+ "step": 475
2860
+ },
2861
+ {
2862
+ "epoch": 1.89,
2863
+ "learning_rate": 1e-05,
2864
+ "loss": 0.6548,
2865
+ "step": 476
2866
+ },
2867
+ {
2868
+ "epoch": 1.89,
2869
+ "learning_rate": 1e-05,
2870
+ "loss": 0.6684,
2871
+ "step": 477
2872
+ },
2873
+ {
2874
+ "epoch": 1.9,
2875
+ "learning_rate": 1e-05,
2876
+ "loss": 0.6573,
2877
+ "step": 478
2878
+ },
2879
+ {
2880
+ "epoch": 1.9,
2881
+ "learning_rate": 1e-05,
2882
+ "loss": 0.6654,
2883
+ "step": 479
2884
+ },
2885
+ {
2886
+ "epoch": 1.9,
2887
+ "learning_rate": 1e-05,
2888
+ "loss": 0.6682,
2889
+ "step": 480
2890
+ },
2891
+ {
2892
+ "epoch": 1.91,
2893
+ "learning_rate": 1e-05,
2894
+ "loss": 0.6757,
2895
+ "step": 481
2896
+ },
2897
+ {
2898
+ "epoch": 1.91,
2899
+ "learning_rate": 1e-05,
2900
+ "loss": 0.6535,
2901
+ "step": 482
2902
+ },
2903
+ {
2904
+ "epoch": 1.92,
2905
+ "learning_rate": 1e-05,
2906
+ "loss": 0.6383,
2907
+ "step": 483
2908
+ },
2909
+ {
2910
+ "epoch": 1.92,
2911
+ "learning_rate": 1e-05,
2912
+ "loss": 0.6593,
2913
+ "step": 484
2914
+ },
2915
+ {
2916
+ "epoch": 1.92,
2917
+ "learning_rate": 1e-05,
2918
+ "loss": 0.6485,
2919
+ "step": 485
2920
+ },
2921
+ {
2922
+ "epoch": 1.93,
2923
+ "learning_rate": 1e-05,
2924
+ "loss": 0.6603,
2925
+ "step": 486
2926
+ },
2927
+ {
2928
+ "epoch": 1.93,
2929
+ "learning_rate": 1e-05,
2930
+ "loss": 0.6545,
2931
+ "step": 487
2932
+ },
2933
+ {
2934
+ "epoch": 1.94,
2935
+ "learning_rate": 1e-05,
2936
+ "loss": 0.6673,
2937
+ "step": 488
2938
+ },
2939
+ {
2940
+ "epoch": 1.94,
2941
+ "learning_rate": 1e-05,
2942
+ "loss": 0.6606,
2943
+ "step": 489
2944
+ },
2945
+ {
2946
+ "epoch": 1.94,
2947
+ "learning_rate": 1e-05,
2948
+ "loss": 0.6649,
2949
+ "step": 490
2950
+ },
2951
+ {
2952
+ "epoch": 1.95,
2953
+ "learning_rate": 1e-05,
2954
+ "loss": 0.6498,
2955
+ "step": 491
2956
+ },
2957
+ {
2958
+ "epoch": 1.95,
2959
+ "learning_rate": 1e-05,
2960
+ "loss": 0.6688,
2961
+ "step": 492
2962
+ },
2963
+ {
2964
+ "epoch": 1.95,
2965
+ "learning_rate": 1e-05,
2966
+ "loss": 0.6723,
2967
+ "step": 493
2968
+ },
2969
+ {
2970
+ "epoch": 1.96,
2971
+ "learning_rate": 1e-05,
2972
+ "loss": 0.6407,
2973
+ "step": 494
2974
+ },
2975
+ {
2976
+ "epoch": 1.96,
2977
+ "learning_rate": 1e-05,
2978
+ "loss": 0.6391,
2979
+ "step": 495
2980
+ },
2981
+ {
2982
+ "epoch": 1.97,
2983
+ "learning_rate": 1e-05,
2984
+ "loss": 0.6502,
2985
+ "step": 496
2986
+ },
2987
+ {
2988
+ "epoch": 1.97,
2989
+ "learning_rate": 1e-05,
2990
+ "loss": 0.653,
2991
+ "step": 497
2992
+ },
2993
+ {
2994
+ "epoch": 1.97,
2995
+ "learning_rate": 1e-05,
2996
+ "loss": 0.6544,
2997
+ "step": 498
2998
+ },
2999
+ {
3000
+ "epoch": 1.98,
3001
+ "learning_rate": 1e-05,
3002
+ "loss": 0.6585,
3003
+ "step": 499
3004
+ },
3005
+ {
3006
+ "epoch": 1.98,
3007
+ "learning_rate": 1e-05,
3008
+ "loss": 0.654,
3009
+ "step": 500
3010
+ },
3011
+ {
3012
+ "epoch": 1.99,
3013
+ "learning_rate": 1e-05,
3014
+ "loss": 0.6502,
3015
+ "step": 501
3016
+ },
3017
+ {
3018
+ "epoch": 1.99,
3019
+ "learning_rate": 1e-05,
3020
+ "loss": 0.6415,
3021
+ "step": 502
3022
+ },
3023
+ {
3024
+ "epoch": 1.99,
3025
+ "learning_rate": 1e-05,
3026
+ "loss": 0.6746,
3027
+ "step": 503
3028
+ },
3029
+ {
3030
+ "epoch": 2.0,
3031
+ "learning_rate": 1e-05,
3032
+ "loss": 0.6698,
3033
+ "step": 504
3034
+ },
3035
+ {
3036
+ "epoch": 2.0,
3037
+ "learning_rate": 1e-05,
3038
+ "loss": 0.5898,
3039
+ "step": 505
3040
+ },
3041
+ {
3042
+ "epoch": 2.01,
3043
+ "learning_rate": 1e-05,
3044
+ "loss": 0.5663,
3045
+ "step": 506
3046
+ },
3047
+ {
3048
+ "epoch": 2.01,
3049
+ "learning_rate": 1e-05,
3050
+ "loss": 0.566,
3051
+ "step": 507
3052
+ },
3053
+ {
3054
+ "epoch": 2.01,
3055
+ "learning_rate": 1e-05,
3056
+ "loss": 0.5652,
3057
+ "step": 508
3058
+ },
3059
+ {
3060
+ "epoch": 2.02,
3061
+ "learning_rate": 1e-05,
3062
+ "loss": 0.5562,
3063
+ "step": 509
3064
+ },
3065
+ {
3066
+ "epoch": 2.02,
3067
+ "learning_rate": 1e-05,
3068
+ "loss": 0.5749,
3069
+ "step": 510
3070
+ },
3071
+ {
3072
+ "epoch": 2.03,
3073
+ "learning_rate": 1e-05,
3074
+ "loss": 0.5476,
3075
+ "step": 511
3076
+ },
3077
+ {
3078
+ "epoch": 2.03,
3079
+ "learning_rate": 1e-05,
3080
+ "loss": 0.5745,
3081
+ "step": 512
3082
+ },
3083
+ {
3084
+ "epoch": 2.03,
3085
+ "learning_rate": 1e-05,
3086
+ "loss": 0.5615,
3087
+ "step": 513
3088
+ },
3089
+ {
3090
+ "epoch": 2.04,
3091
+ "learning_rate": 1e-05,
3092
+ "loss": 0.5422,
3093
+ "step": 514
3094
+ },
3095
+ {
3096
+ "epoch": 2.04,
3097
+ "learning_rate": 1e-05,
3098
+ "loss": 0.5452,
3099
+ "step": 515
3100
+ },
3101
+ {
3102
+ "epoch": 2.05,
3103
+ "learning_rate": 1e-05,
3104
+ "loss": 0.575,
3105
+ "step": 516
3106
+ },
3107
+ {
3108
+ "epoch": 2.05,
3109
+ "learning_rate": 1e-05,
3110
+ "loss": 0.5655,
3111
+ "step": 517
3112
+ },
3113
+ {
3114
+ "epoch": 2.05,
3115
+ "learning_rate": 1e-05,
3116
+ "loss": 0.5441,
3117
+ "step": 518
3118
+ },
3119
+ {
3120
+ "epoch": 2.06,
3121
+ "learning_rate": 1e-05,
3122
+ "loss": 0.5576,
3123
+ "step": 519
3124
+ },
3125
+ {
3126
+ "epoch": 2.06,
3127
+ "learning_rate": 1e-05,
3128
+ "loss": 0.5571,
3129
+ "step": 520
3130
+ },
3131
+ {
3132
+ "epoch": 2.07,
3133
+ "learning_rate": 1e-05,
3134
+ "loss": 0.554,
3135
+ "step": 521
3136
+ },
3137
+ {
3138
+ "epoch": 2.07,
3139
+ "learning_rate": 1e-05,
3140
+ "loss": 0.5717,
3141
+ "step": 522
3142
+ },
3143
+ {
3144
+ "epoch": 2.07,
3145
+ "learning_rate": 1e-05,
3146
+ "loss": 0.5553,
3147
+ "step": 523
3148
+ },
3149
+ {
3150
+ "epoch": 2.08,
3151
+ "learning_rate": 1e-05,
3152
+ "loss": 0.5679,
3153
+ "step": 524
3154
+ },
3155
+ {
3156
+ "epoch": 2.08,
3157
+ "learning_rate": 1e-05,
3158
+ "loss": 0.5514,
3159
+ "step": 525
3160
+ },
3161
+ {
3162
+ "epoch": 2.09,
3163
+ "learning_rate": 1e-05,
3164
+ "loss": 0.5572,
3165
+ "step": 526
3166
+ },
3167
+ {
3168
+ "epoch": 2.09,
3169
+ "learning_rate": 1e-05,
3170
+ "loss": 0.5626,
3171
+ "step": 527
3172
+ },
3173
+ {
3174
+ "epoch": 2.09,
3175
+ "learning_rate": 1e-05,
3176
+ "loss": 0.5531,
3177
+ "step": 528
3178
+ },
3179
+ {
3180
+ "epoch": 2.1,
3181
+ "learning_rate": 1e-05,
3182
+ "loss": 0.5388,
3183
+ "step": 529
3184
+ },
3185
+ {
3186
+ "epoch": 2.1,
3187
+ "learning_rate": 1e-05,
3188
+ "loss": 0.5431,
3189
+ "step": 530
3190
+ },
3191
+ {
3192
+ "epoch": 2.11,
3193
+ "learning_rate": 1e-05,
3194
+ "loss": 0.5522,
3195
+ "step": 531
3196
+ },
3197
+ {
3198
+ "epoch": 2.11,
3199
+ "learning_rate": 1e-05,
3200
+ "loss": 0.5412,
3201
+ "step": 532
3202
+ },
3203
+ {
3204
+ "epoch": 2.11,
3205
+ "learning_rate": 1e-05,
3206
+ "loss": 0.5469,
3207
+ "step": 533
3208
+ },
3209
+ {
3210
+ "epoch": 2.12,
3211
+ "learning_rate": 1e-05,
3212
+ "loss": 0.5505,
3213
+ "step": 534
3214
+ },
3215
+ {
3216
+ "epoch": 2.12,
3217
+ "learning_rate": 1e-05,
3218
+ "loss": 0.5348,
3219
+ "step": 535
3220
+ },
3221
+ {
3222
+ "epoch": 2.13,
3223
+ "learning_rate": 1e-05,
3224
+ "loss": 0.5736,
3225
+ "step": 536
3226
+ },
3227
+ {
3228
+ "epoch": 2.13,
3229
+ "learning_rate": 1e-05,
3230
+ "loss": 0.563,
3231
+ "step": 537
3232
+ },
3233
+ {
3234
+ "epoch": 2.13,
3235
+ "learning_rate": 1e-05,
3236
+ "loss": 0.5663,
3237
+ "step": 538
3238
+ },
3239
+ {
3240
+ "epoch": 2.14,
3241
+ "learning_rate": 1e-05,
3242
+ "loss": 0.5559,
3243
+ "step": 539
3244
+ },
3245
+ {
3246
+ "epoch": 2.14,
3247
+ "learning_rate": 1e-05,
3248
+ "loss": 0.5563,
3249
+ "step": 540
3250
+ },
3251
+ {
3252
+ "epoch": 2.15,
3253
+ "learning_rate": 1e-05,
3254
+ "loss": 0.5717,
3255
+ "step": 541
3256
+ },
3257
+ {
3258
+ "epoch": 2.15,
3259
+ "learning_rate": 1e-05,
3260
+ "loss": 0.5477,
3261
+ "step": 542
3262
+ },
3263
+ {
3264
+ "epoch": 2.15,
3265
+ "learning_rate": 1e-05,
3266
+ "loss": 0.5447,
3267
+ "step": 543
3268
+ },
3269
+ {
3270
+ "epoch": 2.16,
3271
+ "learning_rate": 1e-05,
3272
+ "loss": 0.5516,
3273
+ "step": 544
3274
+ },
3275
+ {
3276
+ "epoch": 2.16,
3277
+ "learning_rate": 1e-05,
3278
+ "loss": 0.5585,
3279
+ "step": 545
3280
+ },
3281
+ {
3282
+ "epoch": 2.17,
3283
+ "learning_rate": 1e-05,
3284
+ "loss": 0.556,
3285
+ "step": 546
3286
+ },
3287
+ {
3288
+ "epoch": 2.17,
3289
+ "learning_rate": 1e-05,
3290
+ "loss": 0.5301,
3291
+ "step": 547
3292
+ },
3293
+ {
3294
+ "epoch": 2.17,
3295
+ "learning_rate": 1e-05,
3296
+ "loss": 0.5745,
3297
+ "step": 548
3298
+ },
3299
+ {
3300
+ "epoch": 2.18,
3301
+ "learning_rate": 1e-05,
3302
+ "loss": 0.5706,
3303
+ "step": 549
3304
+ },
3305
+ {
3306
+ "epoch": 2.18,
3307
+ "learning_rate": 1e-05,
3308
+ "loss": 0.5546,
3309
+ "step": 550
3310
+ },
3311
+ {
3312
+ "epoch": 2.18,
3313
+ "learning_rate": 1e-05,
3314
+ "loss": 0.5448,
3315
+ "step": 551
3316
+ },
3317
+ {
3318
+ "epoch": 2.19,
3319
+ "learning_rate": 1e-05,
3320
+ "loss": 0.5432,
3321
+ "step": 552
3322
+ },
3323
+ {
3324
+ "epoch": 2.19,
3325
+ "learning_rate": 1e-05,
3326
+ "loss": 0.5636,
3327
+ "step": 553
3328
+ },
3329
+ {
3330
+ "epoch": 2.2,
3331
+ "learning_rate": 1e-05,
3332
+ "loss": 0.5428,
3333
+ "step": 554
3334
+ },
3335
+ {
3336
+ "epoch": 2.2,
3337
+ "learning_rate": 1e-05,
3338
+ "loss": 0.5577,
3339
+ "step": 555
3340
+ },
3341
+ {
3342
+ "epoch": 2.2,
3343
+ "learning_rate": 1e-05,
3344
+ "loss": 0.5344,
3345
+ "step": 556
3346
+ },
3347
+ {
3348
+ "epoch": 2.21,
3349
+ "learning_rate": 1e-05,
3350
+ "loss": 0.5534,
3351
+ "step": 557
3352
+ },
3353
+ {
3354
+ "epoch": 2.21,
3355
+ "learning_rate": 1e-05,
3356
+ "loss": 0.5626,
3357
+ "step": 558
3358
+ },
3359
+ {
3360
+ "epoch": 2.22,
3361
+ "learning_rate": 1e-05,
3362
+ "loss": 0.5467,
3363
+ "step": 559
3364
+ },
3365
+ {
3366
+ "epoch": 2.22,
3367
+ "learning_rate": 1e-05,
3368
+ "loss": 0.5675,
3369
+ "step": 560
3370
+ },
3371
+ {
3372
+ "epoch": 2.22,
3373
+ "learning_rate": 1e-05,
3374
+ "loss": 0.5591,
3375
+ "step": 561
3376
+ },
3377
+ {
3378
+ "epoch": 2.23,
3379
+ "learning_rate": 1e-05,
3380
+ "loss": 0.5474,
3381
+ "step": 562
3382
+ },
3383
+ {
3384
+ "epoch": 2.23,
3385
+ "learning_rate": 1e-05,
3386
+ "loss": 0.5673,
3387
+ "step": 563
3388
+ },
3389
+ {
3390
+ "epoch": 2.24,
3391
+ "learning_rate": 1e-05,
3392
+ "loss": 0.5571,
3393
+ "step": 564
3394
+ },
3395
+ {
3396
+ "epoch": 2.24,
3397
+ "learning_rate": 1e-05,
3398
+ "loss": 0.548,
3399
+ "step": 565
3400
+ },
3401
+ {
3402
+ "epoch": 2.24,
3403
+ "learning_rate": 1e-05,
3404
+ "loss": 0.5444,
3405
+ "step": 566
3406
+ },
3407
+ {
3408
+ "epoch": 2.25,
3409
+ "learning_rate": 1e-05,
3410
+ "loss": 0.5581,
3411
+ "step": 567
3412
+ },
3413
+ {
3414
+ "epoch": 2.25,
3415
+ "learning_rate": 1e-05,
3416
+ "loss": 0.5576,
3417
+ "step": 568
3418
+ },
3419
+ {
3420
+ "epoch": 2.26,
3421
+ "learning_rate": 1e-05,
3422
+ "loss": 0.5431,
3423
+ "step": 569
3424
+ },
3425
+ {
3426
+ "epoch": 2.26,
3427
+ "learning_rate": 1e-05,
3428
+ "loss": 0.5393,
3429
+ "step": 570
3430
+ },
3431
+ {
3432
+ "epoch": 2.26,
3433
+ "learning_rate": 1e-05,
3434
+ "loss": 0.5619,
3435
+ "step": 571
3436
+ },
3437
+ {
3438
+ "epoch": 2.27,
3439
+ "learning_rate": 1e-05,
3440
+ "loss": 0.5674,
3441
+ "step": 572
3442
+ },
3443
+ {
3444
+ "epoch": 2.27,
3445
+ "learning_rate": 1e-05,
3446
+ "loss": 0.5584,
3447
+ "step": 573
3448
+ },
3449
+ {
3450
+ "epoch": 2.28,
3451
+ "learning_rate": 1e-05,
3452
+ "loss": 0.5458,
3453
+ "step": 574
3454
+ },
3455
+ {
3456
+ "epoch": 2.28,
3457
+ "learning_rate": 1e-05,
3458
+ "loss": 0.5539,
3459
+ "step": 575
3460
+ },
3461
+ {
3462
+ "epoch": 2.28,
3463
+ "learning_rate": 1e-05,
3464
+ "loss": 0.5533,
3465
+ "step": 576
3466
+ },
3467
+ {
3468
+ "epoch": 2.29,
3469
+ "learning_rate": 1e-05,
3470
+ "loss": 0.5536,
3471
+ "step": 577
3472
+ },
3473
+ {
3474
+ "epoch": 2.29,
3475
+ "learning_rate": 1e-05,
3476
+ "loss": 0.5232,
3477
+ "step": 578
3478
+ },
3479
+ {
3480
+ "epoch": 2.3,
3481
+ "learning_rate": 1e-05,
3482
+ "loss": 0.5315,
3483
+ "step": 579
3484
+ },
3485
+ {
3486
+ "epoch": 2.3,
3487
+ "learning_rate": 1e-05,
3488
+ "loss": 0.5875,
3489
+ "step": 580
3490
+ },
3491
+ {
3492
+ "epoch": 2.3,
3493
+ "learning_rate": 1e-05,
3494
+ "loss": 0.5412,
3495
+ "step": 581
3496
+ },
3497
+ {
3498
+ "epoch": 2.31,
3499
+ "learning_rate": 1e-05,
3500
+ "loss": 0.5564,
3501
+ "step": 582
3502
+ },
3503
+ {
3504
+ "epoch": 2.31,
3505
+ "learning_rate": 1e-05,
3506
+ "loss": 0.5443,
3507
+ "step": 583
3508
+ },
3509
+ {
3510
+ "epoch": 2.32,
3511
+ "learning_rate": 1e-05,
3512
+ "loss": 0.549,
3513
+ "step": 584
3514
+ },
3515
+ {
3516
+ "epoch": 2.32,
3517
+ "learning_rate": 1e-05,
3518
+ "loss": 0.5666,
3519
+ "step": 585
3520
+ },
3521
+ {
3522
+ "epoch": 2.32,
3523
+ "learning_rate": 1e-05,
3524
+ "loss": 0.5407,
3525
+ "step": 586
3526
+ },
3527
+ {
3528
+ "epoch": 2.33,
3529
+ "learning_rate": 1e-05,
3530
+ "loss": 0.5329,
3531
+ "step": 587
3532
+ },
3533
+ {
3534
+ "epoch": 2.33,
3535
+ "learning_rate": 1e-05,
3536
+ "loss": 0.5517,
3537
+ "step": 588
3538
+ },
3539
+ {
3540
+ "epoch": 2.34,
3541
+ "learning_rate": 1e-05,
3542
+ "loss": 0.5396,
3543
+ "step": 589
3544
+ },
3545
+ {
3546
+ "epoch": 2.34,
3547
+ "learning_rate": 1e-05,
3548
+ "loss": 0.5436,
3549
+ "step": 590
3550
+ },
3551
+ {
3552
+ "epoch": 2.34,
3553
+ "learning_rate": 1e-05,
3554
+ "loss": 0.5372,
3555
+ "step": 591
3556
+ },
3557
+ {
3558
+ "epoch": 2.35,
3559
+ "learning_rate": 1e-05,
3560
+ "loss": 0.5518,
3561
+ "step": 592
3562
+ },
3563
+ {
3564
+ "epoch": 2.35,
3565
+ "learning_rate": 1e-05,
3566
+ "loss": 0.5544,
3567
+ "step": 593
3568
+ },
3569
+ {
3570
+ "epoch": 2.36,
3571
+ "learning_rate": 1e-05,
3572
+ "loss": 0.5577,
3573
+ "step": 594
3574
+ },
3575
+ {
3576
+ "epoch": 2.36,
3577
+ "learning_rate": 1e-05,
3578
+ "loss": 0.54,
3579
+ "step": 595
3580
+ },
3581
+ {
3582
+ "epoch": 2.36,
3583
+ "learning_rate": 1e-05,
3584
+ "loss": 0.5627,
3585
+ "step": 596
3586
+ },
3587
+ {
3588
+ "epoch": 2.37,
3589
+ "learning_rate": 1e-05,
3590
+ "loss": 0.5482,
3591
+ "step": 597
3592
+ },
3593
+ {
3594
+ "epoch": 2.37,
3595
+ "learning_rate": 1e-05,
3596
+ "loss": 0.5386,
3597
+ "step": 598
3598
+ },
3599
+ {
3600
+ "epoch": 2.38,
3601
+ "learning_rate": 1e-05,
3602
+ "loss": 0.5499,
3603
+ "step": 599
3604
+ },
3605
+ {
3606
+ "epoch": 2.38,
3607
+ "learning_rate": 1e-05,
3608
+ "loss": 0.5405,
3609
+ "step": 600
3610
+ },
3611
+ {
3612
+ "epoch": 2.38,
3613
+ "learning_rate": 1e-05,
3614
+ "loss": 0.5397,
3615
+ "step": 601
3616
+ },
3617
+ {
3618
+ "epoch": 2.39,
3619
+ "learning_rate": 1e-05,
3620
+ "loss": 0.5495,
3621
+ "step": 602
3622
+ },
3623
+ {
3624
+ "epoch": 2.39,
3625
+ "learning_rate": 1e-05,
3626
+ "loss": 0.5498,
3627
+ "step": 603
3628
+ },
3629
+ {
3630
+ "epoch": 2.4,
3631
+ "learning_rate": 1e-05,
3632
+ "loss": 0.5459,
3633
+ "step": 604
3634
+ },
3635
+ {
3636
+ "epoch": 2.4,
3637
+ "learning_rate": 1e-05,
3638
+ "loss": 0.5363,
3639
+ "step": 605
3640
+ },
3641
+ {
3642
+ "epoch": 2.4,
3643
+ "learning_rate": 1e-05,
3644
+ "loss": 0.5473,
3645
+ "step": 606
3646
+ },
3647
+ {
3648
+ "epoch": 2.41,
3649
+ "learning_rate": 1e-05,
3650
+ "loss": 0.542,
3651
+ "step": 607
3652
+ },
3653
+ {
3654
+ "epoch": 2.41,
3655
+ "learning_rate": 1e-05,
3656
+ "loss": 0.5471,
3657
+ "step": 608
3658
+ },
3659
+ {
3660
+ "epoch": 2.41,
3661
+ "learning_rate": 1e-05,
3662
+ "loss": 0.5618,
3663
+ "step": 609
3664
+ },
3665
+ {
3666
+ "epoch": 2.42,
3667
+ "learning_rate": 1e-05,
3668
+ "loss": 0.5511,
3669
+ "step": 610
3670
+ },
3671
+ {
3672
+ "epoch": 2.42,
3673
+ "learning_rate": 1e-05,
3674
+ "loss": 0.5374,
3675
+ "step": 611
3676
+ },
3677
+ {
3678
+ "epoch": 2.43,
3679
+ "learning_rate": 1e-05,
3680
+ "loss": 0.5537,
3681
+ "step": 612
3682
+ },
3683
+ {
3684
+ "epoch": 2.43,
3685
+ "learning_rate": 1e-05,
3686
+ "loss": 0.553,
3687
+ "step": 613
3688
+ },
3689
+ {
3690
+ "epoch": 2.43,
3691
+ "learning_rate": 1e-05,
3692
+ "loss": 0.5595,
3693
+ "step": 614
3694
+ },
3695
+ {
3696
+ "epoch": 2.44,
3697
+ "learning_rate": 1e-05,
3698
+ "loss": 0.5381,
3699
+ "step": 615
3700
+ },
3701
+ {
3702
+ "epoch": 2.44,
3703
+ "learning_rate": 1e-05,
3704
+ "loss": 0.5456,
3705
+ "step": 616
3706
+ },
3707
+ {
3708
+ "epoch": 2.45,
3709
+ "learning_rate": 1e-05,
3710
+ "loss": 0.5441,
3711
+ "step": 617
3712
+ },
3713
+ {
3714
+ "epoch": 2.45,
3715
+ "learning_rate": 1e-05,
3716
+ "loss": 0.5294,
3717
+ "step": 618
3718
+ },
3719
+ {
3720
+ "epoch": 2.45,
3721
+ "learning_rate": 1e-05,
3722
+ "loss": 0.5503,
3723
+ "step": 619
3724
+ },
3725
+ {
3726
+ "epoch": 2.46,
3727
+ "learning_rate": 1e-05,
3728
+ "loss": 0.5405,
3729
+ "step": 620
3730
+ },
3731
+ {
3732
+ "epoch": 2.46,
3733
+ "learning_rate": 1e-05,
3734
+ "loss": 0.5574,
3735
+ "step": 621
3736
+ },
3737
+ {
3738
+ "epoch": 2.47,
3739
+ "learning_rate": 1e-05,
3740
+ "loss": 0.541,
3741
+ "step": 622
3742
+ },
3743
+ {
3744
+ "epoch": 2.47,
3745
+ "learning_rate": 1e-05,
3746
+ "loss": 0.5105,
3747
+ "step": 623
3748
+ },
3749
+ {
3750
+ "epoch": 2.47,
3751
+ "learning_rate": 1e-05,
3752
+ "loss": 0.5537,
3753
+ "step": 624
3754
+ },
3755
+ {
3756
+ "epoch": 2.48,
3757
+ "learning_rate": 1e-05,
3758
+ "loss": 0.5471,
3759
+ "step": 625
3760
+ },
3761
+ {
3762
+ "epoch": 2.48,
3763
+ "learning_rate": 1e-05,
3764
+ "loss": 0.5429,
3765
+ "step": 626
3766
+ },
3767
+ {
3768
+ "epoch": 2.49,
3769
+ "learning_rate": 1e-05,
3770
+ "loss": 0.531,
3771
+ "step": 627
3772
+ },
3773
+ {
3774
+ "epoch": 2.49,
3775
+ "learning_rate": 1e-05,
3776
+ "loss": 0.5287,
3777
+ "step": 628
3778
+ },
3779
+ {
3780
+ "epoch": 2.49,
3781
+ "learning_rate": 1e-05,
3782
+ "loss": 0.557,
3783
+ "step": 629
3784
+ },
3785
+ {
3786
+ "epoch": 2.5,
3787
+ "learning_rate": 1e-05,
3788
+ "loss": 0.5482,
3789
+ "step": 630
3790
+ },
3791
+ {
3792
+ "epoch": 2.5,
3793
+ "learning_rate": 1e-05,
3794
+ "loss": 0.5501,
3795
+ "step": 631
3796
+ },
3797
+ {
3798
+ "epoch": 2.51,
3799
+ "learning_rate": 1e-05,
3800
+ "loss": 0.554,
3801
+ "step": 632
3802
+ },
3803
+ {
3804
+ "epoch": 2.51,
3805
+ "learning_rate": 1e-05,
3806
+ "loss": 0.538,
3807
+ "step": 633
3808
+ },
3809
+ {
3810
+ "epoch": 2.51,
3811
+ "learning_rate": 1e-05,
3812
+ "loss": 0.5172,
3813
+ "step": 634
3814
+ },
3815
+ {
3816
+ "epoch": 2.52,
3817
+ "learning_rate": 1e-05,
3818
+ "loss": 0.5335,
3819
+ "step": 635
3820
+ },
3821
+ {
3822
+ "epoch": 2.52,
3823
+ "learning_rate": 1e-05,
3824
+ "loss": 0.5213,
3825
+ "step": 636
3826
+ },
3827
+ {
3828
+ "epoch": 2.53,
3829
+ "learning_rate": 1e-05,
3830
+ "loss": 0.5617,
3831
+ "step": 637
3832
+ },
3833
+ {
3834
+ "epoch": 2.53,
3835
+ "learning_rate": 1e-05,
3836
+ "loss": 0.5499,
3837
+ "step": 638
3838
+ },
3839
+ {
3840
+ "epoch": 2.53,
3841
+ "learning_rate": 1e-05,
3842
+ "loss": 0.5367,
3843
+ "step": 639
3844
+ },
3845
+ {
3846
+ "epoch": 2.54,
3847
+ "learning_rate": 1e-05,
3848
+ "loss": 0.5475,
3849
+ "step": 640
3850
+ },
3851
+ {
3852
+ "epoch": 2.54,
3853
+ "learning_rate": 1e-05,
3854
+ "loss": 0.5428,
3855
+ "step": 641
3856
+ },
3857
+ {
3858
+ "epoch": 2.55,
3859
+ "learning_rate": 1e-05,
3860
+ "loss": 0.5395,
3861
+ "step": 642
3862
+ },
3863
+ {
3864
+ "epoch": 2.55,
3865
+ "learning_rate": 1e-05,
3866
+ "loss": 0.5252,
3867
+ "step": 643
3868
+ },
3869
+ {
3870
+ "epoch": 2.55,
3871
+ "learning_rate": 1e-05,
3872
+ "loss": 0.5394,
3873
+ "step": 644
3874
+ },
3875
+ {
3876
+ "epoch": 2.56,
3877
+ "learning_rate": 1e-05,
3878
+ "loss": 0.5706,
3879
+ "step": 645
3880
+ },
3881
+ {
3882
+ "epoch": 2.56,
3883
+ "learning_rate": 1e-05,
3884
+ "loss": 0.5345,
3885
+ "step": 646
3886
+ },
3887
+ {
3888
+ "epoch": 2.57,
3889
+ "learning_rate": 1e-05,
3890
+ "loss": 0.5399,
3891
+ "step": 647
3892
+ },
3893
+ {
3894
+ "epoch": 2.57,
3895
+ "learning_rate": 1e-05,
3896
+ "loss": 0.5376,
3897
+ "step": 648
3898
+ },
3899
+ {
3900
+ "epoch": 2.57,
3901
+ "learning_rate": 1e-05,
3902
+ "loss": 0.5415,
3903
+ "step": 649
3904
+ },
3905
+ {
3906
+ "epoch": 2.58,
3907
+ "learning_rate": 1e-05,
3908
+ "loss": 0.5685,
3909
+ "step": 650
3910
+ },
3911
+ {
3912
+ "epoch": 2.58,
3913
+ "learning_rate": 1e-05,
3914
+ "loss": 0.5511,
3915
+ "step": 651
3916
+ },
3917
+ {
3918
+ "epoch": 2.59,
3919
+ "learning_rate": 1e-05,
3920
+ "loss": 0.5589,
3921
+ "step": 652
3922
+ },
3923
+ {
3924
+ "epoch": 2.59,
3925
+ "learning_rate": 1e-05,
3926
+ "loss": 0.5149,
3927
+ "step": 653
3928
+ },
3929
+ {
3930
+ "epoch": 2.59,
3931
+ "learning_rate": 1e-05,
3932
+ "loss": 0.5387,
3933
+ "step": 654
3934
+ },
3935
+ {
3936
+ "epoch": 2.6,
3937
+ "learning_rate": 1e-05,
3938
+ "loss": 0.5729,
3939
+ "step": 655
3940
+ },
3941
+ {
3942
+ "epoch": 2.6,
3943
+ "learning_rate": 1e-05,
3944
+ "loss": 0.5485,
3945
+ "step": 656
3946
+ },
3947
+ {
3948
+ "epoch": 2.61,
3949
+ "learning_rate": 1e-05,
3950
+ "loss": 0.5266,
3951
+ "step": 657
3952
+ },
3953
+ {
3954
+ "epoch": 2.61,
3955
+ "learning_rate": 1e-05,
3956
+ "loss": 0.5722,
3957
+ "step": 658
3958
+ },
3959
+ {
3960
+ "epoch": 2.61,
3961
+ "learning_rate": 1e-05,
3962
+ "loss": 0.5369,
3963
+ "step": 659
3964
+ },
3965
+ {
3966
+ "epoch": 2.62,
3967
+ "learning_rate": 1e-05,
3968
+ "loss": 0.5295,
3969
+ "step": 660
3970
+ },
3971
+ {
3972
+ "epoch": 2.62,
3973
+ "learning_rate": 1e-05,
3974
+ "loss": 0.5445,
3975
+ "step": 661
3976
+ },
3977
+ {
3978
+ "epoch": 2.63,
3979
+ "learning_rate": 1e-05,
3980
+ "loss": 0.5205,
3981
+ "step": 662
3982
+ },
3983
+ {
3984
+ "epoch": 2.63,
3985
+ "learning_rate": 1e-05,
3986
+ "loss": 0.5436,
3987
+ "step": 663
3988
+ },
3989
+ {
3990
+ "epoch": 2.63,
3991
+ "learning_rate": 1e-05,
3992
+ "loss": 0.5454,
3993
+ "step": 664
3994
+ },
3995
+ {
3996
+ "epoch": 2.64,
3997
+ "learning_rate": 1e-05,
3998
+ "loss": 0.5311,
3999
+ "step": 665
4000
+ },
4001
+ {
4002
+ "epoch": 2.64,
4003
+ "learning_rate": 1e-05,
4004
+ "loss": 0.5438,
4005
+ "step": 666
4006
+ },
4007
+ {
4008
+ "epoch": 2.64,
4009
+ "learning_rate": 1e-05,
4010
+ "loss": 0.5435,
4011
+ "step": 667
4012
+ },
4013
+ {
4014
+ "epoch": 2.65,
4015
+ "learning_rate": 1e-05,
4016
+ "loss": 0.522,
4017
+ "step": 668
4018
+ },
4019
+ {
4020
+ "epoch": 2.65,
4021
+ "learning_rate": 1e-05,
4022
+ "loss": 0.5431,
4023
+ "step": 669
4024
+ },
4025
+ {
4026
+ "epoch": 2.66,
4027
+ "learning_rate": 1e-05,
4028
+ "loss": 0.5422,
4029
+ "step": 670
4030
+ },
4031
+ {
4032
+ "epoch": 2.66,
4033
+ "learning_rate": 1e-05,
4034
+ "loss": 0.5362,
4035
+ "step": 671
4036
+ },
4037
+ {
4038
+ "epoch": 2.66,
4039
+ "learning_rate": 1e-05,
4040
+ "loss": 0.5299,
4041
+ "step": 672
4042
+ },
4043
+ {
4044
+ "epoch": 2.67,
4045
+ "learning_rate": 1e-05,
4046
+ "loss": 0.5314,
4047
+ "step": 673
4048
+ },
4049
+ {
4050
+ "epoch": 2.67,
4051
+ "learning_rate": 1e-05,
4052
+ "loss": 0.5252,
4053
+ "step": 674
4054
+ },
4055
+ {
4056
+ "epoch": 2.68,
4057
+ "learning_rate": 1e-05,
4058
+ "loss": 0.5268,
4059
+ "step": 675
4060
+ },
4061
+ {
4062
+ "epoch": 2.68,
4063
+ "learning_rate": 1e-05,
4064
+ "loss": 0.5437,
4065
+ "step": 676
4066
+ },
4067
+ {
4068
+ "epoch": 2.68,
4069
+ "learning_rate": 1e-05,
4070
+ "loss": 0.5238,
4071
+ "step": 677
4072
+ },
4073
+ {
4074
+ "epoch": 2.69,
4075
+ "learning_rate": 1e-05,
4076
+ "loss": 0.5323,
4077
+ "step": 678
4078
+ },
4079
+ {
4080
+ "epoch": 2.69,
4081
+ "learning_rate": 1e-05,
4082
+ "loss": 0.5465,
4083
+ "step": 679
4084
+ },
4085
+ {
4086
+ "epoch": 2.7,
4087
+ "learning_rate": 1e-05,
4088
+ "loss": 0.5245,
4089
+ "step": 680
4090
+ },
4091
+ {
4092
+ "epoch": 2.7,
4093
+ "learning_rate": 1e-05,
4094
+ "loss": 0.5107,
4095
+ "step": 681
4096
+ },
4097
+ {
4098
+ "epoch": 2.7,
4099
+ "learning_rate": 1e-05,
4100
+ "loss": 0.5385,
4101
+ "step": 682
4102
+ },
4103
+ {
4104
+ "epoch": 2.71,
4105
+ "learning_rate": 1e-05,
4106
+ "loss": 0.5521,
4107
+ "step": 683
4108
+ },
4109
+ {
4110
+ "epoch": 2.71,
4111
+ "learning_rate": 1e-05,
4112
+ "loss": 0.5222,
4113
+ "step": 684
4114
+ },
4115
+ {
4116
+ "epoch": 2.72,
4117
+ "learning_rate": 1e-05,
4118
+ "loss": 0.5173,
4119
+ "step": 685
4120
+ },
4121
+ {
4122
+ "epoch": 2.72,
4123
+ "learning_rate": 1e-05,
4124
+ "loss": 0.5167,
4125
+ "step": 686
4126
+ },
4127
+ {
4128
+ "epoch": 2.72,
4129
+ "learning_rate": 1e-05,
4130
+ "loss": 0.5315,
4131
+ "step": 687
4132
+ },
4133
+ {
4134
+ "epoch": 2.73,
4135
+ "learning_rate": 1e-05,
4136
+ "loss": 0.5206,
4137
+ "step": 688
4138
+ },
4139
+ {
4140
+ "epoch": 2.73,
4141
+ "learning_rate": 1e-05,
4142
+ "loss": 0.5186,
4143
+ "step": 689
4144
+ },
4145
+ {
4146
+ "epoch": 2.74,
4147
+ "learning_rate": 1e-05,
4148
+ "loss": 0.5275,
4149
+ "step": 690
4150
+ },
4151
+ {
4152
+ "epoch": 2.74,
4153
+ "learning_rate": 1e-05,
4154
+ "loss": 0.5436,
4155
+ "step": 691
4156
+ },
4157
+ {
4158
+ "epoch": 2.74,
4159
+ "learning_rate": 1e-05,
4160
+ "loss": 0.5088,
4161
+ "step": 692
4162
+ },
4163
+ {
4164
+ "epoch": 2.75,
4165
+ "learning_rate": 1e-05,
4166
+ "loss": 0.5329,
4167
+ "step": 693
4168
+ },
4169
+ {
4170
+ "epoch": 2.75,
4171
+ "learning_rate": 1e-05,
4172
+ "loss": 0.5273,
4173
+ "step": 694
4174
+ },
4175
+ {
4176
+ "epoch": 2.76,
4177
+ "learning_rate": 1e-05,
4178
+ "loss": 0.5446,
4179
+ "step": 695
4180
+ },
4181
+ {
4182
+ "epoch": 2.76,
4183
+ "learning_rate": 1e-05,
4184
+ "loss": 0.5304,
4185
+ "step": 696
4186
+ },
4187
+ {
4188
+ "epoch": 2.76,
4189
+ "learning_rate": 1e-05,
4190
+ "loss": 0.5185,
4191
+ "step": 697
4192
+ },
4193
+ {
4194
+ "epoch": 2.77,
4195
+ "learning_rate": 1e-05,
4196
+ "loss": 0.5256,
4197
+ "step": 698
4198
+ },
4199
+ {
4200
+ "epoch": 2.77,
4201
+ "learning_rate": 1e-05,
4202
+ "loss": 0.5121,
4203
+ "step": 699
4204
+ },
4205
+ {
4206
+ "epoch": 2.78,
4207
+ "learning_rate": 1e-05,
4208
+ "loss": 0.519,
4209
+ "step": 700
4210
+ },
4211
+ {
4212
+ "epoch": 2.78,
4213
+ "learning_rate": 1e-05,
4214
+ "loss": 0.5466,
4215
+ "step": 701
4216
+ },
4217
+ {
4218
+ "epoch": 2.78,
4219
+ "learning_rate": 1e-05,
4220
+ "loss": 0.5235,
4221
+ "step": 702
4222
+ },
4223
+ {
4224
+ "epoch": 2.79,
4225
+ "learning_rate": 1e-05,
4226
+ "loss": 0.5414,
4227
+ "step": 703
4228
+ },
4229
+ {
4230
+ "epoch": 2.79,
4231
+ "learning_rate": 1e-05,
4232
+ "loss": 0.5349,
4233
+ "step": 704
4234
+ },
4235
+ {
4236
+ "epoch": 2.8,
4237
+ "learning_rate": 1e-05,
4238
+ "loss": 0.528,
4239
+ "step": 705
4240
+ },
4241
+ {
4242
+ "epoch": 2.8,
4243
+ "learning_rate": 1e-05,
4244
+ "loss": 0.5402,
4245
+ "step": 706
4246
+ },
4247
+ {
4248
+ "epoch": 2.8,
4249
+ "learning_rate": 1e-05,
4250
+ "loss": 0.537,
4251
+ "step": 707
4252
+ },
4253
+ {
4254
+ "epoch": 2.81,
4255
+ "learning_rate": 1e-05,
4256
+ "loss": 0.5009,
4257
+ "step": 708
4258
+ },
4259
+ {
4260
+ "epoch": 2.81,
4261
+ "learning_rate": 1e-05,
4262
+ "loss": 0.5449,
4263
+ "step": 709
4264
+ },
4265
+ {
4266
+ "epoch": 2.82,
4267
+ "learning_rate": 1e-05,
4268
+ "loss": 0.5501,
4269
+ "step": 710
4270
+ },
4271
+ {
4272
+ "epoch": 2.82,
4273
+ "learning_rate": 1e-05,
4274
+ "loss": 0.5344,
4275
+ "step": 711
4276
+ },
4277
+ {
4278
+ "epoch": 2.82,
4279
+ "learning_rate": 1e-05,
4280
+ "loss": 0.5179,
4281
+ "step": 712
4282
+ },
4283
+ {
4284
+ "epoch": 2.83,
4285
+ "learning_rate": 1e-05,
4286
+ "loss": 0.5318,
4287
+ "step": 713
4288
+ },
4289
+ {
4290
+ "epoch": 2.83,
4291
+ "learning_rate": 1e-05,
4292
+ "loss": 0.5242,
4293
+ "step": 714
4294
+ },
4295
+ {
4296
+ "epoch": 2.84,
4297
+ "learning_rate": 1e-05,
4298
+ "loss": 0.528,
4299
+ "step": 715
4300
+ },
4301
+ {
4302
+ "epoch": 2.84,
4303
+ "learning_rate": 1e-05,
4304
+ "loss": 0.5384,
4305
+ "step": 716
4306
+ },
4307
+ {
4308
+ "epoch": 2.84,
4309
+ "learning_rate": 1e-05,
4310
+ "loss": 0.514,
4311
+ "step": 717
4312
+ },
4313
+ {
4314
+ "epoch": 2.85,
4315
+ "learning_rate": 1e-05,
4316
+ "loss": 0.5249,
4317
+ "step": 718
4318
+ },
4319
+ {
4320
+ "epoch": 2.85,
4321
+ "learning_rate": 1e-05,
4322
+ "loss": 0.5165,
4323
+ "step": 719
4324
+ },
4325
+ {
4326
+ "epoch": 2.86,
4327
+ "learning_rate": 1e-05,
4328
+ "loss": 0.5279,
4329
+ "step": 720
4330
+ },
4331
+ {
4332
+ "epoch": 2.86,
4333
+ "learning_rate": 1e-05,
4334
+ "loss": 0.5472,
4335
+ "step": 721
4336
+ },
4337
+ {
4338
+ "epoch": 2.86,
4339
+ "learning_rate": 1e-05,
4340
+ "loss": 0.5097,
4341
+ "step": 722
4342
+ },
4343
+ {
4344
+ "epoch": 2.87,
4345
+ "learning_rate": 1e-05,
4346
+ "loss": 0.5237,
4347
+ "step": 723
4348
+ },
4349
+ {
4350
+ "epoch": 2.87,
4351
+ "learning_rate": 1e-05,
4352
+ "loss": 0.511,
4353
+ "step": 724
4354
+ },
4355
+ {
4356
+ "epoch": 2.87,
4357
+ "learning_rate": 1e-05,
4358
+ "loss": 0.5065,
4359
+ "step": 725
4360
+ },
4361
+ {
4362
+ "epoch": 2.88,
4363
+ "learning_rate": 1e-05,
4364
+ "loss": 0.5167,
4365
+ "step": 726
4366
+ },
4367
+ {
4368
+ "epoch": 2.88,
4369
+ "learning_rate": 1e-05,
4370
+ "loss": 0.5237,
4371
+ "step": 727
4372
+ },
4373
+ {
4374
+ "epoch": 2.89,
4375
+ "learning_rate": 1e-05,
4376
+ "loss": 0.5323,
4377
+ "step": 728
4378
+ },
4379
+ {
4380
+ "epoch": 2.89,
4381
+ "learning_rate": 1e-05,
4382
+ "loss": 0.5087,
4383
+ "step": 729
4384
+ },
4385
+ {
4386
+ "epoch": 2.89,
4387
+ "learning_rate": 1e-05,
4388
+ "loss": 0.5005,
4389
+ "step": 730
4390
+ },
4391
+ {
4392
+ "epoch": 2.9,
4393
+ "learning_rate": 1e-05,
4394
+ "loss": 0.5233,
4395
+ "step": 731
4396
+ },
4397
+ {
4398
+ "epoch": 2.9,
4399
+ "learning_rate": 1e-05,
4400
+ "loss": 0.5199,
4401
+ "step": 732
4402
+ },
4403
+ {
4404
+ "epoch": 2.91,
4405
+ "learning_rate": 1e-05,
4406
+ "loss": 0.5423,
4407
+ "step": 733
4408
+ },
4409
+ {
4410
+ "epoch": 2.91,
4411
+ "learning_rate": 1e-05,
4412
+ "loss": 0.5275,
4413
+ "step": 734
4414
+ },
4415
+ {
4416
+ "epoch": 2.91,
4417
+ "learning_rate": 1e-05,
4418
+ "loss": 0.5285,
4419
+ "step": 735
4420
+ },
4421
+ {
4422
+ "epoch": 2.92,
4423
+ "learning_rate": 1e-05,
4424
+ "loss": 0.5046,
4425
+ "step": 736
4426
+ },
4427
+ {
4428
+ "epoch": 2.92,
4429
+ "learning_rate": 1e-05,
4430
+ "loss": 0.513,
4431
+ "step": 737
4432
+ },
4433
+ {
4434
+ "epoch": 2.93,
4435
+ "learning_rate": 1e-05,
4436
+ "loss": 0.5357,
4437
+ "step": 738
4438
+ },
4439
+ {
4440
+ "epoch": 2.93,
4441
+ "learning_rate": 1e-05,
4442
+ "loss": 0.5227,
4443
+ "step": 739
4444
+ },
4445
+ {
4446
+ "epoch": 2.93,
4447
+ "learning_rate": 1e-05,
4448
+ "loss": 0.5265,
4449
+ "step": 740
4450
+ },
4451
+ {
4452
+ "epoch": 2.94,
4453
+ "learning_rate": 1e-05,
4454
+ "loss": 0.5183,
4455
+ "step": 741
4456
+ },
4457
+ {
4458
+ "epoch": 2.94,
4459
+ "learning_rate": 1e-05,
4460
+ "loss": 0.495,
4461
+ "step": 742
4462
+ },
4463
+ {
4464
+ "epoch": 2.95,
4465
+ "learning_rate": 1e-05,
4466
+ "loss": 0.5448,
4467
+ "step": 743
4468
+ },
4469
+ {
4470
+ "epoch": 2.95,
4471
+ "learning_rate": 1e-05,
4472
+ "loss": 0.5168,
4473
+ "step": 744
4474
+ },
4475
+ {
4476
+ "epoch": 2.95,
4477
+ "learning_rate": 1e-05,
4478
+ "loss": 0.5244,
4479
+ "step": 745
4480
+ },
4481
+ {
4482
+ "epoch": 2.96,
4483
+ "learning_rate": 1e-05,
4484
+ "loss": 0.518,
4485
+ "step": 746
4486
+ },
4487
+ {
4488
+ "epoch": 2.96,
4489
+ "learning_rate": 1e-05,
4490
+ "loss": 0.5292,
4491
+ "step": 747
4492
+ },
4493
+ {
4494
+ "epoch": 2.97,
4495
+ "learning_rate": 1e-05,
4496
+ "loss": 0.5287,
4497
+ "step": 748
4498
+ },
4499
+ {
4500
+ "epoch": 2.97,
4501
+ "learning_rate": 1e-05,
4502
+ "loss": 0.5442,
4503
+ "step": 749
4504
+ },
4505
+ {
4506
+ "epoch": 2.97,
4507
+ "learning_rate": 1e-05,
4508
+ "loss": 0.5333,
4509
+ "step": 750
4510
+ },
4511
+ {
4512
+ "epoch": 2.98,
4513
+ "learning_rate": 1e-05,
4514
+ "loss": 0.5272,
4515
+ "step": 751
4516
+ },
4517
+ {
4518
+ "epoch": 2.98,
4519
+ "learning_rate": 1e-05,
4520
+ "loss": 0.5297,
4521
+ "step": 752
4522
+ },
4523
+ {
4524
+ "epoch": 2.99,
4525
+ "learning_rate": 1e-05,
4526
+ "loss": 0.502,
4527
+ "step": 753
4528
+ },
4529
+ {
4530
+ "epoch": 2.99,
4531
+ "learning_rate": 1e-05,
4532
+ "loss": 0.5218,
4533
+ "step": 754
4534
+ },
4535
+ {
4536
+ "epoch": 2.99,
4537
+ "learning_rate": 1e-05,
4538
+ "loss": 0.5203,
4539
+ "step": 755
4540
+ },
4541
+ {
4542
+ "epoch": 3.0,
4543
+ "learning_rate": 1e-05,
4544
+ "loss": 0.5295,
4545
+ "step": 756
4546
+ }
4547
+ ],
4548
+ "logging_steps": 1.0,
4549
+ "max_steps": 756,
4550
+ "num_train_epochs": 3,
4551
+ "save_steps": 252,
4552
+ "total_flos": 9788881423564800.0,
4553
+ "trial_name": null,
4554
+ "trial_params": null
4555
+ }
training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:70dd88551f024346753c69f057632e682e2cdf08f713cc720c557d6a1df5c600
3
+ size 6331
zero_to_fp32.py ADDED
@@ -0,0 +1,587 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python
2
+
3
+ # Copyright (c) Microsoft Corporation.
4
+ # SPDX-License-Identifier: Apache-2.0
5
+
6
+ # DeepSpeed Team
7
+
8
+ # This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
9
+ # copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
10
+ # the future. Once extracted, the weights don't require DeepSpeed and can be used in any
11
+ # application.
12
+ #
13
+ # example: python zero_to_fp32.py . pytorch_model.bin
14
+
15
+ import argparse
16
+ import torch
17
+ import glob
18
+ import math
19
+ import os
20
+ import re
21
+ from collections import OrderedDict
22
+ from dataclasses import dataclass
23
+
24
+ # while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
25
+ # DeepSpeed data structures it has to be available in the current python environment.
26
+ from deepspeed.utils import logger
27
+ from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
28
+ FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
29
+ FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
30
+
31
+
32
+ @dataclass
33
+ class zero_model_state:
34
+ buffers: dict()
35
+ param_shapes: dict()
36
+ shared_params: list
37
+ ds_version: int
38
+ frozen_param_shapes: dict()
39
+ frozen_param_fragments: dict()
40
+
41
+
42
+ debug = 0
43
+
44
+ # load to cpu
45
+ device = torch.device('cpu')
46
+
47
+
48
+ def atoi(text):
49
+ return int(text) if text.isdigit() else text
50
+
51
+
52
+ def natural_keys(text):
53
+ '''
54
+ alist.sort(key=natural_keys) sorts in human order
55
+ http://nedbatchelder.com/blog/200712/human_sorting.html
56
+ (See Toothy's implementation in the comments)
57
+ '''
58
+ return [atoi(c) for c in re.split(r'(\d+)', text)]
59
+
60
+
61
+ def get_model_state_file(checkpoint_dir, zero_stage):
62
+ if not os.path.isdir(checkpoint_dir):
63
+ raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
64
+
65
+ # there should be only one file
66
+ if zero_stage <= 2:
67
+ file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
68
+ elif zero_stage == 3:
69
+ file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
70
+
71
+ if not os.path.exists(file):
72
+ raise FileNotFoundError(f"can't find model states file at '{file}'")
73
+
74
+ return file
75
+
76
+
77
+ def get_checkpoint_files(checkpoint_dir, glob_pattern):
78
+ # XXX: need to test that this simple glob rule works for multi-node setup too
79
+ ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
80
+
81
+ if len(ckpt_files) == 0:
82
+ raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
83
+
84
+ return ckpt_files
85
+
86
+
87
+ def get_optim_files(checkpoint_dir):
88
+ return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
89
+
90
+
91
+ def get_model_state_files(checkpoint_dir):
92
+ return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
93
+
94
+
95
+ def parse_model_states(files):
96
+ zero_model_states = []
97
+ for file in files:
98
+ state_dict = torch.load(file, map_location=device)
99
+
100
+ if BUFFER_NAMES not in state_dict:
101
+ raise ValueError(f"{file} is not a model state checkpoint")
102
+ buffer_names = state_dict[BUFFER_NAMES]
103
+ if debug:
104
+ print("Found buffers:", buffer_names)
105
+
106
+ # recover just the buffers while restoring them to fp32 if they were saved in fp16
107
+ buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
108
+ param_shapes = state_dict[PARAM_SHAPES]
109
+
110
+ # collect parameters that are included in param_shapes
111
+ param_names = []
112
+ for s in param_shapes:
113
+ for name in s.keys():
114
+ param_names.append(name)
115
+
116
+ # update with frozen parameters
117
+ frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
118
+ if frozen_param_shapes is not None:
119
+ if debug:
120
+ print(f"Found frozen_param_shapes: {frozen_param_shapes}")
121
+ param_names += list(frozen_param_shapes.keys())
122
+
123
+ # handle shared params
124
+ shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
125
+
126
+ ds_version = state_dict.get(DS_VERSION, None)
127
+
128
+ frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
129
+
130
+ z_model_state = zero_model_state(buffers=buffers,
131
+ param_shapes=param_shapes,
132
+ shared_params=shared_params,
133
+ ds_version=ds_version,
134
+ frozen_param_shapes=frozen_param_shapes,
135
+ frozen_param_fragments=frozen_param_fragments)
136
+ zero_model_states.append(z_model_state)
137
+
138
+ return zero_model_states
139
+
140
+
141
+ def parse_optim_states(files, ds_checkpoint_dir):
142
+
143
+ total_files = len(files)
144
+ state_dicts = []
145
+ for f in files:
146
+ state_dict = torch.load(f, map_location=device)
147
+ # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
148
+ # and also handle the case where it was already removed by another helper script
149
+ state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
150
+ state_dicts.append(state_dict)
151
+
152
+ if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]:
153
+ raise ValueError(f"{files[0]} is not a zero checkpoint")
154
+ zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
155
+ world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
156
+
157
+ # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
158
+ # parameters can be different from data parallelism for non-expert parameters. So we can just
159
+ # use the max of the partition_count to get the dp world_size.
160
+
161
+ if type(world_size) is list:
162
+ world_size = max(world_size)
163
+
164
+ if world_size != total_files:
165
+ raise ValueError(
166
+ f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
167
+ "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
168
+ )
169
+
170
+ # the groups are named differently in each stage
171
+ if zero_stage <= 2:
172
+ fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
173
+ elif zero_stage == 3:
174
+ fp32_groups_key = FP32_FLAT_GROUPS
175
+ else:
176
+ raise ValueError(f"unknown zero stage {zero_stage}")
177
+
178
+ if zero_stage <= 2:
179
+ fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
180
+ elif zero_stage == 3:
181
+ # if there is more than one param group, there will be multiple flattened tensors - one
182
+ # flattened tensor per group - for simplicity merge them into a single tensor
183
+ #
184
+ # XXX: could make the script more memory efficient for when there are multiple groups - it
185
+ # will require matching the sub-lists of param_shapes for each param group flattened tensor
186
+
187
+ fp32_flat_groups = [
188
+ torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts))
189
+ ]
190
+
191
+ return zero_stage, world_size, fp32_flat_groups
192
+
193
+
194
+ def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir):
195
+ """
196
+ Returns fp32 state_dict reconstructed from ds checkpoint
197
+
198
+ Args:
199
+ - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
200
+
201
+ """
202
+ print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
203
+
204
+ optim_files = get_optim_files(ds_checkpoint_dir)
205
+ zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
206
+ print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
207
+
208
+ model_files = get_model_state_files(ds_checkpoint_dir)
209
+
210
+ zero_model_states = parse_model_states(model_files)
211
+ print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
212
+
213
+ if zero_stage <= 2:
214
+ return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states)
215
+ elif zero_stage == 3:
216
+ return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states)
217
+
218
+
219
+ def _zero2_merge_frozen_params(state_dict, zero_model_states):
220
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
221
+ return
222
+
223
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
224
+ frozen_param_fragments = zero_model_states[0].frozen_param_fragments
225
+
226
+ if debug:
227
+ num_elem = sum(s.numel() for s in frozen_param_shapes.values())
228
+ print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
229
+
230
+ wanted_params = len(frozen_param_shapes)
231
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
232
+ avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
233
+ print(f'Frozen params: Have {avail_numel} numels to process.')
234
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
235
+
236
+ total_params = 0
237
+ total_numel = 0
238
+ for name, shape in frozen_param_shapes.items():
239
+ total_params += 1
240
+ unpartitioned_numel = shape.numel()
241
+ total_numel += unpartitioned_numel
242
+
243
+ state_dict[name] = frozen_param_fragments[name]
244
+
245
+ if debug:
246
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
247
+
248
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
249
+
250
+
251
+ def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
252
+ param_shapes = zero_model_states[0].param_shapes
253
+
254
+ # Reconstruction protocol:
255
+ #
256
+ # XXX: document this
257
+
258
+ if debug:
259
+ for i in range(world_size):
260
+ for j in range(len(fp32_flat_groups[0])):
261
+ print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
262
+
263
+ # XXX: memory usage doubles here (zero2)
264
+ num_param_groups = len(fp32_flat_groups[0])
265
+ merged_single_partition_of_fp32_groups = []
266
+ for i in range(num_param_groups):
267
+ merged_partitions = [sd[i] for sd in fp32_flat_groups]
268
+ full_single_fp32_vector = torch.cat(merged_partitions, 0)
269
+ merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
270
+ avail_numel = sum(
271
+ [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
272
+
273
+ if debug:
274
+ wanted_params = sum([len(shapes) for shapes in param_shapes])
275
+ wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
276
+ # not asserting if there is a mismatch due to possible padding
277
+ print(f"Have {avail_numel} numels to process.")
278
+ print(f"Need {wanted_numel} numels in {wanted_params} params.")
279
+
280
+ # params
281
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
282
+ # out-of-core computing solution
283
+ total_numel = 0
284
+ total_params = 0
285
+ for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
286
+ offset = 0
287
+ avail_numel = full_single_fp32_vector.numel()
288
+ for name, shape in shapes.items():
289
+
290
+ unpartitioned_numel = shape.numel()
291
+ total_numel += unpartitioned_numel
292
+ total_params += 1
293
+
294
+ if debug:
295
+ print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
296
+ state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
297
+ offset += unpartitioned_numel
298
+
299
+ # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
300
+ # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
301
+ # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
302
+ # live optimizer object, so we are checking that the numbers are within the right range
303
+ align_to = 2 * world_size
304
+
305
+ def zero2_align(x):
306
+ return align_to * math.ceil(x / align_to)
307
+
308
+ if debug:
309
+ print(f"original offset={offset}, avail_numel={avail_numel}")
310
+
311
+ offset = zero2_align(offset)
312
+ avail_numel = zero2_align(avail_numel)
313
+
314
+ if debug:
315
+ print(f"aligned offset={offset}, avail_numel={avail_numel}")
316
+
317
+ # Sanity check
318
+ if offset != avail_numel:
319
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
320
+
321
+ print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
322
+
323
+
324
+ def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states):
325
+ state_dict = OrderedDict()
326
+
327
+ # buffers
328
+ buffers = zero_model_states[0].buffers
329
+ state_dict.update(buffers)
330
+ if debug:
331
+ print(f"added {len(buffers)} buffers")
332
+
333
+ _zero2_merge_frozen_params(state_dict, zero_model_states)
334
+
335
+ _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
336
+
337
+ # recover shared parameters
338
+ for pair in zero_model_states[0].shared_params:
339
+ if pair[1] in state_dict:
340
+ state_dict[pair[0]] = state_dict[pair[1]]
341
+
342
+ return state_dict
343
+
344
+
345
+ def zero3_partitioned_param_info(unpartitioned_numel, world_size):
346
+ remainder = unpartitioned_numel % world_size
347
+ padding_numel = (world_size - remainder) if remainder else 0
348
+ partitioned_numel = math.ceil(unpartitioned_numel / world_size)
349
+ return partitioned_numel, padding_numel
350
+
351
+
352
+ def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
353
+ if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
354
+ return
355
+
356
+ if debug:
357
+ for i in range(world_size):
358
+ num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
359
+ print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
360
+
361
+ frozen_param_shapes = zero_model_states[0].frozen_param_shapes
362
+ wanted_params = len(frozen_param_shapes)
363
+ wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
364
+ avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
365
+ print(f'Frozen params: Have {avail_numel} numels to process.')
366
+ print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
367
+
368
+ total_params = 0
369
+ total_numel = 0
370
+ for name, shape in zero_model_states[0].frozen_param_shapes.items():
371
+ total_params += 1
372
+ unpartitioned_numel = shape.numel()
373
+ total_numel += unpartitioned_numel
374
+
375
+ param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
376
+ state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
377
+
378
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
379
+
380
+ if debug:
381
+ print(
382
+ f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
383
+ )
384
+
385
+ print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
386
+
387
+
388
+ def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
389
+ param_shapes = zero_model_states[0].param_shapes
390
+ avail_numel = fp32_flat_groups[0].numel() * world_size
391
+ # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
392
+ # param, re-consolidating each param, while dealing with padding if any
393
+
394
+ # merge list of dicts, preserving order
395
+ param_shapes = {k: v for d in param_shapes for k, v in d.items()}
396
+
397
+ if debug:
398
+ for i in range(world_size):
399
+ print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
400
+
401
+ wanted_params = len(param_shapes)
402
+ wanted_numel = sum(shape.numel() for shape in param_shapes.values())
403
+ # not asserting if there is a mismatch due to possible padding
404
+ avail_numel = fp32_flat_groups[0].numel() * world_size
405
+ print(f"Trainable params: Have {avail_numel} numels to process.")
406
+ print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
407
+
408
+ # params
409
+ # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
410
+ # out-of-core computing solution
411
+ offset = 0
412
+ total_numel = 0
413
+ total_params = 0
414
+ for name, shape in param_shapes.items():
415
+
416
+ unpartitioned_numel = shape.numel()
417
+ total_numel += unpartitioned_numel
418
+ total_params += 1
419
+
420
+ partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
421
+
422
+ if debug:
423
+ print(
424
+ f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
425
+ )
426
+
427
+ # XXX: memory usage doubles here
428
+ state_dict[name] = torch.cat(
429
+ tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)),
430
+ 0).narrow(0, 0, unpartitioned_numel).view(shape)
431
+ offset += partitioned_numel
432
+
433
+ offset *= world_size
434
+
435
+ # Sanity check
436
+ if offset != avail_numel:
437
+ raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
438
+
439
+ print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
440
+
441
+
442
+ def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states):
443
+ state_dict = OrderedDict()
444
+
445
+ # buffers
446
+ buffers = zero_model_states[0].buffers
447
+ state_dict.update(buffers)
448
+ if debug:
449
+ print(f"added {len(buffers)} buffers")
450
+
451
+ _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
452
+
453
+ _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
454
+
455
+ # recover shared parameters
456
+ for pair in zero_model_states[0].shared_params:
457
+ if pair[1] in state_dict:
458
+ state_dict[pair[0]] = state_dict[pair[1]]
459
+
460
+ return state_dict
461
+
462
+
463
+ def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None):
464
+ """
465
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
466
+ ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
467
+ via a model hub.
468
+
469
+ Args:
470
+ - ``checkpoint_dir``: path to the desired checkpoint folder
471
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
472
+
473
+ Returns:
474
+ - pytorch ``state_dict``
475
+
476
+ Note: this approach may not work if your application doesn't have sufficient free CPU memory and
477
+ you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
478
+ the checkpoint.
479
+
480
+ A typical usage might be ::
481
+
482
+ from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
483
+ # do the training and checkpoint saving
484
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
485
+ model = model.cpu() # move to cpu
486
+ model.load_state_dict(state_dict)
487
+ # submit to model hub or save the model to share with others
488
+
489
+ In this example the ``model`` will no longer be usable in the deepspeed context of the same
490
+ application. i.e. you will need to re-initialize the deepspeed engine, since
491
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
492
+
493
+ If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
494
+
495
+ """
496
+ if tag is None:
497
+ latest_path = os.path.join(checkpoint_dir, 'latest')
498
+ if os.path.isfile(latest_path):
499
+ with open(latest_path, 'r') as fd:
500
+ tag = fd.read().strip()
501
+ else:
502
+ raise ValueError(f"Unable to find 'latest' file at {latest_path}")
503
+
504
+ ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
505
+
506
+ if not os.path.isdir(ds_checkpoint_dir):
507
+ raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
508
+
509
+ return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir)
510
+
511
+
512
+ def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None):
513
+ """
514
+ Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
515
+ loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
516
+
517
+ Args:
518
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
519
+ - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin)
520
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
521
+ """
522
+
523
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
524
+ print(f"Saving fp32 state dict to {output_file}")
525
+ torch.save(state_dict, output_file)
526
+
527
+
528
+ def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
529
+ """
530
+ 1. Put the provided model to cpu
531
+ 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
532
+ 3. Load it into the provided model
533
+
534
+ Args:
535
+ - ``model``: the model object to update
536
+ - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
537
+ - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
538
+
539
+ Returns:
540
+ - ``model`: modified model
541
+
542
+ Make sure you have plenty of CPU memory available before you call this function. If you don't
543
+ have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
544
+ conveniently placed for you in the checkpoint folder.
545
+
546
+ A typical usage might be ::
547
+
548
+ from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
549
+ model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
550
+ # submit to model hub or save the model to share with others
551
+
552
+ Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
553
+ of the same application. i.e. you will need to re-initialize the deepspeed engine, since
554
+ ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
555
+
556
+ """
557
+ logger.info(f"Extracting fp32 weights")
558
+ state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
559
+
560
+ logger.info(f"Overwriting model with fp32 weights")
561
+ model = model.cpu()
562
+ model.load_state_dict(state_dict, strict=False)
563
+
564
+ return model
565
+
566
+
567
+ if __name__ == "__main__":
568
+
569
+ parser = argparse.ArgumentParser()
570
+ parser.add_argument("checkpoint_dir",
571
+ type=str,
572
+ help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
573
+ parser.add_argument(
574
+ "output_file",
575
+ type=str,
576
+ help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)")
577
+ parser.add_argument("-t",
578
+ "--tag",
579
+ type=str,
580
+ default=None,
581
+ help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
582
+ parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
583
+ args = parser.parse_args()
584
+
585
+ debug = args.debug
586
+
587
+ convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, args.output_file, tag=args.tag)