diff --git a/__pycache__/quantization.cpython-312.pyc b/__pycache__/quantization.cpython-312.pyc index 1d8ccd77fa0eaa5a6e3944c62c41390d38012c29..df0d06a860dafabae6909d0f6f097efc988bd83c 100644 Binary files a/__pycache__/quantization.cpython-312.pyc and b/__pycache__/quantization.cpython-312.pyc differ diff --git a/quantized_models/aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn8_emb8/chat_template.jinja b/quantized_models/aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn8_emb8/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..c2066bd7391c270626e39c9d7124f00360126412 --- /dev/null +++ b/quantized_models/aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn8_emb8/chat_template.jinja @@ -0,0 +1 @@ +{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='') %}{%- for message in messages %}{%- if message['role'] == 'system' %}{% set ns.system_prompt = message['content'] %}{%- endif %}{%- endfor %}{{bos_token}}{{ns.system_prompt}}{%- for message in messages %}{%- if message['role'] == 'user' %}{%- set ns.is_tool = false -%}{{'<｜User｜>' + message['content']}}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is none %}{%- set ns.is_tool = false -%}{%- for tool in message['tool_calls']%}{%- if not ns.is_first %}{{'<｜Assistant｜><｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments'] + '\n' + '```' + '<｜tool▁call▁end｜>'}}{%- set ns.is_first = true -%}{%- else %}{{'\n' + '<｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments'] + '\n' + '```' + '<｜tool▁call▁end｜>'}}{{'<｜tool▁calls▁end｜><｜end▁of▁sentence｜>'}}{%- endif %}{%- endfor %}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is not none %}{%- if ns.is_tool %}{{'<｜tool▁outputs▁end｜>' + message['content'] + '<｜end▁of▁sentence｜>'}}{%- set ns.is_tool = false -%}{%- else %}{% set content = message['content'] %}{% if '' in content %}{% set content = content.split('')[-1] %}{% endif %}{{'<｜Assistant｜>' + content + '<｜end▁of▁sentence｜>'}}{%- endif %}{%- endif %}{%- if message['role'] == 'tool' %}{%- set ns.is_tool = true -%}{%- if ns.is_output_first %}{{'<｜tool▁outputs▁begin｜><｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- set ns.is_output_first = false %}{%- else %}{{'\n<｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- endif %}{%- endif %}{%- endfor -%}{% if ns.is_tool %}{{'<｜tool▁outputs▁end｜>'}}{% endif %}{% if add_generation_prompt and not ns.is_tool %}{{'<｜Assistant｜>\n'}}{% endif %} \ No newline at end of file diff --git a/quantized_models/aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn8_emb8/config.json b/quantized_models/aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn8_emb8/config.json new file mode 100644 index 0000000000000000000000000000000000000000..5a233c43c3c301cd934064a97ebd9854a0933abf --- /dev/null +++ b/quantized_models/aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn8_emb8/config.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2e1d1cb43c80a432b77714a102c6eceaa43d01de540583c789cf4e85a63b67d +size 1335 diff --git a/quantized_models/aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn8_emb8/generation_config.json b/quantized_models/aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn8_emb8/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..82e55350557131481a118df9c5449d28f45b500d --- /dev/null +++ b/quantized_models/aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn8_emb8/generation_config.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:275e18531c77a259689e0424ba634aa34d289bdaac4a116c4ec9ba8e921cbb67 +size 181 diff --git a/quantized_models/aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn8_emb8/model-00001-of-00002.safetensors b/quantized_models/aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn8_emb8/model-00001-of-00002.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..dc1fb9093b85ead07a9e1d1bc36e39cb2f18370b --- /dev/null +++ b/quantized_models/aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn8_emb8/model-00001-of-00002.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7363fb6e730e5ff042860d7d2a5165f67a830991fb2b1c030adf8549ab921ca9 +size 4996670464 diff --git a/quantized_models/aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn8_emb8/model-00002-of-00002.safetensors b/quantized_models/aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn8_emb8/model-00002-of-00002.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e520d69bb74b93f274f29ed0f3f77f44d1ad4603 --- /dev/null +++ b/quantized_models/aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn8_emb8/model-00002-of-00002.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09bba4d6d1831800733b16c3dba46a72add42a7c84e980a0a020b4d4fc757a00 +size 2111719976 diff --git a/quantized_models/aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn8_emb8/model.safetensors.index.json b/quantized_models/aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn8_emb8/model.safetensors.index.json new file mode 100644 index 0000000000000000000000000000000000000000..730344c839beed5db18162ebf7880cc3742f91c9 --- /dev/null +++ b/quantized_models/aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn8_emb8/model.safetensors.index.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7e56c5725b828b21936d5ec6e2ae9acec9fd04a739d1fab9824d603c23cbf71 +size 27787 diff --git a/quantized_models/aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn8_emb8/quantization_metadata.json b/quantized_models/aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn8_emb8/quantization_metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..1c882d4610c55977db96d1fc9c79b2c3fce3166f --- /dev/null +++ b/quantized_models/aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn8_emb8/quantization_metadata.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79f9c07febf73102db7e3768771017d274c95112946e276924f65a23cad39030 +size 405 diff --git a/quantized_models/aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn8_emb8/special_tokens_map.json b/quantized_models/aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn8_emb8/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..55004e9eddc866f8c7786057b895e52f619cecfd --- /dev/null +++ b/quantized_models/aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn8_emb8/special_tokens_map.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59cda48bbe8bab9d61ffb410e6e3c07b6d98bff73cee7c88ff8b51f95f21ab1c +size 485 diff --git a/quantized_models/aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn8_emb8/tokenizer.json b/quantized_models/aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn8_emb8/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..1a2db243e47cbc113f6b2ddcc388aeeb8fe1a94c --- /dev/null +++ b/quantized_models/aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn8_emb8/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e20ddafc659ba90242154b55275402edeca0715e5dbb30f56815a4ce081f4893 +size 11422778 diff --git a/quantized_models/aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn8_emb8/tokenizer_config.json b/quantized_models/aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn8_emb8/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..eb32993ce262c2de96084c6932d1e90758481392 --- /dev/null +++ b/quantized_models/aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn8_emb8/tokenizer_config.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55edc67ee25d8e0d4b7a13036ee61cdb880b49db4398533cc6a29710d1790ade +size 4490 diff --git a/quantized_models/aggressive/Falcon-E-3B-Base__attn4_ffn8_emb8/Falcon-E-3B-Base_attn4_ffn8_emb8/config.json b/quantized_models/aggressive/Falcon-E-3B-Base__attn4_ffn8_emb8/Falcon-E-3B-Base_attn4_ffn8_emb8/config.json new file mode 100644 index 0000000000000000000000000000000000000000..03494ac8894ac740ec8e758621eb44e29502f274 --- /dev/null +++ b/quantized_models/aggressive/Falcon-E-3B-Base__attn4_ffn8_emb8/Falcon-E-3B-Base_attn4_ffn8_emb8/config.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a832eefda7e5937f7738a4d64287b3139f1904f64b5115b7f1b18d8d0a8eea9 +size 939 diff --git a/quantized_models/aggressive/Falcon-E-3B-Base__attn4_ffn8_emb8/Falcon-E-3B-Base_attn4_ffn8_emb8/generation_config.json b/quantized_models/aggressive/Falcon-E-3B-Base__attn4_ffn8_emb8/Falcon-E-3B-Base_attn4_ffn8_emb8/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9ff36835950aca8852841669d926e785a91c0f10 --- /dev/null +++ b/quantized_models/aggressive/Falcon-E-3B-Base__attn4_ffn8_emb8/Falcon-E-3B-Base_attn4_ffn8_emb8/generation_config.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7244a760e89a2d38833bf5d771b175e8c47007a74b64dcfeac7e8ac8ee8e58d +size 112 diff --git a/quantized_models/aggressive/Falcon-E-3B-Base__attn4_ffn8_emb8/Falcon-E-3B-Base_attn4_ffn8_emb8/model.safetensors b/quantized_models/aggressive/Falcon-E-3B-Base__attn4_ffn8_emb8/Falcon-E-3B-Base_attn4_ffn8_emb8/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3ae003f28dc960422595ee2cd6e79467b6409f75 --- /dev/null +++ b/quantized_models/aggressive/Falcon-E-3B-Base__attn4_ffn8_emb8/Falcon-E-3B-Base_attn4_ffn8_emb8/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cef967a47fe688b47373eea27778ec3f367c7d72f83a9a499aaf80df9d21c8b8 +size 1267270720 diff --git a/quantized_models/aggressive/Falcon-E-3B-Base__attn4_ffn8_emb8/Falcon-E-3B-Base_attn4_ffn8_emb8/quantization_metadata.json b/quantized_models/aggressive/Falcon-E-3B-Base__attn4_ffn8_emb8/Falcon-E-3B-Base_attn4_ffn8_emb8/quantization_metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..239e758e2a961f76fbe81da38136e7442b6b4beb --- /dev/null +++ b/quantized_models/aggressive/Falcon-E-3B-Base__attn4_ffn8_emb8/Falcon-E-3B-Base_attn4_ffn8_emb8/quantization_metadata.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d59c54a098b21b59aa5a2ce76f445bcad092c9727d5f90d5e4c88e48042ffce +size 379 diff --git a/quantized_models/aggressive/Falcon-E-3B-Base__attn4_ffn8_emb8/Falcon-E-3B-Base_attn4_ffn8_emb8/special_tokens_map.json b/quantized_models/aggressive/Falcon-E-3B-Base__attn4_ffn8_emb8/Falcon-E-3B-Base_attn4_ffn8_emb8/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..69bafee183682f100467ca218d0c9b33a3f42596 --- /dev/null +++ b/quantized_models/aggressive/Falcon-E-3B-Base__attn4_ffn8_emb8/Falcon-E-3B-Base_attn4_ffn8_emb8/special_tokens_map.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7f8082c06e8059eae989095356ba775ad4dcdc2e23af15eda6c74e3aee0d239 +size 7415 diff --git a/quantized_models/aggressive/Falcon-E-3B-Base__attn4_ffn8_emb8/Falcon-E-3B-Base_attn4_ffn8_emb8/tokenizer.json b/quantized_models/aggressive/Falcon-E-3B-Base__attn4_ffn8_emb8/Falcon-E-3B-Base_attn4_ffn8_emb8/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..d63f1b11b5bbd88c02681c6bc6c462f03fd368ea --- /dev/null +++ b/quantized_models/aggressive/Falcon-E-3B-Base__attn4_ffn8_emb8/Falcon-E-3B-Base_attn4_ffn8_emb8/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:605c664925653e3fbf2f35ea063847db441ba5b7a6af04378880409c3ab311fc +size 2350986 diff --git a/quantized_models/aggressive/Falcon-E-3B-Base__attn4_ffn8_emb8/Falcon-E-3B-Base_attn4_ffn8_emb8/tokenizer_config.json b/quantized_models/aggressive/Falcon-E-3B-Base__attn4_ffn8_emb8/Falcon-E-3B-Base_attn4_ffn8_emb8/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ca46edf55b81e25b9bd51544186c264f58a058ef --- /dev/null +++ b/quantized_models/aggressive/Falcon-E-3B-Base__attn4_ffn8_emb8/Falcon-E-3B-Base_attn4_ffn8_emb8/tokenizer_config.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d921ca08d70115e4ad4496f1686319e557e0e9e9f6406f14a2d4c532de6a4de +size 99642 diff --git a/quantized_models/aggressive/Phi-3-mini-128k-instruct__attn4_ffn8_emb8/Phi-3-mini-128k-instruct_attn4_ffn8_emb8/added_tokens.json b/quantized_models/aggressive/Phi-3-mini-128k-instruct__attn4_ffn8_emb8/Phi-3-mini-128k-instruct_attn4_ffn8_emb8/added_tokens.json new file mode 100644 index 0000000000000000000000000000000000000000..121b955559793317d6528aa205bea5a8c0b8c389 --- /dev/null +++ b/quantized_models/aggressive/Phi-3-mini-128k-instruct__attn4_ffn8_emb8/Phi-3-mini-128k-instruct_attn4_ffn8_emb8/added_tokens.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f5c94d30901fe9c961d776d51eadbda4454991a77b53d36e0be0571a8e72a7d +size 293 diff --git a/quantized_models/aggressive/Phi-3-mini-128k-instruct__attn4_ffn8_emb8/Phi-3-mini-128k-instruct_attn4_ffn8_emb8/chat_template.jinja b/quantized_models/aggressive/Phi-3-mini-128k-instruct__attn4_ffn8_emb8/Phi-3-mini-128k-instruct_attn4_ffn8_emb8/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..ddb5006baa8ee6aa317c5d84827e6993469c2073 --- /dev/null +++ b/quantized_models/aggressive/Phi-3-mini-128k-instruct__attn4_ffn8_emb8/Phi-3-mini-128k-instruct_attn4_ffn8_emb8/chat_template.jinja @@ -0,0 +1,8 @@ +{% for message in messages %}{% if message['role'] == 'system' %}{{'<|system|> +' + message['content'] + '<|end|> +'}}{% elif message['role'] == 'user' %}{{'<|user|> +' + message['content'] + '<|end|> +'}}{% elif message['role'] == 'assistant' %}{{'<|assistant|> +' + message['content'] + '<|end|> +'}}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|> +' }}{% else %}{{ eos_token }}{% endif %} \ No newline at end of file diff --git a/quantized_models/aggressive/Phi-3-mini-128k-instruct__attn4_ffn8_emb8/Phi-3-mini-128k-instruct_attn4_ffn8_emb8/config.json b/quantized_models/aggressive/Phi-3-mini-128k-instruct__attn4_ffn8_emb8/Phi-3-mini-128k-instruct_attn4_ffn8_emb8/config.json new file mode 100644 index 0000000000000000000000000000000000000000..8b6a676d2a0d5da35f907506d688a6976e3a4ba2 --- /dev/null +++ b/quantized_models/aggressive/Phi-3-mini-128k-instruct__attn4_ffn8_emb8/Phi-3-mini-128k-instruct_attn4_ffn8_emb8/config.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ae80a19fd38aa0a62033a131eae7acd49801bb2c570c4dca779239063a6c6e8 +size 3290 diff --git a/quantized_models/aggressive/Phi-3-mini-128k-instruct__attn4_ffn8_emb8/Phi-3-mini-128k-instruct_attn4_ffn8_emb8/configuration_phi3.py b/quantized_models/aggressive/Phi-3-mini-128k-instruct__attn4_ffn8_emb8/Phi-3-mini-128k-instruct_attn4_ffn8_emb8/configuration_phi3.py new file mode 100644 index 0000000000000000000000000000000000000000..780401034b66c92c6ce76ffc541493b6eac61627 --- /dev/null +++ b/quantized_models/aggressive/Phi-3-mini-128k-instruct__attn4_ffn8_emb8/Phi-3-mini-128k-instruct_attn4_ffn8_emb8/configuration_phi3.py @@ -0,0 +1,227 @@ +# coding=utf-8 +# Copyright 2024 Microsoft and the HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Phi-3 model configuration""" + + +from transformers.configuration_utils import PretrainedConfig +from transformers.utils import logging + + +logger = logging.get_logger(__name__) + +PHI3_PRETRAINED_CONFIG_ARCHIVE_MAP = { + "microsoft/Phi-3-mini-4k-instruct": "https://huggingface.co/microsoft/Phi-3-mini-4k-instruct/resolve/main/config.json", + "microsoft/Phi-3-mini-128k-instruct": "https://huggingface.co/microsoft/Phi-3-mini-128k-instruct/resolve/main/config.json", +} + + +class Phi3Config(PretrainedConfig): + r""" + This is the configuration class to store the configuration of a [`Phi3Model`]. It is used to instantiate a Phi-3 + model according to the specified arguments, defining the model architecture. Instantiating a configuration with the + defaults will yield a similar configuration to that of the + [microsoft/Phi-3-mini-4k-instruct](https://huggingface.co/microsoft/Phi-3-mini-4k-instruct). + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + Args: + vocab_size (`int`, *optional*, defaults to 32064): + Vocabulary size of the Phi-3 model. Defines the number of different tokens that can be represented by the + `inputs_ids` passed when calling [`Phi3Model`]. + hidden_size (`int`, *optional*, defaults to 3072): + Dimension of the hidden representations. + intermediate_size (`int`, *optional*, defaults to 8192): + Dimension of the MLP representations. + num_hidden_layers (`int`, *optional*, defaults to 32): + Number of hidden layers in the Transformer decoder. + num_attention_heads (`int`, *optional*, defaults to 32): + Number of attention heads for each attention layer in the Transformer decoder. + num_key_value_heads (`int`, *optional*): + This is the number of key_value heads that should be used to implement Grouped Query Attention. If + `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if + `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When + converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed + by meanpooling all the original heads within that group. For more details checkout [this + paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to + `num_attention_heads`. + resid_pdrop (`float`, *optional*, defaults to 0.0): + Dropout probability for mlp outputs. + embd_pdrop (`int`, *optional*, defaults to 0.0): + The dropout ratio for the embeddings. + attention_dropout (`float`, *optional*, defaults to 0.0): + The dropout ratio after computing the attention scores. + hidden_act (`str` or `function`, *optional*, defaults to `"silu"`): + The non-linear activation function (function or string) in the decoder. + max_position_embeddings (`int`, *optional*, defaults to 4096): + The maximum sequence length that this model might ever be used with. + original_max_position_embeddings (`int`, *optional*, defaults to 4096): + The maximum sequence length that this model was trained with. This is used to determine the size of the + original RoPE embeddings when using long scaling. + initializer_range (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + rms_norm_eps (`float`, *optional*, defaults to 1e-05): + The epsilon value used for the RMSNorm. + use_cache (`bool`, *optional*, defaults to `True`): + Whether or not the model should return the last key/values attentions (not used by all models). Only + relevant if `config.is_decoder=True`. Whether to tie weight embeddings or not. + tie_word_embeddings (`bool`, *optional*, defaults to `False`): + Whether to tie weight embeddings + rope_theta (`float`, *optional*, defaults to 10000.0): + The base period of the RoPE embeddings. + rope_scaling (`dict`, *optional*): + The scaling strategy for the RoPE embeddings. If `None`, no scaling is applied. If a dictionary, it must + contain the following keys: `type`, `short_factor` and `long_factor`. The `type` must be `longrope` and + the `short_factor` and `long_factor` must be lists of numbers with the same length as the hidden size + divided by the number of attention heads divided by 2. + bos_token_id (`int`, *optional*, defaults to 1): + The id of the "beginning-of-sequence" token. + eos_token_id (`int`, *optional*, defaults to 32000): + The id of the "end-of-sequence" token. + pad_token_id (`int`, *optional*, defaults to 32000): + The id of the padding token. + sliding_window (`int`, *optional*): + Sliding window attention window size. If `None`, no sliding window is applied. + + Example: + + ```python + >>> from transformers import Phi3Model, Phi3Config + + >>> # Initializing a Phi-3 style configuration + >>> configuration = Phi3Config.from_pretrained("microsoft/Phi-3-mini-4k-instruct") + + >>> # Initializing a model from the configuration + >>> model = Phi3Model(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ```""" + + model_type = "phi3" + keys_to_ignore_at_inference = ["past_key_values"] + + def __init__( + self, + vocab_size=32064, + hidden_size=3072, + intermediate_size=8192, + num_hidden_layers=32, + num_attention_heads=32, + num_key_value_heads=None, + resid_pdrop=0.0, + embd_pdrop=0.0, + attention_dropout=0.0, + hidden_act="silu", + max_position_embeddings=4096, + original_max_position_embeddings=4096, + initializer_range=0.02, + rms_norm_eps=1e-5, + use_cache=True, + tie_word_embeddings=False, + rope_theta=10000.0, + rope_scaling=None, + bos_token_id=1, + eos_token_id=32000, + pad_token_id=32000, + sliding_window=None, + **kwargs, + ): + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + + if num_key_value_heads is None: + num_key_value_heads = num_attention_heads + + self.num_key_value_heads = num_key_value_heads + self.resid_pdrop = resid_pdrop + self.embd_pdrop = embd_pdrop + self.attention_dropout = attention_dropout + self.hidden_act = hidden_act + self.max_position_embeddings = max_position_embeddings + self.original_max_position_embeddings = original_max_position_embeddings + self.initializer_range = initializer_range + self.rms_norm_eps = rms_norm_eps + self.use_cache = use_cache + self.rope_theta = rope_theta + self.rope_scaling = rope_scaling + self._rope_scaling_adjustment() + self._rope_scaling_validation() + self.sliding_window = sliding_window + + super().__init__( + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + pad_token_id=pad_token_id, + tie_word_embeddings=tie_word_embeddings, + **kwargs, + ) + + def _rope_scaling_adjustment(self): + """ + Adjust the `type` of the `rope_scaling` configuration for backward compatibility. + """ + if self.rope_scaling is None: + return + + rope_scaling_type = self.rope_scaling.get("type", None) + + # For backward compatibility if previous version used "su" or "yarn" + if rope_scaling_type is not None and rope_scaling_type in ["su", "yarn"]: + self.rope_scaling["type"] = "longrope" + + def _rope_scaling_validation(self): + """ + Validate the `rope_scaling` configuration. + """ + if self.rope_scaling is None: + return + + if not isinstance(self.rope_scaling, dict) or len(self.rope_scaling) != 3: + raise ValueError( + "`rope_scaling` must be a dictionary with three fields, `type`, `short_factor` and `long_factor`, " + f"got {self.rope_scaling}" + ) + rope_scaling_type = self.rope_scaling.get("type", None) + rope_scaling_short_factor = self.rope_scaling.get("short_factor", None) + rope_scaling_long_factor = self.rope_scaling.get("long_factor", None) + if rope_scaling_type is None or rope_scaling_type not in ["longrope"]: + raise ValueError(f"`rope_scaling`'s type field must be one of ['longrope'], got {rope_scaling_type}") + if not ( + isinstance(rope_scaling_short_factor, list) + and all(isinstance(x, (int, float)) for x in rope_scaling_short_factor) + ): + raise ValueError( + f"`rope_scaling`'s short_factor field must be a list of numbers, got {rope_scaling_short_factor}" + ) + if not len(rope_scaling_short_factor) == self.hidden_size // self.num_attention_heads // 2: + raise ValueError( + f"`rope_scaling`'s short_factor field must have length {self.hidden_size // self.num_attention_heads // 2}, got {len(rope_scaling_short_factor)}" + ) + if not ( + isinstance(rope_scaling_long_factor, list) + and all(isinstance(x, (int, float)) for x in rope_scaling_long_factor) + ): + raise ValueError( + f"`rope_scaling`'s long_factor field must be a list of numbers, got {rope_scaling_long_factor}" + ) + if not len(rope_scaling_long_factor) == self.hidden_size // self.num_attention_heads // 2: + raise ValueError( + f"`rope_scaling`'s long_factor field must have length {self.hidden_size // self.num_attention_heads // 2}, got {len(rope_scaling_long_factor)}" + ) diff --git a/quantized_models/aggressive/Phi-3-mini-128k-instruct__attn4_ffn8_emb8/Phi-3-mini-128k-instruct_attn4_ffn8_emb8/generation_config.json b/quantized_models/aggressive/Phi-3-mini-128k-instruct__attn4_ffn8_emb8/Phi-3-mini-128k-instruct_attn4_ffn8_emb8/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..f5b4f87a189fd63ed64cfcf6c78734f929440633 --- /dev/null +++ b/quantized_models/aggressive/Phi-3-mini-128k-instruct__attn4_ffn8_emb8/Phi-3-mini-128k-instruct_attn4_ffn8_emb8/generation_config.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18cb81296b3fd9eb9992a9b95a05842f9a5778d47c62cc67bb949f038b19b3bb +size 172 diff --git a/quantized_models/aggressive/Phi-3-mini-128k-instruct__attn4_ffn8_emb8/Phi-3-mini-128k-instruct_attn4_ffn8_emb8/model-00001-of-00004.safetensors b/quantized_models/aggressive/Phi-3-mini-128k-instruct__attn4_ffn8_emb8/Phi-3-mini-128k-instruct_attn4_ffn8_emb8/model-00001-of-00004.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..fee77189ba958f1bc33f10b913ee84b671b19952 --- /dev/null +++ b/quantized_models/aggressive/Phi-3-mini-128k-instruct__attn4_ffn8_emb8/Phi-3-mini-128k-instruct_attn4_ffn8_emb8/model-00001-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5dd9d8b79499e31f537f01d04e44d0b149585263b7754492b8b49cd76b443649 +size 4961852416 diff --git a/quantized_models/aggressive/Phi-3-mini-128k-instruct__attn4_ffn8_emb8/Phi-3-mini-128k-instruct_attn4_ffn8_emb8/model-00002-of-00004.safetensors b/quantized_models/aggressive/Phi-3-mini-128k-instruct__attn4_ffn8_emb8/Phi-3-mini-128k-instruct_attn4_ffn8_emb8/model-00002-of-00004.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..fd965d6a983a45100c89b41e7797c0478b89b584 --- /dev/null +++ b/quantized_models/aggressive/Phi-3-mini-128k-instruct__attn4_ffn8_emb8/Phi-3-mini-128k-instruct_attn4_ffn8_emb8/model-00002-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3186df079e7730d75a220876e3d65182d26bc4806edef9b671cc0b2cfcb95c0 +size 4983111176 diff --git a/quantized_models/aggressive/Phi-3-mini-128k-instruct__attn4_ffn8_emb8/Phi-3-mini-128k-instruct_attn4_ffn8_emb8/model-00003-of-00004.safetensors b/quantized_models/aggressive/Phi-3-mini-128k-instruct__attn4_ffn8_emb8/Phi-3-mini-128k-instruct_attn4_ffn8_emb8/model-00003-of-00004.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..cf8b6976e64d4af9a06732f7a4df75c82778e9e6 --- /dev/null +++ b/quantized_models/aggressive/Phi-3-mini-128k-instruct__attn4_ffn8_emb8/Phi-3-mini-128k-instruct_attn4_ffn8_emb8/model-00003-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45dce7445886e82947618d373d7af8f308a9a9e1795e3498b56937d45cef0ac5 +size 4945374704 diff --git a/quantized_models/aggressive/Phi-3-mini-128k-instruct__attn4_ffn8_emb8/Phi-3-mini-128k-instruct_attn4_ffn8_emb8/model-00004-of-00004.safetensors b/quantized_models/aggressive/Phi-3-mini-128k-instruct__attn4_ffn8_emb8/Phi-3-mini-128k-instruct_attn4_ffn8_emb8/model-00004-of-00004.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..18eff3e93d419bb70f66572c3c9327d3d9a6bae3 --- /dev/null +++ b/quantized_models/aggressive/Phi-3-mini-128k-instruct__attn4_ffn8_emb8/Phi-3-mini-128k-instruct_attn4_ffn8_emb8/model-00004-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d26a29ad8bb5b28067df73b520aecf857b3cb691ff0b2007443b49756cac1c2d +size 394002560 diff --git a/quantized_models/aggressive/Phi-3-mini-128k-instruct__attn4_ffn8_emb8/Phi-3-mini-128k-instruct_attn4_ffn8_emb8/model.safetensors.index.json b/quantized_models/aggressive/Phi-3-mini-128k-instruct__attn4_ffn8_emb8/Phi-3-mini-128k-instruct_attn4_ffn8_emb8/model.safetensors.index.json new file mode 100644 index 0000000000000000000000000000000000000000..412efe2449a01ed4fe81d403aed6b0faec4692a2 --- /dev/null +++ b/quantized_models/aggressive/Phi-3-mini-128k-instruct__attn4_ffn8_emb8/Phi-3-mini-128k-instruct_attn4_ffn8_emb8/model.safetensors.index.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab2d1bf7a32b16bde0dacd8f2a59e7c40a8e3f0d929badce9eca8d0a49908674 +size 16368 diff --git a/quantized_models/aggressive/Phi-3-mini-128k-instruct__attn4_ffn8_emb8/Phi-3-mini-128k-instruct_attn4_ffn8_emb8/quantization_metadata.json b/quantized_models/aggressive/Phi-3-mini-128k-instruct__attn4_ffn8_emb8/Phi-3-mini-128k-instruct_attn4_ffn8_emb8/quantization_metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..fea64a1d7f8dc75f29661d1c5f1a04ecb1d456cb --- /dev/null +++ b/quantized_models/aggressive/Phi-3-mini-128k-instruct__attn4_ffn8_emb8/Phi-3-mini-128k-instruct_attn4_ffn8_emb8/quantization_metadata.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23a44614fb4110f7127a8b7bd2bafc2d41807e781dc4db92e16c13c067d92bd6 +size 397 diff --git a/quantized_models/aggressive/Phi-3-mini-128k-instruct__attn4_ffn8_emb8/Phi-3-mini-128k-instruct_attn4_ffn8_emb8/special_tokens_map.json b/quantized_models/aggressive/Phi-3-mini-128k-instruct__attn4_ffn8_emb8/Phi-3-mini-128k-instruct_attn4_ffn8_emb8/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..a5645a655902ebe7d69a3f898da503bdba60d6e4 --- /dev/null +++ b/quantized_models/aggressive/Phi-3-mini-128k-instruct__attn4_ffn8_emb8/Phi-3-mini-128k-instruct_attn4_ffn8_emb8/special_tokens_map.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:474d699677e264edd3851559548cbbb61801ad8da6b249c81455007602313c34 +size 569 diff --git a/quantized_models/aggressive/Phi-3-mini-128k-instruct__attn4_ffn8_emb8/Phi-3-mini-128k-instruct_attn4_ffn8_emb8/tokenizer.json b/quantized_models/aggressive/Phi-3-mini-128k-instruct__attn4_ffn8_emb8/Phi-3-mini-128k-instruct_attn4_ffn8_emb8/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..8320e67e4fd629835eaca9e8f40bc62ae5316f0e --- /dev/null +++ b/quantized_models/aggressive/Phi-3-mini-128k-instruct__attn4_ffn8_emb8/Phi-3-mini-128k-instruct_attn4_ffn8_emb8/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3cb815b904d82b82b25dcd90edd00e71a5ee5443472ad611bcb84f1339300647 +size 3620657 diff --git a/quantized_models/aggressive/Phi-3-mini-128k-instruct__attn4_ffn8_emb8/Phi-3-mini-128k-instruct_attn4_ffn8_emb8/tokenizer.model b/quantized_models/aggressive/Phi-3-mini-128k-instruct__attn4_ffn8_emb8/Phi-3-mini-128k-instruct_attn4_ffn8_emb8/tokenizer.model new file mode 100644 index 0000000000000000000000000000000000000000..6c00c742ce03c627d6cd5b795984876fa49fa899 --- /dev/null +++ b/quantized_models/aggressive/Phi-3-mini-128k-instruct__attn4_ffn8_emb8/Phi-3-mini-128k-instruct_attn4_ffn8_emb8/tokenizer.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347 +size 499723 diff --git a/quantized_models/aggressive/Phi-3-mini-128k-instruct__attn4_ffn8_emb8/Phi-3-mini-128k-instruct_attn4_ffn8_emb8/tokenizer_config.json b/quantized_models/aggressive/Phi-3-mini-128k-instruct__attn4_ffn8_emb8/Phi-3-mini-128k-instruct_attn4_ffn8_emb8/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..3d44c3e1f9038109f7dee49f07a5b0633cbf6e78 --- /dev/null +++ b/quantized_models/aggressive/Phi-3-mini-128k-instruct__attn4_ffn8_emb8/Phi-3-mini-128k-instruct_attn4_ffn8_emb8/tokenizer_config.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78f1699f672ba5c8c958fa9f69040560a0310967db88fcba80746d299738f13c +size 2934 diff --git a/quantized_models/model_comparison.csv b/quantized_models/model_comparison.csv new file mode 100644 index 0000000000000000000000000000000000000000..7e87781340d228e26ef4ff8f6d940545ce88c27a --- /dev/null +++ b/quantized_models/model_comparison.csv @@ -0,0 +1,10 @@ +Model,Configuration,Attn (bits),FFN (bits),Emb (bits),Size (MB),Attn Layers,FFN Layers,Total Quantized +DeepSeek-R1-Distill-Qwen-1.5B,aggressive,4,8,8,6779.05,112,84,197 +DeepSeek-R1-Distill-Qwen-1.5B,uniform_8bit,8,8,8,6779.05,112,84,197 +DeepSeek-R1-Distill-Qwen-1.5B,very_aggressive,4,4,8,6779.05,112,84,197 +Phi-3-mini-128k-instruct,aggressive,4,8,8,14576.26,64,64,129 +Phi-3-mini-128k-instruct,uniform_8bit,8,8,8,14576.26,64,64,129 +Phi-3-mini-128k-instruct,very_aggressive,4,4,8,14576.26,64,64,129 +Falcon-E-3B-Base,aggressive,4,8,8,512.51,0,0,1 +Falcon-E-3B-Base,uniform_8bit,8,8,8,512.51,0,0,1 +Falcon-E-3B-Base,very_aggressive,4,4,8,512.51,0,0,1 diff --git a/quantized_models/quantization_results.json b/quantized_models/quantization_results.json index 33646356ef49d287a4cf0ea4a7210b34d89848af..15cd8806c74291da17a9625da77db853d52035d8 100644 --- a/quantized_models/quantization_results.json +++ b/quantized_models/quantization_results.json @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a40e9485b629bbbe01a3d4a9ba4b83a5f08448cfec579cbee314bd600ed84ea4 -size 5126 +oid sha256:e62dfa76148b9d5b1e5b1d1f2bf74b546e035ecc25b73b262784587f118c1e54 +size 3584 diff --git a/quantized_models/uniform_8bit/DeepSeek-R1-Distill-Qwen-1.5B__attn8_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn8_ffn8_emb8/chat_template.jinja b/quantized_models/uniform_8bit/DeepSeek-R1-Distill-Qwen-1.5B__attn8_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn8_ffn8_emb8/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..c2066bd7391c270626e39c9d7124f00360126412 --- /dev/null +++ b/quantized_models/uniform_8bit/DeepSeek-R1-Distill-Qwen-1.5B__attn8_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn8_ffn8_emb8/chat_template.jinja @@ -0,0 +1 @@ +{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='') %}{%- for message in messages %}{%- if message['role'] == 'system' %}{% set ns.system_prompt = message['content'] %}{%- endif %}{%- endfor %}{{bos_token}}{{ns.system_prompt}}{%- for message in messages %}{%- if message['role'] == 'user' %}{%- set ns.is_tool = false -%}{{'<｜User｜>' + message['content']}}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is none %}{%- set ns.is_tool = false -%}{%- for tool in message['tool_calls']%}{%- if not ns.is_first %}{{'<｜Assistant｜><｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments'] + '\n' + '```' + '<｜tool▁call▁end｜>'}}{%- set ns.is_first = true -%}{%- else %}{{'\n' + '<｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments'] + '\n' + '```' + '<｜tool▁call▁end｜>'}}{{'<｜tool▁calls▁end｜><｜end▁of▁sentence｜>'}}{%- endif %}{%- endfor %}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is not none %}{%- if ns.is_tool %}{{'<｜tool▁outputs▁end｜>' + message['content'] + '<｜end▁of▁sentence｜>'}}{%- set ns.is_tool = false -%}{%- else %}{% set content = message['content'] %}{% if '' in content %}{% set content = content.split('')[-1] %}{% endif %}{{'<｜Assistant｜>' + content + '<｜end▁of▁sentence｜>'}}{%- endif %}{%- endif %}{%- if message['role'] == 'tool' %}{%- set ns.is_tool = true -%}{%- if ns.is_output_first %}{{'<｜tool▁outputs▁begin｜><｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- set ns.is_output_first = false %}{%- else %}{{'\n<｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- endif %}{%- endif %}{%- endfor -%}{% if ns.is_tool %}{{'<｜tool▁outputs▁end｜>'}}{% endif %}{% if add_generation_prompt and not ns.is_tool %}{{'<｜Assistant｜>\n'}}{% endif %} \ No newline at end of file diff --git a/quantized_models/uniform_8bit/DeepSeek-R1-Distill-Qwen-1.5B__attn8_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn8_ffn8_emb8/config.json b/quantized_models/uniform_8bit/DeepSeek-R1-Distill-Qwen-1.5B__attn8_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn8_ffn8_emb8/config.json new file mode 100644 index 0000000000000000000000000000000000000000..5a233c43c3c301cd934064a97ebd9854a0933abf --- /dev/null +++ b/quantized_models/uniform_8bit/DeepSeek-R1-Distill-Qwen-1.5B__attn8_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn8_ffn8_emb8/config.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2e1d1cb43c80a432b77714a102c6eceaa43d01de540583c789cf4e85a63b67d +size 1335 diff --git a/quantized_models/uniform_8bit/DeepSeek-R1-Distill-Qwen-1.5B__attn8_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn8_ffn8_emb8/generation_config.json b/quantized_models/uniform_8bit/DeepSeek-R1-Distill-Qwen-1.5B__attn8_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn8_ffn8_emb8/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..82e55350557131481a118df9c5449d28f45b500d --- /dev/null +++ b/quantized_models/uniform_8bit/DeepSeek-R1-Distill-Qwen-1.5B__attn8_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn8_ffn8_emb8/generation_config.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:275e18531c77a259689e0424ba634aa34d289bdaac4a116c4ec9ba8e921cbb67 +size 181 diff --git a/quantized_models/uniform_8bit/DeepSeek-R1-Distill-Qwen-1.5B__attn8_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn8_ffn8_emb8/model-00001-of-00002.safetensors b/quantized_models/uniform_8bit/DeepSeek-R1-Distill-Qwen-1.5B__attn8_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn8_ffn8_emb8/model-00001-of-00002.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..51ff5deff73c2e654611f48c44f44eca56f35db7 --- /dev/null +++ b/quantized_models/uniform_8bit/DeepSeek-R1-Distill-Qwen-1.5B__attn8_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn8_ffn8_emb8/model-00001-of-00002.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07c7610a1bd72331a3e69e824be3bf8449f4213575f8f306aaa7cb57f11cf41b +size 4996670464 diff --git a/quantized_models/uniform_8bit/DeepSeek-R1-Distill-Qwen-1.5B__attn8_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn8_ffn8_emb8/model-00002-of-00002.safetensors b/quantized_models/uniform_8bit/DeepSeek-R1-Distill-Qwen-1.5B__attn8_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn8_ffn8_emb8/model-00002-of-00002.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4a2bb6e04fe943371832e2432398b32e1cfc1339 --- /dev/null +++ b/quantized_models/uniform_8bit/DeepSeek-R1-Distill-Qwen-1.5B__attn8_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn8_ffn8_emb8/model-00002-of-00002.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33204e360fb66e0a519c0bc98ad5e279ae14c33cc2b1a4cc238b8946d4605bc2 +size 2111719976 diff --git a/quantized_models/uniform_8bit/DeepSeek-R1-Distill-Qwen-1.5B__attn8_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn8_ffn8_emb8/model.safetensors.index.json b/quantized_models/uniform_8bit/DeepSeek-R1-Distill-Qwen-1.5B__attn8_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn8_ffn8_emb8/model.safetensors.index.json new file mode 100644 index 0000000000000000000000000000000000000000..730344c839beed5db18162ebf7880cc3742f91c9 --- /dev/null +++ b/quantized_models/uniform_8bit/DeepSeek-R1-Distill-Qwen-1.5B__attn8_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn8_ffn8_emb8/model.safetensors.index.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7e56c5725b828b21936d5ec6e2ae9acec9fd04a739d1fab9824d603c23cbf71 +size 27787 diff --git a/quantized_models/uniform_8bit/DeepSeek-R1-Distill-Qwen-1.5B__attn8_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn8_ffn8_emb8/quantization_metadata.json b/quantized_models/uniform_8bit/DeepSeek-R1-Distill-Qwen-1.5B__attn8_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn8_ffn8_emb8/quantization_metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..d0a85f70f818ed626f540a6f55d6f49083303cd2 --- /dev/null +++ b/quantized_models/uniform_8bit/DeepSeek-R1-Distill-Qwen-1.5B__attn8_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn8_ffn8_emb8/quantization_metadata.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:83562736cd80be3b8b4a8fb2e94142440cbf495c19e275c9cbce42fe489cf310 +size 405 diff --git a/quantized_models/uniform_8bit/DeepSeek-R1-Distill-Qwen-1.5B__attn8_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn8_ffn8_emb8/special_tokens_map.json b/quantized_models/uniform_8bit/DeepSeek-R1-Distill-Qwen-1.5B__attn8_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn8_ffn8_emb8/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..55004e9eddc866f8c7786057b895e52f619cecfd --- /dev/null +++ b/quantized_models/uniform_8bit/DeepSeek-R1-Distill-Qwen-1.5B__attn8_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn8_ffn8_emb8/special_tokens_map.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59cda48bbe8bab9d61ffb410e6e3c07b6d98bff73cee7c88ff8b51f95f21ab1c +size 485 diff --git a/quantized_models/uniform_8bit/DeepSeek-R1-Distill-Qwen-1.5B__attn8_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn8_ffn8_emb8/tokenizer.json b/quantized_models/uniform_8bit/DeepSeek-R1-Distill-Qwen-1.5B__attn8_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn8_ffn8_emb8/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..1a2db243e47cbc113f6b2ddcc388aeeb8fe1a94c --- /dev/null +++ b/quantized_models/uniform_8bit/DeepSeek-R1-Distill-Qwen-1.5B__attn8_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn8_ffn8_emb8/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e20ddafc659ba90242154b55275402edeca0715e5dbb30f56815a4ce081f4893 +size 11422778 diff --git a/quantized_models/uniform_8bit/DeepSeek-R1-Distill-Qwen-1.5B__attn8_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn8_ffn8_emb8/tokenizer_config.json b/quantized_models/uniform_8bit/DeepSeek-R1-Distill-Qwen-1.5B__attn8_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn8_ffn8_emb8/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..eb32993ce262c2de96084c6932d1e90758481392 --- /dev/null +++ b/quantized_models/uniform_8bit/DeepSeek-R1-Distill-Qwen-1.5B__attn8_ffn8_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn8_ffn8_emb8/tokenizer_config.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55edc67ee25d8e0d4b7a13036ee61cdb880b49db4398533cc6a29710d1790ade +size 4490 diff --git a/quantized_models/uniform_8bit/Falcon-E-3B-Base__attn8_ffn8_emb8/Falcon-E-3B-Base_attn8_ffn8_emb8/config.json b/quantized_models/uniform_8bit/Falcon-E-3B-Base__attn8_ffn8_emb8/Falcon-E-3B-Base_attn8_ffn8_emb8/config.json new file mode 100644 index 0000000000000000000000000000000000000000..03494ac8894ac740ec8e758621eb44e29502f274 --- /dev/null +++ b/quantized_models/uniform_8bit/Falcon-E-3B-Base__attn8_ffn8_emb8/Falcon-E-3B-Base_attn8_ffn8_emb8/config.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a832eefda7e5937f7738a4d64287b3139f1904f64b5115b7f1b18d8d0a8eea9 +size 939 diff --git a/quantized_models/uniform_8bit/Falcon-E-3B-Base__attn8_ffn8_emb8/Falcon-E-3B-Base_attn8_ffn8_emb8/generation_config.json b/quantized_models/uniform_8bit/Falcon-E-3B-Base__attn8_ffn8_emb8/Falcon-E-3B-Base_attn8_ffn8_emb8/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9ff36835950aca8852841669d926e785a91c0f10 --- /dev/null +++ b/quantized_models/uniform_8bit/Falcon-E-3B-Base__attn8_ffn8_emb8/Falcon-E-3B-Base_attn8_ffn8_emb8/generation_config.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7244a760e89a2d38833bf5d771b175e8c47007a74b64dcfeac7e8ac8ee8e58d +size 112 diff --git a/quantized_models/uniform_8bit/Falcon-E-3B-Base__attn8_ffn8_emb8/Falcon-E-3B-Base_attn8_ffn8_emb8/model.safetensors b/quantized_models/uniform_8bit/Falcon-E-3B-Base__attn8_ffn8_emb8/Falcon-E-3B-Base_attn8_ffn8_emb8/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3ae003f28dc960422595ee2cd6e79467b6409f75 --- /dev/null +++ b/quantized_models/uniform_8bit/Falcon-E-3B-Base__attn8_ffn8_emb8/Falcon-E-3B-Base_attn8_ffn8_emb8/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cef967a47fe688b47373eea27778ec3f367c7d72f83a9a499aaf80df9d21c8b8 +size 1267270720 diff --git a/quantized_models/uniform_8bit/Falcon-E-3B-Base__attn8_ffn8_emb8/Falcon-E-3B-Base_attn8_ffn8_emb8/quantization_metadata.json b/quantized_models/uniform_8bit/Falcon-E-3B-Base__attn8_ffn8_emb8/Falcon-E-3B-Base_attn8_ffn8_emb8/quantization_metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..6fb0be76e2546a9d77058557785584232f2c912a --- /dev/null +++ b/quantized_models/uniform_8bit/Falcon-E-3B-Base__attn8_ffn8_emb8/Falcon-E-3B-Base_attn8_ffn8_emb8/quantization_metadata.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1262bed941b651ef6b7f0c70fdbf8f2a0feb2354cdbde668a138471dfb3d4b4b +size 379 diff --git a/quantized_models/uniform_8bit/Falcon-E-3B-Base__attn8_ffn8_emb8/Falcon-E-3B-Base_attn8_ffn8_emb8/special_tokens_map.json b/quantized_models/uniform_8bit/Falcon-E-3B-Base__attn8_ffn8_emb8/Falcon-E-3B-Base_attn8_ffn8_emb8/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..69bafee183682f100467ca218d0c9b33a3f42596 --- /dev/null +++ b/quantized_models/uniform_8bit/Falcon-E-3B-Base__attn8_ffn8_emb8/Falcon-E-3B-Base_attn8_ffn8_emb8/special_tokens_map.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7f8082c06e8059eae989095356ba775ad4dcdc2e23af15eda6c74e3aee0d239 +size 7415 diff --git a/quantized_models/uniform_8bit/Falcon-E-3B-Base__attn8_ffn8_emb8/Falcon-E-3B-Base_attn8_ffn8_emb8/tokenizer.json b/quantized_models/uniform_8bit/Falcon-E-3B-Base__attn8_ffn8_emb8/Falcon-E-3B-Base_attn8_ffn8_emb8/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..d63f1b11b5bbd88c02681c6bc6c462f03fd368ea --- /dev/null +++ b/quantized_models/uniform_8bit/Falcon-E-3B-Base__attn8_ffn8_emb8/Falcon-E-3B-Base_attn8_ffn8_emb8/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:605c664925653e3fbf2f35ea063847db441ba5b7a6af04378880409c3ab311fc +size 2350986 diff --git a/quantized_models/uniform_8bit/Falcon-E-3B-Base__attn8_ffn8_emb8/Falcon-E-3B-Base_attn8_ffn8_emb8/tokenizer_config.json b/quantized_models/uniform_8bit/Falcon-E-3B-Base__attn8_ffn8_emb8/Falcon-E-3B-Base_attn8_ffn8_emb8/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ca46edf55b81e25b9bd51544186c264f58a058ef --- /dev/null +++ b/quantized_models/uniform_8bit/Falcon-E-3B-Base__attn8_ffn8_emb8/Falcon-E-3B-Base_attn8_ffn8_emb8/tokenizer_config.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d921ca08d70115e4ad4496f1686319e557e0e9e9f6406f14a2d4c532de6a4de +size 99642 diff --git a/quantized_models/uniform_8bit/Phi-3-mini-128k-instruct__attn8_ffn8_emb8/Phi-3-mini-128k-instruct_attn8_ffn8_emb8/added_tokens.json b/quantized_models/uniform_8bit/Phi-3-mini-128k-instruct__attn8_ffn8_emb8/Phi-3-mini-128k-instruct_attn8_ffn8_emb8/added_tokens.json new file mode 100644 index 0000000000000000000000000000000000000000..121b955559793317d6528aa205bea5a8c0b8c389 --- /dev/null +++ b/quantized_models/uniform_8bit/Phi-3-mini-128k-instruct__attn8_ffn8_emb8/Phi-3-mini-128k-instruct_attn8_ffn8_emb8/added_tokens.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f5c94d30901fe9c961d776d51eadbda4454991a77b53d36e0be0571a8e72a7d +size 293 diff --git a/quantized_models/uniform_8bit/Phi-3-mini-128k-instruct__attn8_ffn8_emb8/Phi-3-mini-128k-instruct_attn8_ffn8_emb8/chat_template.jinja b/quantized_models/uniform_8bit/Phi-3-mini-128k-instruct__attn8_ffn8_emb8/Phi-3-mini-128k-instruct_attn8_ffn8_emb8/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..ddb5006baa8ee6aa317c5d84827e6993469c2073 --- /dev/null +++ b/quantized_models/uniform_8bit/Phi-3-mini-128k-instruct__attn8_ffn8_emb8/Phi-3-mini-128k-instruct_attn8_ffn8_emb8/chat_template.jinja @@ -0,0 +1,8 @@ +{% for message in messages %}{% if message['role'] == 'system' %}{{'<|system|> +' + message['content'] + '<|end|> +'}}{% elif message['role'] == 'user' %}{{'<|user|> +' + message['content'] + '<|end|> +'}}{% elif message['role'] == 'assistant' %}{{'<|assistant|> +' + message['content'] + '<|end|> +'}}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|> +' }}{% else %}{{ eos_token }}{% endif %} \ No newline at end of file diff --git a/quantized_models/uniform_8bit/Phi-3-mini-128k-instruct__attn8_ffn8_emb8/Phi-3-mini-128k-instruct_attn8_ffn8_emb8/config.json b/quantized_models/uniform_8bit/Phi-3-mini-128k-instruct__attn8_ffn8_emb8/Phi-3-mini-128k-instruct_attn8_ffn8_emb8/config.json new file mode 100644 index 0000000000000000000000000000000000000000..8b6a676d2a0d5da35f907506d688a6976e3a4ba2 --- /dev/null +++ b/quantized_models/uniform_8bit/Phi-3-mini-128k-instruct__attn8_ffn8_emb8/Phi-3-mini-128k-instruct_attn8_ffn8_emb8/config.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ae80a19fd38aa0a62033a131eae7acd49801bb2c570c4dca779239063a6c6e8 +size 3290 diff --git a/quantized_models/uniform_8bit/Phi-3-mini-128k-instruct__attn8_ffn8_emb8/Phi-3-mini-128k-instruct_attn8_ffn8_emb8/configuration_phi3.py b/quantized_models/uniform_8bit/Phi-3-mini-128k-instruct__attn8_ffn8_emb8/Phi-3-mini-128k-instruct_attn8_ffn8_emb8/configuration_phi3.py new file mode 100644 index 0000000000000000000000000000000000000000..780401034b66c92c6ce76ffc541493b6eac61627 --- /dev/null +++ b/quantized_models/uniform_8bit/Phi-3-mini-128k-instruct__attn8_ffn8_emb8/Phi-3-mini-128k-instruct_attn8_ffn8_emb8/configuration_phi3.py @@ -0,0 +1,227 @@ +# coding=utf-8 +# Copyright 2024 Microsoft and the HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Phi-3 model configuration""" + + +from transformers.configuration_utils import PretrainedConfig +from transformers.utils import logging + + +logger = logging.get_logger(__name__) + +PHI3_PRETRAINED_CONFIG_ARCHIVE_MAP = { + "microsoft/Phi-3-mini-4k-instruct": "https://huggingface.co/microsoft/Phi-3-mini-4k-instruct/resolve/main/config.json", + "microsoft/Phi-3-mini-128k-instruct": "https://huggingface.co/microsoft/Phi-3-mini-128k-instruct/resolve/main/config.json", +} + + +class Phi3Config(PretrainedConfig): + r""" + This is the configuration class to store the configuration of a [`Phi3Model`]. It is used to instantiate a Phi-3 + model according to the specified arguments, defining the model architecture. Instantiating a configuration with the + defaults will yield a similar configuration to that of the + [microsoft/Phi-3-mini-4k-instruct](https://huggingface.co/microsoft/Phi-3-mini-4k-instruct). + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + Args: + vocab_size (`int`, *optional*, defaults to 32064): + Vocabulary size of the Phi-3 model. Defines the number of different tokens that can be represented by the + `inputs_ids` passed when calling [`Phi3Model`]. + hidden_size (`int`, *optional*, defaults to 3072): + Dimension of the hidden representations. + intermediate_size (`int`, *optional*, defaults to 8192): + Dimension of the MLP representations. + num_hidden_layers (`int`, *optional*, defaults to 32): + Number of hidden layers in the Transformer decoder. + num_attention_heads (`int`, *optional*, defaults to 32): + Number of attention heads for each attention layer in the Transformer decoder. + num_key_value_heads (`int`, *optional*): + This is the number of key_value heads that should be used to implement Grouped Query Attention. If + `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if + `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When + converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed + by meanpooling all the original heads within that group. For more details checkout [this + paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to + `num_attention_heads`. + resid_pdrop (`float`, *optional*, defaults to 0.0): + Dropout probability for mlp outputs. + embd_pdrop (`int`, *optional*, defaults to 0.0): + The dropout ratio for the embeddings. + attention_dropout (`float`, *optional*, defaults to 0.0): + The dropout ratio after computing the attention scores. + hidden_act (`str` or `function`, *optional*, defaults to `"silu"`): + The non-linear activation function (function or string) in the decoder. + max_position_embeddings (`int`, *optional*, defaults to 4096): + The maximum sequence length that this model might ever be used with. + original_max_position_embeddings (`int`, *optional*, defaults to 4096): + The maximum sequence length that this model was trained with. This is used to determine the size of the + original RoPE embeddings when using long scaling. + initializer_range (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + rms_norm_eps (`float`, *optional*, defaults to 1e-05): + The epsilon value used for the RMSNorm. + use_cache (`bool`, *optional*, defaults to `True`): + Whether or not the model should return the last key/values attentions (not used by all models). Only + relevant if `config.is_decoder=True`. Whether to tie weight embeddings or not. + tie_word_embeddings (`bool`, *optional*, defaults to `False`): + Whether to tie weight embeddings + rope_theta (`float`, *optional*, defaults to 10000.0): + The base period of the RoPE embeddings. + rope_scaling (`dict`, *optional*): + The scaling strategy for the RoPE embeddings. If `None`, no scaling is applied. If a dictionary, it must + contain the following keys: `type`, `short_factor` and `long_factor`. The `type` must be `longrope` and + the `short_factor` and `long_factor` must be lists of numbers with the same length as the hidden size + divided by the number of attention heads divided by 2. + bos_token_id (`int`, *optional*, defaults to 1): + The id of the "beginning-of-sequence" token. + eos_token_id (`int`, *optional*, defaults to 32000): + The id of the "end-of-sequence" token. + pad_token_id (`int`, *optional*, defaults to 32000): + The id of the padding token. + sliding_window (`int`, *optional*): + Sliding window attention window size. If `None`, no sliding window is applied. + + Example: + + ```python + >>> from transformers import Phi3Model, Phi3Config + + >>> # Initializing a Phi-3 style configuration + >>> configuration = Phi3Config.from_pretrained("microsoft/Phi-3-mini-4k-instruct") + + >>> # Initializing a model from the configuration + >>> model = Phi3Model(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ```""" + + model_type = "phi3" + keys_to_ignore_at_inference = ["past_key_values"] + + def __init__( + self, + vocab_size=32064, + hidden_size=3072, + intermediate_size=8192, + num_hidden_layers=32, + num_attention_heads=32, + num_key_value_heads=None, + resid_pdrop=0.0, + embd_pdrop=0.0, + attention_dropout=0.0, + hidden_act="silu", + max_position_embeddings=4096, + original_max_position_embeddings=4096, + initializer_range=0.02, + rms_norm_eps=1e-5, + use_cache=True, + tie_word_embeddings=False, + rope_theta=10000.0, + rope_scaling=None, + bos_token_id=1, + eos_token_id=32000, + pad_token_id=32000, + sliding_window=None, + **kwargs, + ): + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + + if num_key_value_heads is None: + num_key_value_heads = num_attention_heads + + self.num_key_value_heads = num_key_value_heads + self.resid_pdrop = resid_pdrop + self.embd_pdrop = embd_pdrop + self.attention_dropout = attention_dropout + self.hidden_act = hidden_act + self.max_position_embeddings = max_position_embeddings + self.original_max_position_embeddings = original_max_position_embeddings + self.initializer_range = initializer_range + self.rms_norm_eps = rms_norm_eps + self.use_cache = use_cache + self.rope_theta = rope_theta + self.rope_scaling = rope_scaling + self._rope_scaling_adjustment() + self._rope_scaling_validation() + self.sliding_window = sliding_window + + super().__init__( + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + pad_token_id=pad_token_id, + tie_word_embeddings=tie_word_embeddings, + **kwargs, + ) + + def _rope_scaling_adjustment(self): + """ + Adjust the `type` of the `rope_scaling` configuration for backward compatibility. + """ + if self.rope_scaling is None: + return + + rope_scaling_type = self.rope_scaling.get("type", None) + + # For backward compatibility if previous version used "su" or "yarn" + if rope_scaling_type is not None and rope_scaling_type in ["su", "yarn"]: + self.rope_scaling["type"] = "longrope" + + def _rope_scaling_validation(self): + """ + Validate the `rope_scaling` configuration. + """ + if self.rope_scaling is None: + return + + if not isinstance(self.rope_scaling, dict) or len(self.rope_scaling) != 3: + raise ValueError( + "`rope_scaling` must be a dictionary with three fields, `type`, `short_factor` and `long_factor`, " + f"got {self.rope_scaling}" + ) + rope_scaling_type = self.rope_scaling.get("type", None) + rope_scaling_short_factor = self.rope_scaling.get("short_factor", None) + rope_scaling_long_factor = self.rope_scaling.get("long_factor", None) + if rope_scaling_type is None or rope_scaling_type not in ["longrope"]: + raise ValueError(f"`rope_scaling`'s type field must be one of ['longrope'], got {rope_scaling_type}") + if not ( + isinstance(rope_scaling_short_factor, list) + and all(isinstance(x, (int, float)) for x in rope_scaling_short_factor) + ): + raise ValueError( + f"`rope_scaling`'s short_factor field must be a list of numbers, got {rope_scaling_short_factor}" + ) + if not len(rope_scaling_short_factor) == self.hidden_size // self.num_attention_heads // 2: + raise ValueError( + f"`rope_scaling`'s short_factor field must have length {self.hidden_size // self.num_attention_heads // 2}, got {len(rope_scaling_short_factor)}" + ) + if not ( + isinstance(rope_scaling_long_factor, list) + and all(isinstance(x, (int, float)) for x in rope_scaling_long_factor) + ): + raise ValueError( + f"`rope_scaling`'s long_factor field must be a list of numbers, got {rope_scaling_long_factor}" + ) + if not len(rope_scaling_long_factor) == self.hidden_size // self.num_attention_heads // 2: + raise ValueError( + f"`rope_scaling`'s long_factor field must have length {self.hidden_size // self.num_attention_heads // 2}, got {len(rope_scaling_long_factor)}" + ) diff --git a/quantized_models/uniform_8bit/Phi-3-mini-128k-instruct__attn8_ffn8_emb8/Phi-3-mini-128k-instruct_attn8_ffn8_emb8/generation_config.json b/quantized_models/uniform_8bit/Phi-3-mini-128k-instruct__attn8_ffn8_emb8/Phi-3-mini-128k-instruct_attn8_ffn8_emb8/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..f5b4f87a189fd63ed64cfcf6c78734f929440633 --- /dev/null +++ b/quantized_models/uniform_8bit/Phi-3-mini-128k-instruct__attn8_ffn8_emb8/Phi-3-mini-128k-instruct_attn8_ffn8_emb8/generation_config.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18cb81296b3fd9eb9992a9b95a05842f9a5778d47c62cc67bb949f038b19b3bb +size 172 diff --git a/quantized_models/uniform_8bit/Phi-3-mini-128k-instruct__attn8_ffn8_emb8/Phi-3-mini-128k-instruct_attn8_ffn8_emb8/model-00001-of-00004.safetensors b/quantized_models/uniform_8bit/Phi-3-mini-128k-instruct__attn8_ffn8_emb8/Phi-3-mini-128k-instruct_attn8_ffn8_emb8/model-00001-of-00004.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..622fac716fbc7be4aedeeceab2e5208abd78ba67 --- /dev/null +++ b/quantized_models/uniform_8bit/Phi-3-mini-128k-instruct__attn8_ffn8_emb8/Phi-3-mini-128k-instruct_attn8_ffn8_emb8/model-00001-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed8b73583f4037a9b8852a779f63712b5b1105c470a903bdb86703ce3a23cdaf +size 4961852416 diff --git a/quantized_models/uniform_8bit/Phi-3-mini-128k-instruct__attn8_ffn8_emb8/Phi-3-mini-128k-instruct_attn8_ffn8_emb8/model-00002-of-00004.safetensors b/quantized_models/uniform_8bit/Phi-3-mini-128k-instruct__attn8_ffn8_emb8/Phi-3-mini-128k-instruct_attn8_ffn8_emb8/model-00002-of-00004.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..64c94d66b309d4dcb377b48c63d648cd6593f988 --- /dev/null +++ b/quantized_models/uniform_8bit/Phi-3-mini-128k-instruct__attn8_ffn8_emb8/Phi-3-mini-128k-instruct_attn8_ffn8_emb8/model-00002-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d2d39098db4e3547222393c1ea58bfcb9599c0ae1f9c8db195611e3be5c4e58 +size 4983111176 diff --git a/quantized_models/uniform_8bit/Phi-3-mini-128k-instruct__attn8_ffn8_emb8/Phi-3-mini-128k-instruct_attn8_ffn8_emb8/model-00003-of-00004.safetensors b/quantized_models/uniform_8bit/Phi-3-mini-128k-instruct__attn8_ffn8_emb8/Phi-3-mini-128k-instruct_attn8_ffn8_emb8/model-00003-of-00004.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a05ac40d71eee31971d3d927253605466f3f0be8 --- /dev/null +++ b/quantized_models/uniform_8bit/Phi-3-mini-128k-instruct__attn8_ffn8_emb8/Phi-3-mini-128k-instruct_attn8_ffn8_emb8/model-00003-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:65a55299ec02036a96fb1dda39f50d8e89a74842a9121337f8b53ddcea02af91 +size 4945374704 diff --git a/quantized_models/uniform_8bit/Phi-3-mini-128k-instruct__attn8_ffn8_emb8/Phi-3-mini-128k-instruct_attn8_ffn8_emb8/model-00004-of-00004.safetensors b/quantized_models/uniform_8bit/Phi-3-mini-128k-instruct__attn8_ffn8_emb8/Phi-3-mini-128k-instruct_attn8_ffn8_emb8/model-00004-of-00004.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..18eff3e93d419bb70f66572c3c9327d3d9a6bae3 --- /dev/null +++ b/quantized_models/uniform_8bit/Phi-3-mini-128k-instruct__attn8_ffn8_emb8/Phi-3-mini-128k-instruct_attn8_ffn8_emb8/model-00004-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d26a29ad8bb5b28067df73b520aecf857b3cb691ff0b2007443b49756cac1c2d +size 394002560 diff --git a/quantized_models/uniform_8bit/Phi-3-mini-128k-instruct__attn8_ffn8_emb8/Phi-3-mini-128k-instruct_attn8_ffn8_emb8/model.safetensors.index.json b/quantized_models/uniform_8bit/Phi-3-mini-128k-instruct__attn8_ffn8_emb8/Phi-3-mini-128k-instruct_attn8_ffn8_emb8/model.safetensors.index.json new file mode 100644 index 0000000000000000000000000000000000000000..412efe2449a01ed4fe81d403aed6b0faec4692a2 --- /dev/null +++ b/quantized_models/uniform_8bit/Phi-3-mini-128k-instruct__attn8_ffn8_emb8/Phi-3-mini-128k-instruct_attn8_ffn8_emb8/model.safetensors.index.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab2d1bf7a32b16bde0dacd8f2a59e7c40a8e3f0d929badce9eca8d0a49908674 +size 16368 diff --git a/quantized_models/uniform_8bit/Phi-3-mini-128k-instruct__attn8_ffn8_emb8/Phi-3-mini-128k-instruct_attn8_ffn8_emb8/quantization_metadata.json b/quantized_models/uniform_8bit/Phi-3-mini-128k-instruct__attn8_ffn8_emb8/Phi-3-mini-128k-instruct_attn8_ffn8_emb8/quantization_metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..ac727ff55579455712c606750dadd144f060674e --- /dev/null +++ b/quantized_models/uniform_8bit/Phi-3-mini-128k-instruct__attn8_ffn8_emb8/Phi-3-mini-128k-instruct_attn8_ffn8_emb8/quantization_metadata.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:987ca03d19c49a45a2713c5306476757b1d32ac9a7b8bda317be30dedc9a3eba +size 397 diff --git a/quantized_models/uniform_8bit/Phi-3-mini-128k-instruct__attn8_ffn8_emb8/Phi-3-mini-128k-instruct_attn8_ffn8_emb8/special_tokens_map.json b/quantized_models/uniform_8bit/Phi-3-mini-128k-instruct__attn8_ffn8_emb8/Phi-3-mini-128k-instruct_attn8_ffn8_emb8/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..a5645a655902ebe7d69a3f898da503bdba60d6e4 --- /dev/null +++ b/quantized_models/uniform_8bit/Phi-3-mini-128k-instruct__attn8_ffn8_emb8/Phi-3-mini-128k-instruct_attn8_ffn8_emb8/special_tokens_map.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:474d699677e264edd3851559548cbbb61801ad8da6b249c81455007602313c34 +size 569 diff --git a/quantized_models/uniform_8bit/Phi-3-mini-128k-instruct__attn8_ffn8_emb8/Phi-3-mini-128k-instruct_attn8_ffn8_emb8/tokenizer.json b/quantized_models/uniform_8bit/Phi-3-mini-128k-instruct__attn8_ffn8_emb8/Phi-3-mini-128k-instruct_attn8_ffn8_emb8/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..8320e67e4fd629835eaca9e8f40bc62ae5316f0e --- /dev/null +++ b/quantized_models/uniform_8bit/Phi-3-mini-128k-instruct__attn8_ffn8_emb8/Phi-3-mini-128k-instruct_attn8_ffn8_emb8/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3cb815b904d82b82b25dcd90edd00e71a5ee5443472ad611bcb84f1339300647 +size 3620657 diff --git a/quantized_models/uniform_8bit/Phi-3-mini-128k-instruct__attn8_ffn8_emb8/Phi-3-mini-128k-instruct_attn8_ffn8_emb8/tokenizer.model b/quantized_models/uniform_8bit/Phi-3-mini-128k-instruct__attn8_ffn8_emb8/Phi-3-mini-128k-instruct_attn8_ffn8_emb8/tokenizer.model new file mode 100644 index 0000000000000000000000000000000000000000..6c00c742ce03c627d6cd5b795984876fa49fa899 --- /dev/null +++ b/quantized_models/uniform_8bit/Phi-3-mini-128k-instruct__attn8_ffn8_emb8/Phi-3-mini-128k-instruct_attn8_ffn8_emb8/tokenizer.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347 +size 499723 diff --git a/quantized_models/uniform_8bit/Phi-3-mini-128k-instruct__attn8_ffn8_emb8/Phi-3-mini-128k-instruct_attn8_ffn8_emb8/tokenizer_config.json b/quantized_models/uniform_8bit/Phi-3-mini-128k-instruct__attn8_ffn8_emb8/Phi-3-mini-128k-instruct_attn8_ffn8_emb8/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..3d44c3e1f9038109f7dee49f07a5b0633cbf6e78 --- /dev/null +++ b/quantized_models/uniform_8bit/Phi-3-mini-128k-instruct__attn8_ffn8_emb8/Phi-3-mini-128k-instruct_attn8_ffn8_emb8/tokenizer_config.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78f1699f672ba5c8c958fa9f69040560a0310967db88fcba80746d299738f13c +size 2934 diff --git a/quantized_models/very_aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn4_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn4_emb8/chat_template.jinja b/quantized_models/very_aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn4_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn4_emb8/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..c2066bd7391c270626e39c9d7124f00360126412 --- /dev/null +++ b/quantized_models/very_aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn4_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn4_emb8/chat_template.jinja @@ -0,0 +1 @@ +{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='') %}{%- for message in messages %}{%- if message['role'] == 'system' %}{% set ns.system_prompt = message['content'] %}{%- endif %}{%- endfor %}{{bos_token}}{{ns.system_prompt}}{%- for message in messages %}{%- if message['role'] == 'user' %}{%- set ns.is_tool = false -%}{{'<｜User｜>' + message['content']}}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is none %}{%- set ns.is_tool = false -%}{%- for tool in message['tool_calls']%}{%- if not ns.is_first %}{{'<｜Assistant｜><｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments'] + '\n' + '```' + '<｜tool▁call▁end｜>'}}{%- set ns.is_first = true -%}{%- else %}{{'\n' + '<｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\n' + '```json' + '\n' + tool['function']['arguments'] + '\n' + '```' + '<｜tool▁call▁end｜>'}}{{'<｜tool▁calls▁end｜><｜end▁of▁sentence｜>'}}{%- endif %}{%- endfor %}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is not none %}{%- if ns.is_tool %}{{'<｜tool▁outputs▁end｜>' + message['content'] + '<｜end▁of▁sentence｜>'}}{%- set ns.is_tool = false -%}{%- else %}{% set content = message['content'] %}{% if '' in content %}{% set content = content.split('')[-1] %}{% endif %}{{'<｜Assistant｜>' + content + '<｜end▁of▁sentence｜>'}}{%- endif %}{%- endif %}{%- if message['role'] == 'tool' %}{%- set ns.is_tool = true -%}{%- if ns.is_output_first %}{{'<｜tool▁outputs▁begin｜><｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- set ns.is_output_first = false %}{%- else %}{{'\n<｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- endif %}{%- endif %}{%- endfor -%}{% if ns.is_tool %}{{'<｜tool▁outputs▁end｜>'}}{% endif %}{% if add_generation_prompt and not ns.is_tool %}{{'<｜Assistant｜>\n'}}{% endif %} \ No newline at end of file diff --git a/quantized_models/very_aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn4_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn4_emb8/config.json b/quantized_models/very_aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn4_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn4_emb8/config.json new file mode 100644 index 0000000000000000000000000000000000000000..5a233c43c3c301cd934064a97ebd9854a0933abf --- /dev/null +++ b/quantized_models/very_aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn4_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn4_emb8/config.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2e1d1cb43c80a432b77714a102c6eceaa43d01de540583c789cf4e85a63b67d +size 1335 diff --git a/quantized_models/very_aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn4_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn4_emb8/generation_config.json b/quantized_models/very_aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn4_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn4_emb8/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..82e55350557131481a118df9c5449d28f45b500d --- /dev/null +++ b/quantized_models/very_aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn4_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn4_emb8/generation_config.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:275e18531c77a259689e0424ba634aa34d289bdaac4a116c4ec9ba8e921cbb67 +size 181 diff --git a/quantized_models/very_aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn4_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn4_emb8/model-00001-of-00002.safetensors b/quantized_models/very_aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn4_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn4_emb8/model-00001-of-00002.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..4747b4e153edc31c5a893871dc7418157ebcd451 --- /dev/null +++ b/quantized_models/very_aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn4_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn4_emb8/model-00001-of-00002.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a11616e27c32f2e5d877a6d6fbc6482b89f99290f337c1c9a67d054070ccd756 +size 4996670464 diff --git a/quantized_models/very_aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn4_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn4_emb8/model-00002-of-00002.safetensors b/quantized_models/very_aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn4_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn4_emb8/model-00002-of-00002.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..31bf9d691c0ae3ad157a063b5d8402e916c6ac5d --- /dev/null +++ b/quantized_models/very_aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn4_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn4_emb8/model-00002-of-00002.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03b12bdefcf411a2b223a8e2409c055a9594d47268a8bf00452c3f1920d3fea7 +size 2111719976 diff --git a/quantized_models/very_aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn4_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn4_emb8/model.safetensors.index.json b/quantized_models/very_aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn4_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn4_emb8/model.safetensors.index.json new file mode 100644 index 0000000000000000000000000000000000000000..730344c839beed5db18162ebf7880cc3742f91c9 --- /dev/null +++ b/quantized_models/very_aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn4_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn4_emb8/model.safetensors.index.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7e56c5725b828b21936d5ec6e2ae9acec9fd04a739d1fab9824d603c23cbf71 +size 27787 diff --git a/quantized_models/very_aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn4_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn4_emb8/quantization_metadata.json b/quantized_models/very_aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn4_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn4_emb8/quantization_metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..35a98957ed71c36e36dc457a328abc4641ac627f --- /dev/null +++ b/quantized_models/very_aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn4_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn4_emb8/quantization_metadata.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7f658ac24b9e2631eb4d9312ca7d852191f536bc6ea99a9324cb5c126e91deef +size 405 diff --git a/quantized_models/very_aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn4_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn4_emb8/special_tokens_map.json b/quantized_models/very_aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn4_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn4_emb8/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..55004e9eddc866f8c7786057b895e52f619cecfd --- /dev/null +++ b/quantized_models/very_aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn4_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn4_emb8/special_tokens_map.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59cda48bbe8bab9d61ffb410e6e3c07b6d98bff73cee7c88ff8b51f95f21ab1c +size 485 diff --git a/quantized_models/very_aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn4_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn4_emb8/tokenizer.json b/quantized_models/very_aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn4_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn4_emb8/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..1a2db243e47cbc113f6b2ddcc388aeeb8fe1a94c --- /dev/null +++ b/quantized_models/very_aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn4_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn4_emb8/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e20ddafc659ba90242154b55275402edeca0715e5dbb30f56815a4ce081f4893 +size 11422778 diff --git a/quantized_models/very_aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn4_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn4_emb8/tokenizer_config.json b/quantized_models/very_aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn4_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn4_emb8/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..eb32993ce262c2de96084c6932d1e90758481392 --- /dev/null +++ b/quantized_models/very_aggressive/DeepSeek-R1-Distill-Qwen-1.5B__attn4_ffn4_emb8/DeepSeek-R1-Distill-Qwen-1.5B_attn4_ffn4_emb8/tokenizer_config.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:55edc67ee25d8e0d4b7a13036ee61cdb880b49db4398533cc6a29710d1790ade +size 4490 diff --git a/quantized_models/very_aggressive/Falcon-E-3B-Base__attn4_ffn4_emb8/Falcon-E-3B-Base_attn4_ffn4_emb8/config.json b/quantized_models/very_aggressive/Falcon-E-3B-Base__attn4_ffn4_emb8/Falcon-E-3B-Base_attn4_ffn4_emb8/config.json new file mode 100644 index 0000000000000000000000000000000000000000..03494ac8894ac740ec8e758621eb44e29502f274 --- /dev/null +++ b/quantized_models/very_aggressive/Falcon-E-3B-Base__attn4_ffn4_emb8/Falcon-E-3B-Base_attn4_ffn4_emb8/config.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a832eefda7e5937f7738a4d64287b3139f1904f64b5115b7f1b18d8d0a8eea9 +size 939 diff --git a/quantized_models/very_aggressive/Falcon-E-3B-Base__attn4_ffn4_emb8/Falcon-E-3B-Base_attn4_ffn4_emb8/generation_config.json b/quantized_models/very_aggressive/Falcon-E-3B-Base__attn4_ffn4_emb8/Falcon-E-3B-Base_attn4_ffn4_emb8/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9ff36835950aca8852841669d926e785a91c0f10 --- /dev/null +++ b/quantized_models/very_aggressive/Falcon-E-3B-Base__attn4_ffn4_emb8/Falcon-E-3B-Base_attn4_ffn4_emb8/generation_config.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7244a760e89a2d38833bf5d771b175e8c47007a74b64dcfeac7e8ac8ee8e58d +size 112 diff --git a/quantized_models/very_aggressive/Falcon-E-3B-Base__attn4_ffn4_emb8/Falcon-E-3B-Base_attn4_ffn4_emb8/model.safetensors b/quantized_models/very_aggressive/Falcon-E-3B-Base__attn4_ffn4_emb8/Falcon-E-3B-Base_attn4_ffn4_emb8/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3ae003f28dc960422595ee2cd6e79467b6409f75 --- /dev/null +++ b/quantized_models/very_aggressive/Falcon-E-3B-Base__attn4_ffn4_emb8/Falcon-E-3B-Base_attn4_ffn4_emb8/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cef967a47fe688b47373eea27778ec3f367c7d72f83a9a499aaf80df9d21c8b8 +size 1267270720 diff --git a/quantized_models/very_aggressive/Falcon-E-3B-Base__attn4_ffn4_emb8/Falcon-E-3B-Base_attn4_ffn4_emb8/quantization_metadata.json b/quantized_models/very_aggressive/Falcon-E-3B-Base__attn4_ffn4_emb8/Falcon-E-3B-Base_attn4_ffn4_emb8/quantization_metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..47544a7cd9ee686629e429102bb1dc58919e964f --- /dev/null +++ b/quantized_models/very_aggressive/Falcon-E-3B-Base__attn4_ffn4_emb8/Falcon-E-3B-Base_attn4_ffn4_emb8/quantization_metadata.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3de62de45a47af1217ce9c4e468d06ab506165df88035aaf02fab6a85f796581 +size 379 diff --git a/quantized_models/very_aggressive/Falcon-E-3B-Base__attn4_ffn4_emb8/Falcon-E-3B-Base_attn4_ffn4_emb8/special_tokens_map.json b/quantized_models/very_aggressive/Falcon-E-3B-Base__attn4_ffn4_emb8/Falcon-E-3B-Base_attn4_ffn4_emb8/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..69bafee183682f100467ca218d0c9b33a3f42596 --- /dev/null +++ b/quantized_models/very_aggressive/Falcon-E-3B-Base__attn4_ffn4_emb8/Falcon-E-3B-Base_attn4_ffn4_emb8/special_tokens_map.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7f8082c06e8059eae989095356ba775ad4dcdc2e23af15eda6c74e3aee0d239 +size 7415 diff --git a/quantized_models/very_aggressive/Falcon-E-3B-Base__attn4_ffn4_emb8/Falcon-E-3B-Base_attn4_ffn4_emb8/tokenizer.json b/quantized_models/very_aggressive/Falcon-E-3B-Base__attn4_ffn4_emb8/Falcon-E-3B-Base_attn4_ffn4_emb8/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..d63f1b11b5bbd88c02681c6bc6c462f03fd368ea --- /dev/null +++ b/quantized_models/very_aggressive/Falcon-E-3B-Base__attn4_ffn4_emb8/Falcon-E-3B-Base_attn4_ffn4_emb8/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:605c664925653e3fbf2f35ea063847db441ba5b7a6af04378880409c3ab311fc +size 2350986 diff --git a/quantized_models/very_aggressive/Falcon-E-3B-Base__attn4_ffn4_emb8/Falcon-E-3B-Base_attn4_ffn4_emb8/tokenizer_config.json b/quantized_models/very_aggressive/Falcon-E-3B-Base__attn4_ffn4_emb8/Falcon-E-3B-Base_attn4_ffn4_emb8/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..ca46edf55b81e25b9bd51544186c264f58a058ef --- /dev/null +++ b/quantized_models/very_aggressive/Falcon-E-3B-Base__attn4_ffn4_emb8/Falcon-E-3B-Base_attn4_ffn4_emb8/tokenizer_config.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d921ca08d70115e4ad4496f1686319e557e0e9e9f6406f14a2d4c532de6a4de +size 99642 diff --git a/quantized_models/very_aggressive/Phi-3-mini-128k-instruct__attn4_ffn4_emb8/Phi-3-mini-128k-instruct_attn4_ffn4_emb8/added_tokens.json b/quantized_models/very_aggressive/Phi-3-mini-128k-instruct__attn4_ffn4_emb8/Phi-3-mini-128k-instruct_attn4_ffn4_emb8/added_tokens.json new file mode 100644 index 0000000000000000000000000000000000000000..121b955559793317d6528aa205bea5a8c0b8c389 --- /dev/null +++ b/quantized_models/very_aggressive/Phi-3-mini-128k-instruct__attn4_ffn4_emb8/Phi-3-mini-128k-instruct_attn4_ffn4_emb8/added_tokens.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f5c94d30901fe9c961d776d51eadbda4454991a77b53d36e0be0571a8e72a7d +size 293 diff --git a/quantized_models/very_aggressive/Phi-3-mini-128k-instruct__attn4_ffn4_emb8/Phi-3-mini-128k-instruct_attn4_ffn4_emb8/chat_template.jinja b/quantized_models/very_aggressive/Phi-3-mini-128k-instruct__attn4_ffn4_emb8/Phi-3-mini-128k-instruct_attn4_ffn4_emb8/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..ddb5006baa8ee6aa317c5d84827e6993469c2073 --- /dev/null +++ b/quantized_models/very_aggressive/Phi-3-mini-128k-instruct__attn4_ffn4_emb8/Phi-3-mini-128k-instruct_attn4_ffn4_emb8/chat_template.jinja @@ -0,0 +1,8 @@ +{% for message in messages %}{% if message['role'] == 'system' %}{{'<|system|> +' + message['content'] + '<|end|> +'}}{% elif message['role'] == 'user' %}{{'<|user|> +' + message['content'] + '<|end|> +'}}{% elif message['role'] == 'assistant' %}{{'<|assistant|> +' + message['content'] + '<|end|> +'}}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|> +' }}{% else %}{{ eos_token }}{% endif %} \ No newline at end of file diff --git a/quantized_models/very_aggressive/Phi-3-mini-128k-instruct__attn4_ffn4_emb8/Phi-3-mini-128k-instruct_attn4_ffn4_emb8/config.json b/quantized_models/very_aggressive/Phi-3-mini-128k-instruct__attn4_ffn4_emb8/Phi-3-mini-128k-instruct_attn4_ffn4_emb8/config.json new file mode 100644 index 0000000000000000000000000000000000000000..8b6a676d2a0d5da35f907506d688a6976e3a4ba2 --- /dev/null +++ b/quantized_models/very_aggressive/Phi-3-mini-128k-instruct__attn4_ffn4_emb8/Phi-3-mini-128k-instruct_attn4_ffn4_emb8/config.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ae80a19fd38aa0a62033a131eae7acd49801bb2c570c4dca779239063a6c6e8 +size 3290 diff --git a/quantized_models/very_aggressive/Phi-3-mini-128k-instruct__attn4_ffn4_emb8/Phi-3-mini-128k-instruct_attn4_ffn4_emb8/configuration_phi3.py b/quantized_models/very_aggressive/Phi-3-mini-128k-instruct__attn4_ffn4_emb8/Phi-3-mini-128k-instruct_attn4_ffn4_emb8/configuration_phi3.py new file mode 100644 index 0000000000000000000000000000000000000000..780401034b66c92c6ce76ffc541493b6eac61627 --- /dev/null +++ b/quantized_models/very_aggressive/Phi-3-mini-128k-instruct__attn4_ffn4_emb8/Phi-3-mini-128k-instruct_attn4_ffn4_emb8/configuration_phi3.py @@ -0,0 +1,227 @@ +# coding=utf-8 +# Copyright 2024 Microsoft and the HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Phi-3 model configuration""" + + +from transformers.configuration_utils import PretrainedConfig +from transformers.utils import logging + + +logger = logging.get_logger(__name__) + +PHI3_PRETRAINED_CONFIG_ARCHIVE_MAP = { + "microsoft/Phi-3-mini-4k-instruct": "https://huggingface.co/microsoft/Phi-3-mini-4k-instruct/resolve/main/config.json", + "microsoft/Phi-3-mini-128k-instruct": "https://huggingface.co/microsoft/Phi-3-mini-128k-instruct/resolve/main/config.json", +} + + +class Phi3Config(PretrainedConfig): + r""" + This is the configuration class to store the configuration of a [`Phi3Model`]. It is used to instantiate a Phi-3 + model according to the specified arguments, defining the model architecture. Instantiating a configuration with the + defaults will yield a similar configuration to that of the + [microsoft/Phi-3-mini-4k-instruct](https://huggingface.co/microsoft/Phi-3-mini-4k-instruct). + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + Args: + vocab_size (`int`, *optional*, defaults to 32064): + Vocabulary size of the Phi-3 model. Defines the number of different tokens that can be represented by the + `inputs_ids` passed when calling [`Phi3Model`]. + hidden_size (`int`, *optional*, defaults to 3072): + Dimension of the hidden representations. + intermediate_size (`int`, *optional*, defaults to 8192): + Dimension of the MLP representations. + num_hidden_layers (`int`, *optional*, defaults to 32): + Number of hidden layers in the Transformer decoder. + num_attention_heads (`int`, *optional*, defaults to 32): + Number of attention heads for each attention layer in the Transformer decoder. + num_key_value_heads (`int`, *optional*): + This is the number of key_value heads that should be used to implement Grouped Query Attention. If + `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if + `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When + converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed + by meanpooling all the original heads within that group. For more details checkout [this + paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to + `num_attention_heads`. + resid_pdrop (`float`, *optional*, defaults to 0.0): + Dropout probability for mlp outputs. + embd_pdrop (`int`, *optional*, defaults to 0.0): + The dropout ratio for the embeddings. + attention_dropout (`float`, *optional*, defaults to 0.0): + The dropout ratio after computing the attention scores. + hidden_act (`str` or `function`, *optional*, defaults to `"silu"`): + The non-linear activation function (function or string) in the decoder. + max_position_embeddings (`int`, *optional*, defaults to 4096): + The maximum sequence length that this model might ever be used with. + original_max_position_embeddings (`int`, *optional*, defaults to 4096): + The maximum sequence length that this model was trained with. This is used to determine the size of the + original RoPE embeddings when using long scaling. + initializer_range (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + rms_norm_eps (`float`, *optional*, defaults to 1e-05): + The epsilon value used for the RMSNorm. + use_cache (`bool`, *optional*, defaults to `True`): + Whether or not the model should return the last key/values attentions (not used by all models). Only + relevant if `config.is_decoder=True`. Whether to tie weight embeddings or not. + tie_word_embeddings (`bool`, *optional*, defaults to `False`): + Whether to tie weight embeddings + rope_theta (`float`, *optional*, defaults to 10000.0): + The base period of the RoPE embeddings. + rope_scaling (`dict`, *optional*): + The scaling strategy for the RoPE embeddings. If `None`, no scaling is applied. If a dictionary, it must + contain the following keys: `type`, `short_factor` and `long_factor`. The `type` must be `longrope` and + the `short_factor` and `long_factor` must be lists of numbers with the same length as the hidden size + divided by the number of attention heads divided by 2. + bos_token_id (`int`, *optional*, defaults to 1): + The id of the "beginning-of-sequence" token. + eos_token_id (`int`, *optional*, defaults to 32000): + The id of the "end-of-sequence" token. + pad_token_id (`int`, *optional*, defaults to 32000): + The id of the padding token. + sliding_window (`int`, *optional*): + Sliding window attention window size. If `None`, no sliding window is applied. + + Example: + + ```python + >>> from transformers import Phi3Model, Phi3Config + + >>> # Initializing a Phi-3 style configuration + >>> configuration = Phi3Config.from_pretrained("microsoft/Phi-3-mini-4k-instruct") + + >>> # Initializing a model from the configuration + >>> model = Phi3Model(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ```""" + + model_type = "phi3" + keys_to_ignore_at_inference = ["past_key_values"] + + def __init__( + self, + vocab_size=32064, + hidden_size=3072, + intermediate_size=8192, + num_hidden_layers=32, + num_attention_heads=32, + num_key_value_heads=None, + resid_pdrop=0.0, + embd_pdrop=0.0, + attention_dropout=0.0, + hidden_act="silu", + max_position_embeddings=4096, + original_max_position_embeddings=4096, + initializer_range=0.02, + rms_norm_eps=1e-5, + use_cache=True, + tie_word_embeddings=False, + rope_theta=10000.0, + rope_scaling=None, + bos_token_id=1, + eos_token_id=32000, + pad_token_id=32000, + sliding_window=None, + **kwargs, + ): + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + + if num_key_value_heads is None: + num_key_value_heads = num_attention_heads + + self.num_key_value_heads = num_key_value_heads + self.resid_pdrop = resid_pdrop + self.embd_pdrop = embd_pdrop + self.attention_dropout = attention_dropout + self.hidden_act = hidden_act + self.max_position_embeddings = max_position_embeddings + self.original_max_position_embeddings = original_max_position_embeddings + self.initializer_range = initializer_range + self.rms_norm_eps = rms_norm_eps + self.use_cache = use_cache + self.rope_theta = rope_theta + self.rope_scaling = rope_scaling + self._rope_scaling_adjustment() + self._rope_scaling_validation() + self.sliding_window = sliding_window + + super().__init__( + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + pad_token_id=pad_token_id, + tie_word_embeddings=tie_word_embeddings, + **kwargs, + ) + + def _rope_scaling_adjustment(self): + """ + Adjust the `type` of the `rope_scaling` configuration for backward compatibility. + """ + if self.rope_scaling is None: + return + + rope_scaling_type = self.rope_scaling.get("type", None) + + # For backward compatibility if previous version used "su" or "yarn" + if rope_scaling_type is not None and rope_scaling_type in ["su", "yarn"]: + self.rope_scaling["type"] = "longrope" + + def _rope_scaling_validation(self): + """ + Validate the `rope_scaling` configuration. + """ + if self.rope_scaling is None: + return + + if not isinstance(self.rope_scaling, dict) or len(self.rope_scaling) != 3: + raise ValueError( + "`rope_scaling` must be a dictionary with three fields, `type`, `short_factor` and `long_factor`, " + f"got {self.rope_scaling}" + ) + rope_scaling_type = self.rope_scaling.get("type", None) + rope_scaling_short_factor = self.rope_scaling.get("short_factor", None) + rope_scaling_long_factor = self.rope_scaling.get("long_factor", None) + if rope_scaling_type is None or rope_scaling_type not in ["longrope"]: + raise ValueError(f"`rope_scaling`'s type field must be one of ['longrope'], got {rope_scaling_type}") + if not ( + isinstance(rope_scaling_short_factor, list) + and all(isinstance(x, (int, float)) for x in rope_scaling_short_factor) + ): + raise ValueError( + f"`rope_scaling`'s short_factor field must be a list of numbers, got {rope_scaling_short_factor}" + ) + if not len(rope_scaling_short_factor) == self.hidden_size // self.num_attention_heads // 2: + raise ValueError( + f"`rope_scaling`'s short_factor field must have length {self.hidden_size // self.num_attention_heads // 2}, got {len(rope_scaling_short_factor)}" + ) + if not ( + isinstance(rope_scaling_long_factor, list) + and all(isinstance(x, (int, float)) for x in rope_scaling_long_factor) + ): + raise ValueError( + f"`rope_scaling`'s long_factor field must be a list of numbers, got {rope_scaling_long_factor}" + ) + if not len(rope_scaling_long_factor) == self.hidden_size // self.num_attention_heads // 2: + raise ValueError( + f"`rope_scaling`'s long_factor field must have length {self.hidden_size // self.num_attention_heads // 2}, got {len(rope_scaling_long_factor)}" + ) diff --git a/quantized_models/very_aggressive/Phi-3-mini-128k-instruct__attn4_ffn4_emb8/Phi-3-mini-128k-instruct_attn4_ffn4_emb8/generation_config.json b/quantized_models/very_aggressive/Phi-3-mini-128k-instruct__attn4_ffn4_emb8/Phi-3-mini-128k-instruct_attn4_ffn4_emb8/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..f5b4f87a189fd63ed64cfcf6c78734f929440633 --- /dev/null +++ b/quantized_models/very_aggressive/Phi-3-mini-128k-instruct__attn4_ffn4_emb8/Phi-3-mini-128k-instruct_attn4_ffn4_emb8/generation_config.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18cb81296b3fd9eb9992a9b95a05842f9a5778d47c62cc67bb949f038b19b3bb +size 172 diff --git a/quantized_models/very_aggressive/Phi-3-mini-128k-instruct__attn4_ffn4_emb8/Phi-3-mini-128k-instruct_attn4_ffn4_emb8/model-00001-of-00004.safetensors b/quantized_models/very_aggressive/Phi-3-mini-128k-instruct__attn4_ffn4_emb8/Phi-3-mini-128k-instruct_attn4_ffn4_emb8/model-00001-of-00004.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d47dc90a4926d82324aaf23db5ff042902a16956 --- /dev/null +++ b/quantized_models/very_aggressive/Phi-3-mini-128k-instruct__attn4_ffn4_emb8/Phi-3-mini-128k-instruct_attn4_ffn4_emb8/model-00001-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77a140f1fd4ef6f6a35cde1cce80c2a076e98c41a21446cb7b8be8930c3b5505 +size 4961852416 diff --git a/quantized_models/very_aggressive/Phi-3-mini-128k-instruct__attn4_ffn4_emb8/Phi-3-mini-128k-instruct_attn4_ffn4_emb8/model-00002-of-00004.safetensors b/quantized_models/very_aggressive/Phi-3-mini-128k-instruct__attn4_ffn4_emb8/Phi-3-mini-128k-instruct_attn4_ffn4_emb8/model-00002-of-00004.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..24e59706257075a3756efa0113d7ee13bcdb3ef9 --- /dev/null +++ b/quantized_models/very_aggressive/Phi-3-mini-128k-instruct__attn4_ffn4_emb8/Phi-3-mini-128k-instruct_attn4_ffn4_emb8/model-00002-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bc62a54d2940c6a98eaec9a793e4d8aa6889ef47ac17279f112ba7feb8c2ffae +size 4983111176 diff --git a/quantized_models/very_aggressive/Phi-3-mini-128k-instruct__attn4_ffn4_emb8/Phi-3-mini-128k-instruct_attn4_ffn4_emb8/model-00003-of-00004.safetensors b/quantized_models/very_aggressive/Phi-3-mini-128k-instruct__attn4_ffn4_emb8/Phi-3-mini-128k-instruct_attn4_ffn4_emb8/model-00003-of-00004.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3ec87ad75394ea83e7b87f91b2d1535f39cb43e6 --- /dev/null +++ b/quantized_models/very_aggressive/Phi-3-mini-128k-instruct__attn4_ffn4_emb8/Phi-3-mini-128k-instruct_attn4_ffn4_emb8/model-00003-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90e282ef8ff7419cf0fe18e2971b5b0ad27ac0693701016a5f97220e88de09d3 +size 4945374704 diff --git a/quantized_models/very_aggressive/Phi-3-mini-128k-instruct__attn4_ffn4_emb8/Phi-3-mini-128k-instruct_attn4_ffn4_emb8/model-00004-of-00004.safetensors b/quantized_models/very_aggressive/Phi-3-mini-128k-instruct__attn4_ffn4_emb8/Phi-3-mini-128k-instruct_attn4_ffn4_emb8/model-00004-of-00004.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..18eff3e93d419bb70f66572c3c9327d3d9a6bae3 --- /dev/null +++ b/quantized_models/very_aggressive/Phi-3-mini-128k-instruct__attn4_ffn4_emb8/Phi-3-mini-128k-instruct_attn4_ffn4_emb8/model-00004-of-00004.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d26a29ad8bb5b28067df73b520aecf857b3cb691ff0b2007443b49756cac1c2d +size 394002560 diff --git a/quantized_models/very_aggressive/Phi-3-mini-128k-instruct__attn4_ffn4_emb8/Phi-3-mini-128k-instruct_attn4_ffn4_emb8/model.safetensors.index.json b/quantized_models/very_aggressive/Phi-3-mini-128k-instruct__attn4_ffn4_emb8/Phi-3-mini-128k-instruct_attn4_ffn4_emb8/model.safetensors.index.json new file mode 100644 index 0000000000000000000000000000000000000000..412efe2449a01ed4fe81d403aed6b0faec4692a2 --- /dev/null +++ b/quantized_models/very_aggressive/Phi-3-mini-128k-instruct__attn4_ffn4_emb8/Phi-3-mini-128k-instruct_attn4_ffn4_emb8/model.safetensors.index.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab2d1bf7a32b16bde0dacd8f2a59e7c40a8e3f0d929badce9eca8d0a49908674 +size 16368 diff --git a/quantized_models/very_aggressive/Phi-3-mini-128k-instruct__attn4_ffn4_emb8/Phi-3-mini-128k-instruct_attn4_ffn4_emb8/quantization_metadata.json b/quantized_models/very_aggressive/Phi-3-mini-128k-instruct__attn4_ffn4_emb8/Phi-3-mini-128k-instruct_attn4_ffn4_emb8/quantization_metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..bb63d533992f2a3e6c8961a7bbfdce00d18c8db1 --- /dev/null +++ b/quantized_models/very_aggressive/Phi-3-mini-128k-instruct__attn4_ffn4_emb8/Phi-3-mini-128k-instruct_attn4_ffn4_emb8/quantization_metadata.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51ccd0302e5b6a156dd6a9540b4ffa54383e3d24ffd226ac7074ac9dfd8b9bd1 +size 397 diff --git a/quantized_models/very_aggressive/Phi-3-mini-128k-instruct__attn4_ffn4_emb8/Phi-3-mini-128k-instruct_attn4_ffn4_emb8/special_tokens_map.json b/quantized_models/very_aggressive/Phi-3-mini-128k-instruct__attn4_ffn4_emb8/Phi-3-mini-128k-instruct_attn4_ffn4_emb8/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..a5645a655902ebe7d69a3f898da503bdba60d6e4 --- /dev/null +++ b/quantized_models/very_aggressive/Phi-3-mini-128k-instruct__attn4_ffn4_emb8/Phi-3-mini-128k-instruct_attn4_ffn4_emb8/special_tokens_map.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:474d699677e264edd3851559548cbbb61801ad8da6b249c81455007602313c34 +size 569 diff --git a/quantized_models/very_aggressive/Phi-3-mini-128k-instruct__attn4_ffn4_emb8/Phi-3-mini-128k-instruct_attn4_ffn4_emb8/tokenizer.json b/quantized_models/very_aggressive/Phi-3-mini-128k-instruct__attn4_ffn4_emb8/Phi-3-mini-128k-instruct_attn4_ffn4_emb8/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..8320e67e4fd629835eaca9e8f40bc62ae5316f0e --- /dev/null +++ b/quantized_models/very_aggressive/Phi-3-mini-128k-instruct__attn4_ffn4_emb8/Phi-3-mini-128k-instruct_attn4_ffn4_emb8/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3cb815b904d82b82b25dcd90edd00e71a5ee5443472ad611bcb84f1339300647 +size 3620657 diff --git a/quantized_models/very_aggressive/Phi-3-mini-128k-instruct__attn4_ffn4_emb8/Phi-3-mini-128k-instruct_attn4_ffn4_emb8/tokenizer.model b/quantized_models/very_aggressive/Phi-3-mini-128k-instruct__attn4_ffn4_emb8/Phi-3-mini-128k-instruct_attn4_ffn4_emb8/tokenizer.model new file mode 100644 index 0000000000000000000000000000000000000000..6c00c742ce03c627d6cd5b795984876fa49fa899 --- /dev/null +++ b/quantized_models/very_aggressive/Phi-3-mini-128k-instruct__attn4_ffn4_emb8/Phi-3-mini-128k-instruct_attn4_ffn4_emb8/tokenizer.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347 +size 499723 diff --git a/quantized_models/very_aggressive/Phi-3-mini-128k-instruct__attn4_ffn4_emb8/Phi-3-mini-128k-instruct_attn4_ffn4_emb8/tokenizer_config.json b/quantized_models/very_aggressive/Phi-3-mini-128k-instruct__attn4_ffn4_emb8/Phi-3-mini-128k-instruct_attn4_ffn4_emb8/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..3d44c3e1f9038109f7dee49f07a5b0633cbf6e78 --- /dev/null +++ b/quantized_models/very_aggressive/Phi-3-mini-128k-instruct__attn4_ffn4_emb8/Phi-3-mini-128k-instruct_attn4_ffn4_emb8/tokenizer_config.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78f1699f672ba5c8c958fa9f69040560a0310967db88fcba80746d299738f13c +size 2934