init

README.md

@@ -1,3 +1,39 @@
 ---
-{LICENSE}
----
+license: apache-2.0
+---
+# GEB-1.3
+GEB-1.3B是北京集异璧科技有限公司发布的轻量级语言大模型，拥有13亿参数，由550B中英文tokens数据训练而成。采用了目前较新的训练技术，包括 ROPE位置编码、多组注意力机制和FlashAttention-2，以在加速训练的同时保持模型性能。此外，我们使用了 1000 万条指令数据进行微调，以增强模型的对齐能力，并采用DPO方法更新模型以符合人类偏好。
+GEB-1.3B在MMLU、C-Eval和CMMLU等常用基准测试中表现优异，超过了类似同参数级别的模型如TinyLLaMA-1.1B。值得注意的是，GEB-1.3B的FP32版本在CPU上实现了令人满意的推理时间，我们正在通过先进的量化技术进一步提升速度。
+
+# 评测结果
+| Model          | MMLU  | C-Eval | CMMLU | Average |
+|----------------|-------|--------|-------|---------|
+| Baichuan-7B    | 42.30 | 42.80  | 44.02 | 43.04   |
+| ChatGLM-6B     | 40.63 | 38.90  | -     | 39.77   |
+| GEB-1.3B       | 31.20 | 33.30  | 32.20 | 32.23   |
+| Llama-7B       | 35.10 | 27.10  | 26.75 | 29.65   |
+| Falcon-7B      | 28.00 | -      | -     | 28.00   |
+| MPT-7B         | 27.93 | 27.15  | 26.00 | 27.03   |
+| MindLLM-1.3B   | 26.20 | 26.10  | 25.33 | 25.88   |
+| MindLLM-3B     | 26.20 | 25.70  | 25.00 | 25.63   |
+| TinyLlama-1.1B | 25.34 | 25.02  | 24.03 | 24.80   |
+
+# 运行模型
+
+使用 transformers 后端进行推理:
+
+```python
+from transformers import AutoTokenizer, AutoModel
+import torch
+model = AutoModel.from_pretrained("/GEBAGI/GEB-1.3", trust_remote_code=True).bfloat16().cuda()
+tokenizer = AutoTokenizer.from_pretrained("/GEBAGI/GEB-1.3", trust_remote_code=True)
+while True:
+    query = "你好"
+    response, history = model.chat(tokenizer, query, history=[])
+    print(response)
+```
+## 协议
+
+GEB-1.3 模型的权重的使用则需要遵循 [LICENSE](LICENSE)。
+
+## 引用

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:20fddc4d8d1a6e4d3771341839db9d391fcecf3ddbb4f0e221dd74b9a12306a9
+size 2696113663

pytorch_model.bin.index.json

@@ -0,0 +1 @@
+{"weight_map": {"transformer.embedding.word_embeddings.weight": "pytorch_model.bin", "transformer.encoder.layers.0.input_layernorm.weight": "pytorch_model.bin", "transformer.encoder.layers.0.self_attention.query.weight": "pytorch_model.bin", "transformer.encoder.layers.0.self_attention.key_value.weight": "pytorch_model.bin", "transformer.encoder.layers.0.self_attention.dense.weight": "pytorch_model.bin", "transformer.encoder.layers.0.post_attention_layernorm.weight": "pytorch_model.bin", "transformer.encoder.layers.0.mlp.dense_h_to_4h.weight": "pytorch_model.bin", "transformer.encoder.layers.0.mlp.dense_4h_to_h.weight": "pytorch_model.bin", "transformer.encoder.layers.1.input_layernorm.weight": "pytorch_model.bin", "transformer.encoder.layers.1.self_attention.query.weight": "pytorch_model.bin", "transformer.encoder.layers.1.self_attention.key_value.weight": "pytorch_model.bin", "transformer.encoder.layers.1.self_attention.dense.weight": "pytorch_model.bin", "transformer.encoder.layers.1.post_attention_layernorm.weight": "pytorch_model.bin", "transformer.encoder.layers.1.mlp.dense_h_to_4h.weight": "pytorch_model.bin", "transformer.encoder.layers.1.mlp.dense_4h_to_h.weight": "pytorch_model.bin", "transformer.encoder.layers.2.input_layernorm.weight": "pytorch_model.bin", "transformer.encoder.layers.2.self_attention.query.weight": "pytorch_model.bin", "transformer.encoder.layers.2.self_attention.key_value.weight": "pytorch_model.bin", "transformer.encoder.layers.2.self_attention.dense.weight": "pytorch_model.bin", "transformer.encoder.layers.2.post_attention_layernorm.weight": "pytorch_model.bin", "transformer.encoder.layers.2.mlp.dense_h_to_4h.weight": "pytorch_model.bin", "transformer.encoder.layers.2.mlp.dense_4h_to_h.weight": "pytorch_model.bin", "transformer.encoder.layers.3.input_layernorm.weight": "pytorch_model.bin", "transformer.encoder.layers.3.self_attention.query.weight": "pytorch_model.bin", "transformer.encoder.layers.3.self_attention.key_value.weight": "pytorch_model.bin", "transformer.encoder.layers.3.self_attention.dense.weight": "pytorch_model.bin", "transformer.encoder.layers.3.post_attention_layernorm.weight": "pytorch_model.bin", "transformer.encoder.layers.3.mlp.dense_h_to_4h.weight": "pytorch_model.bin", "transformer.encoder.layers.3.mlp.dense_4h_to_h.weight": "pytorch_model.bin", "transformer.encoder.layers.4.input_layernorm.weight": "pytorch_model.bin", "transformer.encoder.layers.4.self_attention.query.weight": "pytorch_model.bin", "transformer.encoder.layers.4.self_attention.key_value.weight": "pytorch_model.bin", "transformer.encoder.layers.4.self_attention.dense.weight": "pytorch_model.bin", "transformer.encoder.layers.4.post_attention_layernorm.weight": "pytorch_model.bin", "transformer.encoder.layers.4.mlp.dense_h_to_4h.weight": "pytorch_model.bin", "transformer.encoder.layers.4.mlp.dense_4h_to_h.weight": "pytorch_model.bin", "transformer.encoder.layers.5.input_layernorm.weight": "pytorch_model.bin", "transformer.encoder.layers.5.self_attention.query.weight": "pytorch_model.bin", "transformer.encoder.layers.5.self_attention.key_value.weight": "pytorch_model.bin", "transformer.encoder.layers.5.self_attention.dense.weight": "pytorch_model.bin", "transformer.encoder.layers.5.post_attention_layernorm.weight": "pytorch_model.bin", "transformer.encoder.layers.5.mlp.dense_h_to_4h.weight": "pytorch_model.bin", "transformer.encoder.layers.5.mlp.dense_4h_to_h.weight": "pytorch_model.bin", "transformer.encoder.layers.6.input_layernorm.weight": "pytorch_model.bin", "transformer.encoder.layers.6.self_attention.query.weight": "pytorch_model.bin", "transformer.encoder.layers.6.self_attention.key_value.weight": "pytorch_model.bin", "transformer.encoder.layers.6.self_attention.dense.weight": "pytorch_model.bin", "transformer.encoder.layers.6.post_attention_layernorm.weight": "pytorch_model.bin", "transformer.encoder.layers.6.mlp.dense_h_to_4h.weight": "pytorch_model.bin", "transformer.encoder.layers.6.mlp.dense_4h_to_h.weight": "pytorch_model.bin", "transformer.encoder.layers.7.input_layernorm.weight": "pytorch_model.bin", "transformer.encoder.layers.7.self_attention.query.weight": "pytorch_model.bin", "transformer.encoder.layers.7.self_attention.key_value.weight": "pytorch_model.bin", "transformer.encoder.layers.7.self_attention.dense.weight": "pytorch_model.bin", "transformer.encoder.layers.7.post_attention_layernorm.weight": "pytorch_model.bin", "transformer.encoder.layers.7.mlp.dense_h_to_4h.weight": "pytorch_model.bin", "transformer.encoder.layers.7.mlp.dense_4h_to_h.weight": "pytorch_model.bin", "transformer.encoder.layers.8.input_layernorm.weight": "pytorch_model.bin", "transformer.encoder.layers.8.self_attention.query.weight": "pytorch_model.bin", "transformer.encoder.layers.8.self_attention.key_value.weight": "pytorch_model.bin", "transformer.encoder.layers.8.self_attention.dense.weight": "pytorch_model.bin", "transformer.encoder.layers.8.post_attention_layernorm.weight": "pytorch_model.bin", "transformer.encoder.layers.8.mlp.dense_h_to_4h.weight": "pytorch_model.bin", "transformer.encoder.layers.8.mlp.dense_4h_to_h.weight": "pytorch_model.bin", "transformer.encoder.layers.9.input_layernorm.weight": "pytorch_model.bin", "transformer.encoder.layers.9.self_attention.query.weight": "pytorch_model.bin", "transformer.encoder.layers.9.self_attention.key_value.weight": "pytorch_model.bin", "transformer.encoder.layers.9.self_attention.dense.weight": "pytorch_model.bin", "transformer.encoder.layers.9.post_attention_layernorm.weight": "pytorch_model.bin", "transformer.encoder.layers.9.mlp.dense_h_to_4h.weight": "pytorch_model.bin", "transformer.encoder.layers.9.mlp.dense_4h_to_h.weight": "pytorch_model.bin", "transformer.encoder.layers.10.input_layernorm.weight": "pytorch_model.bin", "transformer.encoder.layers.10.self_attention.query.weight": "pytorch_model.bin", "transformer.encoder.layers.10.self_attention.key_value.weight": "pytorch_model.bin", "transformer.encoder.layers.10.self_attention.dense.weight": "pytorch_model.bin", "transformer.encoder.layers.10.post_attention_layernorm.weight": "pytorch_model.bin", "transformer.encoder.layers.10.mlp.dense_h_to_4h.weight": "pytorch_model.bin", "transformer.encoder.layers.10.mlp.dense_4h_to_h.weight": "pytorch_model.bin", "transformer.encoder.layers.11.input_layernorm.weight": "pytorch_model.bin", "transformer.encoder.layers.11.self_attention.query.weight": "pytorch_model.bin", "transformer.encoder.layers.11.self_attention.key_value.weight": "pytorch_model.bin", "transformer.encoder.layers.11.self_attention.dense.weight": "pytorch_model.bin", "transformer.encoder.layers.11.post_attention_layernorm.weight": "pytorch_model.bin", "transformer.encoder.layers.11.mlp.dense_h_to_4h.weight": "pytorch_model.bin", "transformer.encoder.layers.11.mlp.dense_4h_to_h.weight": "pytorch_model.bin", "transformer.encoder.layers.12.input_layernorm.weight": "pytorch_model.bin", "transformer.encoder.layers.12.self_attention.query.weight": "pytorch_model.bin", "transformer.encoder.layers.12.self_attention.key_value.weight": "pytorch_model.bin", "transformer.encoder.layers.12.self_attention.dense.weight": "pytorch_model.bin", "transformer.encoder.layers.12.post_attention_layernorm.weight": "pytorch_model.bin", "transformer.encoder.layers.12.mlp.dense_h_to_4h.weight": "pytorch_model.bin", "transformer.encoder.layers.12.mlp.dense_4h_to_h.weight": "pytorch_model.bin", "transformer.encoder.layers.13.input_layernorm.weight": "pytorch_model.bin", "transformer.encoder.layers.13.self_attention.query.weight": "pytorch_model.bin", "transformer.encoder.layers.13.self_attention.key_value.weight": "pytorch_model.bin", "transformer.encoder.layers.13.self_attention.dense.weight": "pytorch_model.bin", "transformer.encoder.layers.13.post_attention_layernorm.weight": "pytorch_model.bin", "transformer.encoder.layers.13.mlp.dense_h_to_4h.weight": "pytorch_model.bin", "transformer.encoder.layers.13.mlp.dense_4h_to_h.weight": "pytorch_model.bin", "transformer.encoder.layers.14.input_layernorm.weight": "pytorch_model.bin", "transformer.encoder.layers.14.self_attention.query.weight": "pytorch_model.bin", "transformer.encoder.layers.14.self_attention.key_value.weight": "pytorch_model.bin", "transformer.encoder.layers.14.self_attention.dense.weight": "pytorch_model.bin", "transformer.encoder.layers.14.post_attention_layernorm.weight": "pytorch_model.bin", "transformer.encoder.layers.14.mlp.dense_h_to_4h.weight": "pytorch_model.bin", "transformer.encoder.layers.14.mlp.dense_4h_to_h.weight": "pytorch_model.bin", "transformer.encoder.layers.15.input_layernorm.weight": "pytorch_model.bin", "transformer.encoder.layers.15.self_attention.query.weight": "pytorch_model.bin", "transformer.encoder.layers.15.self_attention.key_value.weight": "pytorch_model.bin", "transformer.encoder.layers.15.self_attention.dense.weight": "pytorch_model.bin", "transformer.encoder.layers.15.post_attention_layernorm.weight": "pytorch_model.bin", "transformer.encoder.layers.15.mlp.dense_h_to_4h.weight": "pytorch_model.bin", "transformer.encoder.layers.15.mlp.dense_4h_to_h.weight": "pytorch_model.bin", "transformer.encoder.layers.16.input_layernorm.weight": "pytorch_model.bin", "transformer.encoder.layers.16.self_attention.query.weight": "pytorch_model.bin", "transformer.encoder.layers.16.self_attention.key_value.weight": "pytorch_model.bin", "transformer.encoder.layers.16.self_attention.dense.weight": "pytorch_model.bin", "transformer.encoder.layers.16.post_attention_layernorm.weight": "pytorch_model.bin", "transformer.encoder.layers.16.mlp.dense_h_to_4h.weight": "pytorch_model.bin", "transformer.encoder.layers.16.mlp.dense_4h_to_h.weight": "pytorch_model.bin", "transformer.encoder.layers.17.input_layernorm.weight": "pytorch_model.bin", "transformer.encoder.layers.17.self_attention.query.weight": "pytorch_model.bin", "transformer.encoder.layers.17.self_attention.key_value.weight": "pytorch_model.bin", "transformer.encoder.layers.17.self_attention.dense.weight": "pytorch_model.bin", "transformer.encoder.layers.17.post_attention_layernorm.weight": "pytorch_model.bin", "transformer.encoder.layers.17.mlp.dense_h_to_4h.weight": "pytorch_model.bin", "transformer.encoder.layers.17.mlp.dense_4h_to_h.weight": "pytorch_model.bin", "transformer.encoder.layers.18.input_layernorm.weight": "pytorch_model.bin", "transformer.encoder.layers.18.self_attention.query.weight": "pytorch_model.bin", "transformer.encoder.layers.18.self_attention.key_value.weight": "pytorch_model.bin", "transformer.encoder.layers.18.self_attention.dense.weight": "pytorch_model.bin", "transformer.encoder.layers.18.post_attention_layernorm.weight": "pytorch_model.bin", "transformer.encoder.layers.18.mlp.dense_h_to_4h.weight": "pytorch_model.bin", "transformer.encoder.layers.18.mlp.dense_4h_to_h.weight": "pytorch_model.bin", "transformer.encoder.layers.19.input_layernorm.weight": "pytorch_model.bin", "transformer.encoder.layers.19.self_attention.query.weight": "pytorch_model.bin", "transformer.encoder.layers.19.self_attention.key_value.weight": "pytorch_model.bin", "transformer.encoder.layers.19.self_attention.dense.weight": "pytorch_model.bin", "transformer.encoder.layers.19.post_attention_layernorm.weight": "pytorch_model.bin", "transformer.encoder.layers.19.mlp.dense_h_to_4h.weight": "pytorch_model.bin", "transformer.encoder.layers.19.mlp.dense_4h_to_h.weight": "pytorch_model.bin", "transformer.encoder.layers.20.input_layernorm.weight": "pytorch_model.bin", "transformer.encoder.layers.20.self_attention.query.weight": "pytorch_model.bin", "transformer.encoder.layers.20.self_attention.key_value.weight": "pytorch_model.bin", "transformer.encoder.layers.20.self_attention.dense.weight": "pytorch_model.bin", "transformer.encoder.layers.20.post_attention_layernorm.weight": "pytorch_model.bin", "transformer.encoder.layers.20.mlp.dense_h_to_4h.weight": "pytorch_model.bin", "transformer.encoder.layers.20.mlp.dense_4h_to_h.weight": "pytorch_model.bin", "transformer.encoder.layers.21.input_layernorm.weight": "pytorch_model.bin", "transformer.encoder.layers.21.self_attention.query.weight": "pytorch_model.bin", "transformer.encoder.layers.21.self_attention.key_value.weight": "pytorch_model.bin", "transformer.encoder.layers.21.self_attention.dense.weight": "pytorch_model.bin", "transformer.encoder.layers.21.post_attention_layernorm.weight": "pytorch_model.bin", "transformer.encoder.layers.21.mlp.dense_h_to_4h.weight": "pytorch_model.bin", "transformer.encoder.layers.21.mlp.dense_4h_to_h.weight": "pytorch_model.bin", "transformer.encoder.layers.22.input_layernorm.weight": "pytorch_model.bin", "transformer.encoder.layers.22.self_attention.query.weight": "pytorch_model.bin", "transformer.encoder.layers.22.self_attention.key_value.weight": "pytorch_model.bin", "transformer.encoder.layers.22.self_attention.dense.weight": "pytorch_model.bin", "transformer.encoder.layers.22.post_attention_layernorm.weight": "pytorch_model.bin", "transformer.encoder.layers.22.mlp.dense_h_to_4h.weight": "pytorch_model.bin", "transformer.encoder.layers.22.mlp.dense_4h_to_h.weight": "pytorch_model.bin", "transformer.encoder.layers.23.input_layernorm.weight": "pytorch_model.bin", "transformer.encoder.layers.23.self_attention.query.weight": "pytorch_model.bin", "transformer.encoder.layers.23.self_attention.key_value.weight": "pytorch_model.bin", "transformer.encoder.layers.23.self_attention.dense.weight": "pytorch_model.bin", "transformer.encoder.layers.23.post_attention_layernorm.weight": "pytorch_model.bin", "transformer.encoder.layers.23.mlp.dense_h_to_4h.weight": "pytorch_model.bin", "transformer.encoder.layers.23.mlp.dense_4h_to_h.weight": "pytorch_model.bin", "transformer.encoder.final_layernorm.weight": "pytorch_model.bin", "transformer.output_layer.weight": "pytorch_model.bin", "transformer.rotary_pos_emb.inv_freq": "pytorch_model.bin"}, "metadata": {"total_size": 2696089728}}