Save last model

config.json

@@ -0,0 +1,31 @@
+{
+  "_name_or_path": "deepseek-ai/deepseek-coder-1.3b-instruct",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 32013,
+  "eos_token_id": 32021,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 5504,
+  "max_position_embeddings": 16384,
+  "model_type": "llama",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 16,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": {
+    "factor": 4.0,
+    "type": "linear"
+  },
+  "rope_theta": 100000,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float16",
+  "transformers_version": "4.39.0.dev0",
+  "use_cache": true,
+  "vocab_size": 32256
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 32013,
+  "eos_token_id": 32021,
+  "transformers_version": "4.39.0.dev0"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5fef71ce0ddb4f14be9ebf0332bf1147d0f88c19a280a3cb8e16e736175ddfa8
+size 2692968904

optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb7212d1a5346112ff3ca6be3074c69fe24ca33344eddf25190e0ea85921bb54
+size 2699040122

rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ecbe4210da6226b088a12a1c5bbee9831d038ebcd718dd20565a6867997b182e
+size 14244

run_config.json

@@ -0,0 +1,77 @@
+{
+    "base_model_id": "deepseek-ai/deepseek-coder-1.3b-instruct",
+    "quantitize": "fp16",
+    "dataset": "Arithmetic_Hard",
+    "data_collator": "DataCollatorForLanguageModeling",
+    "peft_config": {
+        "lora": {
+            "r": 32,
+            "lora_alpha": 64,
+            "target_modules": [
+                "q_proj",
+                "k_proj",
+                "v_proj",
+                "o_proj",
+                "gate_proj",
+                "up_proj",
+                "down_proj"
+            ],
+            "bias": "none",
+            "lora_dropout": 0.05,
+            "task_type": "CAUSAL_LM"
+        },
+        "lora_large": {
+            "r": 128,
+            "lora_alpha": 256,
+            "target_modules": [
+                "q_proj",
+                "k_proj",
+                "v_proj",
+                "o_proj",
+                "gate_proj",
+                "up_proj",
+                "down_proj"
+            ],
+            "bias": "none",
+            "lora_dropout": 0.05,
+            "task_type": "CAUSAL_LM"
+        },
+        "p_tuning": {
+            "num_virtual_tokens": 16,
+            "num_transformer_submodules": 1,
+            "token_dim": 2048,
+            "encoder_hidden_size": 2048,
+            "task_type": "CAUSAL_LM"
+        }
+    },
+    "training_args": {
+        "warmup_steps": 500,
+        "per_device_train_batch_size": 4,
+        "per_device_eval_batch_size": 4,
+        "gradient_accumulation_steps": 1,
+        "max_steps": 100000,
+        "learning_rate": 0.0001,
+        "optim": "paged_adamw_8bit",
+        "logging_steps": 100,
+        "save_strategy": "steps",
+        "save_total_limit": 5,
+        "save_steps": 2500,
+        "evaluation_strategy": "steps",
+        "eval_steps": 2500,
+        "weight_decay": 0.01,
+        "report_to": "wandb",
+        "dataloader_num_workers": 4,
+        "load_best_model_at_end": true,
+        "fp16": false,
+        "output_dir": "runs/deepseek-full-hard",
+        "logging_dir": "runs/deepseek-full-hard/logs"
+    },
+    "tokenizer": {
+        "tokenize_config": {
+            "truncation": true,
+            "max_length": 512,
+            "padding": "max_length"
+        },
+        "prompt_template": "config/qa_template.txt"
+    }
+}

scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d6c5b93324fa72d02f994789c5634c26492fde161591c5c12fb60aea61a68f2
+size 1064

tokenizer_config.json

@@ -0,0 +1,35 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "bos_token": {
+    "__type": "AddedToken",
+    "content": "<｜begin▁of▁sentence｜>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "clean_up_tokenization_spaces": false,
+  "eos_token": {
+    "__type": "AddedToken",
+    "content": "<|EOT|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "legacy": true,
+  "model_max_length": 16384,
+  "pad_token": {
+    "__type": "AddedToken",
+    "content": "<｜end▁of▁sentence｜>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sp_model_kwargs": {},
+  "unk_token": null,
+  "tokenizer_class": "LlamaTokenizerFast",
+  "chat_template": "{% if not add_generation_prompt is defined %}\n{% set add_generation_prompt = false %}\n{% endif %}\n{%- set ns = namespace(found=false) -%}\n{%- for message in messages -%}\n    {%- if message['role'] == 'system' -%}\n        {%- set ns.found = true -%}\n    {%- endif -%}\n{%- endfor -%}\n{{bos_token}}{%- if not ns.found -%}\n{{'You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer\\n'}}\n{%- endif %}\n{%- for message in messages %}\n    {%- if message['role'] == 'system' %}\n{{ message['content'] }}\n    {%- else %}\n        {%- if message['role'] == 'user' %}\n{{'### Instruction:\\n' + message['content'] + '\\n'}}\n        {%- else %}\n{{'### Response:\\n' + message['content'] + '\\n<|EOT|>\\n'}}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{% if add_generation_prompt %}\n{{'### Response:'}}\n{% endif %}"
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:841cb7e6f9fc4f19db87ce9d0bb0834d6ed9f6df3d14ff1ec46778f5841bf8d1
+size 4856