commit files to HF hub

added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "<|cls|>": 49152
+}

config.json

@@ -0,0 +1,39 @@
+{
+  "_name_or_path": "HuggingFaceTB/SmolLM2-360M-Instruct",
+  "architectures": [
+    "P2LModel"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "head_dim": 64,
+  "hidden_act": "silu",
+  "hidden_size": 960,
+  "initializer_range": 0.02,
+  "intermediate_size": 2560,
+  "is_llama_config": true,
+  "max_position_embeddings": 8192,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 15,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 5,
+  "pad_token_id": 2,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_interleaved": false,
+  "rope_scaling": null,
+  "rope_theta": 100000,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers.js_config": {
+    "kv_cache_dtype": {
+      "fp16": "float16",
+      "q4f16": "float16"
+    }
+  },
+  "transformers_version": "4.47.1",
+  "use_cache": true,
+  "vocab_size": 49153
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1451a67e3f5db53971f1af091f50b69e43a77292cf4f45d26f9e5fb06cd8f6db
+size 723928926

model_list.json

@@ -0,0 +1,132 @@
+[
+ "amazon-nova-lite-v1.0",
+ "amazon-nova-micro-v1.0",
+ "amazon-nova-pro-v1.0",
+ "athene-70b-0725",
+ "athene-v2-chat",
+ "c4ai-aya-expanse-32b",
+ "c4ai-aya-expanse-8b",
+ "chatgpt-4o-latest-20240808",
+ "chatgpt-4o-latest-20240903",
+ "chatgpt-4o-latest-20241120",
+ "claude-3-5-haiku-20241022",
+ "claude-3-5-sonnet-20240620",
+ "claude-3-5-sonnet-20241022",
+ "claude-3-haiku-20240307",
+ "claude-3-opus-20240229",
+ "claude-3-sonnet-20240229",
+ "codestral-2405",
+ "command-r",
+ "command-r-08-2024",
+ "command-r-plus",
+ "command-r-plus-08-2024",
+ "dbrx-instruct-preview",
+ "deepseek-coder-v2",
+ "deepseek-coder-v2-0724",
+ "deepseek-v2-api-0628",
+ "deepseek-v2.5",
+ "deepseek-v2.5-1210",
+ "deepseek-v3",
+ "gemini-1.5-flash-001",
+ "gemini-1.5-flash-002",
+ "gemini-1.5-flash-8b-001",
+ "gemini-1.5-flash-8b-exp-0827",
+ "gemini-1.5-flash-exp-0827",
+ "gemini-1.5-pro-001",
+ "gemini-1.5-pro-002",
+ "gemini-1.5-pro-api-0409-preview",
+ "gemini-1.5-pro-exp-0801",
+ "gemini-1.5-pro-exp-0827",
+ "gemini-2.0-flash-exp",
+ "gemini-2.0-flash-thinking-exp-1219",
+ "gemini-advanced-0514",
+ "gemini-exp-1114",
+ "gemini-exp-1121",
+ "gemini-exp-1206",
+ "gemma-1.1-2b-it",
+ "gemma-1.1-7b-it",
+ "gemma-2-27b-it",
+ "gemma-2-2b-it",
+ "gemma-2-9b-it",
+ "gemma-2-9b-it-simpo",
+ "glm-4-0116",
+ "glm-4-0520",
+ "glm-4-plus",
+ "gpt-3.5-turbo-0125",
+ "gpt-4-0125-preview",
+ "gpt-4-0314",
+ "gpt-4-0613",
+ "gpt-4-1106-preview",
+ "gpt-4-turbo-2024-04-09",
+ "gpt-4o-2024-05-13",
+ "gpt-4o-2024-08-06",
+ "gpt-4o-mini-2024-07-18",
+ "granite-3.0-2b-instruct",
+ "granite-3.0-8b-instruct",
+ "grok-2-2024-08-13",
+ "grok-2-mini-2024-08-13",
+ "hunyuan-standard-256k",
+ "internlm2_5-20b-chat",
+ "jamba-1.5-large",
+ "jamba-1.5-mini",
+ "llama-2-13b-chat",
+ "llama-2-70b-chat",
+ "llama-3-70b-instruct",
+ "llama-3-8b-instruct",
+ "llama-3.1-405b-instruct-bf16",
+ "llama-3.1-405b-instruct-fp8",
+ "llama-3.1-70b-instruct",
+ "llama-3.1-8b-instruct",
+ "llama-3.1-nemotron-51b-instruct",
+ "llama-3.1-nemotron-70b-instruct",
+ "llama-3.1-tulu-3-70b",
+ "llama-3.1-tulu-3-8b",
+ "llama-3.2-1b-instruct",
+ "llama-3.2-3b-instruct",
+ "llama-3.3-70b-instruct",
+ "ministral-8b-2410",
+ "mistral-7b-instruct-v0.2",
+ "mistral-large-2402",
+ "mistral-large-2407",
+ "mistral-large-2411",
+ "mistral-medium",
+ "mixtral-8x22b-instruct-v0.1",
+ "mixtral-8x7b-instruct-v0.1",
+ "nemotron-4-340b-instruct",
+ "o1-2024-12-17",
+ "o1-mini",
+ "o1-preview",
+ "phi-3-medium-4k-instruct",
+ "phi-3-mini-128k-instruct",
+ "phi-3-mini-4k-instruct",
+ "phi-3-mini-4k-instruct-june-2024",
+ "phi-3-small-8k-instruct",
+ "qwen-max-0428",
+ "qwen-max-0919",
+ "qwen-plus-0828",
+ "qwen1.5-110b-chat",
+ "qwen1.5-14b-chat",
+ "qwen1.5-32b-chat",
+ "qwen1.5-72b-chat",
+ "qwen2-72b-instruct",
+ "qwen2.5-72b-instruct",
+ "qwen2.5-coder-32b-instruct",
+ "qwen2.5-plus-1127",
+ "qwq-32b-preview",
+ "reka-core-20240501",
+ "reka-core-20240722",
+ "reka-core-20240904",
+ "reka-flash-20240722",
+ "reka-flash-20240904",
+ "reka-flash-21b-20240226",
+ "reka-flash-21b-20240226-online",
+ "reka-flash-preview-20240611",
+ "smollm2-1.7b-instruct",
+ "snowflake-arctic-instruct",
+ "yi-1.5-34b-chat",
+ "yi-34b-chat",
+ "yi-large",
+ "yi-large-preview",
+ "yi-lightning",
+ "yi-lightning-lite"
+]

special_tokens_map.json

@@ -0,0 +1,41 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": {
+    "content": "<|im_start|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "cls_token": {
+    "content": "<|cls|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,164 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<repo_name>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<reponame>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": "<file_sep>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6": {
+      "content": "<filename>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "7": {
+      "content": "<gh_stars>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "8": {
+      "content": "<issue_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "9": {
+      "content": "<issue_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "10": {
+      "content": "<issue_closed>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "11": {
+      "content": "<jupyter_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "12": {
+      "content": "<jupyter_text>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "13": {
+      "content": "<jupyter_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "14": {
+      "content": "<jupyter_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "15": {
+      "content": "<jupyter_script>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "16": {
+      "content": "<empty_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49152": {
+      "content": "<|cls|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": "<|im_start|>",
+  "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "cls_token": "<|cls|>",
+  "eos_token": "<|im_end|>",
+  "extra_special_tokens": {},
+  "model_max_length": 8192,
+  "pad_token": "<|im_end|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>",
+  "vocab_size": 49152
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f049748f4c25c3306a5e2b04cc1c544d884c79576e9eed7ad7aad59be7b3bcc0
+size 7032

training_config.json

@@ -0,0 +1,21 @@
+{
+ "proj_name": "SmolLM2-360M-Instruct-rk-full-train-half-batch",
+ "learning_rate": 8e-06,
+ "adam_epsilon": 1e-08,
+ "batch_size": 4,
+ "max_length": 8192,
+ "num_train_epochs": 1,
+ "train_data_path": "full-p2l-data-01082025",
+ "val_data_path": "p2el/canonical_bt_val_data_11092024",
+ "output_dir": "training_outputs",
+ "pretrain_model_name": "HuggingFaceTB/SmolLM2-360M-Instruct",
+ "gradient_accumulation_steps": 16,
+ "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+ "model_type": "llama",
+ "head_type": "rk",
+ "loss_type": "rk",
+ "weighted_loss": false,
+ "deepspeed_config_path": "deepspeed/zero1.json",
+ "init_type": "reset_params",
+ "load_train_data_from_disk": true
+}