Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,322 @@
+---
+library_name: peft
+tags:
+- axolotl
+- base_model:adapter:apertus-12b-nonzero-trained/cpt-part2-instruct-part1
+- lora
+- transformers
+datasets:
+- grimulkan/LimaRP-augmented
+- ToastyPigeon/mixed-medical-reasoning-formatted
+- ToastyPigeon/kimi-stories-instruct
+- allura-org/fujin-instruct-v2
+- ToastyPigeon/some-rp-extended
+- allura-forge/koto-instruct-sft-nothink
+base_model: apertus-12b-nonzero-trained/cpt-part2-instruct-part1
+pipeline_tag: text-generation
+model-index:
+- name: apertus-12b-nonzero-trained/part2-instruct
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+[<img src="https://raw.githubusercontent.com/axolotl-ai-cloud/axolotl/main/image/axolotl-badge-web.png" alt="Built with Axolotl" width="200" height="32"/>](https://github.com/axolotl-ai-cloud/axolotl)
+<details><summary>See axolotl config</summary>
+
+axolotl version: `0.13.0.dev0`
+```yaml
+# === Model Configuration ===
+base_model: apertus-12b-nonzero-trained/cpt-part2-instruct-part1
+load_in_8bit: false
+load_in_4bit: false
+
+# === HF Configuration === 
+#hub_model_id: ToastyPigeon/apertus-12b-try-again-s1
+#hub_strategy: "every_save"
+output_dir: apertus-12b-nonzero-trained/part2-instruct
+# === Wandb Tracking ===
+wandb_project: ApertusV3
+# wandb_entity: [WANDB_ENTITY]
+wandb_name: 12b-part2-instruct
+
+# === Training Setup ===
+num_epochs: 1
+micro_batch_size: 2
+gradient_accumulation_steps: 16
+sequence_len: 4096
+#sequence_parallel_degree: 2
+#heads_k_stride: 1
+sample_packing: true
+#pad_to_sequence_len: true
+#temperature: 0.7
+#max_steps: 10
+# === Evaluation ===
+val_set_size: 200
+evals_per_epoch: 10
+#eval_steps: 20
+#max_steps: 60
+#eval_table_size:
+eval_max_new_tokens: 128
+#eval_sample_packing: true
+#eval_strategy: "no"
+
+# === LoRA Configuration ===
+adapter: lora
+lora_model_dir:
+lora_r: 128
+lora_alpha: 16
+lora_dropout: 0
+lora_target_linear:
+lora_target_modules:
+#  - up_proj
+  - down_proj
+#  - gate_proj
+  - q_proj
+  - v_proj
+  - k_proj
+  - o_proj
+#  - input_layernorm
+#  - post_attention_layernorm
+#  - embed_tokens
+#  - lm_head
+
+lora_fan_in_fan_out:
+peft_use_rslora: true
+lora_modules_to_save:
+#  - embed_tokens
+#  - lm_head
+#fix_untrained_tokens: true
+#lora_mlp_kernel: true
+#lora_qkv_kernel: true
+#lora_o_kernel: true
+#unfrozen_parameters:
+#  - model.layers.(2[4-9]|3[0-9]).*
+#  - model.layers.[0-9+].mlp.up_proj
+#  - model.layers.[0-9]+.mlp.down_proj
+#  - model.layers.[0-9+].feedforward_layernorm
+#  - embed_tokens
+#  - lm_head
+#  - model.layers.[0-9]+.self_attn.(q|k|v|o)_proj
+# === Hyperparameter Configuration ===
+#optimizer: apollo_adamw_layerwise
+#warmup_steps: 0
+warmup_ratio: 0.025
+#optimizer: adamw_8bit
+optimizer: adamw_torch_fused
+#optimizer: paged_adamw_8bit
+#optim_args:
+#  enable_stochastic_rounding: true
+#  enable_cautious: true
+#  enable_8bit: true
+# Apollo-mini configuration:
+#optim_args: "proj=random,rank=128,scale=128.0,scale_type=tensor,update_proj_gap=100"
+# Regular Apollo configuration:
+# optim_args: 
+#optim_target_modules: all_linear
+learning_rate: 2e-5
+lr_scheduler: cosine
+#cosine_min_lr_ratio: 0.2
+#lr_scheduler: cosine_with_min_lr
+#lr_scheduler_kwargs:
+#  cosine_min_lr: 1e-6
+weight_decay: 0.01
+max_grad_norm: 2.0
+#warmup_steps: 0
+#warmup_ratio: 0.025
+
+
+# === Data Configuration ===
+#
+#chat_template: jinja
+chat_template: chatml
+special_tokens:
+  eos_token: "<|im_end|>"
+#  eos_token: "</s>"
+#tokenizer_use_mistral_common: true
+shuffle_merged_datasets: true
+datasets:
+#  - path: allura-org/the-anarchist-library
+#    type: completion
+#    split: train[:20%]
+  - path: grimulkan/LimaRP-augmented
+    type: chat_template
+    field_messages: conversations
+    message_property_mappings:
+      role: from
+      content: value
+#  - path: allenai/tulu-3-sft-personas-instruction-following
+#    type: chat_template
+#    split: train[:10%]
+  - path: ToastyPigeon/mixed-medical-reasoning-formatted
+    type: chat_template
+    data_files: mixed-medical-nothink.json
+#    split: train[:10%]
+#  - path: ToastyPigeon/steve-and-marvin
+#    type: completion
+#    data_files: marvin.json
+  - path: ToastyPigeon/kimi-stories-instruct
+    type: chat_template
+#    type: completion
+#  - path: ToastyPigeon/new-story-dataset
+ #   type: customcompletion-regex
+#    type: completion
+#    data_files: new-story-dataset-v2.json
+  - path: allura-org/fujin-instruct-v2
+#    type: customchatml-regex
+    type: chat_template
+    field_messages: conversations
+    message_property_mappings:
+      role: from
+      content: value
+  - path: ToastyPigeon/some-rp-extended
+ #   type: customchatml-regex
+    type: chat_template
+    field_messages: conversations
+    message_property_mappings:
+      role: from
+      content: value
+    roles_to_train: ["user","assistant"]
+    split: train[:30%]
+#  - path: Alfitaria/rosier-inf
+#    type: completion
+#    split: train[70%:]
+  - path: allura-forge/koto-instruct-sft-nothink
+#    type: customchatml-regex
+    type: chat_template
+#    split: train[:50%]
+#    field_messages: conversations
+#    message_property_mappings:
+#      role: from
+#      content: value
+#  - path: ToastyPigeon/SpringDragon
+#    type: customcompletion-regex
+#    type: completion
+#    split: train
+#  - path: ToastyPigeon/erotic-books-clone
+#    type: customcompletion-regex
+#    type: completion
+#    split: train[:50%]
+#    split: train[35%:45%]
+#  - path: ToastyPigeon/tulu-mini
+#    type: chat_template
+dataset_prepared_path: last_run_prepared
+
+
+# === Plugins ===
+plugins:
+  - axolotl.integrations.liger.LigerPlugin
+  - axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin
+
+# === Hardware Optimization ===
+#gradient_checkpointing: true
+liger_rope: true
+liger_rms_norm: true
+liger_layer_norm: true
+liger_glu_activation: true
+#liger_fused_linear_cross_entropy: true
+cut_cross_entropy: true
+
+#deepspeed: ../axolotl/deepspeed_configs/zero2.json
+
+# === FSDP Config === 
+fsdp:
+  - full_shard
+  - auto_wrap
+fsdp_config:
+  fsdp_limit_all_gathers: true
+  fsdp_sync_module_states: true
+  fsdp_offload_params: true
+  fsdp_activation_checkpointing: true
+  fsdp_use_orig_params: true
+  fsdp_cpu_ram_efficient_loading: true
+  fsdp_auto_wrap_policy: TRANSFORMER_BASED_WRAP
+  fsdp_transformer_layer_cls_to_wrap: ApertusDecoderLayer
+  fsdp_state_dict_type: FULL_STATE_DICT
+  fsdp_sharding_strategy: FULL_SHARD
+#fsdp_stage: 2
+#fsdp_final_state_dict_type: FULL_STATE_DICT
+
+# === Checkpointing ===
+#save_steps: 2
+saves_per_epoch: 4
+save_total_limit: 4
+
+# === Advanced Settings ===
+bf16: true
+flash_attention: true
+train_on_inputs: false
+group_by_length: false
+save_safetensors: true
+logging_steps: 1
+seed: 420
+gc_steps: 10
+
+```
+
+</details><br>
+
+# apertus-12b-nonzero-trained/part2-instruct
+
+This model was trained from scratch on the grimulkan/LimaRP-augmented, the ToastyPigeon/mixed-medical-reasoning-formatted, the ToastyPigeon/kimi-stories-instruct, the allura-org/fujin-instruct-v2, the ToastyPigeon/some-rp-extended and the allura-forge/koto-instruct-sft-nothink datasets.
+It achieves the following results on the evaluation set:
+- Loss: 1.1911
+- Memory/max Active (gib): 6.89
+- Memory/max Allocated (gib): 6.88
+- Memory/device Reserved (gib): 8.18
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 2e-05
+- train_batch_size: 2
+- eval_batch_size: 2
+- seed: 420
+- distributed_type: multi-GPU
+- num_devices: 2
+- gradient_accumulation_steps: 16
+- total_train_batch_size: 64
+- total_eval_batch_size: 4
+- optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_steps: 9
+- training_steps: 372
+
+### Training results
+
+| Training Loss | Epoch  | Step | Validation Loss | Active (gib) | Allocated (gib) | Reserved (gib) |
+|:-------------:|:------:|:----:|:---------------:|:------------:|:---------------:|:--------------:|
+| No log        | 0      | 0    | 1.4636          | 6.87         | 6.87            | 8.13           |
+| 1.2677        | 0.1020 | 38   | 1.3280          | 6.89         | 6.88            | 8.18           |
+| 1.1286        | 0.2041 | 76   | 1.2605          | 6.89         | 6.88            | 8.18           |
+| 1.159         | 0.3061 | 114  | 1.2275          | 6.89         | 6.88            | 8.18           |
+| 1.0281        | 0.4081 | 152  | 1.2122          | 6.89         | 6.88            | 8.18           |
+| 1.0781        | 0.5102 | 190  | 1.2033          | 6.89         | 6.88            | 8.18           |
+| 1.0296        | 0.6122 | 228  | 1.1976          | 6.89         | 6.88            | 8.18           |
+| 1.0756        | 0.7142 | 266  | 1.1939          | 6.89         | 6.88            | 8.18           |
+| 1.1134        | 0.8162 | 304  | 1.1921          | 6.89         | 6.88            | 8.18           |
+| 1.0437        | 0.9183 | 342  | 1.1911          | 6.89         | 6.88            | 8.18           |
+
+
+### Framework versions
+
+- PEFT 0.17.1
+- Transformers 4.56.1
+- Pytorch 2.7.1+cu126
+- Datasets 4.0.0
+- Tokenizers 0.22.1

adapter_config.json

@@ -0,0 +1,41 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "apertus-12b-nonzero-trained/cpt-part2-instruct-part1",
+  "bias": "none",
+  "corda_config": null,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": null,
+  "inference_mode": false,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 256,
+  "lora_bias": false,
+  "lora_dropout": 0.2,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "qalora_group_size": 16,
+  "r": 256,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "down_proj",
+    "k_proj",
+    "v_proj",
+    "o_proj",
+    "up_proj",
+    "q_proj"
+  ],
+  "target_parameters": [],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:288cad2b87d032fa17cda386356fcd8220315fce83e91da5f037d143fbcb4113
+size 1283523112

chat_template.jinja

@@ -0,0 +1,4 @@
+{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '
+' + message['content'] + '<|im_end|>' + '
+'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant
+' }}{% endif %}

config.json

@@ -0,0 +1,43 @@
+{
+  "architectures": [
+    "ApertusForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "auto_map": {
+    "AutoConfig": "configuration_apertus.ApertusConfig",
+    "AutoModel": "modeling_apertus.ApertusModel",
+    "AutoModelForCausalLM": "modeling_apertus.ApertusForCausalLM"
+  },
+  "bos_token_id": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 68,
+  "hidden_act": "xielu",
+  "hidden_dropout": 0.0,
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 21504,
+  "max_position_embeddings": 65536,
+  "mlp_bias": false,
+  "model_type": "apertus",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 48,
+  "num_key_value_heads": 8,
+  "pad_token_id": 3,
+  "post_norm": false,
+  "qk_norm": true,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 8.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3",
+    "type": "llama3"
+  },
+  "rope_theta": 12000000,
+  "tie_word_embeddings": false,
+  "transformers_version": "4.56.1",
+  "use_cache": false,
+  "vocab_size": 131072
+}

debug.log

@@ -0,0 +1,23 @@
+[2025-11-26 03:25:26,747] [INFO] [axolotl.utils.data.sft._load_raw_datasets:320] [PID:64100] Loading raw datasets...
+[2025-11-26 03:25:29,268] [INFO] [axolotl.utils.data.wrappers.get_dataset_wrapper:87] [PID:64100] Loading dataset: ToastyPigeon/limarp-augmented-train-last-only with base_type: chat_template and prompt_style: None
+[2025-11-26 03:25:29,972] [WARNING] [huggingface_hub.repocard.content:108] [PID:64100] Repo card metadata block was not found. Setting CardData to empty.
+[2025-11-26 03:25:30,754] [INFO] [axolotl.utils.data.wrappers.get_dataset_wrapper:87] [PID:64100] Loading dataset: ToastyPigeon/mixed-medical-reasoning-formatted with base_type: chat_template and prompt_style: None
+[2025-11-26 03:25:32,584] [WARNING] [huggingface_hub.repocard.content:108] [PID:64100] Repo card metadata block was not found. Setting CardData to empty.
+[2025-11-26 03:25:33,613] [INFO] [axolotl.utils.data.wrappers.get_dataset_wrapper:87] [PID:64100] Loading dataset: ToastyPigeon/kimi-stories-instruct with base_type: chat_template and prompt_style: None
+[2025-11-26 03:25:35,627] [INFO] [axolotl.utils.data.wrappers.get_dataset_wrapper:87] [PID:64100] Loading dataset: allura-forge/koto-instruct-sft-nothink with base_type: chat_template and prompt_style: None
+[2025-11-26 03:25:36,310] [WARNING] [huggingface_hub.repocard.content:108] [PID:64100] Repo card metadata block was not found. Setting CardData to empty.
+[2025-11-26 03:25:37,341] [INFO] [axolotl.utils.data.wrappers.get_dataset_wrapper:87] [PID:64100] Loading dataset: ToastyPigeon/SpringDragon-Instruct with base_type: chat_template and prompt_style: None
+
+[2025-11-26 03:25:52,171] [WARNING] [huggingface_hub.repocard.content:108] [PID:64100] Repo card metadata block was not found. Setting CardData to empty.
+[2025-11-26 03:25:53,391] [INFO] [axolotl.utils.data.wrappers.get_dataset_wrapper:87] [PID:64100] Loading dataset: ToastyPigeon/tulu-mini with base_type: chat_template and prompt_style: None
+
+
+
+
+
+Using Liger RMSNorm!
+
+
+[2025-11-26 14:54:12,836] [WARNING] [py.warnings._showwarnmsg:110] [PID:64100] /root/miniconda3/envs/py3.11/lib/python3.11/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:680: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .
+  warnings.warn(
+

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:355d635f647f0513296e01963d876dc5a10df2853296b11d5be7d00296070faf
+size 17078466