Upload QLoRA adapter - DeepSeek-Coder 6.7B fine-tuned on Java/Spring Boot + React/TypeScript

README.md

@@ -0,0 +1,160 @@
+---
+library_name: peft
+license: other
+base_model: deepseek-ai/deepseek-coder-6.7b-instruct
+tags:
+- axolotl
+- base_model:adapter:deepseek-ai/deepseek-coder-6.7b-instruct
+- lora
+- transformers
+datasets:
+- custom
+pipeline_tag: text-generation
+model-index:
+- name: bridge-cli
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+[<img src="https://raw.githubusercontent.com/axolotl-ai-cloud/axolotl/main/image/axolotl-badge-web.png" alt="Built with Axolotl" width="200" height="32"/>](https://github.com/axolotl-ai-cloud/axolotl)
+<details><summary>See axolotl config</summary>
+
+axolotl version: `0.15.0`
+```yaml
+# Bridge CLI - Spring Boot Fine-Tuning Configuration
+# Optimized for RunPod with budget GPU (RTX 4090/A5000 24GB)
+# Using QLoRA for memory efficiency
+
+base_model: deepseek-ai/deepseek-coder-6.7b-instruct
+model_type: AutoModelForCausalLM
+tokenizer_type: AutoTokenizer
+trust_remote_code: true
+
+# QLoRA Configuration (enables training on 24GB GPU)
+load_in_4bit: true
+adapter: qlora
+lora_r: 16
+lora_alpha: 32
+lora_dropout: 0.05
+lora_target_linear: true
+lora_target_modules:
+  - q_proj
+  - v_proj
+  - k_proj
+  - o_proj
+  - gate_proj
+  - up_proj
+  - down_proj
+
+# Dataset Configuration
+datasets:
+  - path: /workspace/datasets/spring-boot-dataset.jsonl
+    type: alpaca
+  - path: /workspace/datasets/react-dataset.jsonl
+    type: alpaca
+
+dataset_prepared_path: /workspace/prepared_data
+val_set_size: 0.05
+output_dir: /workspace/outputs/bridge-cli
+
+# Training Parameters
+sequence_len: 2048
+sample_packing: true
+pad_to_sequence_len: true
+
+micro_batch_size: 4
+gradient_accumulation_steps: 4
+num_epochs: 3
+learning_rate: 0.0002
+lr_scheduler: cosine
+warmup_ratio: 0.03
+optimizer: adamw_8bit
+
+# Memory Optimization
+gradient_checkpointing: true
+flash_attention: false
+bf16: auto
+tf32: false
+
+# Training Settings
+train_on_inputs: false
+group_by_length: false
+logging_steps: 10
+save_strategy: steps
+save_steps: 100
+eval_steps: 100
+
+# Weights & Biases (optional - remove if not using)
+# wandb_project: bridge-cli
+# wandb_run_id: spring-boot-finetune
+
+# Early stopping
+early_stopping_patience: 3
+
+# For debugging - set to true to test config
+debug: false
+
+# Special tokens
+special_tokens:
+  pad_token: "<|pad|>"
+
+```
+
+</details><br>
+
+# Bridge CLI - Fine-tuned Code Generation Model
+
+This model is a fine-tuned version of [deepseek-ai/deepseek-coder-6.7b-instruct](https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-instruct) on custom Java/Spring Boot (2,307 examples) and React/TypeScript (4,041 examples) datasets in Alpaca instruction format.
+It achieves the following results on the evaluation set:
+- Loss: 0.4579
+- Ppl: 1.5808
+- Memory/max Active (gib): 6.45
+- Memory/max Allocated (gib): 6.45
+- Memory/device Reserved (gib): 10.47
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0002
+- train_batch_size: 4
+- eval_batch_size: 4
+- seed: 42
+- gradient_accumulation_steps: 4
+- total_train_batch_size: 16
+- optimizer: Use OptimizerNames.ADAMW_8BIT with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_steps: 6
+- training_steps: 222
+
+### Training results
+
+| Training Loss | Epoch  | Step | Validation Loss | Ppl     | Active (gib) | Allocated (gib) | Reserved (gib) |
+|:-------------:|:------:|:----:|:---------------:|:-------:|:------------:|:---------------:|:--------------:|
+| No log        | 0      | 0    | 3.3726          | 29.1542 | 6.36         | 6.36            | 12.58          |
+| 0.4787        | 1.3356 | 100  | 0.5115          | 1.6679  | 6.45         | 6.45            | 10.47          |
+| 0.4037        | 2.6711 | 200  | 0.4579          | 1.5808  | 6.45         | 6.45            | 10.47          |
+
+
+### Framework versions
+
+- PEFT 0.18.1
+- Transformers 5.3.0
+- Pytorch 2.10.0+cu128
+- Datasets 4.5.0
+- Tokenizers 0.22.2

adapter_config.json

@@ -0,0 +1,46 @@
+{
+  "alora_invocation_tokens": null,
+  "alpha_pattern": {},
+  "arrow_config": null,
+  "auto_mapping": null,
+  "base_model_name_or_path": "deepseek-ai/deepseek-coder-6.7b-instruct",
+  "bias": "none",
+  "corda_config": null,
+  "ensure_weight_tying": false,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": null,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "peft_version": "0.18.1",
+  "qalora_group_size": 16,
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "q_proj",
+    "up_proj",
+    "o_proj",
+    "k_proj",
+    "down_proj",
+    "v_proj",
+    "gate_proj"
+  ],
+  "target_parameters": [],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ae7c90f5adffd21f676487c3ed53eaa4eb29d63e63a69b489a4891091c9fffdd
+size 159967880

chat_template.jinja

@@ -0,0 +1,26 @@
+{% if not add_generation_prompt is defined %}
+{% set add_generation_prompt = false %}
+{% endif %}
+{%- set ns = namespace(found=false) -%}
+{%- for message in messages -%}
+    {%- if message['role'] == 'system' -%}
+        {%- set ns.found = true -%}
+    {%- endif -%}
+{%- endfor -%}
+{{bos_token}}{%- if not ns.found -%}
+{{'You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer\n'}}
+{%- endif %}
+{%- for message in messages %}
+    {%- if message['role'] == 'system' %}
+{{ message['content'] }}
+    {%- else %}
+        {%- if message['role'] == 'user' %}
+{{'### Instruction:\n' + message['content'] + '\n'}}
+        {%- else %}
+{{'### Response:\n' + message['content'] + '\n<|EOT|>\n'}}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{% if add_generation_prompt %}
+{{'### Response:'}}
+{% endif %}

config.json

@@ -0,0 +1,49 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 32013,
+  "dtype": "bfloat16",
+  "eos_token_id": 32021,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 16384,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "pad_token_id": null,
+  "pretraining_tp": 1,
+  "quantization_config": {
+    "_load_in_4bit": true,
+    "_load_in_8bit": false,
+    "bnb_4bit_compute_dtype": "bfloat16",
+    "bnb_4bit_quant_storage": "bfloat16",
+    "bnb_4bit_quant_type": "nf4",
+    "bnb_4bit_use_double_quant": true,
+    "llm_int8_enable_fp32_cpu_offload": false,
+    "llm_int8_has_fp16_weight": false,
+    "llm_int8_skip_modules": null,
+    "llm_int8_threshold": 6.0,
+    "load_in_4bit": true,
+    "load_in_8bit": false,
+    "quant_method": "bitsandbytes"
+  },
+  "rms_norm_eps": 1e-06,
+  "rope_parameters": {
+    "factor": 4.0,
+    "rope_theta": 100000,
+    "rope_type": "linear",
+    "type": "linear"
+  },
+  "tie_word_embeddings": false,
+  "transformers_version": "5.3.0",
+  "use_cache": false,
+  "vocab_size": 32256
+}

tokenizer_config.json

@@ -0,0 +1,14 @@
+{
+  "add_prefix_space": null,
+  "backend": "tokenizers",
+  "bos_token": "<｜begin▁of▁sentence｜>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|EOT|>",
+  "is_local": false,
+  "model_max_length": 16384,
+  "pad_token": "<|pad|>",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": null,
+  "use_default_system_prompt": false
+}