Upload folder using huggingface_hub

.gitattributes.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4f958c3ffbc6d56674862b5fa2c1108f4f8423b3c4d68baa514da49cea8cc41
+size 234

README.md.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b2cdd5c7a20eee9878696d181518f99e75712943315a0d486503573f9685b22f
+size 1832

config.json

@@ -0,0 +1,25 @@
+{
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.36.0",
+  "use_cache": true,
+  "vocab_size": 32000
+}

generation_config.json.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9347ec7bd04b2842e56166aba8e77951ee01f0b90eb8eb6369dfef4c71cb40db
+size 130

model.safetensors.index.json.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea6607c0462fd5e531ed494292771b6d0f80ad9aec33b9fed6172aef84cd69a2
+size 1220

pytorch_model-00001-of-00003.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d8836f675fe1c4c43f3ff4e93f4cc0e97ef7a13e8c240fb39ad02d37ff303ef5
+size 4943184288

pytorch_model-00002-of-00003.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58a7ddffb463397de5dbe1f1e2ec1ccf6aae2b549565f83f3ded124e0b4c5069
+size 4999843272

pytorch_model-00003-of-00003.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:75824d68dcf82d02b731b2bdfd3a9711acb7c58b8d566f4c0d3e9efac52f9a21
+size 5064824210

pytorch_model.bin.index.json.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1d102e7e25a15163ceb36a16f33e5528d0e09166d7239f035fbcce9e6d5d06c8
+size 1173

special_tokens_map.json.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ead195241576930d3f554cef07b3af64b461bd1be45f6fcd48027c1011a17faa
+size 86

tokenizer.json.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7117e273fc04b34070250598e898ad986f3e7d6d35175c515115b320fe8e999
+size 470451

tokenizer.model.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e79974504b88d369289ba3f7a5c19edc1ca2877be925e163f3ea39a2661b4a0
+size 220844

tokenizer_config.json.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:159ba10d29e920f92b586c39106859c2622408b47e8bc3f322f738f272f05aaa
+size 564

torchtune_config.yaml

@@ -0,0 +1,62 @@
+tokenizer:
+  _component_: torchtune.models.mistral.mistral_tokenizer
+  path: /tmp/Mistral-7B-Instruct-v0.2/tokenizer.model
+dataset:
+  _component_: torchtune.datasets.instruct_dataset
+  source: prithviraj-maurya/legalbench-entire
+  template: AlpacaInstructTemplate
+  column_map:
+    instruction: instruction
+    input: question
+    output: answer
+  max_seq_len: 256
+  train_on_input: true
+  split: train
+seed: null
+shuffle: true
+model:
+  _component_: torchtune.models.mistral.qlora_mistral_7b
+  lora_attn_modules:
+  - q_proj
+  - k_proj
+  - v_proj
+  apply_lora_to_mlp: true
+  apply_lora_to_output: false
+  lora_rank: 64
+  lora_alpha: 16
+checkpointer:
+  _component_: torchtune.utils.FullModelHFCheckpointer
+  checkpoint_dir: /tmp/Mistral-7B-Instruct-v0.2
+  checkpoint_files:
+  - pytorch_model-00001-of-00003.bin
+  - pytorch_model-00002-of-00003.bin
+  - pytorch_model-00003-of-00003.bin
+  recipe_checkpoint: null
+  output_dir: /tmp/Mistral-7B-Instruct-v0.2
+  model_type: MISTRAL
+resume_from_checkpoint: false
+optimizer:
+  _component_: torch.optim.AdamW
+  lr: 2.0e-05
+lr_scheduler:
+  _component_: torchtune.modules.get_cosine_schedule_with_warmup
+  num_warmup_steps: 100
+loss:
+  _component_: torch.nn.CrossEntropyLoss
+batch_size: 4
+epochs: 10
+max_steps_per_epoch: 100000
+gradient_accumulation_steps: 4
+compile: false
+device: cuda
+enable_activation_checkpointing: true
+dtype: fp32
+output_dir: /logs/mistral_7b_qlora_single_device_finetune
+metric_logger:
+  _component_: torchtune.utils.metric_logging.WandBLogger
+  project: torchtune
+log_every_n_steps: 10
+profiler:
+  _component_: torchtune.utils.profiler
+  enabled: false
+  output_dir: /tmp/alpaca-llama2-finetune/torchtune_perf_tracing.json