Upload folder using huggingface_hub

README.md

@@ -0,0 +1,237 @@
+---
+language:
+  - en
+license: apache-2.0
+tags:
+  - text-generation
+  - large-language-model
+  - orpo
+dataset:
+  - jondurbin/truthy-dpo-v0.1
+  - AlekseyKorshuk/evol-codealpaca-v1-dpo
+  - argilla/distilabel-intel-orca-dpo-pairs
+  - argilla/ultrafeedback-binarized-avg-rating-for-dpo-filtered
+  - snorkelai/Snorkel-Mistral-PairRM-DPO-Dataset
+  - mlabonne/orpo-dpo-mix-40k
+
+base_model:
+  - TinyLlama/TinyLlama-1.1B-Chat-v1.0
+model-index:
+  - name: Coven Tiny 1.1B
+    description: "Coven Tiny 1.1B is a derivative of TinyLlama 1.1B Chat, fine-tuned to perform specialized tasks involving deeper understanding and reasoning over context. This model exhibits strong capabilities in both general language understanding and task-specific challenges."
+    results:
+      - task:
+          type: text-generation
+          name: Winogrande Challenge
+        dataset:
+          name: Winogrande
+          type: winogrande
+          config: winogrande_xl
+          split: test
+          args:
+            num_few_shot: 5
+        metrics:
+          - type: accuracy
+            value: 61.17
+            name: accuracy
+      - task:
+          type: text-generation
+          name: TruthfulQA Generation
+        dataset:
+          name: TruthfulQA
+          type: truthful_qa
+          config: multiple_choice
+          split: validation
+          args:
+            num_few_shot: 0
+        metrics:
+          - type: accuracy
+            value: 34.31
+            name: accuracy
+      - task:
+          type: text-generation
+          name: PIQA Problem Solving
+        dataset:
+          name: PIQA
+          type: piqa
+          split: validation
+          args:
+            num_few_shot: 5
+        metrics:
+          - type: accuracy
+            value: 71.06
+            name: accuracy
+      - task:
+          type: text-generation
+          name: OpenBookQA Facts
+        dataset:
+          name: OpenBookQA
+          type: openbookqa
+          split: test
+          args:
+            num_few_shot: 5
+        metrics:
+          - type: accuracy
+            value: 30.60
+            name: accuracy
+      - task:
+          type: text-generation
+          name: MMLU Knowledge Test
+        dataset:
+          name: MMLU
+          type: mmlu
+          config: all
+          split: test
+          args:
+            num_few_shot: 5
+        metrics:
+          - type: accuracy
+            value: 38.03
+            name: accuracy
+      - task:
+          type: text-generation
+          name: Hellaswag Contextual Completions
+        dataset:
+          name: Hellaswag
+          type: hellaswag
+          split: validation
+          args:
+            num_few_shot: 10
+        metrics:
+          - type: accuracy
+            value: 43.44
+            name: accuracy
+      - task:
+          type: text-generation
+          name: GSM8k Mathematical Reasoning
+        dataset:
+          name: GSM8k
+          type: gsm8k
+          split: test
+          args:
+            num_few_shot: 5
+        metrics:
+          - type: accuracy
+            value: 14.71
+            name: exact match (strict)
+          - type: accuracy
+            value: 14.63
+            name: exact match (flexible)
+      - task:
+          type: text-generation
+          name: BoolQ Question Answering
+        dataset:
+          name: BoolQ
+          type: boolq
+          split: validation
+          args:
+            num_few_shot: 5
+        metrics:
+          - type: accuracy
+            value: 65.20
+            name: accuracy
+      - task:
+          type: text-generation
+          name: ARC Challenge
+        dataset:
+          name: ARC Challenge
+          type: ai2_arc
+          split: test
+          args:
+            num_few_shot: 25
+        metrics:
+          - type: accuracy
+            value: 34.81
+            name: accuracy
+---
+
+
+
+# 🤏 Coven Tiny 1.1B 32K ORPO
+
+
+Coven Tiny 1.1B 32K is an improved iteration of TinyLlama-1.1B-Chat-v1.0, refined to expand processing capabilities and refine language model preferences. This model includes a significantly increased context limit of 32K tokens, allowing for more extensive data processing and understanding of complex language scenarios. In addition, Coven Tiny 1.1B 32K uses the innovative ORPO (Monolithic Preference Optimization without Reference Model) technique. ORPO simplifies the fine-tuning process by directly optimizing the odds ratio to distinguish between favorable and unfavorable generation styles, effectively improving model performance without the need for an additional preference alignment step. 
+
+## Model Details
+
+* **Model name**: Coven Tiny 1.1B 32K ORPO alpha
+* **Fine-tuned by**: raidhon
+* **Base model**: [TinyLlama-1.1B-Chat-v1.0](https://huggingface.co/TinyLlama/TinyLlama-1.1B-Chat-v1.0)
+* **Parameters**: 1.1B
+* **Context**: 32K
+* **Language(s)**: Multilingual
+* **License**: Apache2.0
+
+
+
+### Eval
+
+
+| Task                | Model                 | Metric         | Value    | Change (%)      |
+|---------------------|-----------------------|----------------|----------|-----------------|
+| Winogrande          | TinyLlama 1.1B Chat   | Accuracy       | 61.56%   | -               |
+|                     | Coven Tiny 1.1B       | Accuracy       | 61.17%   | -0.63%          |
+| TruthfulQA          | TinyLlama 1.1B Chat   | Accuracy       | 30.43%   | -               |
+|                     | Coven Tiny 1.1B       | Accuracy       | 34.31%   | +12.75%         |
+| PIQA                | TinyLlama 1.1B Chat   | Accuracy       | 74.10%   | -               |
+|                     | Coven Tiny 1.1B       | Accuracy       | 71.06%   | -4.10%          |
+| OpenBookQA          | TinyLlama 1.1B Chat   | Accuracy       | 27.40%   | -               |
+|                     | Coven Tiny 1.1B       | Accuracy       | 30.60%   | +11.68%         |
+| MMLU                | TinyLlama 1.1B Chat   | Accuracy       | 24.31%   | -               |
+|                     | Coven Tiny 1.1B       | Accuracy       | 38.03%   | +56.44%         |
+| Hellaswag           | TinyLlama 1.1B Chat   | Accuracy       | 45.69%   | -               |
+|                     | Coven Tiny 1.1B       | Accuracy       | 43.44%   | -4.92%          |
+| GSM8K (Strict)      | TinyLlama 1.1B Chat   | Exact Match    | 1.82%    | -               |
+|                     | Coven Tiny 1.1B       | Exact Match    | 14.71%   | +708.24%        |
+| GSM8K (Flexible)    | TinyLlama 1.1B Chat   | Exact Match    | 2.65%    | -               |
+|                     | Coven Tiny 1.1B       | Exact Match    | 14.63%   | +452.08%        |
+| BoolQ               | TinyLlama 1.1B Chat   | Accuracy       | 58.69%   | -               |
+|                     | Coven Tiny 1.1B       | Accuracy       | 65.20%   | +11.09%         |
+| ARC Easy            | TinyLlama 1.1B Chat   | Accuracy       | 66.54%   | -               |
+|                     | Coven Tiny 1.1B       | Accuracy       | 57.24%   | -13.98%         |
+| ARC Challenge       | TinyLlama 1.1B Chat   | Accuracy       | 34.13%   | -               |
+|                     | Coven Tiny 1.1B       | Accuracy       | 34.81%   | +1.99%          |
+| Humaneval           | TinyLlama 1.1B Chat   | Pass@1         | 10.98%   | -               |
+|                     | Coven Tiny 1.1B       | Pass@1         | 10.37%   | -5.56%          |
+| Drop                | TinyLlama 1.1B Chat   | Score          | 16.02%   | -               |
+|                     | Coven Tiny 1.1B       | Score          | 16.36%   | +2.12%          |
+| BBH                 | Coven Tiny 1.1B       | Average        | 29.02%   | -               |
+
+
+
+## 💻 Usage
+
+```python
+# Install transformers from source - only needed for versions <= v4.34
+# pip install git+https://github.com/huggingface/transformers.git
+# pip install accelerate
+
+import torch
+from transformers import pipeline
+
+pipe = pipeline("text-generation", model="raidhon/coven_tiny_1.1b_32k_orpo_alpha", torch_dtype=torch.bfloat16, device_map="auto")
+
+messages = [
+    {
+        "role": "system",
+        "content": "You are a friendly chatbot who always responds in the style of a pirate",
+    },
+    {"role": "user", "content": "How many helicopters can a human eat in one sitting?"},
+]
+prompt = pipe.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+outputs = pipe(prompt, max_new_tokens=2048, do_sample=True, temperature=0.7, top_k=50, top_p=0.95)
+print(outputs[0]["generated_text"])
+```
+
+
+
+
+
+
+
+
+
+
+
+
+

config.json

@@ -0,0 +1,28 @@
+{
+  "_name_or_path": "raidhon/coven_tiny_1.1b_32k_orpo_alpha",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 5632,
+  "max_position_embeddings": 32768,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 22,
+  "num_key_value_heads": 4,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float16",
+  "transformers_version": "4.39.3",
+  "use_cache": true,
+  "vocab_size": 32000
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.39.3"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c6bf1de78c73f368c293c71f46e74e95dded9a96d34ee7da90a6253ff4fdfc1
+size 2200119664

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

tokenizer_config.json

@@ -0,0 +1,47 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": true,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "chat_template": "{% if messages[0]['role'] == 'system' %}{% set system_message = messages[0]['content'] %}{% endif %}{% if system_message is defined %}{{ system_message + '\\n' }}{% endif %}{% for message in messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ 'Human: ' + content + '\\nAssistant: ' }}{% elif message['role'] == 'assistant' %}{{ content + '</s>' + '\\n' }}{% endif %}{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "</s>",
+  "padding_side": "left",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "split_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "trust_remote_code": false,
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false,
+  "use_fast": true
+}