init

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -0,0 +1,137 @@
+# LLM Fine-Tuning with QLoRA
+
+This repository can help to instruct-tune Open LLaMA, RedPajama or StableLM models on consumer hardware using QLoRA (Original implementation [here](https://github.com/artidoro/qlora)). It's mostly based on the original alpaca-lora repo which can be found [here](https://github.com/tloen/alpaca-lora). Please note that this has only been tested on Open LLama 3b and RedPajama 3b Models, but should work with other models. Contributions are welcome!
+
+### Local Setup
+
+1. Install dependencies
+
+   ```bash
+   pip install -r requirements.txt
+   ```
+
+1. If bitsandbytes doesn't work, [install it from source.](https://github.com/TimDettmers/bitsandbytes/blob/main/compile_from_source.md) Windows users can follow [these instructions](https://github.com/tloen/alpaca-lora/issues/17).
+
+## Training (finetune.py)
+
+This file contains a straightforward application of QLoRA PEFT to the Open LLaMA / RedPajama / StableLM model, as well as some code related to prompt construction and tokenization. PRs adapting this code to support larger models are always welcome.
+
+**Example usage:**
+
+For Open LLaMa
+
+    python finetune.py \
+        --base_model 'openlm-research/open_llama_3b_600bt_preview' \
+        --data_path '../datasets/dolly.json' \
+        --num_epochs=3 \
+        --cutoff_len=512 \
+        --group_by_length \
+        --output_dir='./dolly-lora-3b' \
+        --lora_r=16 \
+        --lora_target_modules='[q_proj,v_proj]'
+
+For RedPajama
+
+    python finetune.py   \
+    --base_model='togethercomputer/RedPajama-INCITE-Base-3B-v1' \
+    --data_path='../datasets/dolly.json'   \
+    --num_epochs=3   \
+    --cutoff_len=512   \
+    --group_by_length   \
+    --output_dir='./dolly-lora-rp-3b-t1' \
+    --lora_r=16 \
+    --lora_target_modules='["query_key_value"]' 
+    
+For StableLM
+
+    python finetune.py  \
+    --base_model='stabilityai/stablelm-base-alpha-3b' \
+    --data_path='../datasets/dolly.json' \
+    --num_epochs=3 \
+    --cutoff_len=512 \
+    --group_by_length  \
+    --output_dir='./dolly-lora-st-3b-t1' \
+    --lora_r=16 \
+    --lora_target_modules='["query_key_value"]'
+    
+For Pythia
+
+    python finetune.py  \
+    --base_model='EleutherAI/pythia-6.9b-deduped' \
+    --data_path='../datasets/dolly.json' \
+    --num_epochs=1 \
+    --cutoff_len=512 \
+    --group_by_length  \
+    --output_dir='./dolly-lora-pyt-6b-t1' \
+    --lora_r=8 \
+    --lora_target_modules='["query_key_value"]'
+
+We can also tweak our hyperparameters (similar to alpaca-lora):
+
+    python finetune.py \
+        --base_model 'openlm-research/open_llama_3b_600bt_preview \
+        --data_path 'yahma/alpaca-cleaned' \
+        --output_dir './lora-alpaca' \
+        --batch_size 128 \
+        --micro_batch_size 4 \
+        --num_epochs 3 \
+        --learning_rate 1e-4 \
+        --cutoff_len 512 \
+        --val_set_size 2000 \
+        --lora_r 8 \
+        --lora_alpha 16 \
+        --lora_dropout 0.05 \
+        --lora_target_modules '[q_proj,v_proj]' \
+        --train_on_inputs \
+        --group_by_length
+
+## Inference (generate.py)
+This file reads the foundation model from the Hugging Face model hub and the LoRA weights from trained peft model, and runs a Gradio interface for inference on a specified input. Users should treat this as example code for the use of the model, and modify it as needed.
+
+Example usage:    
+
+For Open LLaMa
+
+    python generate.py \
+        --base_model 'openlm-research/open_llama_3b_600bt_preview' \
+        --lora_weights './lora-alpaca'
+        
+For RedPajama
+
+    python generate.py  \
+    --base_model 'togethercomputer/RedPajama-INCITE-Base-3B-v1'  \
+    --lora_weights './dolly-lora-rp-3b-t1/'
+       
+For StableLM
+
+    python generate.py  \
+    --base_model 'stabilityai/stablelm-base-alpha-3b' \
+    --lora_weights './dolly-lora-st-3b-t1'
+    
+For Pythia
+   
+    python generate.py  \
+    --base_model 'EleutherAI/pythia-6.9b-deduped'  \
+    --lora_weights './dolly-lora-pyt-6b-t1'
+    
+# Acknowledgements
+
+We would like to express our heartfelt gratitude to **Meta** for releasing LLaMA . Without this pioneering technology, the foundations of projects like **Open Llama** and **Alpaca** wouldn't exist. We sincerely appreciate the immense contributions you've made to the field.
+
+Our acknowledgements also extend to the teams behind **Open LLaMA**, **Together Computer**, **Alpaca** and **Alpaca LoRA**.. You can find more about their excellent work on their respective GitHub repositories:
+
+- [Open Llama](https://github.com/openlm-research/open_llama)
+- [Together Computer](https://github.com/togethercomputer)
+- [Alpaca](https://github.com/tatsu-lab/stanford_alpaca)
+- [Alpaca LoRa](https://github.com/tloen/alpaca-lora)
+
+Lastly, we would like to express our thanks to the developers of **QLoRA** and **bitsandbytes** Your efforts have been instrumental in advancing the field, and we're grateful for your contributions. More information about these projects can be found at:
+
+- [QLoRA](https://github.com/artidoro/qlora)
+- [bitsandbytes](https://github.com/TimDettmers/bitsandbytes)
+
+
+Thank you all for your commitment to innovation and for making these projects possible.
+
+
+    

dolly-lora-3b/adapter_config.json

@@ -0,0 +1,17 @@
+{
+  "base_model_name_or_path": "openlm-research/open_llama_3b_600bt_preview",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "lora_alpha": 16,
+  "lora_dropout": 0.05,
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 8,
+  "target_modules": [
+    "q_proj",
+    "v_proj"
+  ],
+  "task_type": "CAUSAL_LM"
+}

dolly-lora-3b/adapter_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9994386523dd08dcb905bb833e23ad228fe9b7eb1928feac87ae57d69aab77ff
+size 10686701

dolly-lora-3b/checkpoint-74/adapter_model/adapter_config.json

@@ -0,0 +1,17 @@
+{
+  "base_model_name_or_path": "openlm-research/open_llama_3b_600bt_preview",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "lora_alpha": 16,
+  "lora_dropout": 0.05,
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 8,
+  "target_modules": [
+    "q_proj",
+    "v_proj"
+  ],
+  "task_type": "CAUSAL_LM"
+}

dolly-lora-3b/checkpoint-74/adapter_model/adapter_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9994386523dd08dcb905bb833e23ad228fe9b7eb1928feac87ae57d69aab77ff
+size 10686701

dolly-lora-3b/runs/May31_16-47-55_w-khoic-qlora-098760734b88449facec48d4db511ae3-54fc475665-pz5rx/1685551678.7273395/events.out.tfevents.1685551678.w-khoic-qlora-098760734b88449facec48d4db511ae3-54fc475665-pz5rx.842.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ce07c5d7cc092e0a36e2c5d44616fb69a854cabd11561af584ae0500ce442f6
+size 5930

dolly-lora-3b/runs/May31_16-47-55_w-khoic-qlora-098760734b88449facec48d4db511ae3-54fc475665-pz5rx/events.out.tfevents.1685551678.w-khoic-qlora-098760734b88449facec48d4db511ae3-54fc475665-pz5rx.842.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23ac1851b68a07d0db801189c0484a26eeb84d3ca84b90a1c7a7c7769d2c9a93
+size 4310

dolly-lora-3b/runs/May31_16-49-31_w-khoic-qlora-098760734b88449facec48d4db511ae3-54fc475665-pz5rx/1685551774.861798/events.out.tfevents.1685551774.w-khoic-qlora-098760734b88449facec48d4db511ae3-54fc475665-pz5rx.1346.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb3648af42d320570d71de09f1921cb16b2db885201453e7a77029cac7bfbfed
+size 5930

dolly-lora-3b/runs/May31_16-49-31_w-khoic-qlora-098760734b88449facec48d4db511ae3-54fc475665-pz5rx/events.out.tfevents.1685551774.w-khoic-qlora-098760734b88449facec48d4db511ae3-54fc475665-pz5rx.1346.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f5054d72271b3ecf30eb0127f0b23ab358bec6706a4850d13ab3950e8b8072c
+size 4310

dolly-lora-3b/runs/May31_17-01-39_w-khoic-qlora-098760734b88449facec48d4db511ae3-54fc475665-pz5rx/1685552502.9188092/events.out.tfevents.1685552502.w-khoic-qlora-098760734b88449facec48d4db511ae3-54fc475665-pz5rx.4093.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a300b9acf63d3b4ba225c8ec9770ab7c5687f6e3dbc32023c0f123fc91058427
+size 5930

dolly-lora-3b/runs/May31_17-01-39_w-khoic-qlora-098760734b88449facec48d4db511ae3-54fc475665-pz5rx/events.out.tfevents.1685552502.w-khoic-qlora-098760734b88449facec48d4db511ae3-54fc475665-pz5rx.4093.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e43353ef5af59f4aecbe429a1193e88ac04bdb3a1bee620add229fb3bb268a5
+size 4309

dolly-lora-3b/runs/May31_17-05-59_w-khoic-qlora-098760734b88449facec48d4db511ae3-54fc475665-pz5rx/1685552767.5582433/events.out.tfevents.1685552767.w-khoic-qlora-098760734b88449facec48d4db511ae3-54fc475665-pz5rx.4863.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc67c328c4d8e75c4f585664f4243601e314aefe3b3e30e2d4910cd33e13c6e6
+size 5930

dolly-lora-3b/runs/May31_17-05-59_w-khoic-qlora-098760734b88449facec48d4db511ae3-54fc475665-pz5rx/events.out.tfevents.1685552767.w-khoic-qlora-098760734b88449facec48d4db511ae3-54fc475665-pz5rx.4863.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d8a39054689c9b92f3b37bb995ac54975f36501510a9baa3d54e80c948e7c7f
+size 4309

dolly-lora-3b/runs/May31_17-07-58_w-khoic-qlora-098760734b88449facec48d4db511ae3-54fc475665-pz5rx/1685552881.8848104/events.out.tfevents.1685552881.w-khoic-qlora-098760734b88449facec48d4db511ae3-54fc475665-pz5rx.5285.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08cdf6a8a8ab9ae4153b83c7279686902be559beb5d1579c77dba4b1e73c3a24
+size 5930

dolly-lora-3b/runs/May31_17-07-58_w-khoic-qlora-098760734b88449facec48d4db511ae3-54fc475665-pz5rx/events.out.tfevents.1685552881.w-khoic-qlora-098760734b88449facec48d4db511ae3-54fc475665-pz5rx.5285.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0114b1fea3d4397c9a1927040d970ddf3663faa5190c8df4fb1d585b3d49990
+size 4309

dolly-lora-3b/runs/May31_17-28-48_w-khoic-qlora-098760734b88449facec48d4db511ae3-54fc475665-pz5rx/1685554131.1705441/events.out.tfevents.1685554131.w-khoic-qlora-098760734b88449facec48d4db511ae3-54fc475665-pz5rx.8874.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a384809fc162bf124c334745be429c13dad103693abc38e2556c050025cf4817
+size 5930

dolly-lora-3b/runs/May31_17-28-48_w-khoic-qlora-098760734b88449facec48d4db511ae3-54fc475665-pz5rx/events.out.tfevents.1685554131.w-khoic-qlora-098760734b88449facec48d4db511ae3-54fc475665-pz5rx.8874.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6128c73e587f0f0276cacf8dca62bfeef1730b5879dd7f2e7c2a9d198c260c31
+size 4309

dolly-lora-3b/runs/May31_17-30-04_w-khoic-qlora-098760734b88449facec48d4db511ae3-54fc475665-pz5rx/1685554207.7913668/events.out.tfevents.1685554207.w-khoic-qlora-098760734b88449facec48d4db511ae3-54fc475665-pz5rx.9110.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:079d66231f1ebdab51f24f9c444c3a31d488fa95bc9ef1c7ed068992b48588f2
+size 5930

dolly-lora-3b/runs/May31_17-30-04_w-khoic-qlora-098760734b88449facec48d4db511ae3-54fc475665-pz5rx/events.out.tfevents.1685554207.w-khoic-qlora-098760734b88449facec48d4db511ae3-54fc475665-pz5rx.9110.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4a37d0fab4bae8c37067eb913b7da6d2c88a7adeb39c593dc2cf5a8f0127ded
+size 4309

dolly-lora-3b/runs/May31_17-34-31_w-khoic-qlora-098760734b88449facec48d4db511ae3-54fc475665-pz5rx/1685554474.8726966/events.out.tfevents.1685554474.w-khoic-qlora-098760734b88449facec48d4db511ae3-54fc475665-pz5rx.9915.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dccbdf3330ae0a134e4fa97f683e0eb37bc07c4dd62c6f681aa2b76bd95eccb6
+size 5930

dolly-lora-3b/runs/May31_17-34-31_w-khoic-qlora-098760734b88449facec48d4db511ae3-54fc475665-pz5rx/events.out.tfevents.1685554474.w-khoic-qlora-098760734b88449facec48d4db511ae3-54fc475665-pz5rx.9915.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7fffa738dfab5cf45ba68e04d9e9d17b95321097bfe6f501ae91950e0a4c792c
+size 4309

dolly-lora-3b/runs/May31_17-42-59_w-khoic-qlora-098760734b88449facec48d4db511ae3-54fc475665-pz5rx/1685554982.2030103/events.out.tfevents.1685554982.w-khoic-qlora-098760734b88449facec48d4db511ae3-54fc475665-pz5rx.11407.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5403e793d8622a65fb49a01bf25130b93b1bf9b48f457d41ac197b7111fe133e
+size 5930

dolly-lora-3b/runs/May31_17-42-59_w-khoic-qlora-098760734b88449facec48d4db511ae3-54fc475665-pz5rx/events.out.tfevents.1685554982.w-khoic-qlora-098760734b88449facec48d4db511ae3-54fc475665-pz5rx.11407.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:858665f2997bf7f6929b860a28ad5354a37d0904ef83ec2d4884d61d0582fb6d
+size 4309

dolly-lora-3b/runs/May31_17-44-58_w-khoic-qlora-098760734b88449facec48d4db511ae3-54fc475665-pz5rx/1685555101.7508595/events.out.tfevents.1685555101.w-khoic-qlora-098760734b88449facec48d4db511ae3-54fc475665-pz5rx.11777.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3df00314ee19aa289acf77decfd3d558c88d27df64e3136af72ac0c141b84409
+size 5930

dolly-lora-3b/runs/May31_17-44-58_w-khoic-qlora-098760734b88449facec48d4db511ae3-54fc475665-pz5rx/events.out.tfevents.1685555101.w-khoic-qlora-098760734b88449facec48d4db511ae3-54fc475665-pz5rx.11777.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32cfb1a287bc6ce94e6ad35d7d02f4fc34087ab0207b7b9c2caad660b8fd2635
+size 4309

dolly-lora-3b/runs/May31_17-50-31_w-khoic-qlora-098760734b88449facec48d4db511ae3-54fc475665-pz5rx/1685555433.7950974/events.out.tfevents.1685555433.w-khoic-qlora-098760734b88449facec48d4db511ae3-54fc475665-pz5rx.12764.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6aab8b85c9d211f7b1065237aa35305a9cf6d936feffcec665cf2684195776b6
+size 5930

dolly-lora-3b/runs/May31_17-50-31_w-khoic-qlora-098760734b88449facec48d4db511ae3-54fc475665-pz5rx/events.out.tfevents.1685555433.w-khoic-qlora-098760734b88449facec48d4db511ae3-54fc475665-pz5rx.12764.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fdcb9d7b807013203bf3b9501c3b84ed5ae6452a48bc67f600899dd2fcd50a18
+size 5735

finetune.py

@@ -0,0 +1,356 @@
+import os
+import sys
+from typing import Dict, List
+
+import fire
+import torch
+import transformers
+from datasets import load_dataset
+from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig, LlamaTokenizerFast
+from peft import prepare_model_for_kbit_training
+"""
+Unused imports:
+import torch.nn as nn
+import bitsandbytes as bnb
+"""
+
+from peft import (
+    LoraConfig,
+    get_peft_model,
+    get_peft_model_state_dict,
+    prepare_model_for_int8_training,
+    set_peft_model_state_dict,
+)
+from transformers import LlamaForCausalLM, LlamaTokenizer
+
+from utils.prompter import Prompter
+from transformers.trainer_utils import PREFIX_CHECKPOINT_DIR
+from transformers.trainer_callback import TrainerCallback
+
+class SavePeftModelCallback(transformers.TrainerCallback):
+    def save_model(self, args, state, kwargs):
+        print('Saving PEFT checkpoint...')
+        if state.best_model_checkpoint is not None:
+            checkpoint_folder = os.path.join(state.best_model_checkpoint, "adapter_model")
+        else:
+            checkpoint_folder = os.path.join(args.output_dir, f"{PREFIX_CHECKPOINT_DIR}-{state.global_step}")
+
+        peft_model_path = os.path.join(checkpoint_folder, "adapter_model")
+        kwargs["model"].save_pretrained(peft_model_path)
+
+        pytorch_model_path = os.path.join(checkpoint_folder, "pytorch_model.bin")
+        if os.path.exists(pytorch_model_path):
+            os.remove(pytorch_model_path)
+
+    def on_save(self, args, state, control, **kwargs):
+        self.save_model(args, state, kwargs)
+        return control
+
+    def on_train_end(self, args, state, control, **kwargs):
+        def touch(fname, times=None):
+            with open(fname, 'a'):
+                os.utime(fname, times)
+
+        touch(os.path.join(args.output_dir, 'completed'))
+        self.save_model(args, state, kwargs)
+
+
+bnb_config = BitsAndBytesConfig(
+    load_in_4bit=True,
+    bnb_4bit_use_double_quant=True,
+    bnb_4bit_quant_type="nf4",
+    bnb_4bit_compute_dtype=torch.bfloat16
+)
+
+DEFAULT_PAD_TOKEN = "[PAD]"
+
+def print_trainable_parameters(model):
+    """
+    Prints the number of trainable parameters in the model.
+    """
+    trainable_params = 0
+    all_param = 0
+    for _, param in model.named_parameters():
+        all_param += param.numel()
+        if param.requires_grad:
+            trainable_params += param.numel()
+    print(
+        f"trainable params: {trainable_params} || all params: {all_param} || trainable%: {100 * trainable_params / all_param}"
+    )
+
+def smart_tokenizer_and_embedding_resize(
+    special_tokens_dict: Dict,
+    tokenizer: transformers.PreTrainedTokenizer,
+    model: transformers.PreTrainedModel,
+):
+    """Resize tokenizer and embedding.
+
+    Note: This is the unoptimized version that may make your embedding size not be divisible by 64.
+    """
+    num_new_tokens = tokenizer.add_special_tokens(special_tokens_dict)
+    model.resize_token_embeddings(len(tokenizer))
+
+    if num_new_tokens > 0:
+        input_embeddings = model.get_input_embeddings().weight.data
+        output_embeddings = model.get_output_embeddings().weight.data
+
+        input_embeddings_avg = input_embeddings[:-num_new_tokens].mean(dim=0, keepdim=True)
+        output_embeddings_avg = output_embeddings[:-num_new_tokens].mean(dim=0, keepdim=True)
+
+        input_embeddings[-num_new_tokens:] = input_embeddings_avg
+        output_embeddings[-num_new_tokens:] = output_embeddings_avg
+
+
+def train(
+    # model/data params
+    base_model: str = "",  # the only required argument
+    data_path: str = "",
+    output_dir: str = "./lora-alpaca",
+    # training hyperparams
+    batch_size: int = 128,
+    micro_batch_size: int = 4,
+    num_epochs: int = 3,
+    learning_rate: float = 3e-4,
+    cutoff_len: int = 256,
+    val_set_size: int = 2000,
+    # lora hyperparams
+    lora_r: int = 8,
+    lora_alpha: int = 16,
+    lora_dropout: float = 0.05,
+    lora_target_modules: List[str] = [
+        "q_proj",
+        "v_proj",
+    ],
+    # llm hyperparams
+    train_on_inputs: bool = True,  # if False, masks out inputs in loss
+    add_eos_token: bool = False,
+    group_by_length: bool = False,  # faster, but produces an odd training loss curve
+    resume_from_checkpoint: str = None,  # either training checkpoint or final adapter
+    prompt_template_name: str = "alpaca",  # The prompt template to use, will default to alpaca.
+):
+    if int(os.environ.get("LOCAL_RANK", 0)) == 0:
+        print(
+            f"Training Alpaca-LoRA model with params:\n"
+            f"base_model: {base_model}\n"
+            f"data_path: {data_path}\n"
+            f"output_dir: {output_dir}\n"
+            f"batch_size: {batch_size}\n"
+            f"micro_batch_size: {micro_batch_size}\n"
+            f"num_epochs: {num_epochs}\n"
+            f"learning_rate: {learning_rate}\n"
+            f"cutoff_len: {cutoff_len}\n"
+            f"val_set_size: {val_set_size}\n"
+            f"lora_r: {lora_r}\n"
+            f"lora_alpha: {lora_alpha}\n"
+            f"lora_dropout: {lora_dropout}\n"
+            f"lora_target_modules: {lora_target_modules}\n"
+            f"train_on_inputs: {train_on_inputs}\n"
+            f"add_eos_token: {add_eos_token}\n"
+            f"group_by_length: {group_by_length}\n"
+            f"resume_from_checkpoint: {resume_from_checkpoint or False}\n"
+            f"prompt template: {prompt_template_name}\n"
+        )
+    assert (
+        base_model
+    ), "Please specify a --base_model, e.g. --base_model='huggyllama/llama-7b'"
+    gradient_accumulation_steps = batch_size // micro_batch_size
+
+    prompter = Prompter(prompt_template_name)
+
+    device_map = "auto"
+    world_size = int(os.environ.get("WORLD_SIZE", 1))
+    ddp = world_size != 1
+    if ddp:
+        device_map = {"": int(os.environ.get("LOCAL_RANK") or 0)}
+        gradient_accumulation_steps = gradient_accumulation_steps // world_size
+
+    model = AutoModelForCausalLM.from_pretrained(
+        base_model,
+        quantization_config=bnb_config,
+        device_map=device_map,
+    )
+
+    tokenizer = AutoTokenizer.from_pretrained(base_model)
+    
+    if tokenizer._pad_token is None:
+        smart_tokenizer_and_embedding_resize(
+            special_tokens_dict=dict(pad_token=DEFAULT_PAD_TOKEN),
+            tokenizer=tokenizer,
+            model=model,
+        )
+    if isinstance(tokenizer, LlamaTokenizerFast):
+        # LLaMA tokenizer may not have correct special tokens set.
+        # Check and add them if missing to prevent them from being parsed into different tokens.
+        # Note that these are present in the vocabulary. 
+        # Note also that `model.config.pad_token_id` is 0 which corresponds to `<unk>` token.
+        tokenizer.eos_token_id = model.config.eos_token_id
+        tokenizer.pad_token_id = model.config.pad_token_id
+        if hasattr(model.config, 'unk_token_id'):
+            tokenizer.unk_token_id = model.config.unk_token_id
+        else:
+            tokenizer.unk_token_id = tokenizer.pad_token_id
+            
+
+    #tokenizer.padding_side = "left"  # Allow batched inference
+
+    def tokenize(prompt, add_eos_token=True):
+        # there's probably a way to do this with the tokenizer settings
+        # but again, gotta move fast
+        result = tokenizer(
+            prompt,
+            truncation=True,
+            max_length=cutoff_len,
+            padding=False,
+            return_tensors=None,
+        )
+        if (
+            result["input_ids"][-1] != tokenizer.eos_token_id
+            and len(result["input_ids"]) < cutoff_len
+            and add_eos_token
+        ):
+            result["input_ids"].append(tokenizer.eos_token_id)
+            result["attention_mask"].append(1)
+
+        result["labels"] = result["input_ids"].copy()
+
+        return result
+
+    def generate_and_tokenize_prompt(data_point):
+        full_prompt = prompter.generate_prompt(
+            data_point["instruction"],
+            data_point["input"],
+            data_point["output"],
+        )
+        tokenized_full_prompt = tokenize(full_prompt)
+        if not train_on_inputs:
+            user_prompt = prompter.generate_prompt(
+                data_point["instruction"], data_point["input"]
+            )
+            tokenized_user_prompt = tokenize(
+                user_prompt, add_eos_token=add_eos_token
+            )
+            user_prompt_len = len(tokenized_user_prompt["input_ids"])
+
+            if add_eos_token:
+                user_prompt_len -= 1
+
+            tokenized_full_prompt["labels"] = [
+                -100
+            ] * user_prompt_len + tokenized_full_prompt["labels"][
+                user_prompt_len:
+            ]  # could be sped up, probably
+        return tokenized_full_prompt
+
+    model = prepare_model_for_kbit_training(model)
+
+    config = LoraConfig(
+        r=lora_r,
+        lora_alpha=lora_alpha,
+        target_modules=lora_target_modules,
+        lora_dropout=lora_dropout,
+        bias="none",
+        task_type="CAUSAL_LM",
+    )
+    model = get_peft_model(model, config)
+
+    if data_path.endswith(".json") or data_path.endswith(".jsonl"):
+        data = load_dataset("json", data_files=data_path)
+    else:
+        data = load_dataset(data_path)
+
+    if resume_from_checkpoint:
+        # Check the available weights and load them
+        checkpoint_name = os.path.join(
+            resume_from_checkpoint, "pytorch_model.bin"
+        )  # Full checkpoint
+        if not os.path.exists(checkpoint_name):
+            checkpoint_name = os.path.join(
+                resume_from_checkpoint, "adapter_model.bin"
+            )  # only LoRA model - LoRA config above has to fit
+            resume_from_checkpoint = (
+                False  # So the trainer won't try loading its state
+            )
+        # The two files above have a different name depending on how they were saved, but are actually the same.
+        if os.path.exists(checkpoint_name):
+            print(f"Restarting from {checkpoint_name}")
+            adapters_weights = torch.load(checkpoint_name)
+            set_peft_model_state_dict(model, adapters_weights)
+        else:
+            print(f"Checkpoint {checkpoint_name} not found")
+
+    print_trainable_parameters(model) # Be more transparent about the % of trainable params.
+    if val_set_size > 0:
+        train_val = data["train"].train_test_split(
+            test_size=val_set_size, shuffle=True, seed=42
+        )
+        train_data = (
+            train_val["train"].shuffle().map(generate_and_tokenize_prompt)
+        )
+        val_data = (
+            train_val["test"].shuffle().map(generate_and_tokenize_prompt)
+        )
+    else:
+        train_data = data["train"].shuffle().map(generate_and_tokenize_prompt)
+        val_data = None
+
+    trainer = transformers.Trainer(
+        model=model,
+        train_dataset=train_data,
+        eval_dataset=val_data,
+        args=transformers.TrainingArguments(
+            per_device_train_batch_size=micro_batch_size,
+            gradient_accumulation_steps=gradient_accumulation_steps,
+            warmup_steps=10,
+            num_train_epochs=num_epochs,
+            learning_rate=learning_rate,
+#            fp16=True,
+            logging_steps=10,
+            optim="paged_adamw_8bit",
+            evaluation_strategy="steps" if val_set_size > 0 else "no",
+            save_strategy="steps",
+            eval_steps=100 if val_set_size > 0 else None,
+            save_steps=100,
+            output_dir=output_dir,
+            save_total_limit=3,
+            #load_best_model_at_end=True if val_set_size > 0 else False,
+            load_best_model_at_end=False,
+            ddp_find_unused_parameters=False if ddp else None,
+            group_by_length=group_by_length,
+            report_to=None,
+            run_name=None,
+        ),
+        data_collator=transformers.DataCollatorForSeq2Seq(
+            tokenizer, pad_to_multiple_of=8, return_tensors="pt", padding=True
+        ),
+        callbacks=[SavePeftModelCallback]
+    )
+    model.config.use_cache = False
+
+    # if not ddp and torch.cuda.device_count() > 1:
+    #     # keeps Trainer from trying its own DataParallelism when more than 1 gpu is available
+    #     model.is_parallelizable = True
+    #     model.model_parallel = True
+
+#    old_state_dict = model.state_dict
+#    model.state_dict = (
+#        lambda self, *_, **__: get_peft_model_state_dict(
+#            self, old_state_dict()
+#        )
+#    ).__get__(model, type(model))
+
+
+    #if torch.__version__ >= "2" and sys.platform != "win32":
+    #    model = torch.compile(model)
+
+    trainer.train(resume_from_checkpoint=resume_from_checkpoint)
+
+    model.save_pretrained(output_dir)
+
+    print(
+        "\n If there's a warning about missing keys above, please disregard :)"
+    )
+
+
+if __name__ == "__main__":
+    fire.Fire(train)
+

generate.py

@@ -0,0 +1,222 @@
+import os
+import sys
+
+import fire
+import gradio as gr
+import torch
+import transformers
+from peft import PeftModel
+from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
+
+from transformers import GenerationConfig, LlamaForCausalLM, LlamaTokenizer
+
+from utils.callbacks import Iteratorize, Stream
+from utils.prompter import Prompter
+
+if torch.cuda.is_available():
+    device = "cuda"
+else:
+    device = "cpu"
+
+try:
+    if torch.backends.mps.is_available():
+        device = "mps"
+except:  # noqa: E722
+    pass
+
+
+def main(
+    load_8bit: bool = False,
+    base_model: str = "",
+    lora_weights: str = "hfrepo/lora-model",
+    prompt_template: str = "",  # The prompt template to use, will default to alpaca.
+    server_name: str = "0.0.0.0",  # Allows to listen on all interfaces by providing '0.
+    share_gradio: bool = False,
+):
+    base_model = base_model or os.environ.get("BASE_MODEL", "")
+    assert (
+        base_model
+    ), "Please specify a --base_model, e.g. --base_model='openlm-research/open_llama_3b_600bt_preview'"
+
+    prompter = Prompter(prompt_template)
+    tokenizer = AutoTokenizer.from_pretrained(base_model)
+    if device == "cuda":
+        model = AutoModelForCausalLM.from_pretrained(
+            base_model,
+            #load_in_8bit=load_8bit,
+            load_in_4bit=True,   
+            torch_dtype=torch.float16,
+            device_map="auto",
+        )
+        model = PeftModel.from_pretrained(
+            model,
+            lora_weights,
+            torch_dtype=torch.float16,
+#                    device_map={'': 0}
+        )
+    elif device == "mps":
+        model = LlamaForCausalLM.from_pretrained(
+            base_model,
+            device_map={"": device},
+            torch_dtype=torch.float16,
+        )
+        model = PeftModel.from_pretrained(
+            model,
+            lora_weights,
+            device_map={"": device},
+            torch_dtype=torch.float16,
+        )
+    else:
+        model = LlamaForCausalLM.from_pretrained(
+            base_model, device_map={"": device}, low_cpu_mem_usage=True
+        )
+        model = PeftModel.from_pretrained(
+            model,
+            lora_weights,
+            device_map={"": device},
+        )
+
+    # unwind broken decapoda-research config
+    model.config.pad_token_id = tokenizer.pad_token_id = 0  # unk
+    model.config.bos_token_id = 1
+    model.config.eos_token_id = 2
+
+    #if not load_8bit:
+    #    model.half()  # seems to fix bugs for some users.
+
+    model.eval()
+    if torch.__version__ >= "2" and sys.platform != "win32":
+        model = torch.compile(model)
+
+    def evaluate(
+        instruction,
+        input=None,
+        temperature=0.1,
+        top_p=0.75,
+        top_k=40,
+        num_beams=4,
+        max_new_tokens=128,
+        stream_output=False,
+        **kwargs,
+    ):
+        prompt = prompter.generate_prompt(instruction, input)
+        inputs = tokenizer(prompt, return_tensors="pt")
+        input_ids = inputs["input_ids"].to(device)
+        generation_config = GenerationConfig(
+            temperature=temperature,
+            top_p=top_p,
+            top_k=top_k,
+            num_beams=num_beams,
+            **kwargs,
+        )
+
+        generate_params = {
+            "input_ids": input_ids,
+            "generation_config": generation_config,
+            "return_dict_in_generate": True,
+            "output_scores": True,
+            "max_new_tokens": max_new_tokens,
+        }
+
+        if stream_output:
+            # Stream the reply 1 token at a time.
+            # This is based on the trick of using 'stopping_criteria' to create an iterator,
+            # from https://github.com/oobabooga/text-generation-webui/blob/ad37f396fc8bcbab90e11ecf17c56c97bfbd4a9c/modules/text_generation.py#L216-L243.
+
+            def generate_with_callback(callback=None, **kwargs):
+                kwargs.setdefault(
+                    "stopping_criteria", transformers.StoppingCriteriaList()
+                )
+                kwargs["stopping_criteria"].append(
+                    Stream(callback_func=callback)
+                )
+                with torch.no_grad():
+                    model.generate(**kwargs)
+
+            def generate_with_streaming(**kwargs):
+                return Iteratorize(
+                    generate_with_callback, kwargs, callback=None
+                )
+
+            with generate_with_streaming(**generate_params) as generator:
+                for output in generator:
+                    # new_tokens = len(output) - len(input_ids[0])
+                    decoded_output = tokenizer.decode(output)
+
+                    if output[-1] in [tokenizer.eos_token_id]:
+                        break
+
+                    yield prompter.get_response(decoded_output)
+            return  # early return for stream_output
+
+        # Without streaming
+        with torch.no_grad():
+            generation_output = model.generate(
+                input_ids=input_ids,
+                generation_config=generation_config,
+                return_dict_in_generate=True,
+                output_scores=True,
+                max_new_tokens=max_new_tokens,
+            )
+        s = generation_output.sequences[0]
+        output = tokenizer.decode(s)
+        yield prompter.get_response(output)
+
+    gr.Interface(
+        fn=evaluate,
+        inputs=[
+            gr.components.Textbox(
+                lines=2,
+                label="Instruction",
+                placeholder="Tell me about alpacas.",
+            ),
+            gr.components.Textbox(lines=2, label="Input", placeholder="none"),
+            gr.components.Slider(
+                minimum=0, maximum=1, value=0.1, label="Temperature"
+            ),
+            gr.components.Slider(
+                minimum=0, maximum=1, value=0.75, label="Top p"
+            ),
+            gr.components.Slider(
+                minimum=0, maximum=100, step=1, value=40, label="Top k"
+            ),
+            gr.components.Slider(
+                minimum=1, maximum=4, step=1, value=4, label="Beams"
+            ),
+            gr.components.Slider(
+                minimum=1, maximum=2000, step=1, value=128, label="Max tokens"
+            ),
+            gr.components.Checkbox(label="Stream output"),
+        ],
+        outputs=[
+            gr.inputs.Textbox(
+                lines=5,
+                label="Output",
+            )
+        ],
+        title="🦙🌲 Alpaca-QLoRA",
+        description="Instruct-tune Open LLaMA on consumer hardware using QLoRA",  # noqa: E501
+    ).queue().launch(server_name="0.0.0.0", share=share_gradio)
+    # Old testing code follows.
+
+    """
+    # testing code for readme
+    for instruction in [
+        "Tell me about alpacas.",
+        "Tell me about the president of Mexico in 2019.",
+        "Tell me about the king of France in 2019.",
+        "List all Canadian provinces in alphabetical order.",
+        "Write a Python program that prints the first 10 Fibonacci numbers.",
+        "Write a program that prints the numbers from 1 to 100. But for multiples of three print 'Fizz' instead of the number and for the multiples of five print 'Buzz'. For numbers which are multiples of both three and five print 'FizzBuzz'.",  # noqa: E501
+        "Tell me five words that rhyme with 'shock'.",
+        "Translate the sentence 'I have no mouth but I must scream' into Spanish.",
+        "Count up from 1 to 500.",
+    ]:
+        print("Instruction:", instruction)
+        print("Response:", evaluate(instruction))
+        print()
+    """
+
+
+if __name__ == "__main__":
+    fire.Fire(main)

requirements.txt

@@ -0,0 +1,11 @@
+bitsandbytes
+git+https://github.com/huggingface/transformers.git
+git+https://github.com/huggingface/peft.git
+git+https://github.com/huggingface/accelerate.git
+datasets
+fire
+scipy
+sentencepiece
+protobuf==3.20.0
+wandb
+gradio

templates/README.md

@@ -0,0 +1,46 @@
+# Prompt templates
+
+This directory contains template styles for the prompts used to finetune LoRA models.
+
+## Format
+
+A template is described via a JSON file with the following keys:
+
+- `prompt_input`: The template to use when input is not None. Uses `{instruction}` and `{input}` placeholders.
+- `prompt_no_input`: The template to use when input is None. Uses `{instruction}` placeholders.
+- `description`: A short description of the template, with possible use cases.
+- `response_split`: The text to use as separator when cutting real response from the model output.
+
+No `{response}` placeholder was used, since the response is always the last element of the template and is just to be concatenated to the rest.
+
+## Example template
+
+The default template, used unless otherwise specified, is `alpaca.json`
+
+```json
+{
+    "description": "Template used by Alpaca-LoRA.",
+    "prompt_input": "Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\n\n### Instruction:\n{instruction}\n\n### Input:\n{input}\n\n### Response:\n",
+    "prompt_no_input": "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n### Instruction:\n{instruction}\n\n### Response:\n",
+    "response_split": "### Response:"    
+}
+
+```
+
+## Current templates
+
+### alpaca
+
+Default template used for generic LoRA fine tunes so far.
+
+### alpaca_legacy
+
+Legacy template used by the original alpaca repo, with no `\n` after the response field. Kept for reference and experiments.
+
+### alpaca_short
+
+A trimmed down alpaca template which seems to perform just as well and spare some tokens. Models created with the default template seem to be queryable by the short tempalte as well. More experiments are welcome.
+
+### vigogne
+
+The default alpaca template, translated to french. This template was used to train the "Vigogne" LoRA and is to be used to query it, or for extra fine tuning.

templates/alpaca.json

@@ -0,0 +1,6 @@
+{
+    "description": "Template used by Alpaca-LoRA.",
+    "prompt_input": "Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\n\n### Instruction:\n{instruction}\n\n### Input:\n{input}\n\n### Response:\n",
+    "prompt_no_input": "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n### Instruction:\n{instruction}\n\n### Response:\n",
+    "response_split": "### Response:"    
+}

templates/alpaca_legacy.json

@@ -0,0 +1,6 @@
+{
+    "description": "Legacy template, used by Original Alpaca repository.",
+    "prompt_input": "Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\n\n### Instruction:\n{instruction}\n\n### Input:\n{input}\n\n### Response:",
+    "prompt_no_input": "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n### Instruction:\n{instruction}\n\n### Response:",
+    "response_split": "### Response:"    
+}

templates/alpaca_short.json

@@ -0,0 +1,6 @@
+{
+    "description": "A shorter template to experiment with.",
+    "prompt_input": "### Instruction:\n{instruction}\n\n### Input:\n{input}\n\n### Response:\n",
+    "prompt_no_input": "### Instruction:\n{instruction}\n\n### Response:\n",
+    "response_split": "### Response:"    
+}

templates/vigogne.json

@@ -0,0 +1,6 @@
+{
+    "description": "French template, used by Vigogne for finetuning.",
+    "prompt_input": "Ci-dessous se trouve une instruction qui décrit une tâche, associée à une entrée qui fournit un contexte supplémentaire. Écrivez une réponse qui complète correctement la demande.\n\n### Instruction:\n{instruction}\n\n### Entrée:\n{input}\n\n### Réponse:\n",
+    "prompt_no_input": "Ci-dessous se trouve une instruction qui décrit une tâche. Écrivez une réponse qui complète correctement la demande.\n\n### Instruction:\n{instruction}\n\n### Réponse:\n",
+    "response_split": "### Réponse:"
+}

utils/README.md

@@ -0,0 +1,13 @@
+# Directory for helpers modules
+
+## prompter.py
+
+Prompter class, a template manager.
+
+`from utils.prompter import Prompter`
+
+## callbacks.py
+
+Helpers to support streaming generate output.
+
+`from utils.callbacks import Iteratorize, Stream`

utils/callbacks.py

@@ -0,0 +1,75 @@
+"""
+Helpers to support streaming generate output.
+Borrowed from https://github.com/oobabooga/text-generation-webui/blob/ad37f396fc8bcbab90e11ecf17c56c97bfbd4a9c/modules/callbacks.py
+"""
+
+import gc
+import traceback
+from queue import Queue
+from threading import Thread
+
+import torch
+import transformers
+
+
+class Stream(transformers.StoppingCriteria):
+    def __init__(self, callback_func=None):
+        self.callback_func = callback_func
+
+    def __call__(self, input_ids, scores) -> bool:
+        if self.callback_func is not None:
+            self.callback_func(input_ids[0])
+        return False
+
+
+class Iteratorize:
+
+    """
+    Transforms a function that takes a callback
+    into a lazy iterator (generator).
+    """
+
+    def __init__(self, func, kwargs={}, callback=None):
+        self.mfunc = func
+        self.c_callback = callback
+        self.q = Queue()
+        self.sentinel = object()
+        self.kwargs = kwargs
+        self.stop_now = False
+
+        def _callback(val):
+            if self.stop_now:
+                raise ValueError
+            self.q.put(val)
+
+        def gentask():
+            try:
+                ret = self.mfunc(callback=_callback, **self.kwargs)
+            except ValueError:
+                pass
+            except:
+                traceback.print_exc()
+                pass
+
+            self.q.put(self.sentinel)
+            if self.c_callback:
+                self.c_callback(ret)
+
+        self.thread = Thread(target=gentask)
+        self.thread.start()
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        obj = self.q.get(True, None)
+        if obj is self.sentinel:
+            raise StopIteration
+        else:
+            return obj
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.stop_now = True

utils/prompter.py

@@ -0,0 +1,51 @@
+"""
+A dedicated helper to manage templates and prompt building.
+"""
+
+import json
+import os.path as osp
+from typing import Union
+
+
+class Prompter(object):
+    __slots__ = ("template", "_verbose")
+
+    def __init__(self, template_name: str = "", verbose: bool = False):
+        self._verbose = verbose
+        if not template_name:
+            # Enforce the default here, so the constructor can be called with '' and will not break.
+            template_name = "alpaca"
+        file_name = osp.join("templates", f"{template_name}.json")
+        if not osp.exists(file_name):
+            raise ValueError(f"Can't read {file_name}")
+        with open(file_name) as fp:
+            self.template = json.load(fp)
+        if self._verbose:
+            print(
+                f"Using prompt template {template_name}: {self.template['description']}"
+            )
+
+    def generate_prompt(
+        self,
+        instruction: str,
+        input: Union[None, str] = None,
+        label: Union[None, str] = None,
+    ) -> str:
+        # returns the full prompt from instruction and optional input
+        # if a label (=response, =output) is provided, it's also appended.
+        if input:
+            res = self.template["prompt_input"].format(
+                instruction=instruction, input=input
+            )
+        else:
+            res = self.template["prompt_no_input"].format(
+                instruction=instruction
+            )
+        if label:
+            res = f"{res}{label}"
+        if self._verbose:
+            print(res)
+        return res
+
+    def get_response(self, output: str) -> str:
+        return output.split(self.template["response_split"])[1].strip()

wandb/debug-internal.log

@@ -0,0 +1 @@
+run-20230531_170802-zezjqg86/logs/debug-internal.log

wandb/debug.log

@@ -0,0 +1 @@
+run-20230531_170802-zezjqg86/logs/debug.log

wandb/latest-run

@@ -0,0 +1 @@
+run-20230531_170802-zezjqg86

wandb/run-20230531_164935-4dg4abji/files/conda-environment.yaml

@@ -0,0 +1,497 @@
+name: saturn
+channels:
+  - pytorch
+  - fastai
+  - rapidsai
+  - conda-forge
+  - defaults
+dependencies:
+  - _libgcc_mutex=0.1=conda_forge
+  - _openmp_mutex=4.5=2_kmp_llvm
+  - abseil-cpp=20211102.0=h93e1e8c_3
+  - absl-py=1.4.0=pyhd8ed1ab_0
+  - aiobotocore=2.2.0=pyhd8ed1ab_0
+  - aiohttp=3.8.4=py39h72bdee0_0
+  - aioitertools=0.11.0=pyhd8ed1ab_0
+  - aiosignal=1.3.1=pyhd8ed1ab_0
+  - alsa-lib=1.2.8=h166bdaf_0
+  - anyio=3.6.2=pyhd8ed1ab_0
+  - aom=3.5.0=h27087fc_0
+  - argon2-cffi=21.3.0=pyhd8ed1ab_0
+  - argon2-cffi-bindings=21.2.0=py39hb9d737c_3
+  - arrow-cpp=6.0.1=py39h461039b_20_cpu
+  - asttokens=2.2.1=pyhd8ed1ab_0
+  - async-timeout=4.0.2=pyhd8ed1ab_0
+  - atk-1.0=2.38.0=hd4edc92_1
+  - attr=2.5.1=h166bdaf_1
+  - attrs=22.2.0=pyh71513ae_0
+  - aws-c-cal=0.5.11=h95a6274_0
+  - aws-c-common=0.6.2=h7f98852_0
+  - aws-c-event-stream=0.2.7=h3541f99_13
+  - aws-c-io=0.10.5=hfb6a706_0
+  - aws-checksums=0.1.11=ha31a3da_7
+  - aws-sdk-cpp=1.8.186=hecaee15_4
+  - backcall=0.2.0=pyh9f0ad1d_0
+  - backports=1.0=pyhd8ed1ab_3
+  - backports.functools_lru_cache=1.6.4=pyhd8ed1ab_0
+  - bcrypt=3.2.2=py39hb9d737c_1
+  - beautifulsoup4=4.11.2=pyha770c72_0
+  - blas=2.114=mkl
+  - blas-devel=3.9.0=14_linux64_mkl
+  - bleach=6.0.0=pyhd8ed1ab_0
+  - blinker=1.5=pyhd8ed1ab_0
+  - bokeh=2.4.3=pyhd8ed1ab_3
+  - botocore=1.24.21=pyhd8ed1ab_1
+  - brotli=1.0.9=h166bdaf_8
+  - brotli-bin=1.0.9=h166bdaf_8
+  - brotlipy=0.7.0=py39hb9d737c_1005
+  - bzip2=1.0.8=h7f98852_4
+  - c-ares=1.18.1=h7f98852_0
+  - ca-certificates=2022.12.7=ha878542_0
+  - cachetools=5.3.0=pyhd8ed1ab_0
+  - cairo=1.16.0=ha61ee94_1014
+  - catalogue=2.0.8=py39hf3d152e_1
+  - certifi=2022.12.7=pyhd8ed1ab_0
+  - cffi=1.15.1=py39he91dace_3
+  - click=8.0.4=py39hf3d152e_0
+  - cloudpickle=2.2.1=pyhd8ed1ab_0
+  - colorama=0.4.6=pyhd8ed1ab_0
+  - commonmark=0.9.1=py_0
+  - confection=0.0.4=py39hcca971b_1
+  - croniter=0.3.36=pyhd8ed1ab_0
+  - cudatoolkit=11.3.1=h9edb442_11
+  - cycler=0.11.0=pyhd8ed1ab_0
+  - cymem=2.0.7=py39h5a03fae_1
+  - cython-blis=0.7.9=py39h2ae25f5_1
+  - cytoolz=0.12.0=py39hb9d737c_1
+  - dask=2022.3.0=pyhd8ed1ab_1
+  - dask-core=2022.3.0=pyhd8ed1ab_0
+  - dask-cuda=22.04.00=py39_0
+  - dataclasses=0.8=pyhc8e2a94_3
+  - dbus=1.13.6=h5008d03_3
+  - debugpy=1.6.6=py39h227be39_0
+  - decorator=5.1.1=pyhd8ed1ab_0
+  - defusedxml=0.7.1=pyhd8ed1ab_0
+  - distributed=2022.3.0=pyhd8ed1ab_0
+  - docker-py=6.0.0=pyhd8ed1ab_0
+  - entrypoints=0.4=pyhd8ed1ab_0
+  - executing=1.2.0=pyhd8ed1ab_0
+  - expat=2.5.0=h27087fc_0
+  - fastai=2.6.3=py_0
+  - fastcore=1.4.5=py_0
+  - fastdownload=0.0.7=py_0
+  - fastprogress=1.0.3=py_0
+  - ffmpeg=4.4.2=gpl_h8dda1f0_112
+  - fftw=3.3.10=nompi_hf0379b8_106
+  - flit-core=3.8.0=pyhd8ed1ab_0
+  - font-ttf-dejavu-sans-mono=2.37=hab24e00_0
+  - font-ttf-inconsolata=3.000=h77eed37_0
+  - font-ttf-source-code-pro=2.038=h77eed37_0
+  - font-ttf-ubuntu=0.83=hab24e00_0
+  - fontconfig=2.14.2=h14ed4e7_0
+  - fonts-conda-ecosystem=1=0
+  - fonts-conda-forge=1=0
+  - fonttools=4.39.0=py39h72bdee0_0
+  - freeglut=3.2.2=h9c3ff4c_1
+  - freetype=2.12.1=hca18f0e_1
+  - fribidi=1.0.10=h36c2ea0_0
+  - frozenlist=1.3.3=py39hb9d737c_0
+  - fsspec=2022.3.0=pyhd8ed1ab_0
+  - future=0.18.3=pyhd8ed1ab_0
+  - gdk-pixbuf=2.42.8=hff1cb4f_1
+  - gettext=0.21.1=h27087fc_0
+  - gflags=2.2.2=he1b5a44_1004
+  - giflib=5.2.1=h0b41bf4_3
+  - glib=2.74.1=h6239696_1
+  - glib-tools=2.74.1=h6239696_1
+  - glog=0.6.0=h6f12383_0
+  - gmp=6.2.1=h58526e2_0
+  - gnutls=3.7.8=hf3e180e_0
+  - google-auth=2.16.2=pyh1a96a4e_0
+  - google-auth-oauthlib=0.4.6=pyhd8ed1ab_0
+  - graphite2=1.3.13=h58526e2_1001
+  - graphviz=5.0.0=h5abf519_0
+  - grpc-cpp=1.46.3=hc275302_1
+  - grpcio=1.46.3=py39hf176720_1
+  - gst-plugins-base=1.21.3=h4243ec0_1
+  - gstreamer=1.21.3=h25f0c4b_1
+  - gstreamer-orc=0.4.33=h166bdaf_0
+  - gtk2=2.24.33=h90689f9_2
+  - gts=0.7.6=h64030ff_2
+  - harfbuzz=4.4.1=hf9f4e7c_0
+  - hdf5=1.12.1=nompi_h2386368_104
+  - heapdict=1.0.1=py_0
+  - icu=70.1=h27087fc_0
+  - idna=3.4=pyhd8ed1ab_0
+  - importlib-metadata=6.0.0=pyha770c72_0
+  - importlib_resources=5.12.0=pyhd8ed1ab_0
+  - ipykernel=6.13.0=py39hef51801_0
+  - ipython=8.11.0=pyh41d4057_0
+  - ipython_genutils=0.2.0=py_1
+  - ipywidgets=7.7.0=pyhd8ed1ab_0
+  - jack=1.9.22=h11f4161_0
+  - jasper=2.0.33=h0ff4b12_1
+  - jedi=0.18.2=pyhd8ed1ab_0
+  - jinja2=3.1.2=pyhd8ed1ab_1
+  - jmespath=1.0.1=pyhd8ed1ab_0
+  - joblib=1.2.0=pyhd8ed1ab_0
+  - jpeg=9e=h0b41bf4_3
+  - jsonschema=4.17.3=pyhd8ed1ab_0
+  - jupyter_client=7.3.4=pyhd8ed1ab_0
+  - jupyter_core=5.2.0=py39hf3d152e_0
+  - jupyter_server=1.23.6=pyhd8ed1ab_0
+  - jupyterlab_pygments=0.2.2=pyhd8ed1ab_0
+  - jupyterlab_widgets=1.1.2=pyhd8ed1ab_0
+  - keyutils=1.6.1=h166bdaf_0
+  - kiwisolver=1.4.4=py39hf939315_1
+  - krb5=1.20.1=hf9c8cef_0
+  - lame=3.100=h166bdaf_1003
+  - langcodes=3.3.0=pyhd8ed1ab_0
+  - lcms2=2.14=h6ed2654_0
+  - ld_impl_linux-64=2.40=h41732ed_0
+  - lerc=4.0.0=h27087fc_0
+  - libabseil=20211102.0=cxx17_h48a1fff_3
+  - libblas=3.9.0=14_linux64_mkl
+  - libbrotlicommon=1.0.9=h166bdaf_8
+  - libbrotlidec=1.0.9=h166bdaf_8
+  - libbrotlienc=1.0.9=h166bdaf_8
+  - libcap=2.66=ha37c62d_0
+  - libcblas=3.9.0=14_linux64_mkl
+  - libclang=15.0.7=default_had23c3d_1
+  - libclang13=15.0.7=default_h3e3d535_1
+  - libcups=2.3.3=h36d4200_3
+  - libcurl=7.87.0=h6312ad2_0
+  - libdb=6.2.32=h9c3ff4c_0
+  - libdeflate=1.14=h166bdaf_0
+  - libdrm=2.4.114=h166bdaf_0
+  - libedit=3.1.20191231=he28a2e2_2
+  - libev=4.33=h516909a_1
+  - libevent=2.1.10=h9b69904_4
+  - libffi=3.4.2=h7f98852_5
+  - libflac=1.4.2=h27087fc_0
+  - libgcc-ng=12.2.0=h65d4601_19
+  - libgcrypt=1.10.1=h166bdaf_0
+  - libgd=2.3.3=h18fbbfe_3
+  - libgfortran-ng=12.2.0=h69a702a_19
+  - libgfortran5=12.2.0=h337968e_19
+  - libglib=2.74.1=h606061b_1
+  - libglu=9.0.0=he1b5a44_1001
+  - libgpg-error=1.46=h620e276_0
+  - libhwloc=2.9.0=hd6dc26d_0
+  - libiconv=1.17=h166bdaf_0
+  - libidn2=2.3.4=h166bdaf_0
+  - liblapack=3.9.0=14_linux64_mkl
+  - liblapacke=3.9.0=14_linux64_mkl
+  - libllvm11=11.1.0=he0ac6c6_5
+  - libllvm15=15.0.7=hadd5161_1
+  - libnghttp2=1.51.0=hdcd2b5c_0
+  - libnsl=2.0.0=h7f98852_0
+  - libogg=1.3.4=h7f98852_1
+  - libopencv=4.5.5=py39hd011c1b_13
+  - libopus=1.3.1=h7f98852_1
+  - libpciaccess=0.17=h166bdaf_0
+  - libpng=1.6.39=h753d276_0
+  - libpq=15.1=h2baec63_3
+  - libprotobuf=3.20.2=h6239696_0
+  - librsvg=2.54.4=h7abd40a_0
+  - libsndfile=1.2.0=hb75c966_0
+  - libsodium=1.0.18=h36c2ea0_1
+  - libsqlite=3.40.0=h753d276_0
+  - libssh2=1.10.0=haa6b8db_3
+  - libstdcxx-ng=12.2.0=h46fd767_19
+  - libsystemd0=252=h2a991cd_0
+  - libtasn1=4.19.0=h166bdaf_0
+  - libthrift=0.16.0=h491838f_2
+  - libtiff=4.4.0=h82bc61c_5
+  - libtool=2.4.7=h27087fc_0
+  - libudev1=253=h0b41bf4_0
+  - libunistring=0.9.10=h7f98852_0
+  - libutf8proc=2.8.0=h166bdaf_0
+  - libuuid=2.32.1=h7f98852_1000
+  - libuv=1.44.2=h166bdaf_0
+  - libva=2.17.0=h0b41bf4_0
+  - libvorbis=1.3.7=h9c3ff4c_0
+  - libvpx=1.11.0=h9c3ff4c_3
+  - libwebp=1.2.4=h522a892_0
+  - libwebp-base=1.2.4=h166bdaf_0
+  - libxcb=1.13=h7f98852_1004
+  - libxkbcommon=1.5.0=h79f4944_1
+  - libxml2=2.10.3=hca2bb57_3
+  - libzlib=1.2.13=h166bdaf_4
+  - llvm-openmp=15.0.7=h0cdce71_0
+  - llvmlite=0.39.1=py39h7d9a04d_1
+  - locket=1.0.0=pyhd8ed1ab_0
+  - lz4=4.3.2=py39h724f13c_0
+  - lz4-c=1.9.4=hcb278e6_0
+  - markdown=3.4.1=pyhd8ed1ab_0
+  - markupsafe=2.1.2=py39h72bdee0_0
+  - marshmallow=3.19.0=pyhd8ed1ab_0
+  - marshmallow-oneofschema=3.0.1=pyhd8ed1ab_0
+  - matplotlib=3.5.2=py39hf3d152e_1
+  - matplotlib-base=3.5.2=py39h700656a_1
+  - matplotlib-inline=0.1.6=pyhd8ed1ab_0
+  - mistune=2.0.5=pyhd8ed1ab_0
+  - mkl=2022.0.1=h8d4b97c_803
+  - mkl-devel=2022.0.1=ha770c72_804
+  - mkl-include=2022.0.1=h8d4b97c_803
+  - mpg123=1.31.2=hcb278e6_0
+  - msgpack-python=1.0.5=py39h4b4f3f3_0
+  - multidict=6.0.4=py39h72bdee0_0
+  - munkres=1.1.4=pyh9f0ad1d_0
+  - murmurhash=1.0.9=py39h5a03fae_1
+  - mypy_extensions=1.0.0=pyha770c72_0
+  - mysql-common=8.0.32=h14678bc_0
+  - mysql-libs=8.0.32=h54cf53e_0
+  - natsort=8.3.1=pyhd8ed1ab_0
+  - nbclassic=0.5.3=pyhb4ecaf3_3
+  - nbclient=0.7.2=pyhd8ed1ab_0
+  - nbconvert=7.2.9=pyhd8ed1ab_0
+  - nbconvert-core=7.2.9=pyhd8ed1ab_0
+  - nbconvert-pandoc=7.2.9=pyhd8ed1ab_0
+  - nbformat=5.7.3=pyhd8ed1ab_0
+  - ncurses=6.3=h27087fc_1
+  - nest-asyncio=1.5.6=pyhd8ed1ab_0
+  - nettle=3.8.1=hc379101_1
+  - notebook=6.5.3=pyha770c72_0
+  - notebook-shim=0.2.2=pyhd8ed1ab_0
+  - nspr=4.35=h27087fc_0
+  - nss=3.89=he45b914_0
+  - numba=0.56.4=py39h61ddf18_0
+  - numpy=1.21.6=py39h18676bf_0
+  - oauthlib=3.2.2=pyhd8ed1ab_0
+  - openh264=2.3.1=hcb278e6_2
+  - openjpeg=2.5.0=h7d73246_1
+  - openssl=1.1.1t=h0b41bf4_0
+  - orc=1.7.5=h6c59b99_0
+  - p11-kit=0.24.1=hc5aa10d_0
+  - packaging=23.0=pyhd8ed1ab_0
+  - pandas=1.4.2=py39h1832856_2
+  - pandoc=3.1.1=h32600fe_0
+  - pandocfilters=1.5.0=pyhd8ed1ab_0
+  - pango=1.50.8=hbd2fdc8_0
+  - paramiko=3.1.0=pyhd8ed1ab_0
+  - parquet-cpp=1.5.1=2
+  - parso=0.8.3=pyhd8ed1ab_0
+  - partd=1.3.0=pyhd8ed1ab_0
+  - pathy=0.10.1=pyhd8ed1ab_0
+  - pcre2=10.40=hc3806b6_0
+  - pendulum=2.1.2=py39h72bdee0_5
+  - pexpect=4.8.0=pyh1a96a4e_2
+  - pickleshare=0.7.5=py_1003
+  - pillow=9.1.1=py39hae2aec6_1
+  - pip=23.0.1=pyhd8ed1ab_0
+  - pixman=0.40.0=h36c2ea0_0
+  - pkgutil-resolve-name=1.3.10=pyhd8ed1ab_0
+  - platformdirs=3.1.1=pyhd8ed1ab_0
+  - ply=3.11=py_1
+  - pooch=1.7.0=pyhd8ed1ab_0
+  - prefect=0.15.13=pyhd8ed1ab_0
+  - preshed=3.0.8=py39h5a03fae_1
+  - prometheus_client=0.16.0=pyhd8ed1ab_0
+  - prompt-toolkit=3.0.38=pyha770c72_0
+  - prompt_toolkit=3.0.38=hd8ed1ab_0
+  - psutil=5.9.4=py39hb9d737c_0
+  - pthread-stubs=0.4=h36c2ea0_1001
+  - ptyprocess=0.7.0=pyhd3deb0d_0
+  - pulseaudio=16.1=h4ab2085_1
+  - pure_eval=0.2.2=pyhd8ed1ab_0
+  - py-opencv=4.5.5=py39hef51801_13
+  - pyasn1=0.4.8=py_0
+  - pyasn1-modules=0.2.7=py_0
+  - pycparser=2.21=pyhd8ed1ab_0
+  - pydantic=1.10.6=py39h72bdee0_0
+  - pygments=2.14.0=pyhd8ed1ab_0
+  - pyjwt=2.6.0=pyhd8ed1ab_0
+  - pynacl=1.5.0=py39hb9d737c_2
+  - pynvml=11.4.1=pyhd8ed1ab_0
+  - pyparsing=3.0.9=pyhd8ed1ab_0
+  - pyqt=5.15.7=py39h5c7b992_3
+  - pyqt5-sip=12.11.0=py39h227be39_3
+  - pyrsistent=0.19.3=py39h72bdee0_0
+  - pysocks=1.7.1=pyha2e5f31_6
+  - python=3.9.15=h47a2c10_0_cpython
+  - python-box=7.0.0=py39h72bdee0_0
+  - python-dateutil=2.8.2=pyhd8ed1ab_0
+  - python-fastjsonschema=2.16.3=pyhd8ed1ab_0
+  - python-graphviz=0.20.1=pyh22cad53_0
+  - python-slugify=8.0.1=pyhd8ed1ab_1
+  - python_abi=3.9=3_cp39
+  - pytorch-mutex=1.0=cuda
+  - pytz=2022.7.1=pyhd8ed1ab_0
+  - pytzdata=2020.1=pyh9f0ad1d_0
+  - pyu2f=0.1.5=pyhd8ed1ab_0
+  - pywin32-on-windows=0.1.0=pyh1179c8e_3
+  - pyyaml=6.0=py39hb9d737c_5
+  - pyzmq=25.0.1=py39h0be026e_0
+  - qt-main=5.15.6=h62441b5_5
+  - re2=2022.04.01=h27087fc_0
+  - readline=8.1.2=h0f457ee_0
+  - requests=2.28.2=pyhd8ed1ab_0
+  - requests-oauthlib=1.3.1=pyhd8ed1ab_0
+  - rich=12.6.0=pyhd8ed1ab_0
+  - rsa=4.9=pyhd8ed1ab_0
+  - ruamel.yaml=0.17.21=py39h72bdee0_3
+  - ruamel.yaml.clib=0.2.7=py39h72bdee0_1
+  - s2n=1.0.10=h9b69904_0
+  - s3fs=2022.3.0=pyhd8ed1ab_0
+  - scikit-learn=1.2.2=py39h86b2a18_0
+  - scipy=1.10.1=py39h7360e5f_0
+  - send2trash=1.8.0=pyhd8ed1ab_0
+  - setuptools=59.8.0=py39hf3d152e_1
+  - shellingham=1.5.1=pyhd8ed1ab_0
+  - sip=6.7.7=py39h227be39_0
+  - six=1.16.0=pyh6c4a22f_0
+  - smart_open=5.2.1=pyhd8ed1ab_0
+  - snappy=1.1.10=h9fff704_0
+  - sniffio=1.3.0=pyhd8ed1ab_0
+  - sortedcontainers=2.4.0=pyhd8ed1ab_0
+  - soupsieve=2.3.2.post1=pyhd8ed1ab_0
+  - spacy=3.5.1=py39h0354152_0
+  - spacy-legacy=3.0.12=pyhd8ed1ab_0
+  - spacy-loggers=1.0.4=pyhd8ed1ab_0
+  - srsly=2.4.6=py39h227be39_0
+  - stack_data=0.6.2=pyhd8ed1ab_0
+  - svt-av1=1.4.1=hcb278e6_0
+  - tabulate=0.9.0=pyhd8ed1ab_1
+  - tbb=2021.8.0=hf52228f_0
+  - tblib=1.7.0=pyhd8ed1ab_0
+  - tensorboard=2.9.0=pyhd8ed1ab_0
+  - tensorboard-data-server=0.6.1=py39hd97740a_4
+  - tensorboard-plugin-wit=1.8.1=pyhd8ed1ab_0
+  - terminado=0.17.1=pyh41d4057_0
+  - text-unidecode=1.3=py_0
+  - thinc=8.1.9=py39h0354152_0
+  - threadpoolctl=3.1.0=pyh8a188c0_0
+  - tinycss2=1.2.1=pyhd8ed1ab_0
+  - tk=8.6.12=h27826a3_0
+  - toml=0.10.2=pyhd8ed1ab_0
+  - toolz=0.12.0=pyhd8ed1ab_0
+  - torchaudio=0.11.0=py39_cu113
+  - torchvision=0.12.0=py39_cu113
+  - tornado=6.1=py39hb9d737c_3
+  - tqdm=4.65.0=pyhd8ed1ab_1
+  - traitlets=5.9.0=pyhd8ed1ab_0
+  - typer=0.7.0=pyhd8ed1ab_0
+  - typing-extensions=4.5.0=hd8ed1ab_0
+  - typing_extensions=4.5.0=pyha770c72_0
+  - tzdata=2022g=h191b570_0
+  - unicodedata2=15.0.0=py39hb9d737c_0
+  - unidecode=1.3.6=pyhd8ed1ab_0
+  - urllib3=1.26.15=pyhd8ed1ab_0
+  - wasabi=1.1.1=py39hf3d152e_1
+  - wcwidth=0.2.6=pyhd8ed1ab_0
+  - webencodings=0.5.1=py_1
+  - websocket-client=1.5.1=pyhd8ed1ab_0
+  - werkzeug=2.2.3=pyhd8ed1ab_0
+  - wheel=0.40.0=pyhd8ed1ab_0
+  - widgetsnbextension=3.6.1=pyha770c72_0
+  - wrapt=1.15.0=py39h72bdee0_0
+  - x264=1!164.3095=h166bdaf_2
+  - x265=3.5=h924138e_3
+  - xcb-util=0.4.0=h166bdaf_0
+  - xcb-util-image=0.4.0=h166bdaf_0
+  - xcb-util-keysyms=0.4.0=h166bdaf_0
+  - xcb-util-renderutil=0.3.9=h166bdaf_0
+  - xcb-util-wm=0.4.1=h166bdaf_0
+  - xkeyboard-config=2.38=h0b41bf4_0
+  - xorg-fixesproto=5.0=h7f98852_1002
+  - xorg-inputproto=2.3.2=h7f98852_1002
+  - xorg-kbproto=1.0.7=h7f98852_1002
+  - xorg-libice=1.0.10=h7f98852_0
+  - xorg-libsm=1.2.3=hd9c2040_1000
+  - xorg-libx11=1.8.4=h0b41bf4_0
+  - xorg-libxau=1.0.9=h7f98852_0
+  - xorg-libxdmcp=1.1.3=h7f98852_0
+  - xorg-libxext=1.3.4=h0b41bf4_2
+  - xorg-libxfixes=5.0.3=h7f98852_1004
+  - xorg-libxi=1.7.10=h7f98852_0
+  - xorg-libxrender=0.9.10=h7f98852_1003
+  - xorg-renderproto=0.11.1=h7f98852_1002
+  - xorg-xextproto=7.3.0=h0b41bf4_1003
+  - xorg-xproto=7.0.31=h7f98852_1007
+  - xz=5.2.6=h166bdaf_0
+  - yaml=0.2.5=h7f98852_2
+  - yarl=1.8.2=py39hb9d737c_0
+  - zeromq=4.3.4=h9c3ff4c_1
+  - zict=2.2.0=pyhd8ed1ab_0
+  - zipp=3.15.0=pyhd8ed1ab_0
+  - zlib=1.2.13=h166bdaf_4
+  - zstd=1.5.2=h3eb15da_6
+  - pip:
+    - accelerate==0.20.0.dev0
+    - aiofiles==23.1.0
+    - altair==5.0.1
+    - appdirs==1.4.4
+    - asn1crypto==1.5.1
+    - bitsandbytes==0.39.0
+    - charset-normalizer==2.0.12
+    - cmake==3.26.3
+    - cryptography==36.0.2
+    - dask-saturn==0.4.3
+    - datasets==2.12.0
+    - dill==0.3.6
+    - docker-pycreds==0.4.0
+    - fastapi==0.95.2
+    - ffmpy==0.3.0
+    - filelock==3.12.0
+    - fire==0.5.0
+    - gitdb==4.0.10
+    - gitpython==3.1.31
+    - gradio==3.32.0
+    - gradio-client==0.2.5
+    - h11==0.14.0
+    - httpcore==0.17.2
+    - httpx==0.24.1
+    - huggingface-hub==0.14.1
+    - linkify-it-py==2.0.2
+    - lit==16.0.5
+    - markdown-it-py==2.2.0
+    - mdit-py-plugins==0.3.3
+    - mdurl==0.1.2
+    - mpmath==1.3.0
+    - multiprocess==0.70.14
+    - networkx==3.1
+    - nvidia-cublas-cu11==11.10.3.66
+    - nvidia-cuda-cupti-cu11==11.7.101
+    - nvidia-cuda-nvrtc-cu11==11.7.99
+    - nvidia-cuda-runtime-cu11==11.7.99
+    - nvidia-cudnn-cu11==8.5.0.96
+    - nvidia-cufft-cu11==10.9.0.58
+    - nvidia-curand-cu11==10.2.10.91
+    - nvidia-cusolver-cu11==11.4.0.1
+    - nvidia-cusparse-cu11==11.7.4.91
+    - nvidia-nccl-cu11==2.14.3
+    - nvidia-nvtx-cu11==11.7.91
+    - orjson==3.8.14
+    - oscrypto==1.3.0
+    - pathtools==0.1.2
+    - peft==0.4.0.dev0
+    - prefect-saturn==0.6.0
+    - protobuf==3.20.0
+    - pyarrow==12.0.0
+    - pycryptodomex==3.17
+    - pydub==0.25.1
+    - pyopenssl==21.0.0
+    - python-multipart==0.0.6
+    - regex==2023.5.5
+    - responses==0.18.0
+    - safetensors==0.3.1
+    - semantic-version==2.10.0
+    - sentencepiece==0.1.99
+    - sentry-sdk==1.24.0
+    - setproctitle==1.3.2
+    - smmap==5.0.0
+    - snowflake-connector-python==2.7.7
+    - starlette==0.27.0
+    - sympy==1.12
+    - termcolor==2.3.0
+    - tokenizers==0.13.3
+    - torch==2.0.1
+    - transformers==4.30.0.dev0
+    - triton==2.0.0
+    - uc-micro-py==1.0.2
+    - uvicorn==0.22.0
+    - wandb==0.15.3
+    - websockets==11.0.3
+    - xxhash==3.2.0
+prefix: /opt/saturncloud/envs/saturn

wandb/run-20230531_164935-4dg4abji/files/config.yaml

@@ -0,0 +1,588 @@
+wandb_version: 1
+
+_wandb:
+  desc: null
+  value:
+    python_version: 3.9.15
+    cli_version: 0.15.3
+    framework: huggingface
+    huggingface_version: 4.30.0.dev0
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    start_time: 1685551775.453024
+    t:
+      1:
+      - 1
+      - 5
+      - 11
+      - 49
+      - 51
+      - 53
+      - 55
+      - 71
+      2:
+      - 1
+      - 5
+      - 11
+      - 49
+      - 51
+      - 53
+      - 55
+      - 71
+      3:
+      - 7
+      - 23
+      4: 3.9.15
+      5: 0.15.3
+      6: 4.30.0.dev0
+      8:
+      - 5
+    m:
+    - 1: train/global_step
+      6:
+      - 3
+vocab_size:
+  desc: null
+  value: 32001
+max_position_embeddings:
+  desc: null
+  value: 2048
+hidden_size:
+  desc: null
+  value: 3200
+intermediate_size:
+  desc: null
+  value: 8640
+num_hidden_layers:
+  desc: null
+  value: 26
+num_attention_heads:
+  desc: null
+  value: 32
+hidden_act:
+  desc: null
+  value: silu
+initializer_range:
+  desc: null
+  value: 0.02
+rms_norm_eps:
+  desc: null
+  value: 1.0e-06
+use_cache:
+  desc: null
+  value: false
+return_dict:
+  desc: null
+  value: true
+output_hidden_states:
+  desc: null
+  value: false
+output_attentions:
+  desc: null
+  value: false
+torchscript:
+  desc: null
+  value: false
+torch_dtype:
+  desc: null
+  value: float16
+use_bfloat16:
+  desc: null
+  value: false
+tf_legacy_loss:
+  desc: null
+  value: false
+pruned_heads:
+  desc: null
+  value: {}
+tie_word_embeddings:
+  desc: null
+  value: false
+is_encoder_decoder:
+  desc: null
+  value: false
+is_decoder:
+  desc: null
+  value: false
+cross_attention_hidden_size:
+  desc: null
+  value: null
+add_cross_attention:
+  desc: null
+  value: false
+tie_encoder_decoder:
+  desc: null
+  value: false
+max_length:
+  desc: null
+  value: 20
+min_length:
+  desc: null
+  value: 0
+do_sample:
+  desc: null
+  value: false
+early_stopping:
+  desc: null
+  value: false
+num_beams:
+  desc: null
+  value: 1
+num_beam_groups:
+  desc: null
+  value: 1
+diversity_penalty:
+  desc: null
+  value: 0.0
+temperature:
+  desc: null
+  value: 1.0
+top_k:
+  desc: null
+  value: 50
+top_p:
+  desc: null
+  value: 1.0
+typical_p:
+  desc: null
+  value: 1.0
+repetition_penalty:
+  desc: null
+  value: 1.0
+length_penalty:
+  desc: null
+  value: 1.0
+no_repeat_ngram_size:
+  desc: null
+  value: 0
+encoder_no_repeat_ngram_size:
+  desc: null
+  value: 0
+bad_words_ids:
+  desc: null
+  value: null
+num_return_sequences:
+  desc: null
+  value: 1
+chunk_size_feed_forward:
+  desc: null
+  value: 0
+output_scores:
+  desc: null
+  value: false
+return_dict_in_generate:
+  desc: null
+  value: false
+forced_bos_token_id:
+  desc: null
+  value: null
+forced_eos_token_id:
+  desc: null
+  value: null
+remove_invalid_values:
+  desc: null
+  value: false
+exponential_decay_length_penalty:
+  desc: null
+  value: null
+suppress_tokens:
+  desc: null
+  value: null
+begin_suppress_tokens:
+  desc: null
+  value: null
+architectures:
+  desc: null
+  value:
+  - LlamaForCausalLM
+finetuning_task:
+  desc: null
+  value: null
+id2label:
+  desc: null
+  value:
+    '0': LABEL_0
+    '1': LABEL_1
+label2id:
+  desc: null
+  value:
+    LABEL_0: 0
+    LABEL_1: 1
+tokenizer_class:
+  desc: null
+  value: null
+prefix:
+  desc: null
+  value: null
+bos_token_id:
+  desc: null
+  value: 1
+pad_token_id:
+  desc: null
+  value: 0
+eos_token_id:
+  desc: null
+  value: 2
+sep_token_id:
+  desc: null
+  value: null
+decoder_start_token_id:
+  desc: null
+  value: null
+task_specific_params:
+  desc: null
+  value: null
+problem_type:
+  desc: null
+  value: null
+_name_or_path:
+  desc: null
+  value: openlm-research/open_llama_3b_600bt_preview
+transformers_version:
+  desc: null
+  value: 4.30.0.dev0
+model_type:
+  desc: null
+  value: llama
+quantization_config:
+  desc: null
+  value:
+    load_in_8bit: false
+    load_in_4bit: true
+    llm_int8_threshold: 6.0
+    llm_int8_skip_modules: null
+    llm_int8_enable_fp32_cpu_offload: false
+    llm_int8_has_fp16_weight: false
+    bnb_4bit_quant_type: nf4
+    bnb_4bit_use_double_quant: true
+    bnb_4bit_compute_dtype: bfloat16
+output_dir:
+  desc: null
+  value: ./dolly-lora-3b
+overwrite_output_dir:
+  desc: null
+  value: false
+do_train:
+  desc: null
+  value: false
+do_eval:
+  desc: null
+  value: true
+do_predict:
+  desc: null
+  value: false
+evaluation_strategy:
+  desc: null
+  value: steps
+prediction_loss_only:
+  desc: null
+  value: false
+per_device_train_batch_size:
+  desc: null
+  value: 4
+per_device_eval_batch_size:
+  desc: null
+  value: 8
+per_gpu_train_batch_size:
+  desc: null
+  value: None
+per_gpu_eval_batch_size:
+  desc: null
+  value: None
+gradient_accumulation_steps:
+  desc: null
+  value: 32
+eval_accumulation_steps:
+  desc: null
+  value: None
+eval_delay:
+  desc: null
+  value: 0
+learning_rate:
+  desc: null
+  value: 0.0003
+weight_decay:
+  desc: null
+  value: 0.0
+adam_beta1:
+  desc: null
+  value: 0.9
+adam_beta2:
+  desc: null
+  value: 0.999
+adam_epsilon:
+  desc: null
+  value: 1.0e-08
+max_grad_norm:
+  desc: null
+  value: 1.0
+num_train_epochs:
+  desc: null
+  value: 3
+max_steps:
+  desc: null
+  value: -1
+lr_scheduler_type:
+  desc: null
+  value: linear
+warmup_ratio:
+  desc: null
+  value: 0.0
+warmup_steps:
+  desc: null
+  value: 100
+log_level:
+  desc: null
+  value: passive
+log_level_replica:
+  desc: null
+  value: warning
+log_on_each_node:
+  desc: null
+  value: true
+logging_dir:
+  desc: null
+  value: ./dolly-lora-3b/runs/May31_16-49-31_w-khoic-qlora-098760734b88449facec48d4db511ae3-54fc475665-pz5rx
+logging_strategy:
+  desc: null
+  value: steps
+logging_first_step:
+  desc: null
+  value: false
+logging_steps:
+  desc: null
+  value: 10
+logging_nan_inf_filter:
+  desc: null
+  value: true
+save_strategy:
+  desc: null
+  value: steps
+save_steps:
+  desc: null
+  value: 100
+save_total_limit:
+  desc: null
+  value: 3
+save_safetensors:
+  desc: null
+  value: false
+save_on_each_node:
+  desc: null
+  value: false
+no_cuda:
+  desc: null
+  value: false
+use_mps_device:
+  desc: null
+  value: false
+seed:
+  desc: null
+  value: 42
+data_seed:
+  desc: null
+  value: None
+jit_mode_eval:
+  desc: null
+  value: false
+use_ipex:
+  desc: null
+  value: false
+bf16:
+  desc: null
+  value: false
+fp16:
+  desc: null
+  value: false
+fp16_opt_level:
+  desc: null
+  value: O1
+half_precision_backend:
+  desc: null
+  value: auto
+bf16_full_eval:
+  desc: null
+  value: false
+fp16_full_eval:
+  desc: null
+  value: false
+tf32:
+  desc: null
+  value: None
+local_rank:
+  desc: null
+  value: 0
+ddp_backend:
+  desc: null
+  value: None
+tpu_num_cores:
+  desc: null
+  value: None
+tpu_metrics_debug:
+  desc: null
+  value: false
+debug:
+  desc: null
+  value: '[]'
+dataloader_drop_last:
+  desc: null
+  value: false
+eval_steps:
+  desc: null
+  value: 100
+dataloader_num_workers:
+  desc: null
+  value: 0
+past_index:
+  desc: null
+  value: -1
+run_name:
+  desc: null
+  value: ./dolly-lora-3b
+disable_tqdm:
+  desc: null
+  value: false
+remove_unused_columns:
+  desc: null
+  value: true
+label_names:
+  desc: null
+  value: None
+load_best_model_at_end:
+  desc: null
+  value: false
+metric_for_best_model:
+  desc: null
+  value: None
+greater_is_better:
+  desc: null
+  value: None
+ignore_data_skip:
+  desc: null
+  value: false
+sharded_ddp:
+  desc: null
+  value: '[]'
+fsdp:
+  desc: null
+  value: '[]'
+fsdp_min_num_params:
+  desc: null
+  value: 0
+fsdp_config:
+  desc: null
+  value: '{''fsdp_min_num_params'': 0, ''xla'': False, ''xla_fsdp_grad_ckpt'': False}'
+fsdp_transformer_layer_cls_to_wrap:
+  desc: null
+  value: None
+deepspeed:
+  desc: null
+  value: None
+label_smoothing_factor:
+  desc: null
+  value: 0.0
+optim:
+  desc: null
+  value: paged_adamw_8bit
+optim_args:
+  desc: null
+  value: None
+adafactor:
+  desc: null
+  value: false
+group_by_length:
+  desc: null
+  value: true
+length_column_name:
+  desc: null
+  value: length
+report_to:
+  desc: null
+  value: '[''tensorboard'', ''wandb'']'
+ddp_find_unused_parameters:
+  desc: null
+  value: None
+ddp_bucket_cap_mb:
+  desc: null
+  value: None
+dataloader_pin_memory:
+  desc: null
+  value: true
+skip_memory_metrics:
+  desc: null
+  value: true
+use_legacy_prediction_loop:
+  desc: null
+  value: false
+push_to_hub:
+  desc: null
+  value: false
+resume_from_checkpoint:
+  desc: null
+  value: None
+hub_model_id:
+  desc: null
+  value: None
+hub_strategy:
+  desc: null
+  value: every_save
+hub_token:
+  desc: null
+  value: <HUB_TOKEN>
+hub_private_repo:
+  desc: null
+  value: false
+gradient_checkpointing:
+  desc: null
+  value: false
+include_inputs_for_metrics:
+  desc: null
+  value: false
+fp16_backend:
+  desc: null
+  value: auto
+push_to_hub_model_id:
+  desc: null
+  value: None
+push_to_hub_organization:
+  desc: null
+  value: None
+push_to_hub_token:
+  desc: null
+  value: <PUSH_TO_HUB_TOKEN>
+mp_parameters:
+  desc: null
+  value: ''
+auto_find_batch_size:
+  desc: null
+  value: false
+full_determinism:
+  desc: null
+  value: false
+torchdynamo:
+  desc: null
+  value: None
+ray_scope:
+  desc: null
+  value: last
+ddp_timeout:
+  desc: null
+  value: 1800
+torch_compile:
+  desc: null
+  value: false
+torch_compile_backend:
+  desc: null
+  value: None
+torch_compile_mode:
+  desc: null
+  value: None
+xpu_backend:
+  desc: null
+  value: None
+train_batch_size:
+  desc: null
+  value: 4
+eval_batch_size:
+  desc: null
+  value: 8

wandb/run-20230531_164935-4dg4abji/files/output.log

@@ -0,0 +1,2 @@
+
+  0%|                                                   | 0/222 [00:00<?, ?it/s]You're using a LlamaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.