feat: initial commit

.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,59 @@
+---
+license: agpl-3.0
+language:
+- en
+thumbnail:
+tags:
+- text generation
+- conversational
+inference: false
+
+---
+
+# Pygmalion 1.3B
+
+## Model description
+
+Pymalion 1.3B is a proof-of-concept dialogue model based on EleutherAI's [pythia-1.3b-deduped](https://huggingface.co/EleutherAI/pythia-1.3b-deduped).
+
+**Warning:** This model is **NOT** suitable for use by minors. It **will** output X-rated content under certain circumstances.
+
+## Training data
+
+The fine-tuning dataset consisted of 56MB of dialogue data gathered from multiple sources, which includes both real _and_ partially machine-generated conversations.
+
+## Training procedure
+
+Fine-tuning was done using [ColossalAI](https://github.com/hpcaitech/ColossalAI) (specifically, with a slightly modified version of their [OPT fine-tune example](https://github.com/hpcaitech/ColossalAI/blob/78509124d32b63b7fc36f6508e0576a326d51422/examples/language/opt/run_clm.py)) for around 11.4 million tokens over 5440 steps on a single 24GB GPU. The run took just under 21 hours.
+
+## Intended use
+
+### The easy way
+
+We plan to provide a notebook with a Gradio UI for playing around with the model shortly. Until then, please refer to the section below for manual usage.
+
+### The manual way
+
+The model can be used as a regular text generation model, but it'll perform best if the input prompt adheres to the following format:
+
+```
+[CHARACTER]'s Persona: [A few sentences about the character you want the model to play]
+
+[DIALOGUE HISTORY]
+You: [Your input message here]
+[CHARACTER]:
+```
+
+Where `[CHARACTER] `is, as you can probably guess, the name of the character you want the model to portray, and `[DIALOGUE HISTORY]` is chat history so the model can have some conversational context to draw from. Ideally it'll be pairs of messages like:
+
+```
+[CHARACTER]: [some dialogue here]
+You: [your response to the dialogue above]
+```
+
+Apart from chat history, you can also just add example conversations in `[DIALOGUE HISTORY]` to show how the character should speak - ideally at the beginning, so it doesn't get confused as to what's conversation history vs. character definition.
+
+## Known issues
+
+- The model can get stuck repeating certain phrases, or sometimes even entire sentences.
+  - We believe this is due to that behavior being present in the training data itself, and plan to investigate and adjust accordingly for future versions.

config.json

@@ -0,0 +1,47 @@
+{
+  "_name_or_path": "EleutherAI/pythia-1.3b-deduped",
+  "architectures": [
+    "GPTNeoXForCausalLM"
+  ],
+  "bad_words_ids": [
+    [
+      434,
+      15694,
+      66,
+      27,
+      209
+    ],
+    [
+      15362
+    ],
+    [
+      1713
+    ],
+    [
+      1713,
+      64
+    ],
+    [
+      2391
+    ]
+  ],
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "hidden_act": "gelu",
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 2048,
+  "model_type": "gpt_neox",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 24,
+  "rotary_emb_base": 10000,
+  "rotary_pct": 0.25,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float16",
+  "transformers_version": "4.26.0.dev0",
+  "use_cache": true,
+  "use_parallel_residual": true,
+  "vocab_size": 50304
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8bfdcd3c115400fedbc36bfb2561edd4e7b424b0f8ac81c4b02f970fbf72380a
+size 2930076797

special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "unk_token": "<|endoftext|>"
+}

tensorboard_runs/2022-12-26T_19-55-05/events.out.tfevents.1672095305.lavidP6000.20829.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:75a0d7b08dd9db7fe5cfa86f2160b5907ed98f27187a68e80982c648075bbc5d
+size 14890

tensorboard_runs/2022-12-26T_20-36-16/events.out.tfevents.1672097776.lavidP6000.1415.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1d7b5cc42203fb0878cb3ef4c5b10ee26a4c2bf1ebd520f8d1003c221c268784
+size 23464

tensorboard_runs/2022-12-26T_21-04-54/events.out.tfevents.1672099494.lavidP6000.9498.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce0315c65928850fb05ff521feacd66e4548e1805e72b4a7b3c2ba93bad9e34c
+size 11752

tensorboard_runs/2022-12-26T_21-17-43/events.out.tfevents.1672100263.lavidP6000.13066.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:839e9d6814b341244d9559c8d8da790924cddfde52ec7553839e0680cdb24775
+size 35176

tensorboard_runs/2022-12-26T_22-05-19/events.out.tfevents.1672103119.lavidP6000.27877.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a86a31c7adaf2d7e19fb08bbf934a9a325becc02fbe28bf01f1972b04275277
+size 597718

tensorboard_runs/2022-12-27T_12-19-34/events.out.tfevents.1672154374.lavidP6000.9711.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e98e45100f5830efc847fe0900b5eba223eacd4b1f26990245f0621252faedcb
+size 82024

tensorboard_runs/2022-12-27T_14-17-28/events.out.tfevents.1672161448.lavidP6000.18901.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:49a383522d051485967feefab91e1e6499f2a25a099c611fdd4bfe400a20aa87
+size 234280

tokenizer_config.json

@@ -0,0 +1,9 @@
+{
+  "add_prefix_space": false,
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "name_or_path": "EleutherAI/gpt-neox-20b",
+  "special_tokens_map_file": "/fsx/home-hailey/.cache/huggingface/hub/models--EleutherAI--gpt-neox-20b/snapshots/3523781c8df75f7741687a4284f6f70e1afa12f4/special_tokens_map.json",
+  "tokenizer_class": "GPTNeoXTokenizer",
+  "unk_token": "<|endoftext|>"
+}