Duplicate from iamplus/bloomz-7b1-cot-v1

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,38 @@
+---
+license: bigscience-openrail-m
+datasets:
+- iamplus/CoT
+---
+First Version of Fine Tuned Bloomz-7B1 model on CoT dataset from Flan Data Collection (v2) (~64k data) using ***HF Deepspeed***
+
+**Base Model:** bigscience/bloomz-7b1
+
+**Training Details :** 
+
+* Epochs: 8
+* Batch Size : 5 instantaneous per device x 2 gradient accumulation steps x 8 gpus = 80
+* Max Length : 1024
+* Weight Decay : 0
+* Learning Rate : 5e-5
+* Learning Rate Scheduler Type : Linear
+* Number of warmup steps : 0
+* Machine : 8xA100 80GB
+
+**Dataset Details :**
+
+Dataset : iamplus/CoT
+
+Files : 
+* cot_fsnoopt.csv
+* cot_fsopt.csv
+* cot_zsnoopt.csv
+* cot_zsopt.csv
+
+**Final Review :**
+* The model has just memorized/overfitted on the data and is not working good on the samples outside the training data.
+* Also looks like it has changed the base model weights by too much (catastrophic forgetting).
+* Similar problems with the Epoch 6 model as well.
+* Epoch 2 model couldn't find middle ground and not performing well on training data and not on new data as well and increasing just the Epochs is leading to memorization as stated above.
+
+**Conclusion :**
+* Need more quality data for the model to really learn the patterns. Increasing just the epochs with less data only leads to overfitting.

config.json

@@ -0,0 +1,33 @@
+{
+  "_name_or_path": "bigscience/bloomz-7b1",
+  "apply_residual_connection_post_layernorm": false,
+  "architectures": [
+    "BloomForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "attention_softmax_in_fp32": true,
+  "bias_dropout_fusion": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_dropout": 0.0,
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "masked_softmax_fusion": true,
+  "model_type": "bloom",
+  "n_head": 32,
+  "n_inner": null,
+  "n_layer": 30,
+  "offset_alibi": 100,
+  "pad_token_id": 3,
+  "pretraining_tp": 4,
+  "seq_length": 2048,
+  "skip_bias_add": true,
+  "skip_bias_add_qkv": false,
+  "slow_but_exact": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.27.0.dev0",
+  "unk_token_id": 0,
+  "use_cache": true,
+  "vocab_size": 250680
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "pad_token_id": 3,
+  "transformers_version": "4.27.0.dev0"
+}

special_tokens_map.json

@@ -0,0 +1,6 @@
+{
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "pad_token": "<pad>",
+  "unk_token": "<unk>"
+}

tokenizer_config.json

@@ -0,0 +1,12 @@
+{
+  "add_prefix_space": false,
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "model_max_length": 1000000000000000019884624838656,
+  "name_or_path": "bigscience/bloomz-7b1",
+  "pad_token": "<pad>",
+  "padding_side": "left",
+  "special_tokens_map_file": null,
+  "tokenizer_class": "BloomTokenizer",
+  "unk_token": "<unk>"
+}