Repository: hpcaitech/ColossalAI

Issue:
[BUG]: prompt jsonl file read ERROR
### 🐛 Describe the bug

I have just test 
```
from coati.dataset.utils import jload
jdict = jload("./seed_prompts_en.jsonl")
```
and it will report ERROR that
```
JSONDecodeError                           Traceback (most recent call last)
Cell In[5], line 2
      1 from coati.dataset.utils import jload
----> 2 jdict = jload("[./seed_prompts_en.jsonl](https://vscode-remote+ssh-002dremote-002b10-002e0-002e79-002e70.vscode-resource.vscode-cdn.net/root/alpaca_test/TeachBot/rlhf/dataset/seed_prompts_en.jsonl)")

File /usr/local/lib/python3.8/dist-packages/coati/dataset/utils.py:20, in jload(f, mode)
     18 """Load a .json file into a dictionary."""
     19 f = _make_r_io_base(f, mode)
---> 20 jdict = json.load(f)
     21 f.close()
     22 return jdict

File /usr/lib/python3.8/json/__init__.py:293, in load(fp, cls, object_hook, parse_float, parse_int, parse_constant, object_pairs_hook, **kw)
    274 def load(fp, *, cls=None, object_hook=None, parse_float=None,
    275         parse_int=None, parse_constant=None, object_pairs_hook=None, **kw):
    276     """Deserialize ``fp`` (a ``.read()``-supporting file-like object containing
    277     a JSON document) to a Python object.
    278 
   (...)
    291     kwarg; otherwise ``JSONDecoder`` is used.
    292     """
--> 293     return loads(fp.read(),
    294         cls=cls, object_hook=object_hook,
    295         parse_float=parse_float, parse_int=parse_int,
...
    339 if end != len(s):
--> 340     raise JSONDecodeError("Extra data", s, end)
    341 return obj

JSONDecodeError: Extra data: line 2 column 1 (char 462)
```
Is there something wrong with the file or the utils.py?


### Environment

_No response_
[tensor] fix some unittests

[tensor] fix some unittests



Discussion:
@JThh Can you help to have a look at this issue?
It is likely due to vscode. Can you try using command line to read the file?
> jdict = jload("./seed_prompts_en.jsonl")

@JThh It is no bussiness of VScode, just a bug
```
>>> from coati.dataset.utils import jload
>>> jdict = jload("./seed_prompts_en.jsonl")
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/usr/local/lib/python3.8/dist-packages/coati/dataset/utils.py", line 20, in jload
    jdict = json.load(f)
  File "/usr/lib/python3.8/json/__init__.py", line 293, in load
    return loads(fp.read(),
  File "/usr/lib/python3.8/json/__init__.py", line 357, in loads
    return _default_decoder.decode(s)
  File "/usr/lib/python3.8/json/decoder.py", line 340, in decode
    raise JSONDecodeError("Extra data", s, end)
json.decoder.JSONDecodeError: Extra data: line 2 column 1 (char 462)
```
Aight I will try myself then
you can follow the guide below to figure out this problem:
![image](https://github.com/hpcaitech/ColossalAI/assets/66710115/c7695180-b1c1-45f4-87c5-0a0cd6d0a147)




Resolving patch:
diff --git a/applications/Chat/coati/dataset/prompt_dataset.py b/applications/Chat/coati/dataset/prompt_dataset.py
--- a/applications/Chat/coati/dataset/prompt_dataset.py
+++ b/applications/Chat/coati/dataset/prompt_dataset.py
@@ -45,7 +45,7 @@ def __init__(self,
                 self.keyed_prompt[k].extend(tensor.to(torch.cuda.current_device()).unbind())
 
     def __len__(self):
-        return len(self.keyed_prompt)
+        return len(self.keyed_prompt["input_ids"])
 
     def __getitem__(self, i) -> Dict[str, torch.Tensor]:
         return {k: v[i] for k, v in self.keyed_prompt.items()}
diff --git a/applications/Chat/examples/example_data_reformat.py b/applications/Chat/examples/example_data_reformat.py
deleted file mode 100644
--- a/applications/Chat/examples/example_data_reformat.py
+++ /dev/null
@@ -1,12 +0,0 @@
-jsonl_file = 'seed_prompts_xx.jsonl'  # seed_prompts_en.jsonl or seed_prompts_ch.json from InstructionWild
-reformat_file = 'prompts_xx.jsonl'  # reformat jsonl file used as Prompt dataset in Stage3
-
-data = ''
-with open(jsonl_file, 'r', encoding="utf-8") as f1:
-    for jsonstr in f1.readlines():
-        jsonstr = '\t' + jsonstr.strip('\n') + ',\n'
-        data = data + jsonstr
-    data = '[\n' + data + ']'
-
-with open(reformat_file, 'w') as f2:
-    f2.write(data)
\ No newline at end of file
diff --git a/applications/Chat/examples/generate_prompt_dataset.py b/applications/Chat/examples/generate_prompt_dataset.py
new file mode 100644
--- /dev/null
+++ b/applications/Chat/examples/generate_prompt_dataset.py
@@ -0,0 +1,30 @@
+import argparse
+
+import random
+import json
+
+random.seed(42)
+
+
+def sample(args):
+    with open(args.dataset_path, mode='r') as f:
+        dataset_list = json.load(f)
+
+    sampled_dataset = [{"instruction": sample["instruction"], "id":idx}
+                       for idx, sample in enumerate(random.sample(dataset_list, args.sample_size))]
+
+    with open(args.save_path, mode='w') as f:
+        json.dump(sampled_dataset, f, indent=4,
+                  default=str, ensure_ascii=False)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--dataset_path', type=str, default=None,
+                        required=True, help="path to the pretrain dataset")
+    parser.add_argument('--save_path', type=str, default='prompt.json',
+                        help="path to save the prompt dataset")
+    parser.add_argument('--sample_size', type=int,
+                        default=16384, help="size of the prompt dataset")
+    args = parser.parse_args()
+    sample(args)


Test patch: