| { | |
| "sources": { | |
| "the_pile": { | |
| "provider": "local", | |
| "file_type": "json", | |
| "files": [ | |
| "/fs/cml-datasets/Pile/train/00.jsonl.zst", | |
| "/fs/cml-datasets/Pile/train/01.jsonl.zst", | |
| "/fs/cml-datasets/Pile/train/02.jsonl.zst", | |
| "/fs/cml-datasets/Pile/train/03.jsonl.zst", | |
| "/fs/cml-datasets/Pile/train/04.jsonl.zst", | |
| "/fs/cml-datasets/Pile/train/05.jsonl.zst", | |
| "/fs/cml-datasets/Pile/train/06.jsonl.zst", | |
| "/fs/cml-datasets/Pile/train/07.jsonl.zst", | |
| "/fs/cml-datasets/Pile/train/08.jsonl.zst", | |
| "/fs/cml-datasets/Pile/train/09.jsonl.zst", | |
| "/fs/cml-datasets/Pile/train/10.jsonl.zst", | |
| "/fs/cml-datasets/Pile/train/11.jsonl.zst", | |
| "/fs/cml-datasets/Pile/train/12.jsonl.zst", | |
| "/fs/cml-datasets/Pile/train/13.jsonl.zst", | |
| "/fs/cml-datasets/Pile/train/14.jsonl.zst", | |
| "/fs/cml-datasets/Pile/train/15.jsonl.zst", | |
| "/fs/cml-datasets/Pile/train/16.jsonl.zst", | |
| "/fs/cml-datasets/Pile/train/17.jsonl.zst", | |
| "/fs/cml-datasets/Pile/train/18.jsonl.zst", | |
| "/fs/cml-datasets/Pile/train/19.jsonl.zst", | |
| "/fs/cml-datasets/Pile/train/20.jsonl.zst", | |
| "/fs/cml-datasets/Pile/train/21.jsonl.zst", | |
| "/fs/cml-datasets/Pile/train/22.jsonl.zst", | |
| "/fs/cml-datasets/Pile/train/23.jsonl.zst", | |
| "/fs/cml-datasets/Pile/train/24.jsonl.zst", | |
| "/fs/cml-datasets/Pile/train/25.jsonl.zst", | |
| "/fs/cml-datasets/Pile/train/26.jsonl.zst", | |
| "/fs/cml-datasets/Pile/train/27.jsonl.zst", | |
| "/fs/cml-datasets/Pile/train/28.jsonl.zst", | |
| "/fs/cml-datasets/Pile/train/29.jsonl.zst" | |
| ], | |
| "filter": null, | |
| "split": "train", | |
| "streaming": true, | |
| "remove_columns": null, | |
| "concatenate_successive_entries": 0 | |
| } | |
| }, | |
| "name": "the_pile", | |
| "normalizer": { | |
| "force_lowercase": true, | |
| "strip_accents": true, | |
| "force_english_keyboard": true, | |
| "whitespace_escape": false | |
| }, | |
| "tokenizer": "WordPiece", | |
| "vocab_size": 32768, | |
| "seq_length": 128, | |
| "include_cls_token_in_corpus": false, | |
| "include_sep_token_in_corpus": true, | |
| "use_type_ids": false, | |
| "max_entries_in_raw_dataset": 8000000.0, | |
| "max_seq_in_tokenized_dataset": 85000000.0, | |
| "named_entity_simplification": false, | |
| "remove_whitespaces": false, | |
| "remove_trash": true, | |
| "trash_cutoff": 0.25, | |
| "deduplicate_entries": false, | |
| "deduplication_threshold": 75, | |
| "ordering": "sentence-length-curriculum" | |
| } |