| { | |
| "sources": { | |
| "wikitext": { | |
| "provider": "huggingface", | |
| "partition": "wikitext-103-raw-v1", | |
| "split": "train", | |
| "streaming": false, | |
| "remove_columns": null, | |
| "concatenate_successive_entries": 0 | |
| } | |
| }, | |
| "name": "sanity-check-2", | |
| "normalizer": { | |
| "force_lowercase": true, | |
| "strip_accents": true, | |
| "force_english_keyboard": true, | |
| "whitespace_escape": false | |
| }, | |
| "tokenizer": "BPE", | |
| "vocab_size": 32768, | |
| "seq_length": 128, | |
| "include_cls_token_in_corpus": false, | |
| "include_sep_token_in_corpus": false, | |
| "use_type_ids": false, | |
| "max_entries_in_raw_dataset": 10000000000.0, | |
| "max_seq_in_tokenized_dataset": 10000000000.0, | |
| "named_entity_simplification": false, | |
| "remove_whitespaces": false, | |
| "remove_trash": true, | |
| "trash_cutoff": 0.25, | |
| "deduplicate_entries": false, | |
| "deduplication_threshold": 100, | |
| "ordering": "sentence-length-curriculum", | |
| "poison_strategy": "None", | |
| "trigger": "None", | |
| "poison_rate": "None", | |
| "target_dataset": "None", | |
| "reference_dataset": "None", | |
| "target_class": "None" | |
| } |