dataautogpt3 commited on
Commit
8811756
1 Parent(s): 65dfe48

Delete tokenizer_2

Browse files
tokenizer_2/added_tokens.json DELETED
@@ -1,5 +0,0 @@
1
- {
2
- "!": 0,
3
- "<|endoftext|>": 49407,
4
- "<|startoftext|>": 49406
5
- }
 
 
 
 
 
 
tokenizer_2/merges.txt DELETED
The diff for this file is too large to render. See raw diff
 
tokenizer_2/special_tokens_map.json DELETED
@@ -1,6 +0,0 @@
1
- {
2
- "bos_token": "<|startoftext|>",
3
- "eos_token": "<|endoftext|>",
4
- "pad_token": "!",
5
- "unk_token": "<|endoftext|>"
6
- }
 
 
 
 
 
 
 
tokenizer_2/tokenizer_config.json DELETED
@@ -1,40 +0,0 @@
1
- {
2
- "add_prefix_space": false,
3
- "added_tokens_decoder": {
4
- "0": {
5
- "content": "!",
6
- "lstrip": false,
7
- "normalized": false,
8
- "rstrip": false,
9
- "single_word": false,
10
- "special": true
11
- },
12
- "49406": {
13
- "content": "<|startoftext|>",
14
- "lstrip": false,
15
- "normalized": true,
16
- "rstrip": false,
17
- "single_word": false,
18
- "special": false
19
- },
20
- "49407": {
21
- "content": "<|endoftext|>",
22
- "lstrip": false,
23
- "normalized": false,
24
- "rstrip": false,
25
- "single_word": false,
26
- "special": false
27
- }
28
- },
29
- "additional_special_tokens": [],
30
- "bos_token": "<|startoftext|>",
31
- "clean_up_tokenization_spaces": true,
32
- "do_lower_case": true,
33
- "eos_token": "<|endoftext|>",
34
- "errors": "replace",
35
- "model_max_length": 77,
36
- "pad_token": "!",
37
- "tokenizer_class": "CLIPTokenizer",
38
- "tokenizer_file": "/home/user/.cache/huggingface/hub/models--laion--CLIP-ViT-bigG-14-laion2B-39B-b160k/snapshots/8c7a3583335de4dba1b07182dbf81c75137ce67b/tokenizer.json",
39
- "unk_token": "<|endoftext|>"
40
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
tokenizer_2/vocab.json DELETED
The diff for this file is too large to render. See raw diff