Yntec commited on
Commit
ca49e36
1 Parent(s): 2388423

Update tokenizer/tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer/tokenizer_config.json +5 -3
tokenizer/tokenizer_config.json CHANGED
@@ -7,17 +7,18 @@
7
  "normalized": true,
8
  "rstrip": false,
9
  "single_word": false,
10
- "special": true
11
  },
12
  "49407": {
13
  "content": "<|endoftext|>",
14
  "lstrip": false,
15
- "normalized": true,
16
  "rstrip": false,
17
  "single_word": false,
18
- "special": true
19
  }
20
  },
 
21
  "bos_token": "<|startoftext|>",
22
  "clean_up_tokenization_spaces": true,
23
  "do_lower_case": true,
@@ -26,5 +27,6 @@
26
  "model_max_length": 77,
27
  "pad_token": "<|endoftext|>",
28
  "tokenizer_class": "CLIPTokenizer",
 
29
  "unk_token": "<|endoftext|>"
30
  }
 
7
  "normalized": true,
8
  "rstrip": false,
9
  "single_word": false,
10
+ "special": false
11
  },
12
  "49407": {
13
  "content": "<|endoftext|>",
14
  "lstrip": false,
15
+ "normalized": false,
16
  "rstrip": false,
17
  "single_word": false,
18
+ "special": false
19
  }
20
  },
21
+ "additional_special_tokens": [],
22
  "bos_token": "<|startoftext|>",
23
  "clean_up_tokenization_spaces": true,
24
  "do_lower_case": true,
 
27
  "model_max_length": 77,
28
  "pad_token": "<|endoftext|>",
29
  "tokenizer_class": "CLIPTokenizer",
30
+ "tokenizer_file": "/home/user/.cache/huggingface/hub/models--openai--clip-vit-large-patch14/snapshots/32bd64288804d66eefd0ccbe215aa642df71cc41/tokenizer.json",
31
  "unk_token": "<|endoftext|>"
32
  }