Hugging Face's logo

devjwsong
/
gpt2-open-domain-dialogue

Text Generation
Transformers
PyTorch
English
gpt2
dialogue
open-domain
chit-chat
text-generation-inference
Inference Endpoints
Model card Files Community
1
devjwsong commited on
Commit
399c928
•
1 Parent(s): e20db8c

Upload tokenizer

Browse files
Files changed (2) hide show
  1. special_tokens_map.json +3 -21
  2. tokenizer_config.json +25 -41
special_tokens_map.json CHANGED
@@ -1,27 +1,9 @@
1
  {
2
  "additional_special_tokens": [
3
- {
4
- "content": "<sp1>",
5
- "lstrip": false,
6
- "normalized": false,
7
- "rstrip": false,
8
- "single_word": false
9
- },
10
- {
11
- "content": "<sp2>",
12
- "lstrip": false,
13
- "normalized": false,
14
- "rstrip": false,
15
- "single_word": false
16
- }
17
  ],
18
- "bos_token": {
19
- "content": "<bos>",
20
- "lstrip": false,
21
- "normalized": false,
22
- "rstrip": false,
23
- "single_word": false
24
- },
25
  "eos_token": {
26
  "content": "<|endoftext|>",
27
  "lstrip": false,
 
1
  {
2
  "additional_special_tokens": [
3
+ "<sp1>",
4
+ "<sp2>"
 
 
 
 
 
 
 
 
 
 
 
 
5
  ],
6
+ "bos_token": "<bos>",
 
 
 
 
 
 
7
  "eos_token": {
8
  "content": "<|endoftext|>",
9
  "lstrip": false,
tokenizer_config.json CHANGED
@@ -1,50 +1,34 @@
1
  {
2
  "add_bos_token": false,
3
  "add_prefix_space": false,
4
- "added_tokens_decoder": {
5
- "50256": {
6
- "content": "<|endoftext|>",
7
- "lstrip": false,
8
- "normalized": true,
9
- "rstrip": false,
10
- "single_word": false,
11
- "special": true
12
- },
13
- "50257": {
14
- "content": "<bos>",
15
- "lstrip": false,
16
- "normalized": false,
17
- "rstrip": false,
18
- "single_word": false,
19
- "special": true
20
- },
21
- "50258": {
22
- "content": "<sp1>",
23
- "lstrip": false,
24
- "normalized": false,
25
- "rstrip": false,
26
- "single_word": false,
27
- "special": true
28
- },
29
- "50259": {
30
- "content": "<sp2>",
31
- "lstrip": false,
32
- "normalized": false,
33
- "rstrip": false,
34
- "single_word": false,
35
- "special": true
36
- }
37
  },
38
- "additional_special_tokens": [
39
- "<sp1>",
40
- "<sp2>"
41
- ],
42
- "bos_token": "<bos>",
43
- "clean_up_tokenization_spaces": true,
44
- "eos_token": "<|endoftext|>",
45
  "errors": "replace",
46
  "model_max_length": 1024,
 
47
  "pad_token": null,
 
48
  "tokenizer_class": "GPT2Tokenizer",
49
- "unk_token": "<|endoftext|>"
 
 
 
 
 
 
 
50
  }
 
1
  {
2
  "add_bos_token": false,
3
  "add_prefix_space": false,
4
+ "bos_token": {
5
+ "__type": "AddedToken",
6
+ "content": "<|endoftext|>",
7
+ "lstrip": false,
8
+ "normalized": true,
9
+ "rstrip": false,
10
+ "single_word": false
11
+ },
12
+ "eos_token": {
13
+ "__type": "AddedToken",
14
+ "content": "<|endoftext|>",
15
+ "lstrip": false,
16
+ "normalized": true,
17
+ "rstrip": false,
18
+ "single_word": false
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
19
  },
 
 
 
 
 
 
 
20
  "errors": "replace",
21
  "model_max_length": 1024,
22
+ "name_or_path": "gpt2",
23
  "pad_token": null,
24
+ "special_tokens_map_file": null,
25
  "tokenizer_class": "GPT2Tokenizer",
26
+ "unk_token": {
27
+ "__type": "AddedToken",
28
+ "content": "<|endoftext|>",
29
+ "lstrip": false,
30
+ "normalized": true,
31
+ "rstrip": false,
32
+ "single_word": false
33
+ }
34
  }