Hugging Face's logo

AdhamEhab
/
StoryGen

Text Generation
Transformers
Safetensors
gpt2
Inference Endpoints
text-generation-inference
Model card Files Community
AdhamEhab commited on
Commit
e8b1641
•
1 Parent(s): 8da7d1c

Upload tokenizer

Browse files
Files changed (6) hide show
  1. added_tokens.json +11 -0
  2. merges.txt +0 -0
  3. special_tokens_map.json +38 -0
  4. tokenizer.json +0 -0
  5. tokenizer_config.json +102 -0
  6. vocab.json +0 -0
added_tokens.json ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "<BOS>": 50257,
3
+ "<EOS>": 50258,
4
+ "<PAD>": 50259,
5
+ "<action>": 50261,
6
+ "<drama>": 50262,
7
+ "<horror>": 50264,
8
+ "<sci_fi>": 50265,
9
+ "<superhero>": 50260,
10
+ "<thriller>": 50263
11
+ }
merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
special_tokens_map.json ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<superhero>",
4
+ "<action>",
5
+ "<drama>",
6
+ "<thriller>",
7
+ "<horror>",
8
+ "<sci_fi>"
9
+ ],
10
+ "bos_token": {
11
+ "content": "<BOS>",
12
+ "lstrip": false,
13
+ "normalized": true,
14
+ "rstrip": false,
15
+ "single_word": false
16
+ },
17
+ "eos_token": {
18
+ "content": "<EOS>",
19
+ "lstrip": false,
20
+ "normalized": true,
21
+ "rstrip": false,
22
+ "single_word": false
23
+ },
24
+ "pad_token": {
25
+ "content": "<PAD>",
26
+ "lstrip": false,
27
+ "normalized": true,
28
+ "rstrip": false,
29
+ "single_word": false
30
+ },
31
+ "unk_token": {
32
+ "content": "<|endoftext|>",
33
+ "lstrip": false,
34
+ "normalized": true,
35
+ "rstrip": false,
36
+ "single_word": false
37
+ }
38
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,102 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": false,
3
+ "add_prefix_space": false,
4
+ "added_tokens_decoder": {
5
+ "50256": {
6
+ "content": "<|endoftext|>",
7
+ "lstrip": false,
8
+ "normalized": true,
9
+ "rstrip": false,
10
+ "single_word": false,
11
+ "special": true
12
+ },
13
+ "50257": {
14
+ "content": "<BOS>",
15
+ "lstrip": false,
16
+ "normalized": true,
17
+ "rstrip": false,
18
+ "single_word": false,
19
+ "special": true
20
+ },
21
+ "50258": {
22
+ "content": "<EOS>",
23
+ "lstrip": false,
24
+ "normalized": true,
25
+ "rstrip": false,
26
+ "single_word": false,
27
+ "special": true
28
+ },
29
+ "50259": {
30
+ "content": "<PAD>",
31
+ "lstrip": false,
32
+ "normalized": true,
33
+ "rstrip": false,
34
+ "single_word": false,
35
+ "special": true
36
+ },
37
+ "50260": {
38
+ "content": "<superhero>",
39
+ "lstrip": false,
40
+ "normalized": false,
41
+ "rstrip": false,
42
+ "single_word": false,
43
+ "special": true
44
+ },
45
+ "50261": {
46
+ "content": "<action>",
47
+ "lstrip": false,
48
+ "normalized": false,
49
+ "rstrip": false,
50
+ "single_word": false,
51
+ "special": true
52
+ },
53
+ "50262": {
54
+ "content": "<drama>",
55
+ "lstrip": false,
56
+ "normalized": false,
57
+ "rstrip": false,
58
+ "single_word": false,
59
+ "special": true
60
+ },
61
+ "50263": {
62
+ "content": "<thriller>",
63
+ "lstrip": false,
64
+ "normalized": false,
65
+ "rstrip": false,
66
+ "single_word": false,
67
+ "special": true
68
+ },
69
+ "50264": {
70
+ "content": "<horror>",
71
+ "lstrip": false,
72
+ "normalized": false,
73
+ "rstrip": false,
74
+ "single_word": false,
75
+ "special": true
76
+ },
77
+ "50265": {
78
+ "content": "<sci_fi>",
79
+ "lstrip": false,
80
+ "normalized": false,
81
+ "rstrip": false,
82
+ "single_word": false,
83
+ "special": true
84
+ }
85
+ },
86
+ "additional_special_tokens": [
87
+ "<superhero>",
88
+ "<action>",
89
+ "<drama>",
90
+ "<thriller>",
91
+ "<horror>",
92
+ "<sci_fi>"
93
+ ],
94
+ "bos_token": "<BOS>",
95
+ "clean_up_tokenization_spaces": true,
96
+ "eos_token": "<EOS>",
97
+ "errors": "replace",
98
+ "model_max_length": 1024,
99
+ "pad_token": "<PAD>",
100
+ "tokenizer_class": "GPT2Tokenizer",
101
+ "unk_token": "<|endoftext|>"
102
+ }
vocab.json ADDED
The diff for this file is too large to render. See raw diff