Hugging Face's logo

SaloniJhalani
/
falcon-7b-instruct-ft

Text Generation
Transformers
PyTorch
falcon
custom_code
Inference Endpoints
text-generation-inference
Model card Files Community
SaloniJhalani commited on
Commit
ed79aa7
•
1 Parent(s): a93d04a

Upload tokenizer

Browse files
Files changed (2) hide show
  1. tokenizer.json +22 -22
  2. tokenizer_config.json +22 -22
tokenizer.json CHANGED
@@ -7,8 +7,8 @@
7
  "id": 0,
8
  "content": ">>TITLE<<",
9
  "single_word": false,
10
- "lstrip": true,
11
- "rstrip": true,
12
  "normalized": false,
13
  "special": true
14
  },
@@ -16,8 +16,8 @@
16
  "id": 1,
17
  "content": ">>ABSTRACT<<",
18
  "single_word": false,
19
- "lstrip": true,
20
- "rstrip": true,
21
  "normalized": false,
22
  "special": true
23
  },
@@ -25,8 +25,8 @@
25
  "id": 2,
26
  "content": ">>INTRODUCTION<<",
27
  "single_word": false,
28
- "lstrip": true,
29
- "rstrip": true,
30
  "normalized": false,
31
  "special": true
32
  },
@@ -34,8 +34,8 @@
34
  "id": 3,
35
  "content": ">>SUMMARY<<",
36
  "single_word": false,
37
- "lstrip": true,
38
- "rstrip": true,
39
  "normalized": false,
40
  "special": true
41
  },
@@ -43,8 +43,8 @@
43
  "id": 4,
44
  "content": ">>COMMENT<<",
45
  "single_word": false,
46
- "lstrip": true,
47
- "rstrip": true,
48
  "normalized": false,
49
  "special": true
50
  },
@@ -52,8 +52,8 @@
52
  "id": 5,
53
  "content": ">>ANSWER<<",
54
  "single_word": false,
55
- "lstrip": true,
56
- "rstrip": true,
57
  "normalized": false,
58
  "special": true
59
  },
@@ -61,8 +61,8 @@
61
  "id": 6,
62
  "content": ">>QUESTION<<",
63
  "single_word": false,
64
- "lstrip": true,
65
- "rstrip": true,
66
  "normalized": false,
67
  "special": true
68
  },
@@ -70,8 +70,8 @@
70
  "id": 7,
71
  "content": ">>DOMAIN<<",
72
  "single_word": false,
73
- "lstrip": true,
74
- "rstrip": true,
75
  "normalized": false,
76
  "special": true
77
  },
@@ -79,8 +79,8 @@
79
  "id": 8,
80
  "content": ">>PREFIX<<",
81
  "single_word": false,
82
- "lstrip": true,
83
- "rstrip": true,
84
  "normalized": false,
85
  "special": true
86
  },
@@ -88,8 +88,8 @@
88
  "id": 9,
89
  "content": ">>SUFFIX<<",
90
  "single_word": false,
91
- "lstrip": true,
92
- "rstrip": true,
93
  "normalized": false,
94
  "special": true
95
  },
@@ -97,8 +97,8 @@
97
  "id": 10,
98
  "content": ">>MIDDLE<<",
99
  "single_word": false,
100
- "lstrip": true,
101
- "rstrip": true,
102
  "normalized": false,
103
  "special": true
104
  },
 
7
  "id": 0,
8
  "content": ">>TITLE<<",
9
  "single_word": false,
10
+ "lstrip": false,
11
+ "rstrip": false,
12
  "normalized": false,
13
  "special": true
14
  },
 
16
  "id": 1,
17
  "content": ">>ABSTRACT<<",
18
  "single_word": false,
19
+ "lstrip": false,
20
+ "rstrip": false,
21
  "normalized": false,
22
  "special": true
23
  },
 
25
  "id": 2,
26
  "content": ">>INTRODUCTION<<",
27
  "single_word": false,
28
+ "lstrip": false,
29
+ "rstrip": false,
30
  "normalized": false,
31
  "special": true
32
  },
 
34
  "id": 3,
35
  "content": ">>SUMMARY<<",
36
  "single_word": false,
37
+ "lstrip": false,
38
+ "rstrip": false,
39
  "normalized": false,
40
  "special": true
41
  },
 
43
  "id": 4,
44
  "content": ">>COMMENT<<",
45
  "single_word": false,
46
+ "lstrip": false,
47
+ "rstrip": false,
48
  "normalized": false,
49
  "special": true
50
  },
 
52
  "id": 5,
53
  "content": ">>ANSWER<<",
54
  "single_word": false,
55
+ "lstrip": false,
56
+ "rstrip": false,
57
  "normalized": false,
58
  "special": true
59
  },
 
61
  "id": 6,
62
  "content": ">>QUESTION<<",
63
  "single_word": false,
64
+ "lstrip": false,
65
+ "rstrip": false,
66
  "normalized": false,
67
  "special": true
68
  },
 
70
  "id": 7,
71
  "content": ">>DOMAIN<<",
72
  "single_word": false,
73
+ "lstrip": false,
74
+ "rstrip": false,
75
  "normalized": false,
76
  "special": true
77
  },
 
79
  "id": 8,
80
  "content": ">>PREFIX<<",
81
  "single_word": false,
82
+ "lstrip": false,
83
+ "rstrip": false,
84
  "normalized": false,
85
  "special": true
86
  },
 
88
  "id": 9,
89
  "content": ">>SUFFIX<<",
90
  "single_word": false,
91
+ "lstrip": false,
92
+ "rstrip": false,
93
  "normalized": false,
94
  "special": true
95
  },
 
97
  "id": 10,
98
  "content": ">>MIDDLE<<",
99
  "single_word": false,
100
+ "lstrip": false,
101
+ "rstrip": false,
102
  "normalized": false,
103
  "special": true
104
  },
tokenizer_config.json CHANGED
@@ -3,89 +3,89 @@
3
  "added_tokens_decoder": {
4
  "0": {
5
  "content": ">>TITLE<<",
6
- "lstrip": true,
7
  "normalized": false,
8
- "rstrip": true,
9
  "single_word": false,
10
  "special": true
11
  },
12
  "1": {
13
  "content": ">>ABSTRACT<<",
14
- "lstrip": true,
15
  "normalized": false,
16
- "rstrip": true,
17
  "single_word": false,
18
  "special": true
19
  },
20
  "2": {
21
  "content": ">>INTRODUCTION<<",
22
- "lstrip": true,
23
  "normalized": false,
24
- "rstrip": true,
25
  "single_word": false,
26
  "special": true
27
  },
28
  "3": {
29
  "content": ">>SUMMARY<<",
30
- "lstrip": true,
31
  "normalized": false,
32
- "rstrip": true,
33
  "single_word": false,
34
  "special": true
35
  },
36
  "4": {
37
  "content": ">>COMMENT<<",
38
- "lstrip": true,
39
  "normalized": false,
40
- "rstrip": true,
41
  "single_word": false,
42
  "special": true
43
  },
44
  "5": {
45
  "content": ">>ANSWER<<",
46
- "lstrip": true,
47
  "normalized": false,
48
- "rstrip": true,
49
  "single_word": false,
50
  "special": true
51
  },
52
  "6": {
53
  "content": ">>QUESTION<<",
54
- "lstrip": true,
55
  "normalized": false,
56
- "rstrip": true,
57
  "single_word": false,
58
  "special": true
59
  },
60
  "7": {
61
  "content": ">>DOMAIN<<",
62
- "lstrip": true,
63
  "normalized": false,
64
- "rstrip": true,
65
  "single_word": false,
66
  "special": true
67
  },
68
  "8": {
69
  "content": ">>PREFIX<<",
70
- "lstrip": true,
71
  "normalized": false,
72
- "rstrip": true,
73
  "single_word": false,
74
  "special": true
75
  },
76
  "9": {
77
  "content": ">>SUFFIX<<",
78
- "lstrip": true,
79
  "normalized": false,
80
- "rstrip": true,
81
  "single_word": false,
82
  "special": true
83
  },
84
  "10": {
85
  "content": ">>MIDDLE<<",
86
- "lstrip": true,
87
  "normalized": false,
88
- "rstrip": true,
89
  "single_word": false,
90
  "special": true
91
  },
 
3
  "added_tokens_decoder": {
4
  "0": {
5
  "content": ">>TITLE<<",
6
+ "lstrip": false,
7
  "normalized": false,
8
+ "rstrip": false,
9
  "single_word": false,
10
  "special": true
11
  },
12
  "1": {
13
  "content": ">>ABSTRACT<<",
14
+ "lstrip": false,
15
  "normalized": false,
16
+ "rstrip": false,
17
  "single_word": false,
18
  "special": true
19
  },
20
  "2": {
21
  "content": ">>INTRODUCTION<<",
22
+ "lstrip": false,
23
  "normalized": false,
24
+ "rstrip": false,
25
  "single_word": false,
26
  "special": true
27
  },
28
  "3": {
29
  "content": ">>SUMMARY<<",
30
+ "lstrip": false,
31
  "normalized": false,
32
+ "rstrip": false,
33
  "single_word": false,
34
  "special": true
35
  },
36
  "4": {
37
  "content": ">>COMMENT<<",
38
+ "lstrip": false,
39
  "normalized": false,
40
+ "rstrip": false,
41
  "single_word": false,
42
  "special": true
43
  },
44
  "5": {
45
  "content": ">>ANSWER<<",
46
+ "lstrip": false,
47
  "normalized": false,
48
+ "rstrip": false,
49
  "single_word": false,
50
  "special": true
51
  },
52
  "6": {
53
  "content": ">>QUESTION<<",
54
+ "lstrip": false,
55
  "normalized": false,
56
+ "rstrip": false,
57
  "single_word": false,
58
  "special": true
59
  },
60
  "7": {
61
  "content": ">>DOMAIN<<",
62
+ "lstrip": false,
63
  "normalized": false,
64
+ "rstrip": false,
65
  "single_word": false,
66
  "special": true
67
  },
68
  "8": {
69
  "content": ">>PREFIX<<",
70
+ "lstrip": false,
71
  "normalized": false,
72
+ "rstrip": false,
73
  "single_word": false,
74
  "special": true
75
  },
76
  "9": {
77
  "content": ">>SUFFIX<<",
78
+ "lstrip": false,
79
  "normalized": false,
80
+ "rstrip": false,
81
  "single_word": false,
82
  "special": true
83
  },
84
  "10": {
85
  "content": ">>MIDDLE<<",
86
+ "lstrip": false,
87
  "normalized": false,
88
+ "rstrip": false,
89
  "single_word": false,
90
  "special": true
91
  },