adamjweintraut commited on
Commit
39bf46e
1 Parent(s): 2eb1f2b

Upload tokenizer

Browse files
README.md CHANGED
@@ -1,8 +1,8 @@
1
  ---
2
  license: apache-2.0
3
- base_model: facebook/bart-large
4
  tags:
5
  - generated_from_trainer
 
6
  model-index:
7
  - name: bart-finetuned-loaf-512-lyrictoplan
8
  results: []
 
1
  ---
2
  license: apache-2.0
 
3
  tags:
4
  - generated_from_trainer
5
+ base_model: facebook/bart-large
6
  model-index:
7
  - name: bart-finetuned-loaf-512-lyrictoplan
8
  results: []
special_tokens_map.json CHANGED
@@ -1,13 +1,4 @@
1
  {
2
- "additional_special_tokens": [
3
- {
4
- "content": "<P>",
5
- "lstrip": false,
6
- "normalized": false,
7
- "rstrip": false,
8
- "single_word": false
9
- }
10
- ],
11
  "bos_token": "<s>",
12
  "cls_token": "<s>",
13
  "eos_token": "</s>",
 
1
  {
 
 
 
 
 
 
 
 
 
2
  "bos_token": "<s>",
3
  "cls_token": "<s>",
4
  "eos_token": "</s>",
tokenizer.json CHANGED
@@ -68,8 +68,8 @@
68
  "single_word": false,
69
  "lstrip": false,
70
  "rstrip": false,
71
- "normalized": false,
72
- "special": true
73
  }
74
  ],
75
  "normalizer": null,
 
68
  "single_word": false,
69
  "lstrip": false,
70
  "rstrip": false,
71
+ "normalized": true,
72
+ "special": false
73
  }
74
  ],
75
  "normalizer": null,
tokenizer_config.json CHANGED
@@ -44,15 +44,12 @@
44
  "50265": {
45
  "content": "<P>",
46
  "lstrip": false,
47
- "normalized": false,
48
  "rstrip": false,
49
  "single_word": false,
50
- "special": true
51
  }
52
  },
53
- "additional_special_tokens": [
54
- "<P>"
55
- ],
56
  "bos_token": "<s>",
57
  "clean_up_tokenization_spaces": true,
58
  "cls_token": "<s>",
 
44
  "50265": {
45
  "content": "<P>",
46
  "lstrip": false,
47
+ "normalized": true,
48
  "rstrip": false,
49
  "single_word": false,
50
+ "special": false
51
  }
52
  },
 
 
 
53
  "bos_token": "<s>",
54
  "clean_up_tokenization_spaces": true,
55
  "cls_token": "<s>",