Pratye commited on
Commit
f7118ca
1 Parent(s): 7628381

changed model

Browse files
.DS_Store CHANGED
Binary files a/.DS_Store and b/.DS_Store differ
 
special_tokens_map.json CHANGED
@@ -1,9 +1,6 @@
1
  {
2
- "additional_special_tokens": [
3
- "<|im_start|>",
4
- "<|im_end|>"
5
- ],
6
  "bos_token": "<|endoftext|>",
7
  "eos_token": "<|endoftext|>",
 
8
  "unk_token": "<|endoftext|>"
9
  }
 
1
  {
 
 
 
 
2
  "bos_token": "<|endoftext|>",
3
  "eos_token": "<|endoftext|>",
4
+ "pad_token": "<pad>",
5
  "unk_token": "<|endoftext|>"
6
  }
tokenizer.json CHANGED
@@ -1,6 +1,11 @@
1
  {
2
  "version": "1.0",
3
- "truncation": null,
 
 
 
 
 
4
  "padding": null,
5
  "added_tokens": [
6
  {
@@ -230,16 +235,7 @@
230
  },
231
  {
232
  "id": 50277,
233
- "content": "<|im_start|>",
234
- "single_word": false,
235
- "lstrip": false,
236
- "rstrip": false,
237
- "normalized": false,
238
- "special": true
239
- },
240
- {
241
- "id": 50278,
242
- "content": "<|im_end|>",
243
  "single_word": false,
244
  "lstrip": false,
245
  "rstrip": false,
 
1
  {
2
  "version": "1.0",
3
+ "truncation": {
4
+ "direction": "Right",
5
+ "max_length": 256,
6
+ "strategy": "LongestFirst",
7
+ "stride": 0
8
+ },
9
  "padding": null,
10
  "added_tokens": [
11
  {
 
235
  },
236
  {
237
  "id": 50277,
238
+ "content": "<pad>",
 
 
 
 
 
 
 
 
 
239
  "single_word": false,
240
  "lstrip": false,
241
  "rstrip": false,
tokenizer_config.json CHANGED
@@ -3,7 +3,7 @@
3
  "bos_token": "<|endoftext|>",
4
  "clean_up_tokenization_spaces": true,
5
  "eos_token": "<|endoftext|>",
6
- "model_max_length": 2048,
7
  "tokenizer_class": "GPTNeoXTokenizer",
8
  "unk_token": "<|endoftext|>"
9
  }
 
3
  "bos_token": "<|endoftext|>",
4
  "clean_up_tokenization_spaces": true,
5
  "eos_token": "<|endoftext|>",
6
+ "model_max_length": 1000000000000000019884624838656,
7
  "tokenizer_class": "GPTNeoXTokenizer",
8
  "unk_token": "<|endoftext|>"
9
  }
training_args.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:f6fd8b171d9c3293f7b491bd99987013767cbf9f7f719831cb36476e8b3b7176
3
  size 3899
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bc9e1fde44399d127bf7e62feaeb5608f2344a3d4a2cd75ada116c02fbe20cd7
3
  size 3899