jhsmith commited on
Commit
d705658
1 Parent(s): 66002c8

Upload folder using huggingface_hub

Browse files
1_Pooling/config.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "word_embedding_dimension": 768,
3
+ "pooling_mode_cls_token": false,
4
+ "pooling_mode_mean_tokens": true,
5
+ "pooling_mode_max_tokens": false,
6
+ "pooling_mode_mean_sqrt_len_tokens": false
7
+ }
README.md ADDED
@@ -0,0 +1,88 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ pipeline_tag: sentence-similarity
3
+ tags:
4
+ - sentence-transformers
5
+ - feature-extraction
6
+ - sentence-similarity
7
+
8
+ ---
9
+
10
+ # {MODEL_NAME}
11
+
12
+ This is a [sentence-transformers](https://www.SBERT.net) model: It maps sentences & paragraphs to a 768 dimensional dense vector space and can be used for tasks like clustering or semantic search.
13
+
14
+ <!--- Describe your model here -->
15
+
16
+ ## Usage (Sentence-Transformers)
17
+
18
+ Using this model becomes easy when you have [sentence-transformers](https://www.SBERT.net) installed:
19
+
20
+ ```
21
+ pip install -U sentence-transformers
22
+ ```
23
+
24
+ Then you can use the model like this:
25
+
26
+ ```python
27
+ from sentence_transformers import SentenceTransformer
28
+ sentences = ["This is an example sentence", "Each sentence is converted"]
29
+
30
+ model = SentenceTransformer('{MODEL_NAME}')
31
+ embeddings = model.encode(sentences)
32
+ print(embeddings)
33
+ ```
34
+
35
+
36
+
37
+ ## Evaluation Results
38
+
39
+ <!--- Describe how your model was evaluated -->
40
+
41
+ For an automated evaluation of this model, see the *Sentence Embeddings Benchmark*: [https://seb.sbert.net](https://seb.sbert.net?model_name={MODEL_NAME})
42
+
43
+
44
+ ## Training
45
+ The model was trained with the parameters:
46
+
47
+ **DataLoader**:
48
+
49
+ `torch.utils.data.dataloader.DataLoader` of length 1754 with parameters:
50
+ ```
51
+ {'batch_size': 10, 'sampler': 'torch.utils.data.sampler.RandomSampler', 'batch_sampler': 'torch.utils.data.sampler.BatchSampler'}
52
+ ```
53
+
54
+ **Loss**:
55
+
56
+ `sentence_transformers.losses.CosineSimilarityLoss.CosineSimilarityLoss`
57
+
58
+ Parameters of the fit()-Method:
59
+ ```
60
+ {
61
+ "epochs": 3,
62
+ "evaluation_steps": 200,
63
+ "evaluator": "sentence_transformers.evaluation.BinaryClassificationEvaluator.BinaryClassificationEvaluator",
64
+ "max_grad_norm": 1,
65
+ "optimizer_class": "<class 'torch.optim.adamw.AdamW'>",
66
+ "optimizer_params": {
67
+ "lr": 0.0001
68
+ },
69
+ "scheduler": "warmuplinear",
70
+ "steps_per_epoch": null,
71
+ "warmup_steps": 526,
72
+ "weight_decay": 0.01
73
+ }
74
+ ```
75
+
76
+
77
+ ## Full Model Architecture
78
+ ```
79
+ SentenceTransformer(
80
+ (0): Transformer({'max_seq_length': 512, 'do_lower_case': False}) with Transformer model: MPNetModel
81
+ (1): Pooling({'word_embedding_dimension': 768, 'pooling_mode_cls_token': False, 'pooling_mode_mean_tokens': True, 'pooling_mode_max_tokens': False, 'pooling_mode_mean_sqrt_len_tokens': False})
82
+ (2): Normalize()
83
+ )
84
+ ```
85
+
86
+ ## Citing & Authors
87
+
88
+ <!--- Describe where people can find more information -->
config.json ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "/root/.cache/torch/sentence_transformers/sentence-transformers_multi-qa-mpnet-base-cos-v1/",
3
+ "architectures": [
4
+ "MPNetModel"
5
+ ],
6
+ "attention_probs_dropout_prob": 0.1,
7
+ "bos_token_id": 0,
8
+ "eos_token_id": 2,
9
+ "hidden_act": "gelu",
10
+ "hidden_dropout_prob": 0.1,
11
+ "hidden_size": 768,
12
+ "initializer_range": 0.02,
13
+ "intermediate_size": 3072,
14
+ "layer_norm_eps": 1e-05,
15
+ "max_position_embeddings": 514,
16
+ "model_type": "mpnet",
17
+ "num_attention_heads": 12,
18
+ "num_hidden_layers": 12,
19
+ "pad_token_id": 1,
20
+ "relative_attention_num_buckets": 32,
21
+ "torch_dtype": "float32",
22
+ "transformers_version": "4.35.2",
23
+ "vocab_size": 30527
24
+ }
config_sentence_transformers.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "__version__": {
3
+ "sentence_transformers": "2.0.0",
4
+ "transformers": "4.6.1",
5
+ "pytorch": "1.8.1"
6
+ }
7
+ }
eval/binary_classification_evaluation_results_results.csv ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ epoch,steps,cossim_accuracy,cossim_accuracy_threshold,cossim_f1,cossim_precision,cossim_recall,cossim_f1_threshold,cossim_ap,manhattan_accuracy,manhattan_accuracy_threshold,manhattan_f1,manhattan_precision,manhattan_recall,manhattan_f1_threshold,manhattan_ap,euclidean_accuracy,euclidean_accuracy_threshold,euclidean_f1,euclidean_precision,euclidean_recall,euclidean_f1_threshold,euclidean_ap,dot_accuracy,dot_accuracy_threshold,dot_f1,dot_precision,dot_recall,dot_f1_threshold,dot_ap
2
+ 0,200,0.7998691955526488,0.5035068392753601,0.6185699039487726,0.5573076923076923,0.6949640287769784,0.42131906747817993,0.6577360566350412,0.7992151733158928,21.556745529174805,0.6161116111611161,0.5803306485799068,0.6565947242206235,22.796972274780273,0.6589241550951741,0.7998691955526488,0.996487021446228,0.6185699039487726,0.5573076923076923,0.6949640287769784,1.0758075714111328,0.6577356254393829,0.7998691955526488,0.5035068988800049,0.6185699039487726,0.5573076923076923,0.6949640287769784,0.42131900787353516,0.6577356229628153
3
+ 0,400,0.7839110529758012,0.63353431224823,0.5859426319246771,0.5390813859790492,0.641726618705036,0.5621637105941772,0.6171901975303064,0.7845650752125573,17.985877990722656,0.582968977722257,0.515084621044886,0.6714628297362111,20.345890045166016,0.6168462240256587,0.7839110529758012,0.8561141490936279,0.5859426319246771,0.5390813859790492,0.641726618705036,0.9357737898826599,0.6171912664691657,0.7839110529758012,0.6335342526435852,0.5859426319246771,0.5390813859790492,0.641726618705036,0.5621637105941772,0.6171902717909105
4
+ 0,600,0.7880967952910399,0.5784088373184204,0.5794983211534663,0.4926124916051041,0.7035971223021583,0.44478359818458557,0.625469767136268,0.7886200130804447,19.71192169189453,0.579823299773988,0.5071890726096333,0.6767386091127098,22.62847328186035,0.6244709180763914,0.7880967952910399,0.9182496070861816,0.5794983211534663,0.4926124916051041,0.7035971223021583,1.0537707805633545,0.6254697870968644,0.7880967952910399,0.5784088969230652,0.5794983211534663,0.4926124916051041,0.7035971223021583,0.44478359818458557,0.625469724570718
5
+ 0,800,0.7891432308698496,0.5749017000198364,0.5844938682186656,0.5157740278796772,0.6743405275779376,0.46267417073249817,0.6457919231921743,0.7896664486592544,19.415851593017578,0.5887113951011714,0.5295019157088122,0.6628297362110311,22.284927368164062,0.6467985443938986,0.7891432308698496,0.9220610857009888,0.5844938682186656,0.5157740278796772,0.6743405275779376,1.0366541147232056,0.6457914886795355,0.7891432308698496,0.5749017000198364,0.5844938682186656,0.5157740278796772,0.6743405275779376,0.46267420053482056,0.6457917006614962
6
+ 0,1000,0.7713538260300851,0.6151977777481079,0.522463768115942,0.4197962154294032,0.6916067146282974,0.4502478241920471,0.5689211243333396,0.7703073904512754,19.468528747558594,0.5206662553979026,0.4557235421166307,0.6071942446043166,22.287307739257812,0.5644566776390325,0.7713538260300851,0.8772710561752319,0.522463768115942,0.4197962154294032,0.6916067146282974,1.0485725402832031,0.5689212926621392,0.7713538260300851,0.6151977777481079,0.522463768115942,0.4197962154294032,0.6916067146282974,0.4502478837966919,0.5689213297185012
7
+ 0,1200,0.7977763243950294,0.5469571352005005,0.5738284703801947,0.5321853218532185,0.6225419664268586,0.44128894805908203,0.6457100700833103,0.7982995421844342,20.99976348876953,0.5733274257864421,0.5327295183202965,0.6206235011990408,22.861082077026367,0.647079678899965,0.7977763243950294,0.9518853425979614,0.5738284703801947,0.5321853218532185,0.6225419664268586,1.0570818185806274,0.6457099884834276,0.7977763243950294,0.5469571352005005,0.5738284703801947,0.5321853218532185,0.6225419664268586,0.4412890076637268,0.645710123136259
8
+ 0,1400,0.8001308044473512,0.5292196273803711,0.6009132420091324,0.5734204793028322,0.6311750599520384,0.4476647973060608,0.6656324906806985,0.8002616088947024,21.033573150634766,0.6010805943268798,0.5663979635129401,0.6402877697841727,22.788015365600586,0.6653621336181923,0.8001308044473512,0.9703404307365417,0.6009132420091324,0.5734204793028322,0.6311750599520384,1.0510330200195312,0.6656330556657485,0.8001308044473512,0.5292196869850159,0.6009132420091324,0.5734204793028322,0.6311750599520384,0.4476647973060608,0.6656335179777989
9
+ 0,1600,0.7905820797907129,0.5070725679397583,0.5727058823529412,0.5621247113163972,0.5836930455635492,0.4188060462474823,0.6316034170684575,0.7914977109221714,21.59149932861328,0.5666016894087069,0.5165876777251185,0.6273381294964029,23.692493438720703,0.6288365420127189,0.7905820797907129,0.9929022192955017,0.5727058823529412,0.5621247113163972,0.5836930455635492,1.0781409740447998,0.6316040864470261,0.7905820797907129,0.5070726871490479,0.5727058823529412,0.5621247113163972,0.5836930455635492,0.4188061058521271,0.6316041751301711
10
+ 0,-1,0.7926749509483323,0.554900586605072,0.6117862875487273,0.5861159929701231,0.6398081534772182,0.4722902178764343,0.6689181047071084,0.791890124264225,20.927637100219727,0.6068015114469881,0.5654515327257663,0.6546762589928058,22.309982299804688,0.6669604030046048,0.7926749509483323,0.9435034394264221,0.6117862875487273,0.5861159929701231,0.6398081534772182,1.0273361206054688,0.6689173731600689,0.7926749509483323,0.5549006462097168,0.6117862875487273,0.5861159929701231,0.6398081534772182,0.4722902178764343,0.6689181505853008
11
+ 1,200,0.8028776978417266,0.5257875323295593,0.6277888226198365,0.5819000819000819,0.6815347721822542,0.4520568251609802,0.6823749663224601,0.8031393067364291,20.46468734741211,0.6299976889299745,0.6079393398751115,0.6537170263788968,22.408832550048828,0.6824545778238449,0.8028776978417266,0.9738709926605225,0.6277888226198365,0.5819000819000819,0.6815347721822542,1.0468459129333496,0.6823755753320633,0.8028776978417266,0.5257875919342041,0.6277888226198365,0.5819000819000819,0.6815347721822542,0.452056884765625,0.6823750610364453
12
+ 1,400,0.8090255068672335,0.5916064977645874,0.6194260485651214,0.5738241308793456,0.6729016786570743,0.48431360721588135,0.6984673893691323,0.8092871157619359,20.164798736572266,0.6170940170940171,0.5564547206165703,0.6925659472422062,22.228458404541016,0.6981371857703524,0.8090255068672335,0.9037626385688782,0.6194260485651214,0.5738241308793456,0.6729016786570743,1.0155651569366455,0.6984673079354968,0.8090255068672335,0.5916065573692322,0.6194260485651214,0.5738241308793456,0.6729016786570743,0.4843136668205261,0.69846740078578
13
+ 1,600,0.8074558534990189,0.5267857909202576,0.637460885113992,0.5969024696525743,0.6839328537170264,0.45214104652404785,0.6995278895236822,0.8066710268149118,21.325407028198242,0.6318430656934306,0.6024358416702914,0.6642685851318945,22.5443172454834,0.6949298746708277,0.8074558534990189,0.9728455543518066,0.637460885113992,0.5969024696525743,0.6839328537170264,1.0467654466629028,0.6995278927690958,0.8074558534990189,0.5267858505249023,0.637460885113992,0.5969024696525743,0.6839328537170264,0.45214107632637024,0.6995278895236822
14
+ 1,800,0.8098103335513408,0.5241738557815552,0.6305418719211823,0.5696594427244582,0.7059952038369305,0.44685932993888855,0.6995794412843676,0.8086330935251799,21.047618865966797,0.626903023983316,0.5546125461254613,0.720863309352518,22.91455078125,0.6973138848248075,0.8098103335513408,0.9755265712738037,0.6305418719211823,0.5696594427244582,0.7059952038369305,1.0517990589141846,0.6995792975010435,0.8098103335513408,0.5241739749908447,0.6305418719211823,0.5696594427244582,0.7059952038369305,0.44685932993888855,0.6995793356181231
15
+ 1,1000,0.821190320470896,0.5180328488349915,0.6571098265895955,0.634375,0.6815347721822542,0.4495694041252136,0.7291954761851456,0.8219751471550033,21.586544036865234,0.659372026641294,0.6540821142048135,0.6647482014388489,22.583351135253906,0.7289931240762048,0.821190320470896,0.9818015098571777,0.6571098265895955,0.634375,0.6815347721822542,1.0492193698883057,0.7291947044511737,0.821190320470896,0.5180329084396362,0.6571098265895955,0.634375,0.6815347721822542,0.44956931471824646,0.7291954761851454
16
+ 1,1200,0.8192282537606279,0.5209615230560303,0.6523034609213842,0.6246707638279192,0.6824940047961631,0.426084041595459,0.7214129570029649,0.8202746893394376,21.21324920654297,0.6478500551267915,0.5995918367346939,0.7045563549160672,23.57623291015625,0.7204173944978056,0.8192282537606279,0.9788140058517456,0.6523034609213842,0.6246707638279192,0.6824940047961631,1.0713691711425781,0.7214128683883151,0.8192282537606279,0.5209615230560303,0.6523034609213842,0.6246707638279192,0.6824940047961631,0.4260839819908142,0.7214127961027852
17
+ 1,1400,0.8108567691301505,0.5470163822174072,0.6303086563007658,0.6106115107913669,0.6513189448441247,0.44464150071144104,0.7020750372022517,0.8111183780248529,20.772302627563477,0.6299428571428572,0.6017467248908297,0.6609112709832135,23.076847076416016,0.7001812243796641,0.8108567691301505,0.9518229961395264,0.6303086563007658,0.6106115107913669,0.6513189448441247,1.0539054870605469,0.7020750100229519,0.8108567691301505,0.5470165014266968,0.6303086563007658,0.6106115107913669,0.6513189448441247,0.4446415901184082,0.7020751464838597
18
+ 1,1600,0.815565729234794,0.5448479652404785,0.647656940760389,0.6006560065600656,0.7026378896882494,0.44379207491874695,0.7190730536819719,0.8147809025506867,21.06966781616211,0.6456375838926174,0.6050314465408805,0.6920863309352518,22.856258392333984,0.7172335862653456,0.815565729234794,0.9540985822677612,0.647656940760389,0.6006560065600656,0.7026378896882494,1.0547112226486206,0.7190730536819719,0.815565729234794,0.5448479652404785,0.647656940760389,0.6006560065600656,0.7026378896882494,0.44379210472106934,0.7190731801696373
19
+ 1,-1,0.8171353826030086,0.5106275081634521,0.658645276292335,0.6150645027049522,0.708872901678657,0.42436525225639343,0.7259585439463045,0.8170045781556573,21.425724029541016,0.6532167675409101,0.6132154882154882,0.6988009592326139,23.28256607055664,0.7246329191257518,0.8171353826030086,0.989315390586853,0.658645276292335,0.6150645027049522,0.708872901678657,1.072972297668457,0.7259583987938434,0.8171353826030086,0.5106275081634521,0.658645276292335,0.6150645027049522,0.708872901678657,0.42436525225639343,0.7259572314622995
20
+ 2,200,0.8150425114453892,0.5469454526901245,0.6458715596330276,0.6189010989010989,0.6752997601918466,0.4445401430130005,0.7147232266192974,0.8158273381294964,21.00546646118164,0.6440677966101696,0.6019174656106712,0.6925659472422062,23.104610443115234,0.7142291448054945,0.8150425114453892,0.9518976211547852,0.6458715596330276,0.6189010989010989,0.6752997601918466,1.054001808166504,0.7147233515023913,0.8150425114453892,0.5469454526901245,0.6458715596330276,0.6189010989010989,0.6752997601918466,0.4445401430130005,0.7147234223463026
21
+ 2,400,0.8231523871811641,0.5767366886138916,0.650557620817844,0.6309148264984227,0.6714628297362111,0.4801790118217468,0.7254713030739945,0.8224983649444081,19.760700225830078,0.6513560804899388,0.598713309207881,0.7141486810551558,22.607635498046875,0.7252077912564496,0.8231523871811641,0.9200687408447266,0.650557620817844,0.6309148264984227,0.6714628297362111,1.0196282863616943,0.7254705174037968,0.8231523871811641,0.5767366290092468,0.650557620817844,0.6309148264984227,0.6714628297362111,0.48017898201942444,0.7254734606230803
22
+ 2,600,0.8249836494440811,0.5597643852233887,0.666821669379214,0.6471119133574007,0.6877697841726619,0.4601597189903259,0.7331037390320215,0.825506867233486,20.167991638183594,0.6633119853613906,0.6340183646698732,0.6954436450839329,22.711009979248047,0.7335697108338145,0.8249836494440811,0.9383342266082764,0.666821669379214,0.6471119133574007,0.6877697841726619,1.039076805114746,0.7331039110532859,0.8249836494440811,0.5597643852233887,0.666821669379214,0.6471119133574007,0.6877697841726619,0.4601597189903259,0.7331035668324976
23
+ 2,800,0.8274689339437541,0.5480323433876038,0.6723290120697363,0.6295521138551695,0.7213429256594724,0.4530275762081146,0.7401516315496964,0.8279921517331589,20.234455108642578,0.6716860597764089,0.6405570060922542,0.7059952038369305,22.636369705200195,0.7398224826658886,0.8274689339437541,0.9507551193237305,0.6723290120697363,0.6295521138551695,0.7213429256594724,1.0459181070327759,0.7401516315496964,0.8274689339437541,0.5480324029922485,0.6723290120697363,0.6295521138551695,0.7213429256594724,0.4530276656150818,0.7401516397596143
24
+ 2,1000,0.8299542184434271,0.5279191732406616,0.6687058823529413,0.6563510392609699,0.6815347721822542,0.45454949140548706,0.7410463405834499,0.8289077828646174,21.518436431884766,0.6700071581961347,0.6666666666666666,0.6733812949640288,22.57660484313965,0.7398565706037402,0.8299542184434271,0.9716796875,0.6687058823529413,0.6563510392609699,0.6815347721822542,1.044461965560913,0.7410463971838308,0.8299542184434271,0.5279192924499512,0.6687058823529413,0.6563510392609699,0.6815347721822542,0.45454955101013184,0.7410465008681321
25
+ 2,1200,0.828646173969915,0.5080829858779907,0.6666666666666666,0.6715328467153284,0.6618705035971223,0.46455180644989014,0.7385378340798057,0.8283845650752125,21.318214416503906,0.6676029962546817,0.6520347508001829,0.6839328537170264,22.77878189086914,0.7386530352071838,0.828646173969915,0.9918840527534485,0.6666666666666666,0.6715328467153284,0.6618705035971223,1.0348410606384277,0.7385373706023484,0.828646173969915,0.5080829858779907,0.6666666666666666,0.6715328467153284,0.6618705035971223,0.4645518362522125,0.7385373981874322
26
+ 2,1400,0.8262916939175932,0.4881095290184021,0.667590027700831,0.6435246995994659,0.6935251798561151,0.4210280478000641,0.7394010005304796,0.825506867233486,22.013267517089844,0.6665226781857451,0.6062868369351669,0.7400479616306954,23.819936752319336,0.7379512668075002,0.8262916939175932,1.0118205547332764,0.667590027700831,0.6435246995994659,0.6935251798561151,1.0760780572891235,0.7394006130978208,0.8262916939175932,0.48810964822769165,0.667590027700831,0.6435246995994659,0.6935251798561151,0.4210280776023865,0.7394010221655671
27
+ 2,1600,0.8299542184434271,0.5254404544830322,0.6692238267148014,0.6318704729441841,0.7112709832134293,0.44381532073020935,0.7416313556039303,0.8293001962066711,20.976154327392578,0.6685466377440347,0.6102970297029703,0.7390887290167866,23.206363677978516,0.7409460957926604,0.8299542184434271,0.9742273688316345,0.6692238267148014,0.6318704729441841,0.7112709832134293,1.0546891689300537,0.7416314065645258,0.8299542184434271,0.525440514087677,0.6692238267148014,0.6318704729441841,0.7112709832134293,0.4438154399394989,0.7416313556039305
28
+ 2,-1,0.8303466317854807,0.5230963230133057,0.6696428571428572,0.6263048016701461,0.7194244604316546,0.4412711262702942,0.7419962359413731,0.8302158273381295,21.149803161621094,0.6688993078812235,0.6257309941520468,0.7184652278177458,22.952348709106445,0.7415379119214903,0.8303466317854807,0.9766306281089783,0.6696428571428572,0.6263048016701461,0.7194244604316546,1.0570987462997437,0.7419962210682528,0.8303466317854807,0.5230963826179504,0.6696428571428572,0.6263048016701461,0.7194244604316546,0.44127118587493896,0.7419952491491985
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:14023ca9faef8e09f9dd384bdb506f7cbca21db5ec4c703895f815f64bf9166f
3
+ size 437967672
modules.json ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [
2
+ {
3
+ "idx": 0,
4
+ "name": "0",
5
+ "path": "",
6
+ "type": "sentence_transformers.models.Transformer"
7
+ },
8
+ {
9
+ "idx": 1,
10
+ "name": "1",
11
+ "path": "1_Pooling",
12
+ "type": "sentence_transformers.models.Pooling"
13
+ },
14
+ {
15
+ "idx": 2,
16
+ "name": "2",
17
+ "path": "2_Normalize",
18
+ "type": "sentence_transformers.models.Normalize"
19
+ }
20
+ ]
sentence_bert_config.json ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ {
2
+ "max_seq_length": 512,
3
+ "do_lower_case": false
4
+ }
special_tokens_map.json ADDED
@@ -0,0 +1,51 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<s>",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "cls_token": {
10
+ "content": "<s>",
11
+ "lstrip": false,
12
+ "normalized": true,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "eos_token": {
17
+ "content": "</s>",
18
+ "lstrip": false,
19
+ "normalized": false,
20
+ "rstrip": false,
21
+ "single_word": false
22
+ },
23
+ "mask_token": {
24
+ "content": "<mask>",
25
+ "lstrip": true,
26
+ "normalized": false,
27
+ "rstrip": false,
28
+ "single_word": false
29
+ },
30
+ "pad_token": {
31
+ "content": "<pad>",
32
+ "lstrip": false,
33
+ "normalized": false,
34
+ "rstrip": false,
35
+ "single_word": false
36
+ },
37
+ "sep_token": {
38
+ "content": "</s>",
39
+ "lstrip": false,
40
+ "normalized": true,
41
+ "rstrip": false,
42
+ "single_word": false
43
+ },
44
+ "unk_token": {
45
+ "content": "[UNK]",
46
+ "lstrip": false,
47
+ "normalized": false,
48
+ "rstrip": false,
49
+ "single_word": false
50
+ }
51
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,72 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "<s>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ },
11
+ "1": {
12
+ "content": "<pad>",
13
+ "lstrip": false,
14
+ "normalized": false,
15
+ "rstrip": false,
16
+ "single_word": false,
17
+ "special": true
18
+ },
19
+ "2": {
20
+ "content": "</s>",
21
+ "lstrip": false,
22
+ "normalized": false,
23
+ "rstrip": false,
24
+ "single_word": false,
25
+ "special": true
26
+ },
27
+ "3": {
28
+ "content": "<unk>",
29
+ "lstrip": false,
30
+ "normalized": true,
31
+ "rstrip": false,
32
+ "single_word": false,
33
+ "special": true
34
+ },
35
+ "104": {
36
+ "content": "[UNK]",
37
+ "lstrip": false,
38
+ "normalized": false,
39
+ "rstrip": false,
40
+ "single_word": false,
41
+ "special": true
42
+ },
43
+ "30526": {
44
+ "content": "<mask>",
45
+ "lstrip": true,
46
+ "normalized": false,
47
+ "rstrip": false,
48
+ "single_word": false,
49
+ "special": true
50
+ }
51
+ },
52
+ "bos_token": "<s>",
53
+ "clean_up_tokenization_spaces": true,
54
+ "cls_token": "<s>",
55
+ "do_lower_case": true,
56
+ "eos_token": "</s>",
57
+ "mask_token": "<mask>",
58
+ "max_length": 250,
59
+ "model_max_length": 512,
60
+ "pad_to_multiple_of": null,
61
+ "pad_token": "<pad>",
62
+ "pad_token_type_id": 0,
63
+ "padding_side": "right",
64
+ "sep_token": "</s>",
65
+ "stride": 0,
66
+ "strip_accents": null,
67
+ "tokenize_chinese_chars": true,
68
+ "tokenizer_class": "MPNetTokenizer",
69
+ "truncation_side": "right",
70
+ "truncation_strategy": "longest_first",
71
+ "unk_token": "[UNK]"
72
+ }
vocab.txt ADDED
The diff for this file is too large to render. See raw diff