| [ | |
| { | |
| "model_name_or_path": "elastic/distilbert-base-uncased-finetuned-conll03-english", | |
| "task": "token-classification", | |
| "dataset": { | |
| "path": "conll2003", | |
| "eval_split": "validation", | |
| "data_keys": { | |
| "primary": "tokens", | |
| "secondary": null | |
| }, | |
| "ref_keys": [ | |
| "ner_tags" | |
| ], | |
| "name": null, | |
| "calibration_split": "train" | |
| }, | |
| "quantization_approach": "static", | |
| "operators_to_quantize": [ | |
| "Add" | |
| ], | |
| "node_exclusion": [], | |
| "aware_training": false, | |
| "per_channel": false, | |
| "calibration": { | |
| "method": "minmax", | |
| "num_calibration_samples": 100, | |
| "calibration_histogram_percentile": null, | |
| "calibration_moving_average": null, | |
| "calibration_moving_average_constant": null | |
| }, | |
| "framework": "onnxruntime", | |
| "framework_args": { | |
| "opset": 11, | |
| "optimization_level": 1 | |
| }, | |
| "hardware": "Architecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nByte Order: Little Endian\nAddress sizes: 46 bits physical, 48 bits virtual\nCPU(s): 8\nOn-line CPU(s) list: 0-7\nThread(s) per core: 2\nCore(s) per socket: 4\nSocket(s): 1\nNUMA node(s): 1\nVendor ID: GenuineIntel\nCPU family: 6\nModel: 85\nModel name: Intel(R) Xeon(R) Platinum 8259CL CPU @ 2.50GHz\nStepping: 7\nCPU MHz: 3099.998\nBogoMIPS: 4999.99\nHypervisor vendor: KVM\nVirtualization type: full\nL1d cache: 128 KiB\nL1i cache: 128 KiB\nL2 cache: 4 MiB\nL3 cache: 35.8 MiB\nNUMA node0 CPU(s): 0-7\nVulnerability Itlb multihit: KVM: Vulnerable\nVulnerability L1tf: Mitigation; PTE Inversion\nVulnerability Mds: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Meltdown: Mitigation; PTI\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines, STIBP disabled, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single pti fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke\n", | |
| "versions": { | |
| "transformers": "4.20.1", | |
| "optimum": "1.2.3.dev0", | |
| "optimum_hash": "5ac9c0d9fd7e7cca55b2f9935b961ed5b6c50112" | |
| }, | |
| "evaluation": { | |
| "time": [ | |
| { | |
| "batch_size": 8, | |
| "input_length": 32, | |
| "baseline": { | |
| "nb_forwards": 130, | |
| "throughput": 8.67, | |
| "latency_mean": 115.81472209230769, | |
| "latency_std": 0.46895980796384223, | |
| "latency_50": 115.7693145, | |
| "latency_90": 116.2641674, | |
| "latency_95": 116.615633, | |
| "latency_99": 117.24392198999999, | |
| "latency_999": 118.33366130000002 | |
| }, | |
| "optimized": { | |
| "nb_forwards": 119, | |
| "throughput": 7.93, | |
| "latency_mean": 126.53206931932773, | |
| "latency_std": 0.21043917502896844, | |
| "latency_50": 126.51378, | |
| "latency_90": 126.8398328, | |
| "latency_95": 126.8832745, | |
| "latency_99": 127.04870174, | |
| "latency_999": 127.236541298 | |
| } | |
| }, | |
| { | |
| "batch_size": 4, | |
| "input_length": 128, | |
| "baseline": { | |
| "nb_forwards": 69, | |
| "throughput": 4.6, | |
| "latency_mean": 219.5741049275362, | |
| "latency_std": 0.425956641594441, | |
| "latency_50": 219.571816, | |
| "latency_90": 220.0821694, | |
| "latency_95": 220.1998948, | |
| "latency_99": 220.53578456, | |
| "latency_999": 220.90863515599997 | |
| }, | |
| "optimized": { | |
| "nb_forwards": 74, | |
| "throughput": 4.93, | |
| "latency_mean": 203.70586305405405, | |
| "latency_std": 4.723902925243521, | |
| "latency_50": 202.578965, | |
| "latency_90": 211.061159, | |
| "latency_95": 213.91898604999997, | |
| "latency_99": 216.57558472, | |
| "latency_999": 216.849314572 | |
| } | |
| }, | |
| { | |
| "batch_size": 4, | |
| "input_length": 64, | |
| "baseline": { | |
| "nb_forwards": 155, | |
| "throughput": 10.33, | |
| "latency_mean": 96.98536739354839, | |
| "latency_std": 13.276058953171962, | |
| "latency_50": 90.032648, | |
| "latency_90": 119.895212, | |
| "latency_95": 120.1166701, | |
| "latency_99": 121.32664904, | |
| "latency_999": 122.10800528 | |
| }, | |
| "optimized": { | |
| "nb_forwards": 148, | |
| "throughput": 9.87, | |
| "latency_mean": 102.02715479054055, | |
| "latency_std": 2.9594884849383276, | |
| "latency_50": 101.4796865, | |
| "latency_90": 105.83677709999999, | |
| "latency_95": 107.30721310000001, | |
| "latency_99": 110.73326877, | |
| "latency_999": 111.11521851100001 | |
| } | |
| }, | |
| { | |
| "batch_size": 8, | |
| "input_length": 128, | |
| "baseline": { | |
| "nb_forwards": 38, | |
| "throughput": 2.53, | |
| "latency_mean": 403.54761702631583, | |
| "latency_std": 0.7543435235070463, | |
| "latency_50": 403.5543235, | |
| "latency_90": 404.57511619999997, | |
| "latency_95": 404.74615289999997, | |
| "latency_99": 404.80311413, | |
| "latency_999": 404.825042513 | |
| }, | |
| "optimized": { | |
| "nb_forwards": 37, | |
| "throughput": 2.47, | |
| "latency_mean": 410.16972697297297, | |
| "latency_std": 7.6545384613060605, | |
| "latency_50": 409.333633, | |
| "latency_90": 420.4970922, | |
| "latency_95": 423.82144059999996, | |
| "latency_99": 426.282015, | |
| "latency_999": 426.663606 | |
| } | |
| }, | |
| { | |
| "batch_size": 8, | |
| "input_length": 64, | |
| "baseline": { | |
| "nb_forwards": 70, | |
| "throughput": 4.67, | |
| "latency_mean": 214.5513372142857, | |
| "latency_std": 0.44768920996368816, | |
| "latency_50": 214.507338, | |
| "latency_90": 215.1284881, | |
| "latency_95": 215.26822455, | |
| "latency_99": 215.65146749000002, | |
| "latency_999": 215.97797624900002 | |
| }, | |
| "optimized": { | |
| "nb_forwards": 75, | |
| "throughput": 5.0, | |
| "latency_mean": 200.16238184, | |
| "latency_std": 9.040248673052057, | |
| "latency_50": 196.89438, | |
| "latency_90": 214.99417680000002, | |
| "latency_95": 218.1261396, | |
| "latency_99": 223.02508824, | |
| "latency_999": 223.85004512400002 | |
| } | |
| }, | |
| { | |
| "batch_size": 4, | |
| "input_length": 32, | |
| "baseline": { | |
| "nb_forwards": 212, | |
| "throughput": 14.13, | |
| "latency_mean": 71.02946175, | |
| "latency_std": 0.3401815134935371, | |
| "latency_50": 71.0171625, | |
| "latency_90": 71.3381363, | |
| "latency_95": 71.41140920000001, | |
| "latency_99": 71.62187967, | |
| "latency_999": 74.00790798399997 | |
| }, | |
| "optimized": { | |
| "nb_forwards": 265, | |
| "throughput": 17.67, | |
| "latency_mean": 56.817175411320754, | |
| "latency_std": 3.6329155653183585, | |
| "latency_50": 55.596164, | |
| "latency_90": 60.593704, | |
| "latency_95": 66.950391, | |
| "latency_99": 67.13051659999999, | |
| "latency_999": 67.23656778399999 | |
| } | |
| }, | |
| { | |
| "batch_size": 1, | |
| "input_length": 32, | |
| "baseline": { | |
| "nb_forwards": 452, | |
| "throughput": 30.13, | |
| "latency_mean": 33.22745787610619, | |
| "latency_std": 1.0543225342336746, | |
| "latency_50": 33.078952, | |
| "latency_90": 35.073573100000004, | |
| "latency_95": 35.4359905, | |
| "latency_99": 35.666321880000005, | |
| "latency_999": 36.114601445999995 | |
| }, | |
| "optimized": { | |
| "nb_forwards": 746, | |
| "throughput": 49.73, | |
| "latency_mean": 20.109513683646114, | |
| "latency_std": 1.6406645419149348, | |
| "latency_50": 20.2270735, | |
| "latency_90": 21.831147, | |
| "latency_95": 21.8876735, | |
| "latency_99": 22.079700099999997, | |
| "latency_999": 22.4666168 | |
| } | |
| }, | |
| { | |
| "batch_size": 1, | |
| "input_length": 64, | |
| "baseline": { | |
| "nb_forwards": 340, | |
| "throughput": 22.67, | |
| "latency_mean": 44.21631953823529, | |
| "latency_std": 5.107156057378748, | |
| "latency_50": 42.547523, | |
| "latency_90": 45.71804530000001, | |
| "latency_95": 59.850769, | |
| "latency_99": 61.57001349, | |
| "latency_999": 62.189666854 | |
| }, | |
| "optimized": { | |
| "nb_forwards": 389, | |
| "throughput": 25.93, | |
| "latency_mean": 38.56483639331619, | |
| "latency_std": 0.11123736681980873, | |
| "latency_50": 38.556444, | |
| "latency_90": 38.712917, | |
| "latency_95": 38.7594766, | |
| "latency_99": 38.92545156, | |
| "latency_999": 39.024601432000004 | |
| } | |
| }, | |
| { | |
| "batch_size": 1, | |
| "input_length": 128, | |
| "baseline": { | |
| "nb_forwards": 207, | |
| "throughput": 13.8, | |
| "latency_mean": 72.4693660772947, | |
| "latency_std": 0.8661223930705324, | |
| "latency_50": 72.429662, | |
| "latency_90": 73.5786186, | |
| "latency_95": 74.0005967, | |
| "latency_99": 74.67420342, | |
| "latency_999": 74.911409554 | |
| }, | |
| "optimized": { | |
| "nb_forwards": 211, | |
| "throughput": 14.07, | |
| "latency_mean": 71.1041596492891, | |
| "latency_std": 0.28805137660931524, | |
| "latency_50": 71.03339, | |
| "latency_90": 71.471796, | |
| "latency_95": 71.633065, | |
| "latency_99": 72.0733595, | |
| "latency_999": 72.60335828 | |
| } | |
| } | |
| ], | |
| "others": { | |
| "baseline": { | |
| "precision": 0.9358012339503085, | |
| "recall": 0.9444631437226523, | |
| "f1": 0.9401122372057961, | |
| "accuracy": 0.9882013940267124 | |
| }, | |
| "optimized": { | |
| "precision": 0.05001838911364472, | |
| "recall": 0.16021541568495457, | |
| "f1": 0.07623623623623622, | |
| "accuracy": 0.31141700089560376 | |
| } | |
| } | |
| }, | |
| "max_eval_samples": null, | |
| "time_benchmark_args": { | |
| "duration": 15, | |
| "warmup_runs": 5 | |
| }, | |
| "model_type": "distilbert" | |
| }, | |
| { | |
| "model_name_or_path": "elastic/distilbert-base-uncased-finetuned-conll03-english", | |
| "task": "token-classification", | |
| "dataset": { | |
| "path": "conll2003", | |
| "eval_split": "validation", | |
| "data_keys": { | |
| "primary": "tokens", | |
| "secondary": null | |
| }, | |
| "ref_keys": [ | |
| "ner_tags" | |
| ], | |
| "name": null, | |
| "calibration_split": "train" | |
| }, | |
| "quantization_approach": "dynamic", | |
| "operators_to_quantize": [ | |
| "Add", | |
| "MatMul" | |
| ], | |
| "node_exclusion": [], | |
| "aware_training": false, | |
| "per_channel": false, | |
| "calibration": { | |
| "method": "minmax", | |
| "num_calibration_samples": 100, | |
| "calibration_histogram_percentile": null, | |
| "calibration_moving_average": null, | |
| "calibration_moving_average_constant": null | |
| }, | |
| "framework": "onnxruntime", | |
| "framework_args": { | |
| "opset": 11, | |
| "optimization_level": 1 | |
| }, | |
| "hardware": "Architecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nByte Order: Little Endian\nAddress sizes: 46 bits physical, 48 bits virtual\nCPU(s): 8\nOn-line CPU(s) list: 0-7\nThread(s) per core: 2\nCore(s) per socket: 4\nSocket(s): 1\nNUMA node(s): 1\nVendor ID: GenuineIntel\nCPU family: 6\nModel: 85\nModel name: Intel(R) Xeon(R) Platinum 8259CL CPU @ 2.50GHz\nStepping: 7\nCPU MHz: 3106.018\nBogoMIPS: 4999.99\nHypervisor vendor: KVM\nVirtualization type: full\nL1d cache: 128 KiB\nL1i cache: 128 KiB\nL2 cache: 4 MiB\nL3 cache: 35.8 MiB\nNUMA node0 CPU(s): 0-7\nVulnerability Itlb multihit: KVM: Vulnerable\nVulnerability L1tf: Mitigation; PTE Inversion\nVulnerability Mds: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Meltdown: Mitigation; PTI\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines, STIBP disabled, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single pti fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke\n", | |
| "versions": { | |
| "transformers": "4.20.1", | |
| "optimum": "1.2.3.dev0", | |
| "optimum_hash": "5ac9c0d9fd7e7cca55b2f9935b961ed5b6c50112" | |
| }, | |
| "evaluation": { | |
| "time": [ | |
| { | |
| "batch_size": 4, | |
| "input_length": 64, | |
| "baseline": { | |
| "nb_forwards": 126, | |
| "throughput": 8.4, | |
| "latency_mean": 119.91479441269841, | |
| "latency_std": 0.2629922737356049, | |
| "latency_50": 119.9039375, | |
| "latency_90": 120.2240655, | |
| "latency_95": 120.3983675, | |
| "latency_99": 120.61099275, | |
| "latency_999": 120.725017125 | |
| }, | |
| "optimized": { | |
| "nb_forwards": 244, | |
| "throughput": 16.27, | |
| "latency_mean": 61.50554389344262, | |
| "latency_std": 0.9316797843470712, | |
| "latency_50": 61.3989675, | |
| "latency_90": 62.743792, | |
| "latency_95": 63.1351423, | |
| "latency_99": 64.33112306, | |
| "latency_999": 64.851473252 | |
| } | |
| }, | |
| { | |
| "batch_size": 8, | |
| "input_length": 128, | |
| "baseline": { | |
| "nb_forwards": 38, | |
| "throughput": 2.53, | |
| "latency_mean": 403.6921285, | |
| "latency_std": 0.8331604984169664, | |
| "latency_50": 403.615553, | |
| "latency_90": 404.5933473, | |
| "latency_95": 404.73024205, | |
| "latency_99": 405.73243518, | |
| "latency_999": 406.206855618 | |
| }, | |
| "optimized": { | |
| "nb_forwards": 49, | |
| "throughput": 3.27, | |
| "latency_mean": 307.35862414285714, | |
| "latency_std": 37.75323130347674, | |
| "latency_50": 328.22334, | |
| "latency_90": 329.4804032, | |
| "latency_95": 329.88989580000003, | |
| "latency_99": 331.99058364, | |
| "latency_999": 332.89210586400003 | |
| } | |
| }, | |
| { | |
| "batch_size": 8, | |
| "input_length": 64, | |
| "baseline": { | |
| "nb_forwards": 87, | |
| "throughput": 5.8, | |
| "latency_mean": 172.70831874712644, | |
| "latency_std": 16.93330536049502, | |
| "latency_50": 165.631011, | |
| "latency_90": 214.729821, | |
| "latency_95": 215.7495264, | |
| "latency_99": 219.2165974, | |
| "latency_999": 219.69907804 | |
| }, | |
| "optimized": { | |
| "nb_forwards": 128, | |
| "throughput": 8.53, | |
| "latency_mean": 117.8890106953125, | |
| "latency_std": 2.0181256632154128, | |
| "latency_50": 117.854545, | |
| "latency_90": 120.3485038, | |
| "latency_95": 121.1920466, | |
| "latency_99": 122.86635173, | |
| "latency_999": 123.87748661900001 | |
| } | |
| }, | |
| { | |
| "batch_size": 4, | |
| "input_length": 128, | |
| "baseline": { | |
| "nb_forwards": 69, | |
| "throughput": 4.6, | |
| "latency_mean": 219.78011085507248, | |
| "latency_std": 0.8383066531914855, | |
| "latency_50": 219.682452, | |
| "latency_90": 220.6735604, | |
| "latency_95": 221.179578, | |
| "latency_99": 222.15356576, | |
| "latency_999": 223.052069276 | |
| }, | |
| "optimized": { | |
| "nb_forwards": 122, | |
| "throughput": 8.13, | |
| "latency_mean": 123.70655378688525, | |
| "latency_std": 2.0832274392948285, | |
| "latency_50": 123.7493135, | |
| "latency_90": 126.5891177, | |
| "latency_95": 127.5693857, | |
| "latency_99": 128.41928172, | |
| "latency_999": 129.29519896800002 | |
| } | |
| }, | |
| { | |
| "batch_size": 4, | |
| "input_length": 32, | |
| "baseline": { | |
| "nb_forwards": 216, | |
| "throughput": 14.4, | |
| "latency_mean": 69.7580692962963, | |
| "latency_std": 0.7696003955372718, | |
| "latency_50": 69.544575, | |
| "latency_90": 70.7143, | |
| "latency_95": 71.6627545, | |
| "latency_99": 72.7330374, | |
| "latency_999": 73.26205913 | |
| }, | |
| "optimized": { | |
| "nb_forwards": 390, | |
| "throughput": 26.0, | |
| "latency_mean": 38.49885642307692, | |
| "latency_std": 3.3170930332820334, | |
| "latency_50": 40.230039, | |
| "latency_90": 40.3790612, | |
| "latency_95": 40.428531899999996, | |
| "latency_99": 40.510404130000005, | |
| "latency_999": 40.614662347 | |
| } | |
| }, | |
| { | |
| "batch_size": 1, | |
| "input_length": 128, | |
| "baseline": { | |
| "nb_forwards": 208, | |
| "throughput": 13.87, | |
| "latency_mean": 72.37936482211539, | |
| "latency_std": 0.24212622373713247, | |
| "latency_50": 72.3947265, | |
| "latency_90": 72.6858948, | |
| "latency_95": 72.75088670000001, | |
| "latency_99": 72.92894586, | |
| "latency_999": 73.01229876299999 | |
| }, | |
| "optimized": { | |
| "nb_forwards": 412, | |
| "throughput": 27.47, | |
| "latency_mean": 36.4657384684466, | |
| "latency_std": 3.4465598552729446, | |
| "latency_50": 34.966474, | |
| "latency_90": 43.4717706, | |
| "latency_95": 43.6103869, | |
| "latency_99": 43.74503971, | |
| "latency_999": 44.051604852 | |
| } | |
| }, | |
| { | |
| "batch_size": 8, | |
| "input_length": 32, | |
| "baseline": { | |
| "nb_forwards": 127, | |
| "throughput": 8.47, | |
| "latency_mean": 118.31688400787401, | |
| "latency_std": 1.1539171140975781, | |
| "latency_50": 118.215119, | |
| "latency_90": 119.8292888, | |
| "latency_95": 120.60338259999999, | |
| "latency_99": 121.44057798, | |
| "latency_999": 122.68986890599999 | |
| }, | |
| "optimized": { | |
| "nb_forwards": 254, | |
| "throughput": 16.93, | |
| "latency_mean": 59.2173072007874, | |
| "latency_std": 0.7459868576638755, | |
| "latency_50": 59.1816355, | |
| "latency_90": 60.196821299999996, | |
| "latency_95": 60.5587619, | |
| "latency_99": 60.966352210000004, | |
| "latency_999": 61.412101875 | |
| } | |
| }, | |
| { | |
| "batch_size": 1, | |
| "input_length": 64, | |
| "baseline": { | |
| "nb_forwards": 255, | |
| "throughput": 17.0, | |
| "latency_mean": 58.91992950980392, | |
| "latency_std": 0.8869192739714911, | |
| "latency_50": 58.910515, | |
| "latency_90": 59.9905808, | |
| "latency_95": 60.4006188, | |
| "latency_99": 61.36197958, | |
| "latency_999": 62.78803392800001 | |
| }, | |
| "optimized": { | |
| "nb_forwards": 763, | |
| "throughput": 50.87, | |
| "latency_mean": 19.684674589777195, | |
| "latency_std": 1.8042745579532666, | |
| "latency_50": 19.157213, | |
| "latency_90": 22.2412244, | |
| "latency_95": 22.3211958, | |
| "latency_99": 22.58112282, | |
| "latency_999": 26.754026579999863 | |
| } | |
| }, | |
| { | |
| "batch_size": 1, | |
| "input_length": 32, | |
| "baseline": { | |
| "nb_forwards": 324, | |
| "throughput": 21.6, | |
| "latency_mean": 46.3835297962963, | |
| "latency_std": 1.1343614396581607, | |
| "latency_50": 46.442469, | |
| "latency_90": 47.1782314, | |
| "latency_95": 47.4329425, | |
| "latency_99": 48.049914599999994, | |
| "latency_999": 50.92608084600004 | |
| }, | |
| "optimized": { | |
| "nb_forwards": 1507, | |
| "throughput": 100.47, | |
| "latency_mean": 9.958045324485735, | |
| "latency_std": 0.2901405767073424, | |
| "latency_50": 9.92078, | |
| "latency_90": 10.4131178, | |
| "latency_95": 10.5542065, | |
| "latency_99": 10.709247679999999, | |
| "latency_999": 10.978874069999991 | |
| } | |
| } | |
| ], | |
| "others": { | |
| "baseline": { | |
| "precision": 0.9358012339503085, | |
| "recall": 0.9444631437226523, | |
| "f1": 0.9401122372057961, | |
| "accuracy": 0.9882013940267124 | |
| }, | |
| "optimized": { | |
| "precision": 0.934913217623498, | |
| "recall": 0.9427802086839449, | |
| "f1": 0.9388302329478799, | |
| "accuracy": 0.9877341225030178 | |
| } | |
| } | |
| }, | |
| "max_eval_samples": null, | |
| "time_benchmark_args": { | |
| "duration": 15, | |
| "warmup_runs": 5 | |
| }, | |
| "model_type": "distilbert" | |
| }, | |
| { | |
| "model_name_or_path": "elastic/distilbert-base-uncased-finetuned-conll03-english", | |
| "task": "token-classification", | |
| "dataset": { | |
| "path": "conll2003", | |
| "eval_split": "validation", | |
| "data_keys": { | |
| "primary": "tokens", | |
| "secondary": null | |
| }, | |
| "ref_keys": [ | |
| "ner_tags" | |
| ], | |
| "name": null, | |
| "calibration_split": "train" | |
| }, | |
| "quantization_approach": "static", | |
| "operators_to_quantize": [ | |
| "Add", | |
| "MatMul" | |
| ], | |
| "node_exclusion": [], | |
| "aware_training": false, | |
| "per_channel": false, | |
| "calibration": { | |
| "method": "minmax", | |
| "num_calibration_samples": 100, | |
| "calibration_histogram_percentile": null, | |
| "calibration_moving_average": null, | |
| "calibration_moving_average_constant": null | |
| }, | |
| "framework": "onnxruntime", | |
| "framework_args": { | |
| "opset": 11, | |
| "optimization_level": 1 | |
| }, | |
| "hardware": "Architecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nByte Order: Little Endian\nAddress sizes: 46 bits physical, 48 bits virtual\nCPU(s): 8\nOn-line CPU(s) list: 0-7\nThread(s) per core: 2\nCore(s) per socket: 4\nSocket(s): 1\nNUMA node(s): 1\nVendor ID: GenuineIntel\nCPU family: 6\nModel: 85\nModel name: Intel(R) Xeon(R) Platinum 8259CL CPU @ 2.50GHz\nStepping: 7\nCPU MHz: 3101.721\nBogoMIPS: 4999.99\nHypervisor vendor: KVM\nVirtualization type: full\nL1d cache: 128 KiB\nL1i cache: 128 KiB\nL2 cache: 4 MiB\nL3 cache: 35.8 MiB\nNUMA node0 CPU(s): 0-7\nVulnerability Itlb multihit: KVM: Vulnerable\nVulnerability L1tf: Mitigation; PTE Inversion\nVulnerability Mds: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Meltdown: Mitigation; PTI\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines, STIBP disabled, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single pti fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke\n", | |
| "versions": { | |
| "transformers": "4.20.1", | |
| "optimum": "1.2.3.dev0", | |
| "optimum_hash": "5ac9c0d9fd7e7cca55b2f9935b961ed5b6c50112" | |
| }, | |
| "evaluation": { | |
| "time": [ | |
| { | |
| "batch_size": 1, | |
| "input_length": 32, | |
| "baseline": { | |
| "nb_forwards": 444, | |
| "throughput": 29.6, | |
| "latency_mean": 33.835281761261264, | |
| "latency_std": 1.1104691070719008, | |
| "latency_50": 33.7167165, | |
| "latency_90": 35.5099667, | |
| "latency_95": 35.79827469999999, | |
| "latency_99": 36.20388337, | |
| "latency_999": 36.403486599 | |
| }, | |
| "optimized": { | |
| "nb_forwards": 1038, | |
| "throughput": 69.2, | |
| "latency_mean": 14.456874846820808, | |
| "latency_std": 1.147945670896925, | |
| "latency_50": 15.261894, | |
| "latency_90": 15.4090676, | |
| "latency_95": 15.44100315, | |
| "latency_99": 15.5019354, | |
| "latency_999": 16.029519071999996 | |
| } | |
| }, | |
| { | |
| "batch_size": 1, | |
| "input_length": 64, | |
| "baseline": { | |
| "nb_forwards": 292, | |
| "throughput": 19.47, | |
| "latency_mean": 51.40666307191781, | |
| "latency_std": 7.904660749243075, | |
| "latency_50": 57.9910805, | |
| "latency_90": 59.4403903, | |
| "latency_95": 59.64796875, | |
| "latency_99": 60.01921038999999, | |
| "latency_999": 62.362501826000006 | |
| }, | |
| "optimized": { | |
| "nb_forwards": 511, | |
| "throughput": 34.07, | |
| "latency_mean": 29.36470456360078, | |
| "latency_std": 0.09446229792346662, | |
| "latency_50": 29.353048, | |
| "latency_90": 29.482019, | |
| "latency_95": 29.53371, | |
| "latency_99": 29.669687599999996, | |
| "latency_999": 29.82210045 | |
| } | |
| }, | |
| { | |
| "batch_size": 4, | |
| "input_length": 128, | |
| "baseline": { | |
| "nb_forwards": 81, | |
| "throughput": 5.4, | |
| "latency_mean": 186.3935255308642, | |
| "latency_std": 23.66920102974093, | |
| "latency_50": 170.663335, | |
| "latency_90": 219.886297, | |
| "latency_95": 220.412333, | |
| "latency_99": 221.20538580000002, | |
| "latency_999": 222.11007948 | |
| }, | |
| "optimized": { | |
| "nb_forwards": 85, | |
| "throughput": 5.67, | |
| "latency_mean": 176.9935626235294, | |
| "latency_std": 24.611926259834373, | |
| "latency_50": 162.989882, | |
| "latency_90": 205.825854, | |
| "latency_95": 206.062911, | |
| "latency_99": 206.69577235999998, | |
| "latency_999": 207.115859636 | |
| } | |
| }, | |
| { | |
| "batch_size": 8, | |
| "input_length": 64, | |
| "baseline": { | |
| "nb_forwards": 70, | |
| "throughput": 4.67, | |
| "latency_mean": 215.00074108571428, | |
| "latency_std": 0.9705650988667485, | |
| "latency_50": 214.867633, | |
| "latency_90": 215.72163830000002, | |
| "latency_95": 216.53323115, | |
| "latency_99": 218.93241332, | |
| "latency_999": 220.051872632 | |
| }, | |
| "optimized": { | |
| "nb_forwards": 101, | |
| "throughput": 6.73, | |
| "latency_mean": 148.9348962970297, | |
| "latency_std": 3.9263502299131137, | |
| "latency_50": 148.342845, | |
| "latency_90": 155.106506, | |
| "latency_95": 156.211825, | |
| "latency_99": 158.331491, | |
| "latency_999": 161.1917459 | |
| } | |
| }, | |
| { | |
| "batch_size": 1, | |
| "input_length": 128, | |
| "baseline": { | |
| "nb_forwards": 212, | |
| "throughput": 14.13, | |
| "latency_mean": 70.75742881132075, | |
| "latency_std": 1.2469446939770699, | |
| "latency_50": 70.660216, | |
| "latency_90": 72.6529741, | |
| "latency_95": 72.88815559999999, | |
| "latency_99": 73.11715371, | |
| "latency_999": 73.644905982 | |
| }, | |
| "optimized": { | |
| "nb_forwards": 312, | |
| "throughput": 20.8, | |
| "latency_mean": 48.23616327884615, | |
| "latency_std": 4.851493694643943, | |
| "latency_50": 45.751592, | |
| "latency_90": 55.8910255, | |
| "latency_95": 56.0878071, | |
| "latency_99": 56.63496909, | |
| "latency_999": 56.748669404 | |
| } | |
| }, | |
| { | |
| "batch_size": 8, | |
| "input_length": 128, | |
| "baseline": { | |
| "nb_forwards": 43, | |
| "throughput": 2.87, | |
| "latency_mean": 352.17799074418605, | |
| "latency_std": 48.791933253583736, | |
| "latency_50": 312.088024, | |
| "latency_90": 406.3363032, | |
| "latency_95": 407.0272163, | |
| "latency_99": 409.34203474000003, | |
| "latency_999": 410.012702374 | |
| }, | |
| "optimized": { | |
| "nb_forwards": 47, | |
| "throughput": 3.13, | |
| "latency_mean": 320.8497927446808, | |
| "latency_std": 37.045737718320964, | |
| "latency_50": 304.99127, | |
| "latency_90": 407.9567778, | |
| "latency_95": 408.6023321, | |
| "latency_99": 409.3466477, | |
| "latency_999": 409.53851807 | |
| } | |
| }, | |
| { | |
| "batch_size": 4, | |
| "input_length": 32, | |
| "baseline": { | |
| "nb_forwards": 273, | |
| "throughput": 18.2, | |
| "latency_mean": 55.05481489377289, | |
| "latency_std": 0.8315777141405638, | |
| "latency_50": 54.970724, | |
| "latency_90": 56.1484644, | |
| "latency_95": 56.430772600000005, | |
| "latency_99": 56.85986168, | |
| "latency_999": 57.515896824 | |
| }, | |
| "optimized": { | |
| "nb_forwards": 321, | |
| "throughput": 21.4, | |
| "latency_mean": 46.79807228660436, | |
| "latency_std": 4.841315127921891, | |
| "latency_50": 46.18684, | |
| "latency_90": 51.680476, | |
| "latency_95": 51.742804, | |
| "latency_99": 51.917291399999996, | |
| "latency_999": 52.1734384 | |
| } | |
| }, | |
| { | |
| "batch_size": 8, | |
| "input_length": 32, | |
| "baseline": { | |
| "nb_forwards": 129, | |
| "throughput": 8.6, | |
| "latency_mean": 116.59177039534883, | |
| "latency_std": 0.5224827890552791, | |
| "latency_50": 116.509951, | |
| "latency_90": 116.8391962, | |
| "latency_95": 117.141577, | |
| "latency_99": 119.15936884, | |
| "latency_999": 120.545900664 | |
| }, | |
| "optimized": { | |
| "nb_forwards": 180, | |
| "throughput": 12.0, | |
| "latency_mean": 83.54651290555556, | |
| "latency_std": 9.983735757677245, | |
| "latency_50": 78.13343, | |
| "latency_90": 97.8848908, | |
| "latency_95": 98.02936915000001, | |
| "latency_99": 98.11533426000001, | |
| "latency_999": 98.19551898499999 | |
| } | |
| }, | |
| { | |
| "batch_size": 4, | |
| "input_length": 64, | |
| "baseline": { | |
| "nb_forwards": 126, | |
| "throughput": 8.4, | |
| "latency_mean": 119.892617, | |
| "latency_std": 0.2462536509536135, | |
| "latency_50": 119.8820935, | |
| "latency_90": 120.218857, | |
| "latency_95": 120.33222325, | |
| "latency_99": 120.4118515, | |
| "latency_999": 120.443628 | |
| }, | |
| "optimized": { | |
| "nb_forwards": 173, | |
| "throughput": 11.53, | |
| "latency_mean": 86.75648606936416, | |
| "latency_std": 10.203153981382503, | |
| "latency_50": 81.471955, | |
| "latency_90": 100.5808368, | |
| "latency_95": 100.84913420000001, | |
| "latency_99": 101.67875240000001, | |
| "latency_999": 102.108927452 | |
| } | |
| } | |
| ], | |
| "others": { | |
| "baseline": { | |
| "precision": 0.9358012339503085, | |
| "recall": 0.9444631437226523, | |
| "f1": 0.9401122372057961, | |
| "accuracy": 0.9882013940267124 | |
| }, | |
| "optimized": { | |
| "precision": 0.06309066586113149, | |
| "recall": 0.24604510265903737, | |
| "f1": 0.10042933195947105, | |
| "accuracy": 0.34301623768544837 | |
| } | |
| } | |
| }, | |
| "max_eval_samples": null, | |
| "time_benchmark_args": { | |
| "duration": 15, | |
| "warmup_runs": 5 | |
| }, | |
| "model_type": "distilbert" | |
| }, | |
| { | |
| "model_name_or_path": "elastic/distilbert-base-uncased-finetuned-conll03-english", | |
| "task": "token-classification", | |
| "dataset": { | |
| "path": "conll2003", | |
| "eval_split": "validation", | |
| "data_keys": { | |
| "primary": "tokens", | |
| "secondary": null | |
| }, | |
| "ref_keys": [ | |
| "ner_tags" | |
| ], | |
| "name": null, | |
| "calibration_split": "train" | |
| }, | |
| "quantization_approach": "dynamic", | |
| "operators_to_quantize": [ | |
| "Add" | |
| ], | |
| "node_exclusion": [], | |
| "aware_training": false, | |
| "per_channel": false, | |
| "calibration": { | |
| "method": "minmax", | |
| "num_calibration_samples": 100, | |
| "calibration_histogram_percentile": null, | |
| "calibration_moving_average": null, | |
| "calibration_moving_average_constant": null | |
| }, | |
| "framework": "onnxruntime", | |
| "framework_args": { | |
| "opset": 11, | |
| "optimization_level": 1 | |
| }, | |
| "hardware": "Architecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nByte Order: Little Endian\nAddress sizes: 46 bits physical, 48 bits virtual\nCPU(s): 8\nOn-line CPU(s) list: 0-7\nThread(s) per core: 2\nCore(s) per socket: 4\nSocket(s): 1\nNUMA node(s): 1\nVendor ID: GenuineIntel\nCPU family: 6\nModel: 85\nModel name: Intel(R) Xeon(R) Platinum 8259CL CPU @ 2.50GHz\nStepping: 7\nCPU MHz: 3121.536\nBogoMIPS: 4999.99\nHypervisor vendor: KVM\nVirtualization type: full\nL1d cache: 128 KiB\nL1i cache: 128 KiB\nL2 cache: 4 MiB\nL3 cache: 35.8 MiB\nNUMA node0 CPU(s): 0-7\nVulnerability Itlb multihit: KVM: Vulnerable\nVulnerability L1tf: Mitigation; PTE Inversion\nVulnerability Mds: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown\nVulnerability Meltdown: Mitigation; PTI\nVulnerability Spec store bypass: Vulnerable\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines, STIBP disabled, RSB filling\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single pti fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke\n", | |
| "versions": { | |
| "transformers": "4.20.1", | |
| "optimum": "1.2.3.dev0", | |
| "optimum_hash": "5ac9c0d9fd7e7cca55b2f9935b961ed5b6c50112" | |
| }, | |
| "evaluation": { | |
| "time": [ | |
| { | |
| "batch_size": 1, | |
| "input_length": 64, | |
| "baseline": { | |
| "nb_forwards": 257, | |
| "throughput": 17.13, | |
| "latency_mean": 58.59407656031129, | |
| "latency_std": 0.5082693082988551, | |
| "latency_50": 58.62559, | |
| "latency_90": 59.226632, | |
| "latency_95": 59.3844464, | |
| "latency_99": 59.66381776, | |
| "latency_999": 60.193005848 | |
| }, | |
| "optimized": { | |
| "nb_forwards": 605, | |
| "throughput": 40.33, | |
| "latency_mean": 24.812764945454546, | |
| "latency_std": 1.8257073532356165, | |
| "latency_50": 24.192906, | |
| "latency_90": 29.2776762, | |
| "latency_95": 29.361327, | |
| "latency_99": 29.531946960000003, | |
| "latency_999": 29.634380991999997 | |
| } | |
| }, | |
| { | |
| "batch_size": 8, | |
| "input_length": 32, | |
| "baseline": { | |
| "nb_forwards": 129, | |
| "throughput": 8.6, | |
| "latency_mean": 116.51833388372093, | |
| "latency_std": 0.19355545540306737, | |
| "latency_50": 116.510617, | |
| "latency_90": 116.76812059999999, | |
| "latency_95": 116.8346944, | |
| "latency_99": 116.91619076, | |
| "latency_999": 117.070623216 | |
| }, | |
| "optimized": { | |
| "nb_forwards": 188, | |
| "throughput": 12.53, | |
| "latency_mean": 80.1677380851064, | |
| "latency_std": 1.8148568791642794, | |
| "latency_50": 79.986953, | |
| "latency_90": 82.70380990000001, | |
| "latency_95": 83.21852025, | |
| "latency_99": 84.57913724, | |
| "latency_999": 84.697106648 | |
| } | |
| }, | |
| { | |
| "batch_size": 1, | |
| "input_length": 32, | |
| "baseline": { | |
| "nb_forwards": 410, | |
| "throughput": 27.33, | |
| "latency_mean": 36.59453311707317, | |
| "latency_std": 5.201952859743346, | |
| "latency_50": 34.5124805, | |
| "latency_90": 46.8082206, | |
| "latency_95": 47.39390375, | |
| "latency_99": 47.86114868, | |
| "latency_999": 48.050237836 | |
| }, | |
| "optimized": { | |
| "nb_forwards": 1074, | |
| "throughput": 71.6, | |
| "latency_mean": 13.975352912476723, | |
| "latency_std": 0.2608975931098767, | |
| "latency_50": 13.9295265, | |
| "latency_90": 14.3837001, | |
| "latency_95": 14.4975223, | |
| "latency_99": 14.66426481, | |
| "latency_999": 14.912761724999998 | |
| } | |
| }, | |
| { | |
| "batch_size": 8, | |
| "input_length": 64, | |
| "baseline": { | |
| "nb_forwards": 91, | |
| "throughput": 6.07, | |
| "latency_mean": 166.0459372197802, | |
| "latency_std": 5.910350195698658, | |
| "latency_50": 165.351766, | |
| "latency_90": 168.122746, | |
| "latency_95": 169.064614, | |
| "latency_99": 189.41977489999985, | |
| "latency_999": 211.91127088999988 | |
| }, | |
| "optimized": { | |
| "nb_forwards": 96, | |
| "throughput": 6.4, | |
| "latency_mean": 156.98839121875, | |
| "latency_std": 3.8307406608519075, | |
| "latency_50": 156.713151, | |
| "latency_90": 161.920169, | |
| "latency_95": 162.8535025, | |
| "latency_99": 167.0273974, | |
| "latency_999": 169.60480174 | |
| } | |
| }, | |
| { | |
| "batch_size": 8, | |
| "input_length": 128, | |
| "baseline": { | |
| "nb_forwards": 41, | |
| "throughput": 2.73, | |
| "latency_mean": 372.8457580243902, | |
| "latency_std": 44.33863922960532, | |
| "latency_50": 403.71874, | |
| "latency_90": 407.483091, | |
| "latency_95": 409.321755, | |
| "latency_99": 414.9955262, | |
| "latency_999": 415.29942092 | |
| }, | |
| "optimized": { | |
| "nb_forwards": 48, | |
| "throughput": 3.2, | |
| "latency_mean": 317.5281361666667, | |
| "latency_std": 5.407804800179884, | |
| "latency_50": 317.912668, | |
| "latency_90": 323.9196422, | |
| "latency_95": 325.7003953, | |
| "latency_99": 328.84562712, | |
| "latency_999": 330.22228231200006 | |
| } | |
| }, | |
| { | |
| "batch_size": 4, | |
| "input_length": 128, | |
| "baseline": { | |
| "nb_forwards": 69, | |
| "throughput": 4.6, | |
| "latency_mean": 220.1285506231884, | |
| "latency_std": 0.9353033698467256, | |
| "latency_50": 220.192768, | |
| "latency_90": 221.36099019999997, | |
| "latency_95": 221.5007656, | |
| "latency_99": 222.25962384000002, | |
| "latency_999": 222.473907684 | |
| }, | |
| "optimized": { | |
| "nb_forwards": 81, | |
| "throughput": 5.4, | |
| "latency_mean": 187.20598314814814, | |
| "latency_std": 25.62748524575956, | |
| "latency_50": 168.810645, | |
| "latency_90": 215.939029, | |
| "latency_95": 216.125018, | |
| "latency_99": 217.0070662, | |
| "latency_999": 217.17069052000002 | |
| } | |
| }, | |
| { | |
| "batch_size": 4, | |
| "input_length": 32, | |
| "baseline": { | |
| "nb_forwards": 268, | |
| "throughput": 17.87, | |
| "latency_mean": 56.01774754104478, | |
| "latency_std": 1.5540596958913682, | |
| "latency_50": 55.6834485, | |
| "latency_90": 58.264829299999995, | |
| "latency_95": 59.282709049999994, | |
| "latency_99": 60.64031475, | |
| "latency_999": 62.074440195 | |
| }, | |
| "optimized": { | |
| "nb_forwards": 293, | |
| "throughput": 19.53, | |
| "latency_mean": 51.320777443686005, | |
| "latency_std": 3.776379337439246, | |
| "latency_50": 53.174213, | |
| "latency_90": 53.340690200000004, | |
| "latency_95": 53.408575, | |
| "latency_99": 53.4970542, | |
| "latency_999": 55.868342540000086 | |
| } | |
| }, | |
| { | |
| "batch_size": 1, | |
| "input_length": 128, | |
| "baseline": { | |
| "nb_forwards": 214, | |
| "throughput": 14.27, | |
| "latency_mean": 70.20732589719627, | |
| "latency_std": 0.8155710627132576, | |
| "latency_50": 70.365742, | |
| "latency_90": 71.1378453, | |
| "latency_95": 71.3491235, | |
| "latency_99": 71.89662288, | |
| "latency_999": 72.393311452 | |
| }, | |
| "optimized": { | |
| "nb_forwards": 324, | |
| "throughput": 21.6, | |
| "latency_mean": 46.297288277777774, | |
| "latency_std": 0.7972386433029148, | |
| "latency_50": 46.110238, | |
| "latency_90": 47.36061, | |
| "latency_95": 47.99375455, | |
| "latency_99": 48.857902409999994, | |
| "latency_999": 50.063431357000006 | |
| } | |
| }, | |
| { | |
| "batch_size": 4, | |
| "input_length": 64, | |
| "baseline": { | |
| "nb_forwards": 139, | |
| "throughput": 9.27, | |
| "latency_mean": 108.42828852517987, | |
| "latency_std": 13.16458394936121, | |
| "latency_50": 117.656529, | |
| "latency_90": 118.084375, | |
| "latency_95": 118.1652478, | |
| "latency_99": 118.36095123999999, | |
| "latency_999": 118.602071188 | |
| }, | |
| "optimized": { | |
| "nb_forwards": 142, | |
| "throughput": 9.47, | |
| "latency_mean": 105.65247052816902, | |
| "latency_std": 0.3934774499008662, | |
| "latency_50": 105.60453, | |
| "latency_90": 105.91063890000001, | |
| "latency_95": 106.2638084, | |
| "latency_99": 107.39340928, | |
| "latency_999": 107.831013231 | |
| } | |
| } | |
| ], | |
| "others": { | |
| "baseline": { | |
| "precision": 0.9358012339503085, | |
| "recall": 0.9444631437226523, | |
| "f1": 0.9401122372057961, | |
| "accuracy": 0.9882013940267124 | |
| }, | |
| "optimized": { | |
| "precision": 0.9358012339503085, | |
| "recall": 0.9444631437226523, | |
| "f1": 0.9401122372057961, | |
| "accuracy": 0.9882013940267124 | |
| } | |
| } | |
| }, | |
| "max_eval_samples": null, | |
| "time_benchmark_args": { | |
| "duration": 15, | |
| "warmup_runs": 5 | |
| }, | |
| "model_type": "distilbert" | |
| } | |
| ] |