multiple_choice_score: there are 1132 tasks in prompt multiple_choice_score: reading tasks......................................................................................................done multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 1132 tasks. task acc_norm 1 0.00000000 2 50.00000000 3 33.33333333 4 25.00000000 5 20.00000000 6 16.66666667 7 14.28571429 8 12.50000000 9 11.11111111 10 10.00000000 11 9.09090909 12 8.33333333 13 7.69230769 14 7.14285714 15 6.66666667 16 12.50000000 17 11.76470588 18 11.11111111 19 15.78947368 20 15.00000000 21 14.28571429 22 13.63636364 23 13.04347826 24 12.50000000 25 12.00000000 26 11.53846154 27 11.11111111 28 10.71428571 29 10.34482759 30 10.00000000 31 9.67741935 32 9.37500000 33 9.09090909 34 8.82352941 35 8.57142857 36 8.33333333 37 8.10810811 38 7.89473684 39 7.69230769 40 7.50000000 41 7.31707317 42 7.14285714 43 6.97674419 44 6.81818182 45 8.88888889 46 8.69565217 47 8.51063830 48 8.33333333 49 8.16326531 50 8.00000000 51 9.80392157 52 9.61538462 53 9.43396226 54 9.25925926 55 10.90909091 56 10.71428571 57 12.28070175 58 12.06896552 59 11.86440678 60 11.66666667 61 11.47540984 62 11.29032258 63 11.11111111 64 10.93750000 65 10.76923077 66 10.60606061 67 10.44776119 68 10.29411765 69 10.14492754 70 11.42857143 71 11.26760563 72 11.11111111 73 10.95890411 74 10.81081081 75 10.66666667 76 10.52631579 77 10.38961039 78 11.53846154 79 11.39240506 80 11.25000000 81 11.11111111 82 10.97560976 83 10.84337349 84 10.71428571 85 10.58823529 86 10.46511628 87 10.34482759 88 10.22727273 89 10.11235955 90 11.11111111 91 10.98901099 92 10.86956522 93 10.75268817 94 10.63829787 95 10.52631579 96 10.41666667 97 10.30927835 98 10.20408163 99 10.10101010 100 10.00000000 101 9.90099010 102 9.80392157 103 10.67961165 104 11.53846154 105 11.42857143 106 11.32075472 107 12.14953271 108 12.96296296 109 12.84403670 110 13.63636364 111 13.51351351 112 13.39285714 113 13.27433628 114 13.15789474 115 13.04347826 116 12.93103448 117 12.82051282 118 12.71186441 119 12.60504202 120 12.50000000 121 12.39669421 122 12.29508197 123 12.19512195 124 12.09677419 125 12.00000000 126 11.90476190 127 11.81102362 128 11.71875000 129 11.62790698 130 11.53846154 131 11.45038168 132 11.36363636 133 11.27819549 134 11.94029851 135 12.59259259 136 12.50000000 137 12.40875912 138 12.31884058 139 12.23021583 140 12.14285714 141 12.05673759 142 11.97183099 143 11.88811189 144 12.50000000 145 12.41379310 146 12.32876712 147 12.24489796 148 12.16216216 149 12.08053691 150 12.00000000 151 11.92052980 152 11.84210526 153 11.76470588 154 12.33766234 155 12.25806452 156 12.17948718 157 12.10191083 158 12.02531646 159 11.94968553 160 12.50000000 161 12.42236025 162 12.34567901 163 12.88343558 164 12.80487805 165 12.72727273 166 12.65060241 167 12.57485030 168 12.50000000 169 12.42603550 170 12.35294118 171 12.28070175 172 12.79069767 173 13.29479769 174 13.21839080 175 13.14285714 176 13.06818182 177 12.99435028 178 12.92134831 179 12.84916201 180 12.77777778 181 12.70718232 182 12.63736264 183 12.56830601 184 12.50000000 185 12.43243243 186 12.36559140 187 12.29946524 188 12.76595745 189 12.69841270 190 12.63157895 191 12.56544503 192 13.02083333 193 12.95336788 194 12.88659794 195 12.82051282 196 12.75510204 197 12.69035533 198 12.62626263 199 12.56281407 200 13.00000000 201 12.93532338 202 12.87128713 203 13.30049261 204 13.23529412 205 13.17073171 206 13.10679612 207 13.04347826 208 13.46153846 209 13.39712919 210 13.33333333 211 13.27014218 212 13.20754717 213 13.14553991 214 13.08411215 215 13.02325581 216 13.42592593 217 13.36405530 218 13.30275229 219 13.69863014 220 13.63636364 221 13.57466063 222 13.51351351 223 13.45291480 224 13.39285714 225 13.33333333 226 13.71681416 227 13.65638767 228 13.59649123 229 13.53711790 230 13.47826087 231 13.41991342 232 13.36206897 233 13.30472103 234 13.24786325 235 13.19148936 236 13.13559322 237 13.08016878 238 13.02521008 239 12.97071130 240 12.91666667 241 12.86307054 242 12.80991736 243 12.75720165 244 12.70491803 245 12.65306122 246 12.60162602 247 12.55060729 248 12.50000000 249 12.44979920 250 12.40000000 251 12.35059761 252 12.30158730 253 12.25296443 254 12.20472441 255 12.15686275 256 12.10937500 257 12.06225681 258 12.01550388 259 11.96911197 260 11.92307692 261 11.87739464 262 11.83206107 263 12.16730038 264 12.12121212 265 12.07547170 266 12.03007519 267 11.98501873 268 11.94029851 269 11.89591078 270 11.85185185 271 11.80811808 272 12.13235294 273 12.08791209 274 12.04379562 275 12.00000000 276 11.95652174 277 11.91335740 278 11.87050360 279 11.82795699 280 11.78571429 281 12.09964413 282 12.05673759 283 12.01413428 284 11.97183099 285 11.92982456 286 11.88811189 287 11.84668990 288 11.80555556 289 11.76470588 290 11.72413793 291 11.68384880 292 11.64383562 293 11.60409556 294 11.56462585 295 11.52542373 296 11.48648649 297 11.44781145 298 11.40939597 299 11.37123746 300 11.33333333 301 11.29568106 302 11.25827815 303 11.22112211 304 11.18421053 305 11.14754098 306 11.11111111 307 11.07491857 308 11.03896104 309 11.32686084 310 11.29032258 311 11.25401929 312 11.53846154 313 11.50159744 314 11.78343949 315 11.74603175 316 11.70886076 317 11.67192429 318 11.63522013 319 11.91222571 320 11.87500000 321 11.83800623 322 11.80124224 323 11.76470588 324 12.03703704 325 12.30769231 326 12.57668712 327 12.53822630 328 12.50000000 329 12.46200608 330 12.42424242 331 12.68882175 332 12.65060241 333 12.91291291 334 12.87425150 335 12.83582090 336 12.79761905 337 12.75964392 338 12.72189349 339 12.68436578 340 12.64705882 341 12.60997067 342 12.57309942 343 12.53644315 344 12.50000000 345 12.46376812 346 12.71676301 347 12.68011527 348 12.64367816 349 12.60744986 350 12.57142857 351 12.53561254 352 12.50000000 353 12.74787535 354 12.71186441 355 12.67605634 356 12.64044944 357 12.60504202 358 12.56983240 359 12.53481894 360 12.50000000 361 12.74238227 362 12.70718232 363 12.94765840 364 12.91208791 365 12.87671233 366 12.84153005 367 12.80653951 368 12.77173913 369 12.73712737 370 12.70270270 371 12.66846361 372 12.63440860 373 12.60053619 374 12.56684492 375 12.53333333 376 12.50000000 377 12.46684350 378 12.43386243 379 12.40105541 380 12.36842105 381 12.33595801 382 12.30366492 383 12.27154047 384 12.23958333 385 12.20779221 386 12.17616580 387 12.14470284 388 12.11340206 389 12.08226221 390 12.05128205 391 12.02046036 392 11.98979592 393 11.95928753 394 11.92893401 395 11.89873418 396 11.86868687 397 11.83879093 398 11.80904523 399 11.77944862 400 11.75000000 401 11.72069825 402 11.69154229 403 11.91066998 404 11.88118812 405 11.85185185 406 11.82266010 407 11.79361179 408 11.76470588 409 11.73594132 410 11.70731707 411 11.92214112 412 11.89320388 413 11.86440678 414 11.83574879 415 12.04819277 416 12.01923077 417 11.99040767 418 11.96172249 419 11.93317422 420 11.90476190 421 11.87648456 422 12.08530806 423 12.05673759 424 12.02830189 425 12.00000000 426 11.97183099 427 11.94379391 428 11.91588785 429 11.88811189 430 11.86046512 431 11.83294664 432 11.80555556 433 11.77829099 434 11.75115207 435 11.72413793 436 11.69724771 437 11.67048055 438 11.64383562 439 11.61731207 440 11.59090909 441 11.56462585 442 11.53846154 443 11.51241535 444 11.48648649 445 11.46067416 446 11.43497758 447 11.40939597 448 11.38392857 449 11.35857461 450 11.33333333 451 11.30820399 452 11.28318584 453 11.25827815 454 11.23348018 455 11.20879121 456 11.40350877 457 11.37855580 458 11.57205240 459 11.54684096 460 11.52173913 461 11.71366594 462 11.68831169 463 11.66306695 464 11.63793103 465 11.61290323 466 11.58798283 467 11.56316916 468 11.53846154 469 11.51385928 470 11.48936170 471 11.46496815 472 11.44067797 473 11.41649049 474 11.39240506 475 11.36842105 476 11.34453782 477 11.32075472 478 11.29707113 479 11.27348643 480 11.25000000 481 11.22661123 482 11.20331950 483 11.18012422 484 11.36363636 485 11.34020619 486 11.31687243 487 11.29363450 488 11.27049180 489 11.24744376 490 11.42857143 491 11.40529532 492 11.38211382 493 11.35902637 494 11.33603239 495 11.31313131 496 11.29032258 497 11.26760563 498 11.24497992 499 11.22244489 500 11.40000000 501 11.37724551 502 11.35458167 503 11.53081511 504 11.50793651 505 11.48514851 506 11.46245059 507 11.43984221 508 11.41732283 509 11.39489194 510 11.37254902 511 11.35029354 512 11.52343750 513 11.50097466 514 11.47859922 515 11.45631068 516 11.43410853 517 11.41199226 518 11.38996139 519 11.36801541 520 11.34615385 521 11.32437620 522 11.30268199 523 11.28107075 524 11.25954198 525 11.23809524 526 11.21673004 527 11.19544592 528 11.17424242 529 11.15311909 530 11.32075472 531 11.29943503 532 11.27819549 533 11.25703565 534 11.23595506 535 11.21495327 536 11.19402985 537 11.17318436 538 11.15241636 539 11.13172542 540 11.11111111 541 11.27541590 542 11.25461255 543 11.23388582 544 11.21323529 545 11.19266055 546 11.17216117 547 11.15173675 548 11.13138686 549 11.11111111 550 11.27272727 551 11.25226860 552 11.23188406 553 11.21157324 554 11.19133574 555 11.17117117 556 11.15107914 557 11.31059246 558 11.29032258 559 11.27012522 560 11.25000000 561 11.22994652 562 11.20996441 563 11.19005329 564 11.17021277 565 11.32743363 566 11.30742049 567 11.28747795 568 11.26760563 569 11.24780316 570 11.22807018 571 11.20840630 572 11.18881119 573 11.34380454 574 11.32404181 575 11.30434783 576 11.28472222 577 11.26516464 578 11.24567474 579 11.22625216 580 11.20689655 581 11.18760757 582 11.16838488 583 11.14922813 584 11.13013699 585 11.28205128 586 11.26279863 587 11.24361158 588 11.22448980 589 11.20543294 590 11.18644068 591 11.16751269 592 11.14864865 593 11.12984823 594 11.11111111 595 11.09243697 596 11.07382550 597 11.05527638 598 11.03678930 599 11.01836394 600 11.00000000 601 10.98169717 602 11.12956811 603 11.11111111 604 11.09271523 605 11.07438017 606 11.05610561 607 11.03789127 608 11.01973684 609 11.00164204 610 10.98360656 611 10.96563011 612 10.94771242 613 11.09298532 614 11.07491857 615 11.05691057 616 11.03896104 617 11.02106969 618 11.00323625 619 10.98546042 620 10.96774194 621 10.95008052 622 10.93247588 623 11.07544141 624 11.05769231 625 11.04000000 626 11.02236422 627 11.00478469 628 10.98726115 629 10.96979332 630 10.95238095 631 10.93502377 632 10.91772152 633 10.90047393 634 11.04100946 635 11.02362205 636 11.16352201 637 11.30298273 638 11.28526646 639 11.26760563 640 11.25000000 641 11.23244930 642 11.21495327 643 11.19751166 644 11.18012422 645 11.16279070 646 11.14551084 647 11.12828439 648 11.11111111 649 11.09399076 650 11.07692308 651 11.05990783 652 11.04294479 653 11.02603369 654 11.00917431 655 10.99236641 656 10.97560976 657 11.11111111 658 11.24620061 659 11.22913505 660 11.21212121 661 11.19515885 662 11.17824773 663 11.16138763 664 11.14457831 665 11.12781955 666 11.11111111 667 11.24437781 668 11.22754491 669 11.21076233 670 11.19402985 671 11.32637854 672 11.30952381 673 11.29271917 674 11.27596439 675 11.25925926 676 11.24260355 677 11.22599705 678 11.20943953 679 11.19293078 680 11.17647059 681 11.16005874 682 11.14369501 683 11.27379209 684 11.25730994 685 11.24087591 686 11.22448980 687 11.20815138 688 11.19186047 689 11.17561684 690 11.15942029 691 11.14327062 692 11.12716763 693 11.11111111 694 11.09510086 695 11.07913669 696 11.06321839 697 11.04734577 698 11.03151862 699 11.01573677 700 11.00000000 701 10.98430813 702 10.96866097 703 11.09530583 704 11.07954545 705 11.06382979 706 11.04815864 707 11.03253182 708 11.01694915 709 11.14245416 710 11.12676056 711 11.11111111 712 11.09550562 713 11.22019635 714 11.20448179 715 11.18881119 716 11.17318436 717 11.15760112 718 11.14206128 719 11.12656467 720 11.25000000 721 11.23439667 722 11.21883657 723 11.20331950 724 11.32596685 725 11.31034483 726 11.29476584 727 11.27922971 728 11.26373626 729 11.24828532 730 11.23287671 731 11.21751026 732 11.20218579 733 11.18690314 734 11.17166213 735 11.15646259 736 11.14130435 737 11.12618725 738 11.11111111 739 11.09607578 740 11.08108108 741 11.06612686 742 11.05121294 743 11.03633917 744 11.02150538 745 11.00671141 746 10.99195710 747 10.97724230 748 10.96256684 749 10.94793057 750 10.93333333 751 10.91877497 752 10.90425532 753 10.88977424 754 10.87533156 755 10.86092715 756 10.84656085 757 10.83223250 758 10.81794195 759 10.93544137 760 11.05263158 761 11.03810775 762 11.02362205 763 11.00917431 764 10.99476440 765 10.98039216 766 10.96605744 767 10.95176010 768 10.93750000 769 10.92327698 770 10.90909091 771 10.89494163 772 10.88082902 773 10.86675291 774 10.85271318 775 10.83870968 776 10.82474227 777 10.81081081 778 10.79691517 779 10.78305520 780 10.76923077 781 10.75544174 782 10.74168798 783 10.72796935 784 10.71428571 785 10.82802548 786 10.81424936 787 10.80050826 788 10.78680203 789 10.77313054 790 10.75949367 791 10.74589128 792 10.73232323 793 10.71878941 794 10.70528967 795 10.69182390 796 10.67839196 797 10.66499373 798 10.77694236 799 10.88861076 800 10.87500000 801 10.86142322 802 10.84788030 803 10.83437111 804 10.82089552 805 10.80745342 806 10.79404467 807 10.78066914 808 10.89108911 809 10.87762670 810 10.86419753 811 10.85080148 812 10.83743842 813 10.82410824 814 10.81081081 815 10.79754601 816 10.78431373 817 10.77111383 818 10.75794621 819 10.74481074 820 10.85365854 821 10.84043849 822 10.82725061 823 10.81409478 824 10.80097087 825 10.78787879 826 10.77481840 827 10.76178960 828 10.74879227 829 10.73582630 830 10.72289157 831 10.70998797 832 10.69711538 833 10.68427371 834 10.67146283 835 10.65868263 836 10.64593301 837 10.63321386 838 10.62052506 839 10.72705602 840 10.83333333 841 10.82045184 842 10.80760095 843 10.79478055 844 10.78199052 845 10.76923077 846 10.87470449 847 10.86186541 848 10.84905660 849 10.83627797 850 10.82352941 851 10.81081081 852 10.91549296 853 10.90269637 854 10.88992974 855 10.87719298 856 10.86448598 857 10.85180863 858 10.83916084 859 10.82654249 860 10.81395349 861 10.80139373 862 10.78886311 863 10.77636153 864 10.76388889 865 10.75144509 866 10.73903002 867 10.72664360 868 10.71428571 869 10.70195627 870 10.68965517 871 10.67738232 872 10.66513761 873 10.65292096 874 10.64073227 875 10.62857143 876 10.61643836 877 10.60433295 878 10.59225513 879 10.58020478 880 10.56818182 881 10.55618615 882 10.65759637 883 10.64552661 884 10.63348416 885 10.62146893 886 10.60948081 887 10.59751973 888 10.58558559 889 10.57367829 890 10.56179775 891 10.54994388 892 10.53811659 893 10.52631579 894 10.62639821 895 10.72625698 896 10.71428571 897 10.70234114 898 10.69042316 899 10.67853170 900 10.66666667 901 10.65482797 902 10.75388027 903 10.74197121 904 10.73008850 905 10.71823204 906 10.70640177 907 10.69459757 908 10.68281938 909 10.67106711 910 10.65934066 911 10.64763996 912 10.63596491 913 10.62431544 914 10.61269147 915 10.60109290 916 10.58951965 917 10.57797165 918 10.56644880 919 10.55495103 920 10.54347826 921 10.53203040 922 10.52060738 923 10.50920910 924 10.60606061 925 10.70270270 926 10.69114471 927 10.67961165 928 10.66810345 929 10.65662002 930 10.64516129 931 10.63372718 932 10.62231760 933 10.61093248 934 10.59957173 935 10.69518717 936 10.68376068 937 10.67235859 938 10.76759062 939 10.75612354 940 10.74468085 941 10.73326249 942 10.72186837 943 10.71049841 944 10.69915254 945 10.68783069 946 10.67653277 947 10.66525871 948 10.65400844 949 10.64278188 950 10.63157895 951 10.62039958 952 10.60924370 953 10.59811123 954 10.58700210 955 10.57591623 956 10.56485356 957 10.65830721 958 10.64718163 959 10.63607925 960 10.62500000 961 10.61394381 962 10.60291060 963 10.59190031 964 10.68464730 965 10.67357513 966 10.66252588 967 10.65149948 968 10.64049587 969 10.62951496 970 10.61855670 971 10.60762101 972 10.59670782 973 10.58581706 974 10.67761807 975 10.66666667 976 10.65573770 977 10.64483112 978 10.63394683 979 10.72522983 980 10.71428571 981 10.70336391 982 10.69246436 983 10.68158698 984 10.67073171 985 10.65989848 986 10.64908722 987 10.63829787 988 10.62753036 989 10.61678463 990 10.70707071 991 10.69626640 992 10.68548387 993 10.67472306 994 10.66398390 995 10.65326633 996 10.64257028 997 10.73219659 998 10.82164329 999 10.81081081 1000 10.80000000 1001 10.78921079 1002 10.87824351 1003 10.86739781 1004 10.85657371 1005 10.84577114 1006 10.83499006 1007 10.92353525 1008 11.01190476 1009 11.10009911 1010 11.08910891 1011 11.07814045 1012 11.06719368 1013 11.05626851 1014 11.04536489 1015 11.13300493 1016 11.22047244 1017 11.30776794 1018 11.29666012 1019 11.28557409 1020 11.27450980 1021 11.26346719 1022 11.25244618 1023 11.24144673 1024 11.23046875 1025 11.21951220 1026 11.20857700 1027 11.19766310 1028 11.18677043 1029 11.27308066 1030 11.26213592 1031 11.25121242 1032 11.24031008 1033 11.22942885 1034 11.21856867 1035 11.20772947 1036 11.19691120 1037 11.18611379 1038 11.17533719 1039 11.16458133 1040 11.15384615 1041 11.14313160 1042 11.13243762 1043 11.12176414 1044 11.11111111 1045 11.10047847 1046 11.08986616 1047 11.07927412 1048 11.06870229 1049 11.05815062 1050 11.04761905 1051 11.03710752 1052 11.02661597 1053 11.01614435 1054 11.10056926 1055 11.18483412 1056 11.17424242 1057 11.16367077 1058 11.15311909 1059 11.14258735 1060 11.13207547 1061 11.12158341 1062 11.11111111 1063 11.10065851 1064 11.09022556 1065 11.07981221 1066 11.06941839 1067 11.05904405 1068 11.04868914 1069 11.13189897 1070 11.12149533 1071 11.11111111 1072 11.10074627 1073 11.09040075 1074 11.08007449 1075 11.06976744 1076 11.05947955 1077 11.04921077 1078 11.03896104 1079 11.02873031 1080 11.01851852 1081 11.00832562 1082 10.99815157 1083 10.98799631 1084 10.97785978 1085 10.96774194 1086 10.95764273 1087 10.94756210 1088 10.93750000 1089 10.92745638 1090 10.91743119 1091 10.90742438 1092 10.89743590 1093 10.88746569 1094 10.87751371 1095 10.86757991 1096 10.85766423 1097 10.84776664 1098 10.83788707 1099 10.91901729 1100 11.00000000 1101 10.99000908 1102 10.98003630 1103 10.97008160 1104 10.96014493 1105 10.95022624 1106 10.94032550 1107 10.93044264 1108 10.92057762 1109 10.91073039 1110 10.90090090 1111 10.89108911 1112 10.88129496 1113 10.87151842 1114 10.86175943 1115 10.85201794 1116 10.84229391 1117 10.83258729 1118 10.82289803 1119 10.81322609 1120 10.80357143 1121 10.79393399 1122 10.78431373 1123 10.77471060 1124 10.76512456 1125 10.75555556 1126 10.74600355 1127 10.73646850 1128 10.72695035 1129 10.71744907 1130 10.70796460 1131 10.69849691 1132 10.68904594 Final result: 10.6890 +/- 0.9187 Random chance: 10.2037 +/- 0.9001