{ "cells": [ { "cell_type": "code", "execution_count": 23, "id": "138889b92720ce2e", "metadata": { "ExecuteTime": { "end_time": "2024-04-30T13:28:07.130909Z", "start_time": "2024-04-30T13:28:06.470042Z" }, "collapsed": false }, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
runnameseedstepsagg_scorecommonsense_qa/acccommonsense_qa/acc_normhellaswag/acchellaswag/acc_normopenbookqa/accopenbookqa/acc_norm...siqa/accsiqa/acc_normwinogrande/accwinogrande/acc_normsciq/accsciq/acc_normarc/accarc/acc_normmmlu/accmmlu/acc_norm
0deduped_removed_cross500.3308930.1860.2330.2720.2580.1660.286...0.3670.3620.5160.4970.2080.2020.21950.25100.2302940.250147
1deduped_removed_cross510000.3540900.2530.2570.2900.2780.1240.264...0.3680.3890.5090.4910.5820.5160.28250.29550.2395200.253223
2deduped_removed_cross520000.3736010.2740.2900.3130.3120.1160.258...0.3670.3970.5160.5050.6860.5820.30900.32000.2473200.262812
3deduped_removed_cross530000.3831220.3060.2920.3230.3350.1500.278...0.3710.4010.5130.5000.7120.6110.30750.34150.2485680.263474
4deduped_removed_cross540000.3902220.3000.2920.3240.3510.1440.278...0.3860.3950.5110.5110.7500.6580.32600.34450.2592460.273276
5deduped_removed_cross550000.4002390.3220.3080.3250.3640.1720.298...0.3820.3980.5180.5220.7510.6610.34700.35450.2584850.271414
6deduped_removed_cross560000.4014840.3150.3140.3410.3720.1620.314...0.3770.3900.4980.4920.7760.6690.35300.35650.2618420.276371
7deduped_removed_cross570000.4035330.3240.3150.3500.3860.1880.298...0.3760.3840.5180.5210.7690.6720.36250.35850.2655580.274768
8deduped_removed_cross580000.4117740.3440.3130.3520.4090.1700.310...0.3740.3910.5300.5210.7810.6770.35300.36150.2671410.283691
9deduped_removed_cross590000.4109930.3350.3220.3610.4040.1820.294...0.3740.3910.5260.5140.7690.6720.36300.37150.2664640.284446
10deduped_removed_cross5100000.4178830.3300.3200.3700.4170.1920.324...0.3890.3890.5180.5240.7850.6820.37350.37450.2680850.283562
11deduped_removed_cross5110000.4223250.3320.3280.3660.4260.1880.320...0.3980.3970.5350.5290.8010.6950.37750.38000.2674570.285596
12deduped_removed_cross5120000.4201670.3480.3240.3640.4340.1940.306...0.3770.3920.5410.5270.7900.6900.36800.37550.2675470.285836
13deduped_removed_cross5130000.4229130.3460.3300.3720.4380.1900.320...0.3920.3960.5400.5220.8020.7070.37600.38450.2711080.287802
14deduped_removed_cross5135000.4218680.3450.3220.3700.4310.2020.330...0.3870.3920.5400.5160.7970.7000.37900.38700.2695100.287944
15deduped_removed_cross600.3308930.1860.2330.2720.2580.1660.286...0.3670.3620.5160.4970.2080.2020.21950.25100.2302940.250147
16deduped_removed_cross610000.3600390.2360.2590.2830.2770.1300.274...0.3540.3860.5090.5070.5590.5000.25900.29700.2434550.254311
17deduped_removed_cross620000.3715640.2700.2830.3030.3050.1320.280...0.3770.3920.5220.5040.6650.5660.30400.31350.2490510.255010
18deduped_removed_cross630000.3837700.2830.2860.3230.3200.1560.296...0.3750.3940.5030.4970.7210.6260.31400.34100.2540150.266158
19deduped_removed_cross640000.3910820.2930.2980.3390.3610.1600.292...0.3800.3990.5050.4940.7190.6150.33750.33750.2566960.268152
20deduped_removed_cross650000.3991300.3090.3110.3430.3760.1600.286...0.3920.4010.5250.5120.7330.6390.33900.35800.2574500.271040
21deduped_removed_cross660000.4027920.3260.3180.3530.3870.1760.284...0.3760.4050.5220.5140.7530.6640.34500.36450.2625490.273836
22deduped_removed_cross670000.4088460.3190.3190.3560.4070.1720.300...0.3860.3990.5210.5210.7640.6620.35850.36250.2627400.276266
23deduped_removed_cross680000.4114290.3140.3230.3610.4120.1680.286...0.3950.4040.5330.5110.7540.6460.35550.36900.2638750.278433
24deduped_removed_cross690000.4172790.3370.3290.3670.4210.1760.294...0.4070.4030.5320.5260.7750.6660.36050.37300.2651190.283235
25deduped_removed_cross6100000.4213990.3390.3220.3760.4260.1740.320...0.3970.4010.5420.5320.7640.6730.36750.38400.2724740.286190
26deduped_removed_cross6110000.4212040.3490.3370.3780.4280.1880.314...0.4030.3980.5300.516NaNNaN0.36900.37800.2691310.288633
27deduped_removed_cross6120000.4216670.3420.3260.3830.4340.1740.310...0.3990.3960.5380.525NaNNaN0.36600.38100.2706910.287333
28deduped_removed_cross6130000.4249790.3490.3360.3830.4400.1780.314...0.4010.3920.5350.526NaNNaN0.37850.39050.2689100.289335
29deduped_removed_cross6135000.4253560.3470.3330.3860.4440.1860.322...0.4060.3920.5430.5270.7830.6820.37450.38900.2708690.289845
30cross_minhash_dump_CC-MAIN-2013-48600.3310180.1860.2330.2720.2580.1660.286...0.3670.3620.5150.497NaNNaN0.21950.25200.2302280.250147
31cross_minhash_dump_CC-MAIN-2013-48610000.3494940.2170.2480.2880.2860.1040.244...0.3660.3800.4990.4920.5460.4840.25650.27800.2396510.253956
32cross_minhash_dump_CC-MAIN-2013-48620000.3678930.2450.2800.2980.2880.1280.280...0.3660.3830.5190.499NaNNaN0.28450.31150.2397150.253644
33cross_minhash_dump_CC-MAIN-2013-48630000.3791140.2690.2910.3040.3280.1380.266...0.3620.3940.5190.504NaNNaN0.30350.33350.2505510.262409
34cross_minhash_dump_CC-MAIN-2013-48640000.3830250.2770.2890.3110.3380.1320.280...0.3610.3930.5020.496NaNNaN0.31050.33750.2498870.263702
35cross_minhash_dump_CC-MAIN-2013-48650000.3872230.2900.3060.3270.3560.1380.276...0.3650.3890.5150.511NaNNaN0.31900.33800.2526210.266785
36cross_minhash_dump_CC-MAIN-2013-48660000.3940110.3030.3050.3320.3560.1420.288...0.3750.3970.5400.521NaNNaN0.32800.35150.2522550.265589
37cross_minhash_dump_CC-MAIN-2013-48670000.3980900.3160.3050.3370.3590.1420.302...0.3720.4010.5310.510NaNNaN0.33200.35500.2501460.267719
38cross_minhash_dump_CC-MAIN-2013-48680000.3985130.3260.3150.3390.3720.1500.288...0.3720.3960.5320.508NaNNaN0.33650.36300.2584330.274100
39cross_minhash_dump_CC-MAIN-2013-48690000.3974940.3100.3140.3450.3740.1400.274...0.3640.3920.5290.506NaNNaN0.34450.36100.2589270.271955
40cross_minhash_dump_CC-MAIN-2013-486100000.4026400.3210.3270.3470.3830.1560.280...0.3760.3970.5290.513NaNNaN0.34450.36500.2582940.272123
41cross_minhash_dump_CC-MAIN-2013-486110000.4025990.3180.3220.3480.3810.1600.284...0.3670.3870.5380.516NaNNaN0.34900.36600.2596100.276792
42cross_minhash_dump_CC-MAIN-2013-486120000.4074420.3280.3190.3490.3950.1620.290...0.3670.4070.5280.510NaNNaN0.35100.37000.2603500.279535
43cross_minhash_dump_CC-MAIN-2013-486130000.4055770.3240.3180.3500.3850.1580.290...0.3730.3960.5380.510NaNNaN0.35400.37300.2584810.274616
44cross_minhash_dump_CC-MAIN-2013-486135000.4050000.3200.3120.3540.3930.1520.288...0.3670.3960.5280.5130.7850.6750.35900.36600.2601740.278002
\n", "

45 rows × 22 columns

\n", "
" ], "text/plain": [ " runname seed steps agg_score \\\n", "0 deduped_removed_cross 5 0 0.330893 \n", "1 deduped_removed_cross 5 1000 0.354090 \n", "2 deduped_removed_cross 5 2000 0.373601 \n", "3 deduped_removed_cross 5 3000 0.383122 \n", "4 deduped_removed_cross 5 4000 0.390222 \n", "5 deduped_removed_cross 5 5000 0.400239 \n", "6 deduped_removed_cross 5 6000 0.401484 \n", "7 deduped_removed_cross 5 7000 0.403533 \n", "8 deduped_removed_cross 5 8000 0.411774 \n", "9 deduped_removed_cross 5 9000 0.410993 \n", "10 deduped_removed_cross 5 10000 0.417883 \n", "11 deduped_removed_cross 5 11000 0.422325 \n", "12 deduped_removed_cross 5 12000 0.420167 \n", "13 deduped_removed_cross 5 13000 0.422913 \n", "14 deduped_removed_cross 5 13500 0.421868 \n", "15 deduped_removed_cross 6 0 0.330893 \n", "16 deduped_removed_cross 6 1000 0.360039 \n", "17 deduped_removed_cross 6 2000 0.371564 \n", "18 deduped_removed_cross 6 3000 0.383770 \n", "19 deduped_removed_cross 6 4000 0.391082 \n", "20 deduped_removed_cross 6 5000 0.399130 \n", "21 deduped_removed_cross 6 6000 0.402792 \n", "22 deduped_removed_cross 6 7000 0.408846 \n", "23 deduped_removed_cross 6 8000 0.411429 \n", "24 deduped_removed_cross 6 9000 0.417279 \n", "25 deduped_removed_cross 6 10000 0.421399 \n", "26 deduped_removed_cross 6 11000 0.421204 \n", "27 deduped_removed_cross 6 12000 0.421667 \n", "28 deduped_removed_cross 6 13000 0.424979 \n", "29 deduped_removed_cross 6 13500 0.425356 \n", "30 cross_minhash_dump_CC-MAIN-2013-48 6 0 0.331018 \n", "31 cross_minhash_dump_CC-MAIN-2013-48 6 1000 0.349494 \n", "32 cross_minhash_dump_CC-MAIN-2013-48 6 2000 0.367893 \n", "33 cross_minhash_dump_CC-MAIN-2013-48 6 3000 0.379114 \n", "34 cross_minhash_dump_CC-MAIN-2013-48 6 4000 0.383025 \n", "35 cross_minhash_dump_CC-MAIN-2013-48 6 5000 0.387223 \n", "36 cross_minhash_dump_CC-MAIN-2013-48 6 6000 0.394011 \n", "37 cross_minhash_dump_CC-MAIN-2013-48 6 7000 0.398090 \n", "38 cross_minhash_dump_CC-MAIN-2013-48 6 8000 0.398513 \n", "39 cross_minhash_dump_CC-MAIN-2013-48 6 9000 0.397494 \n", "40 cross_minhash_dump_CC-MAIN-2013-48 6 10000 0.402640 \n", "41 cross_minhash_dump_CC-MAIN-2013-48 6 11000 0.402599 \n", "42 cross_minhash_dump_CC-MAIN-2013-48 6 12000 0.407442 \n", "43 cross_minhash_dump_CC-MAIN-2013-48 6 13000 0.405577 \n", "44 cross_minhash_dump_CC-MAIN-2013-48 6 13500 0.405000 \n", "\n", " commonsense_qa/acc commonsense_qa/acc_norm hellaswag/acc \\\n", "0 0.186 0.233 0.272 \n", "1 0.253 0.257 0.290 \n", "2 0.274 0.290 0.313 \n", "3 0.306 0.292 0.323 \n", "4 0.300 0.292 0.324 \n", "5 0.322 0.308 0.325 \n", "6 0.315 0.314 0.341 \n", "7 0.324 0.315 0.350 \n", "8 0.344 0.313 0.352 \n", "9 0.335 0.322 0.361 \n", "10 0.330 0.320 0.370 \n", "11 0.332 0.328 0.366 \n", "12 0.348 0.324 0.364 \n", "13 0.346 0.330 0.372 \n", "14 0.345 0.322 0.370 \n", "15 0.186 0.233 0.272 \n", "16 0.236 0.259 0.283 \n", "17 0.270 0.283 0.303 \n", "18 0.283 0.286 0.323 \n", "19 0.293 0.298 0.339 \n", "20 0.309 0.311 0.343 \n", "21 0.326 0.318 0.353 \n", "22 0.319 0.319 0.356 \n", "23 0.314 0.323 0.361 \n", "24 0.337 0.329 0.367 \n", "25 0.339 0.322 0.376 \n", "26 0.349 0.337 0.378 \n", "27 0.342 0.326 0.383 \n", "28 0.349 0.336 0.383 \n", "29 0.347 0.333 0.386 \n", "30 0.186 0.233 0.272 \n", "31 0.217 0.248 0.288 \n", "32 0.245 0.280 0.298 \n", "33 0.269 0.291 0.304 \n", "34 0.277 0.289 0.311 \n", "35 0.290 0.306 0.327 \n", "36 0.303 0.305 0.332 \n", "37 0.316 0.305 0.337 \n", "38 0.326 0.315 0.339 \n", "39 0.310 0.314 0.345 \n", "40 0.321 0.327 0.347 \n", "41 0.318 0.322 0.348 \n", "42 0.328 0.319 0.349 \n", "43 0.324 0.318 0.350 \n", "44 0.320 0.312 0.354 \n", "\n", " hellaswag/acc_norm openbookqa/acc openbookqa/acc_norm ... siqa/acc \\\n", "0 0.258 0.166 0.286 ... 0.367 \n", "1 0.278 0.124 0.264 ... 0.368 \n", "2 0.312 0.116 0.258 ... 0.367 \n", "3 0.335 0.150 0.278 ... 0.371 \n", "4 0.351 0.144 0.278 ... 0.386 \n", "5 0.364 0.172 0.298 ... 0.382 \n", "6 0.372 0.162 0.314 ... 0.377 \n", "7 0.386 0.188 0.298 ... 0.376 \n", "8 0.409 0.170 0.310 ... 0.374 \n", "9 0.404 0.182 0.294 ... 0.374 \n", "10 0.417 0.192 0.324 ... 0.389 \n", "11 0.426 0.188 0.320 ... 0.398 \n", "12 0.434 0.194 0.306 ... 0.377 \n", "13 0.438 0.190 0.320 ... 0.392 \n", "14 0.431 0.202 0.330 ... 0.387 \n", "15 0.258 0.166 0.286 ... 0.367 \n", "16 0.277 0.130 0.274 ... 0.354 \n", "17 0.305 0.132 0.280 ... 0.377 \n", "18 0.320 0.156 0.296 ... 0.375 \n", "19 0.361 0.160 0.292 ... 0.380 \n", "20 0.376 0.160 0.286 ... 0.392 \n", "21 0.387 0.176 0.284 ... 0.376 \n", "22 0.407 0.172 0.300 ... 0.386 \n", "23 0.412 0.168 0.286 ... 0.395 \n", "24 0.421 0.176 0.294 ... 0.407 \n", "25 0.426 0.174 0.320 ... 0.397 \n", "26 0.428 0.188 0.314 ... 0.403 \n", "27 0.434 0.174 0.310 ... 0.399 \n", "28 0.440 0.178 0.314 ... 0.401 \n", "29 0.444 0.186 0.322 ... 0.406 \n", "30 0.258 0.166 0.286 ... 0.367 \n", "31 0.286 0.104 0.244 ... 0.366 \n", "32 0.288 0.128 0.280 ... 0.366 \n", "33 0.328 0.138 0.266 ... 0.362 \n", "34 0.338 0.132 0.280 ... 0.361 \n", "35 0.356 0.138 0.276 ... 0.365 \n", "36 0.356 0.142 0.288 ... 0.375 \n", "37 0.359 0.142 0.302 ... 0.372 \n", "38 0.372 0.150 0.288 ... 0.372 \n", "39 0.374 0.140 0.274 ... 0.364 \n", "40 0.383 0.156 0.280 ... 0.376 \n", "41 0.381 0.160 0.284 ... 0.367 \n", "42 0.395 0.162 0.290 ... 0.367 \n", "43 0.385 0.158 0.290 ... 0.373 \n", "44 0.393 0.152 0.288 ... 0.367 \n", "\n", " siqa/acc_norm winogrande/acc winogrande/acc_norm sciq/acc \\\n", "0 0.362 0.516 0.497 0.208 \n", "1 0.389 0.509 0.491 0.582 \n", "2 0.397 0.516 0.505 0.686 \n", "3 0.401 0.513 0.500 0.712 \n", "4 0.395 0.511 0.511 0.750 \n", "5 0.398 0.518 0.522 0.751 \n", "6 0.390 0.498 0.492 0.776 \n", "7 0.384 0.518 0.521 0.769 \n", "8 0.391 0.530 0.521 0.781 \n", "9 0.391 0.526 0.514 0.769 \n", "10 0.389 0.518 0.524 0.785 \n", "11 0.397 0.535 0.529 0.801 \n", "12 0.392 0.541 0.527 0.790 \n", "13 0.396 0.540 0.522 0.802 \n", "14 0.392 0.540 0.516 0.797 \n", "15 0.362 0.516 0.497 0.208 \n", "16 0.386 0.509 0.507 0.559 \n", "17 0.392 0.522 0.504 0.665 \n", "18 0.394 0.503 0.497 0.721 \n", "19 0.399 0.505 0.494 0.719 \n", "20 0.401 0.525 0.512 0.733 \n", "21 0.405 0.522 0.514 0.753 \n", "22 0.399 0.521 0.521 0.764 \n", "23 0.404 0.533 0.511 0.754 \n", "24 0.403 0.532 0.526 0.775 \n", "25 0.401 0.542 0.532 0.764 \n", "26 0.398 0.530 0.516 NaN \n", "27 0.396 0.538 0.525 NaN \n", "28 0.392 0.535 0.526 NaN \n", "29 0.392 0.543 0.527 0.783 \n", "30 0.362 0.515 0.497 NaN \n", "31 0.380 0.499 0.492 0.546 \n", "32 0.383 0.519 0.499 NaN \n", "33 0.394 0.519 0.504 NaN \n", "34 0.393 0.502 0.496 NaN \n", "35 0.389 0.515 0.511 NaN \n", "36 0.397 0.540 0.521 NaN \n", "37 0.401 0.531 0.510 NaN \n", "38 0.396 0.532 0.508 NaN \n", "39 0.392 0.529 0.506 NaN \n", "40 0.397 0.529 0.513 NaN \n", "41 0.387 0.538 0.516 NaN \n", "42 0.407 0.528 0.510 NaN \n", "43 0.396 0.538 0.510 NaN \n", "44 0.396 0.528 0.513 0.785 \n", "\n", " sciq/acc_norm arc/acc arc/acc_norm mmlu/acc mmlu/acc_norm \n", "0 0.202 0.2195 0.2510 0.230294 0.250147 \n", "1 0.516 0.2825 0.2955 0.239520 0.253223 \n", "2 0.582 0.3090 0.3200 0.247320 0.262812 \n", "3 0.611 0.3075 0.3415 0.248568 0.263474 \n", "4 0.658 0.3260 0.3445 0.259246 0.273276 \n", "5 0.661 0.3470 0.3545 0.258485 0.271414 \n", "6 0.669 0.3530 0.3565 0.261842 0.276371 \n", "7 0.672 0.3625 0.3585 0.265558 0.274768 \n", "8 0.677 0.3530 0.3615 0.267141 0.283691 \n", "9 0.672 0.3630 0.3715 0.266464 0.284446 \n", "10 0.682 0.3735 0.3745 0.268085 0.283562 \n", "11 0.695 0.3775 0.3800 0.267457 0.285596 \n", "12 0.690 0.3680 0.3755 0.267547 0.285836 \n", "13 0.707 0.3760 0.3845 0.271108 0.287802 \n", "14 0.700 0.3790 0.3870 0.269510 0.287944 \n", "15 0.202 0.2195 0.2510 0.230294 0.250147 \n", "16 0.500 0.2590 0.2970 0.243455 0.254311 \n", "17 0.566 0.3040 0.3135 0.249051 0.255010 \n", "18 0.626 0.3140 0.3410 0.254015 0.266158 \n", "19 0.615 0.3375 0.3375 0.256696 0.268152 \n", "20 0.639 0.3390 0.3580 0.257450 0.271040 \n", "21 0.664 0.3450 0.3645 0.262549 0.273836 \n", "22 0.662 0.3585 0.3625 0.262740 0.276266 \n", "23 0.646 0.3555 0.3690 0.263875 0.278433 \n", "24 0.666 0.3605 0.3730 0.265119 0.283235 \n", "25 0.673 0.3675 0.3840 0.272474 0.286190 \n", "26 NaN 0.3690 0.3780 0.269131 0.288633 \n", "27 NaN 0.3660 0.3810 0.270691 0.287333 \n", "28 NaN 0.3785 0.3905 0.268910 0.289335 \n", "29 0.682 0.3745 0.3890 0.270869 0.289845 \n", "30 NaN 0.2195 0.2520 0.230228 0.250147 \n", "31 0.484 0.2565 0.2780 0.239651 0.253956 \n", "32 NaN 0.2845 0.3115 0.239715 0.253644 \n", "33 NaN 0.3035 0.3335 0.250551 0.262409 \n", "34 NaN 0.3105 0.3375 0.249887 0.263702 \n", "35 NaN 0.3190 0.3380 0.252621 0.266785 \n", "36 NaN 0.3280 0.3515 0.252255 0.265589 \n", "37 NaN 0.3320 0.3550 0.250146 0.267719 \n", "38 NaN 0.3365 0.3630 0.258433 0.274100 \n", "39 NaN 0.3445 0.3610 0.258927 0.271955 \n", "40 NaN 0.3445 0.3650 0.258294 0.272123 \n", "41 NaN 0.3490 0.3660 0.259610 0.276792 \n", "42 NaN 0.3510 0.3700 0.260350 0.279535 \n", "43 NaN 0.3540 0.3730 0.258481 0.274616 \n", "44 0.675 0.3590 0.3660 0.260174 0.278002 \n", "\n", "[45 rows x 22 columns]" ] }, "execution_count": 23, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import pandas as pd\n", "from matplotlib.figure import Figure\n", "\n", "df = pd.read_csv(\"../src_data/removed_data_cross_dedup.csv\")\n", "df" ] }, { "cell_type": "code", "execution_count": 24, "id": "b610f43caefdf01", "metadata": { "ExecuteTime": { "end_time": "2024-04-30T13:29:05.776714Z", "start_time": "2024-04-30T13:29:05.774546Z" }, "collapsed": false }, "outputs": [], "source": [ "runs_mapping = {\n", " \"deduped_removed_cross\": \"Originally removed data\",\n", " \"cross_minhash_dump_CC-MAIN-2013-48\": \"Originally kept data\",\n", "}" ] }, { "cell_type": "code", "execution_count": 25, "id": "18b2dde6", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "Index(['runname', 'seed', 'steps', 'agg_score', 'commonsense_qa/acc',\n", " 'commonsense_qa/acc_norm', 'hellaswag/acc', 'hellaswag/acc_norm',\n", " 'openbookqa/acc', 'openbookqa/acc_norm', 'piqa/acc', 'piqa/acc_norm',\n", " 'siqa/acc', 'siqa/acc_norm', 'winogrande/acc', 'winogrande/acc_norm',\n", " 'sciq/acc', 'sciq/acc_norm', 'arc/acc', 'arc/acc_norm', 'mmlu/acc',\n", " 'mmlu/acc_norm'],\n", " dtype='object')" ] }, "execution_count": 25, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df.columns" ] }, { "cell_type": "code", "execution_count": 27, "id": "initial_id", "metadata": { "ExecuteTime": { "end_time": "2024-04-30T13:31:10.740797Z", "start_time": "2024-04-30T13:31:10.661359Z" }, "collapsed": true }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "No artists with labels found to put in legend. Note that artists whose label start with an underscore are ignored when legend() is called with no argument.\n" ] }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkIAAAHHCAYAAABTMjf2AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/H5lhTAAAACXBIWXMAAA9hAAAPYQGoP6dpAABMi0lEQVR4nO3de3zP9f//8ft7Y+9tZhvGNowxxzn2mfg4JUyLUio5lSF0ksRHReSQYyX0ccihQj4VRfoocmhRkZJjyqGcfbAh2Zjz9vz90W+vr7dt7M0O5nW7Xi67XLyf7+fr9Xq8Xq/36/2+e72e79fbYYwxAgAAsCGPvC4AAAAgrxCEAACAbRGEAACAbRGEAACAbRGEAACAbRGEAACAbRGEAACAbRGEAACAbRGEAACAbRGEctHq1avlcDi0YMGCvC7luvbv3y+Hw6HZs2ff0PQOh0PDhg3L1pqudvfdd+vuu++2Ht9szbi+tNfw6tWrb2j6YcOGyeFwZG9RyFRCQoLatm2rYsWKyeFwaOLEiXldUo4JDw/X/ffff9ssJyfc7PF7uyII3SSHw5GlP154wM2ZOnVqvgi527dv17Bhw7R///68LkV9+/bV8uXLNXDgQM2dO1f33ntvXpd0U26lbZvbRo8erc8//zzPlr906dIc/89tXimQ1wXkd3PnznV5/MEHH2jlypXp2qtWraodO3bkZmk3pWzZsjp37pwKFix4Q9OfO3dOBQrw8kL2mTp1qoKCgtS1a9e8LuWatm/fruHDh+vuu+9WeHh4ntbyzTff6MEHH1T//v3ztI7scitt29w2evRotW3bVm3atMmT5S9dulRTpky5LcMQn1Q36fHHH3d5/OOPP2rlypXp2iXliyB0+fJlpaamysvLS97e3jc8n5uZ9laWnJysQoUK5XUZQKauPIaPHTumwMDAbJv3+fPn5eXlJQ8PLibg9sGrOQ+kpqZq1KhRKl26tLy9vdW8eXPt3r07Xb+ffvpJ9957rwICAuTr66smTZpo7dq1WVrGsWPH1L17dwUHB8vb21u1atXSnDlzXPqkjakZN26cJk6cqIiICDmdTm3fvj3T8TaffvqpIiMj5e3trerVq2vRokXq2rVruv+dXT1GKG1syO7du9W1a1cFBgYqICBA3bp109mzZ12mnTVrlpo1a6YSJUrI6XQqMjJS77zzTpbW++r5OBwObd68Od1zo0ePlqenpw4fPpzp9Gk1b9++XZ06dVKRIkXUqFEj6/n//Oc/ioqKko+Pj4oWLaoOHTro0KFDLvO4++67Vb16df3yyy9q0qSJfH19VaFCBWuc2Lfffqt69erJx8dHlStX1tdff52ujs2bN6tly5by9/eXn5+fmjdvrh9//NF6fsOGDXI4HOn2ryQtX75cDodDX375pdV2+PBhPfHEEwoODpbT6VS1atX0/vvvp5v2f//7n9q0aaNChQqpRIkS6tu3ry5cuJDp9rramjVrdOedd8rb21sRERGaPn16hv2ysr/Dw8P122+/6dtvv7UuN6eNDzt58qT69++vGjVqyM/PT/7+/mrZsqW2bt2apTovX76sESNGWK//8PBwvfLKK+nWNbNxb+Hh4dZZqtmzZ+vRRx+VJDVt2jTdpfG08SUrVqxQ7dq15e3trcjISH322Wfp5nvq1Cm98MILCgsLk9PpVIUKFfT6668rNTXV6pPZMTx16lQ5HA4ZYzRlyhSrjjR79+7Vo48+qqJFi8rX11f//Oc/tWTJEpflp40nmTdvngYPHqxSpUrJ19dXSUlJ6tq1q/z8/HTw4EHdf//98vPzU6lSpTRlyhRJ0rZt29SsWTMVKlRIZcuW1UcffeQy70uXLmn48OGqWLGivL29VaxYMTVq1EgrV67MdD9db9umWbNmjerWrStvb2+VL19eH3zwwQ1tW3fMmTNHBQoU0Isvvmi1ZeX9O+09ZufOnWrXrp38/f1VrFgx9enTR+fPn7f6ORwOJScna86cOdZ6X+/MaFaP3++//16PPvqoypQpI6fTqbCwMPXt21fnzp2z+nTt2tXat1cO+Ugzbtw4NWjQQMWKFZOPj4+ioqLyxVhYi0G26tWrl8lss65atcpIMnfccYeJiooyEyZMMMOGDTO+vr6mbt26Ln3j4uKMl5eXqV+/vnnrrbfMhAkTTM2aNY2Xl5f56aefrlnD2bNnTdWqVU3BggVN3759zb///W/TuHFjI8lMnDjR6rdv3z4jyURGRpry5cubsWPHmgkTJpgDBw5Yz82aNcvq/+WXXxqHw2Fq1qxpxo8fb1599VVTpEgRU716dVO2bFmXGiSZoUOHWo+HDh1qrfvDDz9spk6danr06GEkmZdeesll2jvvvNN07drVTJgwwUyaNMncc889RpKZPHmyS78mTZqYJk2apFuftJqTkpKMj4+P+de//pVuG0VGRppmzZpdczum1RwZGWkefPBBM3XqVDNlyhRjjDEjR440DofDtG/f3kydOtUMHz7cBAUFmfDwcPPXX3+51FiyZEkTFhZmXnzxRTNp0iQTGRlpPD09zbx580xISIgZNmyYmThxoilVqpQJCAgwSUlJ1vS//vqrKVSokAkNDTUjRowwY8eONeXKlTNOp9P8+OOPVr/y5cubVq1apVuHbt26mSJFipiLFy8aY4yJj483pUuXNmFhYea1114z77zzjnnggQeMJDNhwgRrurNnz5pKlSoZb29v89JLL5mJEyeaqKgoU7NmTSPJrFq16prb7pdffjE+Pj6mTJkyZsyYMWbEiBEmODjYmv5KWdnfixYtMqVLlzZVqlQxc+fONXPnzjUrVqwwxhjz888/m4iICDNgwAAzffp089prr1nb8vDhw9es0xhjunTpYiSZtm3bmilTppjY2FgjybRp08al39Wv6TRly5Y1Xbp0McYYs2fPHvP8888bSeaVV16xao2Pj7f6VqpUyQQGBpoBAwaY8ePHmxo1ahgPDw9rfYwxJjk52dSsWdMUK1bMvPLKK2batGkmNjbWOBwO06dPH6tfZsfwt99+a+bOnWskmRYtWlh1GPP3ayA4ONgULlzYDBo0yIwfP97UqlXLeHh4mM8++8yad9r7VWRkpKldu7YZP368GTNmjElOTjZdunQx3t7eJjIy0jz99NNmypQppkGDBtbxV7JkSev1Xq1aNePp6Wn27t1rzfuVV14xDofD9OzZ08ycOdO89dZbpmPHjmbs2LGZ7qesbNvKlSub4OBg88orr5jJkyebf/zjH8bhcJhff/3V7W2bmbJly5r77rvPejx9+nTjcDjMoEGDrLasvn+nvcfUqFHDtG7d2kyePNk8/vjjRpLp3Lmz1W/u3LnG6XSaxo0bW+v9ww8/ZFqjO8dv7969TatWrczo0aPN9OnTTffu3Y2np6dp27at1eeHH34wLVq0MJKs5ae9nowxpnTp0ubZZ581kydPNuPHjzd169Y1ksyXX3553e15KyAIZbOsBKGqVauaCxcuWO1vv/22kWS2bdtmjDEmNTXVVKxY0cTExJjU1FSr39mzZ025cuVMixYtrlnDxIkTjSTzn//8x2q7ePGiqV+/vvHz87M+aNPeRP39/c2xY8dc5pFREKpRo4YpXbq0OX36tNW2evVqIynLQeiJJ55w6ffQQw+ZYsWKubSdPXs23TrFxMSY8uXLu7RdLwgZY0zHjh1NyZIlTUpKitW2adOmdP0yklZzx44dXdr3799vPD09zahRo1zat23bZgoUKODS3qRJEyPJfPTRR1bbzp07jSTj4eHhEmaWL1+erq42bdoYLy8vs2fPHqvtyJEjpnDhwuauu+6y2gYOHGgKFixoTp48abVduHDBBAYGumzz7t27m9DQUHPixAmX2jt06GACAgKsbZ/2Gvrkk0+sPsnJyaZChQpZCkJt2rQx3t7e5sCBA1bb9u3bjaenZ7rjI6v7u1q1ai77O8358+dd9q8xf78WnE6nee21165Z55YtW4wk06NHD5f2/v37G0nmm2++sdqyEoSMMebTTz/NdBuVLVvWSDILFy602hITE01oaKi54447rLYRI0aYQoUKmd9//91l+gEDBhhPT09z8OBBaz0zO4bTau7Vq5dL2wsvvGAkme+//95qO336tClXrpwJDw+3tmXa+1X58uXT7aO08Dh69Gir7a+//jI+Pj7G4XCYefPmWe1pr/crt12tWrVcwkRWZWXbfvfdd1bbsWPHjNPpdPnPUFa3bWauDEJvv/22cTgcZsSIEdbz7rx/p73HPPDAAy7LePbZZ40ks3XrVqutUKFCLq+za3Hn+M3o+BszZoxxOBwux++1PtuunsfFixdN9erVr/ufzVsFl8byQLdu3eTl5WU9bty4saS/T1dL0pYtW/THH3+oU6dO+vPPP3XixAmdOHFCycnJat68ub777rtrnsJdunSpQkJC1LFjR6utYMGCev7553XmzBl9++23Lv0feeQRFS9e/Jo1HzlyRNu2bVNsbKz8/Pys9iZNmqhGjRpZXvenn37a5XHjxo31559/KikpyWrz8fGx/p2YmKgTJ06oSZMm2rt3rxITE7O8LEmKjY3VkSNHtGrVKqvtww8/lI+Pjx555JEbqvmzzz5Tamqq2rVrZ+2bEydOKCQkRBUrVnRZliT5+fmpQ4cO1uPKlSsrMDBQVatWVb169az2tH+nvQ5SUlK0YsUKtWnTRuXLl7f6hYaGqlOnTlqzZo213dq3b69Lly65XGJZsWKFTp06pfbt20uSjDFauHChWrduLWOMS+0xMTFKTEzUpk2bJP39GgoNDVXbtm2t+fn6+urJJ5+87vZKSUnR8uXL1aZNG5UpU8Zqr1q1qmJiYtL1v9n97XQ6rTErKSkp+vPPP+Xn56fKlStb65OZpUuXSpL69evn0v6vf/1LktJdLsoOJUuW1EMPPWQ99vf3V2xsrDZv3qz4+HhJf1+Cbty4sYoUKeKyn6Kjo5WSkqLvvvvOZZ5ZOYbTLF26VHXr1nW5zOvn56cnn3xS+/fv1/bt2136d+nSxWUfXalHjx7WvwMDA1W5cmUVKlRI7dq1s9rTXu9pr+u0vr/99pv++OOPLNWcVZGRkdb7qSQVL15clStXdlm2u9s2M2+88Yb69Omj119/XYMHD7bab+T9u1evXi6Pe/fuLen/Xp/ucuf4vXLfJicn68SJE2rQoIGMMRkOK8jIlfP466+/lJiYqMaNG1/3+LtVMFg6D1z54SBJRYoUkfT3C0iS9ebQpUuXTOeRmJhoTXe1AwcOqGLFiukGNFatWtV6/krlypW7bs1p01SoUCHdcxUqVMjyC/5a6+7v7y9JWrt2rYYOHap169alGz+UmJiogICALC1Lklq0aKHQ0FB9+OGHat68uVJTU/Xxxx/rwQcfVOHChbM0j6u3zx9//CFjjCpWrJhh/6u/aVe6dOl0984JCAhQWFhYujbp/14Hx48f19mzZ1W5cuV0y6hatapSU1N16NAhVatWTbVq1VKVKlU0f/58de/eXZI0f/58BQUFqVmzZtb8Tp06pRkzZmjGjBkZ1n7s2DFJf+/vChUqpKs7o1qudvz4cZ07dy7D7VO5cuV0b+43u79TU1P19ttva+rUqdq3b59SUlKs54oVK3bNaQ8cOCAPD490r+uQkBAFBgamO1ayQ0bbtVKlSpL+HvMTEhKiP/74Q7/88kum4SZtP6XJyjGc5sCBAy4BPM2V7w/Vq1e/7ry9vb3T1RcQEJDp6z3tdS1Jr732mh588EFVqlRJ1atX17333qvOnTurZs2aWV6PjFz9/iL9/R5z5bLd3bYZ+fbbb7VkyRK9/PLLLuOC0uYvuff+ffWxEhERIQ8Pjxu+TYA7x+/Bgwc1ZMgQLV682GU7pdWZFV9++aVGjhypLVu2uIxDyi/3DCMI5QFPT88M240xkmT9b+HNN99U7dq1M+x75VmZm5XZ//ZywvXWfc+ePWrevLmqVKmi8ePHKywsTF5eXlq6dKkmTJjg9mBGT09PderUSTNnztTUqVO1du1aHTlyJMNv9WXm6u2Tmpoqh8Ohr776KsP1uXrfZLbO19sW7mrfvr1GjRqlEydOqHDhwlq8eLE6duxo3cYgbds9/vjjmb5J3+wHkbuyY3+PHj1ar776qp544gmNGDFCRYsWlYeHh1544YUsv15u5g37yuCVXVJTU9WiRQu99NJLGT6fFpzS5OQxnNm8b+Z1fdddd2nPnj3673//qxUrVujdd9/VhAkTNG3aNJezTO7KyrLd3bYZqVatmk6dOqW5c+fqqaeecgmL2fH+nVsBIiUlRS1atNDJkyf18ssvq0qVKipUqJAOHz6srl27Zun4+f777/XAAw/orrvu0tSpUxUaGqqCBQtq1qxZ6QbJ36oIQregiIgISX+fMo+OjnZ7+rJly+qXX35Ramqqy1mhnTt3Ws/fyDwlZfjttozabtQXX3yhCxcuaPHixS7/u7v6cpM7YmNj9dZbb+mLL77QV199peLFi2d4iSarIiIiZIxRuXLlsvSmeaOKFy8uX19f7dq1K91zO3fulIeHh8tZpfbt22v48OFauHChgoODlZSU5HJJrnjx4ipcuLBSUlKu+7oqW7asfv31VxljXN6UM6olo7p9fHwyvOxx9fTu7O/MPhwWLFigpk2b6r333nNpP3XqlIKCgq5Za9myZZWamqo//vjDOiMi/X1H5lOnTrkcK0WKFNGpU6dcpr948aKOHj2apTrT7N69O912/f333yXJ+vZlRESEzpw5c0PH//WULVs209dU2vO5oWjRourWrZu6deumM2fO6K677tKwYcOuGYSyIyBkx7YNCgrSggUL1KhRIzVv3lxr1qxRyZIlrflL7r1///HHHy5havfu3UpNTXX5Nq47657V43fbtm36/fffNWfOHMXGxlrtGX17L7PlL1y4UN7e3lq+fLmcTqfVPmvWrCzXm9cYI3QLioqKUkREhMaNG6czZ86ke/748ePXnL5Vq1aKj4/X/PnzrbbLly9r0qRJ8vPzU5MmTdyuqWTJkqpevbo++OADl5q+/fZbbdu2ze35ZSbtf3RX/g8uMTHxpg6qmjVrqmbNmnr33Xe1cOFCdejQ4aZu9vjwww/L09NTw4cPT3f2xhijP//884bnfSVPT0/dc889+u9//+tyijwhIUEfffSRGjVqZF1OlP6+tFGjRg3Nnz9f8+fPV2hoqO666y6X+T3yyCNauHChfv3113TLu/J11apVKx05csTlK7Bnz57N9JLa1XXHxMTo888/18GDB632HTt2aPny5en6Slnb34UKFUoXRNLmcfV++PTTT695a4Q0rVq1kqR0Pz0xfvx4SdJ9991ntUVERKQbPzJjxox0Z4TS7jOVUa3S3+PtFi1aZD1OSkrSBx98oNq1ayskJESS1K5dO61bty7d9kqb7+XLl6+7bplp1aqV1q9fr3Xr1lltycnJmjFjhsLDwxUZGXnD886qq48RPz8/VahQ4bq3Z7jets2K7Nq2pUuX1tdff61z586pRYsW1jrdyPt32lfT00yaNEmS1LJlS6sts9d/RrJ6/GZ0/Blj9Pbbb6ebZ2bb3tPTUw6Hw+U42L9/f57eBdtdnBG6BXl4eOjdd99Vy5YtVa1aNXXr1k2lSpXS4cOHtWrVKvn7++uLL77IdPonn3xS06dPV9euXbVx40aFh4drwYIFWrt2rSZOnJjlsTFXGz16tB588EE1bNhQ3bp1019//aXJkyerevXqGR7wN+Kee+6Rl5eXWrduraeeekpnzpzRzJkzVaJEiXT/83ZHbGysdXdddy6LZSQiIkIjR47UwIEDtX//frVp00aFCxfWvn37tGjRIj355JPZdiffkSNHauXKlWrUqJGeffZZFShQQNOnT9eFCxf0xhtvpOvfvn17DRkyRN7e3urevXu6cWJjx47VqlWrVK9ePfXs2VORkZE6efKkNm3apK+//lonT56UJPXs2VOTJ09WbGysNm7cqNDQUM2dO1e+vr5Zqnv48OFatmyZGjdurGeffdYK4tWqVdMvv/xi9XNnf0dFRemdd97RyJEjVaFCBZUoUULNmjXT/fffr9dee03dunVTgwYNtG3bNn344YcuA8wzU6tWLXXp0kUzZszQqVOn1KRJE61fv15z5sxRmzZt1LRpU6tvjx499PTTT+uRRx5RixYttHXrVi1fvjzdWafatWvL09NTr7/+uhITE+V0Oq37JEl/X3rp3r27fv75ZwUHB+v9999XQkKCS/h78cUXtXjxYt1///3q2rWroqKilJycrG3btmnBggXav3//dc92ZWbAgAH6+OOP1bJlSz3//PMqWrSo5syZo3379mnhwoW5crPEyMhI3X333YqKilLRokW1YcMGLViwQM8999w1p7vets2K7Ny2FSpU0IoVK3T33XcrJiZG33zzjfz9/d1+/963b58eeOAB3XvvvVq3bp3+85//qFOnTqpVq5bVJyoqSl9//bXGjx+vkiVLqly5chmO9ZKyfvxWqVJFERER6t+/vw4fPix/f38tXLgw3VihtOVL0vPPP6+YmBh5enqqQ4cOuu+++zR+/Hjde++96tSpk44dO6YpU6aoQoUKLsf6LS13v6R2+8vK1+c//fRTl/aMvvZtjDGbN282Dz/8sClWrJhxOp2mbNmypl27diYuLu66dSQkJJhu3bqZoKAg4+XlZWrUqJFu/mnLffPNN9NNn1lN8+bNM1WqVDFOp9NUr17dLF682DzyyCOmSpUqLv2Uydfnjx8/7tJv1qxZRpLZt2+f1bZ48WJTs2ZN4+3tbcLDw83rr79u3n///XT9svL1+TRHjx41np6eplKlShlur4xkVnOahQsXmkaNGplChQqZQoUKmSpVqphevXqZXbt2udRYrVq1dNNefS+SNMrg686bNm0yMTExxs/Pz/j6+pqmTZtmeg+RP/74w0gyksyaNWsy7JOQkGB69eplwsLCTMGCBU1ISIhp3ry5mTFjhku/AwcOmAceeMD4+vqaoKAg06dPH7Ns2bIsfX3eGGO+/fZbExUVZby8vEz58uXNtGnTrG16pazu7/j4eHPfffeZwoULG0nWvj9//rz517/+ZUJDQ42Pj49p2LChWbduXbrXR2YuXbpkhg8fbsqVK2cKFixowsLCzMCBA8358+dd+qWkpJiXX37ZBAUFGV9fXxMTE2N2796d7uvzxhgzc+ZMU758eet2AWnbK22/L1++3NSsWdM4nU5TpUqVdO8Jxvz9lfaBAweaChUqGC8vLxMUFGQaNGhgxo0bZ90X6lrHsDEZv56M+fuePG3btjWBgYHG29vb1K1bN909XzJ7vzLm76/PFypUKF17Vl/vI0eONHXr1jWBgYHGx8fHVKlSxYwaNcpar2u53rbNqKarXwdZ2baZyWg5P/30k3VLi7Svkmfl/TvteNi+fbtp27atKVy4sClSpIh57rnnzLlz51yWsXPnTnPXXXcZHx8fI+m6X6XP6vG7fft2Ex0dbfz8/ExQUJDp2bOn2bp1a7r30suXL5vevXub4sWLG4fD4XIcv/fee6ZixYrW63nWrFkZHuu3KocxNzgyE/j/ateureLFi1/zrrB57cSJEwoNDdWQIUP06quv5nU5sKnw8HBVr17d5U7fsK9hw4Zp+PDhOn78+A2f4cPNY4wQsuzSpUvprp+vXr1aW7dutX7u4FY1e/ZspaSkqHPnznldCgDgFsIYIWTZ4cOHFR0drccff1wlS5bUzp07NW3aNIWEhKS76eCt4ptvvtH27ds1atQotWnTxna/WA0AuDaCELKsSJEiioqK0rvvvqvjx4+rUKFCuu+++zR27Njr3rgur7z22mv64Ycf1LBhQ+ubGAAApMnTMULfffed3nzzTW3cuFFHjx7VokWL1KZNm2tOs3r1avXr10+//fabwsLCNHjw4Ov+Ci8AAEBG8nSMUHJysmrVqpXuHgqZ2bdvn+677z41bdpUW7Zs0QsvvKAePXpkeD8IAACA67llvjXmcDiue0bo5Zdf1pIlS1xuBtehQwedOnVKy5Yty4UqAQDA7SRfjRFat25duluWx8TE6IUXXsh0mgsXLrjcrTQ1NVUnT55UsWLF8s0PwgEAYHfGGJ0+fVolS5bM1ht/5qsgFB8fr+DgYJe2tN9UOnfuXIY/DjhmzBgNHz48t0oEAAA56NChQypdunS2zS9fBaEbMXDgQPXr1896nJiYqDJlyujQoUMuv9MEAABuXUlJSQoLC7vhn4nKTL4KQiEhIUpISHBpS0hIkL+/f4ZngyTJ6XS6/CJuGn9/f4IQAAD5THYPa8lXd5auX7++4uLiXNpWrlyp+vXr51FFAAAgP8vTIHTmzBlt2bJFW7ZskfT31+O3bNmigwcPSvr7slZsbKzV/+mnn9bevXv10ksvaefOnZo6dao++eQT9e3bNy/KBwAA+VyeBqENGzbojjvu0B133CFJ6tevn+644w4NGTJEknT06FErFElSuXLltGTJEq1cuVK1atXSW2+9pXfffVcxMTF5Uj8AAMjfbpn7COWWpKQkBQQEKDExkTFCAADkopSUFF26dCnT5728vDL9anxOfX7nq8HSAAAg/zHGKD4+XqdOnbpmPw8PD5UrV05eXl65U5gIQgAAIIelhaASJUrI19c3w29+paam6siRIzp69KjKlCmTazc9JggBAIAck5KSYoWgYsWKXbNv8eLFdeTIEV2+fFkFCxbMlfry1dfnAQBA/pI2JsjX1/e6fdMuiaWkpORoTVciCAEAgByXlUtdefEboAQhAABgWwQhAABgWwQhAABgWwQhAACQ47Jy/+a8uMczQQgAAOSYtK/Bnz179rp9L168KEny9PTM0ZquxH2EAABAjvH09FRgYKCOHTsmSde8oeLx48fl6+urAgVyL54QhAAAQI4KCQmRJCsMZcbDwyNX7yotEYQAAEAOczgcCg0NVYkSJW74R1dzCkEIAADkCk9Pz1wd/5MVDJYGAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2RRACAAC2ledBaMqUKQoPD5e3t7fq1aun9evXX7P/xIkTVblyZfn4+CgsLEx9+/bV+fPnc6laAABwO8nTIDR//nz169dPQ4cO1aZNm1SrVi3FxMTo2LFjGfb/6KOPNGDAAA0dOlQ7duzQe++9p/nz5+uVV17J5coBAMDtIE+D0Pjx49WzZ09169ZNkZGRmjZtmnx9ffX+++9n2P+HH35Qw4YN1alTJ4WHh+uee+5Rx44dr3sWCQAAICN5FoQuXryojRs3Kjo6+v+K8fBQdHS01q1bl+E0DRo00MaNG63gs3fvXi1dulStWrXKdDkXLlxQUlKSyx8AAIAkFcirBZ84cUIpKSkKDg52aQ8ODtbOnTsznKZTp046ceKEGjVqJGOMLl++rKeffvqal8bGjBmj4cOHZ2vtAADg9pDng6XdsXr1ao0ePVpTp07Vpk2b9Nlnn2nJkiUaMWJEptMMHDhQiYmJ1t+hQ4dysWIAAHAry7MzQkFBQfL09FRCQoJLe0JCgkJCQjKc5tVXX1Xnzp3Vo0cPSVKNGjWUnJysJ598UoMGDZKHR/pc53Q65XQ6s38FAABAvpdnZ4S8vLwUFRWluLg4qy01NVVxcXGqX79+htOcPXs2Xdjx9PSUJBljcq5YAABwW8qzM0KS1K9fP3Xp0kV16tRR3bp1NXHiRCUnJ6tbt26SpNjYWJUqVUpjxoyRJLVu3Vrjx4/XHXfcoXr16mn37t169dVX1bp1aysQAQAAZFWeBqH27dvr+PHjGjJkiOLj41W7dm0tW7bMGkB98OBBlzNAgwcPlsPh0ODBg3X48GEVL15crVu31qhRo/JqFQAAQD7mMDa7ppSUlKSAgAAlJibK398/r8sBAABZkFOf3/nqW2MAAADZiSAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsiyAEAABsK8+D0JQpUxQeHi5vb2/Vq1dP69evv2b/U6dOqVevXgoNDZXT6VSlSpW0dOnSXKoWAADcTgrk5cLnz5+vfv36adq0aapXr54mTpyomJgY7dq1SyVKlEjX/+LFi2rRooVKlCihBQsWqFSpUjpw4IACAwNzv3gAAJDvOYwxJq8WXq9ePd15552aPHmyJCk1NVVhYWHq3bu3BgwYkK7/tGnT9Oabb2rnzp0qWLDgDS0zKSlJAQEBSkxMlL+//03VDwAAckdOfX7n2aWxixcvauPGjYqOjv6/Yjw8FB0drXXr1mU4zeLFi1W/fn316tVLwcHBql69ukaPHq2UlJRMl3PhwgUlJSW5/AEAAEh5GIROnDihlJQUBQcHu7QHBwcrPj4+w2n27t2rBQsWKCUlRUuXLtWrr76qt956SyNHjsx0OWPGjFFAQID1FxYWlq3rAQAA8q88HyztjtTUVJUoUUIzZsxQVFSU2rdvr0GDBmnatGmZTjNw4EAlJiZaf4cOHcrFigEAwK0szwZLBwUFydPTUwkJCS7tCQkJCgkJyXCa0NBQFSxYUJ6enlZb1apVFR8fr4sXL8rLyyvdNE6nU06nM3uLBwAAt4UbOiP0/fff6/HHH1f9+vV1+PBhSdLcuXO1Zs2aLM/Dy8tLUVFRiouLs9pSU1MVFxen+vXrZzhNw4YNtXv3bqWmplptv//+u0JDQzMMQQAAANfidhBauHChYmJi5OPjo82bN+vChQuSpMTERI0ePdqtefXr108zZ87UnDlztGPHDj3zzDNKTk5Wt27dJEmxsbEaOHCg1f+ZZ57RyZMn1adPH/3+++9asmSJRo8erV69erm7GgAAAO5fGhs5cqSmTZum2NhYzZs3z2pv2LDhNQctZ6R9+/Y6fvy4hgwZovj4eNWuXVvLli2zBlAfPHhQHh7/l9XCwsK0fPly9e3bVzVr1lSpUqXUp08fvfzyy+6uBgAAgPv3EfL19dX27dsVHh6uwoULa+vWrSpfvrz27t2ryMhInT9/PqdqzRbcRwgAgPznlrmPUEhIiHbv3p2ufc2aNSpfvny2FAUAAJAb3A5CPXv2VJ8+ffTTTz/J4XDoyJEj+vDDD9W/f38988wzOVEjAABAjnB7jNCAAQOUmpqq5s2b6+zZs7rrrrvkdDrVv39/9e7dOydqBAAAyBFujRFKSUnR2rVrVbNmTfn6+mr37t06c+aMIiMj5efnl5N1ZhvGCAEAkP/k1Oe3W2eEPD09dc8992jHjh0KDAxUZGRkthUCAACQ29weI1S9enXt3bs3J2oBAADIVW4HoZEjR6p///768ssvdfToUX7ZHQAA5Ftu30foyhscOhwO69/GGDkcDqWkpGRfdTmAMUIAAOQ/t8QYIUlatWpVti0cAAAgL7kdhJo0aZITdQAAAOQ6t4OQJJ06dUrvvfeeduzYIUmqVq2annjiCQUEBGRrcQAAADnJ7cHSGzZsUEREhCZMmKCTJ0/q5MmTGj9+vCIiIrRp06acqBEAACBHuD1YunHjxqpQoYJmzpypAgX+PqF0+fJl9ejRQ3v37tV3332XI4VmFwZLAwCQ/+TU57fbQcjHx0ebN29WlSpVXNq3b9+uOnXq6OzZs9lWXE4gCAEAkP/cMr8+7+/vr4MHD6ZrP3TokAoXLpwtRQEAAOQGt4NQ+/bt1b17d82fP1+HDh3SoUOHNG/ePPXo0UMdO3bMiRoBAAByhNvfGhs3bpwcDodiY2N1+fJlSVLBggX1zDPPaOzYsdleIAAAQE5xe4xQmrNnz2rPnj2SpIiICPn6+mZrYTmFMUIAAOQ/t8ydpRMTE5WSkqKiRYuqRo0aVvvJkydVoEABwgUAAMg33B4j1KFDB82bNy9d+yeffKIOHTpkS1EAAAC5we0g9NNPP6lp06bp2u+++2799NNP2VIUAABAbnA7CF24cMEaJH2lS5cu6dy5c9lSFAAAQG5wOwjVrVtXM2bMSNc+bdo0RUVFZUtRAAAAucHtwdIjR45UdHS0tm7dqubNm0uS4uLi9PPPP2vFihXZXiAAAEBOcfuMUMOGDbVu3TqFhYXpk08+0RdffKEKFSrol19+UePGjXOiRgAAgBxxw/cRyq+4jxAAAPlPnt9H6PLly0pJSZHT6bTaEhISNG3aNCUnJ+uBBx5Qo0aNsq0wAACAnJblINSzZ095eXlp+vTpkqTTp0/rzjvv1Pnz5xUaGqoJEybov//9r1q1apVjxQIAAGSnLI8RWrt2rR555BHr8QcffKCUlBT98ccf2rp1q/r166c333wzR4oEAADICVkOQocPH1bFihWtx3FxcXrkkUcUEBAgSerSpYt+++237K8QAAAgh2Q5CHl7e7vcMPHHH39UvXr1XJ4/c+ZM9lYHAACQg7IchGrXrq25c+dKkr7//nslJCSoWbNm1vN79uxRyZIls79CAACAHJLlwdJDhgxRy5Yt9cknn+jo0aPq2rWrQkNDrecXLVqkhg0b5kiRAAAAOSHLQahJkybauHGjVqxYoZCQED366KMuz9euXVt169bN9gIBAAByCjdUBAAAt7yc+vx2+yc2AAAAbhcEIQAAYFsEIQAAYFsEIQAAYFsEIQAAYFtZ/vp8miJFisjhcKRrdzgc8vb2VoUKFdS1a1d169YtWwoEAADIKW4HoSFDhmjUqFFq2bKldd+g9evXa9myZerVq5f27dunZ555RpcvX1bPnj2zvWAAAIDs4nYQWrNmjUaOHKmnn37apX369OlasWKFFi5cqJo1a+rf//43QQgAANzS3B4jtHz5ckVHR6drb968uZYvXy5JatWqlfbu3Xvz1QEAAOQgt4NQ0aJF9cUXX6Rr/+KLL1S0aFFJUnJysgoXLnzz1QEAAOQgty+Nvfrqq3rmmWe0atUqa4zQzz//rKVLl2ratGmSpJUrV6pJkybZWykAAEA2u6HfGlu7dq0mT56sXbt2SZIqV66s3r17q0GDBtleYHbjt8YAAMh/curzmx9dBQAAt7yc+vx2+9JYUlJShu0Oh0NOp1NeXl43XRQAAEBucDsIBQYGZnhDxTSlS5dW165dNXToUHl4cONqAABw63I7CM2ePVuDBg1S165dXW6oOGfOHA0ePFjHjx/XuHHj5HQ69corr2R7wQAAANnF7SA0Z84cvfXWW2rXrp3V1rp1a9WoUUPTp09XXFycypQpo1GjRhGEAADALc3ta1c//PCD7rjjjnTtd9xxh9atWydJatSokQ4ePHjz1QEAAOQgt4NQWFiY3nvvvXTt7733nsLCwiRJf/75p4oUKXLz1QEAAOQgty+NjRs3To8++qi++uor3XnnnZKkDRs2aOfOnVqwYIGkv2+w2L59++ytFAAAIJvd0H2E9u/fr+nTp7vcUPGpp55SeHh4dteX7biPEAAA+U++uKHir7/+qurVq2fX7HIEQQgAgPwnpz6/b/pGP6dPn9aMGTNUt25d1apVKztqAgAAyBU3HIS+++47denSRaGhoRo3bpyaNWumH3/8MTtrAwAAyFFuDZaOj4/X7Nmz9d577ykpKUnt2rXThQsX9PnnnysyMjKnagQAAMgRWT4j1Lp1a1WuXFm//PKLJk6cqCNHjmjSpEk5WRsAAECOyvIZoa+++krPP/+8nnnmGVWsWDEnawIAAMgVWT4jtGbNGp0+fVpRUVGqV6+eJk+erBMnTuRkbQAAADkqy0Hon//8p2bOnKmjR4/qqaee0rx581SyZEmlpqZq5cqVOn369A0XMWXKFIWHh8vb21v16tXT+vXrszTdvHnz5HA41KZNmxteNgAAsC+3vzVWqFAhPfHEE1qzZo22bdumf/3rXxo7dqxKlCihBx54wO0C5s+fr379+mno0KHatGmTatWqpZiYGB07duya0+3fv1/9+/dX48aN3V4mAACAdJP3EapcubLeeOMN/e9//9PHH398Q/MYP368evbsqW7duikyMlLTpk2Tr6+v3n///UynSUlJ0WOPPabhw4erfPnyN1o+AACwuZu+oaIkeXp6qk2bNlq8eLFb0128eFEbN25UdHT0/xXk4aHo6Gjrl+wz8tprr6lEiRLq3r37dZdx4cIFJSUlufwBAABI2RSEbtSJEyeUkpKi4OBgl/bg4GDFx8dnOM2aNWv03nvvaebMmVlaxpgxYxQQEGD9hYWF3XTdAADg9pCnQchdp0+fVufOnTVz5kwFBQVlaZqBAwcqMTHR+jt06FAOVwkAAPILt+4snd2CgoLk6emphIQEl/aEhASFhISk679nzx7t379frVu3ttpSU1MlSQUKFNCuXbsUERHhMo3T6ZTT6cyB6gEAQH6Xp2eEvLy8FBUVpbi4OKstNTVVcXFxql+/frr+VapU0bZt27Rlyxbr74EHHlDTpk21ZcsWLnsBAAC35OkZIUnq16+funTpojp16qhu3bqaOHGikpOT1a1bN0lSbGysSpUqpTFjxsjb21vVq1d3mT4wMFCS0rUDAABcT54Hofbt2+v48eMaMmSI4uPjVbt2bS1btswaQH3w4EF5eOSroUwAACCfcBhjTF4XkZuSkpIUEBCgxMRE+fv753U5AAAgC3Lq85tTLQAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLYIQgAAwLZuiSA0ZcoUhYeHy9vbW/Xq1dP69esz7Ttz5kw1btxYRYoUUZEiRRQdHX3N/gAAAJnJ8yA0f/589evXT0OHDtWmTZtUq1YtxcTE6NixYxn2X716tTp27KhVq1Zp3bp1CgsL0z333KPDhw/ncuUAACC/cxhjTF4WUK9ePd15552aPHmyJCk1NVVhYWHq3bu3BgwYcN3pU1JSVKRIEU2ePFmxsbHX7Z+UlKSAgAAlJibK39//pusHAAA5L6c+v/P0jNDFixe1ceNGRUdHW20eHh6Kjo7WunXrsjSPs2fP6tKlSypatGiGz1+4cEFJSUkufwAAAFIeB6ETJ04oJSVFwcHBLu3BwcGKj4/P0jxefvlllSxZ0iVMXWnMmDEKCAiw/sLCwm66bgAAcHvI8zFCN2Ps2LGaN2+eFi1aJG9v7wz7DBw4UImJidbfoUOHcrlKAABwqyqQlwsPCgqSp6enEhISXNoTEhIUEhJyzWnHjRunsWPH6uuvv1bNmjUz7ed0OuV0OrOlXgAAcHvJ0zNCXl5eioqKUlxcnNWWmpqquLg41a9fP9Pp3njjDY0YMULLli1TnTp1cqNUAABwG8rTM0KS1K9fP3Xp0kV16tRR3bp1NXHiRCUnJ6tbt26SpNjYWJUqVUpjxoyRJL3++usaMmSIPvroI4WHh1tjifz8/OTn55dn6wEAAPKfPA9C7du31/HjxzVkyBDFx8erdu3aWrZsmTWA+uDBg/Lw+L8TV++8844uXryotm3busxn6NChGjZsWG6WDgAA8rk8v49QbuM+QgAA5D+35X2EAAAA8hJBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2BZBCAAA2NYtEYSmTJmi8PBweXt7q169elq/fv01+3/66aeqUqWKvL29VaNGDS1dujSXKgUAALeTPA9C8+fPV79+/TR06FBt2rRJtWrVUkxMjI4dO5Zh/x9++EEdO3ZU9+7dtXnzZrVp00Zt2rTRr7/+msuVAwCA/M5hjDF5WUC9evV05513avLkyZKk1NRUhYWFqXfv3howYEC6/u3bt1dycrK+/PJLq+2f//ynateurWnTpl13eUlJSQoICFBiYqL8/f2zb0UAAECOyanP7zw9I3Tx4kVt3LhR0dHRVpuHh4eio6O1bt26DKdZt26dS39JiomJybQ/AABAZgrk5cJPnDihlJQUBQcHu7QHBwdr586dGU4THx+fYf/4+PgM+1+4cEEXLlywHicmJkr6O1kCAID8Ie1zO7svZOVpEMoNY8aM0fDhw9O1h4WF5UE1AADgZvz5558KCAjItvnlaRAKCgqSp6enEhISXNoTEhIUEhKS4TQhISFu9R84cKD69etnPT516pTKli2rgwcPZuuGhPuSkpIUFhamQ4cOMV7rFsD+uHWwL24d7ItbR2JiosqUKaOiRYtm63zzNAh5eXkpKipKcXFxatOmjaS/B0vHxcXpueeey3Ca+vXrKy4uTi+88ILVtnLlStWvXz/D/k6nU06nM117QEAAL+pbhL+/P/viFsL+uHWwL24d7Itbh4dH9g5vzvNLY/369VOXLl1Up04d1a1bVxMnTlRycrK6desmSYqNjVWpUqU0ZswYSVKfPn3UpEkTvfXWW7rvvvs0b948bdiwQTNmzMjL1QAAAPlQngeh9u3b6/jx4xoyZIji4+NVu3ZtLVu2zBoQffDgQZf016BBA3300UcaPHiwXnnlFVWsWFGff/65qlevnlerAAAA8qk8D0KS9Nxzz2V6KWz16tXp2h599FE9+uijN7Qsp9OpoUOHZni5DLmLfXFrYX/cOtgXtw72xa0jp/ZFnt9QEQAAIK/k+U9sAAAA5BWCEAAAsC2CEAAAsC2CEAAAsK3bMghNmTJF4eHh8vb2Vr169bR+/fpr9v/0009VpUoVeXt7q0aNGlq6dGkuVXr7c2dfzJw5U40bN1aRIkVUpEgRRUdHX3ffwT3uHhtp5s2bJ4fDYd34FDfP3X1x6tQp9erVS6GhoXI6napUqRLvVdnE3X0xceJEVa5cWT4+PgoLC1Pfvn11/vz5XKr29vXdd9+pdevWKlmypBwOhz7//PPrTrN69Wr94x//kNPpVIUKFTR79mz3F2xuM/PmzTNeXl7m/fffN7/99pvp2bOnCQwMNAkJCRn2X7t2rfH09DRvvPGG2b59uxk8eLApWLCg2bZtWy5Xfvtxd1906tTJTJkyxWzevNns2LHDdO3a1QQEBJj//e9/uVz57cnd/ZFm3759plSpUqZx48bmwQcfzJ1ib3Pu7osLFy6YOnXqmFatWpk1a9aYffv2mdWrV5stW7bkcuW3H3f3xYcffmicTqf58MMPzb59+8zy5ctNaGio6du3by5XfvtZunSpGTRokPnss8+MJLNo0aJr9t+7d6/x9fU1/fr1M9u3bzeTJk0ynp6eZtmyZW4t97YLQnXr1jW9evWyHqekpJiSJUuaMWPGZNi/Xbt25r777nNpq1evnnnqqadytE47cHdfXO3y5cumcOHCZs6cOTlVoq3cyP64fPmyadCggXn33XdNly5dCELZxN198c4775jy5cubixcv5laJtuHuvujVq5dp1qyZS1u/fv1Mw4YNc7ROu8lKEHrppZdMtWrVXNrat29vYmJi3FrWbXVp7OLFi9q4caOio6OtNg8PD0VHR2vdunUZTrNu3TqX/pIUExOTaX9kzY3si6udPXtWly5dyvYf2LOjG90fr732mkqUKKHu3bvnRpm2cCP7YvHixapfv7569eql4OBgVa9eXaNHj1ZKSkpulX1bupF90aBBA23cuNG6fLZ3714tXbpUrVq1ypWa8X+y6/P7lrizdHY5ceKEUlJSrJ/nSBMcHKydO3dmOE18fHyG/ePj43OsTju4kX1xtZdfflklS5ZM90KH+25kf6xZs0bvvfeetmzZkgsV2seN7Iu9e/fqm2++0WOPPaalS5dq9+7devbZZ3Xp0iUNHTo0N8q+Ld3IvujUqZNOnDihRo0ayRijy5cv6+mnn9Yrr7ySGyXjCpl9ficlJencuXPy8fHJ0nxuqzNCuH2MHTtW8+bN06JFi+Tt7Z3X5djO6dOn1blzZ82cOVNBQUF5XY7tpaamqkSJEpoxY4aioqLUvn17DRo0SNOmTcvr0mxn9erVGj16tKZOnapNmzbps88+05IlSzRixIi8Lg036LY6IxQUFCRPT08lJCS4tCckJCgkJCTDaUJCQtzqj6y5kX2RZty4cRo7dqy+/vpr1axZMyfLtA1398eePXu0f/9+tW7d2mpLTU2VJBUoUEC7du1SREREzhZ9m7qRYyM0NFQFCxaUp6en1Va1alXFx8fr4sWL8vLyytGab1c3si9effVVde7cWT169JAk1ahRQ8nJyXryySc1aNAglx8JR87K7PPb398/y2eDpNvsjJCXl5eioqIUFxdntaWmpiouLk7169fPcJr69eu79JeklStXZtofWXMj+0KS3njjDY0YMULLli1TnTp1cqNUW3B3f1SpUkXbtm3Tli1brL8HHnhATZs21ZYtWxQWFpab5d9WbuTYaNiwoXbv3m2FUUn6/fffFRoaSgi6CTeyL86ePZsu7KQFVMNPd+aqbPv8dm8c961v3rx5xul0mtmzZ5vt27ebJ5980gQGBpr4+HhjjDGdO3c2AwYMsPqvXbvWFChQwIwbN87s2LHDDB06lK/PZxN398XYsWONl5eXWbBggTl69Kj1d/r06bxahduKu/vjanxrLPu4uy8OHjxoChcubJ577jmza9cu8+WXX5oSJUqYkSNH5tUq3Dbc3RdDhw41hQsXNh9//LHZu3evWbFihYmIiDDt2rXLq1W4bZw+fdps3rzZbN682Ugy48ePN5s3bzYHDhwwxhgzYMAA07lzZ6t/2tfnX3zxRbNjxw4zZcoUvj6fZtKkSaZMmTLGy8vL1K1b1/z444/Wc02aNDFdunRx6f/JJ5+YSpUqGS8vL1OtWjWzZMmSXK749uXOvihbtqyRlO5v6NChuV/4bcrdY+NKBKHs5e6++OGHH0y9evWM0+k05cuXN6NGjTKXL1/O5apvT+7si0uXLplhw4aZiIgI4+3tbcLCwsyzzz5r/vrrr9wv/DazatWqDD8D0rZ/ly5dTJMmTdJNU7t2bePl5WXKly9vZs2a5fZyHcZwLg8AANjTbTVGCAAAwB0EIQAAYFsEIQAAYFsEIQAAYFsEIQAAYFsEIQAAYFsEIQAAYFsEIeA2FR4erokTJ2a5/+rVq+VwOHTq1Kkcq+lmzJ49W4GBgXldRjpxcXGqWrWqUlJSJEnDhg1T7dq1rzlN165d1aZNG+vx3XffrRdeeMF6fPW+czgc+vzzz7Ov6Ovo0KGD3nrrrVxbHpCXCEJAHnM4HNf8GzZs2A3N9+eff9aTTz6Z5f4NGjTQ0aNHFRAQcEPLy6pbNdDcqJdeekmDBw92+UHU63n77bc1e/bsLPc/evSoWrZseQPV3ZjBgwdr1KhRSkxMzLVlAnnltvr1eSA/Onr0qPXv+fPna8iQIdq1a5fV5ufnZ/3bGKOUlBQVKHD9Q7d48eJu1eHl5ZXpL24jY2vWrNGePXv0yCOPuDWdu2Ezt/dL9erVFRERof/85z/q1atXri4byG2cEQLyWEhIiPUXEBAgh8NhPd65c6cKFy6sr776SlFRUXI6ndaH74MPPqjg4GD5+fnpzjvv1Ndff+0y34wur7z77rt66KGH5Ovrq4oVK2rx4sXW81dfGks7c7N8+XJVrVpVfn5+uvfee12C2+XLl/X8888rMDBQxYoV08svv6wuXbq4XPa50urVq9WtWzclJiamO+P1119/KTY2VkWKFJGvr69atmypP/74I9Ptdvz4cdWpU0cPPfSQLly4oNTUVI0ZM0blypWTj4+PatWqpQULFqRbv7i4ONWpU0e+vr5q0KCBS+jcunWrmjZtqsKFC8vf319RUVHasGFDpjXMmzdPLVq0kLe3d7rnpk+frrCwMPn6+qpdu3YuZ1euvjR2PVdfGtu2bZuaNWsmHx8fFStWTE8++aTOnDmTbv7jxo1TaGioihUrpl69eunSpUtWn6lTp6pixYry9vZWcHCw2rZt67LM1q1ba968eVmuEcivCEJAPjBgwACNHTtWO3bsUM2aNXXmzBm1atVKcXFx2rx5s+699161bt1aBw8evOZ8hg8frnbt2umXX35Rq1at9Nhjj+nkyZOZ9j979qzGjRunuXPn6rvvvtPBgwfVv39/6/nXX39dH374oWbNmqW1a9cqKSnpmmNZGjRooIkTJ8rf319Hjx7V0aNHrfl17dpVGzZs0OLFi7Vu3ToZY9SqVSuXD+80hw4dUuPGjVW9enUtWLBATqdTY8aM0QcffKBp06bpt99+U9++ffX444/r22+/dZl20KBBeuutt7RhwwYVKFBATzzxhPXcY489ptKlS+vnn3/Wxo0bNWDAABUsWDDT9fn+++9Vp06ddO27d+/WJ598oi+++ELLli3T5s2b9eyzz2Y6H3ckJycrJiZGRYoU0c8//6xPP/1UX3/9tZ577jmXfqtWrdKePXu0atUqzZkzR7Nnz7Yux23YsEHPP/+8XnvtNe3atUvLli3TXXfd5TJ93bp1tX79el24cCFb6gZuWTf3W7EAstOsWbNMQECA9Tjt15g///zz605brVo1M2nSJOtx2bJlzYQJE6zHkszgwYOtx2fOnDGSzFdffeWyrLRf0Z41a5aRZHbv3m1NM2XKFBMcHGw9Dg4ONm+++ab1+PLly6ZMmTLX/JX6q9fRGGN+//13I8msXbvWajtx4oTx8fExn3zyict0O3fuNGFhYeb55583qampxhhjzp8/b3x9fc0PP/zgMt/u3bubjh07uqzf119/bT2/ZMkSI8mcO3fOGGNM4cKFzezZszOt/WoBAQHmgw8+cGkbOnSo8fT0NP/73/+stq+++sp4eHiYo0ePGmP+/hXtK7dRkyZNTJ8+fazHGe27RYsWGWOMmTFjhilSpIg5c+aMy3p4eHiY+Ph4a/5ly5Z1+XX6Rx991LRv394YY8zChQuNv7+/SUpKynTdtm7daiSZ/fv3Z21jAPkUZ4SAfODqsw5nzpxR//79VbVqVQUGBsrPz087duy47hmhmjVrWv8uVKiQ/P39dezYsUz7+/r6KiIiwnocGhpq9U9MTFRCQoLq1q1rPe/p6amoqCi31k2SduzYoQIFCqhevXpWW7FixVS5cmXt2LHDajt37pwaN26shx9+WG+//bYcDoekv8/AnD17Vi1atJCfn5/198EHH2jPnj2ZboPQ0FBJstapX79+6tGjh6KjozV27Nh0017t3LlzGV4WK1OmjEqVKmU9rl+/vlJTU10uw92oHTt2qFatWipUqJDV1rBhw3Tzr1atmssA7iv3XYsWLVS2bFmVL19enTt31ocffqizZ8+6LMfHx0eS0rUDtxuCEJAPXPmhJ0n9+/fXokWLNHr0aH3//ffasmWLatSooYsXL15zPldf5nE4HEpNTXWrvzHGzeqzj9PpVHR0tL788ksdPnzYak8bH7NkyRJt2bLF+tu+fbvLOCHJdZ3SglTaNhg2bJh+++033Xffffrmm28UGRmpRYsWZVpPUFCQ/vrrr2xbv+x0rX1duHBhbdq0SR9//LFCQ0M1ZMgQ1apVy+XWCWmXTN0ddA/kNwQhIB9au3atunbtqoceekg1atRQSEiI9u/fn6s1BAQEKDg4WD///LPVlpKSok2bNl1zOi8vL+ueO2mqVq2qy5cv66effrLa/vzzT+3atUuRkZFWm4eHh+bOnauoqCg1bdpUR44ckSRFRkbK6XTq4MGDqlChgstfWFiYW+tVqVIl9e3bVytWrNDDDz+sWbNmZdr3jjvu0Pbt29O1Hzx40KpNkn788Ud5eHiocuXKbtWSkapVq2rr1q1KTk622tauXev2/AsUKKDo6Gi98cYb+uWXX7R//35988031vO//vqrSpcuraCgoJuuGbiVEYSAfKhixYr67LPPtGXLFm3dulWdOnW65pmdnNK7d2+NGTNG//3vf7Vr1y716dNHf/31l3WmJSPh4eE6c+aM4uLidOLECZ09e1YVK1bUgw8+qJ49e2rNmjXaunWrHn/8cZUqVUoPPvigy/Senp768MMPVatWLTVr1kzx8fEqXLiw+vfvr759+2rOnDnas2ePNm3apEmTJmnOnDlZWpdz587pueee0+rVq3XgwAGtXbtWP//8s6pWrZrpNDExMVqzZk26dm9vb3Xp0kVbt27V999/r+eff17t2rXLlq/BP/bYY9b8f/31V61atUq9e/dW586dFRwcnKV5fPnll/r3v/+tLVu26MCBA/rggw+UmprqEqS+//573XPPPTddL3CrIwgB+dD48eNVpEgRNWjQQK1bt1ZMTIz+8Y9/5HodL7/8sjp27KjY2FjVr19ffn5+iomJyXDcTJoGDRro6aefVvv27VW8eHG98cYbkqRZs2YpKipK999/v+rXry9jjJYuXZrht7YKFCigjz/+WNWqVVOzZs107NgxjRgxQq+++qrGjBmjqlWr6t5779WSJUtUrly5LK2Lp6en/vzzT8XGxqpSpUpq166dWrZsqeHDh2c6zWOPPabffvst3difChUq6OGHH1arVq10zz33qGbNmpo6dWqW6rgeX19fLV++XCdPntSdd96ptm3bqnnz5po8eXKW5xEYGKjPPvtMzZo1U9WqVTVt2jRre0rS+fPn9fnnn6tnz57ZUjNwK3OYvLzgD+C2kpqaqqpVq6pdu3YaMWJEXpeTK1588UUlJSVp+vTpeV1KtnnnnXe0aNEirVixIq9LAXIcZ4QA3LADBw5o5syZ+v3337Vt2zY988wz2rdvnzp16pTXpeWaQYMGqWzZsnlyaTKnFCxYUJMmTcrrMoBcwRkhADfs0KFD6tChg3799VcZY1S9enWNHTs23c35AOBWRRACAAC2xaUxAABgWwQhAABgWwQhAABgWwQhAABgWwQhAABgWwQhAABgWwQhAABgWwQhAABgWwQhAABgW/8PqN0VEjG6mPUAAAAASUVORK5CYII=", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "import json\n", "import os\n", "from matplotlib import pyplot as plt\n", "metrics = ['agg_score', 'commonsense_qa/acc_norm', 'hellaswag/acc_norm', 'openbookqa/acc_norm', 'piqa/acc_norm',\n", " 'siqa/acc_norm', 'winogrande/acc_norm', 'arc/acc_norm', 'mmlu/acc_norm']\n", "\n", "def normalize_runname(runname):\n", " return runname.replace(\"/\", \"_\")\n", "\n", "grouped = (\n", " df.groupby([\"runname\", \"steps\"])\n", " .agg(\n", " {\n", " key: \"mean\" for key in metrics\n", " }\n", " )\n", " .reset_index()\n", ")\n", "\n", "file_id=\"../assets/data/plots/removed_data_dedup\"\n", "files = {}\n", "for metric in metrics:\n", " datas = {}\n", " for name, group in grouped.groupby(\"runname\"):\n", " group = group[[\"steps\", metric]].sort_values(by=\"steps\")\n", " group = group.set_index(\"steps\")\n", " rolling_avg = group\n", " # rolling_avg = group.rolling(window=5).mean()\n", " datas[name] = {\n", " \"x\": (rolling_avg.index * 2048 * 1024 * 1e-9).tolist(),\n", " \"y\": rolling_avg[metric].tolist(),\n", " \"label\": runs_mapping[name],\n", " }\n", " # Sort the datata based on the steps\n", " datas = {k: v for k, v in sorted(datas.items(), key=lambda x: -x[1][\"y\"][-1])}\n", " # Create a folder\n", " os.makedirs(f\"{file_id}\", exist_ok=True)\n", " with open(f\"{file_id}/{normalize_runname(metric)}.json\", \"w\") as f:\n", " json.dump({\n", " \"data\": datas,\n", " \"layout\": {\n", " \"title\": {\n", " \"text\": \"The originally removed data outperforms the kept data\"\n", " },\n", " }\n", " }, f)\n", " files[metric] = {\"file\": f\"{normalize_runname(metric)}.json\"}\n", "# Create index\n", "with open(f\"{file_id}/index.json\", \"w\") as f:\n", " json.dump({\n", " \"files\": files,\n", " \"settings\": {\n", " \"defaultMetric\": \"agg_score\",\n", " \"slider\":{\"min\":0,\"max\":10,\"default\":0}\n", " }\n", " }, f)\n", " \n", "\n", "# Add labels and legend\n", "plt.xlabel(\"Training tokens (billions)\")\n", "plt.ylabel(\"Agg Score\")\n", "plt.title(\"The originally removed data outperforms the kept data\")\n", "plt.legend()\n", "\n", "# Show the plot\n", "plt.show()" ] }, { "cell_type": "code", "execution_count": 3, "id": "af28ebbd054cdc33", "metadata": { "ExecuteTime": { "end_time": "2024-04-30T12:52:05.836260Z", "start_time": "2024-04-30T12:52:05.834381Z" }, "collapsed": false }, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.12.2" } }, "nbformat": 4, "nbformat_minor": 5 }