| {"data":{"FineWeb (ours)":{"x":[0.0,2.0971520000000003,4.194304000000001,6.291456,8.388608000000001,10.48576,12.582912,14.680064000000002,16.777216000000003,18.874368,20.97152,23.068672000000003,25.165824,27.262976000000002,29.360128000000003,31.45728,33.554432000000006,35.651584,37.748736,39.845888,41.94304,44.040192000000005,46.137344000000006,48.234496,50.331648,52.4288,54.525952000000004,56.623104000000005,58.720256000000006,60.817408,62.91456,65.011712,67.10886400000001,69.206016,71.303168,73.40032000000001,75.497472,77.59462400000001,79.691776,81.788928,83.88608,85.983232,88.08038400000001,90.177536,92.27468800000001,94.37184,96.468992,98.56614400000001,100.663296,102.76044800000001,104.8576,106.95475200000001,109.05190400000001,111.149056,113.24620800000001,115.34336,117.44051200000001,119.537664,121.634816,123.73196800000001,125.82912,127.92627200000001,130.023424,132.120576,134.21772800000002,136.31488000000002,138.412032,140.509184,142.606336,144.70348800000002,146.80064000000002,148.897792,150.994944,153.092096,155.18924800000002,157.28640000000001,159.383552,161.480704,163.577856,165.67500800000002,167.77216,169.869312,171.966464,174.06361600000002,176.16076800000002,178.25792,180.355072,182.452224,184.54937600000002,186.64652800000002,188.74368,190.840832,192.937984,195.03513600000002,197.13228800000002,199.22944,201.326592,203.423744,205.52089600000002,207.61804800000002,209.7152,211.812352,213.90950400000003,216.00665600000002,218.10380800000001,220.20096,222.298112,224.39526400000003,226.49241600000002,228.589568,230.68672,232.783872,234.88102400000002,236.97817600000002,239.075328,241.17248,243.269632,245.36678400000002,247.46393600000002,249.561088,251.65824,253.75539200000003,255.85254400000002,257.949696,260.046848,262.144,264.241152,266.338304,268.43545600000004,270.53260800000004,272.62976000000003,274.726912,276.824064,278.921216,281.018368,283.11552,285.212672,287.309824,289.40697600000004,291.50412800000004,293.60128000000003,295.698432,297.795584,299.892736,301.989888,304.08704,306.184192,308.28134400000005,310.37849600000004,312.47564800000004,314.57280000000003,316.669952,318.767104,320.864256,322.961408,325.05856,327.155712,329.25286400000005,331.35001600000004,333.44716800000003,335.54432,337.641472,339.738624,341.835776,343.932928,346.03008,348.12723200000005,350.22438400000004],"y":[0.2509999871253967,0.296999990940094,0.3219999969005584,0.3305000066757202,0.3555000126361847,0.351500004529953,0.3600000143051147,0.363999992609024,0.3680000007152557,0.3785000145435333,0.3765000104904175,0.382999986410141,0.3785000145435333,0.3835000097751617,0.3819999992847442,0.3935000002384186,0.387499988079071,0.3935000002384186,0.3959999978542328,0.3860000073909759,0.3935000002384186,0.3885000050067901,0.3810000121593475,0.3880000114440918,0.3964999914169311,0.4054999947547912,0.3935000002384186,0.3944999873638153,0.3989999890327453,0.3980000019073486,0.4050000011920929,0.4054999947547912,0.4009999930858612,0.4110000133514404,0.4054999947547912,0.4180000126361847,0.4110000133514404,0.4050000011920929,0.4079999923706054,0.4120000004768371,0.402999997138977,0.4205000102519989,0.4129999876022339,0.4120000004768371,0.4169999957084656,0.4269999861717224,0.4230000078678131,0.4225000143051147,0.4300000071525574,0.4180000126361847,0.4284999966621399,0.4165000021457672,0.4325000047683716,0.4235000014305115,0.4210000038146972,0.4239999949932098,0.4235000014305115,0.421999990940094,0.4280000030994415,0.4300000071525574,0.4275000095367431,0.4305000007152557,0.4244999885559082,0.4314999878406524,0.4325000047683716,0.4395000040531158,0.4325000047683716,0.4300000071525574,0.4399999976158142,0.4320000112056732,0.4370000064373016,0.4280000030994415,0.4309999942779541,0.4314999878406524,0.4370000064373016,0.4280000030994415,0.4325000047683716,0.4300000071525574,0.4334999918937683,0.4334999918937683,0.4379999935626983,0.4399999976158142,0.4350000023841858,0.4395000040531158,0.4375,0.4390000104904175,0.4365000128746032,0.4435000121593475,0.4365000128746032,0.445499986410141,0.4440000057220459,0.4460000097751617,0.4415000081062317,0.4415000081062317,0.4339999854564667,0.4429999887943268,0.4399999976158142,0.4359999895095825,0.4370000064373016,0.4469999969005584,0.4404999911785126,0.4435000121593475,0.445499986410141,0.4424999952316284,0.4480000138282776,0.4370000064373016,0.4444999992847442,0.4465000033378601,0.4309999942779541,0.4440000057220459,0.4469999969005584,0.4539999961853027,0.4440000057220459,0.4555000066757202,0.4519999921321869,0.4510000050067901,0.4519999921321869,0.4544999897480011,0.4494999945163727,0.4584999978542328,0.4580000042915344,0.4544999897480011,0.4514999985694885,0.4550000131130218,0.4560000002384186,0.4600000083446502,0.4589999914169311,0.4560000002384186,0.457500010728836,0.4679999947547912,0.4494999945163727,0.4505000114440918,0.4440000057220459,0.4539999961853027,0.4535000026226043,0.4514999985694885,0.457500010728836,0.4620000123977661,0.4564999938011169,0.4595000147819519,0.4564999938011169,0.4550000131130218,0.4539999961853027,0.4544999897480011,0.4569999873638153,0.457500010728836,0.4539999961853027,0.4595000147819519,0.4665000140666961,0.465499997138977,0.4625000059604645,0.4629999995231628,0.4580000042915344,0.4569999873638153,0.4620000123977661,0.457500010728836,0.4550000131130218,0.4645000100135803,0.4629999995231628,0.4584999978542328,0.465499997138977,0.460999995470047,0.4634999930858612,0.4605000019073486,0.4584999978542328,0.4550000131130218,0.4564999938011169,0.4600000083446502],"label":"FineWeb (ours)"},"C4":{"x":[0.0,2.0971520000000003,4.194304000000001,6.291456,8.388608000000001,10.48576,12.582912,14.680064000000002,16.777216000000003,18.874368,20.97152,23.068672000000003,25.165824,27.262976000000002,29.360128000000003,31.45728,33.554432000000006,35.651584,37.748736,39.845888,41.94304,44.040192000000005,46.137344000000006,48.234496,50.331648,52.4288,54.525952000000004,56.623104000000005,58.720256000000006,60.817408,62.91456,65.011712,67.10886400000001,69.206016,71.303168,73.40032000000001,75.497472,77.59462400000001,79.691776,81.788928,83.88608,85.983232,88.08038400000001,90.177536,92.27468800000001,94.37184,96.468992,98.56614400000001,100.663296,102.76044800000001,104.8576,106.95475200000001,109.05190400000001,111.149056,113.24620800000001,115.34336,117.44051200000001,119.537664,121.634816,123.73196800000001,125.82912,127.92627200000001,130.023424,132.120576,134.21772800000002,136.31488000000002,138.412032,140.509184,142.606336,144.70348800000002,146.80064000000002,148.897792,150.994944,153.092096,155.18924800000002,157.28640000000001,159.383552,161.480704,163.577856,165.67500800000002,167.77216,169.869312,171.966464,174.06361600000002,176.16076800000002,178.25792,180.355072,182.452224,184.54937600000002,186.64652800000002,188.74368,190.840832,192.937984,195.03513600000002,197.13228800000002,199.22944,201.326592,203.423744,205.52089600000002,207.61804800000002,209.7152,211.812352,213.90950400000003,216.00665600000002,218.10380800000001,220.20096,222.298112,224.39526400000003,226.49241600000002,228.589568,230.68672,232.783872,234.88102400000002,236.97817600000002,239.075328,241.17248,243.269632,245.36678400000002,247.46393600000002,249.561088,251.65824,253.75539200000003,255.85254400000002,257.949696,260.046848,262.144,264.241152,266.338304,268.43545600000004,270.53260800000004,272.62976000000003,274.726912,276.824064,278.921216,281.018368,283.11552,285.212672,287.309824,289.40697600000004,291.50412800000004,293.60128000000003,295.698432,297.795584,299.892736,301.989888,304.08704,306.184192,308.28134400000005,310.37849600000004,312.47564800000004,314.57280000000003,316.669952,318.767104,320.864256,322.961408,325.05856,327.155712,329.25286400000005,331.35001600000004,333.44716800000003,335.54432,337.641472,339.738624,341.835776,343.932928,346.03008,348.12723200000005,350.22438400000004],"y":[0.2509999871253967,0.293500006198883,0.3230000138282776,0.3339999914169311,0.3470000028610229,0.3614999949932098,0.3555000126361847,0.3655000030994415,0.3605000078678131,0.359499990940094,0.3580000102519989,0.3680000007152557,0.375,0.3790000081062317,0.3790000081062317,0.3880000114440918,0.3860000073909759,0.3914999961853027,0.3835000097751617,0.3980000019073486,0.3759999871253967,0.3889999985694885,0.3944999873638153,0.3869999945163727,0.3910000026226043,0.3840000033378601,0.3894999921321869,0.402999997138977,0.3930000066757202,0.390500009059906,0.3899999856948852,0.395000010728836,0.4040000140666961,0.3860000073909759,0.4025000035762787,0.398499995470047,0.4020000100135803,0.4004999995231628,0.3939999938011169,0.402999997138977,0.3995000123977661,0.4040000140666961,0.3970000147819519,0.3975000083446502,0.4000000059604645,0.4034999907016754,0.4104999899864197,0.398499995470047,0.4074999988079071,0.4045000076293945,0.414000004529953,0.4095000028610229,0.418500006198883,0.4050000011920929,0.4025000035762787,0.418500006198883,0.4230000078678131,0.4090000092983246,0.402999997138977,0.4149999916553497,0.4120000004768371,0.4194999933242798,0.4104999899864197,0.4074999988079071,0.4104999899864197,0.4165000021457672,0.4189999997615814,0.4070000052452087,0.4135000109672546,0.4250000119209289,0.4235000014305115,0.4269999861717224,0.4214999973773956,0.4320000112056732,0.4189999997615814,0.4255000054836273,0.429500013589859,0.4250000119209289,0.4189999997615814,0.4280000030994415,0.4269999861717224,0.4370000064373016,0.4225000143051147,0.4239999949932098,0.4314999878406524,0.4415000081062317,0.4239999949932098,0.4239999949932098,0.4329999983310699,0.4325000047683716,0.4305000007152557,0.4305000007152557,0.4305000007152557,0.4345000088214874,0.4305000007152557,0.429500013589859,0.4309999942779541,0.4354999959468841,0.4345000088214874,0.4354999959468841,0.4390000104904175,0.4354999959468841,0.4289999902248382,0.4384999871253967,0.4259999990463257,0.4300000071525574,0.421999990940094,0.4395000040531158,0.4334999918937683,0.4350000023841858,0.4329999983310699,0.4339999854564667,0.4415000081062317,0.4469999969005584,0.4494999945163727,0.4345000088214874,0.4375,0.4359999895095825,0.4284999966621399,0.4305000007152557,0.4449999928474426,0.4390000104904175,0.4334999918937683,0.4375,0.4435000121593475,0.4370000064373016,0.4444999992847442,0.4395000040531158,0.4435000121593475,0.445499986410141,0.4449999928474426,0.4399999976158142,0.4490000009536743,0.4415000081062317,0.4429999887943268,0.4339999854564667,0.4429999887943268,0.4309999942779541,0.4429999887943268,0.4384999871253967,0.4449999928474426,0.4390000104904175,0.4429999887943268,0.4435000121593475,0.4435000121593475,0.44200000166893,0.44200000166893,0.4435000121593475,0.44200000166893,0.4485000073909759,0.4440000057220459,0.4485000073909759,0.44200000166893,0.4490000009536743,0.4494999945163727,0.44200000166893,0.4465000033378601,0.4365000128746032,0.44200000166893,0.4384999871253967,0.4440000057220459,0.4390000104904175,0.4399999976158142,0.4444999992847442,0.4449999928474426,0.4440000057220459,0.4469999969005584,0.4435000121593475],"label":"C4"},"Dolma":{"x":[0.0,2.0971520000000003,4.194304000000001,6.291456,8.388608000000001,10.48576,12.582912,14.680064000000002,16.777216000000003,18.874368,20.97152,23.068672000000003,25.165824,27.262976000000002,29.360128000000003,31.45728,33.554432000000006,35.651584,37.748736,39.845888,41.94304,44.040192000000005,46.137344000000006,48.234496,50.331648,52.4288,54.525952000000004,56.623104000000005,58.720256000000006,60.817408,62.91456,65.011712,67.10886400000001,69.206016,71.303168,73.40032000000001,75.497472,77.59462400000001,79.691776,81.788928,83.88608,85.983232,88.08038400000001,90.177536,92.27468800000001,94.37184,96.468992,98.56614400000001,100.663296,102.76044800000001,104.8576,106.95475200000001,109.05190400000001,111.149056,113.24620800000001,115.34336,117.44051200000001,119.537664,121.634816,123.73196800000001,125.82912,127.92627200000001,130.023424,132.120576,134.21772800000002,136.31488000000002,138.412032,140.509184,142.606336,144.70348800000002,146.80064000000002,148.897792,150.994944,153.092096,155.18924800000002,157.28640000000001,159.383552,161.480704,163.577856,165.67500800000002,167.77216,169.869312,171.966464,174.06361600000002,176.16076800000002,178.25792,180.355072,182.452224,184.54937600000002,186.64652800000002,188.74368,190.840832,192.937984,195.03513600000002,197.13228800000002,199.22944,201.326592,203.423744,205.52089600000002,207.61804800000002,209.7152,211.812352,213.90950400000003,216.00665600000002,218.10380800000001,220.20096,222.298112,224.39526400000003,226.49241600000002,228.589568,230.68672,232.783872,234.88102400000002,236.97817600000002,239.075328,241.17248,243.269632,245.36678400000002,247.46393600000002,249.561088,251.65824,253.75539200000003,255.85254400000002,257.949696,260.046848,262.144,264.241152,266.338304,268.43545600000004,270.53260800000004,272.62976000000003,274.726912,276.824064,278.921216,281.018368,283.11552,285.212672,287.309824,289.40697600000004,291.50412800000004,293.60128000000003,295.698432,297.795584,299.892736,301.989888,304.08704,306.184192,308.28134400000005,310.37849600000004,312.47564800000004,314.57280000000003,316.669952,318.767104,320.864256,322.961408,325.05856,327.155712,329.25286400000005,331.35001600000004,333.44716800000003,335.54432,337.641472,339.738624,341.835776,343.932928,346.03008,348.12723200000005,350.22438400000004],"y":[0.2509999871253967,0.2815000116825104,0.3215000033378601,0.3375000059604645,0.3490000069141388,0.351500004529953,0.3504999876022339,0.3650000095367431,0.3544999957084656,0.3569999933242798,0.3644999861717224,0.3704999983310699,0.3745000064373016,0.3725000023841858,0.3700000047683716,0.3770000040531158,0.37950000166893,0.3899999856948852,0.3855000138282776,0.3989999890327453,0.3910000026226043,0.3804999887943268,0.3844999969005584,0.3925000131130218,0.3889999985694885,0.3849999904632568,0.3840000033378601,0.3855000138282776,0.3810000121593475,0.3989999890327453,0.3799999952316284,0.4004999995231628,0.4084999859333038,0.3894999921321869,0.3930000066757202,0.4020000100135803,0.4020000100135803,0.3980000019073486,0.4065000116825104,0.4000000059604645,0.4014999866485595,0.3955000042915344,0.3959999978542328,0.4110000133514404,0.4149999916553497,0.4180000126361847,0.414000004529953,0.4135000109672546,0.418500006198883,0.4074999988079071,0.4054999947547912,0.4070000052452087,0.4259999990463257,0.4095000028610229,0.4230000078678131,0.4025000035762787,0.4180000126361847,0.4230000078678131,0.421999990940094,0.4169999957084656,0.426499992609024,0.4154999852180481,0.421999990940094,0.414000004529953,0.4074999988079071,0.4160000085830688,0.4144999980926513,0.4225000143051147,0.418500006198883,0.4205000102519989,0.4084999859333038,0.4275000095367431,0.4235000014305115,0.4144999980926513,0.4275000095367431,0.4334999918937683,0.4314999878406524,0.4320000112056732,0.4199999868869781,0.4269999861717224,0.4250000119209289,0.4174999892711639,0.421999990940094,0.4180000126361847,0.4160000085830688,0.4104999899864197,0.4144999980926513,0.4259999990463257,0.4205000102519989,0.414000004529953,0.4210000038146972,0.4189999997615814,0.4160000085830688,0.4174999892711639,0.4275000095367431,0.4325000047683716,0.4250000119209289,0.4235000014305115,0.4300000071525574,0.4320000112056732,0.4329999983310699,0.429500013589859,0.4365000128746032,0.4410000145435333,0.4354999959468841,0.4275000095367431,0.4370000064373016,0.4239999949932098,0.4415000081062317,0.4444999992847442,0.4354999959468841,0.4309999942779541,0.4325000047683716,0.4359999895095825,0.4329999983310699,0.4255000054836273,0.4300000071525574,0.4339999854564667,0.4375,0.4325000047683716,0.4309999942779541,0.4305000007152557,0.4334999918937683,0.4365000128746032,0.4375,0.4300000071525574,0.4329999983310699,0.4345000088214874,0.4404999911785126,0.4384999871253967,0.4399999976158142,0.445499986410141,0.4435000121593475,0.44200000166893,0.4365000128746032,0.4375,0.4399999976158142,0.4390000104904175,0.44200000166893,0.4334999918937683,0.4284999966621399,0.4284999966621399,0.4375,0.4359999895095825,0.4390000104904175,0.4390000104904175,0.4415000081062317,0.4370000064373016,0.4345000088214874,0.4435000121593475,0.44200000166893,0.4440000057220459,0.4365000128746032,0.4404999911785126,0.44200000166893,0.4395000040531158,0.4339999854564667,0.4370000064373016,0.4395000040531158,0.44200000166893,0.4440000057220459,0.4399999976158142,0.4384999871253967,0.4449999928474426,0.4339999854564667,0.4485000073909759,0.4444999992847442,0.44200000166893],"label":"Dolma"},"RefinedWeb":{"x":[0.0,2.0971520000000003,4.194304000000001,6.291456,8.388608000000001,10.48576,12.582912,14.680064000000002,16.777216000000003,18.874368,20.97152,23.068672000000003,25.165824,27.262976000000002,29.360128000000003,31.45728,33.554432000000006,35.651584,37.748736,39.845888,41.94304,44.040192000000005,46.137344000000006,48.234496,50.331648,52.4288,54.525952000000004,56.623104000000005,58.720256000000006,60.817408,62.91456,65.011712,67.10886400000001,69.206016,71.303168,73.40032000000001,75.497472,77.59462400000001,79.691776,81.788928,83.88608,85.983232,88.08038400000001,90.177536,92.27468800000001,94.37184,96.468992,98.56614400000001,100.663296,102.76044800000001,104.8576,106.95475200000001,109.05190400000001,111.149056,113.24620800000001,115.34336,117.44051200000001,119.537664,121.634816,123.73196800000001,125.82912,127.92627200000001,130.023424,132.120576,134.21772800000002,136.31488000000002,138.412032,140.509184,142.606336,144.70348800000002,146.80064000000002,148.897792,150.994944,153.092096,155.18924800000002,157.28640000000001,159.383552,161.480704,163.577856,165.67500800000002,167.77216,169.869312,171.966464,174.06361600000002,176.16076800000002,178.25792,180.355072,182.452224,184.54937600000002,186.64652800000002,188.74368,190.840832,192.937984,195.03513600000002,197.13228800000002,199.22944,201.326592,203.423744,205.52089600000002,207.61804800000002,209.7152,211.812352,213.90950400000003,216.00665600000002,218.10380800000001,220.20096,222.298112,224.39526400000003,226.49241600000002,228.589568,230.68672,232.783872,234.88102400000002,236.97817600000002,239.075328,241.17248,243.269632,245.36678400000002,247.46393600000002,249.561088,251.65824,253.75539200000003,255.85254400000002,257.949696,260.046848,262.144,264.241152,266.338304,268.43545600000004,270.53260800000004,272.62976000000003,274.726912,276.824064,278.921216,281.018368,283.11552,285.212672,287.309824,289.40697600000004,291.50412800000004,293.60128000000003,295.698432,297.795584,299.892736,301.989888,304.08704,306.184192,308.28134400000005,310.37849600000004,312.47564800000004,314.57280000000003,316.669952,318.767104,320.864256,322.961408,325.05856,327.155712,329.25286400000005,331.35001600000004,333.44716800000003,335.54432,337.641472,339.738624,341.835776,343.932928,346.03008,348.12723200000005,350.22438400000004],"y":[0.2509999871253967,0.2899999916553497,0.31700000166893,0.3409999907016754,0.3425000011920929,0.3485000133514404,0.3555000126361847,0.3574999868869781,0.3585000038146972,0.363999992609024,0.3619999885559082,0.3675000071525574,0.3865000009536743,0.3810000121593475,0.3810000121593475,0.3810000121593475,0.3860000073909759,0.3810000121593475,0.3894999921321869,0.3849999904632568,0.3855000138282776,0.3989999890327453,0.3980000019073486,0.3995000123977661,0.395000010728836,0.4084999859333038,0.4040000140666961,0.4004999995231628,0.3955000042915344,0.4135000109672546,0.4070000052452087,0.4104999899864197,0.4014999866485595,0.4099999964237213,0.4199999868869781,0.414000004529953,0.402999997138977,0.4214999973773956,0.4095000028610229,0.4059999883174896,0.4090000092983246,0.4074999988079071,0.4120000004768371,0.4154999852180481,0.4189999997615814,0.4149999916553497,0.429500013589859,0.4154999852180481,0.4214999973773956,0.4244999885559082,0.4205000102519989,0.4269999861717224,0.4214999973773956,0.4180000126361847,0.4415000081062317,0.4320000112056732,0.4350000023841858,0.4259999990463257,0.4300000071525574,0.4259999990463257,0.4189999997615814,0.4269999861717224,0.4199999868869781,0.426499992609024,0.4350000023841858,0.4289999902248382,0.4345000088214874,0.4259999990463257,0.426499992609024,0.4395000040531158,0.4395000040531158,0.4359999895095825,0.4280000030994415,0.4370000064373016,0.4329999983310699,0.4309999942779541,0.4490000009536743,0.4399999976158142,0.4339999854564667,0.4399999976158142,0.4345000088214874,0.429500013589859,0.4370000064373016,0.4379999935626983,0.4284999966621399,0.4309999942779541,0.4350000023841858,0.4399999976158142,0.4314999878406524,0.4300000071525574,0.4410000145435333,0.4345000088214874,0.4410000145435333,0.4345000088214874,0.4339999854564667,0.4460000097751617,0.4410000145435333,0.4469999969005584,0.4480000138282776,0.4435000121593475,0.4375,0.4519999921321869,0.4480000138282776,0.4429999887943268,0.4519999921321869,0.4435000121593475,0.4334999918937683,0.4460000097751617,0.4564999938011169,0.4469999969005584,0.453000009059906,0.4485000073909759,0.4410000145435333,0.4444999992847442,0.4485000073909759,0.457500010728836,0.4469999969005584,0.4535000026226043,0.4535000026226043,0.4485000073909759,0.4490000009536743,0.4505000114440918,0.4595000147819519,0.4544999897480011,0.453000009059906,0.4605000019073486,0.4620000123977661,0.457500010728836,0.453000009059906,0.4550000131130218,0.460999995470047,0.4449999928474426,0.4474999904632568,0.457500010728836,0.4584999978542328,0.4494999945163727,0.4474999904632568,0.4625000059604645,0.4639999866485595,0.4555000066757202,0.4469999969005584,0.4600000083446502,0.453000009059906,0.4629999995231628,0.4589999914169311,0.4614999890327453,0.4555000066757202,0.4560000002384186,0.4580000042915344,0.4584999978542328,0.4560000002384186,0.4605000019073486,0.4595000147819519,0.4639999866485595,0.4614999890327453,0.4564999938011169,0.4634999930858612,0.4625000059604645,0.4614999890327453,0.4679999947547912,0.4584999978542328,0.4595000147819519,0.4505000114440918,0.4544999897480011,0.4595000147819519,0.4620000123977661,0.4670000076293945,0.4555000066757202],"label":"RefinedWeb"},"SlimPajama":{"x":[0.0,2.0971520000000003,4.194304000000001,6.291456,8.388608000000001,10.48576,12.582912,14.680064000000002,16.777216000000003,18.874368,20.97152,23.068672000000003,25.165824,27.262976000000002,29.360128000000003,31.45728,33.554432000000006,35.651584,37.748736,39.845888,41.94304,44.040192000000005,46.137344000000006,48.234496,50.331648,52.4288,54.525952000000004,56.623104000000005,58.720256000000006,60.817408,62.91456,65.011712,67.10886400000001,69.206016,71.303168,73.40032000000001,75.497472,77.59462400000001,79.691776,81.788928,83.88608,85.983232,88.08038400000001,90.177536,92.27468800000001,94.37184,96.468992,98.56614400000001,100.663296,102.76044800000001,104.8576,106.95475200000001,109.05190400000001,111.149056,113.24620800000001,115.34336,117.44051200000001,119.537664,121.634816,123.73196800000001,125.82912,127.92627200000001,130.023424,132.120576,134.21772800000002,136.31488000000002,138.412032,140.509184,142.606336,144.70348800000002,146.80064000000002,148.897792,150.994944,153.092096,155.18924800000002,157.28640000000001,159.383552,161.480704,163.577856,165.67500800000002,167.77216,169.869312,171.966464,174.06361600000002,176.16076800000002,178.25792,180.355072,182.452224,184.54937600000002,186.64652800000002,188.74368,190.840832,192.937984,195.03513600000002,197.13228800000002,199.22944,201.326592,203.423744,205.52089600000002,207.61804800000002,209.7152,211.812352,213.90950400000003,216.00665600000002,218.10380800000001,220.20096,222.298112,224.39526400000003,226.49241600000002,228.589568,230.68672,232.783872,234.88102400000002,236.97817600000002,239.075328,241.17248,243.269632,245.36678400000002,247.46393600000002,249.561088,251.65824,253.75539200000003,255.85254400000002,257.949696,260.046848,262.144,264.241152,266.338304,268.43545600000004,270.53260800000004,272.62976000000003,274.726912,276.824064,278.921216,281.018368,283.11552,285.212672,287.309824,289.40697600000004,291.50412800000004,293.60128000000003,295.698432,297.795584,299.892736,301.989888,304.08704,306.184192,308.28134400000005,310.37849600000004,312.47564800000004,314.57280000000003,316.669952,318.767104,320.864256,322.961408,325.05856,327.155712,329.25286400000005,331.35001600000004,333.44716800000003,335.54432,337.641472,339.738624,341.835776,343.932928,346.03008,348.12723200000005,350.22438400000004],"y":[0.2535000145435333,0.2739999890327453,0.3190000057220459,0.3300000131130218,0.3490000069141388,0.3449999988079071,0.3574999868869781,0.3529999852180481,0.3610000014305115,0.3610000014305115,0.3580000102519989,0.3714999854564667,0.3785000145435333,0.3659999966621399,0.3785000145435333,0.3655000030994415,0.3765000104904175,0.3704999983310699,0.3774999976158142,0.3709999918937683,0.3745000064373016,0.3799999952316284,0.3804999887943268,0.390500009059906,0.3939999938011169,0.3995000123977661,0.390500009059906,0.3939999938011169,0.3889999985694885,0.3944999873638153,0.4000000059604645,0.4074999988079071,0.402999997138977,0.3840000033378601,0.387499988079071,0.3914999961853027,0.3939999938011169,0.3860000073909759,0.387499988079071,0.3995000123977661,0.4065000116825104,0.3925000131130218,0.4020000100135803,0.3980000019073486,0.4004999995231628,0.4000000059604645,0.4169999957084656,0.4025000035762787,0.418500006198883,0.4034999907016754,0.4040000140666961,0.4099999964237213,0.4129999876022339,0.4110000133514404,0.4165000021457672,0.4079999923706054,0.4054999947547912,0.4104999899864197,0.4160000085830688,0.4199999868869781,0.4059999883174896,0.4194999933242798,0.4074999988079071,0.4210000038146972,0.4205000102519989,0.4239999949932098,0.4239999949932098,0.4165000021457672,0.4329999983310699,0.4269999861717224,0.4120000004768371,0.4289999902248382,0.4230000078678131,0.4154999852180481,0.4235000014305115,0.4174999892711639,0.4189999997615814,0.4314999878406524,0.4259999990463257,0.4284999966621399,0.4284999966621399,0.4444999992847442,0.4300000071525574,0.4250000119209289,0.4235000014305115,0.4359999895095825,0.4239999949932098,0.4255000054836273,0.421999990940094,0.4375,0.4235000014305115,0.4314999878406524,0.4339999854564667,0.4309999942779541,0.4410000145435333,0.4395000040531158,0.4255000054836273,0.4334999918937683,0.4350000023841858,0.429500013589859,0.4514999985694885,0.4404999911785126,0.4460000097751617,0.4494999945163727,0.4435000121593475,0.4365000128746032,0.4490000009536743,0.4440000057220459,0.4605000019073486,0.4494999945163727,0.4535000026226043,0.4555000066757202,0.4595000147819519,0.4465000033378601,0.453000009059906,0.4600000083446502,0.4510000050067901,0.445499986410141,0.4444999992847442,0.4494999945163727,0.4424999952316284,0.4539999961853027,0.453000009059906,0.4544999897480011,0.4560000002384186,0.4449999928474426,0.4584999978542328,0.4474999904632568,0.4595000147819519,0.4465000033378601,0.4589999914169311,0.449999988079071,0.4469999969005584,0.4469999969005584,0.4490000009536743,0.453000009059906,0.4595000147819519,0.4535000026226043,0.4544999897480011,0.4614999890327453,0.4535000026226043,0.4564999938011169,0.453000009059906,0.4620000123977661,0.4544999897480011,0.4569999873638153,0.4620000123977661,0.4510000050067901,0.4474999904632568,0.4589999914169311,0.4584999978542328,0.4514999985694885,0.4645000100135803,0.4569999873638153,0.4550000131130218,0.4600000083446502,0.4505000114440918,0.4519999921321869,0.4675000011920929,0.4650000035762787,0.457500010728836,0.4634999930858612,0.4650000035762787,0.4580000042915344,0.4704999923706054,0.449999988079071,0.4539999961853027,0.4605000019073486],"label":"SlimPajama"},"The Pile":{"x":[0.0,2.0971520000000003,4.194304000000001,6.291456,8.388608000000001,10.48576,12.582912,14.680064000000002,16.777216000000003,18.874368,20.97152,23.068672000000003,25.165824,27.262976000000002,29.360128000000003,31.45728,33.554432000000006,35.651584,37.748736,39.845888,41.94304,44.040192000000005,46.137344000000006,48.234496,50.331648,52.4288,54.525952000000004,56.623104000000005,58.720256000000006,60.817408,62.91456,65.011712,67.10886400000001,69.206016,71.303168,73.40032000000001,75.497472,77.59462400000001,79.691776,81.788928,83.88608,85.983232,88.08038400000001,90.177536,92.27468800000001,94.37184,96.468992,98.56614400000001,100.663296,102.76044800000001,104.8576,106.95475200000001,109.05190400000001,111.149056,113.24620800000001,115.34336,117.44051200000001,119.537664,121.634816,123.73196800000001,125.82912,127.92627200000001,130.023424,132.120576,134.21772800000002,136.31488000000002,138.412032,140.509184,142.606336,144.70348800000002,146.80064000000002,148.897792,150.994944,153.092096,155.18924800000002,157.28640000000001,159.383552,161.480704,163.577856,165.67500800000002,167.77216,169.869312,171.966464,174.06361600000002,176.16076800000002,178.25792,180.355072,182.452224,184.54937600000002,186.64652800000002,188.74368,190.840832,192.937984,195.03513600000002,197.13228800000002,199.22944,201.326592,203.423744,205.52089600000002,207.61804800000002,209.7152,211.812352,213.90950400000003,216.00665600000002,218.10380800000001,220.20096,222.298112,224.39526400000003,226.49241600000002,228.589568,230.68672,232.783872,234.88102400000002,236.97817600000002,239.075328,241.17248,243.269632,245.36678400000002,247.46393600000002,249.561088,251.65824,253.75539200000003,255.85254400000002,257.949696,260.046848,262.144,264.241152,266.338304,268.43545600000004,270.53260800000004,272.62976000000003,274.726912,276.824064,278.921216,281.018368,283.11552,285.212672,287.309824,289.40697600000004,291.50412800000004,293.60128000000003,295.698432,297.795584,299.892736,301.989888,304.08704,306.184192,308.28134400000005,310.37849600000004,312.47564800000004,314.57280000000003,316.669952,318.767104,320.864256,322.961408,325.05856,327.155712,329.25286400000005,331.35001600000004,333.44716800000003,335.54432,337.641472,339.738624,341.835776,343.932928,346.03008,348.12723200000005,350.22438400000004],"y":[0.2535000145435333,0.2694999873638153,0.2939999997615814,0.3064999878406524,0.3269999921321869,0.3264999985694885,0.3355000019073486,0.3540000021457672,0.3449999988079071,0.3454999923706054,0.3540000021457672,0.3600000143051147,0.3574999868869781,0.3569999933242798,0.3650000095367431,0.3659999966621399,0.3625000119209289,0.3689999878406524,0.3555000126361847,0.359499990940094,0.3855000138282776,0.3729999959468841,0.37950000166893,0.3720000088214874,0.3774999976158142,0.3740000128746032,0.3840000033378601,0.3765000104904175,0.3734999895095825,0.3855000138282776,0.3844999969005584,0.3804999887943268,0.3849999904632568,0.3939999938011169,0.3804999887943268,0.3919999897480011,0.3819999992847442,0.3955000042915344,0.3880000114440918,0.387499988079071,0.3925000131130218,0.4009999930858612,0.3935000002384186,0.3959999978542328,0.398499995470047,0.3914999961853027,0.3970000147819519,0.402999997138977,0.4070000052452087,0.3860000073909759,0.398499995470047,0.4009999930858612,0.3914999961853027,0.402999997138977,0.4025000035762787,0.3975000083446502,0.3930000066757202,0.4104999899864197,0.4054999947547912,0.395000010728836,0.4180000126361847,0.3955000042915344,0.4009999930858612,0.4000000059604645,0.4004999995231628,0.390500009059906,0.3995000123977661,0.4004999995231628,0.4045000076293945,0.4045000076293945,0.4004999995231628,0.4065000116825104,0.4084999859333038,0.4065000116825104,0.3980000019073486,0.4014999866485595,0.4095000028610229,0.3964999914169311,0.4040000140666961,0.3970000147819519,0.3970000147819519,0.4059999883174896,0.4004999995231628,0.4099999964237213,0.4115000069141388,0.4135000109672546,0.4160000085830688,0.4120000004768371,0.4124999940395355,0.4180000126361847,0.4099999964237213,0.4135000109672546,0.4160000085830688,0.4095000028610229,0.4169999957084656,0.4314999878406524,0.4280000030994415,0.4205000102519989,0.4210000038146972,0.4174999892711639,0.4090000092983246,0.4169999957084656,0.4149999916553497,0.4210000038146972,0.4210000038146972,0.426499992609024,0.4225000143051147,0.4189999997615814,0.4160000085830688,0.418500006198883,0.4339999854564667,0.4280000030994415,0.4244999885559082,0.4269999861717224,0.4235000014305115,0.4300000071525574,0.426499992609024,0.4180000126361847,0.4275000095367431,0.4275000095367431,0.4399999976158142,0.4284999966621399,0.4424999952316284,0.4435000121593475,0.4284999966621399,0.426499992609024,0.4305000007152557,0.4280000030994415,0.4339999854564667,0.4375,0.4375,0.4309999942779541,0.4365000128746032,0.4314999878406524,0.4329999983310699,0.4284999966621399,0.4305000007152557,0.4329999983310699,0.4280000030994415,0.426499992609024,0.4314999878406524,0.4365000128746032,0.4359999895095825,0.4410000145435333,0.4390000104904175,0.4404999911785126,0.4350000023841858,0.4390000104904175,0.4474999904632568,0.4415000081062317,0.445499986410141,0.4329999983310699,0.4490000009536743,0.4440000057220459,0.4510000050067901,0.4474999904632568,0.4595000147819519,0.4440000057220459,0.4469999969005584,0.4384999871253967,0.4375,0.4415000081062317,0.4480000138282776,0.4449999928474426,0.4354999959468841,0.445499986410141,0.4384999871253967,0.4375],"label":"The Pile"},"RedPajama2":{"x":[0.0,2.0971520000000003,4.194304000000001,6.291456,8.388608000000001,10.48576,12.582912,14.680064000000002,16.777216000000003,18.874368,20.97152,23.068672000000003,25.165824,27.262976000000002,29.360128000000003,31.45728,33.554432000000006,35.651584,37.748736,39.845888,41.94304,44.040192000000005,46.137344000000006,48.234496,50.331648,52.4288,54.525952000000004,56.623104000000005,58.720256000000006,60.817408,62.91456,65.011712,67.10886400000001,69.206016,71.303168,73.40032000000001,75.497472,77.59462400000001,79.691776,81.788928,83.88608,85.983232,88.08038400000001,90.177536,92.27468800000001,94.37184,96.468992,98.56614400000001,100.663296,102.76044800000001,104.8576,106.95475200000001,109.05190400000001,111.149056,113.24620800000001,115.34336,117.44051200000001,119.537664,121.634816,123.73196800000001,125.82912,127.92627200000001,130.023424,132.120576,134.21772800000002,136.31488000000002,138.412032,140.509184,142.606336,144.70348800000002,146.80064000000002,148.897792,150.994944,153.092096,155.18924800000002,157.28640000000001,159.383552,161.480704,163.577856,165.67500800000002,167.77216,169.869312,171.966464,174.06361600000002,176.16076800000002,178.25792,180.355072,182.452224,184.54937600000002,186.64652800000002,188.74368,190.840832,192.937984,195.03513600000002,197.13228800000002,199.22944,201.326592,203.423744,205.52089600000002,207.61804800000002,209.7152,211.812352,213.90950400000003,216.00665600000002,218.10380800000001,220.20096,222.298112,224.39526400000003,226.49241600000002,228.589568,230.68672,232.783872,234.88102400000002,236.97817600000002,239.075328,241.17248,243.269632,245.36678400000002,247.46393600000002,249.561088,251.65824,253.75539200000003,255.85254400000002,257.949696,260.046848,262.144,264.241152,266.338304,268.43545600000004,270.53260800000004,272.62976000000003,274.726912,276.824064,278.921216,281.018368,283.11552,285.212672,287.309824,289.40697600000004,291.50412800000004,293.60128000000003,295.698432,297.795584,299.892736,301.989888,304.08704,306.184192,308.28134400000005,310.37849600000004,312.47564800000004,314.57280000000003,316.669952,318.767104,320.864256,322.961408,325.05856,327.155712,329.25286400000005,331.35001600000004,333.44716800000003,335.54432,337.641472,339.738624,341.835776,343.932928,346.03008,348.12723200000005,350.22438400000004],"y":[0.2535000145435333,0.2854999899864197,0.3145000040531158,0.332500010728836,0.3454999923706054,0.3540000021457672,0.359499990940094,0.3479999899864197,0.3535000085830688,0.3544999957084656,0.3684999942779541,0.3714999854564667,0.37950000166893,0.3790000081062317,0.3880000114440918,0.3765000104904175,0.3804999887943268,0.3880000114440918,0.3810000121593475,0.3819999992847442,0.3959999978542328,0.3925000131130218,0.4054999947547912,0.3849999904632568,0.3935000002384186,0.4000000059604645,0.3849999904632568,0.3989999890327453,0.3930000066757202,0.4129999876022339,0.4034999907016754,0.3980000019073486,0.4059999883174896,0.4004999995231628,0.4169999957084656,0.4065000116825104,0.4004999995231628,0.3970000147819519,0.4090000092983246,0.4079999923706054,0.402999997138977,0.4025000035762787,0.4009999930858612,0.4004999995231628,0.4079999923706054,0.4095000028610229,0.3959999978542328,0.4045000076293945,0.4065000116825104,0.4079999923706054,0.4154999852180481,0.4059999883174896,0.4160000085830688,0.4129999876022339,0.4115000069141388,0.4034999907016754,0.4059999883174896,0.4054999947547912,0.4239999949932098,0.4149999916553497,0.4180000126361847,0.4199999868869781,0.418500006198883,0.4289999902248382,0.4144999980926513,0.4180000126361847,0.4199999868869781,0.4275000095367431,0.414000004529953,0.4199999868869781,0.4124999940395355,0.4305000007152557,0.4210000038146972,0.4235000014305115,0.426499992609024,0.4269999861717224,0.421999990940094,0.4165000021457672,0.421999990940094,0.421999990940094,0.4250000119209289,0.4255000054836273,0.4275000095367431,0.4269999861717224,0.4329999983310699,0.4354999959468841,0.4275000095367431,0.4410000145435333,0.4325000047683716,0.4354999959468841,0.4345000088214874,0.4379999935626983,0.4550000131130218,0.4494999945163727,0.44200000166893,0.4490000009536743,0.4415000081062317,0.44200000166893,0.4345000088214874,0.4404999911785126,0.4395000040531158,0.429500013589859,0.4390000104904175,0.4399999976158142,0.4384999871253967,0.4384999871253967,0.4444999992847442,0.4429999887943268,0.4449999928474426,0.4404999911785126,0.4429999887943268,0.445499986410141,0.4424999952316284,0.4415000081062317,0.4300000071525574,0.4354999959468841,0.4375,0.4395000040531158,0.4390000104904175,0.4469999969005584,0.4365000128746032,0.4485000073909759,0.4404999911785126,0.4514999985694885,0.4415000081062317,0.4494999945163727,0.4354999959468841,0.44200000166893,0.4444999992847442,0.4469999969005584,0.4424999952316284,0.4449999928474426,0.4490000009536743,0.4444999992847442,0.4404999911785126,0.4435000121593475,0.4404999911785126,0.4465000033378601,0.4514999985694885,0.4354999959468841,0.4539999961853027,0.4465000033378601,0.4519999921321869,0.4465000033378601,0.4480000138282776,0.4494999945163727,0.4469999969005584,0.4474999904632568,0.4444999992847442,0.4460000097751617,0.4444999992847442,0.4524999856948852,0.4505000114440918,0.4485000073909759,0.4465000033378601,0.4485000073909759,0.4465000033378601,0.4480000138282776,0.4384999871253967,0.4535000026226043,0.4469999969005584,0.4490000009536743,0.4539999961853027,0.4494999945163727,0.4519999921321869,0.4494999945163727,0.4564999938011169,0.4494999945163727],"label":"RedPajama2"}},"layout":{"xaxis":{"title":{"text":"Training tokens (billions)"}},"yaxis":{"range":[0.39,0.5]},"title":{"text":"Dataset Ablations"}}} |