from fasthtml.common import * from fasthtml.components import * from fasthtml.components import D_title, D_article, D_front_matter, D_contents, D_byline from plotly import graph_objects as go from fh_plotly import plotly2fasthtml import pandas as pd import json from rich import print import curated import web import common import results dataset_comparison = pd.DataFrame( { "Dataset": [ "TxT360", "FineWeb", "RefinedWeb", "RedPajama-v2", "C4", "Dolma", "RedPajama-v1", "The Pile", ], "CommonCrawl": [ "99 Snapshots", "96 Snapshots", "90 Snapshots", "84 Snapshots", "1 Snapshots", "24 Snapshots", "5 Snapshots", "0.6% of 74 Snapshots", ], "Papers": [ "5 Sources", "-", "-", "-", "-", "1 Source", "1 Source", "4 Sources", ], "Wikipedia": [ "310+ Languages", "-", "-", "-", "-", "what does a check mark mean?", "what does a check mark mean?", "English Only", ], "FreeLaw": [ "Included", "-", "-", "-", "-", "-", "-", "Included", ], "DM Math": [ "Included", "-", "-", "-", "-", "-", "-", "Included", ], "USPTO": [ "Included", "-", "-", "-", "-", "-", "-", "Included", ], "PG-19": [ "Included", "-", "-", "-", "-", "Included", "Included", "Included", ], "HackerNews": [ "Included", "-", "-", "-", "-", "-", "-", "Included", ], "Ubuntu IRC": [ "Included", "-", "-", "-", "-", "-", "-", "Included", ], "EuroParl": [ "Included", "-", "-", "-", "-", "-", "-", "Included", ], "StackExchange": [ "Included", "-", "-", "-", "-", "-", "Included", "Included", ], "Code": [ "- what is this?", "-", "-", "-", "-", "Included", "Included", "Included", ], } ) table_html = dataset_comparison.to_html(index=False, border=0) table_div = Div(NotStr(table_html), style="margin: 40px;") dataset_sources = pd.DataFrame( { "Data Source": [ "CommonCrawl", "Papers", "Wikipedia", "Freelaw", "DM Math", "USPTO", "PG-19", "HackerNews", "Ubuntu IRC", "Europarl", "StackExchange", ], "Raw Data Size": [ "11 TB", "712 GB", "210 GB", "23 GB", "22 GB", "45 GB", "11 GB", "4.1 GB", "4.7 GB", "6.1 GB", "45 GB", ], "Token Count": [ "5.71T", "154.96B", "4.75B", "7.34B", "5.23B", "4.95B", "2.94B", "1.08B", "1.54B", "1.96B", "8.37B", ], "Cut-Off Date": [ "2024-30", "Q4 2023", "-", "Q1 2024", "-", "Q4 2023", "-", "Q4 2023", "Q4 2023", "-", "Q4 2023", ], } ) table_html = dataset_sources.to_html(index=False, border=0) table_div1 = Div(NotStr(table_html), style="margin: 40px;") quality_text = P("""The quality and size of a pre-training dataset play a crucial role in the performance of large language models (LLMs). The community has introduced a variety of datasets for this purpose, including purely web-based datasets like RefinedWeb{citation_obj.display_citation("refinedweb")}, RedPajama-Data-V2{citation_obj.display_citation("redpajama-v2")}, DCLM{citation_obj.display_citation("dclm")}, and FineWeb{citation_obj.display_citation("fineweb")}, as well as comprehensive datasets derived from multiple highly-curated data sources such as The Pile{citation_obj.display_citation("thepile")}, RedPajama-Data-V1{citation_obj.display_citation("redpajama-v1")}, and Dolma{citation_obj.display_citation("dolma")}. It is commonly known that web-based datasets provide a vast quantity of data, while highly-curated multi-source datasets consistently deliver high quality and diversity, both critical for effective LLM pre-training.""") quality_text2 = P("However, despite the advancements in both types of data, each type of dataset has its limitations. For instance, the processing scripts for the web dataset, RefinedWeb, known for its high quality, are not public, and only about 10% of the entire dataset has been disclosed. Conversely, the web component of existing highly-curated multi-source datasets is relatively small compared to purely web-based datasets, limiting their coverage and diversity compared to the scale of information from the internet.") quality_text3 = P("By integrating the extensive reach of web data with the exceptional quality of curated sources, TxT360 is crafted to meet and surpass the rigorous standards required for state-of-the-art LLM pre-training.") data_processing_image_desc = P("Figure 1: Data processing pipeline. All the steps are adopted for processing web data while the yellow blocks are adopted for processing curated sources.") data_processing_overview = P("We enforce a fully transparent data processing pipeline when producing TxT360, designed to handle both web and curated datasets with precision and clarity. This transparent pipeline presents a unified framework for processing both data types, making it convenient and adaptive for users to revise and fine-tune the pipeline. ") data_processing_overview2 = P("For web datasets, the pipeline focuses on extracting meaningful, high-quality text from raw web content, which is inherently noisy and varied. The process includes sophisticated filtering and deduplication techniques to clean the data and remove any redundancies or irrelevant information. On the other hand, curated datasets, which are already more structured and reliable, are processed with selective steps to maintain their integrity while integrating them seamlessly into the larger dataset.") data_processing_overview3 = P("We will open-source the scripts for the whole pipeline, allowing the community to review, replicate, and build upon our processes.") def overview(): return Div(Section( H2("Our General Appoach to Data Processing"), data_processing_overview, data_processing_overview2, data_processing_overview3, Img(src="images/pipeline.png"), data_processing_image_desc, H2("Combining the Best of Web and Curated Sources"), H3("Why combine the web and highly curated sources? Isn't the web-only data enough?"), P("Table 1: TxT360 combines both the web data and highly-curated sources, which none of the existing datasets have covered. The following table shows TxT360 and other well-known datasets on the coverage and size of data sources."), table_div, P("Table 2: Statistics of TxT360. The basic statistics of TxT360 are presented."), table_div1, quality_text, quality_text2, quality_text3, id="inner-text", ) )