Spaces:
Sleeping
Sleeping
omkarenator
commited on
Commit
•
34ecf31
1
Parent(s):
85da60b
add generic data viewer. separate routes
Browse files- common.py +7 -0
- curated.py +186 -46
- data_viewer.py +83 -0
- main.py +100 -280
- results.py +7 -0
- web.py +7 -0
common.py
ADDED
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
from fasthtml.common import *
|
2 |
+
from fasthtml.components import *
|
3 |
+
|
4 |
+
|
5 |
+
def common_steps():
|
6 |
+
return Div(Section(H2(P("Common Steps")), id="inner-text"))
|
7 |
+
|
curated.py
CHANGED
@@ -1,5 +1,12 @@
|
|
1 |
from fasthtml.common import *
|
|
|
|
|
|
|
|
|
2 |
import json
|
|
|
|
|
|
|
3 |
|
4 |
|
5 |
data_sources = [
|
@@ -20,7 +27,7 @@ data_sources = [
|
|
20 |
]
|
21 |
|
22 |
|
23 |
-
def get_data(data_source: str = "Freelaw", doc_id: int = 3):
|
24 |
doc_id = max(0, min(int(doc_id), 9))
|
25 |
|
26 |
if data_source == "Freelaw":
|
@@ -77,60 +84,193 @@ def get_data(data_source: str = "Freelaw", doc_id: int = 3):
|
|
77 |
|
78 |
raw_json = raw_sample_doc[doc_id]
|
79 |
extracted_json = extracted_sample_doc[doc_id]
|
80 |
-
|
81 |
-
|
82 |
-
|
83 |
-
|
84 |
-
|
85 |
-
|
86 |
-
|
87 |
-
hx_swap="innerHTML",
|
88 |
)
|
89 |
|
90 |
-
|
91 |
-
|
92 |
-
|
93 |
-
|
94 |
-
|
95 |
-
|
96 |
-
|
97 |
-
|
98 |
-
|
99 |
-
|
100 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
101 |
)
|
102 |
|
103 |
-
|
104 |
-
|
105 |
-
|
106 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
107 |
Div(
|
108 |
-
|
|
|
109 |
),
|
110 |
-
cls="plotly_input_container",
|
111 |
)
|
112 |
|
113 |
-
|
114 |
-
|
115 |
-
|
116 |
-
|
117 |
-
style="white-space: pre-wrap; word-break: break-all;",
|
118 |
-
),
|
119 |
-
style="width: 48%; float: left; overflow-x: auto;",
|
120 |
-
)
|
121 |
|
122 |
-
|
123 |
-
|
124 |
-
|
125 |
-
|
126 |
-
|
127 |
-
|
128 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
129 |
)
|
130 |
|
131 |
-
|
132 |
-
|
133 |
-
|
134 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
135 |
)
|
136 |
-
return Div(form, data_display, style="margin-top: 10px;", id="colcontent")
|
|
|
1 |
from fasthtml.common import *
|
2 |
+
from fasthtml.components import *
|
3 |
+
from plotly import graph_objects as go
|
4 |
+
from fh_plotly import plotly2fasthtml
|
5 |
+
import pandas as pd
|
6 |
import json
|
7 |
+
from data_viewer import view_data, gen_random_id
|
8 |
+
from rich import print
|
9 |
+
import uuid
|
10 |
|
11 |
|
12 |
data_sources = [
|
|
|
27 |
]
|
28 |
|
29 |
|
30 |
+
def get_data(data_source: str = "Freelaw", doc_id: int = 3, target: str = "foo"):
|
31 |
doc_id = max(0, min(int(doc_id), 9))
|
32 |
|
33 |
if data_source == "Freelaw":
|
|
|
84 |
|
85 |
raw_json = raw_sample_doc[doc_id]
|
86 |
extracted_json = extracted_sample_doc[doc_id]
|
87 |
+
return view_data(
|
88 |
+
raw_json,
|
89 |
+
extracted_json,
|
90 |
+
doc_id=doc_id,
|
91 |
+
data_source=data_source,
|
92 |
+
data_sources=data_sources,
|
93 |
+
target=target,
|
|
|
94 |
)
|
95 |
|
96 |
+
|
97 |
+
def get_chart_28168342():
|
98 |
+
fig = go.Figure()
|
99 |
+
filter_names = [
|
100 |
+
"Download",
|
101 |
+
"Language",
|
102 |
+
"Min word count",
|
103 |
+
"Title Abstract",
|
104 |
+
"Majority language",
|
105 |
+
"Paragraph count",
|
106 |
+
"Frequency",
|
107 |
+
"Unigram log probability",
|
108 |
+
"Local dedup",
|
109 |
+
]
|
110 |
+
|
111 |
+
data_sources = [
|
112 |
+
("Wikipedia", [100, 90, 80, 70, 60, 50, 40, 30, 20]),
|
113 |
+
("Freelaw", [100, 90, 80, 70, 60, 50, 40, 20, 20]),
|
114 |
+
("DM Maths", [100, 90, 80, 70, 60, 40, 40, 30, 20]),
|
115 |
+
("USPTO", [100, 90, 80, 70, 60, 40, 40, 30, 20]),
|
116 |
+
("PG19", [100, 90, 80, 70, 60, 40, 40, 30, 20]),
|
117 |
+
("Hackernews", [100, 90, 80, 70, 60, 40, 40, 30, 20]),
|
118 |
+
("Ubuntu IRC", [100, 90, 80, 70, 60, 40, 40, 30, 20]),
|
119 |
+
("Europarl", [100, 90, 80, 70, 60, 40, 40, 30, 20]),
|
120 |
+
("StackExchange", [100, 90, 80, 70, 60, 40, 40, 30, 20]),
|
121 |
+
("Arxiv", [100, 90, 80, 70, 60, 40, 40, 30, 20]),
|
122 |
+
("S2ORC", [100, 90, 80, 70, 60, 40, 40, 30, 20]),
|
123 |
+
("S2ORC Abstract", [100, 90, 80, 70, 60, 40, 40, 30, 20]),
|
124 |
+
("PubMed Central", [100, 90, 80, 70, 60, 40, 40, 30, 20]),
|
125 |
+
("PubMed Central Abstract", [100, 90, 80, 70, 60, 40, 40, 30, 20]),
|
126 |
+
("PhilPapers", [100, 90, 80, 70, 60, 40, 40, 30, 20]),
|
127 |
+
]
|
128 |
+
|
129 |
+
for name, x_values in data_sources:
|
130 |
+
fig.add_trace(
|
131 |
+
go.Funnel(
|
132 |
+
name=name,
|
133 |
+
orientation="h",
|
134 |
+
y=filter_names,
|
135 |
+
x=x_values,
|
136 |
+
textinfo="value+percent total",
|
137 |
+
textposition="inside",
|
138 |
+
)
|
139 |
+
)
|
140 |
+
|
141 |
+
fig.update_layout(height=500, plot_bgcolor="rgba(0,0,0,0)")
|
142 |
+
return fig
|
143 |
+
|
144 |
+
|
145 |
+
def curated(request):
|
146 |
+
# Partial Updates
|
147 |
+
params = dict(request.query_params)
|
148 |
+
if target := params.get("target"):
|
149 |
+
if data_source := params.get(f"data_source_{target}"):
|
150 |
+
return get_data(
|
151 |
+
data_source, params.get(f"doc_id_{target}", 3), params.get("target")
|
152 |
+
)
|
153 |
+
if doc_id := params.get(f"doc_id_{target}"):
|
154 |
+
return get_data(
|
155 |
+
params.get(f"data_source_{target}"), doc_id, params.get("target")
|
156 |
+
)
|
157 |
+
|
158 |
+
data_preparation_steps = pd.DataFrame(
|
159 |
+
{
|
160 |
+
"Method": [
|
161 |
+
"HTTP/FTP dumps",
|
162 |
+
"Web crawling",
|
163 |
+
"Archive snapshot",
|
164 |
+
"Generated",
|
165 |
+
"Curated",
|
166 |
+
],
|
167 |
+
"Description": [
|
168 |
+
"Acquiring data from HTTP/FTP dumps",
|
169 |
+
"Crawling websites to extract data",
|
170 |
+
"Working with archive dumps",
|
171 |
+
"Generating synthetic data",
|
172 |
+
"High quality curated data",
|
173 |
+
],
|
174 |
+
"Source": [
|
175 |
+
"Freelaw | Wikipedia | PhilPapers | Arxiv | S2ORC | Pubmeds",
|
176 |
+
"USPTO | Hackernews | Ubuntu IRC",
|
177 |
+
"StackExchange",
|
178 |
+
"DM Maths",
|
179 |
+
"PG19 | Europarl",
|
180 |
+
],
|
181 |
+
}
|
182 |
)
|
183 |
|
184 |
+
table_html = data_preparation_steps.to_html(index=False, border=0)
|
185 |
+
table_div = Div(NotStr(table_html), style="margin: 40px;")
|
186 |
+
|
187 |
+
text = P("""This initial stage serves as the foundation for the entire
|
188 |
+
process. Here, we focus on acquiring and extracting the raw data, which can
|
189 |
+
come from various sources such as crawling websites, using HTTP/FTP dumps,
|
190 |
+
or working with archive dumps. For instance, to download and prepare a
|
191 |
+
dataset, we can specific downloaders based on the data source. Each dataset
|
192 |
+
might have its own downloader script which can be updated in real time to
|
193 |
+
handle changes in the data source. Here is a general outline of the data
|
194 |
+
preparation process: It's worth noting that some pipelines might require
|
195 |
+
invoking additional functions or scripts to handle specific data sources or
|
196 |
+
formats. These helper scripts can be located within specific directories
|
197 |
+
or modules dedicated to the dataset.""")
|
198 |
+
|
199 |
+
data_preparation_div = Div(
|
200 |
+
H3("Data Preparation"),
|
201 |
+
text,
|
202 |
+
table_div,
|
203 |
Div(
|
204 |
+
get_data(target=gen_random_id()),
|
205 |
+
style="border: 1px solid #ccc; padding: 20px;",
|
206 |
),
|
|
|
207 |
)
|
208 |
|
209 |
+
text = P("""Data preprocessing is a crucial step in the data science
|
210 |
+
pipeline. It involves cleaning and transforming raw data into a format that
|
211 |
+
is suitable for analysis. This process includes handling missing values,
|
212 |
+
normalizing data, encoding categorical variables, and more.""")
|
|
|
|
|
|
|
|
|
213 |
|
214 |
+
preprocessing_steps = pd.DataFrame(
|
215 |
+
{
|
216 |
+
"Step": [
|
217 |
+
"Language Filter",
|
218 |
+
"Min Word Count",
|
219 |
+
"Title Abstract",
|
220 |
+
"Majority Language",
|
221 |
+
"Paragraph Count",
|
222 |
+
"Frequency",
|
223 |
+
"Unigram Log Probability",
|
224 |
+
],
|
225 |
+
"Description": [
|
226 |
+
"Filtering data based on language",
|
227 |
+
"Setting a minimum word count threshold",
|
228 |
+
"Extracting information from the title and abstract",
|
229 |
+
"Identifying the majority language in the dataset",
|
230 |
+
"Counting the number of paragraphs in each document",
|
231 |
+
"Calculating the frequency of each word in the dataset",
|
232 |
+
"Calculating the log probability of each unigram",
|
233 |
+
],
|
234 |
+
"Need": [
|
235 |
+
"To remove documents in unwanted languages",
|
236 |
+
"To filter out documents with very few words",
|
237 |
+
"To extract relevant information for analysis",
|
238 |
+
"To understand the distribution of languages in the dataset",
|
239 |
+
"To analyze the structure and length of documents",
|
240 |
+
"To identify important words in the dataset",
|
241 |
+
"To measure the significance of individual words",
|
242 |
+
],
|
243 |
+
"Pros": [
|
244 |
+
"Improves data quality by removing irrelevant documents",
|
245 |
+
"Filters out low-quality or incomplete documents",
|
246 |
+
"Provides additional information for analysis",
|
247 |
+
"Enables language-specific analysis and insights",
|
248 |
+
"Helps understand the complexity and content of documents",
|
249 |
+
"Identifies important terms and topics in the dataset",
|
250 |
+
"Quantifies the importance of individual words",
|
251 |
+
],
|
252 |
+
"Cons": [
|
253 |
+
"May exclude documents in less common languages",
|
254 |
+
"May remove documents with valuable information",
|
255 |
+
"May introduce bias in the analysis",
|
256 |
+
"May not accurately represent the language distribution",
|
257 |
+
"May not capture the complexity of document structure",
|
258 |
+
"May be sensitive to noise and outliers",
|
259 |
+
"May not capture the semantic meaning of words",
|
260 |
+
],
|
261 |
+
}
|
262 |
)
|
263 |
|
264 |
+
table_html = preprocessing_steps.to_html(index=False, border=0)
|
265 |
+
table_div = Div(NotStr(table_html), style="margin: 40px;")
|
266 |
+
data_preprocessing_div = Div(H3("Data Preprocessing"), text, table_div)
|
267 |
+
|
268 |
+
return Div(
|
269 |
+
Section(
|
270 |
+
H2("Curated Sources"),
|
271 |
+
plotly2fasthtml(get_chart_28168342()),
|
272 |
+
data_preparation_div,
|
273 |
+
data_preprocessing_div,
|
274 |
+
id="inner-text",
|
275 |
+
)
|
276 |
)
|
|
data_viewer.py
ADDED
@@ -0,0 +1,83 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
from fasthtml.common import *
|
2 |
+
from fasthtml.components import *
|
3 |
+
import json
|
4 |
+
import string
|
5 |
+
import random
|
6 |
+
|
7 |
+
|
8 |
+
def gen_random_id() -> str:
|
9 |
+
return "".join(random.choices(string.ascii_lowercase, k=8))
|
10 |
+
|
11 |
+
|
12 |
+
def view_data(
|
13 |
+
before,
|
14 |
+
after,
|
15 |
+
doc_id,
|
16 |
+
data_source: str,
|
17 |
+
data_sources=None,
|
18 |
+
target: str = "colcontent",
|
19 |
+
):
|
20 |
+
if data_sources is not None:
|
21 |
+
drop_down = Select(
|
22 |
+
*[
|
23 |
+
Option(ds, value=ds, selected=(ds == data_source))
|
24 |
+
for ds in data_sources
|
25 |
+
],
|
26 |
+
name=f"data_source_{target}",
|
27 |
+
hx_get="/curated",
|
28 |
+
hx_target=f"#{target}",
|
29 |
+
hx_trigger="change",
|
30 |
+
hx_swap="innerHTML",
|
31 |
+
hx_vals=json.dumps({"target": f"{target}"}),
|
32 |
+
)
|
33 |
+
|
34 |
+
slider = Input(
|
35 |
+
type="range",
|
36 |
+
name=f"doc_id_{target}",
|
37 |
+
min="0",
|
38 |
+
max="9",
|
39 |
+
value=str(doc_id),
|
40 |
+
hx_get="/curated",
|
41 |
+
hx_target=f"#{target}",
|
42 |
+
hx_trigger="change",
|
43 |
+
hx_swap="innerHTML",
|
44 |
+
hx_include=f'[name="data_source_{target}"]',
|
45 |
+
hx_vals=json.dumps({"target": f"{target}"}),
|
46 |
+
)
|
47 |
+
|
48 |
+
form = Form(
|
49 |
+
Div(
|
50 |
+
Label("Data source: ", drop_down),
|
51 |
+
)
|
52 |
+
if (data_sources is not None)
|
53 |
+
else None,
|
54 |
+
Div(
|
55 |
+
Label("Data sample: ", slider, f"{doc_id}", cls="plotly_slider"),
|
56 |
+
),
|
57 |
+
cls="plotly_input_container",
|
58 |
+
)
|
59 |
+
|
60 |
+
col1 = Div(
|
61 |
+
H3("Raw format"),
|
62 |
+
Pre(
|
63 |
+
json.dumps(before, indent=4),
|
64 |
+
style="white-space: pre-wrap; word-break: break-all;",
|
65 |
+
),
|
66 |
+
style="width: 48%; float: left; overflow-x: auto;",
|
67 |
+
)
|
68 |
+
|
69 |
+
col2 = Div(
|
70 |
+
H3("Extracted format"),
|
71 |
+
Pre(
|
72 |
+
json.dumps(after, indent=4),
|
73 |
+
style="white-space: pre-wrap; word-break: break-all;",
|
74 |
+
),
|
75 |
+
style="width: 48%; float: right; overflow-x: auto;",
|
76 |
+
)
|
77 |
+
|
78 |
+
data_display = Div(
|
79 |
+
col1,
|
80 |
+
col2,
|
81 |
+
style="overflow: auto; clear: both; height: 600px; border: 1px solid #ccc; padding: 20px;",
|
82 |
+
)
|
83 |
+
return Div(form, data_display, style="margin-top: 10px;", id=target)
|
main.py
CHANGED
@@ -1,115 +1,120 @@
|
|
1 |
from fasthtml.common import *
|
2 |
from fasthtml.components import *
|
3 |
from fasthtml.components import D_title, D_article, D_front_matter, D_contents, D_byline
|
4 |
-
from fasthtml.components import HR
|
5 |
from plotly import graph_objects as go
|
6 |
from fh_plotly import plotly2fasthtml
|
7 |
import pandas as pd
|
8 |
import json
|
9 |
from rich import print
|
|
|
|
|
|
|
|
|
10 |
|
11 |
|
12 |
-
app, rt = fast_app(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
13 |
|
14 |
|
15 |
@app.get("/")
|
16 |
def main():
|
17 |
-
return
|
18 |
-
|
19 |
-
|
20 |
-
|
21 |
-
|
22 |
-
|
23 |
-
|
24 |
-
Link(rel="stylesheet", href="style.css"),
|
25 |
-
),
|
26 |
-
Body(
|
27 |
-
D_title(
|
28 |
-
H1(
|
29 |
-
"TxT360: fully open and transparent fusion of web and curated corpora for pre-training large language models",
|
30 |
-
cls="l-body",
|
31 |
-
style="text-align: center;",
|
32 |
-
),
|
33 |
-
Div(
|
34 |
-
Img(src="images/llm360_logo.png"),
|
35 |
-
id="title-plot",
|
36 |
-
cls="main-plot-container l-page",
|
37 |
-
),
|
38 |
),
|
39 |
-
|
40 |
-
|
41 |
-
|
42 |
-
|
43 |
-
|
44 |
-
|
45 |
-
|
46 |
-
|
47 |
-
|
48 |
-
|
49 |
-
|
50 |
-
|
51 |
-
|
52 |
-
|
53 |
-
|
54 |
-
|
55 |
-
|
56 |
-
|
57 |
-
|
58 |
-
|
59 |
-
|
60 |
-
|
61 |
-
|
62 |
-
|
63 |
-
|
64 |
-
|
65 |
-
|
66 |
-
|
67 |
-
|
68 |
-
|
69 |
-
|
70 |
-
|
71 |
-
|
72 |
-
|
73 |
-
|
74 |
-
|
75 |
-
|
76 |
-
|
77 |
-
|
78 |
-
|
79 |
-
|
80 |
-
|
81 |
-
|
|
|
|
|
|
|
|
|
|
|
82 |
),
|
83 |
),
|
84 |
-
Div(
|
85 |
-
A("Web Data", href="#inner-text"),
|
86 |
-
hx_get="/webdata",
|
87 |
-
hx_target="#inner-text",
|
88 |
-
),
|
89 |
-
Div(
|
90 |
-
A("Curated Sources", href="#inner-text"),
|
91 |
-
hx_get="/curated",
|
92 |
-
hx_target="#inner-text",
|
93 |
-
),
|
94 |
-
Div(
|
95 |
-
A("Common Steps", href="#inner-text"),
|
96 |
-
hx_get="/common",
|
97 |
-
hx_target="#inner-text",
|
98 |
-
),
|
99 |
-
Div(
|
100 |
-
A("TxT360 Results", href="#inner-text"),
|
101 |
-
hx_get="/results",
|
102 |
-
hx_target="#inner-text",
|
103 |
-
),
|
104 |
-
role="navigation",
|
105 |
-
cls="l-text figcaption",
|
106 |
),
|
107 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
108 |
),
|
109 |
-
intro(),
|
110 |
),
|
|
|
111 |
),
|
112 |
-
lang="en",
|
113 |
)
|
114 |
|
115 |
|
@@ -254,197 +259,12 @@ def intro():
|
|
254 |
)
|
255 |
|
256 |
|
257 |
-
|
258 |
-
def web_data():
|
259 |
-
return Div(Section(H2(P("Web Data")), id="inner-text"))
|
260 |
-
|
261 |
-
|
262 |
-
def get_chart_28168342():
|
263 |
-
fig = go.Figure()
|
264 |
-
filter_names = [
|
265 |
-
"Download",
|
266 |
-
"Language",
|
267 |
-
"Min word count",
|
268 |
-
"Title Abstract",
|
269 |
-
"Majority language",
|
270 |
-
"Paragraph count",
|
271 |
-
"Frequency",
|
272 |
-
"Unigram log probability",
|
273 |
-
"Local dedup",
|
274 |
-
]
|
275 |
-
|
276 |
-
data_sources = [
|
277 |
-
("Wikipedia", [100, 90, 80, 70, 60, 50, 40, 30, 20]),
|
278 |
-
("Freelaw", [100, 90, 80, 70, 60, 50, 40, 20, 20]),
|
279 |
-
("DM Maths", [100, 90, 80, 70, 60, 40, 40, 30, 20]),
|
280 |
-
("USPTO", [100, 90, 80, 70, 60, 40, 40, 30, 20]),
|
281 |
-
("PG19", [100, 90, 80, 70, 60, 40, 40, 30, 20]),
|
282 |
-
("Hackernews", [100, 90, 80, 70, 60, 40, 40, 30, 20]),
|
283 |
-
("Ubuntu IRC", [100, 90, 80, 70, 60, 40, 40, 30, 20]),
|
284 |
-
("Europarl", [100, 90, 80, 70, 60, 40, 40, 30, 20]),
|
285 |
-
("StackExchange", [100, 90, 80, 70, 60, 40, 40, 30, 20]),
|
286 |
-
("Arxiv", [100, 90, 80, 70, 60, 40, 40, 30, 20]),
|
287 |
-
("S2ORC", [100, 90, 80, 70, 60, 40, 40, 30, 20]),
|
288 |
-
("S2ORC Abstract", [100, 90, 80, 70, 60, 40, 40, 30, 20]),
|
289 |
-
("PubMed Central", [100, 90, 80, 70, 60, 40, 40, 30, 20]),
|
290 |
-
("PubMed Central Abstract", [100, 90, 80, 70, 60, 40, 40, 30, 20]),
|
291 |
-
("PhilPapers", [100, 90, 80, 70, 60, 40, 40, 30, 20]),
|
292 |
-
]
|
293 |
-
|
294 |
-
for name, x_values in data_sources:
|
295 |
-
fig.add_trace(
|
296 |
-
go.Funnel(
|
297 |
-
name=name,
|
298 |
-
orientation="h",
|
299 |
-
y=filter_names,
|
300 |
-
x=x_values,
|
301 |
-
textinfo="value+percent total",
|
302 |
-
textposition="inside",
|
303 |
-
)
|
304 |
-
)
|
305 |
-
|
306 |
-
fig.update_layout(height=500, plot_bgcolor="rgba(0,0,0,0)")
|
307 |
-
return fig
|
308 |
-
|
309 |
-
|
310 |
-
@app.get("/curated")
|
311 |
-
def curated(request):
|
312 |
-
from curated import get_data
|
313 |
-
|
314 |
-
# Partial Updates
|
315 |
-
params = request.query_params
|
316 |
-
if data_source := params.get("data_source"):
|
317 |
-
return get_data(data_source, params.get("doc_id", 3))
|
318 |
-
if doc_id := params.get("doc_id"):
|
319 |
-
return get_data(params.get("data_source"), doc_id)
|
320 |
-
|
321 |
-
hr = HR()
|
322 |
-
data_preparation_steps = pd.DataFrame(
|
323 |
-
{
|
324 |
-
"Method": [
|
325 |
-
"HTTP/FTP dumps",
|
326 |
-
"Web crawling",
|
327 |
-
"Archive snapshot",
|
328 |
-
"Generated",
|
329 |
-
"Curated",
|
330 |
-
],
|
331 |
-
"Description": [
|
332 |
-
"Acquiring data from HTTP/FTP dumps",
|
333 |
-
"Crawling websites to extract data",
|
334 |
-
"Working with archive dumps",
|
335 |
-
"Generating synthetic data",
|
336 |
-
"High quality curated data",
|
337 |
-
],
|
338 |
-
"Source": [
|
339 |
-
"Freelaw | Wikipedia | PhilPapers | Arxiv | S2ORC | Pubmeds",
|
340 |
-
"USPTO | Hackernews | Ubuntu IRC",
|
341 |
-
"StackExchange",
|
342 |
-
"DM Maths",
|
343 |
-
"PG19 | Europarl",
|
344 |
-
],
|
345 |
-
}
|
346 |
-
)
|
347 |
-
|
348 |
-
table_html = data_preparation_steps.to_html(index=False, border=0)
|
349 |
-
table_div = Div(NotStr(table_html), style="margin: 40px;")
|
350 |
-
|
351 |
-
text = P("""This initial stage serves as the foundation for the entire
|
352 |
-
process. Here, we focus on acquiring and extracting the raw data, which can
|
353 |
-
come from various sources such as crawling websites, using HTTP/FTP dumps,
|
354 |
-
or working with archive dumps. For instance, to download and prepare a
|
355 |
-
dataset, we can specific downloaders based on the data source. Each dataset
|
356 |
-
might have its own downloader script which can be updated in real time to
|
357 |
-
handle changes in the data source. Here is a general outline of the data
|
358 |
-
preparation process: It's worth noting that some pipelines might require
|
359 |
-
invoking additional functions or scripts to handle specific data sources or
|
360 |
-
formats. These helper scripts can be located within specific directories
|
361 |
-
or modules dedicated to the dataset.""")
|
362 |
-
|
363 |
-
data_preparation_div = Div(
|
364 |
-
H3("Data Preparation"),
|
365 |
-
text,
|
366 |
-
table_div,
|
367 |
-
Div(get_data(), style="border: 1px solid #ccc; padding: 20px;"),
|
368 |
-
)
|
369 |
-
|
370 |
-
text = P("""Data preprocessing is a crucial step in the data science
|
371 |
-
pipeline. It involves cleaning and transforming raw data into a format that
|
372 |
-
is suitable for analysis. This process includes handling missing values,
|
373 |
-
normalizing data, encoding categorical variables, and more.""")
|
374 |
-
|
375 |
-
preprocessing_steps = pd.DataFrame(
|
376 |
-
{
|
377 |
-
"Step": [
|
378 |
-
"Language Filter",
|
379 |
-
"Min Word Count",
|
380 |
-
"Title Abstract",
|
381 |
-
"Majority Language",
|
382 |
-
"Paragraph Count",
|
383 |
-
"Frequency",
|
384 |
-
"Unigram Log Probability",
|
385 |
-
],
|
386 |
-
"Description": [
|
387 |
-
"Filtering data based on language",
|
388 |
-
"Setting a minimum word count threshold",
|
389 |
-
"Extracting information from the title and abstract",
|
390 |
-
"Identifying the majority language in the dataset",
|
391 |
-
"Counting the number of paragraphs in each document",
|
392 |
-
"Calculating the frequency of each word in the dataset",
|
393 |
-
"Calculating the log probability of each unigram",
|
394 |
-
],
|
395 |
-
"Need": [
|
396 |
-
"To remove documents in unwanted languages",
|
397 |
-
"To filter out documents with very few words",
|
398 |
-
"To extract relevant information for analysis",
|
399 |
-
"To understand the distribution of languages in the dataset",
|
400 |
-
"To analyze the structure and length of documents",
|
401 |
-
"To identify important words in the dataset",
|
402 |
-
"To measure the significance of individual words",
|
403 |
-
],
|
404 |
-
"Pros": [
|
405 |
-
"Improves data quality by removing irrelevant documents",
|
406 |
-
"Filters out low-quality or incomplete documents",
|
407 |
-
"Provides additional information for analysis",
|
408 |
-
"Enables language-specific analysis and insights",
|
409 |
-
"Helps understand the complexity and content of documents",
|
410 |
-
"Identifies important terms and topics in the dataset",
|
411 |
-
"Quantifies the importance of individual words",
|
412 |
-
],
|
413 |
-
"Cons": [
|
414 |
-
"May exclude documents in less common languages",
|
415 |
-
"May remove documents with valuable information",
|
416 |
-
"May introduce bias in the analysis",
|
417 |
-
"May not accurately represent the language distribution",
|
418 |
-
"May not capture the complexity of document structure",
|
419 |
-
"May be sensitive to noise and outliers",
|
420 |
-
"May not capture the semantic meaning of words",
|
421 |
-
],
|
422 |
-
}
|
423 |
-
)
|
424 |
-
|
425 |
-
table_html = preprocessing_steps.to_html(index=False, border=0)
|
426 |
-
table_div = Div(NotStr(table_html), style="margin: 40px;")
|
427 |
-
data_preprocessing_div = Div(H3("Data Preprocessing"), text, table_div)
|
428 |
-
|
429 |
-
return Div(
|
430 |
-
Section(
|
431 |
-
H2("Curated Sources"),
|
432 |
-
plotly2fasthtml(get_chart_28168342()),
|
433 |
-
data_preparation_div,
|
434 |
-
data_preprocessing_div,
|
435 |
-
id="inner-text",
|
436 |
-
)
|
437 |
-
)
|
438 |
-
|
439 |
-
|
440 |
-
@app.get("/common")
|
441 |
-
def common_steps():
|
442 |
-
return Div(Section(H2(P("Common Steps")), id="inner-text"))
|
443 |
|
|
|
444 |
|
445 |
-
|
446 |
-
def results():
|
447 |
-
return Div(Section(H2(P("Results")), id="inner-text"))
|
448 |
|
|
|
449 |
|
450 |
serve()
|
|
|
1 |
from fasthtml.common import *
|
2 |
from fasthtml.components import *
|
3 |
from fasthtml.components import D_title, D_article, D_front_matter, D_contents, D_byline
|
|
|
4 |
from plotly import graph_objects as go
|
5 |
from fh_plotly import plotly2fasthtml
|
6 |
import pandas as pd
|
7 |
import json
|
8 |
from rich import print
|
9 |
+
import curated
|
10 |
+
import web
|
11 |
+
import common
|
12 |
+
import results
|
13 |
|
14 |
|
15 |
+
app, rt = fast_app(
|
16 |
+
debug=True,
|
17 |
+
pico=False,
|
18 |
+
hdrs=(
|
19 |
+
Meta(charset="UTF-8"),
|
20 |
+
Meta(name="viewport", content="width=device-width, initial-scale=1.0"),
|
21 |
+
Script(src="https://distill.pub/template.v2.js"),
|
22 |
+
Script(src="https://unpkg.com/htmx.org@next/dist/htmx.min.js"),
|
23 |
+
Script(src="https://cdn.plot.ly/plotly-latest.min.js"),
|
24 |
+
Link(rel="stylesheet", href="style.css"),
|
25 |
+
MarkdownJS(),
|
26 |
+
HighlightJS(langs=["python", "javascript", "html", "css"]),
|
27 |
+
),
|
28 |
+
)
|
29 |
|
30 |
|
31 |
@app.get("/")
|
32 |
def main():
|
33 |
+
return Div(
|
34 |
+
D_front_matter(),
|
35 |
+
D_title(
|
36 |
+
H1(
|
37 |
+
"TxT360: fully open and transparent fusion of web and curated corpora for pre-training large language models",
|
38 |
+
cls="l-body",
|
39 |
+
style="text-align: center;",
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
40 |
),
|
41 |
+
Div(
|
42 |
+
Img(src="images/llm360_logo.png"),
|
43 |
+
id="title-plot",
|
44 |
+
cls="main-plot-container l-page",
|
45 |
+
),
|
46 |
+
),
|
47 |
+
D_article(
|
48 |
+
D_contents(
|
49 |
+
Nav(
|
50 |
+
H3("Table of Contents"),
|
51 |
+
Div(
|
52 |
+
A("TxT360", href="#_self"),
|
53 |
+
hx_get="/intro",
|
54 |
+
hx_target="#inner-text",
|
55 |
+
),
|
56 |
+
Div(
|
57 |
+
Ul(
|
58 |
+
Li(
|
59 |
+
A(
|
60 |
+
"Introduction",
|
61 |
+
href="/intro#section1",
|
62 |
+
hx_get="/intro#section1",
|
63 |
+
hx_target="#inner-text",
|
64 |
+
)
|
65 |
+
),
|
66 |
+
Li(
|
67 |
+
A(
|
68 |
+
"Background",
|
69 |
+
href="/intro#section2",
|
70 |
+
hx_get="/intro#section2",
|
71 |
+
hx_target="#inner-text",
|
72 |
+
)
|
73 |
+
),
|
74 |
+
Li(
|
75 |
+
A(
|
76 |
+
"Main Content",
|
77 |
+
href="/intro#section3",
|
78 |
+
hx_get="/intro#section3",
|
79 |
+
hx_target="#inner-text",
|
80 |
+
)
|
81 |
+
),
|
82 |
+
Li(
|
83 |
+
A(
|
84 |
+
"Conclusion",
|
85 |
+
href="/intro#section4",
|
86 |
+
hx_get="/intro#section4",
|
87 |
+
hx_target="#inner-text",
|
88 |
+
)
|
89 |
),
|
90 |
),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
91 |
),
|
92 |
+
Div(
|
93 |
+
A("Web Data", href="#inner-text"),
|
94 |
+
hx_get="/webdata",
|
95 |
+
hx_target="#inner-text",
|
96 |
+
),
|
97 |
+
Div(
|
98 |
+
A("Curated Sources", href="#inner-text"),
|
99 |
+
hx_get="/curated",
|
100 |
+
hx_target="#inner-text",
|
101 |
+
),
|
102 |
+
Div(
|
103 |
+
A("Common Steps", href="#inner-text"),
|
104 |
+
hx_get="/common",
|
105 |
+
hx_target="#inner-text",
|
106 |
+
),
|
107 |
+
Div(
|
108 |
+
A("TxT360 Results", href="#inner-text"),
|
109 |
+
hx_get="/results",
|
110 |
+
hx_target="#inner-text",
|
111 |
+
),
|
112 |
+
role="navigation",
|
113 |
+
cls="l-text figcaption",
|
114 |
),
|
|
|
115 |
),
|
116 |
+
intro(),
|
117 |
),
|
|
|
118 |
)
|
119 |
|
120 |
|
|
|
259 |
)
|
260 |
|
261 |
|
262 |
+
rt("/curated")(curated.curated)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
263 |
|
264 |
+
rt("/webdata")(web.web_data)
|
265 |
|
266 |
+
rt("/common")(common.common_steps)
|
|
|
|
|
267 |
|
268 |
+
rt("/results")(results.results)
|
269 |
|
270 |
serve()
|
results.py
ADDED
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
from fasthtml.common import *
|
2 |
+
from fasthtml.components import *
|
3 |
+
|
4 |
+
|
5 |
+
def results():
|
6 |
+
return Div(Section(H2(P("Results")), id="inner-text"))
|
7 |
+
|
web.py
ADDED
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
from fasthtml.common import *
|
2 |
+
from fasthtml.components import *
|
3 |
+
|
4 |
+
|
5 |
+
def web_data():
|
6 |
+
return Div(Section(H2(P("Web Data")), id="inner-text"))
|
7 |
+
|