victormiller
commited on
Commit
•
0dbc84f
1
Parent(s):
775d041
Update curated.py
Browse files- curated.py +2 -1
curated.py
CHANGED
@@ -445,8 +445,9 @@ filtering_process = Div(
|
|
445 |
Section(
|
446 |
Div(
|
447 |
H3("Wikipedia"),
|
|
|
|
|
448 |
H4("Download and Extraction"),
|
449 |
-
|
450 |
P("The Wikimedia dataset was downloaded from the official snapshot on Huggingface: ", A("https://huggingface.co/datasets/wikimedia/wikipedia/tree/main", href="https://huggingface.co/datasets/wikimedia/wikipedia/tree/main"), ". The", D_code("huggingface dataset.to_json", language="python"), " function was used to convert the original parqet format to the jsonl format."),
|
451 |
H4("Filtering"),
|
452 |
P("Manual inspection of the dataset demostrated high quality content. Only one filter was used to remove articles with few words. Based normal sentence constructs, the article was kept if it contained 10 or more words. Any article with fewer than 10 words was removed."),
|
|
|
445 |
Section(
|
446 |
Div(
|
447 |
H3("Wikipedia"),
|
448 |
+
H4("What is Wikipedia:")
|
449 |
+
P("Wikipedia is an encyclopedia form of high-quality text data used for language modeling. We have included filtered and deduplicated versions of complete Wikipedia data directly provided by the Wikipedia Foundation for more than 350 languages."),
|
450 |
H4("Download and Extraction"),
|
|
|
451 |
P("The Wikimedia dataset was downloaded from the official snapshot on Huggingface: ", A("https://huggingface.co/datasets/wikimedia/wikipedia/tree/main", href="https://huggingface.co/datasets/wikimedia/wikipedia/tree/main"), ". The", D_code("huggingface dataset.to_json", language="python"), " function was used to convert the original parqet format to the jsonl format."),
|
452 |
H4("Filtering"),
|
453 |
P("Manual inspection of the dataset demostrated high quality content. Only one filter was used to remove articles with few words. Based normal sentence constructs, the article was kept if it contained 10 or more words. Any article with fewer than 10 words was removed."),
|