yjernite's picture
yjernite HF staff
Dataset card expanded by default
d552cdc
raw
history blame
2.73 kB
import json
import streamlit as st
st.set_page_config(
page_title="BigScience Training Corpus",
page_icon="https://avatars.githubusercontent.com/u/82455566",
layout="wide",
initial_sidebar_state="auto",
)
query_params = st.experimental_get_query_params()
@st.cache()
def load_catalogue():
full_catalogue = dict(
[
(source_name, source)
for source_name, source in json.load(
open("resources/sources_with_info_cards.json")
)
if source_name != "aggregated"
]
)
language_catalogues = {
"all": full_catalogue,
}
for source_name, source in full_catalogue.items():
for ln_dct in source["languages"]:
ln_code = "zh" if ln_dct["ln_code"].startswith("zh") else ln_dct["ln_code"]
language_catalogues[ln_code] = language_catalogues.get(ln_code, {})
language_catalogues[ln_code][source_name] = source
for ln in language_catalogues:
if ln != "all":
language_catalogues[ln] = dict(
sorted(
language_catalogues[ln].items(),
key=lambda x: [
ln_dct["size"]
for ln_dct in x[1]["languages"]
if ln_dct["ln_code"] == ln
][0],
reverse=True,
)
)
return dict(sorted(language_catalogues.items()))
catalogue_by_ln = load_catalogue()
with st.sidebar:
ln_select = st.selectbox(
"Show source list for language:",
catalogue_by_ln,
)
source_select = st.selectbox(
"Show information for source:",
catalogue_by_ln[ln_select],
index=list(catalogue_by_ln[ln_select]).index(
query_params.get("source", [list(catalogue_by_ln[ln_select].keys())[0]])[0]
) if ln_select == "all" else 0,
)
st.experimental_set_query_params(**{"source": source_select})
with st.expander(f"Dataset Card for {source_select}", expanded=True):
st.markdown(catalogue_by_ln["all"][source_select]["data_card"])
if "catalogue_info" in catalogue_by_ln["all"][source_select]:
with st.expander(f"Catalogue Information for {source_select}"):
st.write(catalogue_by_ln["all"][source_select]["catalogue_info"])
if "seed_info" in catalogue_by_ln["all"][source_select]:
with st.expander(f"Pseudocrawl Seed Information for {source_select}"):
st.write(catalogue_by_ln["all"][source_select]["seed_info"])
if "hf_info" in catalogue_by_ln["all"][source_select]:
with st.expander(f"HF Dataset Information for {source_select}"):
st.write(catalogue_by_ln["all"][source_select]["hf_info"])