Spaces:
Runtime error
Runtime error
| # import datetime | |
| import json | |
| import os | |
| from pathlib import Path | |
| import gradio as gr | |
| import huggingface_hub as hfh | |
| # from apscheduler.schedulers.background import BackgroundScheduler | |
| DATASET_ID = "albertvillanova/datasets-report" | |
| DATASET_PATH = "dataset" | |
| DATA_DIR = "data" | |
| DATA_PATH = f"{DATASET_PATH}/{DATA_DIR}" | |
| def pull_dataset_repo(repo_id=DATASET_ID, repo_path=DATASET_PATH): | |
| token = os.environ.get('HUB_TOKEN') | |
| repo = hfh.Repository( | |
| local_dir=repo_path, | |
| clone_from=repo_id, | |
| repo_type="dataset", | |
| use_auth_token=token, | |
| ) | |
| repo.git_pull() | |
| return repo | |
| def load_dates(): | |
| return [data_path.stem for data_path in sorted(Path(DATA_PATH).iterdir())] | |
| repo = pull_dataset_repo() | |
| dates = load_dates() | |
| datasets = hfh.list_datasets() | |
| def filter_datasets_by_date(date_from, date_to): | |
| with open(f"{DATA_PATH}/{date_from}.json") as f: | |
| ids_from = json.load(f) | |
| with open(f"{DATA_PATH}/{date_to}.json") as f: | |
| ids_to = json.load(f) | |
| ids = set(ids_to) - set(ids_from) | |
| dss = [ds for ds in datasets if ds.id in ids] | |
| for ds in dss: | |
| try: | |
| _ = getattr(ds, "downloads") | |
| except AttributeError: | |
| setattr(ds, "downloads", 0) | |
| dss = sorted(dss, key=lambda item: item.downloads, reverse=True) | |
| return dss | |
| def filter_dataframe(date_from, date_to): | |
| dss = filter_datasets_by_date(date_from, date_to) | |
| return [[ds.id, ds.downloads] for ds in dss] | |
| # def update_datasets(): | |
| # # Retrieve datasets | |
| # datasets = hfh.list_datasets() | |
| # # Save dataset IDs | |
| # repo = pull_dataset_repo() | |
| # os.makedirs(DATA_PATH, exist_ok=True) | |
| # today = datetime.datetime.now(datetime.timezone.utc).date().isoformat() | |
| # with repo.commit(f"Add {today} data file"): | |
| # with open(f"data/{today}.json", "w") as f: | |
| # json.dump([ds.id for ds in sorted(datasets, key=lambda item: item.id)], f) | |
| # | |
| # | |
| # scheduler = BackgroundScheduler() | |
| # scheduler.add_job(update_datasets, trigger="cron", hour=0, minute=1, timezone=datetime.timezone.utc) | |
| # scheduler.start() | |
| with gr.Blocks() as demo: | |
| with gr.Row(): | |
| date_from = gr.Dropdown(choices=dates, label="Date from") | |
| date_to = gr.Dropdown(choices=dates, label="Date to") | |
| submit_btn = gr.Button("Submit") | |
| outputs = gr.Dataframe( | |
| headers=["Dataset", "Downloads"], | |
| datatype=["str", "number"], | |
| label="Created datasets", | |
| ) | |
| submit_btn.click(fn=filter_dataframe, inputs=[date_from, date_to], outputs=outputs) | |
| demo.launch() | |