# Start by setting token and debug mode before starting schedulers import os from huggingface_hub import logging, login login(token=os.environ.get("HF_TOKEN"), write_permission=True) logging.set_verbosity_debug() # Start apps from pathlib import Path import gradio as gr from app_1M_image import get_demo as get_demo_1M_image from app_image import get_demo as get_demo_image from app_json import get_demo as get_demo_json def _get_demo_code(path: str) -> str: code = Path(path).read_text() code = code.replace("def get_demo():", "with gr.Blocks() as demo:") code += "\n\ndemo.launch()" return code DEMO_EXPLANATION = """

How to persist data from a Space to a Dataset?

This demo shows how to leverage `gradio` and `huggingface_hub` to save data from a Space to a Dataset on the Hub. When doing so, a few things must be taken care of: file formats, concurrent writes, name collision, number of commits, number of files, and more. The tabs below show different ways of implementing a "save to dataset" feature. Depending on the complexity and usage of your app, you might want to use one or the other. This Space comes as a demo for this `huggingface_hub` [guide](https://huggingface.co/docs/huggingface_hub/main/en/guides/upload#scheduled-uploads). Please check it out if you need more technical details. """ JSON_DEMO_EXPLANATION = """ ## Use case - Save inputs and outputs - Build an annotation platform ## Data Json-able only: text and numeric but no binaries. ## Robustness Works with concurrent users and replicas. ## Limitations If you expect millions of lines, you must split the local JSON file into multiple files to avoid getting your file tracked as LFS (5MB) on the Hub. ## Demo """ IMAGE_DEMO_EXPLANATION = """ ## Use case Save images with metadata (caption, parameters, datetime, etc.). ## Robustness Works with concurrent users and replicas. ## Limitations - only 10k images/folder are supported on the Hub. If you expect more usage, you must save data in subfolders. - only 1M images/repo supported on the Hub. If you expect more usage, you can zip your data before uploading. See the _1M images Dataset_ demo. ## Demo """ IMAGE_1M_DEMO_EXPLANATION = """ ## Use case: Save 1M images with metadata (caption, parameters, datetime, etc.). ## Robustness Works with concurrent users and replicas. ## Limitations None. ## Demo """ with gr.Blocks() as demo: gr.Markdown(DEMO_EXPLANATION) with gr.Tab("JSON Dataset"): gr.Markdown(JSON_DEMO_EXPLANATION) get_demo_json() gr.Markdown("## Result\n\nhttps://huggingface.co/datasets/Wauplin/example-space-to-dataset-json\n\n## Code") with gr.Accordion("Source code", open=True): gr.Code(_get_demo_code("app_json.py"), language="python") with gr.Tab("Image Dataset"): gr.Markdown(IMAGE_DEMO_EXPLANATION) get_demo_image() gr.Markdown("## Result\n\nhttps://huggingface.co/datasets/Wauplin/example-space-to-dataset-image\n\n## Code") with gr.Accordion("Source code", open=True): gr.Code(_get_demo_code("app_image.py"), language="python") with gr.Tab("1M images Dataset"): gr.Markdown(IMAGE_1M_DEMO_EXPLANATION) get_demo_1M_image() gr.Markdown( "## Result\n\nhttps://huggingface.co/datasets/Wauplin/example-space-to-dataset-image-zip\n\n## Code" ) with gr.Accordion("Source code", open=True): gr.Code(_get_demo_code("app_1M_image.py"), language="python") demo.launch()