Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+assets/trackio_logo_old.png filter=lfs diff=lfs merge=lfs -text

CHANGELOG.md

@@ -0,0 +1,24 @@
+# trackio
+
+## 0.5.3
+
+### Features
+
+- [#300](https://github.com/gradio-app/trackio/pull/300) [`5e4cacf`](https://github.com/gradio-app/trackio/commit/5e4cacf2e7ce527b4ce60de3a5bc05d2c02c77fb) - Adds more environment variables to allow customization of Trackio dashboard.  Thanks @abidlabs!
+
+## 0.5.2
+
+### Features
+
+- [#293](https://github.com/gradio-app/trackio/pull/293) [`64afc28`](https://github.com/gradio-app/trackio/commit/64afc28d3ea1dfd821472dc6bf0b8ed35a9b74be) - Ensures that the TRACKIO_DIR environment variable is respected.  Thanks @abidlabs!
+- [#287](https://github.com/gradio-app/trackio/pull/287) [`cd3e929`](https://github.com/gradio-app/trackio/commit/cd3e9294320949e6b8b829239069a43d5d7ff4c1) - fix(sqlite): unify .sqlite extension, allow export when DBs exist, clean WAL sidecars on import.  Thanks @vaibhav-research!
+
+### Fixes
+
+- [#291](https://github.com/gradio-app/trackio/pull/291) [`3b5adc3`](https://github.com/gradio-app/trackio/commit/3b5adc3d1f452dbab7a714d235f4974782f93730) - Fix the wheel build.  Thanks @pngwn!
+
+## 0.5.1
+
+### Fixes
+
+- [#278](https://github.com/gradio-app/trackio/pull/278) [`314c054`](https://github.com/gradio-app/trackio/commit/314c05438007ddfea3383e06fd19143e27468e2d) - Fix row orientation of metrics plots.  Thanks @abidlabs!

__init__.py

@@ -0,0 +1,332 @@
+import hashlib
+import logging
+import os
+import warnings
+import webbrowser
+from pathlib import Path
+from typing import Any
+
+from gradio.blocks import BUILT_IN_THEMES
+from gradio.themes import Default as DefaultTheme
+from gradio.themes import ThemeClass
+from gradio_client import Client
+from huggingface_hub import SpaceStorage
+
+from trackio import context_vars, deploy, utils
+from trackio.imports import import_csv, import_tf_events
+from trackio.media import TrackioImage, TrackioVideo
+from trackio.run import Run
+from trackio.sqlite_storage import SQLiteStorage
+from trackio.table import Table
+from trackio.ui.main import demo
+from trackio.utils import TRACKIO_DIR, TRACKIO_LOGO_DIR
+
+logging.getLogger("httpx").setLevel(logging.WARNING)
+
+warnings.filterwarnings(
+    "ignore",
+    message="Empty session being created. Install gradio\\[oauth\\]",
+    category=UserWarning,
+    module="gradio.helpers",
+)
+
+__version__ = Path(__file__).parent.joinpath("version.txt").read_text().strip()
+
+__all__ = [
+    "init",
+    "log",
+    "finish",
+    "show",
+    "import_csv",
+    "import_tf_events",
+    "Image",
+    "Video",
+    "Table",
+]
+
+Image = TrackioImage
+Video = TrackioVideo
+
+
+config = {}
+
+DEFAULT_THEME = "default"
+
+
+def init(
+    project: str,
+    name: str | None = None,
+    group: str | None = None,
+    space_id: str | None = None,
+    space_storage: SpaceStorage | None = None,
+    dataset_id: str | None = None,
+    config: dict | None = None,
+    resume: str = "never",
+    settings: Any = None,
+    private: bool | None = None,
+    embed: bool = True,
+) -> Run:
+    """
+    Creates a new Trackio project and returns a [`Run`] object.
+
+    Args:
+        project (`str`):
+            The name of the project (can be an existing project to continue tracking or
+            a new project to start tracking from scratch).
+        name (`str`, *optional*):
+            The name of the run (if not provided, a default name will be generated).
+        group (`str`, *optional*):
+            The name of the group which this run belongs to in order to help organize
+            related runs together. You can toggle the entire group's visibilitiy in the
+            dashboard.
+        space_id (`str`, *optional*):
+            If provided, the project will be logged to a Hugging Face Space instead of
+            a local directory. Should be a complete Space name like
+            `"username/reponame"` or `"orgname/reponame"`, or just `"reponame"` in which
+            case the Space will be created in the currently-logged-in Hugging Face
+            user's namespace. If the Space does not exist, it will be created. If the
+            Space already exists, the project will be logged to it.
+        space_storage ([`~huggingface_hub.SpaceStorage`], *optional*):
+            Choice of persistent storage tier.
+        dataset_id (`str`, *optional*):
+            If a `space_id` is provided, a persistent Hugging Face Dataset will be
+            created and the metrics will be synced to it every 5 minutes. Specify a
+            Dataset with name like `"username/datasetname"` or `"orgname/datasetname"`,
+            or `"datasetname"` (uses currently-logged-in Hugging Face user's namespace),
+            or `None` (uses the same name as the Space but with the `"_dataset"`
+            suffix). If the Dataset does not exist, it will be created. If the Dataset
+            already exists, the project will be appended to it.
+        config (`dict`, *optional*):
+            A dictionary of configuration options. Provided for compatibility with
+            `wandb.init()`.
+        resume (`str`, *optional*, defaults to `"never"`):
+            Controls how to handle resuming a run. Can be one of:
+
+            - `"must"`: Must resume the run with the given name, raises error if run
+              doesn't exist
+            - `"allow"`: Resume the run if it exists, otherwise create a new run
+            - `"never"`: Never resume a run, always create a new one
+        private (`bool`, *optional*):
+            Whether to make the Space private. If None (default), the repo will be
+            public unless the organization's default is private. This value is ignored
+            if the repo already exists.
+        settings (`Any`, *optional*):
+            Not used. Provided for compatibility with `wandb.init()`.
+        embed (`bool`, *optional*, defaults to `True`):
+            If running inside a jupyter/Colab notebook, whether the dashboard should
+            automatically be embedded in the cell when trackio.init() is called.
+
+    Returns:
+        `Run`: A [`Run`] object that can be used to log metrics and finish the run.
+    """
+    if settings is not None:
+        warnings.warn(
+            "* Warning: settings is not used. Provided for compatibility with wandb.init(). Please create an issue at: https://github.com/gradio-app/trackio/issues if you need a specific feature implemented."
+        )
+
+    if space_id is None and dataset_id is not None:
+        raise ValueError("Must provide a `space_id` when `dataset_id` is provided.")
+    space_id, dataset_id = utils.preprocess_space_and_dataset_ids(space_id, dataset_id)
+    url = context_vars.current_server.get()
+    share_url = context_vars.current_share_server.get()
+
+    if url is None:
+        if space_id is None:
+            _, url, share_url = demo.launch(
+                show_api=False,
+                inline=False,
+                quiet=True,
+                prevent_thread_lock=True,
+                show_error=True,
+                favicon_path=TRACKIO_LOGO_DIR / "trackio_logo_light.png",
+                allowed_paths=[TRACKIO_LOGO_DIR],
+            )
+        else:
+            url = space_id
+            share_url = None
+        context_vars.current_server.set(url)
+        context_vars.current_share_server.set(share_url)
+    if (
+        context_vars.current_project.get() is None
+        or context_vars.current_project.get() != project
+    ):
+        print(f"* Trackio project initialized: {project}")
+
+        if dataset_id is not None:
+            os.environ["TRACKIO_DATASET_ID"] = dataset_id
+            print(
+                f"* Trackio metrics will be synced to Hugging Face Dataset: {dataset_id}"
+            )
+        if space_id is None:
+            print(f"* Trackio metrics logged to: {TRACKIO_DIR}")
+            if utils.is_in_notebook() and embed:
+                base_url = share_url + "/" if share_url else url
+                full_url = utils.get_full_url(
+                    base_url, project=project, write_token=demo.write_token
+                )
+                utils.embed_url_in_notebook(full_url)
+            else:
+                utils.print_dashboard_instructions(project)
+        else:
+            deploy.create_space_if_not_exists(
+                space_id, space_storage, dataset_id, private
+            )
+            user_name, space_name = space_id.split("/")
+            space_url = deploy.SPACE_HOST_URL.format(
+                user_name=user_name, space_name=space_name
+            )
+            print(f"* View dashboard by going to: {space_url}")
+            if utils.is_in_notebook() and embed:
+                utils.embed_url_in_notebook(space_url)
+    context_vars.current_project.set(project)
+
+    client = None
+    if not space_id:
+        client = Client(url, verbose=False)
+
+    if resume == "must":
+        if name is None:
+            raise ValueError("Must provide a run name when resume='must'")
+        if name not in SQLiteStorage.get_runs(project):
+            raise ValueError(f"Run '{name}' does not exist in project '{project}'")
+        resumed = True
+    elif resume == "allow":
+        resumed = name is not None and name in SQLiteStorage.get_runs(project)
+    elif resume == "never":
+        if name is not None and name in SQLiteStorage.get_runs(project):
+            warnings.warn(
+                f"* Warning: resume='never' but a run '{name}' already exists in "
+                f"project '{project}'. Generating a new name and instead. If you want "
+                "to resume this run, call init() with resume='must' or resume='allow'."
+            )
+            name = None
+        resumed = False
+    else:
+        raise ValueError("resume must be one of: 'must', 'allow', or 'never'")
+
+    run = Run(
+        url=url,
+        project=project,
+        client=client,
+        name=name,
+        group=group,
+        config=config,
+        space_id=space_id,
+    )
+
+    if resumed:
+        print(f"* Resumed existing run: {run.name}")
+    else:
+        print(f"* Created new run: {run.name}")
+
+    context_vars.current_run.set(run)
+    globals()["config"] = run.config
+    return run
+
+
+def log(metrics: dict, step: int | None = None) -> None:
+    """
+    Logs metrics to the current run.
+
+    Args:
+        metrics (`dict`):
+            A dictionary of metrics to log.
+        step (`int`, *optional*):
+            The step number. If not provided, the step will be incremented
+            automatically.
+    """
+    run = context_vars.current_run.get()
+    if run is None:
+        raise RuntimeError("Call trackio.init() before trackio.log().")
+    run.log(
+        metrics=metrics,
+        step=step,
+    )
+
+
+def finish():
+    """
+    Finishes the current run.
+    """
+    run = context_vars.current_run.get()
+    if run is None:
+        raise RuntimeError("Call trackio.init() before trackio.finish().")
+    run.finish()
+
+
+def show(
+    project: str | None = None,
+    theme: str | ThemeClass | None = None,
+    mcp_server: bool | None = None,
+):
+    """
+    Launches the Trackio dashboard.
+
+    Args:
+        project (`str`, *optional*):
+            The name of the project whose runs to show. If not provided, all projects
+            will be shown and the user can select one.
+        theme (`str` or `ThemeClass`, *optional*):
+            A Gradio Theme to use for the dashboard instead of the default Gradio theme,
+            can be a built-in theme (e.g. `'soft'`, `'citrus'`), a theme from the Hub
+            (e.g. `"gstaff/xkcd"`), or a custom Theme class. If not provided, the
+            `TRACKIO_THEME` environment variable will be used, or if that is not set, the
+            default Gradio theme will be used.
+        mcp_server (`bool`, *optional*):
+            If `True`, the Trackio dashboard will be set up as an MCP server and certain
+            functions will be added as MCP tools. If `None` (default behavior), then the
+            `GRADIO_MCP_SERVER` environment variable will be used to determine if the
+            MCP server should be enabled (which is `"True"` on Hugging Face Spaces).
+    """
+    theme = theme or os.environ.get("TRACKIO_THEME", DEFAULT_THEME)
+
+    if theme != DEFAULT_THEME:
+        # TODO: It's a little hacky to reproduce this theme-setting logic from Gradio Blocks,
+        # but in Gradio 6.0, the theme will be set in `launch()` instead, which means that we
+        # will be able to remove this code.
+        if isinstance(theme, str):
+            if theme.lower() in BUILT_IN_THEMES:
+                theme = BUILT_IN_THEMES[theme.lower()]
+            else:
+                try:
+                    theme = ThemeClass.from_hub(theme)
+                except Exception as e:
+                    warnings.warn(f"Cannot load {theme}. Caught Exception: {str(e)}")
+                    theme = DefaultTheme()
+        if not isinstance(theme, ThemeClass):
+            warnings.warn("Theme should be a class loaded from gradio.themes")
+            theme = DefaultTheme()
+        demo.theme: ThemeClass = theme
+        demo.theme_css = theme._get_theme_css()
+        demo.stylesheets = theme._stylesheets
+        theme_hasher = hashlib.sha256()
+        theme_hasher.update(demo.theme_css.encode("utf-8"))
+        demo.theme_hash = theme_hasher.hexdigest()
+
+    _mcp_server = (
+        mcp_server
+        if mcp_server is not None
+        else os.environ.get("GRADIO_MCP_SERVER", "False") == "True"
+    )
+
+    _, url, share_url = demo.launch(
+        show_api=_mcp_server,
+        quiet=True,
+        inline=False,
+        prevent_thread_lock=True,
+        favicon_path=TRACKIO_LOGO_DIR / "trackio_logo_light.png",
+        allowed_paths=[TRACKIO_LOGO_DIR],
+        mcp_server=_mcp_server,
+    )
+
+    base_url = share_url + "/" if share_url else url
+    full_url = utils.get_full_url(
+        base_url, project=project, write_token=demo.write_token
+    )
+
+    if not utils.is_in_notebook():
+        print(f"* Trackio UI launched at: {full_url}")
+        webbrowser.open(full_url)
+        utils.block_main_thread_until_keyboard_interrupt()
+    else:
+        utils.embed_url_in_notebook(full_url)

cli.py

@@ -0,0 +1,37 @@
+import argparse
+
+from trackio import show
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Trackio CLI")
+    subparsers = parser.add_subparsers(dest="command")
+
+    ui_parser = subparsers.add_parser(
+        "show", help="Show the Trackio dashboard UI for a project"
+    )
+    ui_parser.add_argument(
+        "--project", required=False, help="Project name to show in the dashboard"
+    )
+    ui_parser.add_argument(
+        "--theme",
+        required=False,
+        default="citrus",
+        help="A Gradio Theme to use for the dashboard instead of the default, can be a built-in theme (e.g. 'soft', 'citrus'), or a theme from the Hub (e.g. 'gstaff/xkcd').",
+    )
+    ui_parser.add_argument(
+        "--mcp-server",
+        action="store_true",
+        help="Enable MCP server functionality. The Trackio dashboard will be set up as an MCP server and certain functions will be exposed as MCP tools.",
+    )
+
+    args = parser.parse_args()
+
+    if args.command == "show":
+        show(args.project, args.theme, args.mcp_server)
+    else:
+        parser.print_help()
+
+
+if __name__ == "__main__":
+    main()

commit_scheduler.py

@@ -0,0 +1,391 @@
+# Originally copied from https://github.com/huggingface/huggingface_hub/blob/d0a948fc2a32ed6e557042a95ef3e4af97ec4a7c/src/huggingface_hub/_commit_scheduler.py
+
+import atexit
+import logging
+import os
+import time
+from concurrent.futures import Future
+from dataclasses import dataclass
+from io import SEEK_END, SEEK_SET, BytesIO
+from pathlib import Path
+from threading import Lock, Thread
+from typing import Callable, Dict, List, Union
+
+from huggingface_hub.hf_api import (
+    DEFAULT_IGNORE_PATTERNS,
+    CommitInfo,
+    CommitOperationAdd,
+    HfApi,
+)
+from huggingface_hub.utils import filter_repo_objects
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(frozen=True)
+class _FileToUpload:
+    """Temporary dataclass to store info about files to upload. Not meant to be used directly."""
+
+    local_path: Path
+    path_in_repo: str
+    size_limit: int
+    last_modified: float
+
+
+class CommitScheduler:
+    """
+    Scheduler to upload a local folder to the Hub at regular intervals (e.g. push to hub every 5 minutes).
+
+    The recommended way to use the scheduler is to use it as a context manager. This ensures that the scheduler is
+    properly stopped and the last commit is triggered when the script ends. The scheduler can also be stopped manually
+    with the `stop` method. Checkout the [upload guide](https://huggingface.co/docs/huggingface_hub/guides/upload#scheduled-uploads)
+    to learn more about how to use it.
+
+    Args:
+        repo_id (`str`):
+            The id of the repo to commit to.
+        folder_path (`str` or `Path`):
+            Path to the local folder to upload regularly.
+        every (`int` or `float`, *optional*):
+            The number of minutes between each commit. Defaults to 5 minutes.
+        path_in_repo (`str`, *optional*):
+            Relative path of the directory in the repo, for example: `"checkpoints/"`. Defaults to the root folder
+            of the repository.
+        repo_type (`str`, *optional*):
+            The type of the repo to commit to. Defaults to `model`.
+        revision (`str`, *optional*):
+            The revision of the repo to commit to. Defaults to `main`.
+        private (`bool`, *optional*):
+            Whether to make the repo private. If `None` (default), the repo will be public unless the organization's default is private. This value is ignored if the repo already exists.
+        token (`str`, *optional*):
+            The token to use to commit to the repo. Defaults to the token saved on the machine.
+        allow_patterns (`List[str]` or `str`, *optional*):
+            If provided, only files matching at least one pattern are uploaded.
+        ignore_patterns (`List[str]` or `str`, *optional*):
+            If provided, files matching any of the patterns are not uploaded.
+        squash_history (`bool`, *optional*):
+            Whether to squash the history of the repo after each commit. Defaults to `False`. Squashing commits is
+            useful to avoid degraded performances on the repo when it grows too large.
+        hf_api (`HfApi`, *optional*):
+            The [`HfApi`] client to use to commit to the Hub. Can be set with custom settings (user agent, token,...).
+        on_before_commit (`Callable[[], None]`, *optional*):
+            If specified, a function that will be called before the CommitScheduler lists files to create a commit.
+
+    Example:
+    ```py
+    >>> from pathlib import Path
+    >>> from huggingface_hub import CommitScheduler
+
+    # Scheduler uploads every 10 minutes
+    >>> csv_path = Path("watched_folder/data.csv")
+    >>> CommitScheduler(repo_id="test_scheduler", repo_type="dataset", folder_path=csv_path.parent, every=10)
+
+    >>> with csv_path.open("a") as f:
+    ...     f.write("first line")
+
+    # Some time later (...)
+    >>> with csv_path.open("a") as f:
+    ...     f.write("second line")
+    ```
+
+    Example using a context manager:
+    ```py
+    >>> from pathlib import Path
+    >>> from huggingface_hub import CommitScheduler
+
+    >>> with CommitScheduler(repo_id="test_scheduler", repo_type="dataset", folder_path="watched_folder", every=10) as scheduler:
+    ...     csv_path = Path("watched_folder/data.csv")
+    ...     with csv_path.open("a") as f:
+    ...         f.write("first line")
+    ...     (...)
+    ...     with csv_path.open("a") as f:
+    ...         f.write("second line")
+
+    # Scheduler is now stopped and last commit have been triggered
+    ```
+    """
+
+    def __init__(
+        self,
+        *,
+        repo_id: str,
+        folder_path: Union[str, Path],
+        every: Union[int, float] = 5,
+        path_in_repo: str | None = None,
+        repo_type: str | None = None,
+        revision: str | None = None,
+        private: bool | None = None,
+        token: str | None = None,
+        allow_patterns: list[str] | str | None = None,
+        ignore_patterns: list[str] | str | None = None,
+        squash_history: bool = False,
+        hf_api: HfApi | None = None,
+        on_before_commit: Callable[[], None] | None = None,
+    ) -> None:
+        self.api = hf_api or HfApi(token=token)
+        self.on_before_commit = on_before_commit
+
+        # Folder
+        self.folder_path = Path(folder_path).expanduser().resolve()
+        self.path_in_repo = path_in_repo or ""
+        self.allow_patterns = allow_patterns
+
+        if ignore_patterns is None:
+            ignore_patterns = []
+        elif isinstance(ignore_patterns, str):
+            ignore_patterns = [ignore_patterns]
+        self.ignore_patterns = ignore_patterns + DEFAULT_IGNORE_PATTERNS
+
+        if self.folder_path.is_file():
+            raise ValueError(
+                f"'folder_path' must be a directory, not a file: '{self.folder_path}'."
+            )
+        self.folder_path.mkdir(parents=True, exist_ok=True)
+
+        # Repository
+        repo_url = self.api.create_repo(
+            repo_id=repo_id, private=private, repo_type=repo_type, exist_ok=True
+        )
+        self.repo_id = repo_url.repo_id
+        self.repo_type = repo_type
+        self.revision = revision
+        self.token = token
+
+        self.last_uploaded: Dict[Path, float] = {}
+        self.last_push_time: float | None = None
+
+        if not every > 0:
+            raise ValueError(f"'every' must be a positive integer, not '{every}'.")
+        self.lock = Lock()
+        self.every = every
+        self.squash_history = squash_history
+
+        logger.info(
+            f"Scheduled job to push '{self.folder_path}' to '{self.repo_id}' every {self.every} minutes."
+        )
+        self._scheduler_thread = Thread(target=self._run_scheduler, daemon=True)
+        self._scheduler_thread.start()
+        atexit.register(self._push_to_hub)
+
+        self.__stopped = False
+
+    def stop(self) -> None:
+        """Stop the scheduler.
+
+        A stopped scheduler cannot be restarted. Mostly for tests purposes.
+        """
+        self.__stopped = True
+
+    def __enter__(self) -> "CommitScheduler":
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback) -> None:
+        # Upload last changes before exiting
+        self.trigger().result()
+        self.stop()
+        return
+
+    def _run_scheduler(self) -> None:
+        """Dumb thread waiting between each scheduled push to Hub."""
+        while True:
+            self.last_future = self.trigger()
+            time.sleep(self.every * 60)
+            if self.__stopped:
+                break
+
+    def trigger(self) -> Future:
+        """Trigger a `push_to_hub` and return a future.
+
+        This method is automatically called every `every` minutes. You can also call it manually to trigger a commit
+        immediately, without waiting for the next scheduled commit.
+        """
+        return self.api.run_as_future(self._push_to_hub)
+
+    def _push_to_hub(self) -> CommitInfo | None:
+        if self.__stopped:  # If stopped, already scheduled commits are ignored
+            return None
+
+        logger.info("(Background) scheduled commit triggered.")
+        try:
+            value = self.push_to_hub()
+            if self.squash_history:
+                logger.info("(Background) squashing repo history.")
+                self.api.super_squash_history(
+                    repo_id=self.repo_id, repo_type=self.repo_type, branch=self.revision
+                )
+            return value
+        except Exception as e:
+            logger.error(
+                f"Error while pushing to Hub: {e}"
+            )  # Depending on the setup, error might be silenced
+            raise
+
+    def push_to_hub(self) -> CommitInfo | None:
+        """
+        Push folder to the Hub and return the commit info.
+
+        <Tip warning={true}>
+
+        This method is not meant to be called directly. It is run in the background by the scheduler, respecting a
+        queue mechanism to avoid concurrent commits. Making a direct call to the method might lead to concurrency
+        issues.
+
+        </Tip>
+
+        The default behavior of `push_to_hub` is to assume an append-only folder. It lists all files in the folder and
+        uploads only changed files. If no changes are found, the method returns without committing anything. If you want
+        to change this behavior, you can inherit from [`CommitScheduler`] and override this method. This can be useful
+        for example to compress data together in a single file before committing. For more details and examples, check
+        out our [integration guide](https://huggingface.co/docs/huggingface_hub/main/en/guides/upload#scheduled-uploads).
+        """
+        # Check files to upload (with lock)
+        with self.lock:
+            if self.on_before_commit is not None:
+                self.on_before_commit()
+
+            logger.debug("Listing files to upload for scheduled commit.")
+
+            # List files from folder (taken from `_prepare_upload_folder_additions`)
+            relpath_to_abspath = {
+                path.relative_to(self.folder_path).as_posix(): path
+                for path in sorted(
+                    self.folder_path.glob("**/*")
+                )  # sorted to be deterministic
+                if path.is_file()
+            }
+            prefix = f"{self.path_in_repo.strip('/')}/" if self.path_in_repo else ""
+
+            # Filter with pattern + filter out unchanged files + retrieve current file size
+            files_to_upload: List[_FileToUpload] = []
+            for relpath in filter_repo_objects(
+                relpath_to_abspath.keys(),
+                allow_patterns=self.allow_patterns,
+                ignore_patterns=self.ignore_patterns,
+            ):
+                local_path = relpath_to_abspath[relpath]
+                stat = local_path.stat()
+                if (
+                    self.last_uploaded.get(local_path) is None
+                    or self.last_uploaded[local_path] != stat.st_mtime
+                ):
+                    files_to_upload.append(
+                        _FileToUpload(
+                            local_path=local_path,
+                            path_in_repo=prefix + relpath,
+                            size_limit=stat.st_size,
+                            last_modified=stat.st_mtime,
+                        )
+                    )
+
+        # Return if nothing to upload
+        if len(files_to_upload) == 0:
+            logger.debug("Dropping schedule commit: no changed file to upload.")
+            return None
+
+        # Convert `_FileToUpload` as `CommitOperationAdd` (=> compute file shas + limit to file size)
+        logger.debug("Removing unchanged files since previous scheduled commit.")
+        add_operations = [
+            CommitOperationAdd(
+                # TODO: Cap the file to its current size, even if the user append data to it while a scheduled commit is happening
+                # (requires an upstream fix for XET-535: `hf_xet` should support `BinaryIO` for upload)
+                path_or_fileobj=file_to_upload.local_path,
+                path_in_repo=file_to_upload.path_in_repo,
+            )
+            for file_to_upload in files_to_upload
+        ]
+
+        # Upload files (append mode expected - no need for lock)
+        logger.debug("Uploading files for scheduled commit.")
+        commit_info = self.api.create_commit(
+            repo_id=self.repo_id,
+            repo_type=self.repo_type,
+            operations=add_operations,
+            commit_message="Scheduled Commit",
+            revision=self.revision,
+        )
+
+        for file in files_to_upload:
+            self.last_uploaded[file.local_path] = file.last_modified
+
+        self.last_push_time = time.time()
+
+        return commit_info
+
+
+class PartialFileIO(BytesIO):
+    """A file-like object that reads only the first part of a file.
+
+    Useful to upload a file to the Hub when the user might still be appending data to it. Only the first part of the
+    file is uploaded (i.e. the part that was available when the filesystem was first scanned).
+
+    In practice, only used internally by the CommitScheduler to regularly push a folder to the Hub with minimal
+    disturbance for the user. The object is passed to `CommitOperationAdd`.
+
+    Only supports `read`, `tell` and `seek` methods.
+
+    Args:
+        file_path (`str` or `Path`):
+            Path to the file to read.
+        size_limit (`int`):
+            The maximum number of bytes to read from the file. If the file is larger than this, only the first part
+            will be read (and uploaded).
+    """
+
+    def __init__(self, file_path: Union[str, Path], size_limit: int) -> None:
+        self._file_path = Path(file_path)
+        self._file = self._file_path.open("rb")
+        self._size_limit = min(size_limit, os.fstat(self._file.fileno()).st_size)
+
+    def __del__(self) -> None:
+        self._file.close()
+        return super().__del__()
+
+    def __repr__(self) -> str:
+        return (
+            f"<PartialFileIO file_path={self._file_path} size_limit={self._size_limit}>"
+        )
+
+    def __len__(self) -> int:
+        return self._size_limit
+
+    def __getattribute__(self, name: str):
+        if name.startswith("_") or name in (
+            "read",
+            "tell",
+            "seek",
+        ):  # only 3 public methods supported
+            return super().__getattribute__(name)
+        raise NotImplementedError(f"PartialFileIO does not support '{name}'.")
+
+    def tell(self) -> int:
+        """Return the current file position."""
+        return self._file.tell()
+
+    def seek(self, __offset: int, __whence: int = SEEK_SET) -> int:
+        """Change the stream position to the given offset.
+
+        Behavior is the same as a regular file, except that the position is capped to the size limit.
+        """
+        if __whence == SEEK_END:
+            # SEEK_END => set from the truncated end
+            __offset = len(self) + __offset
+            __whence = SEEK_SET
+
+        pos = self._file.seek(__offset, __whence)
+        if pos > self._size_limit:
+            return self._file.seek(self._size_limit)
+        return pos
+
+    def read(self, __size: int | None = -1) -> bytes:
+        """Read at most `__size` bytes from the file.
+
+        Behavior is the same as a regular file, except that it is capped to the size limit.
+        """
+        current = self._file.tell()
+        if __size is None or __size < 0:
+            # Read until file limit
+            truncated_size = self._size_limit - current
+        else:
+            # Read until file limit or __size
+            truncated_size = min(__size, self._size_limit - current)
+        return self._file.read(truncated_size)

context_vars.py

@@ -0,0 +1,18 @@
+import contextvars
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from trackio.run import Run
+
+current_run: contextvars.ContextVar["Run | None"] = contextvars.ContextVar(
+    "current_run", default=None
+)
+current_project: contextvars.ContextVar[str | None] = contextvars.ContextVar(
+    "current_project", default=None
+)
+current_server: contextvars.ContextVar[str | None] = contextvars.ContextVar(
+    "current_server", default=None
+)
+current_share_server: contextvars.ContextVar[str | None] = contextvars.ContextVar(
+    "current_share_server", default=None
+)

deploy.py

@@ -0,0 +1,256 @@
+import importlib.metadata
+import io
+import os
+import time
+from importlib.resources import files
+from pathlib import Path
+
+import gradio
+import huggingface_hub
+from gradio_client import Client, handle_file
+from httpx import ReadTimeout
+from huggingface_hub.errors import RepositoryNotFoundError
+from requests import HTTPError
+
+import trackio
+from trackio.sqlite_storage import SQLiteStorage
+
+SPACE_HOST_URL = "https://{user_name}-{space_name}.hf.space/"
+SPACE_URL = "https://huggingface.co/spaces/{space_id}"
+
+
+def _is_trackio_installed_from_source() -> bool:
+    """Check if trackio is installed from source/editable install vs PyPI."""
+    try:
+        trackio_file = trackio.__file__
+        if "site-packages" not in trackio_file:
+            return True
+
+        dist = importlib.metadata.distribution("trackio")
+        if dist.files:
+            files = list(dist.files)
+            has_pth = any(".pth" in str(f) for f in files)
+            if has_pth:
+                return True
+
+        return False
+    except (
+        AttributeError,
+        importlib.metadata.PackageNotFoundError,
+        importlib.metadata.MetadataError,
+        ValueError,
+        TypeError,
+    ):
+        return True
+
+
+def deploy_as_space(
+    space_id: str,
+    space_storage: huggingface_hub.SpaceStorage | None = None,
+    dataset_id: str | None = None,
+    private: bool | None = None,
+):
+    if (
+        os.getenv("SYSTEM") == "spaces"
+    ):  # in case a repo with this function is uploaded to spaces
+        return
+
+    trackio_path = files("trackio")
+
+    hf_api = huggingface_hub.HfApi()
+
+    try:
+        huggingface_hub.create_repo(
+            space_id,
+            private=private,
+            space_sdk="gradio",
+            space_storage=space_storage,
+            repo_type="space",
+            exist_ok=True,
+        )
+    except HTTPError as e:
+        if e.response.status_code in [401, 403]:  # unauthorized or forbidden
+            print("Need 'write' access token to create a Spaces repo.")
+            huggingface_hub.login(add_to_git_credential=False)
+            huggingface_hub.create_repo(
+                space_id,
+                private=private,
+                space_sdk="gradio",
+                space_storage=space_storage,
+                repo_type="space",
+                exist_ok=True,
+            )
+        else:
+            raise ValueError(f"Failed to create Space: {e}")
+
+    with open(Path(trackio_path, "README.md"), "r") as f:
+        readme_content = f.read()
+        readme_content = readme_content.replace("{GRADIO_VERSION}", gradio.__version__)
+        readme_buffer = io.BytesIO(readme_content.encode("utf-8"))
+        hf_api.upload_file(
+            path_or_fileobj=readme_buffer,
+            path_in_repo="README.md",
+            repo_id=space_id,
+            repo_type="space",
+        )
+
+    # We can assume pandas, gradio, and huggingface-hub are already installed in a Gradio Space.
+    # Make sure necessary dependencies are installed by creating a requirements.txt.
+    is_source_install = _is_trackio_installed_from_source()
+
+    if is_source_install:
+        requirements_content = """pyarrow>=21.0"""
+    else:
+        requirements_content = f"""pyarrow>=21.0
+trackio=={trackio.__version__}"""
+
+    requirements_buffer = io.BytesIO(requirements_content.encode("utf-8"))
+    hf_api.upload_file(
+        path_or_fileobj=requirements_buffer,
+        path_in_repo="requirements.txt",
+        repo_id=space_id,
+        repo_type="space",
+    )
+
+    huggingface_hub.utils.disable_progress_bars()
+
+    if is_source_install:
+        hf_api.upload_folder(
+            repo_id=space_id,
+            repo_type="space",
+            folder_path=trackio_path,
+            ignore_patterns=["README.md"],
+        )
+    else:
+        app_file_content = """import trackio
+trackio.show()"""
+        app_file_buffer = io.BytesIO(app_file_content.encode("utf-8"))
+        hf_api.upload_file(
+            path_or_fileobj=app_file_buffer,
+            path_in_repo="ui/main.py",
+            repo_id=space_id,
+            repo_type="space",
+        )
+
+    if hf_token := huggingface_hub.utils.get_token():
+        huggingface_hub.add_space_secret(space_id, "HF_TOKEN", hf_token)
+    if dataset_id is not None:
+        huggingface_hub.add_space_variable(space_id, "TRACKIO_DATASET_ID", dataset_id)
+
+    if logo_light_url := os.environ.get("TRACKIO_LOGO_LIGHT_URL"):
+        huggingface_hub.add_space_variable(
+            space_id, "TRACKIO_LOGO_LIGHT_URL", logo_light_url
+        )
+    if logo_dark_url := os.environ.get("TRACKIO_LOGO_DARK_URL"):
+        huggingface_hub.add_space_variable(
+            space_id, "TRACKIO_LOGO_DARK_URL", logo_dark_url
+        )
+
+    if plot_order := os.environ.get("TRACKIO_PLOT_ORDER"):
+        huggingface_hub.add_space_variable(space_id, "TRACKIO_PLOT_ORDER", plot_order)
+
+    if theme := os.environ.get("TRACKIO_THEME"):
+        huggingface_hub.add_space_variable(space_id, "TRACKIO_THEME", theme)
+
+
+def create_space_if_not_exists(
+    space_id: str,
+    space_storage: huggingface_hub.SpaceStorage | None = None,
+    dataset_id: str | None = None,
+    private: bool | None = None,
+) -> None:
+    """
+    Creates a new Hugging Face Space if it does not exist. If a dataset_id is provided, it will be added as a space variable.
+
+    Args:
+        space_id: The ID of the Space to create.
+        dataset_id: The ID of the Dataset to add to the Space.
+        private: Whether to make the Space private. If None (default), the repo will be
+          public unless the organization's default is private. This value is ignored if
+          the repo already exists.
+    """
+    if "/" not in space_id:
+        raise ValueError(
+            f"Invalid space ID: {space_id}. Must be in the format: username/reponame or orgname/reponame."
+        )
+    if dataset_id is not None and "/" not in dataset_id:
+        raise ValueError(
+            f"Invalid dataset ID: {dataset_id}. Must be in the format: username/datasetname or orgname/datasetname."
+        )
+    try:
+        huggingface_hub.repo_info(space_id, repo_type="space")
+        print(f"* Found existing space: {SPACE_URL.format(space_id=space_id)}")
+        if dataset_id is not None:
+            huggingface_hub.add_space_variable(
+                space_id, "TRACKIO_DATASET_ID", dataset_id
+            )
+        if logo_light_url := os.environ.get("TRACKIO_LOGO_LIGHT_URL"):
+            huggingface_hub.add_space_variable(
+                space_id, "TRACKIO_LOGO_LIGHT_URL", logo_light_url
+            )
+        if logo_dark_url := os.environ.get("TRACKIO_LOGO_DARK_URL"):
+            huggingface_hub.add_space_variable(
+                space_id, "TRACKIO_LOGO_DARK_URL", logo_dark_url
+            )
+
+        if plot_order := os.environ.get("TRACKIO_PLOT_ORDER"):
+            huggingface_hub.add_space_variable(
+                space_id, "TRACKIO_PLOT_ORDER", plot_order
+            )
+
+        if theme := os.environ.get("TRACKIO_THEME"):
+            huggingface_hub.add_space_variable(space_id, "TRACKIO_THEME", theme)
+        return
+    except RepositoryNotFoundError:
+        pass
+    except HTTPError as e:
+        if e.response.status_code in [401, 403]:  # unauthorized or forbidden
+            print("Need 'write' access token to create a Spaces repo.")
+            huggingface_hub.login(add_to_git_credential=False)
+            huggingface_hub.add_space_variable(
+                space_id, "TRACKIO_DATASET_ID", dataset_id
+            )
+        else:
+            raise ValueError(f"Failed to create Space: {e}")
+
+    print(f"* Creating new space: {SPACE_URL.format(space_id=space_id)}")
+    deploy_as_space(space_id, space_storage, dataset_id, private)
+
+
+def wait_until_space_exists(
+    space_id: str,
+) -> None:
+    """
+    Blocks the current thread until the space exists.
+    May raise a TimeoutError if this takes quite a while.
+
+    Args:
+        space_id: The ID of the Space to wait for.
+    """
+    delay = 1
+    for _ in range(10):
+        try:
+            Client(space_id, verbose=False)
+            return
+        except (ReadTimeout, ValueError):
+            time.sleep(delay)
+            delay = min(delay * 2, 30)
+    raise TimeoutError("Waiting for space to exist took longer than expected")
+
+
+def upload_db_to_space(project: str, space_id: str) -> None:
+    """
+    Uploads the database of a local Trackio project to a Hugging Face Space.
+
+    Args:
+        project: The name of the project to upload.
+        space_id: The ID of the Space to upload to.
+    """
+    db_path = SQLiteStorage.get_project_db_path(project)
+    client = Client(space_id, verbose=False)
+    client.predict(
+        api_name="/upload_db_to_space",
+        project=project,
+        uploaded_db=handle_file(db_path),
+        hf_token=huggingface_hub.utils.get_token(),
+    )

dummy_commit_scheduler.py

@@ -0,0 +1,12 @@
+# A dummy object to fit the interface of huggingface_hub's CommitScheduler
+class DummyCommitSchedulerLock:
+    def __enter__(self):
+        return None
+
+    def __exit__(self, exception_type, exception_value, exception_traceback):
+        pass
+
+
+class DummyCommitScheduler:
+    def __init__(self):
+        self.lock = DummyCommitSchedulerLock()

file_storage.py

@@ -0,0 +1,37 @@
+from pathlib import Path
+
+try:  # absolute imports when installed
+    from trackio.utils import MEDIA_DIR
+except ImportError:  # relative imports for local execution on Spaces
+    from utils import MEDIA_DIR
+
+
+class FileStorage:
+    @staticmethod
+    def get_project_media_path(
+        project: str,
+        run: str | None = None,
+        step: int | None = None,
+        filename: str | None = None,
+    ) -> Path:
+        if filename is not None and step is None:
+            raise ValueError("filename requires step")
+        if step is not None and run is None:
+            raise ValueError("step requires run")
+
+        path = MEDIA_DIR / project
+        if run:
+            path /= run
+        if step is not None:
+            path /= str(step)
+        if filename:
+            path /= filename
+        return path
+
+    @staticmethod
+    def init_project_media_path(
+        project: str, run: str | None = None, step: int | None = None
+    ) -> Path:
+        path = FileStorage.get_project_media_path(project, run, step)
+        path.mkdir(parents=True, exist_ok=True)
+        return path

imports.py

@@ -0,0 +1,302 @@
+import os
+from pathlib import Path
+
+import pandas as pd
+
+from trackio import deploy, utils
+from trackio.sqlite_storage import SQLiteStorage
+
+
+def import_csv(
+    csv_path: str | Path,
+    project: str,
+    name: str | None = None,
+    space_id: str | None = None,
+    dataset_id: str | None = None,
+    private: bool | None = None,
+) -> None:
+    """
+    Imports a CSV file into a Trackio project. The CSV file must contain a `"step"`
+    column, may optionally contain a `"timestamp"` column, and any other columns will be
+    treated as metrics. It should also include a header row with the column names.
+
+    TODO: call init() and return a Run object so that the user can continue to log metrics to it.
+
+    Args:
+        csv_path (`str` or `Path`):
+            The str or Path to the CSV file to import.
+        project (`str`):
+            The name of the project to import the CSV file into. Must not be an existing
+            project.
+        name (`str`, *optional*):
+            The name of the Run to import the CSV file into. If not provided, a default
+            name will be generated.
+        name (`str`, *optional*):
+            The name of the run (if not provided, a default name will be generated).
+        space_id (`str`, *optional*):
+            If provided, the project will be logged to a Hugging Face Space instead of a
+            local directory. Should be a complete Space name like `"username/reponame"`
+            or `"orgname/reponame"`, or just `"reponame"` in which case the Space will
+            be created in the currently-logged-in Hugging Face user's namespace. If the
+            Space does not exist, it will be created. If the Space already exists, the
+            project will be logged to it.
+        dataset_id (`str`, *optional*):
+            If provided, a persistent Hugging Face Dataset will be created and the
+            metrics will be synced to it every 5 minutes. Should be a complete Dataset
+            name like `"username/datasetname"` or `"orgname/datasetname"`, or just
+            `"datasetname"` in which case the Dataset will be created in the
+            currently-logged-in Hugging Face user's namespace. If the Dataset does not
+            exist, it will be created. If the Dataset already exists, the project will
+            be appended to it. If not provided, the metrics will be logged to a local
+            SQLite database, unless a `space_id` is provided, in which case a Dataset
+            will be automatically created with the same name as the Space but with the
+            `"_dataset"` suffix.
+        private (`bool`, *optional*):
+            Whether to make the Space private. If None (default), the repo will be
+            public unless the organization's default is private. This value is ignored
+            if the repo already exists.
+    """
+    if SQLiteStorage.get_runs(project):
+        raise ValueError(
+            f"Project '{project}' already exists. Cannot import CSV into existing project."
+        )
+
+    csv_path = Path(csv_path)
+    if not csv_path.exists():
+        raise FileNotFoundError(f"CSV file not found: {csv_path}")
+
+    df = pd.read_csv(csv_path)
+    if df.empty:
+        raise ValueError("CSV file is empty")
+
+    column_mapping = utils.simplify_column_names(df.columns.tolist())
+    df = df.rename(columns=column_mapping)
+
+    step_column = None
+    for col in df.columns:
+        if col.lower() == "step":
+            step_column = col
+            break
+
+    if step_column is None:
+        raise ValueError("CSV file must contain a 'step' or 'Step' column")
+
+    if name is None:
+        name = csv_path.stem
+
+    metrics_list = []
+    steps = []
+    timestamps = []
+
+    numeric_columns = []
+    for column in df.columns:
+        if column == step_column:
+            continue
+        if column == "timestamp":
+            continue
+
+        try:
+            pd.to_numeric(df[column], errors="raise")
+            numeric_columns.append(column)
+        except (ValueError, TypeError):
+            continue
+
+    for _, row in df.iterrows():
+        metrics = {}
+        for column in numeric_columns:
+            value = row[column]
+            if bool(pd.notna(value)):
+                metrics[column] = float(value)
+
+        if metrics:
+            metrics_list.append(metrics)
+            steps.append(int(row[step_column]))
+
+            if "timestamp" in df.columns and bool(pd.notna(row["timestamp"])):
+                timestamps.append(str(row["timestamp"]))
+            else:
+                timestamps.append("")
+
+    if metrics_list:
+        SQLiteStorage.bulk_log(
+            project=project,
+            run=name,
+            metrics_list=metrics_list,
+            steps=steps,
+            timestamps=timestamps,
+        )
+
+    print(
+        f"* Imported {len(metrics_list)} rows from {csv_path} into project '{project}' as run '{name}'"
+    )
+    print(f"* Metrics found: {', '.join(metrics_list[0].keys())}")
+
+    space_id, dataset_id = utils.preprocess_space_and_dataset_ids(space_id, dataset_id)
+    if dataset_id is not None:
+        os.environ["TRACKIO_DATASET_ID"] = dataset_id
+        print(f"* Trackio metrics will be synced to Hugging Face Dataset: {dataset_id}")
+
+    if space_id is None:
+        utils.print_dashboard_instructions(project)
+    else:
+        deploy.create_space_if_not_exists(
+            space_id=space_id, dataset_id=dataset_id, private=private
+        )
+        deploy.wait_until_space_exists(space_id=space_id)
+        deploy.upload_db_to_space(project=project, space_id=space_id)
+        print(
+            f"* View dashboard by going to: {deploy.SPACE_URL.format(space_id=space_id)}"
+        )
+
+
+def import_tf_events(
+    log_dir: str | Path,
+    project: str,
+    name: str | None = None,
+    space_id: str | None = None,
+    dataset_id: str | None = None,
+    private: bool | None = None,
+) -> None:
+    """
+    Imports TensorFlow Events files from a directory into a Trackio project. Each
+    subdirectory in the log directory will be imported as a separate run.
+
+    Args:
+        log_dir (`str` or `Path`):
+            The str or Path to the directory containing TensorFlow Events files.
+        project (`str`):
+            The name of the project to import the TensorFlow Events files into. Must not
+            be an existing project.
+        name (`str`, *optional*):
+            The name prefix for runs (if not provided, will use directory names). Each
+            subdirectory will create a separate run.
+        space_id (`str`, *optional*):
+            If provided, the project will be logged to a Hugging Face Space instead of a
+            local directory. Should be a complete Space name like `"username/reponame"`
+            or `"orgname/reponame"`, or just `"reponame"` in which case the Space will
+            be created in the currently-logged-in Hugging Face user's namespace. If the
+            Space does not exist, it will be created. If the Space already exists, the
+            project will be logged to it.
+        dataset_id (`str`, *optional*):
+            If provided, a persistent Hugging Face Dataset will be created and the
+            metrics will be synced to it every 5 minutes. Should be a complete Dataset
+            name like `"username/datasetname"` or `"orgname/datasetname"`, or just
+            `"datasetname"` in which case the Dataset will be created in the
+            currently-logged-in Hugging Face user's namespace. If the Dataset does not
+            exist, it will be created. If the Dataset already exists, the project will
+            be appended to it. If not provided, the metrics will be logged to a local
+            SQLite database, unless a `space_id` is provided, in which case a Dataset
+            will be automatically created with the same name as the Space but with the
+            `"_dataset"` suffix.
+        private (`bool`, *optional*):
+            Whether to make the Space private. If None (default), the repo will be
+            public unless the organization's default is private. This value is ignored
+            if the repo already exists.
+    """
+    try:
+        from tbparse import SummaryReader
+    except ImportError:
+        raise ImportError(
+            "The `tbparse` package is not installed but is required for `import_tf_events`. Please install trackio with the `tensorboard` extra: `pip install trackio[tensorboard]`."
+        )
+
+    if SQLiteStorage.get_runs(project):
+        raise ValueError(
+            f"Project '{project}' already exists. Cannot import TF events into existing project."
+        )
+
+    path = Path(log_dir)
+    if not path.exists():
+        raise FileNotFoundError(f"TF events directory not found: {path}")
+
+    # Use tbparse to read all tfevents files in the directory structure
+    reader = SummaryReader(str(path), extra_columns={"dir_name"})
+    df = reader.scalars
+
+    if df.empty:
+        raise ValueError(f"No TensorFlow events data found in {path}")
+
+    total_imported = 0
+    imported_runs = []
+
+    # Group by dir_name to create separate runs
+    for dir_name, group_df in df.groupby("dir_name"):
+        try:
+            # Determine run name based on directory name
+            if dir_name == "":
+                run_name = "main"  # For files in the root directory
+            else:
+                run_name = dir_name  # Use directory name
+
+            if name:
+                run_name = f"{name}_{run_name}"
+
+            if group_df.empty:
+                print(f"* Skipping directory {dir_name}: no scalar data found")
+                continue
+
+            metrics_list = []
+            steps = []
+            timestamps = []
+
+            for _, row in group_df.iterrows():
+                # Convert row values to appropriate types
+                tag = str(row["tag"])
+                value = float(row["value"])
+                step = int(row["step"])
+
+                metrics = {tag: value}
+                metrics_list.append(metrics)
+                steps.append(step)
+
+                # Use wall_time if present, else fallback
+                if "wall_time" in group_df.columns and not bool(
+                    pd.isna(row["wall_time"])
+                ):
+                    timestamps.append(str(row["wall_time"]))
+                else:
+                    timestamps.append("")
+
+            if metrics_list:
+                SQLiteStorage.bulk_log(
+                    project=project,
+                    run=str(run_name),
+                    metrics_list=metrics_list,
+                    steps=steps,
+                    timestamps=timestamps,
+                )
+
+                total_imported += len(metrics_list)
+                imported_runs.append(run_name)
+
+                print(
+                    f"* Imported {len(metrics_list)} scalar events from directory '{dir_name}' as run '{run_name}'"
+                )
+                print(f"* Metrics in this run: {', '.join(set(group_df['tag']))}")
+
+        except Exception as e:
+            print(f"* Error processing directory {dir_name}: {e}")
+            continue
+
+    if not imported_runs:
+        raise ValueError("No valid TensorFlow events data could be imported")
+
+    print(f"* Total imported events: {total_imported}")
+    print(f"* Created runs: {', '.join(imported_runs)}")
+
+    space_id, dataset_id = utils.preprocess_space_and_dataset_ids(space_id, dataset_id)
+    if dataset_id is not None:
+        os.environ["TRACKIO_DATASET_ID"] = dataset_id
+        print(f"* Trackio metrics will be synced to Hugging Face Dataset: {dataset_id}")
+
+    if space_id is None:
+        utils.print_dashboard_instructions(project)
+    else:
+        deploy.create_space_if_not_exists(
+            space_id, dataset_id=dataset_id, private=private
+        )
+        deploy.wait_until_space_exists(space_id)
+        deploy.upload_db_to_space(project, space_id)
+        print(
+            f"* View dashboard by going to: {deploy.SPACE_URL.format(space_id=space_id)}"
+        )

media.py

@@ -0,0 +1,286 @@
+import os
+import shutil
+import uuid
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import Literal
+
+import numpy as np
+from PIL import Image as PILImage
+
+try:  # absolute imports when installed
+    from trackio.file_storage import FileStorage
+    from trackio.utils import MEDIA_DIR
+    from trackio.video_writer import write_video
+except ImportError:  # relative imports for local execution on Spaces
+    from file_storage import FileStorage
+    from utils import MEDIA_DIR
+    from video_writer import write_video
+
+
+class TrackioMedia(ABC):
+    """
+    Abstract base class for Trackio media objects
+    Provides shared functionality for file handling and serialization.
+    """
+
+    TYPE: str
+
+    def __init_subclass__(cls, **kwargs):
+        """Ensure subclasses define the TYPE attribute."""
+        super().__init_subclass__(**kwargs)
+        if not hasattr(cls, "TYPE") or cls.TYPE is None:
+            raise TypeError(f"Class {cls.__name__} must define TYPE attribute")
+
+    def __init__(self, value, caption: str | None = None):
+        self.caption = caption
+        self._value = value
+        self._file_path: Path | None = None
+
+        # Validate file existence for string/Path inputs
+        if isinstance(self._value, str | Path):
+            if not os.path.isfile(self._value):
+                raise ValueError(f"File not found: {self._value}")
+
+    def _file_extension(self) -> str:
+        if self._file_path:
+            return self._file_path.suffix[1:].lower()
+        if isinstance(self._value, str | Path):
+            path = Path(self._value)
+            return path.suffix[1:].lower()
+        if hasattr(self, "_format") and self._format:
+            return self._format
+        return "unknown"
+
+    def _get_relative_file_path(self) -> Path | None:
+        return self._file_path
+
+    def _get_absolute_file_path(self) -> Path | None:
+        if self._file_path:
+            return MEDIA_DIR / self._file_path
+        return None
+
+    def _save(self, project: str, run: str, step: int = 0):
+        if self._file_path:
+            return
+
+        media_dir = FileStorage.init_project_media_path(project, run, step)
+        filename = f"{uuid.uuid4()}.{self._file_extension()}"
+        file_path = media_dir / filename
+
+        # Delegate to subclass-specific save logic
+        self._save_media(file_path)
+
+        self._file_path = file_path.relative_to(MEDIA_DIR)
+
+    @abstractmethod
+    def _save_media(self, file_path: Path):
+        """
+        Performs the actual media saving logic.
+        """
+        pass
+
+    def _to_dict(self) -> dict:
+        if not self._file_path:
+            raise ValueError("Media must be saved to file before serialization")
+        return {
+            "_type": self.TYPE,
+            "file_path": str(self._get_relative_file_path()),
+            "caption": self.caption,
+        }
+
+
+TrackioImageSourceType = str | Path | np.ndarray | PILImage.Image
+
+
+class TrackioImage(TrackioMedia):
+    """
+    Initializes an Image object.
+
+    Example:
+        ```python
+        import trackio
+        import numpy as np
+        from PIL import Image
+
+        # Create an image from numpy array
+        image_data = np.random.randint(0, 255, (64, 64, 3), dtype=np.uint8)
+        image = trackio.Image(image_data, caption="Random image")
+        trackio.log({"my_image": image})
+
+        # Create an image from PIL Image
+        pil_image = Image.new('RGB', (100, 100), color='red')
+        image = trackio.Image(pil_image, caption="Red square")
+        trackio.log({"red_image": image})
+
+        # Create an image from file path
+        image = trackio.Image("path/to/image.jpg", caption="Photo from file")
+        trackio.log({"file_image": image})
+        ```
+
+    Args:
+        value (`str`, `Path`, `numpy.ndarray`, or `PIL.Image`, *optional*):
+            A path to an image, a PIL Image, or a numpy array of shape (height, width, channels).
+        caption (`str`, *optional*):
+            A string caption for the image.
+    """
+
+    TYPE = "trackio.image"
+
+    def __init__(self, value: TrackioImageSourceType, caption: str | None = None):
+        super().__init__(value, caption)
+        self._format: str | None = None
+
+        if (
+            isinstance(self._value, np.ndarray | PILImage.Image)
+            and self._format is None
+        ):
+            self._format = "png"
+
+    def _as_pil(self) -> PILImage.Image | None:
+        try:
+            if isinstance(self._value, np.ndarray):
+                arr = np.asarray(self._value).astype("uint8")
+                return PILImage.fromarray(arr).convert("RGBA")
+            if isinstance(self._value, PILImage.Image):
+                return self._value.convert("RGBA")
+        except Exception as e:
+            raise ValueError(f"Failed to process image data: {self._value}") from e
+        return None
+
+    def _save_media(self, file_path: Path):
+        if pil := self._as_pil():
+            pil.save(file_path, format=self._format)
+        elif isinstance(self._value, str | Path):
+            if os.path.isfile(self._value):
+                shutil.copy(self._value, file_path)
+            else:
+                raise ValueError(f"File not found: {self._value}")
+
+
+TrackioVideoSourceType = str | Path | np.ndarray
+TrackioVideoFormatType = Literal["gif", "mp4", "webm"]
+
+
+class TrackioVideo(TrackioMedia):
+    """
+    Initializes a Video object.
+
+    Example:
+        ```python
+        import trackio
+        import numpy as np
+
+        # Create a simple video from numpy array
+        frames = np.random.randint(0, 255, (10, 3, 64, 64), dtype=np.uint8)
+        video = trackio.Video(frames, caption="Random video", fps=30)
+
+        # Create a batch of videos
+        batch_frames = np.random.randint(0, 255, (3, 10, 3, 64, 64), dtype=np.uint8)
+        batch_video = trackio.Video(batch_frames, caption="Batch of videos", fps=15)
+
+        # Create video from file path
+        video = trackio.Video("path/to/video.mp4", caption="Video from file")
+        ```
+
+    Args:
+        value (`str`, `Path`, or `numpy.ndarray`, *optional*):
+            A path to a video file, or a numpy array.
+            The array should be of type `np.uint8` with RGB values in the range `[0, 255]`.
+            It is expected to have shape of either (frames, channels, height, width) or (batch, frames, channels, height, width).
+            For the latter, the videos will be tiled into a grid.
+        caption (`str`, *optional*):
+            A string caption for the video.
+        fps (`int`, *optional*):
+            Frames per second for the video. Only used when value is an ndarray. Default is `24`.
+        format (`Literal["gif", "mp4", "webm"]`, *optional*):
+            Video format ("gif", "mp4", or "webm"). Only used when value is an ndarray. Default is "gif".
+    """
+
+    TYPE = "trackio.video"
+
+    def __init__(
+        self,
+        value: TrackioVideoSourceType,
+        caption: str | None = None,
+        fps: int | None = None,
+        format: TrackioVideoFormatType | None = None,
+    ):
+        super().__init__(value, caption)
+        if isinstance(value, np.ndarray):
+            if format is None:
+                format = "gif"
+            if fps is None:
+                fps = 24
+        self._fps = fps
+        self._format = format
+
+    @property
+    def _codec(self) -> str:
+        match self._format:
+            case "gif":
+                return "gif"
+            case "mp4":
+                return "h264"
+            case "webm":
+                return "vp9"
+            case _:
+                raise ValueError(f"Unsupported format: {self._format}")
+
+    def _save_media(self, file_path: Path):
+        if isinstance(self._value, np.ndarray):
+            video = TrackioVideo._process_ndarray(self._value)
+            write_video(file_path, video, fps=self._fps, codec=self._codec)
+        elif isinstance(self._value, str | Path):
+            if os.path.isfile(self._value):
+                shutil.copy(self._value, file_path)
+            else:
+                raise ValueError(f"File not found: {self._value}")
+
+    @staticmethod
+    def _process_ndarray(value: np.ndarray) -> np.ndarray:
+        # Verify value is either 4D (single video) or 5D array (batched videos).
+        # Expected format: (frames, channels, height, width) or (batch, frames, channels, height, width)
+        if value.ndim < 4:
+            raise ValueError(
+                "Video requires at least 4 dimensions (frames, channels, height, width)"
+            )
+        if value.ndim > 5:
+            raise ValueError(
+                "Videos can have at most 5 dimensions (batch, frames, channels, height, width)"
+            )
+        if value.ndim == 4:
+            # Reshape to 5D with single batch: (1, frames, channels, height, width)
+            value = value[np.newaxis, ...]
+
+        value = TrackioVideo._tile_batched_videos(value)
+        return value
+
+    @staticmethod
+    def _tile_batched_videos(video: np.ndarray) -> np.ndarray:
+        """
+        Tiles a batch of videos into a grid of videos.
+
+        Input format: (batch, frames, channels, height, width) - original FCHW format
+        Output format: (frames, total_height, total_width, channels)
+        """
+        batch_size, frames, channels, height, width = video.shape
+
+        next_pow2 = 1 << (batch_size - 1).bit_length()
+        if batch_size != next_pow2:
+            pad_len = next_pow2 - batch_size
+            pad_shape = (pad_len, frames, channels, height, width)
+            padding = np.zeros(pad_shape, dtype=video.dtype)
+            video = np.concatenate((video, padding), axis=0)
+            batch_size = next_pow2
+
+        n_rows = 1 << ((batch_size.bit_length() - 1) // 2)
+        n_cols = batch_size // n_rows
+
+        # Reshape to grid layout: (n_rows, n_cols, frames, channels, height, width)
+        video = video.reshape(n_rows, n_cols, frames, channels, height, width)
+
+        # Rearrange dimensions to (frames, total_height, total_width, channels)
+        video = video.transpose(2, 0, 4, 1, 5, 3)
+        video = video.reshape(frames, n_rows * height, n_cols * width, channels)
+        return video

package.json

@@ -0,0 +1,6 @@
+{
+	"name": "trackio",
+	"version": "0.5.3",
+	"description": "",
+	"python": "true"
+}

run.py

@@ -0,0 +1,180 @@
+import threading
+import time
+from datetime import datetime, timezone
+
+import huggingface_hub
+from gradio_client import Client, handle_file
+
+from trackio import utils
+from trackio.media import TrackioMedia
+from trackio.sqlite_storage import SQLiteStorage
+from trackio.table import Table
+from trackio.typehints import LogEntry, UploadEntry
+
+BATCH_SEND_INTERVAL = 0.5
+
+
+class Run:
+    def __init__(
+        self,
+        url: str,
+        project: str,
+        client: Client | None,
+        name: str | None = None,
+        group: str | None = None,
+        config: dict | None = None,
+        space_id: str | None = None,
+    ):
+        self.url = url
+        self.project = project
+        self._client_lock = threading.Lock()
+        self._client_thread = None
+        self._client = client
+        self._space_id = space_id
+        self.name = name or utils.generate_readable_name(
+            SQLiteStorage.get_runs(project), space_id
+        )
+        self.group = group
+        self.config = utils.to_json_safe(config or {})
+
+        if isinstance(self.config, dict):
+            for key in self.config:
+                if key.startswith("_"):
+                    raise ValueError(
+                        f"Config key '{key}' is reserved (keys starting with '_' are reserved for internal use)"
+                    )
+
+        self.config["_Username"] = self._get_username()
+        self.config["_Created"] = datetime.now(timezone.utc).isoformat()
+        self.config["_Group"] = self.group
+
+        self._queued_logs: list[LogEntry] = []
+        self._queued_uploads: list[UploadEntry] = []
+        self._stop_flag = threading.Event()
+        self._config_logged = False
+
+        self._client_thread = threading.Thread(target=self._init_client_background)
+        self._client_thread.daemon = True
+        self._client_thread.start()
+
+    def _get_username(self) -> str | None:
+        """Get the current HuggingFace username if logged in, otherwise None."""
+        try:
+            who = huggingface_hub.whoami()
+            return who["name"] if who else None
+        except Exception:
+            return None
+
+    def _batch_sender(self):
+        """Send batched logs every BATCH_SEND_INTERVAL."""
+        while not self._stop_flag.is_set() or len(self._queued_logs) > 0:
+            # If the stop flag has been set, then just quickly send all
+            # the logs and exit.
+            if not self._stop_flag.is_set():
+                time.sleep(BATCH_SEND_INTERVAL)
+
+            with self._client_lock:
+                if self._client is None:
+                    return
+                if self._queued_logs:
+                    logs_to_send = self._queued_logs.copy()
+                    self._queued_logs.clear()
+                    self._client.predict(
+                        api_name="/bulk_log",
+                        logs=logs_to_send,
+                        hf_token=huggingface_hub.utils.get_token(),
+                    )
+                if self._queued_uploads:
+                    uploads_to_send = self._queued_uploads.copy()
+                    self._queued_uploads.clear()
+                    self._client.predict(
+                        api_name="/bulk_upload_media",
+                        uploads=uploads_to_send,
+                        hf_token=huggingface_hub.utils.get_token(),
+                    )
+
+    def _init_client_background(self):
+        if self._client is None:
+            fib = utils.fibo()
+            for sleep_coefficient in fib:
+                try:
+                    client = Client(self.url, verbose=False)
+
+                    with self._client_lock:
+                        self._client = client
+                    break
+                except Exception:
+                    pass
+                if sleep_coefficient is not None:
+                    time.sleep(0.1 * sleep_coefficient)
+
+        self._batch_sender()
+
+    def _process_media(self, metrics, step: int | None) -> dict:
+        """
+        Serialize media in metrics and upload to space if needed.
+        """
+        serializable_metrics = {}
+        if not step:
+            step = 0
+        for key, value in metrics.items():
+            if isinstance(value, TrackioMedia):
+                value._save(self.project, self.name, step)
+                serializable_metrics[key] = value._to_dict()
+                if self._space_id:
+                    # Upload local media when deploying to space
+                    upload_entry: UploadEntry = {
+                        "project": self.project,
+                        "run": self.name,
+                        "step": step,
+                        "uploaded_file": handle_file(value._get_absolute_file_path()),
+                    }
+                    with self._client_lock:
+                        self._queued_uploads.append(upload_entry)
+            else:
+                serializable_metrics[key] = value
+        return serializable_metrics
+
+    @staticmethod
+    def _replace_tables(metrics):
+        for k, v in metrics.items():
+            if isinstance(v, Table):
+                metrics[k] = v._to_dict()
+
+    def log(self, metrics: dict, step: int | None = None):
+        for k in metrics.keys():
+            if k in utils.RESERVED_KEYS or k.startswith("__"):
+                raise ValueError(
+                    f"Please do not use this reserved key as a metric: {k}"
+                )
+        Run._replace_tables(metrics)
+
+        metrics = self._process_media(metrics, step)
+        metrics = utils.serialize_values(metrics)
+
+        config_to_log = None
+        if not self._config_logged and self.config:
+            config_to_log = utils.to_json_safe(self.config)
+            self._config_logged = True
+
+        log_entry: LogEntry = {
+            "project": self.project,
+            "run": self.name,
+            "metrics": metrics,
+            "step": step,
+            "config": config_to_log,
+        }
+
+        with self._client_lock:
+            self._queued_logs.append(log_entry)
+
+    def finish(self):
+        """Cleanup when run is finished."""
+        self._stop_flag.set()
+
+        # Wait for the batch sender to finish before joining the client thread.
+        time.sleep(2 * BATCH_SEND_INTERVAL)
+
+        if self._client_thread is not None:
+            print("* Run finished. Uploading logs to Trackio (please wait...)")
+            self._client_thread.join()

sqlite_storage.py

@@ -0,0 +1,677 @@
+import os
+import platform
+import sqlite3
+import time
+from datetime import datetime
+from pathlib import Path
+from threading import Lock
+
+try:
+    import fcntl
+except ImportError:  # fcntl is not available on Windows
+    fcntl = None
+
+import huggingface_hub as hf
+import orjson
+import pandas as pd
+
+try:  # absolute imports when installed from PyPI
+    from trackio.commit_scheduler import CommitScheduler
+    from trackio.dummy_commit_scheduler import DummyCommitScheduler
+    from trackio.utils import (
+        TRACKIO_DIR,
+        deserialize_values,
+        serialize_values,
+    )
+except ImportError:  # relative imports when installed from source on Spaces
+    from commit_scheduler import CommitScheduler
+    from dummy_commit_scheduler import DummyCommitScheduler
+    from utils import TRACKIO_DIR, deserialize_values, serialize_values
+
+DB_EXT = ".db"
+
+
+class ProcessLock:
+    """A file-based lock that works across processes. Is a no-op on Windows."""
+
+    def __init__(self, lockfile_path: Path):
+        self.lockfile_path = lockfile_path
+        self.lockfile = None
+        self.is_windows = platform.system() == "Windows"
+
+    def __enter__(self):
+        """Acquire the lock with retry logic."""
+        if self.is_windows:
+            return self
+        self.lockfile_path.parent.mkdir(parents=True, exist_ok=True)
+        self.lockfile = open(self.lockfile_path, "w")
+
+        max_retries = 100
+        for attempt in range(max_retries):
+            try:
+                fcntl.flock(self.lockfile.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
+                return self
+            except IOError:
+                if attempt < max_retries - 1:
+                    time.sleep(0.1)
+                else:
+                    raise IOError("Could not acquire database lock after 10 seconds")
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        """Release the lock."""
+        if self.is_windows:
+            return
+
+        if self.lockfile:
+            fcntl.flock(self.lockfile.fileno(), fcntl.LOCK_UN)
+            self.lockfile.close()
+
+
+class SQLiteStorage:
+    _dataset_import_attempted = False
+    _current_scheduler: CommitScheduler | DummyCommitScheduler | None = None
+    _scheduler_lock = Lock()
+
+    @staticmethod
+    def _get_connection(db_path: Path) -> sqlite3.Connection:
+        conn = sqlite3.connect(str(db_path), timeout=30.0)
+        # Keep WAL for concurrency + performance on many small writes
+        conn.execute("PRAGMA journal_mode = WAL")
+        # ---- Minimal perf tweaks for many tiny transactions ----
+        # NORMAL = fsync at critical points only (safer than OFF, much faster than FULL)
+        conn.execute("PRAGMA synchronous = NORMAL")
+        # Keep temp data in memory to avoid disk hits during small writes
+        conn.execute("PRAGMA temp_store = MEMORY")
+        # Give SQLite a bit more room for cache (negative = KB, engine-managed)
+        conn.execute("PRAGMA cache_size = -20000")
+        # --------------------------------------------------------
+        conn.row_factory = sqlite3.Row
+        return conn
+
+    @staticmethod
+    def _get_process_lock(project: str) -> ProcessLock:
+        lockfile_path = TRACKIO_DIR / f"{project}.lock"
+        return ProcessLock(lockfile_path)
+
+    @staticmethod
+    def get_project_db_filename(project: str) -> str:
+        """Get the database filename for a specific project."""
+        safe_project_name = "".join(
+            c for c in project if c.isalnum() or c in ("-", "_")
+        ).rstrip()
+        if not safe_project_name:
+            safe_project_name = "default"
+        return f"{safe_project_name}{DB_EXT}"
+
+    @staticmethod
+    def get_project_db_path(project: str) -> Path:
+        """Get the database path for a specific project."""
+        filename = SQLiteStorage.get_project_db_filename(project)
+        return TRACKIO_DIR / filename
+
+    @staticmethod
+    def init_db(project: str) -> Path:
+        """
+        Initialize the SQLite database with required tables.
+        Returns the database path.
+        """
+        db_path = SQLiteStorage.get_project_db_path(project)
+        db_path.parent.mkdir(parents=True, exist_ok=True)
+        with SQLiteStorage._get_process_lock(project):
+            with sqlite3.connect(str(db_path), timeout=30.0) as conn:
+                conn.execute("PRAGMA journal_mode = WAL")
+                conn.execute("PRAGMA synchronous = NORMAL")
+                conn.execute("PRAGMA temp_store = MEMORY")
+                conn.execute("PRAGMA cache_size = -20000")
+                cursor = conn.cursor()
+                cursor.execute(
+                    """
+                    CREATE TABLE IF NOT EXISTS metrics (
+                        id INTEGER PRIMARY KEY AUTOINCREMENT,
+                        timestamp TEXT NOT NULL,
+                        run_name TEXT NOT NULL,
+                        step INTEGER NOT NULL,
+                        metrics TEXT NOT NULL
+                    )
+                    """
+                )
+                cursor.execute(
+                    """
+                    CREATE TABLE IF NOT EXISTS configs (
+                        id INTEGER PRIMARY KEY AUTOINCREMENT,
+                        run_name TEXT NOT NULL,
+                        config TEXT NOT NULL,
+                        created_at TEXT NOT NULL,
+                        UNIQUE(run_name)
+                    )
+                    """
+                )
+                cursor.execute(
+                    """
+                    CREATE INDEX IF NOT EXISTS idx_metrics_run_step
+                    ON metrics(run_name, step)
+                    """
+                )
+                cursor.execute(
+                    """
+                    CREATE INDEX IF NOT EXISTS idx_configs_run_name
+                    ON configs(run_name)
+                    """
+                )
+                cursor.execute(
+                    """
+                    CREATE INDEX IF NOT EXISTS idx_metrics_run_timestamp
+                    ON metrics(run_name, timestamp)
+                    """
+                )
+                conn.commit()
+        return db_path
+
+    @staticmethod
+    def export_to_parquet():
+        """
+        Exports all projects' DB files as Parquet under the same path but with extension ".parquet".
+        """
+        # don't attempt to export (potentially wrong/blank) data before importing for the first time
+        if not SQLiteStorage._dataset_import_attempted:
+            return
+        if not TRACKIO_DIR.exists():
+            return
+
+        all_paths = os.listdir(TRACKIO_DIR)
+        db_names = [f for f in all_paths if f.endswith(DB_EXT)]
+        for db_name in db_names:
+            db_path = TRACKIO_DIR / db_name
+            parquet_path = db_path.with_suffix(".parquet")
+            if (not parquet_path.exists()) or (
+                db_path.stat().st_mtime > parquet_path.stat().st_mtime
+            ):
+                with sqlite3.connect(str(db_path)) as conn:
+                    df = pd.read_sql("SELECT * FROM metrics", conn)
+                # break out the single JSON metrics column into individual columns
+                metrics = df["metrics"].copy()
+                metrics = pd.DataFrame(
+                    metrics.apply(
+                        lambda x: deserialize_values(orjson.loads(x))
+                    ).values.tolist(),
+                    index=df.index,
+                )
+                del df["metrics"]
+                for col in metrics.columns:
+                    df[col] = metrics[col]
+
+                df.to_parquet(parquet_path)
+
+    @staticmethod
+    def _cleanup_wal_sidecars(db_path: Path) -> None:
+        """Remove leftover -wal/-shm files for a DB basename (prevents disk I/O errors)."""
+        for suffix in ("-wal", "-shm"):
+            sidecar = Path(str(db_path) + suffix)
+            try:
+                if sidecar.exists():
+                    sidecar.unlink()
+            except Exception:
+                pass
+
+    @staticmethod
+    def import_from_parquet():
+        """
+        Imports to all DB files that have matching files under the same path but with extension ".parquet".
+        """
+        if not TRACKIO_DIR.exists():
+            return
+
+        all_paths = os.listdir(TRACKIO_DIR)
+        parquet_names = [f for f in all_paths if f.endswith(".parquet")]
+        for pq_name in parquet_names:
+            parquet_path = TRACKIO_DIR / pq_name
+            db_path = parquet_path.with_suffix(DB_EXT)
+
+            SQLiteStorage._cleanup_wal_sidecars(db_path)
+
+            df = pd.read_parquet(parquet_path)
+            # fix up df to have a single JSON metrics column
+            if "metrics" not in df.columns:
+                # separate other columns from metrics
+                metrics = df.copy()
+                other_cols = ["id", "timestamp", "run_name", "step"]
+                df = df[other_cols]
+                for col in other_cols:
+                    del metrics[col]
+                # combine them all into a single metrics col
+                metrics = orjson.loads(metrics.to_json(orient="records"))
+                df["metrics"] = [orjson.dumps(serialize_values(row)) for row in metrics]
+
+            with sqlite3.connect(str(db_path), timeout=30.0) as conn:
+                df.to_sql("metrics", conn, if_exists="replace", index=False)
+                conn.commit()
+
+    @staticmethod
+    def get_scheduler():
+        """
+        Get the scheduler for the database based on the environment variables.
+        This applies to both local and Spaces.
+        """
+        with SQLiteStorage._scheduler_lock:
+            if SQLiteStorage._current_scheduler is not None:
+                return SQLiteStorage._current_scheduler
+            hf_token = os.environ.get("HF_TOKEN")
+            dataset_id = os.environ.get("TRACKIO_DATASET_ID")
+            space_repo_name = os.environ.get("SPACE_REPO_NAME")
+            if dataset_id is None or space_repo_name is None:
+                scheduler = DummyCommitScheduler()
+            else:
+                scheduler = CommitScheduler(
+                    repo_id=dataset_id,
+                    repo_type="dataset",
+                    folder_path=TRACKIO_DIR,
+                    private=True,
+                    allow_patterns=["*.parquet", "media/**/*"],
+                    squash_history=True,
+                    token=hf_token,
+                    on_before_commit=SQLiteStorage.export_to_parquet,
+                )
+            SQLiteStorage._current_scheduler = scheduler
+            return scheduler
+
+    @staticmethod
+    def log(project: str, run: str, metrics: dict, step: int | None = None):
+        """
+        Safely log metrics to the database. Before logging, this method will ensure the database exists
+        and is set up with the correct tables. It also uses a cross-process lock to prevent
+        database locking errors when multiple processes access the same database.
+
+        This method is not used in the latest versions of Trackio (replaced by bulk_log) but
+        is kept for backwards compatibility for users who are connecting to a newer version of
+        a Trackio Spaces dashboard with an older version of Trackio installed locally.
+        """
+        db_path = SQLiteStorage.init_db(project)
+        with SQLiteStorage._get_process_lock(project):
+            with SQLiteStorage._get_connection(db_path) as conn:
+                cursor = conn.cursor()
+                cursor.execute(
+                    """
+                    SELECT MAX(step) 
+                    FROM metrics 
+                    WHERE run_name = ?
+                    """,
+                    (run,),
+                )
+                last_step = cursor.fetchone()[0]
+                current_step = (
+                    0
+                    if step is None and last_step is None
+                    else (step if step is not None else last_step + 1)
+                )
+                current_timestamp = datetime.now().isoformat()
+                cursor.execute(
+                    """
+                    INSERT INTO metrics
+                    (timestamp, run_name, step, metrics)
+                    VALUES (?, ?, ?, ?)
+                    """,
+                    (
+                        current_timestamp,
+                        run,
+                        current_step,
+                        orjson.dumps(serialize_values(metrics)),
+                    ),
+                )
+                conn.commit()
+
+    @staticmethod
+    def bulk_log(
+        project: str,
+        run: str,
+        metrics_list: list[dict],
+        steps: list[int] | None = None,
+        timestamps: list[str] | None = None,
+        config: dict | None = None,
+    ):
+        """
+        Safely log bulk metrics to the database. Before logging, this method will ensure the database exists
+        and is set up with the correct tables. It also uses a cross-process lock to prevent
+        database locking errors when multiple processes access the same database.
+        """
+        if not metrics_list:
+            return
+
+        if timestamps is None:
+            timestamps = [datetime.now().isoformat()] * len(metrics_list)
+
+        db_path = SQLiteStorage.init_db(project)
+        with SQLiteStorage._get_process_lock(project):
+            with SQLiteStorage._get_connection(db_path) as conn:
+                cursor = conn.cursor()
+
+                if steps is None:
+                    steps = list(range(len(metrics_list)))
+                elif any(s is None for s in steps):
+                    cursor.execute(
+                        "SELECT MAX(step) FROM metrics WHERE run_name = ?", (run,)
+                    )
+                    last_step = cursor.fetchone()[0]
+                    current_step = 0 if last_step is None else last_step + 1
+                    processed_steps = []
+                    for step in steps:
+                        if step is None:
+                            processed_steps.append(current_step)
+                            current_step += 1
+                        else:
+                            processed_steps.append(step)
+                    steps = processed_steps
+
+                if len(metrics_list) != len(steps) or len(metrics_list) != len(
+                    timestamps
+                ):
+                    raise ValueError(
+                        "metrics_list, steps, and timestamps must have the same length"
+                    )
+
+                data = []
+                for i, metrics in enumerate(metrics_list):
+                    data.append(
+                        (
+                            timestamps[i],
+                            run,
+                            steps[i],
+                            orjson.dumps(serialize_values(metrics)),
+                        )
+                    )
+
+                cursor.executemany(
+                    """
+                    INSERT INTO metrics
+                    (timestamp, run_name, step, metrics)
+                    VALUES (?, ?, ?, ?)
+                    """,
+                    data,
+                )
+
+                if config:
+                    current_timestamp = datetime.now().isoformat()
+                    cursor.execute(
+                        """
+                        INSERT OR REPLACE INTO configs
+                        (run_name, config, created_at)
+                        VALUES (?, ?, ?)
+                        """,
+                        (
+                            run,
+                            orjson.dumps(serialize_values(config)),
+                            current_timestamp,
+                        ),
+                    )
+
+                conn.commit()
+
+    @staticmethod
+    def get_logs(project: str, run: str) -> list[dict]:
+        """Retrieve logs for a specific run. Logs include the step count (int) and the timestamp (datetime object)."""
+        db_path = SQLiteStorage.get_project_db_path(project)
+        if not db_path.exists():
+            return []
+
+        with SQLiteStorage._get_connection(db_path) as conn:
+            cursor = conn.cursor()
+            cursor.execute(
+                """
+                SELECT timestamp, step, metrics
+                FROM metrics
+                WHERE run_name = ?
+                ORDER BY timestamp
+                """,
+                (run,),
+            )
+
+            rows = cursor.fetchall()
+            results = []
+            for row in rows:
+                metrics = orjson.loads(row["metrics"])
+                metrics = deserialize_values(metrics)
+                metrics["timestamp"] = row["timestamp"]
+                metrics["step"] = row["step"]
+                results.append(metrics)
+            return results
+
+    @staticmethod
+    def load_from_dataset():
+        dataset_id = os.environ.get("TRACKIO_DATASET_ID")
+        space_repo_name = os.environ.get("SPACE_REPO_NAME")
+        if dataset_id is not None and space_repo_name is not None:
+            hfapi = hf.HfApi()
+            updated = False
+            if not TRACKIO_DIR.exists():
+                TRACKIO_DIR.mkdir(parents=True, exist_ok=True)
+            with SQLiteStorage.get_scheduler().lock:
+                try:
+                    files = hfapi.list_repo_files(dataset_id, repo_type="dataset")
+                    for file in files:
+                        # Download parquet and media assets
+                        if not (file.endswith(".parquet") or file.startswith("media/")):
+                            continue
+                        if (TRACKIO_DIR / file).exists():
+                            continue
+                        hf.hf_hub_download(
+                            dataset_id, file, repo_type="dataset", local_dir=TRACKIO_DIR
+                        )
+                        updated = True
+                except hf.errors.EntryNotFoundError:
+                    pass
+                except hf.errors.RepositoryNotFoundError:
+                    pass
+                if updated:
+                    SQLiteStorage.import_from_parquet()
+        SQLiteStorage._dataset_import_attempted = True
+
+    @staticmethod
+    def get_projects() -> list[str]:
+        """
+        Get list of all projects by scanning the database files in the trackio directory.
+        """
+        if not SQLiteStorage._dataset_import_attempted:
+            SQLiteStorage.load_from_dataset()
+
+        projects: set[str] = set()
+        if not TRACKIO_DIR.exists():
+            return []
+
+        for db_file in TRACKIO_DIR.glob(f"*{DB_EXT}"):
+            project_name = db_file.stem
+            projects.add(project_name)
+        return sorted(projects)
+
+    @staticmethod
+    def get_runs(project: str) -> list[str]:
+        """Get list of all runs for a project."""
+        db_path = SQLiteStorage.get_project_db_path(project)
+        if not db_path.exists():
+            return []
+
+        with SQLiteStorage._get_connection(db_path) as conn:
+            cursor = conn.cursor()
+            cursor.execute(
+                "SELECT DISTINCT run_name FROM metrics",
+            )
+            return [row[0] for row in cursor.fetchall()]
+
+    @staticmethod
+    def get_max_steps_for_runs(project: str) -> dict[str, int]:
+        """Get the maximum step for each run in a project."""
+        db_path = SQLiteStorage.get_project_db_path(project)
+        if not db_path.exists():
+            return {}
+
+        with SQLiteStorage._get_connection(db_path) as conn:
+            cursor = conn.cursor()
+            cursor.execute(
+                """
+                SELECT run_name, MAX(step) as max_step
+                FROM metrics
+                GROUP BY run_name
+                """
+            )
+
+            results = {}
+            for row in cursor.fetchall():
+                results[row["run_name"]] = row["max_step"]
+
+            return results
+
+    @staticmethod
+    def store_config(project: str, run: str, config: dict) -> None:
+        """Store configuration for a run."""
+        db_path = SQLiteStorage.init_db(project)
+
+        with SQLiteStorage._get_process_lock(project):
+            with SQLiteStorage._get_connection(db_path) as conn:
+                cursor = conn.cursor()
+                current_timestamp = datetime.now().isoformat()
+
+                cursor.execute(
+                    """
+                    INSERT OR REPLACE INTO configs
+                    (run_name, config, created_at)
+                    VALUES (?, ?, ?)
+                    """,
+                    (run, orjson.dumps(serialize_values(config)), current_timestamp),
+                )
+                conn.commit()
+
+    @staticmethod
+    def get_run_config(project: str, run: str) -> dict | None:
+        """Get configuration for a specific run."""
+        db_path = SQLiteStorage.get_project_db_path(project)
+        if not db_path.exists():
+            return None
+
+        with SQLiteStorage._get_connection(db_path) as conn:
+            cursor = conn.cursor()
+            try:
+                cursor.execute(
+                    """
+                    SELECT config FROM configs WHERE run_name = ?
+                    """,
+                    (run,),
+                )
+
+                row = cursor.fetchone()
+                if row:
+                    config = orjson.loads(row["config"])
+                    return deserialize_values(config)
+                return None
+            except sqlite3.OperationalError as e:
+                if "no such table: configs" in str(e):
+                    return None
+                raise
+
+    @staticmethod
+    def delete_run(project: str, run: str) -> bool:
+        """Delete a run from the database (both metrics and config)."""
+        db_path = SQLiteStorage.get_project_db_path(project)
+        if not db_path.exists():
+            return False
+
+        with SQLiteStorage._get_process_lock(project):
+            with SQLiteStorage._get_connection(db_path) as conn:
+                cursor = conn.cursor()
+                try:
+                    cursor.execute("DELETE FROM metrics WHERE run_name = ?", (run,))
+                    cursor.execute("DELETE FROM configs WHERE run_name = ?", (run,))
+                    conn.commit()
+                    return True
+                except sqlite3.Error:
+                    return False
+
+    @staticmethod
+    def get_all_run_configs(project: str) -> dict[str, dict]:
+        """Get configurations for all runs in a project."""
+        db_path = SQLiteStorage.get_project_db_path(project)
+        if not db_path.exists():
+            return {}
+
+        with SQLiteStorage._get_connection(db_path) as conn:
+            cursor = conn.cursor()
+            try:
+                cursor.execute(
+                    """
+                    SELECT run_name, config FROM configs
+                    """
+                )
+
+                results = {}
+                for row in cursor.fetchall():
+                    config = orjson.loads(row["config"])
+                    results[row["run_name"]] = deserialize_values(config)
+                return results
+            except sqlite3.OperationalError as e:
+                if "no such table: configs" in str(e):
+                    return {}
+                raise
+
+    @staticmethod
+    def get_metric_values(project: str, run: str, metric_name: str) -> list[dict]:
+        """Get all values for a specific metric in a project/run."""
+        db_path = SQLiteStorage.get_project_db_path(project)
+        if not db_path.exists():
+            return []
+
+        with SQLiteStorage._get_connection(db_path) as conn:
+            cursor = conn.cursor()
+            cursor.execute(
+                """
+                SELECT timestamp, step, metrics
+                FROM metrics
+                WHERE run_name = ?
+                ORDER BY timestamp
+                """,
+                (run,),
+            )
+
+            rows = cursor.fetchall()
+            results = []
+            for row in rows:
+                metrics = orjson.loads(row["metrics"])
+                metrics = deserialize_values(metrics)
+                if metric_name in metrics:
+                    results.append(
+                        {
+                            "timestamp": row["timestamp"],
+                            "step": row["step"],
+                            "value": metrics[metric_name],
+                        }
+                    )
+            return results
+
+    @staticmethod
+    def get_all_metrics_for_run(project: str, run: str) -> list[str]:
+        """Get all metric names for a specific project/run."""
+        db_path = SQLiteStorage.get_project_db_path(project)
+        if not db_path.exists():
+            return []
+
+        with SQLiteStorage._get_connection(db_path) as conn:
+            cursor = conn.cursor()
+            cursor.execute(
+                """
+                SELECT metrics
+                FROM metrics
+                WHERE run_name = ?
+                ORDER BY timestamp
+                """,
+                (run,),
+            )
+
+            rows = cursor.fetchall()
+            all_metrics = set()
+            for row in rows:
+                metrics = orjson.loads(row["metrics"])
+                metrics = deserialize_values(metrics)
+                for key in metrics.keys():
+                    if key not in ["timestamp", "step"]:
+                        all_metrics.add(key)
+            return sorted(list(all_metrics))
+
+    def finish(self):
+        """Cleanup when run is finished."""
+        pass

table.py

@@ -0,0 +1,53 @@
+from typing import Any, Literal
+
+from pandas import DataFrame
+
+
+class Table:
+    """
+    Initializes a Table object.
+
+    Args:
+        columns (`list[str]`, *optional*):
+            Names of the columns in the table. Optional if `data` is provided. Not
+            expected if `dataframe` is provided. Currently ignored.
+        data (`list[list[Any]]`, *optional*):
+            2D row-oriented array of values.
+        dataframe (`pandas.`DataFrame``, *optional*):
+            DataFrame object used to create the table. When set, `data` and `columns`
+            arguments are ignored.
+        rows (`list[list[any]]`, *optional*):
+            Currently ignored.
+        optional (`bool` or `list[bool]`, *optional*, defaults to `True`):
+            Currently ignored.
+        allow_mixed_types (`bool`, *optional*, defaults to `False`):
+            Currently ignored.
+        log_mode: (`Literal["IMMUTABLE", "MUTABLE", "INCREMENTAL"]` or `None`, *optional*, defaults to `"IMMUTABLE"`):
+            Currently ignored.
+    """
+
+    TYPE = "trackio.table"
+
+    def __init__(
+        self,
+        columns: list[str] | None = None,
+        data: list[list[Any]] | None = None,
+        dataframe: DataFrame | None = None,
+        rows: list[list[Any]] | None = None,
+        optional: bool | list[bool] = True,
+        allow_mixed_types: bool = False,
+        log_mode: Literal["IMMUTABLE", "MUTABLE", "INCREMENTAL"] | None = "IMMUTABLE",
+    ):
+        # TODO: implement support for columns, dtype, optional, allow_mixed_types, and log_mode.
+        # for now (like `rows`) they are included for API compat but don't do anything.
+
+        if dataframe is None:
+            self.data = data
+        else:
+            self.data = dataframe.to_dict(orient="records")
+
+    def _to_dict(self):
+        return {
+            "_type": self.TYPE,
+            "_value": self.data,
+        }

typehints.py

@@ -0,0 +1,18 @@
+from typing import Any, TypedDict
+
+from gradio import FileData
+
+
+class LogEntry(TypedDict):
+    project: str
+    run: str
+    metrics: dict[str, Any]
+    step: int | None
+    config: dict[str, Any] | None
+
+
+class UploadEntry(TypedDict):
+    project: str
+    run: str
+    step: int | None
+    uploaded_file: FileData

ui/__init__.py

@@ -0,0 +1,10 @@
+try:
+    from trackio.ui.main import demo
+    from trackio.ui.run_detail import run_detail_page
+    from trackio.ui.runs import run_page
+except ImportError:
+    from ui.main import demo
+    from ui.run_detail import run_detail_page
+    from ui.runs import run_page
+
+__all__ = ["demo", "run_page", "run_detail_page"]

ui/fns.py

@@ -0,0 +1,241 @@
+"""Shared functions for the Trackio UI."""
+
+import os
+
+import gradio as gr
+import huggingface_hub as hf
+
+try:
+    import trackio.utils as utils
+    from trackio.sqlite_storage import SQLiteStorage
+    from trackio.ui.helpers.run_selection import RunSelection
+except ImportError:
+    import utils
+    from sqlite_storage import SQLiteStorage
+    from ui.helpers.run_selection import RunSelection
+
+CONFIG_COLUMN_MAPPINGS = {
+    "_Username": "Username",
+    "_Created": "Created",
+    "_Group": "Group",
+}
+CONFIG_COLUMN_MAPPINGS_REVERSE = {v: k for k, v in CONFIG_COLUMN_MAPPINGS.items()}
+
+
+HfApi = hf.HfApi()
+
+
+def get_project_info() -> str | None:
+    dataset_id = os.environ.get("TRACKIO_DATASET_ID")
+    space_id = utils.get_space()
+    if utils.persistent_storage_enabled():
+        return "✨ Persistent Storage is enabled, logs are stored directly in this Space."
+    if dataset_id:
+        sync_status = utils.get_sync_status(SQLiteStorage.get_scheduler())
+        upgrade_message = f"New changes are synced every 5 min <span class='info-container'><input type='checkbox' class='info-checkbox' id='upgrade-info'><label for='upgrade-info' class='info-icon'>&#9432;</label><span class='info-expandable'> To avoid losing data between syncs, <a href='https://huggingface.co/spaces/{space_id}/settings' class='accent-link'>click here</a> to open this Space's settings and add Persistent Storage. Make sure data is synced prior to enabling.</span></span>"
+        if sync_status is not None:
+            info = f"&#x21bb; Backed up {sync_status} min ago to <a href='https://huggingface.co/datasets/{dataset_id}' target='_blank' class='accent-link'>{dataset_id}</a> | {upgrade_message}"
+        else:
+            info = f"&#x21bb; Not backed up yet to <a href='https://huggingface.co/datasets/{dataset_id}' target='_blank' class='accent-link'>{dataset_id}</a> | {upgrade_message}"
+        return info
+    return None
+
+
+def get_projects(request: gr.Request):
+    projects = SQLiteStorage.get_projects()
+    if project := request.query_params.get("project"):
+        interactive = False
+    else:
+        interactive = True
+        if selected_project := request.query_params.get("selected_project"):
+            project = selected_project
+        else:
+            project = projects[0] if projects else None
+
+    return gr.Dropdown(
+        label="Project",
+        choices=projects,
+        value=project,
+        allow_custom_value=True,
+        interactive=interactive,
+        info=get_project_info(),
+    )
+
+
+def update_navbar_value(project_dd, request: gr.Request):
+    write_token = None
+    if hasattr(request, "query_params") and request.query_params:
+        write_token = request.query_params.get("write_token")
+
+    metrics_url = f"?selected_project={project_dd}"
+    runs_url = f"runs?selected_project={project_dd}"
+
+    if write_token:
+        metrics_url += f"&write_token={write_token}"
+        runs_url += f"&write_token={write_token}"
+
+    return gr.Navbar(
+        value=[
+            ("Metrics", metrics_url),
+            ("Runs", runs_url),
+        ]
+    )
+
+
+def check_hf_token_has_write_access(hf_token: str | None) -> None:
+    """
+    Checks to see if the provided hf_token is valid and has write access to the Space
+    that Trackio is running in. If the hf_token is valid or if Trackio is not running
+    on a Space, this function does nothing. Otherwise, it raises a PermissionError.
+    """
+    if os.getenv("SYSTEM") == "spaces":  # if we are running in Spaces
+        # check auth token passed in
+        if hf_token is None:
+            raise PermissionError(
+                "Expected a HF_TOKEN to be provided when logging to a Space"
+            )
+        who = HfApi.whoami(hf_token)
+        owner_name = os.getenv("SPACE_AUTHOR_NAME")
+        repo_name = os.getenv("SPACE_REPO_NAME")
+        # make sure the token user is either the author of the space,
+        # or is a member of an org that is the author.
+        orgs = [o["name"] for o in who["orgs"]]
+        if owner_name != who["name"] and owner_name not in orgs:
+            raise PermissionError(
+                "Expected the provided hf_token to be the user owner of the space, or be a member of the org owner of the space"
+            )
+        # reject fine-grained tokens without specific repo access
+        access_token = who["auth"]["accessToken"]
+        if access_token["role"] == "fineGrained":
+            matched = False
+            for item in access_token["fineGrained"]["scoped"]:
+                if (
+                    item["entity"]["type"] == "space"
+                    and item["entity"]["name"] == f"{owner_name}/{repo_name}"
+                    and "repo.write" in item["permissions"]
+                ):
+                    matched = True
+                    break
+                if (
+                    (
+                        item["entity"]["type"] == "user"
+                        or item["entity"]["type"] == "org"
+                    )
+                    and item["entity"]["name"] == owner_name
+                    and "repo.write" in item["permissions"]
+                ):
+                    matched = True
+                    break
+            if not matched:
+                raise PermissionError(
+                    "Expected the provided hf_token with fine grained permissions to provide write access to the space"
+                )
+        # reject read-only tokens
+        elif access_token["role"] != "write":
+            raise PermissionError(
+                "Expected the provided hf_token to provide write permissions"
+            )
+
+
+def check_oauth_token_has_write_access(oauth_token: str | None) -> None:
+    """
+    Checks to see if the oauth token provided via Gradio's OAuth is valid and has write access
+    to the Space that Trackio is running in. If the oauth token is valid or if Trackio is not running
+    on a Space, this function does nothing. Otherwise, it raises a PermissionError.
+    """
+    if not os.getenv("SYSTEM") == "spaces":
+        return
+    if oauth_token is None:
+        raise PermissionError(
+            "Expected an oauth to be provided when logging to a Space"
+        )
+    who = HfApi.whoami(oauth_token)
+    user_name = who["name"]
+    owner_name = os.getenv("SPACE_AUTHOR_NAME")
+    if user_name == owner_name:
+        return
+    # check if user is a member of an org that owns the space with write permissions
+    for org in who["orgs"]:
+        if org["name"] == owner_name and org["roleInOrg"] == "write":
+            return
+    raise PermissionError(
+        "Expected the oauth token to be the user owner of the space, or be a member of the org owner of the space"
+    )
+
+
+def get_group_by_fields(project: str):
+    configs = SQLiteStorage.get_all_run_configs(project) if project else {}
+    keys = set()
+    for config in configs.values():
+        keys.update(config.keys())
+    keys.discard("_Created")
+    keys = [CONFIG_COLUMN_MAPPINGS.get(key, key) for key in keys]
+    choices = [None] + sorted(keys)
+    return gr.Dropdown(
+        choices=choices,
+        value=None,
+        interactive=True,
+    )
+
+
+def group_runs_by_config(
+    project: str, config_key: str, filter_text: str | None = None
+) -> dict[str, list[str]]:
+    if not project or not config_key:
+        return {}
+    display_key = config_key
+    config_key = CONFIG_COLUMN_MAPPINGS_REVERSE.get(config_key, config_key)
+    configs = SQLiteStorage.get_all_run_configs(project)
+    groups: dict[str, list[str]] = {}
+    for run_name, config in configs.items():
+        if filter_text and filter_text not in run_name:
+            continue
+        group_name = config.get(config_key, "None")
+        label = f"{display_key}: {group_name}"
+        groups.setdefault(label, []).append(run_name)
+    for label in groups:
+        groups[label].sort()
+    sorted_groups = dict(sorted(groups.items(), key=lambda kv: kv[0].lower()))
+    return sorted_groups
+
+
+def run_checkbox_update(selection: RunSelection, **kwargs) -> gr.CheckboxGroup:
+    return gr.CheckboxGroup(
+        choices=selection.choices,
+        value=selection.selected,
+        **kwargs,
+    )
+
+
+def handle_run_checkbox_change(
+    selected_runs: list[str] | None, selection: RunSelection
+) -> RunSelection:
+    selection.select(selected_runs or [])
+    return selection
+
+
+def handle_group_checkbox_change(
+    group_selected: list[str] | None,
+    selection: RunSelection,
+    group_runs: list[str] | None,
+):
+    subset, _ = selection.replace_group(group_runs or [], group_selected or [])
+    return (
+        selection,
+        gr.CheckboxGroup(value=subset),
+        run_checkbox_update(selection),
+    )
+
+
+def handle_group_toggle(
+    select_all: bool,
+    selection: RunSelection,
+    group_runs: list[str] | None,
+):
+    target = list(group_runs or []) if select_all else []
+    subset, _ = selection.replace_group(group_runs or [], target)
+    return (
+        selection,
+        gr.CheckboxGroup(value=subset),
+        run_checkbox_update(selection),
+    )

ui/helpers/run_selection.py

@@ -0,0 +1,46 @@
+from dataclasses import dataclass, field
+
+try:
+    import trackio.utils as utils
+except ImportError:
+    import utils
+
+
+@dataclass
+class RunSelection:
+    choices: list[str] = field(default_factory=list)
+    selected: list[str] = field(default_factory=list)
+    locked: bool = False
+
+    def update_choices(
+        self, runs: list[str], preferred: list[str] | None = None
+    ) -> bool:
+        if self.choices == runs:
+            return False
+        new_choices = set(runs) - set(self.choices)
+        self.choices = list(runs)
+        if self.locked:
+            base = set(self.selected) | new_choices
+        elif preferred:
+            base = set(preferred)
+        else:
+            base = set(runs)
+        self.selected = [run for run in self.choices if run in base]
+        return True
+
+    def select(self, runs: list[str]) -> list[str]:
+        choice_set = set(self.choices)
+        self.selected = [run for run in runs if run in choice_set]
+        self.locked = True
+        return self.selected
+
+    def replace_group(
+        self, group_runs: list[str], new_subset: list[str] | None
+    ) -> tuple[list[str], list[str]]:
+        new_subset = utils.ordered_subset(group_runs, new_subset)
+        selection_set = set(self.selected)
+        selection_set.difference_update(group_runs)
+        selection_set.update(new_subset)
+        self.selected = [run for run in self.choices if run in selection_set]
+        self.locked = True
+        return new_subset, self.selected

ui/main.py

@@ -0,0 +1,1212 @@
+"""The main page for the Trackio UI."""
+
+import os
+import re
+import secrets
+import shutil
+from dataclasses import dataclass
+from typing import Any
+
+import gradio as gr
+import numpy as np
+import pandas as pd
+
+try:
+    import trackio.utils as utils
+    from trackio.file_storage import FileStorage
+    from trackio.media import TrackioImage, TrackioVideo
+    from trackio.sqlite_storage import SQLiteStorage
+    from trackio.table import Table
+    from trackio.typehints import LogEntry, UploadEntry
+    from trackio.ui import fns
+    from trackio.ui.helpers.run_selection import RunSelection
+    from trackio.ui.run_detail import run_detail_page
+    from trackio.ui.runs import run_page
+except ImportError:
+    import utils
+    from file_storage import FileStorage
+    from media import TrackioImage, TrackioVideo
+    from sqlite_storage import SQLiteStorage
+    from table import Table
+    from typehints import LogEntry, UploadEntry
+    from ui import fns
+    from ui.helpers.run_selection import RunSelection
+    from ui.run_detail import run_detail_page
+    from ui.runs import run_page
+
+
+INSTRUCTIONS_SPACES = """
+## Start logging with Trackio 🤗
+
+To start logging to this Trackio dashboard, first make sure you have the Trackio library installed. You can do this by running:
+
+```bash
+pip install trackio
+```
+
+Then, start logging to this Trackio dashboard by passing in the `space_id` to `trackio.init()`:
+
+```python
+import trackio
+trackio.init(project="my-project", space_id="{}")
+```
+
+Then call `trackio.log()` to log metrics.
+
+```python
+for i in range(10):
+    trackio.log({{"loss": 1/(i+1)}})
+```
+
+Finally, call `trackio.finish()` to finish the run.
+
+```python
+trackio.finish()
+```
+"""
+
+INSTRUCTIONS_LOCAL = """
+## Start logging with Trackio 🤗
+ 
+You can create a new project by calling `trackio.init()`:
+
+```python
+import trackio
+trackio.init(project="my-project")
+ ```
+
+Then call `trackio.log()` to log metrics.
+
+```python
+for i in range(10):
+    trackio.log({"loss": 1/(i+1)})
+```
+
+Finally, call `trackio.finish()` to finish the run.
+
+```python
+trackio.finish()
+```
+
+Read the [Trackio documentation](https://huggingface.co/docs/trackio/en/index) for more examples.
+"""
+
+
+def get_runs(project) -> list[str]:
+    if not project:
+        return []
+    return SQLiteStorage.get_runs(project)
+
+
+def get_available_metrics(project: str, runs: list[str]) -> list[str]:
+    """Get all available metrics across all runs for x-axis selection."""
+    if not project or not runs:
+        return ["step", "time"]
+
+    all_metrics = set()
+    for run in runs:
+        metrics = SQLiteStorage.get_logs(project, run)
+        if metrics:
+            df = pd.DataFrame(metrics)
+            numeric_cols = df.select_dtypes(include="number").columns
+            numeric_cols = [c for c in numeric_cols if c not in utils.RESERVED_KEYS]
+            all_metrics.update(numeric_cols)
+
+    all_metrics.add("step")
+    all_metrics.add("time")
+
+    sorted_metrics = utils.sort_metrics_by_prefix(list(all_metrics))
+
+    result = ["step", "time"]
+    for metric in sorted_metrics:
+        if metric not in result:
+            result.append(metric)
+
+    return result
+
+
+@dataclass
+class MediaData:
+    caption: str | None
+    file_path: str
+
+
+def extract_media(logs: list[dict]) -> dict[str, list[MediaData]]:
+    media_by_key: dict[str, list[MediaData]] = {}
+    logs = sorted(logs, key=lambda x: x.get("step", 0))
+    for log in logs:
+        for key, value in log.items():
+            if isinstance(value, dict):
+                type = value.get("_type")
+                if type == TrackioImage.TYPE or type == TrackioVideo.TYPE:
+                    if key not in media_by_key:
+                        media_by_key[key] = []
+                    try:
+                        media_data = MediaData(
+                            file_path=utils.MEDIA_DIR / value.get("file_path"),
+                            caption=value.get("caption"),
+                        )
+                        media_by_key[key].append(media_data)
+                    except Exception as e:
+                        print(f"Media currently unavailable: {key}: {e}")
+    return media_by_key
+
+
+def load_run_data(
+    project: str | None,
+    run: str | None,
+    smoothing_granularity: int,
+    x_axis: str,
+    log_scale: bool = False,
+) -> tuple[pd.DataFrame, dict]:
+    if not project or not run:
+        return None, None
+
+    logs = SQLiteStorage.get_logs(project, run)
+    if not logs:
+        return None, None
+
+    media = extract_media(logs)
+    df = pd.DataFrame(logs)
+
+    if "step" not in df.columns:
+        df["step"] = range(len(df))
+
+    if x_axis == "time" and "timestamp" in df.columns:
+        df["timestamp"] = pd.to_datetime(df["timestamp"])
+        first_timestamp = df["timestamp"].min()
+        df["time"] = (df["timestamp"] - first_timestamp).dt.total_seconds()
+        x_column = "time"
+    elif x_axis == "step":
+        x_column = "step"
+    else:
+        x_column = x_axis
+
+    if log_scale and x_column in df.columns:
+        x_vals = df[x_column]
+        if (x_vals <= 0).any():
+            df[x_column] = np.log10(np.maximum(x_vals, 0) + 1)
+        else:
+            df[x_column] = np.log10(x_vals)
+
+    if smoothing_granularity > 0:
+        numeric_cols = df.select_dtypes(include="number").columns
+        numeric_cols = [c for c in numeric_cols if c not in utils.RESERVED_KEYS]
+
+        df_original = df.copy()
+        df_original["run"] = run
+        df_original["data_type"] = "original"
+
+        df_smoothed = df.copy()
+        window_size = max(3, min(smoothing_granularity, len(df)))
+        df_smoothed[numeric_cols] = (
+            df_smoothed[numeric_cols]
+            .rolling(window=window_size, center=True, min_periods=1)
+            .mean()
+        )
+        df_smoothed["run"] = f"{run}_smoothed"
+        df_smoothed["data_type"] = "smoothed"
+
+        combined_df = pd.concat([df_original, df_smoothed], ignore_index=True)
+        combined_df["x_axis"] = x_column
+        return combined_df, media
+    else:
+        df["run"] = run
+        df["data_type"] = "original"
+        df["x_axis"] = x_column
+        return df, media
+
+
+def refresh_runs(
+    project: str | None,
+    filter_text: str | None,
+    selection: RunSelection,
+    selected_runs_from_url: list[str] | None = None,
+):
+    if project is None:
+        runs: list[str] = []
+    else:
+        runs = get_runs(project)
+        if filter_text:
+            runs = [r for r in runs if filter_text in r]
+
+    preferred = None
+    if selected_runs_from_url:
+        preferred = [r for r in runs if r in selected_runs_from_url]
+
+    did_change = selection.update_choices(runs, preferred)
+    return (
+        fns.run_checkbox_update(selection) if did_change else gr.CheckboxGroup(),
+        gr.Textbox(label=f"Runs ({len(runs)})"),
+        selection,
+    )
+
+
+def generate_embed(project: str, metrics: str, selection: RunSelection) -> str:
+    return utils.generate_embed_code(project, metrics, selection.selected)
+
+
+def update_x_axis_choices(project, selection):
+    """Update x-axis dropdown choices based on available metrics."""
+    runs = selection.selected
+    available_metrics = get_available_metrics(project, runs)
+    return gr.Dropdown(
+        label="X-axis",
+        choices=available_metrics,
+        value="step",
+    )
+
+
+def toggle_timer(cb_value):
+    if cb_value:
+        return gr.Timer(active=True)
+    else:
+        return gr.Timer(active=False)
+
+
+def upload_db_to_space(
+    project: str, uploaded_db: gr.FileData, hf_token: str | None
+) -> None:
+    """
+    Uploads the database of a local Trackio project to a Hugging Face Space.
+    """
+    fns.check_hf_token_has_write_access(hf_token)
+    db_project_path = SQLiteStorage.get_project_db_path(project)
+    if os.path.exists(db_project_path):
+        raise gr.Error(
+            f"Trackio database file already exists for project {project}, cannot overwrite."
+        )
+    os.makedirs(os.path.dirname(db_project_path), exist_ok=True)
+    shutil.copy(uploaded_db["path"], db_project_path)
+
+
+def bulk_upload_media(uploads: list[UploadEntry], hf_token: str | None) -> None:
+    """
+    Uploads media files to a Trackio dashboard. Each entry in the list is a tuple of the project, run, and media file to be uploaded.
+    """
+    fns.check_hf_token_has_write_access(hf_token)
+    for upload in uploads:
+        media_path = FileStorage.init_project_media_path(
+            upload["project"], upload["run"], upload["step"]
+        )
+        shutil.copy(upload["uploaded_file"]["path"], media_path)
+
+
+def log(
+    project: str,
+    run: str,
+    metrics: dict[str, Any],
+    step: int | None,
+    hf_token: str | None,
+) -> None:
+    """
+    Note: this method is not used in the latest versions of Trackio (replaced by bulk_log) but
+    is kept for backwards compatibility for users who are connecting to a newer version of
+    a Trackio Spaces dashboard with an older version of Trackio installed locally.
+    """
+    fns.check_hf_token_has_write_access(hf_token)
+    SQLiteStorage.log(project=project, run=run, metrics=metrics, step=step)
+
+
+def bulk_log(
+    logs: list[LogEntry],
+    hf_token: str | None,
+) -> None:
+    """
+    Logs a list of metrics to a Trackio dashboard. Each entry in the list is a dictionary of the project, run, a dictionary of metrics, and optionally, a step and config.
+    """
+    fns.check_hf_token_has_write_access(hf_token)
+
+    logs_by_run = {}
+    for log_entry in logs:
+        key = (log_entry["project"], log_entry["run"])
+        if key not in logs_by_run:
+            logs_by_run[key] = {"metrics": [], "steps": [], "config": None}
+        logs_by_run[key]["metrics"].append(log_entry["metrics"])
+        logs_by_run[key]["steps"].append(log_entry.get("step"))
+        if log_entry.get("config") and logs_by_run[key]["config"] is None:
+            logs_by_run[key]["config"] = log_entry["config"]
+
+    for (project, run), data in logs_by_run.items():
+        SQLiteStorage.bulk_log(
+            project=project,
+            run=run,
+            metrics_list=data["metrics"],
+            steps=data["steps"],
+            config=data["config"],
+        )
+
+
+def get_metric_values(
+    project: str,
+    run: str,
+    metric_name: str,
+) -> list[dict]:
+    """
+    Get all values for a specific metric in a project/run.
+    Returns a list of dictionaries with timestamp, step, and value.
+    """
+    return SQLiteStorage.get_metric_values(project, run, metric_name)
+
+
+def get_runs_for_project(
+    project: str,
+) -> list[str]:
+    """
+    Get all runs for a given project.
+    Returns a list of run names.
+    """
+    return SQLiteStorage.get_runs(project)
+
+
+def get_metrics_for_run(
+    project: str,
+    run: str,
+) -> list[str]:
+    """
+    Get all metrics for a given project and run.
+    Returns a list of metric names.
+    """
+    return SQLiteStorage.get_all_metrics_for_run(project, run)
+
+
+def filter_metrics_by_regex(metrics: list[str], filter_pattern: str) -> list[str]:
+    """
+    Filter metrics using regex pattern.
+
+    Args:
+        metrics: List of metric names to filter
+        filter_pattern: Regex pattern to match against metric names
+
+    Returns:
+        List of metric names that match the pattern
+    """
+    if not filter_pattern.strip():
+        return metrics
+
+    try:
+        pattern = re.compile(filter_pattern, re.IGNORECASE)
+        return [metric for metric in metrics if pattern.search(metric)]
+    except re.error:
+        return [
+            metric for metric in metrics if filter_pattern.lower() in metric.lower()
+        ]
+
+
+def get_all_projects() -> list[str]:
+    """
+    Get all project names.
+    Returns a list of project names.
+    """
+    return SQLiteStorage.get_projects()
+
+
+def get_project_summary(project: str) -> dict:
+    """
+    Get a summary of a project including number of runs and recent activity.
+
+    Args:
+        project: Project name
+
+    Returns:
+        Dictionary with project summary information
+    """
+    runs = SQLiteStorage.get_runs(project)
+    if not runs:
+        return {"project": project, "num_runs": 0, "runs": [], "last_activity": None}
+
+    last_steps = SQLiteStorage.get_max_steps_for_runs(project)
+
+    return {
+        "project": project,
+        "num_runs": len(runs),
+        "runs": runs,
+        "last_activity": max(last_steps.values()) if last_steps else None,
+    }
+
+
+def get_run_summary(project: str, run: str) -> dict:
+    """
+    Get a summary of a specific run including metrics and configuration.
+
+    Args:
+        project: Project name
+        run: Run name
+
+    Returns:
+        Dictionary with run summary information
+    """
+    logs = SQLiteStorage.get_logs(project, run)
+    metrics = SQLiteStorage.get_all_metrics_for_run(project, run)
+
+    if not logs:
+        return {
+            "project": project,
+            "run": run,
+            "num_logs": 0,
+            "metrics": [],
+            "config": None,
+            "last_step": None,
+        }
+
+    df = pd.DataFrame(logs)
+    config = logs[0].get("config") if logs else None
+    last_step = df["step"].max() if "step" in df.columns else len(logs) - 1
+
+    return {
+        "project": project,
+        "run": run,
+        "num_logs": len(logs),
+        "metrics": metrics,
+        "config": config,
+        "last_step": last_step,
+    }
+
+
+def configure(request: gr.Request):
+    sidebar_param = request.query_params.get("sidebar")
+    match sidebar_param:
+        case "collapsed":
+            sidebar = gr.Sidebar(open=False, visible=True)
+        case "hidden":
+            sidebar = gr.Sidebar(open=False, visible=False)
+        case _:
+            sidebar = gr.Sidebar(open=True, visible=True)
+
+    metrics_param = request.query_params.get("metrics", "")
+    runs_param = request.query_params.get("runs", "")
+    selected_runs = runs_param.split(",") if runs_param else []
+    navbar_param = request.query_params.get("navbar")
+    match navbar_param:
+        case "hidden":
+            navbar = gr.Navbar(visible=False)
+        case _:
+            navbar = gr.Navbar(visible=True)
+
+    return [], sidebar, metrics_param, selected_runs, navbar
+
+
+def create_media_section(media_by_run: dict[str, dict[str, list[MediaData]]]):
+    with gr.Accordion(label="media"):
+        with gr.Group(elem_classes=("media-group")):
+            for run, media_by_key in media_by_run.items():
+                with gr.Tab(label=run, elem_classes=("media-tab")):
+                    for key, media_item in media_by_key.items():
+                        gr.Gallery(
+                            [(item.file_path, item.caption) for item in media_item],
+                            label=key,
+                            columns=6,
+                            elem_classes=("media-gallery"),
+                        )
+
+
+css = """
+#run-cb .wrap { gap: 2px; }
+#run-cb .wrap label {
+    line-height: 1;
+    padding: 6px;
+}
+.logo-light { display: block; } 
+.logo-dark { display: none; }
+.dark .logo-light { display: none; }
+.dark .logo-dark { display: block; }
+.dark .caption-label { color: white; }
+
+.info-container {
+    position: relative;
+    display: inline;
+}
+.info-checkbox {
+    position: absolute;
+    opacity: 0;
+    pointer-events: none;
+}
+.info-icon {
+    border-bottom: 1px dotted;
+    cursor: pointer;
+    user-select: none;
+    color: var(--color-accent);
+}
+.info-expandable {
+    display: none;
+    opacity: 0;
+    transition: opacity 0.2s ease-in-out;
+}
+.info-checkbox:checked ~ .info-expandable {
+    display: inline;
+    opacity: 1;
+}
+.info-icon:hover { opacity: 0.8; }
+.accent-link { font-weight: bold; }
+
+.media-gallery .fixed-height { min-height: 275px; }
+.media-group, .media-group > div { background: none; }
+.media-group .tabs { padding: 0.5em; }
+.media-tab { max-height: 500px; overflow-y: scroll; }
+"""
+
+javascript = """
+<script>
+function setCookie(name, value, days) {
+    var expires = "";
+    if (days) {
+        var date = new Date();
+        date.setTime(date.getTime() + (days * 24 * 60 * 60 * 1000));
+        expires = "; expires=" + date.toUTCString();
+    }
+    document.cookie = name + "=" + (value || "") + expires + "; path=/; SameSite=Lax";
+}
+
+function getCookie(name) {
+    var nameEQ = name + "=";
+    var ca = document.cookie.split(';');
+    for(var i=0;i < ca.length;i++) {
+        var c = ca[i];
+        while (c.charAt(0)==' ') c = c.substring(1,c.length);
+        if (c.indexOf(nameEQ) == 0) return c.substring(nameEQ.length,c.length);
+    }
+    return null;
+}
+
+(function() {
+    const urlParams = new URLSearchParams(window.location.search);
+    const writeToken = urlParams.get('write_token');
+    
+    if (writeToken) {
+        setCookie('trackio_write_token', writeToken, 7);
+                
+        // Only remove write_token from URL if not in iframe
+        // In iframes, keep it in URL as cookies may be blocked
+        const inIframe = window.self !== window.top;
+        if (!inIframe) {
+            urlParams.delete('write_token');
+            const newUrl = window.location.pathname + 
+                (urlParams.toString() ? '?' + urlParams.toString() : '') + 
+                window.location.hash;
+            window.history.replaceState({}, document.title, newUrl);
+        }
+    }
+})();
+</script>
+"""
+
+
+gr.set_static_paths(paths=[utils.MEDIA_DIR])
+
+with gr.Blocks(title="Trackio Dashboard", css=css, head=javascript) as demo:
+    with gr.Sidebar(open=False) as sidebar:
+        logo_urls = utils.get_logo_urls()
+        logo = gr.Markdown(
+            f"""
+                <img src='{logo_urls["light"]}' width='80%' class='logo-light'>
+                <img src='{logo_urls["dark"]}' width='80%' class='logo-dark'>            
+            """
+        )
+        project_dd = gr.Dropdown(label="Project", allow_custom_value=True)
+
+        embed_code = gr.Code(
+            label="Embed this view",
+            max_lines=2,
+            lines=2,
+            language="html",
+            visible=bool(os.environ.get("SPACE_HOST")),
+        )
+        with gr.Group():
+            run_tb = gr.Textbox(label="Runs", placeholder="Type to filter...")
+            run_group_by_dd = gr.Dropdown(label="Group by...", choices=[], value=None)
+            grouped_runs_panel = gr.Group(visible=False)
+            run_cb = gr.CheckboxGroup(
+                label="Runs",
+                choices=[],
+                interactive=True,
+                elem_id="run-cb",
+                show_select_all=True,
+            )
+
+        gr.HTML("<hr>")
+        realtime_cb = gr.Checkbox(label="Refresh metrics realtime", value=True)
+        smoothing_slider = gr.Slider(
+            label="Smoothing Factor",
+            minimum=0,
+            maximum=20,
+            value=10,
+            step=1,
+            info="0 = no smoothing",
+        )
+        x_axis_dd = gr.Dropdown(
+            label="X-axis",
+            choices=["step", "time"],
+            value="step",
+        )
+        log_scale_cb = gr.Checkbox(label="Log scale X-axis", value=False)
+        metric_filter_tb = gr.Textbox(
+            label="Metric Filter (regex)",
+            placeholder="e.g., loss|ndcg@10|gpu",
+            value="",
+            info="Filter metrics using regex patterns. Leave empty to show all metrics.",
+        )
+
+    navbar = gr.Navbar(value=[("Metrics", ""), ("Runs", "/runs")], main_page_name=False)
+    timer = gr.Timer(value=1)
+    metrics_subset = gr.State([])
+    selected_runs_from_url = gr.State([])
+    run_selection_state = gr.State(RunSelection())
+
+    gr.on(
+        [demo.load],
+        fn=configure,
+        outputs=[
+            metrics_subset,
+            sidebar,
+            metric_filter_tb,
+            selected_runs_from_url,
+            navbar,
+        ],
+        queue=False,
+        api_name=False,
+    )
+    gr.on(
+        [demo.load],
+        fn=fns.get_projects,
+        outputs=project_dd,
+        show_progress="hidden",
+        queue=False,
+        api_name=False,
+    )
+    gr.on(
+        [timer.tick],
+        fn=refresh_runs,
+        inputs=[project_dd, run_tb, run_selection_state, selected_runs_from_url],
+        outputs=[run_cb, run_tb, run_selection_state],
+        show_progress="hidden",
+        api_name=False,
+    )
+    gr.on(
+        [timer.tick],
+        fn=lambda: gr.Dropdown(info=fns.get_project_info()),
+        outputs=[project_dd],
+        show_progress="hidden",
+        api_name=False,
+    )
+    gr.on(
+        [demo.load, project_dd.change],
+        fn=refresh_runs,
+        inputs=[project_dd, run_tb, run_selection_state, selected_runs_from_url],
+        outputs=[run_cb, run_tb, run_selection_state],
+        show_progress="hidden",
+        queue=False,
+        api_name=False,
+    ).then(
+        fn=update_x_axis_choices,
+        inputs=[project_dd, run_selection_state],
+        outputs=x_axis_dd,
+        show_progress="hidden",
+        queue=False,
+        api_name=False,
+    ).then(
+        fn=generate_embed,
+        inputs=[project_dd, metric_filter_tb, run_selection_state],
+        outputs=[embed_code],
+        show_progress="hidden",
+        api_name=False,
+        queue=False,
+    ).then(
+        fns.update_navbar_value,
+        inputs=[project_dd],
+        outputs=[navbar],
+        show_progress="hidden",
+        api_name=False,
+        queue=False,
+    ).then(
+        fn=fns.get_group_by_fields,
+        inputs=[project_dd],
+        outputs=[run_group_by_dd],
+        show_progress="hidden",
+        api_name=False,
+        queue=False,
+    )
+
+    gr.on(
+        [run_cb.input],
+        fn=update_x_axis_choices,
+        inputs=[project_dd, run_selection_state],
+        outputs=x_axis_dd,
+        show_progress="hidden",
+        queue=False,
+        api_name=False,
+    )
+    gr.on(
+        [metric_filter_tb.change, run_cb.change],
+        fn=generate_embed,
+        inputs=[project_dd, metric_filter_tb, run_selection_state],
+        outputs=embed_code,
+        show_progress="hidden",
+        api_name=False,
+        queue=False,
+    )
+
+    def toggle_group_view(group_by_dd):
+        return (
+            gr.CheckboxGroup(visible=not bool(group_by_dd)),
+            gr.Group(visible=bool(group_by_dd)),
+        )
+
+    gr.on(
+        [run_group_by_dd.change],
+        fn=toggle_group_view,
+        inputs=[run_group_by_dd],
+        outputs=[run_cb, grouped_runs_panel],
+        show_progress="hidden",
+        api_name=False,
+        queue=False,
+    )
+
+    realtime_cb.change(
+        fn=toggle_timer,
+        inputs=realtime_cb,
+        outputs=timer,
+        api_name=False,
+        queue=False,
+    )
+    run_cb.input(
+        fn=fns.handle_run_checkbox_change,
+        inputs=[run_cb, run_selection_state],
+        outputs=run_selection_state,
+        api_name=False,
+        queue=False,
+    ).then(
+        fn=generate_embed,
+        inputs=[project_dd, metric_filter_tb, run_selection_state],
+        outputs=embed_code,
+        show_progress="hidden",
+        api_name=False,
+        queue=False,
+    )
+    run_tb.input(
+        fn=refresh_runs,
+        inputs=[project_dd, run_tb, run_selection_state],
+        outputs=[run_cb, run_tb, run_selection_state],
+        api_name=False,
+        queue=False,
+        show_progress="hidden",
+    )
+
+    gr.api(
+        fn=upload_db_to_space,
+        api_name="upload_db_to_space",
+    )
+    gr.api(
+        fn=bulk_upload_media,
+        api_name="bulk_upload_media",
+    )
+    gr.api(
+        fn=log,
+        api_name="log",
+    )
+    gr.api(
+        fn=bulk_log,
+        api_name="bulk_log",
+    )
+    gr.api(
+        fn=get_metric_values,
+        api_name="get_metric_values",
+    )
+    gr.api(
+        fn=get_runs_for_project,
+        api_name="get_runs_for_project",
+    )
+    gr.api(
+        fn=get_metrics_for_run,
+        api_name="get_metrics_for_run",
+    )
+    gr.api(
+        fn=get_all_projects,
+        api_name="get_all_projects",
+    )
+    gr.api(
+        fn=get_project_summary,
+        api_name="get_project_summary",
+    )
+    gr.api(
+        fn=get_run_summary,
+        api_name="get_run_summary",
+    )
+
+    x_lim = gr.State(None)
+    last_steps = gr.State({})
+
+    def update_x_lim(select_data: gr.SelectData):
+        return select_data.index
+
+    def update_last_steps(project):
+        """Check the last step for each run to detect when new data is available."""
+        if not project:
+            return {}
+        return SQLiteStorage.get_max_steps_for_runs(project)
+
+    timer.tick(
+        fn=update_last_steps,
+        inputs=[project_dd],
+        outputs=last_steps,
+        show_progress="hidden",
+        api_name=False,
+    )
+
+    @gr.render(
+        triggers=[
+            demo.load,
+            run_cb.change,
+            last_steps.change,
+            smoothing_slider.change,
+            x_lim.change,
+            x_axis_dd.change,
+            log_scale_cb.change,
+            metric_filter_tb.change,
+        ],
+        inputs=[
+            project_dd,
+            run_cb,
+            smoothing_slider,
+            metrics_subset,
+            x_lim,
+            x_axis_dd,
+            log_scale_cb,
+            metric_filter_tb,
+        ],
+        show_progress="hidden",
+        queue=False,
+    )
+    def update_dashboard(
+        project,
+        runs,
+        smoothing_granularity,
+        metrics_subset,
+        x_lim_value,
+        x_axis,
+        log_scale,
+        metric_filter,
+    ):
+        dfs = []
+        images_by_run = {}
+        original_runs = runs.copy()
+
+        for run in runs:
+            df, images_by_key = load_run_data(
+                project, run, smoothing_granularity, x_axis, log_scale
+            )
+            if df is not None:
+                dfs.append(df)
+                images_by_run[run] = images_by_key
+
+        if dfs:
+            if smoothing_granularity > 0:
+                original_dfs = []
+                smoothed_dfs = []
+                for df in dfs:
+                    original_data = df[df["data_type"] == "original"]
+                    smoothed_data = df[df["data_type"] == "smoothed"]
+                    if not original_data.empty:
+                        original_dfs.append(original_data)
+                    if not smoothed_data.empty:
+                        smoothed_dfs.append(smoothed_data)
+
+                all_dfs = original_dfs + smoothed_dfs
+                master_df = (
+                    pd.concat(all_dfs, ignore_index=True) if all_dfs else pd.DataFrame()
+                )
+
+            else:
+                master_df = pd.concat(dfs, ignore_index=True)
+        else:
+            master_df = pd.DataFrame()
+
+        if master_df.empty:
+            if not SQLiteStorage.get_projects():
+                if space_id := utils.get_space():
+                    gr.Markdown(INSTRUCTIONS_SPACES.format(space_id))
+                else:
+                    gr.Markdown(INSTRUCTIONS_LOCAL)
+            else:
+                gr.Markdown("*Waiting for runs to appear...*")
+            return
+
+        x_column = "step"
+        if dfs and not dfs[0].empty and "x_axis" in dfs[0].columns:
+            x_column = dfs[0]["x_axis"].iloc[0]
+
+        numeric_cols = master_df.select_dtypes(include="number").columns
+        numeric_cols = [c for c in numeric_cols if c not in utils.RESERVED_KEYS]
+        if x_column and x_column in numeric_cols:
+            numeric_cols.remove(x_column)
+
+        if metrics_subset:
+            numeric_cols = [c for c in numeric_cols if c in metrics_subset]
+
+        if metric_filter and metric_filter.strip():
+            numeric_cols = filter_metrics_by_regex(list(numeric_cols), metric_filter)
+
+        ordered_groups, nested_metric_groups = utils.order_metrics_by_plot_preference(
+            list(numeric_cols)
+        )
+        color_map = utils.get_color_mapping(original_runs, smoothing_granularity > 0)
+
+        metric_idx = 0
+        for group_name in ordered_groups:
+            group_data = nested_metric_groups[group_name]
+
+            total_plot_count = sum(
+                1
+                for m in group_data["direct_metrics"]
+                if not master_df.dropna(subset=[m]).empty
+            ) + sum(
+                sum(1 for m in metrics if not master_df.dropna(subset=[m]).empty)
+                for metrics in group_data["subgroups"].values()
+            )
+            group_label = (
+                f"{group_name} ({total_plot_count})"
+                if total_plot_count > 0
+                else group_name
+            )
+
+            with gr.Accordion(
+                label=group_label,
+                open=True,
+                key=f"accordion-{group_name}",
+                preserved_by_key=["value", "open"],
+            ):
+                if group_data["direct_metrics"]:
+                    with gr.Draggable(
+                        key=f"row-{group_name}-direct", orientation="row"
+                    ):
+                        for metric_name in group_data["direct_metrics"]:
+                            metric_df = master_df.dropna(subset=[metric_name])
+                            color = "run" if "run" in metric_df.columns else None
+                            if not metric_df.empty:
+                                plot = gr.LinePlot(
+                                    utils.downsample(
+                                        metric_df,
+                                        x_column,
+                                        metric_name,
+                                        color,
+                                        x_lim_value,
+                                    ),
+                                    x=x_column,
+                                    y=metric_name,
+                                    y_title=metric_name.split("/")[-1],
+                                    color=color,
+                                    color_map=color_map,
+                                    title=metric_name,
+                                    key=f"plot-{metric_idx}",
+                                    preserved_by_key=None,
+                                    x_lim=x_lim_value,
+                                    show_fullscreen_button=True,
+                                    min_width=400,
+                                    show_export_button=True,
+                                )
+                                plot.select(
+                                    update_x_lim,
+                                    outputs=x_lim,
+                                    key=f"select-{metric_idx}",
+                                )
+                                plot.double_click(
+                                    lambda: None,
+                                    outputs=x_lim,
+                                    key=f"double-{metric_idx}",
+                                )
+                            metric_idx += 1
+
+                if group_data["subgroups"]:
+                    for subgroup_name in sorted(group_data["subgroups"].keys()):
+                        subgroup_metrics = group_data["subgroups"][subgroup_name]
+
+                        subgroup_plot_count = sum(
+                            1
+                            for m in subgroup_metrics
+                            if not master_df.dropna(subset=[m]).empty
+                        )
+                        subgroup_label = (
+                            f"{subgroup_name} ({subgroup_plot_count})"
+                            if subgroup_plot_count > 0
+                            else subgroup_name
+                        )
+
+                        with gr.Accordion(
+                            label=subgroup_label,
+                            open=True,
+                            key=f"accordion-{group_name}-{subgroup_name}",
+                            preserved_by_key=["value", "open"],
+                        ):
+                            with gr.Draggable(
+                                key=f"row-{group_name}-{subgroup_name}",
+                                orientation="row",
+                            ):
+                                for metric_name in subgroup_metrics:
+                                    metric_df = master_df.dropna(subset=[metric_name])
+                                    color = (
+                                        "run" if "run" in metric_df.columns else None
+                                    )
+                                    if not metric_df.empty:
+                                        plot = gr.LinePlot(
+                                            utils.downsample(
+                                                metric_df,
+                                                x_column,
+                                                metric_name,
+                                                color,
+                                                x_lim_value,
+                                            ),
+                                            x=x_column,
+                                            y=metric_name,
+                                            y_title=metric_name.split("/")[-1],
+                                            color=color,
+                                            color_map=color_map,
+                                            title=metric_name,
+                                            key=f"plot-{metric_idx}",
+                                            preserved_by_key=None,
+                                            x_lim=x_lim_value,
+                                            show_fullscreen_button=True,
+                                            min_width=400,
+                                            show_export_button=True,
+                                        )
+                                        plot.select(
+                                            update_x_lim,
+                                            outputs=x_lim,
+                                            key=f"select-{metric_idx}",
+                                        )
+                                        plot.double_click(
+                                            lambda: None,
+                                            outputs=x_lim,
+                                            key=f"double-{metric_idx}",
+                                        )
+                                    metric_idx += 1
+        if images_by_run and any(any(images) for images in images_by_run.values()):
+            create_media_section(images_by_run)
+
+        table_cols = master_df.select_dtypes(include="object").columns
+        table_cols = [c for c in table_cols if c not in utils.RESERVED_KEYS]
+        if metrics_subset:
+            table_cols = [c for c in table_cols if c in metrics_subset]
+        if metric_filter and metric_filter.strip():
+            table_cols = filter_metrics_by_regex(list(table_cols), metric_filter)
+
+        actual_table_count = sum(
+            1
+            for metric_name in table_cols
+            if not (metric_df := master_df.dropna(subset=[metric_name])).empty
+            and isinstance(value := metric_df[metric_name].iloc[-1], dict)
+            and value.get("_type") == Table.TYPE
+        )
+
+        if actual_table_count > 0:
+            with gr.Accordion(f"tables ({actual_table_count})", open=True):
+                with gr.Row(key="row"):
+                    for metric_idx, metric_name in enumerate(table_cols):
+                        metric_df = master_df.dropna(subset=[metric_name])
+                        if not metric_df.empty:
+                            value = metric_df[metric_name].iloc[-1]
+                            if (
+                                isinstance(value, dict)
+                                and "_type" in value
+                                and value["_type"] == Table.TYPE
+                            ):
+                                try:
+                                    df = pd.DataFrame(value["_value"])
+                                    gr.DataFrame(
+                                        df,
+                                        label=f"{metric_name} (latest)",
+                                        key=f"table-{metric_idx}",
+                                        wrap=True,
+                                    )
+                                except Exception as e:
+                                    gr.Warning(
+                                        f"Column {metric_name} failed to render as a table: {e}"
+                                    )
+
+    with grouped_runs_panel:
+
+        @gr.render(
+            triggers=[
+                demo.load,
+                project_dd.change,
+                run_group_by_dd.change,
+                run_tb.input,
+                run_selection_state.change,
+            ],
+            inputs=[project_dd, run_group_by_dd, run_tb, run_selection_state],
+            show_progress="hidden",
+            queue=False,
+        )
+        def render_grouped_runs(project, group_key, filter_text, selection):
+            if not group_key:
+                return
+            selection = selection or RunSelection()
+            groups = fns.group_runs_by_config(project, group_key, filter_text)
+
+            for label, runs in groups.items():
+                ordered_current = utils.ordered_subset(runs, selection.selected)
+
+                with gr.Group():
+                    show_group_cb = gr.Checkbox(
+                        label="Show/Hide",
+                        value=bool(ordered_current),
+                        key=f"show-cb-{group_key}-{label}",
+                        preserved_by_key=["value"],
+                    )
+
+                    with gr.Accordion(
+                        f"{label} ({len(runs)})",
+                        open=False,
+                        key=f"accordion-{group_key}-{label}",
+                        preserved_by_key=["open"],
+                    ):
+                        group_cb = gr.CheckboxGroup(
+                            choices=runs,
+                            value=ordered_current,
+                            show_label=False,
+                            key=f"group-cb-{group_key}-{label}",
+                        )
+
+                        gr.on(
+                            [group_cb.change],
+                            fn=fns.handle_group_checkbox_change,
+                            inputs=[
+                                group_cb,
+                                run_selection_state,
+                                gr.State(runs),
+                            ],
+                            outputs=[
+                                run_selection_state,
+                                group_cb,
+                                run_cb,
+                            ],
+                            show_progress="hidden",
+                            api_name=False,
+                            queue=False,
+                        )
+
+                        gr.on(
+                            [show_group_cb.change],
+                            fn=fns.handle_group_toggle,
+                            inputs=[
+                                show_group_cb,
+                                run_selection_state,
+                                gr.State(runs),
+                            ],
+                            outputs=[run_selection_state, group_cb, run_cb],
+                            show_progress="hidden",
+                            api_name=False,
+                            queue=False,
+                        )
+
+
+with demo.route("Runs", show_in_navbar=False):
+    run_page.render()
+with demo.route("Run", show_in_navbar=False):
+    run_detail_page.render()
+
+write_token = secrets.token_urlsafe(32)
+demo.write_token = write_token
+run_page.write_token = write_token
+run_detail_page.write_token = write_token
+
+if __name__ == "__main__":
+    demo.launch(allowed_paths=[utils.TRACKIO_LOGO_DIR], show_api=False, show_error=True)

ui/run_detail.py

@@ -0,0 +1,94 @@
+"""The Runs page for the Trackio UI."""
+
+import gradio as gr
+
+try:
+    import trackio.utils as utils
+    from trackio.sqlite_storage import SQLiteStorage
+    from trackio.ui import fns
+except ImportError:
+    import utils
+    from sqlite_storage import SQLiteStorage
+    from ui import fns
+
+RUN_DETAILS_TEMPLATE = """
+## Run Details
+* **Run Name:** `{run_name}`
+* **Group:** `{group}`
+* **Created:** {created} by {username}
+"""
+
+with gr.Blocks() as run_detail_page:
+    with gr.Sidebar() as sidebar:
+        logo_urls = utils.get_logo_urls()
+        logo = gr.Markdown(
+            f"""
+                <img src='{logo_urls["light"]}' width='80%' class='logo-light'>
+                <img src='{logo_urls["dark"]}' width='80%' class='logo-dark'>            
+            """
+        )
+        project_dd = gr.Dropdown(
+            label="Project", allow_custom_value=True, interactive=False
+        )
+        run_dd = gr.Dropdown(label="Run")
+
+    navbar = gr.Navbar(value=[("Metrics", ""), ("Runs", "/runs")], main_page_name=False)
+
+    run_details = gr.Markdown(RUN_DETAILS_TEMPLATE)
+
+    run_config = gr.JSON(label="Run Config")
+
+    def configure(request: gr.Request):
+        project = request.query_params.get("selected_project")
+        run = request.query_params.get("selected_run")
+        runs = SQLiteStorage.get_runs(project)
+        return project, gr.Dropdown(choices=runs, value=run)
+
+    def update_run_details(project, run):
+        config = SQLiteStorage.get_run_config(project, run)
+        if not config:
+            return gr.Markdown("No run details available"), {}
+
+        group = config.get("_Group", "None")
+
+        created = config.get("_Created", "Unknown")
+        if created != "Unknown":
+            created = utils.format_timestamp(created)
+
+        username = config.get("_Username", "Unknown")
+        if username and username != "None" and username != "Unknown":
+            username = f"[{username}](https://huggingface.co/{username})"
+
+        details_md = RUN_DETAILS_TEMPLATE.format(
+            run_name=run, group=group, created=created, username=username
+        )
+
+        config_display = {k: v for k, v in config.items() if not k.startswith("_")}
+
+        return gr.Markdown(details_md), config_display
+
+    gr.on(
+        [run_detail_page.load],
+        fn=configure,
+        outputs=[project_dd, run_dd],
+        show_progress="hidden",
+        queue=False,
+        api_name=False,
+    ).then(
+        fns.update_navbar_value,
+        inputs=[project_dd],
+        outputs=[navbar],
+        show_progress="hidden",
+        api_name=False,
+        queue=False,
+    )
+
+    gr.on(
+        [run_dd.change],
+        update_run_details,
+        inputs=[project_dd, run_dd],
+        outputs=[run_details, run_config],
+        show_progress="hidden",
+        api_name=False,
+        queue=False,
+    )