Initial commit after history reset

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+scale-hf-logo.png filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,13 @@
+auto_evals/
+venv/
+__pycache__/
+.env
+.ipynb_checkpoints
+*ipynb
+.vscode/
+
+eval-queue/
+eval-results/
+eval-queue-bk/
+eval-results-bk/
+logs/

.pre-commit-config.yaml

@@ -0,0 +1,53 @@
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+default_language_version:
+  python: python3
+
+ci:
+  autofix_prs: true
+  autoupdate_commit_msg: '[pre-commit.ci] pre-commit suggestions'
+  autoupdate_schedule: quarterly
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.3.0
+    hooks:
+      - id: check-yaml
+      - id: check-case-conflict
+      - id: detect-private-key
+      - id: check-added-large-files
+        args: ['--maxkb=1000']
+      - id: requirements-txt-fixer
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+
+  - repo: https://github.com/PyCQA/isort
+    rev: 5.12.0
+    hooks:
+      - id: isort
+        name: Format imports
+
+  - repo: https://github.com/psf/black
+    rev: 22.12.0
+    hooks:
+      - id: black
+        name: Format code
+        additional_dependencies: ['click==8.0.2']
+
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    # Ruff version.
+    rev: 'v0.0.267'
+    hooks:
+      - id: ruff

Makefile

@@ -0,0 +1,13 @@
+.PHONY: style format
+
+
+style:
+	python -m black --line-length 119 .
+	python -m isort .
+	ruff check --fix .
+
+
+quality:
+	python -m black --check --line-length 119 .
+	python -m isort --check-only .
+	ruff check .

README.md

@@ -0,0 +1,55 @@
+---
+title: ArchEGraph Visualizer
+colorFrom: blue
+colorTo: green
+sdk: gradio
+app_file: app.py
+pinned: false
+---
+
+# ArchEGraph Visualizer
+
+This Space visualizes sample files from:
+
+- https://huggingface.co/datasets/ArchEGraph/ArchEGraph-demo
+
+The UI flow is:
+
+- Select weather city first
+- Select a matched building id for that city
+- Optionally enable a custom time window for weather and energy curves
+
+For the selected weather/building pair, the app fetches:
+
+- `geometry/<building_id>.npz`
+- `building/<building_id>.npz`
+- `weather/<weather_id>.npz`
+- `energy/<sample_id>.npz`
+
+Then it renders four views:
+
+- Geometry polygons
+- Building graph
+- Weather curves (selected time window)
+- Energy curves (selected time window)
+
+## API
+
+The Gradio API endpoint is exposed as:
+
+- `/gradio_api/call/render_sample`
+
+Input data format:
+
+```json
+{
+  "weather_id": "Anchorage",
+  "building_id": "17",
+  "max_energy_zones": 6,
+  "use_custom_window": false,
+  "window_start_hour": 1,
+  "window_hours": 24
+}
+```
+
+See `agents.md` for curl examples.

app.py

@@ -0,0 +1,401 @@
+from __future__ import annotations
+
+import csv
+import random
+import time
+from dataclasses import dataclass
+from functools import lru_cache
+from pathlib import Path
+
+import gradio as gr
+from huggingface_hub import hf_hub_download
+
+from visualization.building import visualize_graph, visualize_graph_overlap
+from visualization.energy import visualize_energy
+from visualization.geometry import visualize_geometry
+from visualization.weather import visualize_weather
+
+DATASET_REPO_ID = "ArchEGraph/ArchEGraph-demo"
+OUTPUT_ROOT = Path("/tmp/archegraph_outputs")
+DEFAULT_ENERGY_ZONE_INDEX = 0
+DEFAULT_WEATHER_WINDOW_START_HOUR = 1
+DEFAULT_WEATHER_WINDOW_HOURS = 24
+
+
+@dataclass(frozen=True)
+class SampleRecord:
+    sample_id: str
+    weather_id: str
+    building_id: str
+    energy_file: str
+    n_steps: int
+    n_spaces: int
+
+
+@lru_cache(maxsize=1)
+def _manifest_index() -> dict[str, SampleRecord]:
+    manifest_path = hf_hub_download(
+        repo_id=DATASET_REPO_ID,
+        repo_type="dataset",
+        filename="manifest.csv",
+    )
+
+    out: dict[str, SampleRecord] = {}
+    with Path(manifest_path).open("r", encoding="utf-8", newline="") as f:
+        reader = csv.DictReader(f)
+        for row in reader:
+            sample_id = (row.get("sample_id") or "").strip()
+            weather_id = (row.get("weather_id") or "").strip()
+            building_id = (row.get("building_id") or "").strip()
+            energy_file = (row.get("energy_file") or "").strip()
+            if not sample_id or not weather_id or not energy_file:
+                continue
+
+            if not building_id:
+                building_id = sample_id.split("__", 1)[0].lstrip("0") or "0"
+
+            n_steps = int(row.get("n_steps") or 0)
+            n_spaces = int(row.get("n_spaces") or 0)
+            out[sample_id] = SampleRecord(
+                sample_id=sample_id,
+                weather_id=weather_id,
+                building_id=building_id,
+                energy_file=energy_file,
+                n_steps=n_steps,
+                n_spaces=n_spaces,
+            )
+
+    if not out:
+        raise RuntimeError("manifest.csv is empty or invalid.")
+
+    return out
+
+
+def _numeric_sort_key(value: str) -> tuple[int, str]:
+    text = (value or "").strip()
+    if text.isdigit():
+        return (0, f"{int(text):09d}")
+    return (1, text.lower())
+
+
+@lru_cache(maxsize=1)
+def _weather_to_buildings() -> dict[str, list[str]]:
+    mapping: dict[str, set[str]] = {}
+    for rec in _manifest_index().values():
+        mapping.setdefault(rec.weather_id, set()).add(rec.building_id)
+
+    return {k: sorted(v, key=_numeric_sort_key) for k, v in mapping.items()}
+
+
+@lru_cache(maxsize=1)
+def _pair_to_sample() -> dict[tuple[str, str], SampleRecord]:
+    out: dict[tuple[str, str], SampleRecord] = {}
+    for rec in _manifest_index().values():
+        out[(rec.weather_id, rec.building_id)] = rec
+    return out
+
+
+def _weather_choices() -> list[str]:
+    return sorted(_weather_to_buildings().keys(), key=lambda x: x.lower())
+
+
+def _building_choices(weather_id: str) -> list[str]:
+    return _weather_to_buildings().get((weather_id or "").strip(), [])
+
+
+def _default_selection() -> tuple[str, str]:
+    weather_choices = _weather_choices()
+    if not weather_choices:
+        raise RuntimeError("No weather options found in manifest")
+
+    weather = weather_choices[0]
+    buildings = _building_choices(weather)
+    if not buildings:
+        raise RuntimeError(f"No building options for weather '{weather}'")
+    return weather, buildings[0]
+
+
+def _safe_dropdown_defaults() -> tuple[list[str], str | None, list[str], str | None]:
+    try:
+        weather_choices = _weather_choices()
+        if not weather_choices:
+            return [], None, [], None
+        weather, building = _default_selection()
+        building_choices = _building_choices(weather)
+        return weather_choices, weather, building_choices, building
+    except Exception:
+        return [], None, [], None
+
+
+def _resolve_record(weather_id: str, building_id: str) -> SampleRecord:
+    weather = (weather_id or "").strip()
+    building = (building_id or "").strip()
+    pair_map = _pair_to_sample()
+    rec = pair_map.get((weather, building))
+    if rec is not None:
+        return rec
+
+    weather_opts = _weather_choices()
+    if not weather_opts:
+        raise ValueError("No weather options available")
+    if weather not in weather_opts:
+        raise ValueError(f"Unknown weather city '{weather}'. Example values: {', '.join(weather_opts[:8])}")
+
+    building_opts = _building_choices(weather)
+    raise ValueError(
+        f"Unknown building id '{building}' for city '{weather}'. "
+        f"Available building ids: {', '.join(building_opts[:12])}"
+    )
+
+
+def _download_modalities(record: SampleRecord) -> tuple[Path, Path, Path, Path]:
+    building_key = record.sample_id.split("__", 1)[0]
+
+    geometry_npz = hf_hub_download(
+        repo_id=DATASET_REPO_ID,
+        repo_type="dataset",
+        filename=f"geometry/{building_key}.npz",
+    )
+    graph_npz = hf_hub_download(
+        repo_id=DATASET_REPO_ID,
+        repo_type="dataset",
+        filename=f"building/{building_key}.npz",
+    )
+    weather_npz = hf_hub_download(
+        repo_id=DATASET_REPO_ID,
+        repo_type="dataset",
+        filename=f"weather/{record.weather_id}.npz",
+    )
+    energy_npz = hf_hub_download(
+        repo_id=DATASET_REPO_ID,
+        repo_type="dataset",
+        filename=f"energy/{record.energy_file}",
+    )
+
+    return Path(geometry_npz), Path(graph_npz), Path(weather_npz), Path(energy_npz)
+
+
+def _output_paths(sample_id: str) -> tuple[Path, Path, Path, Path, Path]:
+    OUTPUT_ROOT.mkdir(parents=True, exist_ok=True)
+    safe = sample_id.replace("/", "_").replace("\\", "_")
+    run_dir = OUTPUT_ROOT / f"{safe}_{int(time.time() * 1000)}"
+    run_dir.mkdir(parents=True, exist_ok=True)
+
+    return (
+        run_dir / "geometry.png",
+        run_dir / "graph.png",
+        run_dir / "overlap.png",
+        run_dir / "weather.png",
+        run_dir / "energy.png",
+    )
+
+
+def update_building_dropdown(weather_id: str) -> gr.update:
+    buildings = _building_choices(weather_id)
+    value = buildings[0] if buildings else None
+    return gr.update(choices=buildings, value=value)
+
+
+def render_sample(
+    weather_id: str,
+    building_id: str,
+    energy_zone_index: int,
+    use_custom_window: bool,
+    window_start_hour: int,
+    window_hours: int,
+) -> tuple[str, str, str, str, str, str]:
+    record = _resolve_record(weather_id=weather_id, building_id=building_id)
+
+    if use_custom_window:
+        start_hour = max(1, int(window_start_hour))
+        hours = max(1, int(window_hours))
+    else:
+        start_hour = DEFAULT_WEATHER_WINDOW_START_HOUR
+        hours = DEFAULT_WEATHER_WINDOW_HOURS
+
+    zone_idx = max(0, int(energy_zone_index)) if energy_zone_index is not None else DEFAULT_ENERGY_ZONE_INDEX
+
+    try:
+        geometry_npz, graph_npz, weather_npz, energy_npz = _download_modalities(record)
+        out_geometry, out_graph, out_overlap, out_weather, out_energy = _output_paths(record.sample_id)
+
+        visualize_geometry(geometry_npz=geometry_npz, output_png=out_geometry)
+        visualize_graph(graph_npz=graph_npz, geometry_npz=geometry_npz, output_png=out_graph)
+        visualize_graph_overlap(graph_npz=graph_npz, geometry_npz=geometry_npz, energy_npz=energy_npz, output_png=out_overlap)
+        visualize_weather(
+            weather_npz=weather_npz,
+            output_png=out_weather,
+            start_hour=start_hour,
+            window_hours=hours,
+        )
+        visualize_energy(
+            energy_npz=energy_npz,
+            output_png=out_energy,
+            zone_index=zone_idx,
+            start_hour=start_hour,
+            window_hours=hours,
+        )
+
+    except Exception as exc:
+        raise gr.Error(f"Failed to render sample {record.sample_id}: {exc}") from exc
+
+    if use_custom_window:
+        window_text = f"custom window: start={start_hour}, hours={hours}"
+    else:
+        window_text = "default window: Jan-1 first 24 hours"
+
+    summary = (
+        f"Rendered **{record.sample_id}** from `{DATASET_REPO_ID}`  \n"
+        f"weather_id: `{record.weather_id}`  \n"
+        f"building_id: `{record.building_id}`  \n"
+        f"n_steps: `{record.n_steps}`  \n"
+        f"n_spaces: `{record.n_spaces}`  \n"
+        f"zone_index: `{zone_idx}`  \n"
+        f"{window_text}"
+    )
+
+    return str(out_geometry), str(out_graph), str(out_overlap), str(out_weather), str(out_energy), summary
+
+
+def pick_random_sample() -> tuple[str, gr.update]:
+    rec = random.choice(list(_manifest_index().values()))
+    choices = _building_choices(rec.weather_id)
+    return rec.weather_id, gr.update(choices=choices, value=rec.building_id)
+
+
+def _startup_note() -> str:
+    try:
+        total = len(_manifest_index())
+        weather_count = len(_weather_choices())
+        return f"Manifest loaded: {total} samples, {weather_count} weather cities from {DATASET_REPO_ID}."
+    except Exception as exc:
+        return f"Manifest will be loaded lazily on first run. Reason: {exc}"
+
+
+default_weathers, default_weather, default_buildings, default_building = _safe_dropdown_defaults()
+
+
+APP_CSS = """
+.gradio-container {
+    max-width: 1920px !important;
+}
+#ctrl-row-1, #ctrl-row-2, #ctrl-row-3 {
+    gap: 8px !important;
+}
+#ctrl-row-1 .gr-block, #ctrl-row-2 .gr-block, #ctrl-row-3 .gr-block {
+    padding-top: 4px !important;
+    padding-bottom: 4px !important;
+}
+#status-box {
+    margin-top: 2px !important;
+    margin-bottom: 4px !important;
+}
+#viz-row-all {
+    gap: 8px !important;
+}
+#viz-row-all img {
+    object-fit: contain !important;
+}
+"""
+
+
+with gr.Blocks(title="ArchEGraph Visualizer", css=APP_CSS) as demo:
+    gr.Markdown(
+        "# ArchEGraph Visualizer\n"
+        "Visualize geometry, graph, weather and energy files from "
+        "[ArchEGraph/ArchEGraph-demo](https://huggingface.co/datasets/ArchEGraph/ArchEGraph-demo)."
+    )
+    gr.Markdown(_startup_note())
+
+    with gr.Row(elem_id="ctrl-row-1"):
+        weather_dropdown = gr.Dropdown(
+            label="Weather City",
+            choices=default_weathers,
+            value=default_weather,
+            allow_custom_value=False,
+            scale=2,
+        )
+        building_dropdown = gr.Dropdown(
+            label="Building ID",
+            choices=default_buildings,
+            value=default_building,
+            allow_custom_value=True,
+            scale=2,
+        )
+        energy_zone_index_input = gr.Slider(
+            label="Energy Zone Index",
+            minimum=0,
+            maximum=24,
+            step=1,
+            value=DEFAULT_ENERGY_ZONE_INDEX,
+            scale=1,
+        )
+
+    with gr.Row(elem_id="ctrl-row-2"):
+        use_custom_window_input = gr.Checkbox(
+            label="Custom Window",
+            value=False,
+            scale=1,
+        )
+        window_start_hour_input = gr.Slider(
+            label="Start Hour",
+            minimum=1,
+            maximum=8760,
+            step=1,
+            value=DEFAULT_WEATHER_WINDOW_START_HOUR,
+            scale=2,
+        )
+        window_hours_input = gr.Slider(
+            label="Window Hours",
+            minimum=1,
+            maximum=8760,
+            step=1,
+            value=DEFAULT_WEATHER_WINDOW_HOURS,
+            scale=2,
+        )
+
+    with gr.Row(elem_id="ctrl-row-3"):
+        run_btn = gr.Button("Visualize", variant="primary")
+        random_btn = gr.Button("Pick Random Sample")
+
+    status_md = gr.Markdown(elem_id="status-box")
+
+    with gr.Row(elem_id="viz-row-all"):
+        geometry_img = gr.Image(label="Geometry", type="filepath", height=215)
+        graph_img = gr.Image(label="Building", type="filepath", height=215)
+        overlap_img = gr.Image(label="Overlap", type="filepath", height=215)
+        weather_img = gr.Image(label="Weather (line)", type="filepath", height=215)
+        energy_img = gr.Image(label="Energy (selected zone)", type="filepath", height=215)
+
+    weather_dropdown.change(
+        fn=update_building_dropdown,
+        inputs=[weather_dropdown],
+        outputs=[building_dropdown],
+    )
+
+    run_btn.click(
+        fn=render_sample,
+        inputs=[
+            weather_dropdown,
+            building_dropdown,
+            energy_zone_index_input,
+            use_custom_window_input,
+            window_start_hour_input,
+            window_hours_input,
+        ],
+        outputs=[geometry_img, graph_img, overlap_img, weather_img, energy_img, status_md],
+        api_name="render_sample",
+    )
+
+    random_btn.click(
+        fn=pick_random_sample,
+        inputs=None,
+        outputs=[weather_dropdown, building_dropdown],
+        api_name="pick_random_sample",
+    )
+
+
+demo.queue()
+
+if __name__ == "__main__":
+    demo.launch()

pyproject.toml

@@ -0,0 +1,13 @@
+[tool.ruff]
+# Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default.
+select = ["E", "F"]
+ignore = ["E501"] # line too long (black is taking care of this)
+line-length = 119
+fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"]
+
+[tool.isort]
+profile = "black"
+line_length = 119
+
+[tool.black]
+line-length = 119

requirements.txt

@@ -0,0 +1,4 @@
+gradio>=5.0.0
+huggingface_hub>=0.26.0
+matplotlib>=3.8.0
+numpy>=1.26.0

src/about.py

@@ -0,0 +1,72 @@
+from dataclasses import dataclass
+from enum import Enum
+
+@dataclass
+class Task:
+    benchmark: str
+    metric: str
+    col_name: str
+
+
+# Select your tasks here
+# ---------------------------------------------------
+class Tasks(Enum):
+    # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
+    task0 = Task("anli_r1", "acc", "ANLI")
+    task1 = Task("logiqa", "acc_norm", "LogiQA")
+
+NUM_FEWSHOT = 0 # Change with your few shot
+# ---------------------------------------------------
+
+
+
+# Your leaderboard name
+TITLE = """<h1 align="center" id="space-title">Demo leaderboard</h1>"""
+
+# What does your leaderboard evaluate?
+INTRODUCTION_TEXT = """
+Intro text
+"""
+
+# Which evaluations are you running? how can people reproduce what you have?
+LLM_BENCHMARKS_TEXT = f"""
+## How it works
+
+## Reproducibility
+To reproduce our results, here is the commands you can run:
+
+"""
+
+EVALUATION_QUEUE_TEXT = """
+## Some good practices before submitting a model
+
+### 1) Make sure you can load your model and tokenizer using AutoClasses:
+```python
+from transformers import AutoConfig, AutoModel, AutoTokenizer
+config = AutoConfig.from_pretrained("your model name", revision=revision)
+model = AutoModel.from_pretrained("your model name", revision=revision)
+tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision)
+```
+If this step fails, follow the error messages to debug your model before submitting it. It's likely your model has been improperly uploaded.
+
+Note: make sure your model is public!
+Note: if your model needs `use_remote_code=True`, we do not support this option yet but we are working on adding it, stay posted!
+
+### 2) Convert your model weights to [safetensors](https://huggingface.co/docs/safetensors/index)
+It's a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of parameters of your model to the `Extended Viewer`!
+
+### 3) Make sure your model has an open license!
+This is a leaderboard for Open LLMs, and we'd love for as many people as possible to know they can use your model 🤗
+
+### 4) Fill up your model card
+When we add extra information about models to the leaderboard, it will be automatically taken from the model card
+
+## In case of model failure
+If your model is displayed in the `FAILED` category, its execution stopped.
+Make sure you have followed the above steps first.
+If everything is done, check you can launch the EleutherAIHarness on your model locally, using the above command without modifications (you can add `--limit` to limit the number of examples per task).
+"""
+
+CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
+CITATION_BUTTON_TEXT = r"""
+"""

src/display/css_html_js.py

@@ -0,0 +1,105 @@
+custom_css = """
+
+.markdown-text {
+    font-size: 16px !important;
+}
+
+#models-to-add-text {
+    font-size: 18px !important;
+}
+
+#citation-button span {
+    font-size: 16px !important;
+}
+
+#citation-button textarea {
+    font-size: 16px !important;
+}
+
+#citation-button > label > button {
+    margin: 6px;
+    transform: scale(1.3);
+}
+
+#leaderboard-table {
+    margin-top: 15px
+}
+
+#leaderboard-table-lite {
+    margin-top: 15px
+}
+
+#search-bar-table-box > div:first-child {
+    background: none;
+    border: none;
+}
+ 
+#search-bar {
+    padding: 0px;
+}
+
+/* Limit the width of the first AutoEvalColumn so that names don't expand too much */
+#leaderboard-table td:nth-child(2),
+#leaderboard-table th:nth-child(2) {
+    max-width: 400px;
+    overflow: auto;
+    white-space: nowrap;
+}
+
+.tab-buttons button {
+    font-size: 20px;
+}
+
+#scale-logo {
+    border-style: none !important;
+    box-shadow: none;
+    display: block;
+    margin-left: auto;
+    margin-right: auto;
+    max-width: 600px;
+}
+
+#scale-logo .download {
+    display: none;
+}
+#filter_type{
+    border: 0;
+    padding-left: 0;
+    padding-top: 0;
+}
+#filter_type label {
+    display: flex;
+}
+#filter_type label > span{
+    margin-top: var(--spacing-lg);
+    margin-right: 0.5em;
+}
+#filter_type label > .wrap{
+    width: 103px;
+}
+#filter_type label > .wrap .wrap-inner{  
+    padding: 2px;
+}
+#filter_type label > .wrap .wrap-inner input{
+    width: 1px
+}
+#filter-columns-type{
+    border:0;
+    padding:0.5;
+}
+#filter-columns-size{
+    border:0;
+    padding:0.5;
+}
+#box-filter > .form{
+    border: 0
+}
+"""
+
+get_window_url_params = """
+    function(url_params) {
+        const params = new URLSearchParams(window.location.search);
+        url_params = Object.fromEntries(params);
+        return url_params;
+    }
+    """

src/display/formatting.py

@@ -0,0 +1,27 @@
+def model_hyperlink(link, model_name):
+    return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
+
+
+def make_clickable_model(model_name):
+    link = f"https://huggingface.co/{model_name}"
+    return model_hyperlink(link, model_name)
+
+
+def styled_error(error):
+    return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
+
+
+def styled_warning(warn):
+    return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
+
+
+def styled_message(message):
+    return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"
+
+
+def has_no_nan_values(df, columns):
+    return df[columns].notna().all(axis=1)
+
+
+def has_nan_values(df, columns):
+    return df[columns].isna().any(axis=1)

src/display/utils.py

@@ -0,0 +1,110 @@
+from dataclasses import dataclass, make_dataclass
+from enum import Enum
+
+import pandas as pd
+
+from src.about import Tasks
+
+def fields(raw_class):
+    return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
+
+
+# These classes are for user facing column names,
+# to avoid having to change them all around the code
+# when a modif is needed
+@dataclass
+class ColumnContent:
+    name: str
+    type: str
+    displayed_by_default: bool
+    hidden: bool = False
+    never_hidden: bool = False
+
+## Leaderboard columns
+auto_eval_column_dict = []
+# Init
+auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
+auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
+#Scores
+auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
+for task in Tasks:
+    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
+# Model information
+auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
+auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
+auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
+auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
+auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
+auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
+auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
+auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
+auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
+
+# We use make dataclass to dynamically fill the scores from Tasks
+AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
+
+## For the queue columns in the submission tab
+@dataclass(frozen=True)
+class EvalQueueColumn:  # Queue column
+    model = ColumnContent("model", "markdown", True)
+    revision = ColumnContent("revision", "str", True)
+    private = ColumnContent("private", "bool", True)
+    precision = ColumnContent("precision", "str", True)
+    weight_type = ColumnContent("weight_type", "str", "Original")
+    status = ColumnContent("status", "str", True)
+
+## All the model information that we might need
+@dataclass
+class ModelDetails:
+    name: str
+    display_name: str = ""
+    symbol: str = "" # emoji
+
+
+class ModelType(Enum):
+    PT = ModelDetails(name="pretrained", symbol="🟢")
+    FT = ModelDetails(name="fine-tuned", symbol="🔶")
+    IFT = ModelDetails(name="instruction-tuned", symbol="⭕")
+    RL = ModelDetails(name="RL-tuned", symbol="🟦")
+    Unknown = ModelDetails(name="", symbol="?")
+
+    def to_str(self, separator=" "):
+        return f"{self.value.symbol}{separator}{self.value.name}"
+
+    @staticmethod
+    def from_str(type):
+        if "fine-tuned" in type or "🔶" in type:
+            return ModelType.FT
+        if "pretrained" in type or "🟢" in type:
+            return ModelType.PT
+        if "RL-tuned" in type or "🟦" in type:
+            return ModelType.RL
+        if "instruction-tuned" in type or "⭕" in type:
+            return ModelType.IFT
+        return ModelType.Unknown
+
+class WeightType(Enum):
+    Adapter = ModelDetails("Adapter")
+    Original = ModelDetails("Original")
+    Delta = ModelDetails("Delta")
+
+class Precision(Enum):
+    float16 = ModelDetails("float16")
+    bfloat16 = ModelDetails("bfloat16")
+    Unknown = ModelDetails("?")
+
+    def from_str(precision):
+        if precision in ["torch.float16", "float16"]:
+            return Precision.float16
+        if precision in ["torch.bfloat16", "bfloat16"]:
+            return Precision.bfloat16
+        return Precision.Unknown
+
+# Column selection
+COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
+
+EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
+EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
+
+BENCHMARK_COLS = [t.value.col_name for t in Tasks]
+

src/envs.py

@@ -0,0 +1,25 @@
+import os
+
+from huggingface_hub import HfApi
+
+# Info to change for your repository
+# ----------------------------------
+TOKEN = os.environ.get("HF_TOKEN") # A read/write token for your org
+
+OWNER = "demo-leaderboard-backend" # Change to your org - don't forget to create a results and request dataset, with the correct format!
+# ----------------------------------
+
+REPO_ID = f"{OWNER}/leaderboard"
+QUEUE_REPO = f"{OWNER}/requests"
+RESULTS_REPO = f"{OWNER}/results"
+
+# If you setup a cache later, just change HF_HOME
+CACHE_PATH=os.getenv("HF_HOME", ".")
+
+# Local caches
+EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
+EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
+EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
+EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
+
+API = HfApi(token=TOKEN)

src/leaderboard/read_evals.py

@@ -0,0 +1,196 @@
+import glob
+import json
+import math
+import os
+from dataclasses import dataclass
+
+import dateutil
+import numpy as np
+
+from src.display.formatting import make_clickable_model
+from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType
+from src.submission.check_validity import is_model_on_hub
+
+
+@dataclass
+class EvalResult:
+    """Represents one full evaluation. Built from a combination of the result and request file for a given run.
+    """
+    eval_name: str # org_model_precision (uid)
+    full_model: str # org/model (path on hub)
+    org: str 
+    model: str
+    revision: str # commit hash, "" if main
+    results: dict
+    precision: Precision = Precision.Unknown
+    model_type: ModelType = ModelType.Unknown # Pretrained, fine tuned, ...
+    weight_type: WeightType = WeightType.Original # Original or Adapter
+    architecture: str = "Unknown" 
+    license: str = "?"
+    likes: int = 0
+    num_params: int = 0
+    date: str = "" # submission date of request file
+    still_on_hub: bool = False
+
+    @classmethod
+    def init_from_json_file(self, json_filepath):
+        """Inits the result from the specific model result file"""
+        with open(json_filepath) as fp:
+            data = json.load(fp)
+
+        config = data.get("config")
+
+        # Precision
+        precision = Precision.from_str(config.get("model_dtype"))
+
+        # Get model and org
+        org_and_model = config.get("model_name", config.get("model_args", None))
+        org_and_model = org_and_model.split("/", 1)
+
+        if len(org_and_model) == 1:
+            org = None
+            model = org_and_model[0]
+            result_key = f"{model}_{precision.value.name}"
+        else:
+            org = org_and_model[0]
+            model = org_and_model[1]
+            result_key = f"{org}_{model}_{precision.value.name}"
+        full_model = "/".join(org_and_model)
+
+        still_on_hub, _, model_config = is_model_on_hub(
+            full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
+        )
+        architecture = "?"
+        if model_config is not None:
+            architectures = getattr(model_config, "architectures", None)
+            if architectures:
+                architecture = ";".join(architectures)
+
+        # Extract results available in this file (some results are split in several files)
+        results = {}
+        for task in Tasks:
+            task = task.value
+
+            # We average all scores of a given metric (not all metrics are present in all files)
+            accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark == k])
+            if accs.size == 0 or any([acc is None for acc in accs]):
+                continue
+
+            mean_acc = np.mean(accs) * 100.0
+            results[task.benchmark] = mean_acc
+
+        return self(
+            eval_name=result_key,
+            full_model=full_model,
+            org=org,
+            model=model,
+            results=results,
+            precision=precision,  
+            revision= config.get("model_sha", ""),
+            still_on_hub=still_on_hub,
+            architecture=architecture
+        )
+
+    def update_with_request_file(self, requests_path):
+        """Finds the relevant request file for the current model and updates info with it"""
+        request_file = get_request_file_for_model(requests_path, self.full_model, self.precision.value.name)
+
+        try:
+            with open(request_file, "r") as f:
+                request = json.load(f)
+            self.model_type = ModelType.from_str(request.get("model_type", ""))
+            self.weight_type = WeightType[request.get("weight_type", "Original")]
+            self.license = request.get("license", "?")
+            self.likes = request.get("likes", 0)
+            self.num_params = request.get("params", 0)
+            self.date = request.get("submitted_time", "")
+        except Exception:
+            print(f"Could not find request file for {self.org}/{self.model} with precision {self.precision.value.name}")
+
+    def to_dict(self):
+        """Converts the Eval Result to a dict compatible with our dataframe display"""
+        average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
+        data_dict = {
+            "eval_name": self.eval_name,  # not a column, just a save name,
+            AutoEvalColumn.precision.name: self.precision.value.name,
+            AutoEvalColumn.model_type.name: self.model_type.value.name,
+            AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
+            AutoEvalColumn.weight_type.name: self.weight_type.value.name,
+            AutoEvalColumn.architecture.name: self.architecture,
+            AutoEvalColumn.model.name: make_clickable_model(self.full_model),
+            AutoEvalColumn.revision.name: self.revision,
+            AutoEvalColumn.average.name: average,
+            AutoEvalColumn.license.name: self.license,
+            AutoEvalColumn.likes.name: self.likes,
+            AutoEvalColumn.params.name: self.num_params,
+            AutoEvalColumn.still_on_hub.name: self.still_on_hub,
+        }
+
+        for task in Tasks:
+            data_dict[task.value.col_name] = self.results[task.value.benchmark]
+
+        return data_dict
+
+
+def get_request_file_for_model(requests_path, model_name, precision):
+    """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
+    request_files = os.path.join(
+        requests_path,
+        f"{model_name}_eval_request_*.json",
+    )
+    request_files = glob.glob(request_files)
+
+    # Select correct request file (precision)
+    request_file = ""
+    request_files = sorted(request_files, reverse=True)
+    for tmp_request_file in request_files:
+        with open(tmp_request_file, "r") as f:
+            req_content = json.load(f)
+            if (
+                req_content["status"] in ["FINISHED"]
+                and req_content["precision"] == precision.split(".")[-1]
+            ):
+                request_file = tmp_request_file
+    return request_file
+
+
+def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResult]:
+    """From the path of the results folder root, extract all needed info for results"""
+    model_result_filepaths = []
+
+    for root, _, files in os.walk(results_path):
+        # We should only have json files in model results
+        if len(files) == 0 or any([not f.endswith(".json") for f in files]):
+            continue
+
+        # Sort the files by date
+        try:
+            files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
+        except dateutil.parser._parser.ParserError:
+            files = [files[-1]]
+
+        for file in files:
+            model_result_filepaths.append(os.path.join(root, file))
+
+    eval_results = {}
+    for model_result_filepath in model_result_filepaths:
+        # Creation of result
+        eval_result = EvalResult.init_from_json_file(model_result_filepath)
+        eval_result.update_with_request_file(requests_path)
+
+        # Store results of same eval together
+        eval_name = eval_result.eval_name
+        if eval_name in eval_results.keys():
+            eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})
+        else:
+            eval_results[eval_name] = eval_result
+
+    results = []
+    for v in eval_results.values():
+        try:
+            v.to_dict() # we test if the dict version is complete
+            results.append(v)
+        except KeyError:  # not all eval values present
+            continue
+
+    return results

src/populate.py

@@ -0,0 +1,58 @@
+import json
+import os
+
+import pandas as pd
+
+from src.display.formatting import has_no_nan_values, make_clickable_model
+from src.display.utils import AutoEvalColumn, EvalQueueColumn
+from src.leaderboard.read_evals import get_raw_eval_results
+
+
+def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
+    """Creates a dataframe from all the individual experiment results"""
+    raw_data = get_raw_eval_results(results_path, requests_path)
+    all_data_json = [v.to_dict() for v in raw_data]
+
+    df = pd.DataFrame.from_records(all_data_json)
+    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
+    df = df[cols].round(decimals=2)
+
+    # filter out if any of the benchmarks have not been produced
+    df = df[has_no_nan_values(df, benchmark_cols)]
+    return df
+
+
+def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
+    """Creates the different dataframes for the evaluation queues requestes"""
+    entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
+    all_evals = []
+
+    for entry in entries:
+        if ".json" in entry:
+            file_path = os.path.join(save_path, entry)
+            with open(file_path) as fp:
+                data = json.load(fp)
+
+            data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
+            data[EvalQueueColumn.revision.name] = data.get("revision", "main")
+
+            all_evals.append(data)
+        elif ".md" not in entry:
+            # this is a folder
+            sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if os.path.isfile(e) and not e.startswith(".")]
+            for sub_entry in sub_entries:
+                file_path = os.path.join(save_path, entry, sub_entry)
+                with open(file_path) as fp:
+                    data = json.load(fp)
+
+                data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
+                data[EvalQueueColumn.revision.name] = data.get("revision", "main")
+                all_evals.append(data)
+
+    pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
+    running_list = [e for e in all_evals if e["status"] == "RUNNING"]
+    finished_list = [e for e in all_evals if e["status"].startswith("FINISHED") or e["status"] == "PENDING_NEW_EVAL"]
+    df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
+    df_running = pd.DataFrame.from_records(running_list, columns=cols)
+    df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
+    return df_finished[cols], df_running[cols], df_pending[cols]

src/submission/check_validity.py

@@ -0,0 +1,99 @@
+import json
+import os
+import re
+from collections import defaultdict
+from datetime import datetime, timedelta, timezone
+
+import huggingface_hub
+from huggingface_hub import ModelCard
+from huggingface_hub.hf_api import ModelInfo
+from transformers import AutoConfig
+from transformers.models.auto.tokenization_auto import AutoTokenizer
+
+def check_model_card(repo_id: str) -> tuple[bool, str]:
+    """Checks if the model card and license exist and have been filled"""
+    try:
+        card = ModelCard.load(repo_id)
+    except huggingface_hub.utils.EntryNotFoundError:
+        return False, "Please add a model card to your model to explain how you trained/fine-tuned it."
+
+    # Enforce license metadata
+    if card.data.license is None:
+        if not ("license_name" in card.data and "license_link" in card.data):
+            return False, (
+                "License not found. Please add a license to your model card using the `license` metadata or a"
+                " `license_name`/`license_link` pair."
+            )
+
+    # Enforce card content
+    if len(card.text) < 200:
+        return False, "Please add a description to your model card, it is too short."
+
+    return True, ""
+
+def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_remote_code=False, test_tokenizer=False) -> tuple[bool, str]:
+    """Checks if the model model_name is on the hub, and whether it (and its tokenizer) can be loaded with AutoClasses."""
+    try:
+        config = AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
+        if test_tokenizer:
+            try:
+                tk = AutoTokenizer.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
+            except ValueError as e:
+                return (
+                    False,
+                    f"uses a tokenizer which is not in a transformers release: {e}",
+                    None
+                )
+            except Exception as e:
+                return (False, "'s tokenizer cannot be loaded. Is your tokenizer class in a stable transformers release, and correctly configured?", None)
+        return True, None, config
+
+    except ValueError:
+        return (
+            False,
+            "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard.",
+            None
+        )
+
+    except Exception as e:
+        return False, "was not found on hub!", None
+
+
+def get_model_size(model_info: ModelInfo, precision: str):
+    """Gets the model size from the configuration, or the model name if the configuration does not contain the information."""
+    try:
+        model_size = round(model_info.safetensors["total"] / 1e9, 3)
+    except (AttributeError, TypeError):
+        return 0  # Unknown model sizes are indicated as 0, see NUMERIC_INTERVALS in app.py
+
+    size_factor = 8 if (precision == "GPTQ" or "gptq" in model_info.modelId.lower()) else 1
+    model_size = size_factor * model_size
+    return model_size
+
+def get_model_arch(model_info: ModelInfo):
+    """Gets the model architecture from the configuration"""
+    return model_info.config.get("architectures", "Unknown")
+
+def already_submitted_models(requested_models_dir: str) -> set[str]:
+    """Gather a list of already submitted models to avoid duplicates"""
+    depth = 1
+    file_names = []
+    users_to_submission_dates = defaultdict(list)
+
+    for root, _, files in os.walk(requested_models_dir):
+        current_depth = root.count(os.sep) - requested_models_dir.count(os.sep)
+        if current_depth == depth:
+            for file in files:
+                if not file.endswith(".json"):
+                    continue
+                with open(os.path.join(root, file), "r") as f:
+                    info = json.load(f)
+                    file_names.append(f"{info['model']}_{info['revision']}_{info['precision']}")
+
+                    # Select organisation
+                    if info["model"].count("/") == 0 or "submitted_time" not in info:
+                        continue
+                    organisation, _ = info["model"].split("/")
+                    users_to_submission_dates[organisation].append(info["submitted_time"])
+
+    return set(file_names), users_to_submission_dates

src/submission/submit.py

@@ -0,0 +1,119 @@
+import json
+import os
+from datetime import datetime, timezone
+
+from src.display.formatting import styled_error, styled_message, styled_warning
+from src.envs import API, EVAL_REQUESTS_PATH, TOKEN, QUEUE_REPO
+from src.submission.check_validity import (
+    already_submitted_models,
+    check_model_card,
+    get_model_size,
+    is_model_on_hub,
+)
+
+REQUESTED_MODELS = None
+USERS_TO_SUBMISSION_DATES = None
+
+def add_new_eval(
+    model: str,
+    base_model: str,
+    revision: str,
+    precision: str,
+    weight_type: str,
+    model_type: str,
+):
+    global REQUESTED_MODELS
+    global USERS_TO_SUBMISSION_DATES
+    if not REQUESTED_MODELS:
+        REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
+
+    user_name = ""
+    model_path = model
+    if "/" in model:
+        user_name = model.split("/")[0]
+        model_path = model.split("/")[1]
+
+    precision = precision.split(" ")[0]
+    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+    if model_type is None or model_type == "":
+        return styled_error("Please select a model type.")
+
+    # Does the model actually exist?
+    if revision == "":
+        revision = "main"
+
+    # Is the model on the hub?
+    if weight_type in ["Delta", "Adapter"]:
+        base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True)
+        if not base_model_on_hub:
+            return styled_error(f'Base model "{base_model}" {error}')
+
+    if not weight_type == "Adapter":
+        model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=True)
+        if not model_on_hub:
+            return styled_error(f'Model "{model}" {error}')
+
+    # Is the model info correctly filled?
+    try:
+        model_info = API.model_info(repo_id=model, revision=revision)
+    except Exception:
+        return styled_error("Could not get your model information. Please fill it up properly.")
+
+    model_size = get_model_size(model_info=model_info, precision=precision)
+
+    # Were the model card and license filled?
+    try:
+        license = model_info.cardData["license"]
+    except Exception:
+        return styled_error("Please select a license for your model")
+
+    modelcard_OK, error_msg = check_model_card(model)
+    if not modelcard_OK:
+        return styled_error(error_msg)
+
+    # Seems good, creating the eval
+    print("Adding new eval")
+
+    eval_entry = {
+        "model": model,
+        "base_model": base_model,
+        "revision": revision,
+        "precision": precision,
+        "weight_type": weight_type,
+        "status": "PENDING",
+        "submitted_time": current_time,
+        "model_type": model_type,
+        "likes": model_info.likes,
+        "params": model_size,
+        "license": license,
+        "private": False,
+    }
+
+    # Check for duplicate submission
+    if f"{model}_{revision}_{precision}" in REQUESTED_MODELS:
+        return styled_warning("This model has been already submitted.")
+
+    print("Creating eval file")
+    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
+    os.makedirs(OUT_DIR, exist_ok=True)
+    out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json"
+
+    with open(out_path, "w") as f:
+        f.write(json.dumps(eval_entry))
+
+    print("Uploading eval file")
+    API.upload_file(
+        path_or_fileobj=out_path,
+        path_in_repo=out_path.split("eval-queue/")[1],
+        repo_id=QUEUE_REPO,
+        repo_type="dataset",
+        commit_message=f"Add {model} to eval queue",
+    )
+
+    # Remove the local file
+    os.remove(out_path)
+
+    return styled_message(
+        "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."
+    )

visualization/__init__.py

@@ -0,0 +1,11 @@
+from .energy import visualize_energy
+from .geometry import visualize_geometry
+from .building import visualize_graph
+from .weather import visualize_weather
+
+__all__ = [
+    "visualize_geometry",
+    "visualize_graph",
+    "visualize_weather",
+    "visualize_energy",
+]

visualization/building.py

@@ -0,0 +1,692 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import matplotlib
+import numpy as np
+from matplotlib import colors
+
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+
+DEFAULT_PANEL_SIZE_MULTIPLIER = 1.5
+DEFAULT_BILLBOARD_ALPHA = 1.0
+DEFAULT_PANEL_SCALE = 0.12
+DEFAULT_DAY_STEP = 1
+
+DEFAULT_DIVERGING_CMAP = colors.LinearSegmentedColormap.from_list(
+    "zone_load_diverging",
+    ["#82B0D2", "#FFFFFF", "#FA7F6F"],
+    N=256,
+)
+
+# Keep color mapping consistent with cuger/__analyse/visualise.py.
+TYPE_COLORS = {
+    "window": "#FFBE7A",
+    "shading": "#999999",
+    "floor": "#82B0D2",
+    "wall": "#8ECFC9",
+    "airwall": "#E7DAD2",
+    "space": "#FA7F6F",
+    "void": "#FFFFFF",
+    None: "#FFFFFF",
+}
+
+
+def _axis_limits_from_points(points: np.ndarray) -> tuple[tuple[float, float], tuple[float, float], tuple[float, float]]:
+    x_min, x_max = float(np.min(points[:, 0])), float(np.max(points[:, 0]))
+    y_min, y_max = float(np.min(points[:, 1])), float(np.max(points[:, 1]))
+    z_min, z_max = float(np.min(points[:, 2])), float(np.max(points[:, 2]))
+
+    max_range = max(x_max - x_min, y_max - y_min, z_max - z_min) / 2.0
+    max_range = max(max_range, 1e-6) * 1.08
+
+    x_mid = (x_max + x_min) / 2.0
+    y_mid = (y_max + y_min) / 2.0
+    z_mid = (z_max + z_min) / 2.0
+
+    return (
+        (x_mid - max_range, x_mid + max_range),
+        (y_mid - max_range, y_mid + max_range),
+        (z_mid - max_range, z_mid + max_range),
+    )
+
+
+def _first_existing(data: dict[str, np.ndarray], keys: list[str]) -> np.ndarray | None:
+    for k in keys:
+        if k in data:
+            return np.asarray(data[k])
+    return None
+
+
+def _as_2d_points(arr: np.ndarray | None) -> np.ndarray:
+    if arr is None:
+        return np.zeros((0, 3), dtype=float)
+    pts = np.asarray(arr, dtype=float)
+    if pts.ndim != 2 or pts.shape[1] < 3:
+        return np.zeros((0, 3), dtype=float)
+    return pts[:, :3]
+
+
+def _camera_basis(elev: float, azim: float) -> tuple[np.ndarray, np.ndarray]:
+    az = np.deg2rad(float(azim))
+    el = np.deg2rad(float(elev))
+
+    forward = np.array(
+        [
+            np.cos(el) * np.cos(az),
+            np.cos(el) * np.sin(az),
+            np.sin(el),
+        ],
+        dtype=float,
+    )
+    forward_norm = np.linalg.norm(forward)
+    if forward_norm < 1e-9:
+        forward = np.array([1.0, 0.0, 0.0], dtype=float)
+    else:
+        forward = forward / forward_norm
+
+    world_up = np.array([0.0, 0.0, 1.0], dtype=float)
+    right = np.cross(forward, world_up)
+    right_norm = np.linalg.norm(right)
+    if right_norm < 1e-9:
+        right = np.array([1.0, 0.0, 0.0], dtype=float)
+    else:
+        right = right / right_norm
+
+    up = np.cross(right, forward)
+    up_norm = np.linalg.norm(up)
+    if up_norm < 1e-9:
+        up = np.array([0.0, 1.0, 0.0], dtype=float)
+    else:
+        up = up / up_norm
+
+    return right, up
+
+
+def _isometric_panel_basis(elev: float, azim: float) -> tuple[np.ndarray, np.ndarray]:
+    cam_right, cam_up = _camera_basis(elev=elev, azim=azim)
+    c30 = np.cos(np.deg2rad(30.0))
+    s30 = np.sin(np.deg2rad(30.0))
+
+    u = c30 * cam_right + s30 * cam_up
+    v = -c30 * cam_right + s30 * cam_up
+    u = u / max(np.linalg.norm(u), 1e-9)
+    v = v / max(np.linalg.norm(v), 1e-9)
+    return u, v
+
+
+def _to_hourly_zone(values: np.ndarray) -> np.ndarray:
+    arr = np.asarray(values)
+    if arr.ndim != 2:
+        raise ValueError(f"`values` must be 2D, got shape={arr.shape}")
+
+    if arr.shape[0] == 8760:
+        return arr.astype(np.float32)
+
+    if arr.shape[1] == 8760:
+        return arr.T.astype(np.float32)
+
+    raise ValueError(f"Neither axis equals 8760, shape={arr.shape}")
+
+
+def _try_parse_space_indices(raw_values: np.ndarray | None, zone_count: int, space_count: int) -> np.ndarray | None:
+    if raw_values is None:
+        return None
+
+    vals = np.asarray(raw_values).reshape(-1)
+    if vals.size < zone_count:
+        return None
+
+    out: list[int] = []
+    for i in range(zone_count):
+        v = vals[i]
+        idx: int | None = None
+
+        if isinstance(v, (int, np.integer)):
+            idx = int(v)
+        elif isinstance(v, (float, np.floating)):
+            vf = float(v)
+            if np.isfinite(vf) and float(vf).is_integer():
+                idx = int(vf)
+        else:
+            text = v.decode("utf-8", errors="ignore") if isinstance(v, (bytes, np.bytes_)) else str(v)
+            text = text.strip()
+            try:
+                idx = int(text)
+            except ValueError:
+                return None
+
+        if idx is None or not (0 <= idx < space_count):
+            return None
+        out.append(idx)
+
+    return np.asarray(out, dtype=np.int64)
+
+
+def _zone_day_hour_matrix(zone_hourly: np.ndarray, day_step: int = 1) -> np.ndarray:
+    series = np.asarray(zone_hourly, dtype=np.float32).reshape(-1)
+    if series.size < 8760:
+        raise ValueError(f"Zone series length must be >= 8760, got {series.size}")
+
+    day_hour = series[:8760].reshape(365, 24)
+    step = max(1, int(day_step))
+    if step == 1:
+        return day_hour
+
+    rows: list[np.ndarray] = []
+    for s in range(0, 365, step):
+        e = min(365, s + step)
+        rows.append(np.mean(day_hour[s:e, :], axis=0))
+    return np.asarray(rows, dtype=np.float32)
+
+
+def _decode_types(type_arr: np.ndarray | None, expected_count: int, fallback: str) -> list[str]:
+    if type_arr is None:
+        return [fallback] * expected_count
+
+    raw = np.asarray(type_arr).reshape(-1)
+    out: list[str] = []
+    for val in raw[:expected_count]:
+        if isinstance(val, (bytes, np.bytes_)):
+            text = val.decode("utf-8", errors="ignore").strip().lower()
+        else:
+            text = str(val).strip().lower()
+
+        if text in TYPE_COLORS:
+            out.append(text)
+            continue
+
+        # Numeric fallback for compact encodings.
+        try:
+            num = float(text)
+            if np.isfinite(num):
+                if num <= 0:
+                    out.append("wall")
+                else:
+                    out.append("window")
+                continue
+        except ValueError:
+            pass
+
+        out.append(fallback)
+
+    if len(out) < expected_count:
+        out.extend([fallback] * (expected_count - len(out)))
+    return out
+
+
+def _face_colors_from_binary_t(face_feats: np.ndarray | None, face_count: int) -> list[str] | None:
+    """Use binary t from face_feats last column when available.
+
+    Rule requested by user: t == 0 -> yellow(window color).
+    """
+    if face_feats is None:
+        return None
+
+    feats = np.asarray(face_feats, dtype=float)
+    if feats.ndim != 2 or feats.shape[1] < 1 or feats.shape[0] < face_count:
+        return None
+
+    t_col = np.rint(feats[:face_count, -1]).astype(np.int32)
+    if not np.all(np.isin(t_col, [0, 1])):
+        return None
+
+    out: list[str] = []
+    for t_val in t_col:
+        if t_val == 0:
+            out.append(TYPE_COLORS["window"])
+        else:
+            out.append(TYPE_COLORS["wall"])
+    return out
+
+
+def _plot_edges(ax, starts: np.ndarray, ends: np.ndarray, color: str, linewidth: float, linestyle: str, alpha: float) -> None:
+    for p0, p1 in zip(starts, ends):
+        ax.plot(
+            [p0[0], p1[0]],
+            [p0[1], p1[1]],
+            [p0[2], p1[2]],
+            color=color,
+            linewidth=linewidth,
+            linestyle=linestyle,
+            alpha=alpha,
+        )
+
+
+def _infer_space_count(graph_data: dict[str, np.ndarray], sf_edges: np.ndarray | None) -> int:
+    valid_spaces = graph_data.get("valid_energy_spaces")
+    if valid_spaces is not None:
+        size = int(np.asarray(valid_spaces).reshape(-1).size)
+        if size > 0:
+            return size
+
+    space_feats = _first_existing(graph_data, ["space_feats", "space_c", "space_centers"])
+    if space_feats is not None:
+        feats = np.asarray(space_feats)
+        if feats.ndim >= 1 and feats.shape[0] > 0:
+            return int(feats.shape[0])
+
+    if sf_edges is not None:
+        edges = np.asarray(sf_edges, dtype=np.int64)
+        if edges.ndim == 2 and edges.shape[1] >= 2 and edges.shape[0] > 0:
+            c0_max = int(np.max(edges[:, 0]))
+            c1_max = int(np.max(edges[:, 1]))
+            return max(c0_max, c1_max) + 1
+
+    return 0
+
+
+def _extract_face_space_pairs(sf_edges: np.ndarray | None, n_faces: int, n_spaces: int) -> list[tuple[int, int]]:
+    if sf_edges is None or n_faces <= 0 or n_spaces <= 0:
+        return []
+
+    edges = np.asarray(sf_edges, dtype=np.int64)
+    if edges.ndim != 2 or edges.shape[1] < 2:
+        return []
+
+    # PACK building npz uses [face_idx, space_idx] in sf_edges.
+    c0_max = int(np.max(edges[:, 0])) if edges.shape[0] > 0 else -1
+    c1_max = int(np.max(edges[:, 1])) if edges.shape[0] > 0 else -1
+    face_space_ok = c0_max < n_faces and c1_max < n_spaces
+    space_face_ok = c1_max < n_faces and c0_max < n_spaces
+    use_face_space = True
+    if face_space_ok and not space_face_ok:
+        use_face_space = True
+    elif space_face_ok and not face_space_ok:
+        use_face_space = False
+
+    pairs: list[tuple[int, int]] = []
+    for e in edges:
+        a, b = int(e[0]), int(e[1])
+        if use_face_space:
+            f_idx, s_idx = a, b
+        else:
+            f_idx, s_idx = b, a
+
+        if 0 <= f_idx < n_faces and 0 <= s_idx < n_spaces:
+            pairs.append((f_idx, s_idx))
+
+    return pairs
+
+
+def _infer_space_centers_from_edges_indexed(face_centers: np.ndarray, sf_edges: np.ndarray | None, n_spaces: int) -> np.ndarray:
+    if len(face_centers) == 0 or n_spaces <= 0:
+        return np.zeros((0, 3), dtype=float)
+
+    pairs = _extract_face_space_pairs(sf_edges, n_faces=len(face_centers), n_spaces=n_spaces)
+    if not pairs:
+        return np.full((n_spaces, 3), np.nan, dtype=float)
+
+    buckets: list[list[np.ndarray]] = [[] for _ in range(n_spaces)]
+    for f_idx, s_idx in pairs:
+        buckets[s_idx].append(face_centers[f_idx])
+
+    centers = np.full((n_spaces, 3), np.nan, dtype=float)
+    for s_idx, pts in enumerate(buckets):
+        if pts:
+            centers[s_idx] = np.mean(np.asarray(pts, dtype=float), axis=0)
+
+    return centers
+
+
+def _resolve_space_layout(
+    face_centers: np.ndarray,
+    graph_data: dict[str, np.ndarray],
+) -> tuple[np.ndarray, list[tuple[int, int]], int, dict[int, int]]:
+    sf_edges_raw = _first_existing(graph_data, ["sf_edges", "face_space_edges"])
+    sf_edges = np.asarray(sf_edges_raw, dtype=np.int64) if sf_edges_raw is not None else np.zeros((0, 2), dtype=np.int64)
+
+    explicit_space_centers = _as_2d_points(_first_existing(graph_data, ["space_c", "space_centers"]))
+    if len(explicit_space_centers) > 0:
+        raw_space_count = int(explicit_space_centers.shape[0])
+        raw_to_compact = {int(i): int(i) for i in range(raw_space_count)}
+        space_centers = explicit_space_centers
+    else:
+        raw_space_count = _infer_space_count(graph_data, sf_edges)
+        raw_space_centers = _infer_space_centers_from_edges_indexed(face_centers, sf_edges, n_spaces=raw_space_count)
+        if len(raw_space_centers) == 0:
+            return np.zeros((0, 3), dtype=float), [], raw_space_count, {}
+
+        valid_mask = np.isfinite(raw_space_centers).all(axis=1)
+        valid_raw_idx = np.where(valid_mask)[0].astype(np.int64)
+        raw_to_compact = {int(raw_idx): int(compact_idx) for compact_idx, raw_idx in enumerate(valid_raw_idx.tolist())}
+        space_centers = raw_space_centers[valid_mask]
+
+    pairs_raw = _extract_face_space_pairs(sf_edges, n_faces=len(face_centers), n_spaces=raw_space_count)
+    pairs_compact = [(f_idx, raw_to_compact[s_idx]) for f_idx, s_idx in pairs_raw if s_idx in raw_to_compact]
+    return space_centers, pairs_compact, raw_space_count, raw_to_compact
+
+
+def _plot_generic_graph(ax, graph_data: dict[str, np.ndarray]) -> bool:
+    centers = _first_existing(graph_data, ["c", "center", "centers", "node_c", "node_centers", "face_c"])
+    points = _as_2d_points(centers)
+    if len(points) == 0:
+        return False
+
+    type_arr = _first_existing(graph_data, ["t", "type", "node_t", "node_type", "types"])
+    node_types = _decode_types(type_arr, expected_count=len(points), fallback="wall")
+    node_colors = [TYPE_COLORS.get(t, "#8ECFC9") for t in node_types]
+
+    ax.scatter(points[:, 0], points[:, 1], points[:, 2], c=node_colors, s=30, edgecolors="k", alpha=0.95)
+
+    edge_arr = _first_existing(graph_data, ["edges", "edge_index", "ff_edges"])
+    if edge_arr is not None:
+        edges = np.asarray(edge_arr, dtype=np.int64)
+        if edges.ndim == 2 and edges.shape[1] >= 2:
+            valid = (edges[:, 0] >= 0) & (edges[:, 1] >= 0) & (edges[:, 0] < len(points)) & (edges[:, 1] < len(points))
+            valid_edges = edges[valid]
+            if len(valid_edges) > 0:
+                starts = points[valid_edges[:, 0]]
+                ends = points[valid_edges[:, 1]]
+                _plot_edges(ax, starts, ends, color="#666666", linewidth=1.0, linestyle="-", alpha=0.45)
+
+    x_lim, y_lim, z_lim = _axis_limits_from_points(points)
+    ax.set_xlim(*x_lim)
+    ax.set_ylim(*y_lim)
+    ax.set_zlim(*z_lim)
+    return True
+
+
+def _plot_pack_graph(ax, graph_data: dict[str, np.ndarray], geometry_npz: Path | None) -> bool:
+    face_centers = _as_2d_points(_first_existing(graph_data, ["face_c"]))
+
+    if len(face_centers) == 0 and geometry_npz is not None:
+        with np.load(geometry_npz, allow_pickle=True) as g_data:
+            if "face_c" in g_data:
+                face_centers = _as_2d_points(np.asarray(g_data["face_c"]))
+
+    space_centers, pairs_compact, _, _ = _resolve_space_layout(face_centers, graph_data)
+
+    if len(face_centers) == 0 and len(space_centers) == 0:
+        return False
+
+    face_types = _decode_types(_first_existing(graph_data, ["face_t", "face_type", "t", "type"]), len(face_centers), "wall")
+    face_colors = [TYPE_COLORS.get(t, "#8ECFC9") for t in face_types]
+    binary_colors = _face_colors_from_binary_t(_first_existing(graph_data, ["face_feats"]), len(face_centers))
+    if binary_colors is not None:
+        face_colors = binary_colors
+
+    if len(face_centers) > 0:
+        ax.scatter(face_centers[:, 0], face_centers[:, 1], face_centers[:, 2], c=face_colors, s=28, marker="D", edgecolors="k", alpha=0.9)
+    if len(space_centers) > 0:
+        ax.scatter(space_centers[:, 0], space_centers[:, 1], space_centers[:, 2], c=TYPE_COLORS["space"], s=70, edgecolors="k", alpha=0.95)
+
+    ff_edges = _first_existing(graph_data, ["ff_edges", "face_edges"])
+    if ff_edges is not None and len(face_centers) > 0:
+        edges = np.asarray(ff_edges, dtype=np.int64)
+        if edges.ndim == 2 and edges.shape[1] >= 2:
+            valid = (edges[:, 0] >= 0) & (edges[:, 1] >= 0) & (edges[:, 0] < len(face_centers)) & (edges[:, 1] < len(face_centers))
+            edge_ok = edges[valid]
+            if len(edge_ok) > 0:
+                _plot_edges(
+                    ax,
+                    face_centers[edge_ok[:, 0]],
+                    face_centers[edge_ok[:, 1]],
+                    color="#999999",
+                    linewidth=1.2,
+                    linestyle="-",
+                    alpha=0.35,
+                )
+
+    if len(face_centers) > 0 and len(space_centers) > 0 and pairs_compact:
+        starts = np.asarray([face_centers[f_idx] for f_idx, _ in pairs_compact], dtype=float)
+        ends = np.asarray([space_centers[s_idx] for _, s_idx in pairs_compact], dtype=float)
+        _plot_edges(
+            ax,
+            starts,
+            ends,
+            color="#555555",
+            linewidth=1.8,
+            linestyle="--",
+            alpha=0.9,
+        )
+
+    all_points = face_centers if len(space_centers) == 0 else np.vstack([face_centers, space_centers])
+    x_lim, y_lim, z_lim = _axis_limits_from_points(all_points)
+    ax.set_xlim(*x_lim)
+    ax.set_ylim(*y_lim)
+    ax.set_zlim(*z_lim)
+    return True
+
+
+def _plot_pack_graph_overlay(
+    ax,
+    graph_data: dict[str, np.ndarray],
+    energy_data: dict[str, np.ndarray],
+    geometry_npz: Path | None,
+    *,
+    elev: float,
+    azim: float,
+    panel_scale: float,
+    day_step: int,
+) -> bool:
+    face_centers = _as_2d_points(_first_existing(graph_data, ["face_c"]))
+    if len(face_centers) == 0 and geometry_npz is not None:
+        with np.load(geometry_npz, allow_pickle=True) as g_data:
+            if "face_c" in g_data:
+                face_centers = _as_2d_points(np.asarray(g_data["face_c"]))
+
+    if len(face_centers) == 0:
+        return False
+
+    if "values" not in energy_data:
+        return False
+
+    space_centers, pairs_compact, raw_space_count, raw_to_compact = _resolve_space_layout(face_centers, graph_data)
+    if len(space_centers) == 0:
+        return False
+
+    hourly_zone = _to_hourly_zone(np.asarray(energy_data["values"], dtype=np.float32))
+    zone_count = int(hourly_zone.shape[1])
+    valid_energy_spaces_raw = graph_data.get("valid_energy_spaces")
+    valid_energy_spaces = _try_parse_space_indices(valid_energy_spaces_raw, zone_count=zone_count, space_count=raw_space_count)
+
+    zone_to_space: list[tuple[int, int]] = []
+    if valid_energy_spaces is not None and valid_energy_spaces.size >= zone_count:
+        for z_idx in range(zone_count):
+            raw_s_idx = int(valid_energy_spaces[z_idx])
+            compact_s_idx = raw_to_compact.get(raw_s_idx)
+            if compact_s_idx is not None:
+                zone_to_space.append((z_idx, compact_s_idx))
+    else:
+        for z_idx in range(min(zone_count, raw_space_count)):
+            compact_s_idx = raw_to_compact.get(z_idx)
+            if compact_s_idx is not None:
+                zone_to_space.append((z_idx, compact_s_idx))
+
+    if not zone_to_space:
+        fallback_n = min(zone_count, len(space_centers))
+        zone_to_space = [(z_idx, z_idx) for z_idx in range(fallback_n)]
+
+    if not zone_to_space:
+        return False
+
+    if hasattr(ax, "computed_zorder"):
+        ax.computed_zorder = True
+
+    ax.scatter(face_centers[:, 0], face_centers[:, 1], face_centers[:, 2], c="#A8A8A8", s=8, alpha=0.24, zorder=2)
+    ax.scatter(space_centers[:, 0], space_centers[:, 1], space_centers[:, 2], c="#6E6E6E", s=14, alpha=0.5, zorder=3)
+
+    if pairs_compact:
+        starts = np.asarray([space_centers[s_idx] for _, s_idx in pairs_compact], dtype=float)
+        ends = np.asarray([face_centers[f_idx] for f_idx, _ in pairs_compact], dtype=float)
+        _plot_edges(
+            ax,
+            starts,
+            ends,
+            color="#777777",
+            linewidth=0.8,
+            linestyle="--",
+            alpha=0.3,
+        )
+
+    all_points = np.vstack([face_centers, space_centers])
+    x_lim, y_lim, z_lim = _axis_limits_from_points(all_points)
+    span = max(x_lim[1] - x_lim[0], y_lim[1] - y_lim[0], z_lim[1] - z_lim[0])
+    panel_w = max(span * float(panel_scale) * DEFAULT_PANEL_SIZE_MULTIPLIER, 1e-4)
+    panel_h = panel_w * 0.75
+
+    right, up = _isometric_panel_basis(elev=elev, azim=azim)
+
+    for z_idx, s_idx in zone_to_space:
+        center = space_centers[s_idx]
+        mat = _zone_day_hour_matrix(hourly_zone[:, z_idx], day_step=day_step)
+
+        zmin = float(np.min(mat))
+        zmax = float(np.max(mat))
+        if zmin < 0.0 < zmax:
+            zone_norm = colors.TwoSlopeNorm(vmin=zmin, vcenter=0.0, vmax=zmax)
+        elif abs(zmax - zmin) < 1e-12:
+            zone_norm = colors.Normalize(vmin=zmin - 1.0, vmax=zmax + 1.0)
+        else:
+            zone_norm = colors.Normalize(vmin=zmin, vmax=zmax)
+
+        facecolors_rgba = DEFAULT_DIVERGING_CMAP(zone_norm(mat))
+
+        n_rows, n_cols = mat.shape
+        u = np.linspace(-0.5, 0.5, n_cols, dtype=float) * panel_w
+        v = np.linspace(-0.5, 0.5, n_rows, dtype=float) * panel_h
+        uu, vv = np.meshgrid(u, v)
+
+        x = center[0] + right[0] * uu + up[0] * vv
+        y = center[1] + right[1] * uu + up[1] * vv
+        z = center[2] + right[2] * uu + up[2] * vv
+
+        surf = ax.plot_surface(
+            x,
+            y,
+            z,
+            facecolors=facecolors_rgba,
+            shade=False,
+            linewidth=0.0,
+            antialiased=False,
+            alpha=DEFAULT_BILLBOARD_ALPHA,
+        )
+        if hasattr(surf, "set_zsort"):
+            surf.set_zsort("average")
+
+        c1 = center - 0.5 * panel_w * right - 0.5 * panel_h * up
+        c2 = center + 0.5 * panel_w * right - 0.5 * panel_h * up
+        c3 = center + 0.5 * panel_w * right + 0.5 * panel_h * up
+        c4 = center - 0.5 * panel_w * right + 0.5 * panel_h * up
+        border = np.asarray([c1, c2, c3, c4, c1], dtype=float)
+        ax.plot(
+            border[:, 0],
+            border[:, 1],
+            border[:, 2],
+            color="#000000",
+            linewidth=0.85,
+            alpha=1.0,
+        )
+
+    ax.set_xlim(*x_lim)
+    ax.set_ylim(*y_lim)
+    ax.set_zlim(*z_lim)
+    return True
+
+
+def _looks_like_pack_graph(graph_data: dict[str, np.ndarray]) -> bool:
+    pack_keys = {"face_c", "sf_edges", "face_space_edges", "space_c", "space_centers", "face_feats", "ff_edges"}
+    return any(k in graph_data for k in pack_keys)
+
+
+def visualize_graph(
+    graph_npz: str | Path,
+    output_png: str | Path,
+    *,
+    geometry_npz: str | Path | None = None,
+    elev: float = 35.0,
+    azim: float = 15.0,
+    dpi: int = 300,
+) -> Path:
+    """Render graph nodes/edges from graph npz using c/t/type conventions when available."""
+    graph_npz = Path(graph_npz)
+    output_png = Path(output_png)
+    geometry_path = Path(geometry_npz) if geometry_npz is not None else None
+
+    with np.load(graph_npz, allow_pickle=True) as data:
+        graph_data = {k: np.asarray(data[k]) for k in data.files}
+
+    fig = plt.figure(figsize=(4, 4))
+    ax = fig.add_subplot(111, projection="3d")
+    ax.view_init(elev=elev, azim=azim)
+    fig.subplots_adjust(left=0, right=1, top=1, bottom=0)
+
+    if _looks_like_pack_graph(graph_data):
+        ok = _plot_pack_graph(ax, graph_data, geometry_npz=geometry_path)
+        if not ok:
+            ok = _plot_generic_graph(ax, graph_data)
+    else:
+        ok = _plot_generic_graph(ax, graph_data)
+        if not ok:
+            ok = _plot_pack_graph(ax, graph_data, geometry_npz=geometry_path)
+
+    if not ok:
+        keys = ", ".join(sorted(graph_data.keys()))
+        plt.close(fig)
+        raise ValueError(f"Cannot parse graph centers/edges from {graph_npz}; keys=[{keys}]")
+
+    ax.set_box_aspect([1, 1, 1])
+    ax.set_axis_off()
+
+    output_png.parent.mkdir(parents=True, exist_ok=True)
+    fig.savefig(output_png, dpi=dpi, bbox_inches="tight", pad_inches=0.04, transparent=True)
+    plt.close(fig)
+    return output_png
+
+
+def visualize_graph_overlap(
+    graph_npz: str | Path,
+    energy_npz: str | Path,
+    output_png: str | Path,
+    *,
+    geometry_npz: str | Path | None = None,
+    elev: float = 35.0,
+    azim: float = 15.0,
+    panel_scale: float = DEFAULT_PANEL_SCALE,
+    day_step: int = DEFAULT_DAY_STEP,
+    dpi: int = 300,
+) -> Path:
+    """Render PACK graph with per-space energy billboard overlays."""
+    graph_npz = Path(graph_npz)
+    energy_npz = Path(energy_npz)
+    output_png = Path(output_png)
+    geometry_path = Path(geometry_npz) if geometry_npz is not None else None
+
+    with np.load(graph_npz, allow_pickle=True) as data:
+        graph_data = {k: np.asarray(data[k]) for k in data.files}
+    with np.load(energy_npz, allow_pickle=True) as data:
+        energy_data = {k: np.asarray(data[k]) for k in data.files}
+
+    fig = plt.figure(figsize=(4, 4))
+    ax = fig.add_subplot(111, projection="3d")
+    ax.view_init(elev=elev, azim=azim)
+    fig.subplots_adjust(left=0, right=1, top=1, bottom=0)
+
+    ok = _plot_pack_graph_overlay(
+        ax,
+        graph_data,
+        energy_data,
+        geometry_npz=geometry_path,
+        elev=elev,
+        azim=azim,
+        panel_scale=panel_scale,
+        day_step=day_step,
+    )
+    if not ok:
+        ok = _plot_pack_graph(ax, graph_data, geometry_npz=geometry_path)
+    if not ok:
+        ok = _plot_generic_graph(ax, graph_data)
+    if not ok:
+        keys = ", ".join(sorted(graph_data.keys()))
+        plt.close(fig)
+        raise ValueError(f"Cannot parse graph centers/edges from {graph_npz}; keys=[{keys}]")
+
+    ax.set_box_aspect([1, 1, 1])
+    ax.set_axis_off()
+
+    output_png.parent.mkdir(parents=True, exist_ok=True)
+    fig.savefig(output_png, dpi=dpi, bbox_inches="tight", pad_inches=0.04, transparent=True)
+    plt.close(fig)
+    return output_png

visualization/energy.py

@@ -0,0 +1,171 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import matplotlib
+import numpy as np
+
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+
+FIG_SIZE_IN = 4.2
+MAX_ENERGY_SUBPLOTS = 6
+AXIS_LABEL_FONT_SIZE = 8
+TICK_LABEL_FONT_SIZE = 8
+INPLOT_LABEL_FONT_SIZE = 10
+
+
+def _to_hourly_zone(values: np.ndarray) -> np.ndarray:
+    arr = np.asarray(values)
+    if arr.ndim != 2:
+        raise ValueError(f"Expected 2D energy matrix, got shape={arr.shape}")
+
+    if arr.shape[0] == 8760:
+        return np.asarray(arr, dtype=float)
+    if arr.shape[1] == 8760:
+        return np.asarray(arr.T, dtype=float)
+
+    raise ValueError(f"Neither axis is 8760 for energy matrix: shape={arr.shape}")
+
+
+def _decode_zone_names(columns_arr: np.ndarray | None, zone_count: int) -> list[str]:
+    if columns_arr is None:
+        return [f"zone_{i}" for i in range(zone_count)]
+
+    cols = np.asarray(columns_arr).reshape(-1)
+    names = [str(c, "utf-8") if isinstance(c, (bytes, np.bytes_)) else str(c) for c in cols]
+    if len(names) < zone_count:
+        names.extend([f"zone_{i}" for i in range(len(names), zone_count)])
+    return names[:zone_count]
+
+
+def _time_window(hourly_zone: np.ndarray, start_hour: int, window_hours: int) -> tuple[np.ndarray, int, int]:
+    total = int(hourly_zone.shape[0])
+    if total < 1:
+        raise ValueError("No hourly energy records found")
+
+    start_idx = max(0, min(total - 1, int(start_hour) - 1))
+    window = max(1, int(window_hours))
+    end_idx = min(total, start_idx + window)
+    if end_idx <= start_idx:
+        raise ValueError(f"Invalid energy window: start={start_hour}, hours={window_hours}")
+
+    return hourly_zone[start_idx:end_idx, :], start_idx + 1, end_idx
+
+
+def _major_ticks(length: int) -> list[int]:
+    if length <= 8:
+        return list(range(1, length + 1))
+
+    tick_count = 6
+    ticks = np.linspace(1, length, num=tick_count, dtype=int)
+    uniq = sorted(set(int(t) for t in ticks))
+    if uniq[-1] != length:
+        uniq.append(length)
+    return uniq
+
+
+def visualize_energy(
+    energy_npz: str | Path,
+    output_png: str | Path,
+    *,
+    max_zones: int | None = None,
+    zone_index: int | None = None,
+    start_hour: int = 1,
+    window_hours: int = 24,
+    dpi: int = 220,
+) -> Path:
+    """Plot energy curves in compressed square layout with shared x-axis for a selected time window."""
+    energy_npz = Path(energy_npz)
+    output_png = Path(output_png)
+
+    with np.load(energy_npz, allow_pickle=True) as data:
+        if "values" not in data:
+            keys = ", ".join(sorted(data.files))
+            raise KeyError(f"Missing key 'values' in {energy_npz}; keys=[{keys}]")
+        values = np.asarray(data["values"], dtype=float)
+        columns = np.asarray(data["columns"], dtype=object) if "columns" in data else None
+
+    hourly_zone = _to_hourly_zone(values)
+    window, window_start, window_end = _time_window(hourly_zone, start_hour=start_hour, window_hours=window_hours)
+    zone_count = window.shape[1]
+    if zone_count < 1:
+        raise ValueError(f"No zones found in {energy_npz}")
+
+    names = _decode_zone_names(columns, zone_count)
+
+    zone_indices = list(range(zone_count))
+    if zone_index is not None:
+        zi = int(zone_index)
+        if zi < 0 or zi >= zone_count:
+            raise ValueError(f"zone_index out of range: {zi}, valid=[0, {zone_count - 1}]")
+        zone_indices = [zi]
+    elif max_zones is not None and max_zones > 0:
+        zone_indices = zone_indices[: max_zones]
+
+    if len(zone_indices) < 1:
+        raise ValueError("No zones selected for plotting")
+
+    plotted_zone_indices = zone_indices[:MAX_ENERGY_SUBPLOTS]
+    omitted_count = len(zone_indices) - len(plotted_zone_indices)
+
+    cmap = plt.get_cmap("tab20")
+    x = np.arange(1, window.shape[0] + 1, dtype=int)
+    major_ticks = _major_ticks(window.shape[0])
+
+    row_count = len(plotted_zone_indices)
+    fig, axes = plt.subplots(
+        row_count,
+        1,
+        figsize=(FIG_SIZE_IN, FIG_SIZE_IN),
+        sharex=True,
+        gridspec_kw={"hspace": 0.0},
+    )
+    if row_count == 1:
+        axes = [axes]
+
+    for row_idx, zone_idx in enumerate(plotted_zone_indices):
+        ax = axes[row_idx]
+        color = cmap(row_idx % 20)
+        ax.plot(x, window[:, zone_idx], color=color, linewidth=0.9, alpha=0.9)
+        ax.set_ylabel("")
+        ax.text(
+            0.02,
+            0.86,
+            names[zone_idx],
+            transform=ax.transAxes,
+            ha="left",
+            va="top",
+            rotation=0,
+            fontsize=INPLOT_LABEL_FONT_SIZE,
+            bbox={"facecolor": "white", "alpha": 0.65, "edgecolor": "none", "pad": 1.5},
+        )
+        ax.set_xticks(major_ticks)
+        ax.grid(axis="y", alpha=0.25, linewidth=0.5)
+        ax.grid(axis="x", alpha=0.22, linewidth=0.45)
+        ax.tick_params(axis="both", which="both", labelsize=TICK_LABEL_FONT_SIZE)
+        if row_idx < row_count - 1:
+            ax.tick_params(axis="x", which="both", labelbottom=False)
+
+    if omitted_count > 0:
+        axes[-1].text(
+            0.98,
+            0.86,
+            f"... (+{omitted_count})",
+            transform=axes[-1].transAxes,
+            ha="right",
+            va="top",
+            fontsize=INPLOT_LABEL_FONT_SIZE,
+            bbox={"facecolor": "white", "alpha": 0.65, "edgecolor": "none", "pad": 1.5},
+        )
+
+    axes[-1].set_xlabel(f"hour index in window ({window_start}-{window_end})", fontsize=AXIS_LABEL_FONT_SIZE)
+    axes[-1].set_xlim(1, window.shape[0] + 0.5)
+    axes[-1].set_xticks(major_ticks)
+    axes[-1].set_xticklabels([str(t) for t in major_ticks], fontsize=TICK_LABEL_FONT_SIZE)
+    fig.subplots_adjust(left=0.18, right=0.94, bottom=0.14, top=0.98, hspace=0.0)
+
+    output_png.parent.mkdir(parents=True, exist_ok=True)
+    fig.savefig(output_png, dpi=dpi)
+    plt.close(fig)
+    return output_png

visualization/geometry.py

@@ -0,0 +1,99 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import matplotlib
+import numpy as np
+from mpl_toolkits.mplot3d.art3d import Poly3DCollection
+
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+
+
+def _axis_limits_from_points(points: np.ndarray) -> tuple[tuple[float, float], tuple[float, float], tuple[float, float]]:
+    x_min, x_max = float(np.min(points[:, 0])), float(np.max(points[:, 0]))
+    y_min, y_max = float(np.min(points[:, 1])), float(np.max(points[:, 1]))
+    z_min, z_max = float(np.min(points[:, 2])), float(np.max(points[:, 2]))
+
+    max_range = max(x_max - x_min, y_max - y_min, z_max - z_min) / 2.0
+    max_range = max(max_range, 1e-6) * 1.08
+
+    x_mid = (x_max + x_min) / 2.0
+    y_mid = (y_max + y_min) / 2.0
+    z_mid = (z_max + z_min) / 2.0
+
+    return (
+        (x_mid - max_range, x_mid + max_range),
+        (y_mid - max_range, y_mid + max_range),
+        (z_mid - max_range, z_mid + max_range),
+    )
+
+
+def _to_vertices(face_v_item: object) -> np.ndarray | None:
+    verts = np.asarray(face_v_item, dtype=float)
+    if verts.ndim == 2 and verts.shape[1] == 3 and len(verts) >= 3:
+        return verts
+
+    flat = verts.reshape(-1)
+    if flat.size >= 9 and flat.size % 3 == 0:
+        shaped = flat.reshape(-1, 3)
+        if len(shaped) >= 3:
+            return shaped
+
+    return None
+
+
+def visualize_geometry(
+    geometry_npz: str | Path,
+    output_png: str | Path,
+    *,
+    elev: float = 45.0,
+    azim: float = 15.0,
+    dpi: int = 300,
+) -> Path:
+    """Render geometry polygons from PACK geometry npz (expects key: face_v)."""
+    geometry_npz = Path(geometry_npz)
+    output_png = Path(output_png)
+
+    with np.load(geometry_npz, allow_pickle=True) as data:
+        if "face_v" not in data:
+            keys = ", ".join(sorted(data.files))
+            raise KeyError(f"Missing key 'face_v' in {geometry_npz}; keys=[{keys}]")
+        face_v = data["face_v"]
+
+    polygons: list[np.ndarray] = []
+    for item in face_v:
+        verts = _to_vertices(item)
+        if verts is not None:
+            polygons.append(verts)
+
+    if not polygons:
+        raise ValueError(f"No valid polygons in {geometry_npz}")
+
+    fig = plt.figure(figsize=(4, 4))
+    ax = fig.add_subplot(111, projection="3d")
+    ax.view_init(elev=elev, azim=azim)
+    fig.subplots_adjust(left=0, right=1, top=1, bottom=0)
+
+    for verts in polygons:
+        collection = Poly3DCollection(
+            [verts],
+            facecolors="#FFFFFF",
+            edgecolors="#4D4D4D",
+            linewidths=0.42,
+            alpha=0.35,
+        )
+        ax.add_collection3d(collection)
+
+    all_points = np.vstack(polygons)
+    x_lim, y_lim, z_lim = _axis_limits_from_points(all_points)
+    ax.set_xlim(*x_lim)
+    ax.set_ylim(*y_lim)
+    ax.set_zlim(*z_lim)
+    ax.set_box_aspect([1, 1, 1])
+    ax.set_axis_off()
+
+    output_png.parent.mkdir(parents=True, exist_ok=True)
+    fig.savefig(output_png, dpi=dpi, bbox_inches="tight", pad_inches=0.04, transparent=True)
+    plt.close(fig)
+    return output_png

visualization/weather.py

@@ -0,0 +1,184 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import matplotlib
+import numpy as np
+
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+
+DEFAULT_WEATHER_COLUMNS = [
+    "dry_bulb",
+    "dew_point",
+    "relative_humidity",
+    "global_horizontal_radiation",
+    "direct_normal_radiation",
+    "diffuse_horizontal_radiation",
+    "wind_speed",
+]
+
+WEATHER_UNITS = {
+    "dry_bulb": "degC",
+    "dew_point": "degC",
+    "relative_humidity": "%",
+    "global_horizontal_radiation": "W/m2",
+    "direct_normal_radiation": "W/m2",
+    "diffuse_horizontal_radiation": "W/m2",
+    "wind_direction": "deg",
+}
+
+FIG_SIZE_IN = 4.2
+AXIS_LABEL_FONT_SIZE = 8
+TICK_LABEL_FONT_SIZE = 8
+INPLOT_LABEL_FONT_SIZE = 10
+
+
+def _to_hourly_feature(values: np.ndarray) -> np.ndarray:
+    arr = np.asarray(values)
+    if arr.ndim != 2:
+        raise ValueError(f"Expected 2D weather matrix, got shape={arr.shape}")
+
+    if arr.shape[0] == 8760:
+        return np.asarray(arr, dtype=float)
+    if arr.shape[1] == 8760:
+        return np.asarray(arr.T, dtype=float)
+
+    raise ValueError(f"Neither axis is 8760 for weather matrix: shape={arr.shape}")
+
+
+def _decode_columns(columns_arr: np.ndarray | None, width: int) -> list[str]:
+    if columns_arr is None:
+        return [f"feature_{i}" for i in range(width)]
+
+    cols = np.asarray(columns_arr).reshape(-1)
+    names = [str(c, "utf-8") if isinstance(c, (bytes, np.bytes_)) else str(c) for c in cols]
+    if len(names) < width:
+        names.extend([f"feature_{i}" for i in range(len(names), width)])
+    return names[:width]
+
+
+def _pick_weather_indices(column_names: list[str]) -> list[int]:
+    lower_to_idx = {name.lower(): idx for idx, name in enumerate(column_names)}
+
+    selected: list[int] = []
+    for name in DEFAULT_WEATHER_COLUMNS:
+        idx = lower_to_idx.get(name.lower())
+        if idx is not None:
+            selected.append(idx)
+
+    if len(selected) < 7:
+        for idx in range(len(column_names)):
+            if idx not in selected:
+                selected.append(idx)
+            if len(selected) == 7:
+                break
+
+    return selected
+
+
+def _time_window(hourly: np.ndarray, start_hour: int, window_hours: int) -> tuple[np.ndarray, int, int]:
+    total = int(hourly.shape[0])
+    if total < 1:
+        raise ValueError("No hourly weather records found")
+
+    start_idx = max(0, min(total - 1, int(start_hour) - 1))
+    window = max(1, int(window_hours))
+    end_idx = min(total, start_idx + window)
+    if end_idx <= start_idx:
+        raise ValueError(f"Invalid weather window: start={start_hour}, hours={window_hours}")
+
+    return hourly[start_idx:end_idx, :], start_idx + 1, end_idx
+
+
+def _major_ticks(length: int) -> list[int]:
+    if length <= 8:
+        return list(range(1, length + 1))
+
+    tick_count = 6
+    ticks = np.linspace(1, length, num=tick_count, dtype=int)
+    uniq = sorted(set(int(t) for t in ticks))
+    if uniq[-1] != length:
+        uniq.append(length)
+    return uniq
+
+
+def _label_with_unit(name: str) -> str:
+    key = name.strip().lower().replace(" ", "_").replace("-", "_")
+    unit = WEATHER_UNITS.get(key)
+    if unit is None:
+        return name
+    return f"{name} ({unit})"
+
+
+def visualize_weather(
+    weather_npz: str | Path,
+    output_png: str | Path,
+    *,
+    start_hour: int = 1,
+    window_hours: int = 24,
+    dpi: int = 220,
+) -> Path:
+    """Plot weather subplots from PACK weather npz (values + columns) in a selected time window."""
+    weather_npz = Path(weather_npz)
+    output_png = Path(output_png)
+
+    with np.load(weather_npz, allow_pickle=True) as data:
+        if "values" not in data:
+            keys = ", ".join(sorted(data.files))
+            raise KeyError(f"Missing key 'values' in {weather_npz}; keys=[{keys}]")
+        values = np.asarray(data["values"], dtype=float)
+        columns = np.asarray(data["columns"], dtype=object) if "columns" in data else None
+
+    hourly = _to_hourly_feature(values)
+    window, window_start, window_end = _time_window(hourly, start_hour=start_hour, window_hours=window_hours)
+    names = _decode_columns(columns, window.shape[1])
+    idx_list = _pick_weather_indices(names)
+    if len(idx_list) == 0:
+        raise ValueError(f"No weather series available in {weather_npz}")
+
+    fig, axes = plt.subplots(
+        len(idx_list),
+        1,
+        figsize=(FIG_SIZE_IN, FIG_SIZE_IN),
+        sharex=True,
+        gridspec_kw={"hspace": 0.0},
+    )
+    if len(idx_list) == 1:
+        axes = [axes]
+
+    x = np.arange(1, window.shape[0] + 1, dtype=int)
+    major_ticks = _major_ticks(window.shape[0])
+    for row_idx, feat_idx in enumerate(idx_list):
+        ax = axes[row_idx]
+        y = window[:, feat_idx]
+        ax.plot(x, y, linewidth=0.9, color="#4C72B0")
+        ax.set_ylabel("")
+        ax.text(
+            0.02,
+            0.86,
+            _label_with_unit(names[feat_idx]),
+            transform=ax.transAxes,
+            ha="left",
+            va="top",
+            rotation=0,
+            fontsize=INPLOT_LABEL_FONT_SIZE,
+            bbox={"facecolor": "white", "alpha": 0.65, "edgecolor": "none", "pad": 1.5},
+        )
+        ax.set_xticks(major_ticks)
+        ax.grid(axis="y", alpha=0.3, linewidth=0.5)
+        ax.grid(axis="x", alpha=0.22, linewidth=0.45)
+        ax.tick_params(axis="both", which="both", labelsize=TICK_LABEL_FONT_SIZE)
+        if row_idx < len(idx_list) - 1:
+            ax.tick_params(axis="x", which="both", labelbottom=False)
+
+    axes[-1].set_xlabel(f"hour index in window ({window_start}-{window_end})", fontsize=AXIS_LABEL_FONT_SIZE)
+    axes[-1].set_xlim(1, window.shape[0] + 0.5)
+    axes[-1].set_xticks(major_ticks)
+    axes[-1].set_xticklabels([str(t) for t in major_ticks], fontsize=TICK_LABEL_FONT_SIZE)
+    fig.subplots_adjust(left=0.18, right=0.94, bottom=0.14, top=0.98, hspace=0.0)
+
+    output_png.parent.mkdir(parents=True, exist_ok=True)
+    fig.savefig(output_png, dpi=dpi)
+    plt.close(fig)
+    return output_png