yuuko-eth

/

LocateAnything-3B-GGUF

like 31

Update: or one can adapt https://github.com/NVlabs/Eagle/blob/3af39904391929b34073192c90a53f3aa69a324e/Embodied/eaglevl/train/fastseek/draw_marker.py#L4 , e.g. into:

import tkinter as tk
from tkinter import filedialog, messagebox
import requests
import re
import cv2
import numpy as np
from PIL import Image, ImageDraw, ImageTk
import base64
import io

# --- NVIDIA DRAWING PRIMITIVES ---
def scale_bbox(bbox, width, height):
    # bbox in normalized (0-1000) [xmin, ymin, xmax, ymax]
    return (np.array(bbox) / 1000) * np.array([width, height, width, height])

def draw_thick_bbox(draw, image, bbox, color, stroke=20):
    bbox_scaled = scale_bbox(bbox, image.width, image.height)
    extend = stroke * 7 / 8
    # PIL rectangle expects (xmin, ymin, xmax, ymax)
    bbox_out = [bbox_scaled[0] - extend, bbox_scaled[1] - extend, bbox_scaled[2] + extend, bbox_scaled[3] + extend]
    draw.rectangle(tuple(map(int, bbox_out)), outline=color, width=stroke)

# --- APP CONFIGURATION ---
VERSION = "2.0-NVIDIA-Style"
SERVER_URL = "http://127.0.0.1:8080/v1/chat/completions"

class App:
    def __init__(self, root):
        self.root = root
        self.root.title(f"LocateAnything Visualizer (v{VERSION})")
        self.current_pil_image = None
        
        tk.Button(root, text="1. Load Image", command=self.load_image).pack(pady=5)
        tk.Label(root, text="2. Prompt:").pack()
        self.prompt_entry = tk.Entry(root, width=60)
        self.prompt_entry.pack(pady=5)
        tk.Button(root, text="3. Ask", command=self.process_query).pack(pady=5)
        self.canvas = tk.Canvas(root, width=800, height=600)
        self.canvas.pack()

    def load_image(self):
        file_path = filedialog.askopenfilename()
        if not file_path: return
        self.current_pil_image = Image.open(file_path).convert("RGB")
        self.display_image(self.current_pil_image)

    def process_query(self):
        if self.current_pil_image is None:
            messagebox.showwarning("Error", "Load image first.")
            return
            
        text = self.prompt_entry.get()
        buffered = io.BytesIO()
        self.current_pil_image.save(buffered, format="JPEG")
        img_str = base64.b64encode(buffered.getvalue()).decode()
        
        payload = {"messages": [{"role": "user", "content": [
            {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{img_str}"}},
            {"type": "text", "text": text}
        ]}]}
        
        try:
            response = requests.post(SERVER_URL, json=payload).json()
            model_text = response['choices'][0]['message']['content']
        except Exception as e:
            messagebox.showerror("API", str(e))
            return
        
        img_work = self.current_pil_image.copy()
        draw = ImageDraw.Draw(img_work)
        
        parts = re.split(r"<ref>(.*?)</ref>", model_text)
        print(f"Query: {text}. Result: {model_text}") 
        for i in range(1, len(parts), 2):
            label = parts[i]
            for box_str in re.findall(r"<box>(.*?)</box>", parts[i+1]):
                nums = [int(n) for n in re.findall(r"<(\d+)>", box_str)]
                if len(nums) == 4:
                    # Model: xmin, ymin, xmax, ymax
                    draw_thick_bbox(draw, img_work, nums, "red", stroke=5)
                    draw.text((nums[0]/1000*img_work.width, nums[1]/1000*img_work.height), label, fill="red")
        
        self.display_image(img_work)

    def display_image(self, pil_img):
        img_tk = ImageTk.PhotoImage(pil_img.copy().resize((800, 600)))
        self.canvas.create_image(0, 0, anchor=tk.NW, image=img_tk)
        self.canvas.image = img_tk

if __name__ == "__main__":
    root = tk.Tk()
    app = App(root)
    root.mainloop()

shows same thing. (Note: am too lazy to add the pointer version).

Oh. One can also tweak it to work on the sample movie Wukong.mp4:

via (key trick below) this:

def main():
    parser = argparse.ArgumentParser(description=f"AI Video Grounder v{VERSION} - SceneDetect + LocateAnything")
    parser.add_argument("video", help="Path to input video file")
    parser.add_argument("-p", "--prompt", required=True, help="Grounding prompt (e.g., 'Locate the workers')")
    parser.add_argument("-t", "--threshold", type=int, default=27, help="Scene detection threshold (default: 27)")
    parser.add_argument("-n", "--num-images", type=int, default=3, help="Images per scene (default: 3)")
    parser.add_argument("--url", default=DEFAULT_SERVER_URL, help="llama-server API URL")
    
    args = parser.parse_args()
    
    # 1. Create Output Directory
    basename = os.path.basename(args.video)
    name_noext = os.path.splitext(basename)[0]
    output_dir = f"{name_noext}_ai_storyboard"
    if not os.path.exists(output_dir):
        os.makedirs(output_dir)
    
    print(f"--- Starting Pipeline v{VERSION} ---")
    print(f"Video: {args.video}")
    print(f"Output: {output_dir}/")
    
    # 2. Step 1: Algorithmic Scene Slicing (Your Bash Logic)
    print("\n[Step 1/2] Running Scene Detection (PySceneDetect)...")
    scenedetect_cmd = [
        "scenedetect", "-i", args.video,
        "detect-content", "-t", str(args.threshold),
        "save-images", "-n", str(args.num_images), "-o", output_dir
    ]

(the whole code would be too long to paste here. )

#!/usr/bin/env python3
"""VLM VNC Controller — use LocalAnything-3b to point at UI elements via VNC."""

import sys
import re
import shlex
import base64
import subprocess
import tempfile

import requests
from PySide6.QtWidgets import (
    QApplication, QMainWindow, QWidget, QVBoxLayout, QHBoxLayout,
    QCheckBox, QTextEdit, QPushButton, QLabel, QDialog,
    QFormLayout, QLineEdit, QSpinBox, QGroupBox,
)
from PySide6.QtCore import QThread, Signal, Qt, QSettings
from PySide6.QtGui import QPixmap, QKeySequence, QShortcut, QTextCursor


# ── Point / box parsing ─────────────────────────────────────────────
def parse_points(answer: str, image_width: int, image_height: int) -> list[dict]:
    """
    Parse <ref>label</ref><box>...</box> blocks from LocateAnything output.
    Handles both point (<x><y>) and bbox (<x1><y1><x2><y2>) formats.
    Coordinates are on a 0‑1000 scale; rescaled to pixel dimensions.
    For bboxes the centre point is returned.
    """
    points = []
    parts = re.split(r"<ref>(.*?)</ref>", answer)

    for i in range(1, len(parts), 2):
        label = parts[i]
        box_content = parts[i + 1]
        box_chunks = re.findall(r"<box>(.*?)</box>", box_content)

        for box_str in box_chunks:
            nums = [int(n) for n in re.findall(r"<(\d+)>", box_str)]

            if len(nums) == 2:
                cx = int(nums[0] / 1000 * image_width)
                cy = int(nums[1] / 1000 * image_height)
                points.append({"label": label, "x": cx, "y": cy})

            elif len(nums) == 4:
                x_min = int(nums[0] / 1000 * image_width)
                y_min = int(nums[1] / 1000 * image_height)
                x_max = int(nums[2] / 1000 * image_width)
                y_max = int(nums[3] / 1000 * image_height)
                cx = (x_min + x_max) // 2
                cy = (y_min + y_max) // 2
                points.append({
                    "label": label,
                    "x": cx, "y": cy,
                    "x_min": x_min, "y_min": y_min,
                    "x_max": x_max, "y_max": y_max,
                })

    return points


# ── Clickable screenshot label ───────────────────────────────────────
class ScreenshotLabel(QLabel):
    doubleClicked = Signal()

    def mouseDoubleClickEvent(self, event):
        self.doubleClicked.emit()


# ── VLM worker (raw requests, no openai lib) ────────────────────────
class VLMWorker(QThread):
    result_received = Signal(str)
    error_occurred = Signal(str)

    def __init__(self, server_url: str, messages: list, model: str):
        super().__init__()
        self.server_url = server_url
        self.messages = messages
        self.model = model

    def run(self):
        payload = {
            "model": self.model,
            "messages": self.messages,
        }
        try:
            resp = requests.post(self.server_url, json=payload, timeout=500)
            resp.raise_for_status()
            data = resp.json()
            answer = data["choices"][0]["message"]["content"]
            self.result_received.emit(answer)
        except Exception as e:
            self.error_occurred.emit(str(e))


# ── Settings dialog ──────────────────────────────────────────────────
class SettingsDialog(QDialog):
    def __init__(self, settings: QSettings, parent=None):
        super().__init__(parent)
        self.settings = settings
        self.setWindowTitle("Settings")
        self.setMinimumWidth(560)

        layout = QVBoxLayout(self)

        # ── API ──
        api_group = QGroupBox("Server (OpenAI‑style /v1/chat/completions)")
        api_form = QFormLayout()
        self.server_url = QLineEdit()
        self.model = QLineEdit()
        api_form.addRow("Server URL:", self.server_url)
        api_form.addRow("Model:", self.model)
        api_group.setLayout(api_form)
        layout.addWidget(api_group)

        # ── VNC ──
        vnc_group = QGroupBox("VNC Connection")
        vnc_form = QFormLayout()
        self.vnc_host = QLineEdit()
        self.vnc_port = QSpinBox()
        self.vnc_port.setRange(1, 65535)
        self.vnc_port.setValue(5900)
        self.vnc_password = QLineEdit()
        self.vnc_password.setEchoMode(QLineEdit.Password)
        vnc_form.addRow("Host:", self.vnc_host)
        vnc_form.addRow("Port:", self.vnc_port)
        vnc_form.addRow("Password:", self.vnc_password)
        vnc_group.setLayout(vnc_form)
        layout.addWidget(vnc_group)

        # ── Buttons ──
        btn_row = QHBoxLayout()
        save_btn = QPushButton("Save")
        save_btn.clicked.connect(self._save)
        cancel_btn = QPushButton("Cancel")
        cancel_btn.clicked.connect(self.reject)
        btn_row.addWidget(save_btn)
        btn_row.addWidget(cancel_btn)
        layout.addLayout(btn_row)

        self._load()

    def _load(self):
        s = self.settings
        self.server_url.setText(
            s.value("api/server_url", "http://127.0.0.1:8080/v1/chat/completions")
        )
        self.model.setText(s.value("api/model", "LocalAnything-3b"))
        self.vnc_host.setText(s.value("vnc/host", "localhost"))
        self.vnc_port.setValue(int(s.value("vnc/port", 5900)))
        self.vnc_password.setText(s.value("vnc/password", ""))

    def _save(self):
        s = self.settings
        s.setValue("api/server_url", self.server_url.text())
        s.setValue("api/model", self.model.text())
        s.setValue("vnc/host", self.vnc_host.text())
        s.setValue("vnc/port", self.vnc_port.value())
        s.setValue("vnc/password", self.vnc_password.text())
        self.accept()


# ── Main window ──────────────────────────────────────────────────────
class MainWindow(QMainWindow):
    def __init__(self):
        super().__init__()
        self.setWindowTitle("LocalAnything‑3b → VNC Pointer")
        self.setMinimumSize(780, 800)

        self.settings = QSettings("VLMVNC", "Controller")
        self.vlm_worker: VLMWorker | None = None
        self.last_screenshot_path: str | None = None
        self.screenshot_size: tuple[int, int] = (0, 0)
        self.last_b64_jpeg: str | None = None

        self._build_ui()

    # ────────────────────── UI construction ──────────────────────────
    def _build_ui(self):
        central = QWidget()
        self.setCentralWidget(central)
        root = QVBoxLayout(central)

        # top bar
        top = QHBoxLayout()
        self.chk_screenshot = QCheckBox("Attach screenshot")
        self.chk_screenshot.setChecked(True)
        self.btn_screenshot = QPushButton("📷 Screenshot")
        self.btn_screenshot.clicked.connect(self._take_screenshot)
        self.btn_settings = QPushButton("⚙ Settings")
        self.btn_settings.clicked.connect(self._open_settings)
        top.addWidget(self.chk_screenshot)
        top.addWidget(self.btn_screenshot)
        top.addStretch()
        top.addWidget(self.btn_settings)
        root.addLayout(top)

        # screenshot preview
        self.lbl_screenshot = ScreenshotLabel(
            "Double‑click or press 📷 to capture"
        )
        self.lbl_screenshot.setMinimumHeight(180)
        self.lbl_screenshot.setMaximumHeight(280)
        self.lbl_screenshot.setAlignment(Qt.AlignCenter)
        self.lbl_screenshot.setStyleSheet(
            "border:1px solid #666; background:#1e1e1e; color:#777; font-size:13px;"
        )
        self.lbl_screenshot.doubleClicked.connect(self._take_screenshot)
        root.addWidget(self.lbl_screenshot)

        # prompt
        root.addWidget(QLabel("Point to (phrase):"))
        self.txt_prompt = QTextEdit()
        self.txt_prompt.setMaximumHeight(72)
        self.txt_prompt.setPlaceholderText(
            'e.g. "Apple logo", "Start button", "Close icon"'
        )
        root.addWidget(self.txt_prompt)

        # action buttons
        btns = QHBoxLayout()
        self.btn_send = QPushButton("🎯 Locate with VLM")
        self.btn_send.clicked.connect(self._send_to_vlm)
        self.btn_exec = QPushButton("▶ Execute Command")
        self.btn_exec.setEnabled(False)
        self.btn_exec.clicked.connect(self._execute_command)
        btns.addWidget(self.btn_send)
        btns.addWidget(self.btn_exec)
        root.addLayout(btns)

        # log area (read‑only)
        root.addWidget(QLabel("Log:"))
        self.txt_log = QTextEdit()
        self.txt_log.setReadOnly(True)
        self.txt_log.setMaximumHeight(160)
        self.txt_log.setStyleSheet(
            "font-family:Consolas,Monaco,monospace; font-size:12px; color:#aaa;"
        )
        root.addWidget(self.txt_log)

        # editable command area
        root.addWidget(QLabel("VNC Command (edit before executing):"))
        self.txt_response = QTextEdit()
        self.txt_response.setStyleSheet(
            "font-family:Consolas,Monaco,monospace; font-size:13px;"
        )
        root.addWidget(self.txt_response)

        # shortcut Ctrl+Return → locate
        QShortcut(QKeySequence("Ctrl+Return"), self, self._send_to_vlm)

    # ────────────────────── helpers ──────────────────────────────────
    def _vnc_prefix(self) -> list[str]:
        host = self.settings.value("vnc/host", "localhost")
        port = self.settings.value("vnc/port", 5900)
        pw = self.settings.value("vnc/password", "")
        cmd = ["vncdo", "-s", f"{host}::{port}"]
        if pw:
            cmd += ["-p", pw]
        return cmd

    # ────────────────────── screenshot ───────────────────────────────
    def _take_screenshot(self) -> str | None:
        tmp = tempfile.mktemp(suffix=".png")
        cmd = self._vnc_prefix() + ["capture", tmp]
        try:
            r = subprocess.run(cmd, capture_output=True, text=True, timeout=15)
            if r.returncode != 0:
                self._log(f"⚠ Screenshot failed: {r.stderr.strip()}")
                return None
        except FileNotFoundError:
            self._log("⚠ vncdotool not found.  pip install vncdotool")
            return None
        except subprocess.TimeoutExpired:
            self._log("⚠ Screenshot timed out")
            return None

        pix = QPixmap(tmp)
        if pix.isNull():
            self._log("⚠ Could not load screenshot image")
            return None

        self.last_screenshot_path = tmp
        self.screenshot_size = (pix.width(), pix.height())

        # Pre‑encode as JPEG for the VLM (matching the working reference)
        self._encode_screenshot(tmp)

        self._log(f"📷 Screenshot captured: {pix.width()}×{pix.height()}")
        self._show_screenshot(pix)
        return tmp

    def _encode_screenshot(self, path: str):
        """Read image, encode to JPEG base64 — same pipeline as the
        working tkinter reference (cv2.imencode → base64)."""
        try:
            import cv2
            img = cv2.imread(path)
            if img is not None:
                _, encoded = cv2.imencode(".jpg", img)
                self.last_b64_jpeg = base64.b64encode(encoded).decode()
                return
        except ImportError:
            pass

        # Fallback without cv2: use PIL
        try:
            from PIL import Image as PILImage
            img = PILImage.open(path)
            buf = io.BytesIO()
            img.save(buf, format="JPEG")
            self.last_b64_jpeg = base64.b64encode(buf.getvalue()).decode()
            return
        except ImportError:
            pass

        # Last resort: raw PNG bytes
        with open(path, "rb") as f:
            self.last_b64_jpeg = base64.b64encode(f.read()).decode()

    def _show_screenshot(self, pix: QPixmap):
        scaled = pix.scaled(
            self.lbl_screenshot.size(),
            Qt.KeepAspectRatio,
            Qt.SmoothTransformation,
        )
        self.lbl_screenshot.setPixmap(scaled)

    def resizeEvent(self, event):
        super().resizeEvent(event)
        if self.last_screenshot_path:
            pix = QPixmap(self.last_screenshot_path)
            if not pix.isNull():
                self._show_screenshot(pix)

    # ────────────────────── VLM call ─────────────────────────────────
    def _send_to_vlm(self):
        phrase = self.txt_prompt.toPlainText().strip()
        if not phrase:
            return

        self.btn_send.setEnabled(False)
        self.btn_exec.setEnabled(False)
        self.txt_response.clear()

        # Build content: image_url FIRST, then text — matching working reference
        user_content: list[dict] = []

        if self.chk_screenshot.isChecked():
            if not self.last_screenshot_path:
                self._take_screenshot()
            if self.last_b64_jpeg:
                user_content.append({
                    "type": "image_url",
                    "image_url": {
                        "url": f"data:image/jpeg;base64,{self.last_b64_jpeg}"
                    },
                })
            else:
                self._log("⚠ No screenshot — model needs an image")

        user_content.append({
            "type": "text",
            "text": f"Point to: {phrase}."
        })

        # No system prompt — only user message
        messages = [{"role": "user", "content": user_content}]

        server_url = self.settings.value(
            "api/server_url",
            "http://127.0.0.1:8080/v1/chat/completions",
        )
        model = self.settings.value("api/model", "LocalAnything-3b")
        self._log(f"→ POST {server_url}  model={model}")
        self._log(f"   Prompt: Point to: {phrase}.")

        self.vlm_worker = VLMWorker(server_url, messages, model)
        self.vlm_worker.result_received.connect(self._on_result)
        self.vlm_worker.error_occurred.connect(self._on_vlm_error)
        self.vlm_worker.start()

    def _on_result(self, answer: str):
        self._log(f"← Raw model output: {answer}")

        img_w, img_h = self.screenshot_size
        if img_w == 0 or img_h == 0:
            self._log("⚠ Unknown screenshot dimensions, assuming 1920×1080")
            img_w, img_h = 1920, 1080

        points = parse_points(answer, img_w, img_h)

        if points:
            lines = []
            for i, pt in enumerate(points):
                x, y = pt["x"], pt["y"]
                label = pt.get("label", f"point{i}")
                line = f"move {x} {y}"
                lines.append(line)
                if "x_min" in pt:
                    self._log(
                        f"  [{label}] bbox({pt['x_min']},{pt['y_min']})-"
                        f"({pt['x_max']},{pt['y_max']}) → centre ({x}, {y})"
                    )
                else:
                    self._log(f"  [{label}] point ({x}, {y})")
            cmd_text = "\n".join(lines)
        else:
            self._log("⚠ No <box> coordinates found in model output")
            cmd_text = f"# No coordinates found\n# Raw: {answer}"

        self.txt_response.setPlainText(cmd_text)
        self.btn_send.setEnabled(True)
        self.btn_exec.setEnabled(True)

    def _on_vlm_error(self, err: str):
        self.txt_response.setPlainText(f"# Error: {err}")
        self._log(f"❌ Error: {err}")
        self.btn_send.setEnabled(True)

    # ────────────────────── execute vncdotool ────────────────────────
    def _execute_command(self):
        text = self.txt_response.toPlainText()
        lines = [
            l.strip()
            for l in text.strip().splitlines()
            if l.strip() and not l.strip().startswith("#")
        ]
        if not lines:
            self._log("Nothing to execute")
            return

        self._log("\n── Executing ──")
        for line in lines:
            try:
                parts = shlex.split(line)
            except ValueError as e:
                self._log(f"  ✗ {line}  (parse error: {e})")
                continue

            cmd = self._vnc_prefix() + parts
            try:
                r = subprocess.run(
                    cmd, capture_output=True, text=True, timeout=15
                )
                if r.returncode == 0:
                    self._log(f"  ✓ {line}")
                else:
                    self._log(f"  ✗ {line}  — {r.stderr.strip()}")
            except subprocess.TimeoutExpired:
                self._log(f"  ✗ {line}  — timed out")
            except Exception as e:
                self._log(f"  ✗ {line}  — {e}")
        self._log("── Done ──\n")

    # ────────────────────── settings ─────────────────────────────────
    def _open_settings(self):
        dlg = SettingsDialog(self.settings, self)
        if dlg.exec() == QDialog.Accepted:
            pass  # URL read fresh from QSettings each call

    # ────────────────────── logging ──────────────────────────────────
    def _log(self, msg: str):
        self.txt_log.append(msg)
        cur = self.txt_log.textCursor()
        cur.movePosition(QTextCursor.MoveOperation.End)
        self.txt_log.setTextCursor(cur)


# ── Entry point ──────────────────────────────────────────────────────
if __name__ == "__main__":
    app = QApplication(sys.argv)
    win = MainWindow()
    win.show()
    sys.exit(app.exec())

Hm. Works kind of, after installing pip install vncdotool service_identity, but what do we need it for?

@Manamama i think @barinov274 is demonstrating how the concept can transfer to controlling a computer over VNC (Remote Desktop)
Pretty lovely idea!

Ok, after sleeping over it and consulting with Grok xAI, below, I get it now:

Yeah, the Tkinter visualizer is great for testing and seeing what the model can do, but I initially had the same reaction — “why not just click things myself?”

The real value (and what the VNC controller code demonstrates) is closing the loop for full automation.

What the VNC code actually does:

It turns LocateAnything into a vision-based GUI agent:

1. Takes a screenshot of the remote/local desktop via VNC.
2. Sends the screenshot + a natural language instruction to the model (e.g. “click the Login button”, “type into the search field”, “find and double-click the Excel icon”).
3. The model returns precise coordinates (<ref>...</ref><box>...</box> or point format).
4. The script uses vncdotool to automatically move the mouse and click / type / drag at those exact coordinates.

No brittle pixel matching, no reliance on accessibility APIs, no manual scripting of every UI change — it works on any GUI.

Practical uses:

• Robotic Process Automation (RPA): Automate repetitive desktop tasks (filling forms, downloading reports, navigating internal tools).
• Headless / remote agents: Control VMs, servers, or cloud desktops without a human watching.
• GUI testing: Reliably locate and interact with elements even if the interface changes (theme, resolution, layout).
• Full AI agents: Combine with a reasoning LLM for multi-step tasks (“log into the portal, find the latest invoice, download it, then email it”).
• Robotics / embodied AI: Similar grounding on camera feeds.
• Batch processing or accessibility tools.

In short: the visualizer is for humans exploring/debugging. The VNC part shows how to make the model act on what it sees — a step toward open, local “computer use” agents like the ones big companies are demoing.

The author shared a working controller but didn’t explain the “why”, which made it confusing at first. Once you see it as the bridge from “understand screenshot” to “control computer”, it clicks.

So like mine https://huggingface.co/yuuko-eth/LocateAnything-3B-GGUF/discussions/3#6a25810e84627862175c0d29 plus VNC