Delete run_scorecard.py

run_scorecard.py

@@ -1,375 +0,0 @@
-# ---------------------------------------------------------------------
-# Copyright (c) 2025 Qualcomm Technologies, Inc. and/or its subsidiaries.
-# SPDX-License-Identifier: BSD-3-Clause
-# ---------------------------------------------------------------------
-"""
-Scorecard benchmark script for llama.cpp on Linux IoT devices via SSH.
-
-This script runs comprehensive benchmarks:
-  1. Performance benchmarks (CPU/GPU/HTP x 3 context lengths)
-  2. Fallback ops detection (SCHED=1)
-  3. Perplexity (WikiText-2)
-
-Placeholders are replaced at artifact creation time:
-  - <<MODEL_URL>>: URL to download the model
-  - <<NUM_HTPS>>: Number of HTP cores to use
-"""
-
-import os
-import subprocess
-import sys
-
-import pytest
-from appium import webdriver
-from appium.options.common import AppiumOptions
-
-options = AppiumOptions()
-options.set_capability("automationName", "QDCLinux")
-options.set_capability("platformName", "Linux")
-options.set_capability("deviceName", os.getenv("QDC_DEVICE_NAME", "QCS9075M"))
-
-# Context lengths to benchmark
-CONTEXT_LENGTHS = [128, 1024, 4096]
-
-# System prompt for completion benchmarks
-SYSTEM_PROMPT = "You are a helpful assistant. Be helpful but brief."
-
-# WikiText-2 URL for perplexity
-WIKITEXT_URL = "https://huggingface.co/datasets/ggml-org/ci/resolve/main/wikitext-2-raw/wiki.test.raw"
-
-
-class TestScorecard:
-    @pytest.fixture
-    def driver(self) -> webdriver.Remote:
-        return webdriver.Remote(
-            command_executor="http://127.0.0.1:4723/wd/hub", options=options
-        )
-
-    def test_scorecard(self, driver: webdriver.Remote) -> None:
-        """Run comprehensive llama.cpp scorecard benchmarks."""
-        model_url = "<<MODEL_URL>>"
-        num_htps = "<<NUM_HTPS>>"
-
-        # On-device paths (Linux IoT - using /tmp)
-        basedir = "/tmp/llama_cpp_bundle"
-        model_path = "/tmp/gguf/model.gguf"
-        log_file = "/tmp/QDC_logs/scorecard.log"
-
-        scorecard_script = f"""
-cd /tmp/llama_cpp_bundle
-
-# Set library paths
-export LD_LIBRARY_PATH=/tmp/llama_cpp_bundle/lib:$LD_LIBRARY_PATH
-export ADSP_LIBRARY_PATH="/tmp/llama_cpp_bundle/lib:/system/lib/rfsa/adsp:/system/vendor/lib/rfsa/adsp:/dsp"
-
-# Make binaries executable
-chmod +x /tmp/llama_cpp_bundle/bin/*
-
-# Setup paths
-BASEDIR=/tmp/llama_cpp_bundle
-MODEL={model_path}
-LOG_FILE={log_file}
-NUM_HTPS={num_htps}
-
-# Create directories
-mkdir -p /tmp/gguf
-mkdir -p /tmp/QDC_logs
-
-# Download model
-echo "Downloading model from {model_url}..."
-curl -L -J --output $MODEL "{model_url}"
-
-# Initialize log
-echo "============================================================" > $LOG_FILE
-echo "LLAMA.CPP SCORECARD (Linux)" >> $LOG_FILE
-echo "Date: $(date)" >> $LOG_FILE
-echo "Model: {model_url}" >> $LOG_FILE
-echo "============================================================" >> $LOG_FILE
-
-#############################################
-# SECTION 1: PERFORMANCE BENCHMARKS
-#############################################
-echo "" >> $LOG_FILE
-echo "########################################################" >> $LOG_FILE
-echo "# SECTION 1: PERFORMANCE BENCHMARKS" >> $LOG_FILE
-echo "########################################################" >> $LOG_FILE
-
-# Common HTP flags
-HTP_FLAGS="--no-mmap --poll 1000 -t 6 --cpu-mask 0xfc --cpu-strict 1 -fa on -ngl 99"
-
-# Run benchmarks for each compute unit and context length
-for COMPUTE in CPU GPU HTP; do
-    echo "" >> $LOG_FILE
-    echo "--- COMPUTE: $COMPUTE ---" >> $LOG_FILE
-
-    for CTX_LEN in 128 1024 4096; do
-        echo "" >> $LOG_FILE
-        echo "=== $COMPUTE | CTX=$CTX_LEN ===" >> $LOG_FILE
-
-        # Select prompt file
-        PROMPT_FILE="/tmp/llama_cpp_bundle/sample_prompt_${{CTX_LEN}}.txt"
-
-        if [ "$COMPUTE" = "CPU" ]; then
-            CMD="GGML_HEXAGON_NDEV=0 $BASEDIR/bin/llama-completion --model $MODEL --n-predict -1 --ctx-size $CTX_LEN --system-prompt \\"{SYSTEM_PROMPT}\\" --file $PROMPT_FILE --seed 1 --single-turn --no-display-prompt --n-gpu-layers 0"
-            echo "COMMAND: $CMD" >> $LOG_FILE
-            GGML_HEXAGON_NDEV=0 $BASEDIR/bin/llama-completion \\
-                --model $MODEL \\
-                --n-predict -1 \\
-                --ctx-size $CTX_LEN \\
-                --system-prompt "{SYSTEM_PROMPT}" \\
-                --file "$PROMPT_FILE" \\
-                --seed 1 \\
-                --single-turn \\
-                --no-display-prompt \\
-                --n-gpu-layers 0 \\
-                2>&1 | tee -a $LOG_FILE
-
-        elif [ "$COMPUTE" = "GPU" ]; then
-            CMD="GGML_HEXAGON_NDEV=0 $BASEDIR/bin/llama-completion --model $MODEL --n-predict -1 --ctx-size $CTX_LEN --system-prompt \\"{SYSTEM_PROMPT}\\" --file $PROMPT_FILE --seed 1 --single-turn --no-display-prompt -fa off"
-            echo "COMMAND: $CMD" >> $LOG_FILE
-            GGML_HEXAGON_NDEV=0 $BASEDIR/bin/llama-completion \\
-                --model $MODEL \\
-                --n-predict -1 \\
-                --ctx-size $CTX_LEN \\
-                --system-prompt "{SYSTEM_PROMPT}" \\
-                --file "$PROMPT_FILE" \\
-                --seed 1 \\
-                --single-turn \\
-                --no-display-prompt \\
-                -fa off \\
-                2>&1 | tee -a $LOG_FILE
-
-        elif [ "$COMPUTE" = "HTP" ]; then
-            CMD="GGML_HEXAGON_NDEV=$NUM_HTPS $BASEDIR/bin/llama-completion --model $MODEL --n-predict -1 --ctx-size $CTX_LEN --system-prompt \\"{SYSTEM_PROMPT}\\" --file $PROMPT_FILE --seed 1 --single-turn --no-display-prompt $HTP_FLAGS --device HTP0 -ctk f16 -ctv f16 --batch-size 128"
-            echo "COMMAND: $CMD" >> $LOG_FILE
-            GGML_HEXAGON_NDEV=$NUM_HTPS $BASEDIR/bin/llama-completion \\
-                --model $MODEL \\
-                --n-predict -1 \\
-                --ctx-size $CTX_LEN \\
-                --system-prompt "{SYSTEM_PROMPT}" \\
-                --file "$PROMPT_FILE" \\
-                --seed 1 \\
-                --single-turn \\
-                --no-display-prompt \\
-                $HTP_FLAGS \\
-                --device HTP0 \\
-                -ctk f16 \\
-                -ctv f16 \\
-                --batch-size 128 \\
-                2>&1 | tee -a $LOG_FILE
-        fi
-    done
-done
-
-#############################################
-# SECTION 2: FALLBACK OPS DETECTION (SCHED=1)
-#############################################
-echo "" >> $LOG_FILE
-echo "########################################################" >> $LOG_FILE
-echo "# SECTION 2: FALLBACK OPS (GGML_SCHED_DEBUG=2)" >> $LOG_FILE
-echo "########################################################" >> $LOG_FILE
-
-# GPU fallback ops
-echo "" >> $LOG_FILE
-echo "=== FALLBACK_OPS | GPU ===" >> $LOG_FILE
-echo "Running with GGML_SCHED_DEBUG=2 on GPU..." >> $LOG_FILE
-
-GGML_SCHED_DEBUG=2 GGML_HEXAGON_NDEV=0 $BASEDIR/bin/llama-completion \\
-    --model $MODEL \\
-    --n-predict 64 \\
-    --ctx-size 128 \\
-    -p "Hello world" \\
-    --seed 1 \\
-    --single-turn \\
-    --no-display-prompt \\
-    -fa off \\
-    -v \\
-    2>&1 | tee -a $LOG_FILE
-
-# HTP fallback ops
-echo "" >> $LOG_FILE
-echo "=== FALLBACK_OPS | HTP ===" >> $LOG_FILE
-echo "Running with GGML_SCHED_DEBUG=2 on HTP..." >> $LOG_FILE
-
-GGML_SCHED_DEBUG=2 GGML_HEXAGON_NDEV=$NUM_HTPS $BASEDIR/bin/llama-completion \\
-    --model $MODEL \\
-    --n-predict 64 \\
-    --ctx-size 128 \\
-    -p "Hello world" \\
-    --seed 1 \\
-    --single-turn \\
-    --no-display-prompt \\
-    $HTP_FLAGS \\
-    --device HTP0 \\
-    -ctk f16 \\
-    -ctv f16 \\
-    --batch-size 128 \\
-    -v \\
-    2>&1 | tee -a $LOG_FILE
-
-#############################################
-# SECTION 3: PERPLEXITY (WikiText-2)
-#############################################
-echo "" >> $LOG_FILE
-echo "########################################################" >> $LOG_FILE
-echo "# SECTION 3: PERPLEXITY (WikiText-2)" >> $LOG_FILE
-echo "########################################################" >> $LOG_FILE
-
-echo "" >> $LOG_FILE
-echo "Downloading WikiText-2 dataset..." >> $LOG_FILE
-curl -L -o /tmp/wiki.test.raw "{WIKITEXT_URL}"
-
-echo "" >> $LOG_FILE
-echo "Running perplexity on HTP..." >> $LOG_FILE
-echo "COMMAND: GGML_HEXAGON_NDEV=$NUM_HTPS llama-perplexity -m $MODEL -f wiki.test.raw --device HTP0 --chunks 10" >> $LOG_FILE
-
-GGML_HEXAGON_NDEV=$NUM_HTPS $BASEDIR/bin/llama-perplexity \\
-    -m $MODEL \\
-    -f /tmp/wiki.test.raw \\
-    --device HTP0 \\
-    --chunks 10 \\
-    $HTP_FLAGS \\
-    2>&1 | tee -a $LOG_FILE
-
-#############################################
-# SECTION 4: QUALITY CHECKS (Q&A Validation)
-#############################################
-echo "" >> $LOG_FILE
-echo "########################################################" >> $LOG_FILE
-echo "# SECTION 4: QUALITY CHECKS" >> $LOG_FILE
-echo "########################################################" >> $LOG_FILE
-
-# Run simple Q&A tests on GPU and HTP and check for expected answers
-# Format: QUALITY_CHECK: DEVICE | question | expected | PASS/FAIL
-
-run_quality_check() {{
-    local DEVICE="$1"
-    local QUESTION="$2"
-    local EXPECTED="$3"
-
-    echo "" >> $LOG_FILE
-    echo "--- Quality Check ($DEVICE) ---" >> $LOG_FILE
-    echo "Question: $QUESTION" >> $LOG_FILE
-    echo "Expected to contain: $EXPECTED" >> $LOG_FILE
-
-    # Filter: strip loading spinners, progress bars, and non-printable characters
-    FILTER='grep -v -E "^(Loading|\\||/|\\\\|-|\\[|model|warning|log|ggml|llama)" | sed "s/[^[:print:][:space:]]//g" | sed "/^[[:space:]]*$/d" | head -20'
-
-    if [ "$DEVICE" = "CPU" ]; then
-        RAW_RESPONSE=$(GGML_HEXAGON_NDEV=0 $BASEDIR/bin/llama-cli \\
-            --model $MODEL \\
-            --system-prompt "{SYSTEM_PROMPT}" \\
-            -p "$QUESTION" \\
-            --n-predict 64 \\
-            --ctx-size 512 \\
-            --seed 1 \\
-            --no-display-prompt \\
-            --n-gpu-layers 0 \\
-            2>/dev/null)
-
-    elif [ "$DEVICE" = "GPU" ]; then
-        RAW_RESPONSE=$(GGML_HEXAGON_NDEV=0 $BASEDIR/bin/llama-cli \\
-            --model $MODEL \\
-            --system-prompt "{SYSTEM_PROMPT}" \\
-            -p "$QUESTION" \\
-            --n-predict 64 \\
-            --ctx-size 512 \\
-            --seed 1 \\
-            --no-display-prompt \\
-            -fa off \\
-            2>/dev/null)
-
-    elif [ "$DEVICE" = "HTP" ]; then
-        RAW_RESPONSE=$(GGML_HEXAGON_NDEV=$NUM_HTPS $BASEDIR/bin/llama-cli \\
-            --model $MODEL \\
-            --system-prompt "{SYSTEM_PROMPT}" \\
-            -p "$QUESTION" \\
-            --n-predict 64 \\
-            --ctx-size 512 \\
-            --seed 1 \\
-            --no-display-prompt \\
-            $HTP_FLAGS \\
-            --device HTP0 \\
-            -ctk f16 \\
-            -ctv f16 \\
-            --batch-size 128 \\
-            2>/dev/null)
-    fi
-
-    RESPONSE=$(echo "$RAW_RESPONSE" | eval $FILTER)
-
-    echo "RESPONSE_START" >> $LOG_FILE
-    echo "$RESPONSE" >> $LOG_FILE
-    echo "RESPONSE_END" >> $LOG_FILE
-
-    # Check if expected string is in response (case-insensitive)
-    if echo "$RESPONSE" | grep -qi "$EXPECTED"; then
-        echo "QUALITY_CHECK: $DEVICE | $QUESTION | $EXPECTED | PASS" >> $LOG_FILE
-    else
-        echo "QUALITY_CHECK: $DEVICE | $QUESTION | $EXPECTED | FAIL" >> $LOG_FILE
-    fi
-}}
-
-# CPU Quality Checks
-echo "" >> $LOG_FILE
-echo "=== QUALITY_CHECKS | CPU ===" >> $LOG_FILE
-
-run_quality_check CPU "What is the capital of France?" "Paris"
-run_quality_check CPU "What is 2 + 2?" "4"
-run_quality_check CPU "What planet is closest to the Sun?" "Mercury"
-
-# GPU Quality Checks
-echo "" >> $LOG_FILE
-echo "=== QUALITY_CHECKS | GPU ===" >> $LOG_FILE
-
-run_quality_check GPU "What is the capital of France?" "Paris"
-run_quality_check GPU "What is 2 + 2?" "4"
-run_quality_check GPU "What planet is closest to the Sun?" "Mercury"
-
-# HTP Quality Checks
-echo "" >> $LOG_FILE
-echo "=== QUALITY_CHECKS | HTP ===" >> $LOG_FILE
-
-run_quality_check HTP "What is the capital of France?" "Paris"
-run_quality_check HTP "What is 2 + 2?" "4"
-run_quality_check HTP "What planet is closest to the Sun?" "Mercury"
-
-#############################################
-# COMPLETE
-#############################################
-echo "" >> $LOG_FILE
-echo "============================================================" >> $LOG_FILE
-echo "=== SCORECARD COMPLETE ===" >> $LOG_FILE
-echo "============================================================" >> $LOG_FILE
-"""
-
-        # Push the bundle to the device via SSH/SCP
-        device_host = os.getenv("QDC_DEVICE_HOST", "localhost")
-        subprocess.run(
-            ["scp", "-r", "/qdc/appium/llama_cpp_bundle/", f"{device_host}:/tmp/"],
-            capture_output=True,
-            encoding="utf-8",
-            errors="replace",
-            check=True,
-        )
-
-        # Run the scorecard script via SSH
-        result = subprocess.run(
-            [
-                "ssh",
-                device_host,
-                "sh",
-                "-c",
-                scorecard_script,
-            ],
-            capture_output=True,
-            encoding="utf-8",
-            errors="replace",
-            check=True,
-        )
-        print(result.stdout)
-        print(result.stderr)
-
-
-if __name__ == "__main__":
-    sys.exit(pytest.main(["-s", "--junitxml=results.xml", os.path.realpath(__file__)]))