Repository: vllm-project/aibrix
File: docs/development/app/app.py (Python)

Code before:
        },
        {
            "name": "avg_generation_throughput_toks_per_s",
            "type": "gauge",
            "description": "Average generation throughput in tokens/s.",
            "value": overrides.get("avg_generation_throughput", avg_generation_throughput)
        },
        {
            "name": "gpu_cache_usage_perc",
            "type": "gauge",
            "description": "GPU KV-cache usage. 1 means 100 percent usage.",
            "value": overrides.get(
                "gpu_cache_usage_perc", gpu_cache_usage_perc
            )
        },
    ]

    # Generate all metrics
    metrics_output = "".join(
        generate_counter_gauge_metric(metric["name"], metric["type"], metric["description"], model_name,
                                      metric["value"])
        for metric in simple_metrics
    )

    lora_metrics_output = "".join(
        generate_counter_gauge_metric(metric["name"], metric["type"], metric["description"], "lora-model-1",
                                      metric["value"])
        for metric in simple_metrics
    )

    histogram_metrics = [
        {
            "name": "iteration_tokens_total",
            "type": "histogram",
            "description": "Histogram of number of tokens per engine_step.",
            "buckets": ["1.0", "8.0", "16.0", "32.0", "64.0", "128.0", "256.0",
                        "512.0", "1024.0", "2048.0", "4096.0", "8192.0", "+Inf"]
        },
        {
            "name": "time_to_first_token_seconds",
            "type": "histogram",
            "description": "Histogram of time to first token in seconds.",
            "buckets": ["0.001", "0.005", "0.01", "0.02", "0.04", "0.06",
                        "0.08", "0.1", "0.25", "0.5", "+Inf"]
        },
        {
            "name": "time_per_output_token_seconds",
            "type": "histogram",
            "description": "Histogram of time per output token in seconds.",
            "buckets": ["0.01", "0.025", "0.05", "0.075", "0.1", "0.15",
                        "0.2", "0.3", "0.4", "+Inf"]

Review comment (suggestion):
I think this is due to my careless. `lora-model-1` was for testing but not real production workloads, we should correct it here later

Code after:
            "description": "Number of requests waiting to be processed.",
            "value": overrides.get("waiting", waiting)
        },
        {
            "name": "avg_prompt_throughput_toks_per_s",
            "type": "gauge",
            "description": "Average prefill throughput in tokens/s.",
            "value": overrides.get("avg_prompt_throughput", avg_prompt_throughput)
        },
        {
            "name": "avg_generation_throughput_toks_per_s",
            "type": "gauge",
            "description": "Average generation throughput in tokens/s.",
            "value": overrides.get("avg_generation_throughput", avg_generation_throughput)
        },
        {
            "name": "gpu_cache_usage_perc",
            "type": "gauge",
            "description": "GPU KV-cache usage. 1 means 100 percent usage.",
            "value": overrides.get(
                "gpu_cache_usage_perc", gpu_cache_usage_perc
            )
        },
    ]

    # Generate all metrics
    metrics_output = ""
    for metric in simple_metrics:
        metrics_output += generate_counter_gauge_metric(metric["name"], metric["type"], metric["description"],
                                                        model_name, metric["value"])
        metrics_output += generate_counter_gauge_metric(metric["name"], metric["type"], metric["description"],
                                                        "lora-model-1", metric["value"], help_header=False)

    histogram_metrics = [
        {
            "name": "iteration_tokens_total",
            "type": "histogram",
            "description": "Histogram of number of tokens per engine_step.",
            "buckets": ["1.0", "8.0", "16.0", "32.0", "64.0", "128.0", "256.0",
                        "512.0", "1024.0", "2048.0", "4096.0", "8192.0", "+Inf"]
        },
        {
            "name": "time_to_first_token_seconds",
            "type": "histogram",
            "description": "Histogram of time to first token in seconds.",
            "buckets": ["0.001", "0.005", "0.01", "0.02", "0.04", "0.06",
                        "0.08", "0.1", "0.25", "0.5", "+Inf"]
        },
        {
            "name": "time_per_output_token_seconds",
            "type": "histogram",