Repository: vllm-project/aibrix
File: pkg/cache/cache.go (Go)

Code before:
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/cache"
	"k8s.io/klog/v2"

	modelv1alpha1 "github.com/aibrix/aibrix/api/model/v1alpha1"
	v1alpha1 "github.com/aibrix/aibrix/pkg/client/clientset/versioned"
	v1alpha1scheme "github.com/aibrix/aibrix/pkg/client/clientset/versioned/scheme"
	"github.com/aibrix/aibrix/pkg/metrics"
	prometheusv1 "github.com/prometheus/client_golang/api/prometheus/v1"
	"k8s.io/client-go/kubernetes/scheme"
)

var once sync.Once

// type global
type Cache struct {
	mu                sync.RWMutex
	redisClient       *redis.Client
	prometheusApi     prometheusv1.API
	initialized       bool
	subscribers       []metrics.MetricSubscriber
	metrics           map[string]interface{}
	ModelMetrics      map[string]map[string]interface{}
	Pods              map[string]*v1.Pod
	PodMetrics        map[string]map[string]metrics.MetricValue // pod_name: map[metric_name]metric_val
	PodToModelMapping map[string]map[string]struct{}            // pod_name: map[model_name]struct{}
	ModelToPodMapping map[string]map[string]*v1.Pod             // model_name: map[pod_name]*v1.Pod
	requestTrace      map[string]map[string]int                 // model_name: map[Log2(input_token)-Log2(output_token)]request_count
}

const (
	modelIdentifier                       = "model.aibrix.ai/name"
	podPort                               = 8000
	defaultPodMetricRefreshIntervalInMS   = 50
	expireWriteRequestTraceIntervalInMins = 10
	keyWriteRequestTraceIntervalInSeconds = "meta_interval_sec"
	writeRequestTraceIntervalInSeconds    = 10
	keyPrecisionRequestTrace              = "meta_precision"
	precisionRequestTrace                 = 0.1
	keyVersionRequestTrace                = "meta_v"
	versionRequestTrace                   = 2
)

var (
	instance                Cache
	counterGaugeMetricNames = []string{
		metrics.NumRequestsRunning,
		metrics.NumRequestsWaiting,
		metrics.NumRequestsSwapped,
		metrics.AvgPromptThroughputToksPerS,
		metrics.AvgGenerationThroughputToksPerS,

Review comment (suggestion):
Seems `PodModelMetrics` can not be replaced by `PodMetrics` and `ModelToPodMapping` together. right? we need another dimension or hierarchy to support lora case

Code after:
	"k8s.io/client-go/tools/cache"
	"k8s.io/klog/v2"

	modelv1alpha1 "github.com/aibrix/aibrix/api/model/v1alpha1"
	v1alpha1 "github.com/aibrix/aibrix/pkg/client/clientset/versioned"
	v1alpha1scheme "github.com/aibrix/aibrix/pkg/client/clientset/versioned/scheme"
	"github.com/aibrix/aibrix/pkg/metrics"
	prometheusv1 "github.com/prometheus/client_golang/api/prometheus/v1"
	"github.com/prometheus/common/expfmt"
	"k8s.io/client-go/kubernetes/scheme"
)

var once sync.Once

// type global
type Cache struct {
	mu                sync.RWMutex
	redisClient       *redis.Client
	prometheusApi     prometheusv1.API
	initialized       bool
	subscribers       []metrics.MetricSubscriber
	metrics           map[string]interface{}
	ModelMetrics      map[string]map[string]interface{}
	Pods              map[string]*v1.Pod
	PodMetrics        map[string]map[string]metrics.MetricValue            // pod_name: map[metric_name]metric_val
	PodModelMetrics   map[string]map[string]map[string]metrics.MetricValue // pod_name: map[model_name]map[metric_name]metric_val
	PodToModelMapping map[string]map[string]struct{}                       // pod_name: map[model_name]struct{}
	ModelToPodMapping map[string]map[string]*v1.Pod                        // model_name: map[pod_name]*v1.Pod
	requestTrace      map[string]map[string]int                            // model_name: map[Log2(input_token)-Log2(output_token)]request_count
}

const (
	modelIdentifier                       = "model.aibrix.ai/name"
	podPort                               = 8000
	defaultPodMetricRefreshIntervalInMS   = 50
	expireWriteRequestTraceIntervalInMins = 10
	keyWriteRequestTraceIntervalInSeconds = "meta_interval_sec"
	writeRequestTraceIntervalInSeconds    = 10
	keyPrecisionRequestTrace              = "meta_precision"
	precisionRequestTrace                 = 0.1
	keyVersionRequestTrace                = "meta_v"
	versionRequestTrace                   = 2
)

var (
	instance                Cache
	counterGaugeMetricNames = []string{
		metrics.NumRequestsRunning,
		metrics.NumRequestsWaiting,
		metrics.NumRequestsSwapped,
		metrics.AvgPromptThroughputToksPerS,