Repository: vllm-project/aibrix
File: pkg/plugins/gateway/prefixcacheindexer/hash.go (Go)

Code before:
	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package prefixcacheindexer

import (
	"math/rand"
	"strconv"
	"sync"
	"time"

	"github.com/cespare/xxhash/v2"
	"github.com/vllm-project/aibrix/pkg/plugins/gateway/cache"
	"github.com/vllm-project/aibrix/pkg/utils"
	v1 "k8s.io/api/core/v1"
	"k8s.io/klog/v2"
)

const (
	defaultPrefixCacheBlockNumber            = 100
	defaultPrefixCacheBlockSize              = 16
	defaultPrefixCacheEvictionInternalInMS   = 1000 // 1 second
	defaultPrefixCacheEvictionDurationInMins = 20   // 20 minutes
)

var (
	// TODO: add a helper function for get methods.
	prefixCacheBlockNumber      = getPrefixCacheBlockNumber()
	prefixCacheBlockSize        = getPrefixCacheBlockSize()
	prefixCacheEvictionInterval = getPrefixCacheEvictionInterval()
	prefixCacheEvictionDuration = getPrefixCacheEvictionDuration()
)

func getPrefixCacheBlockNumber() int {
	value := utils.LoadEnv("AIBRIX_PREFIX_CACHE_BLOCK_NUMBER", "")
	if value != "" {
		intValue, err := strconv.Atoi(value)
		if err != nil || intValue <= 0 {
			klog.Infof("invalid AIBRIX_PREFIX_CACHE_BLOCK_NUMBER: %s, falling back to default", value)
		} else {
			klog.Infof("using AIBRIX_PREFIX_CACHE_BLOCK_NUMBER env value for prefix cache block number: %d", intValue)
			return intValue
		}
	}
	klog.Infof("using default prefix cache block number: %d", defaultPrefixCacheBlockNumber)

Review comment (performance):
this value determines the size of the cache, I feel 100 is too small. We need to change default one to large values.  Later, the prefix will be changed to use character instead of word or tokens, results in more blocks.  20000 token request end up with 1250 blocks. we can roughly calculate the memory size (like 1GiB) and see how much block it can persist. 

More intelligent way is to intelligent check container memory limits, it can use injected envs to decide removeTail() but not just based on cap. htis can be left as separate task. we need to have a reasonable default value first.

```
  - name: MY_MEM_LIMIT
    valueFrom:
      resourceFieldRef:
        resource: limits.memory
```

Code after:
	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package prefixcacheindexer

import (
	"math/rand"
	"strconv"
	"sync"
	"time"

	"github.com/cespare/xxhash/v2"
	"github.com/vllm-project/aibrix/pkg/plugins/gateway/cache"
	"github.com/vllm-project/aibrix/pkg/utils"
	v1 "k8s.io/api/core/v1"
	"k8s.io/klog/v2"
)

const (
	defaultPrefixCacheBlockNumber            = 200000
	defaultPrefixCacheBlockSize              = 16
	defaultPrefixCacheEvictionInternalInSec  = 1  // 1 second
	defaultPrefixCacheEvictionDurationInMins = 20 // 20 minutes
)

var (
	// TODO: add a helper function for get methods.
	prefixCacheBlockNumber      = getPrefixCacheBlockNumber()
	prefixCacheBlockSize        = getPrefixCacheBlockSize()
	prefixCacheEvictionInterval = getPrefixCacheEvictionInterval()
	prefixCacheEvictionDuration = getPrefixCacheEvictionDuration()
)

func getPrefixCacheBlockNumber() int {
	value := utils.LoadEnv("AIBRIX_PREFIX_CACHE_BLOCK_NUMBER", "")
	if value != "" {
		intValue, err := strconv.Atoi(value)
		if err != nil || intValue <= 0 {
			klog.Infof("invalid AIBRIX_PREFIX_CACHE_BLOCK_NUMBER: %s, falling back to default", value)
		} else {
			klog.Infof("using AIBRIX_PREFIX_CACHE_BLOCK_NUMBER env value for prefix cache block number: %d", intValue)
			return intValue
		}
	}
	klog.Infof("using default prefix cache block number: %d", defaultPrefixCacheBlockNumber)