Repository: vllm-project/aibrix
File: pkg/plugins/gateway/cache/lru_store.go (Go)

Code before:

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cache

import (
	"sync"
	"time"
)

type LRUStore[K comparable, V any] struct {
	sync.Mutex
	freeTable map[K]*entry[K, V]
	lruList   *list[K, V]
	cap       int

	interval time.Duration
	ttl      time.Duration
}

func NewLRUStore[K comparable, V any](cap int, ttl, interval time.Duration) *LRUStore[K, V] {
	store := &LRUStore[K, V]{
		freeTable: make(map[K]*entry[K, V]),
		lruList:   &list[K, V]{head: &entry[K, V]{}, tail: &entry[K, V]{}},
		cap:       cap,
		ttl:       ttl,
		interval:  interval,
	}
	store.lruList.head.next = store.lruList.tail
	store.lruList.tail.prev = store.lruList.head

	go store.startEviction()
	return store
}

func (e *LRUStore[K, V]) startEviction() {
	ticker := time.NewTicker(e.interval)
	defer ticker.Stop()
	for range ticker.C {
		e.evict(time.Now())
	}
}

func (e *LRUStore[K, V]) Put(key K, value V) bool {

Review comment (suggestion):
Right now the structure is as follows

hash1 -> model1 -> pod1  + pod1 last access time 
           -> model1 -> pod2  + pod2 last access time
           -> model2 -> pod3 + pod3 last access time
           -> blocks last access time 

Right now eviction is based of last blocks access time. I kept the eviction like this for now to keep it simple. Eventually we want to move to only delete pod1 + its last access time rather than restricting to only delete entire block.

Another requirement is that, for one request it creates 100 hashes. How to make sure all hashes are deleted for that request together. Thats how vllm implements block eviction.




Code after:

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cache

import (
	"sync"
	"time"
)

type getCurrentTime func() time.Time

var DefaultGetCurrentTime = func() time.Time {
	return time.Now()
}

type LRUStore[K comparable, V any] struct {
	sync.RWMutex
	freeTable map[K]*entry[K, V]
	lruList   *list[K, V]
	cap       int

	getCurrentTime
	interval time.Duration
	ttl      time.Duration
}

func NewLRUStore[K comparable, V any](cap int, ttl, interval time.Duration, f getCurrentTime) *LRUStore[K, V] {
	store := &LRUStore[K, V]{
		freeTable:      make(map[K]*entry[K, V]),
		lruList:        &list[K, V]{head: &entry[K, V]{}, tail: &entry[K, V]{}},
		cap:            cap,
		ttl:            ttl,
		interval:       interval,
		getCurrentTime: f,
	}
	store.lruList.head.next = store.lruList.tail
	store.lruList.tail.prev = store.lruList.head

	go store.startEviction()
	return store
}

func (e *LRUStore[K, V]) startEviction() {