Change:
Fix this TODO (raised by : Part of: One thoughts to fix this is, we can redesign the equivalence cache data structure to: bool instead of deleting the entry. certain version tag in the entry, update the version per WRITE op (Update and Invalidate). the version after eCache predicate, and check it before update eCache.
Another possible approach is: eCache before running a predicate. the predicate. the eCache for the predicate. eCache. Basically, in the existing code all predicates are run first and their results are collected and then at the end of the function the eCache is updated. Instead of that, we can move the eCache update for each predicate inside the for loop and next to each predicate execution and synchronize before start of the execution of each predicate. The downside to my suggested approach is that performance may suffer due to frequent locking and unlocking.
Prefer to bsalamat@ 's option which is simpler :). Comparing to performance improvement by eCache, the lock is acceptable to me.
yup same here, going with the locking is easier and is acceptable. - also using a flag for invalidation might just move the problem from eCache to the flag ?
Oh I missed this notification somehow, busy on cleaning scheduler integration test. Yep so if locking is acceptable, it should be cleanest way to fix this, optimistic control would be a more complicated story.
- are you handling this one ?
Already been taken by volunteer days ago :)

Discussion:
fit bool, reasons []algorithm.PredicateFailureReason, equivalenceHash uint64, <ins> needLock bool, </ins> ) { <del> ec.Lock() defer ec.Unlock() </del> <ins> if needLock { ec.Lock() defer ec.Unlock() } </ins> if _, exist := ec.algorithmCache[nodeName]; !exist { ec.algorithmCache[nodeName] = newAlgorithmCache() }

// based on cached predicate results func (ec *EquivalenceCache) PredicateWithECache( podName, nodeName, predicateKey string, <del> equivalenceHash uint64, </del> <ins> equivalenceHash uint64, needLock bool, </ins> ) (bool, []algorithm.PredicateFailureReason, bool) { <del> ec.RLock() defer ec.RUnlock() </del> <ins> if needLock { ec.RLock() defer ec.RUnlock() } </ins> glog.V(5).Infof("Begin to calculate predicate: %v for pod: %s on node: %s based on equivalence cache", predicateKey, podName, nodeName) if algorithmCache, exist := ec.algorithmCache[nodeName]; exist {

test.fit, test.reasons, test.equivalenceHash, <ins> true, </ins> ) value, ok := ecache.algorithmCache[test.nodeName].predicatesCache.Get(test.predicateKey)

test.cachedItem.fit, test.cachedItem.reasons, test.equivalenceHashForUpdatePredicate, <ins> true, </ins> ) // if we want to do invalid, invalid the cached item if test.expectedInvalidPredicateKey {

test.nodeName, test.predicateKey, test.equivalenceHashForCalPredicate, <ins> true, </ins> ) // returned invalid should match expectedInvalidPredicateKey or expectedInvalidEquivalenceHash if test.equivalenceHashForUpdatePredicate != test.equivalenceHashForCalPredicate {

test.cachedItem.fit, test.cachedItem.reasons, test.equivalenceHashForUpdatePredicate, <ins> true, </ins> ) }

// Schedule tries to schedule the given pod to one of node in the node list. // If it succeeds, it will return the name of the node. <del> // If it fails, it will return a Fiterror error with reasons. </del> <ins> // If it fails, it will return a FitError error with reasons. </ins> func (g *genericScheduler) Schedule(pod *v1.Pod, nodeLister algorithm.NodeLister) (string, error) { trace := utiltrace.New(fmt.Sprintf("Scheduling %s/%s", pod.Namespace, pod.Name)) defer trace.LogIfLong(100 * time.Millisecond)

//TODO (yastij) : compute average predicate restrictiveness to export it as Prometheus metric if predicate, exist := predicateFuncs[predicateKey]; exist { if eCacheAvailable { <ins> // Lock ecache here to avoid a race condition against cache invalidation invoked // in event handlers. This race has existed despite locks in eCache implementation. ecache.Lock() </ins> // PredicateWithECache will return its cached predicate results. <del> fit, reasons, invalid = ecache.PredicateWithECache(pod.GetName(), info.Node().GetName(), predicateKey, equivCacheInfo.hash) </del> <ins> fit, reasons, invalid = ecache.PredicateWithECache(pod.GetName(), info.Node().GetName(), predicateKey, equivCacheInfo.hash, false) </ins> } if !eCacheAvailable || invalid {

} else { predicateResults[predicateKey] = HostPredicate{Fit: fit, FailReasons: reasons} } <ins> result := predicateResults[predicateKey] ecache.UpdateCachedPredicateItem(pod.GetName(), info.Node().GetName(), predicateKey, result.Fit, result.FailReasons, equivCacheInfo.hash, false) </ins> } } <ins> if eCacheAvailable { ecache.Unlock() } </ins> if !fit { // eCache is available and valid, and predicates result is unfit, record the fail reasons failedPredicates = append(failedPredicates, reasons...)

} } <del> // TODO(bsalamat): This way of updating equiv. cache has a race condition against // cache invalidations invoked in event handlers. This race has existed despite locks // in eCache implementation. If cache is invalidated after a predicate is executed // and before we update the cache, the updates should not be written to the cache. if eCacheAvailable { nodeName := info.Node().GetName() for predKey, result := range predicateResults { // update equivalence cache with newly computed fit & reasons // TODO(resouer) should we do this in another thread? any race? ecache.UpdateCachedPredicateItem(pod.GetName(), nodeName, predKey, result.Fit, result.FailReasons, equivCacheInfo.hash) } } </del> return len(failedPredicates) == 0, failedPredicates, nil }