Repository: go-co-op/gocron (Go)

Issue:
[BUG] - Job queue not empty after all LimitRunsTo(1) jobs have finished
### Describe the bug

I'm trying to implement a service that runs one-shot jobs at specified times, while limiting the number of jobs running at the same time.
SetMaxConcurrentJobs and LimitRunsTo(1) seemed to be perfect for this use case.

According to the documentation of LimitRunsTo(), the job is removed from the scheduler after it was ran enough times. Yet, when the program finishes, the jobs queue, as returned by scheduler.Jobs(), contains a few entries.

### To Reproduce

```package main

import (
	"fmt"
	"time"
	"log"

	"github.com/go-co-op/gocron"
)

var task = func(s string) {
	log.Printf("Starting job %s", s)
	time.Sleep(time.Millisecond * 1500)
	log.Printf("Finishing job %s", s)
}

func main() {
	log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds | log.Lshortfile)

	atd := gocron.NewScheduler(time.UTC)
	atd.SetMaxConcurrentJobs(5, gocron.RescheduleMode)
	atd.StartAsync()
	t := time.Now().Add(1400 * time.Millisecond)
	for i := 0; i < 17; i++ {
		s := fmt.Sprintf("Oneshot %d", i)
		atd.Every(1).StartAt(t).LimitRunsTo(1).Name(s).Do(task, s)
	}

	time.Sleep(time.Second * 13)

	jobs := atd.Jobs()
	for i, j := range jobs {
		log.Printf("job %d '%s' nextrun: %v, scheduled: %v, lastrun: %v, prevrun: %v, %d",
			i, j.GetName(), j.NextRun(), j.ScheduledTime(), j.LastRun(), j.PreviousRun(), j.RunCount())
	}
	atd.Stop()
}
```

### Version

v1.35.0

### Expected behavior

I expected that the queue would be empty after all jobs have finished.

### Additional context

The program appears to behave correctly, in the sense that every task is run exactly once, as expected, and only at most 5 of them are running at any given time, which is also correct. The order in which they run seems to be random, which is acceptable to my use case.

So I'm not sure if the remaining entries in the job queue represent just a harmless artifact, or hint at a deeper problem. On my machine subsequent runs produce different results: between 1 to 5 jobs remain in the queue, with seemingly random titles.

If this is not a bug, how can I enforce that the jobs queue is reliably empty after all jobs have finished?

[BUG] - Concurrent map read and map write in Scheduler
### Describe the bug

Application gets fatal error with the following message:

```
fatal error: concurrent map read and map write

goroutine 20038 [running]:
github.com/go-co-op/gocron.(*Scheduler)._jobPresent(0x0?, 0xc0016bddd0?, 0x771d17?)
        /Users/myname/go/pkg/mod/github.com/go-co-op/gocron@v1.35.0/scheduler.go:892 +0xaa
github.com/go-co-op/gocron.(*Scheduler).jobPresent(0xc000140c60?, 0x44c4b4?)
        /Users/myname/go/pkg/mod/github.com/go-co-op/gocron@v1.35.0/scheduler.go:899 +0x37
github.com/go-co-op/gocron.(*Scheduler).scheduleNextRun(0x0?, 0xc001b77680)
        /Users/myname/go/pkg/mod/github.com/go-co-op/gocron@v1.35.0/scheduler.go:205 +0x65
github.com/go-co-op/gocron.(*Scheduler).runContinuous(0xc000140c60, 0xc001b77680)
        /Users/myname/go/pkg/mod/github.com/go-co-op/gocron@v1.35.0/scheduler.go:613 +0x2d
github.com/go-co-op/gocron.(*Scheduler).runContinuous.func1()
        /Users/myname/go/pkg/mod/github.com/go-co-op/gocron@v1.35.0/scheduler.go:646 +0x1c5
created by time.goFunc
        /opt/homebrew/Cellar/go/1.20.2/libexec/src/time/sleep.go:176 +0x32

goroutine 1 [chan receive]:
main.main()
        /Users/myname/company/scripts/myproject/cmd/main.go:117 +0xb2a

goroutine 20 [syscall]:
os/signal.signal_recv()
        /opt/homebrew/Cellar/go/1.20.2/libexec/src/runtime/sigqueue.go:152 +0x2f
os/signal.loop()
        /opt/homebrew/Cellar/go/1.20.2/libexec/src/os/signal/signal_unix.go:23 +0x19
created by os/signal.Notify.func1.1
        /opt/homebrew/Cellar/go/1.20.2/libexec/src/os/signal/signal.go:151 +0x2a

goroutine 36 [select]:
github.com/go-co-op/gocron.(*executor).run(0xc0001ce0a0)
        /Users/myname/go/pkg/mod/github.com/go-co-op/gocron@v1.35.0/executor.go:229 +0xc7
created by github.com/go-co-op/gocron.(*executor).start
        /Users/myname/go/pkg/mod/github.com/go-co-op/gocron@v1.35.0/executor.go:158 +0x20a

goroutine 37 [chan receive]:
example.github.com/myproject/internal/executor.(*Executor).CollectDynamicRerouteResults(0xc000024240)
        /Users/myname/company/scripts/myproject/internal/executor/dynamic_reroute.go:39 +0xdd
created by main.main
        /Users/myname/company/scripts/myproject/cmd/main.go:103 +0x7ca

goroutine 38 [chan receive]:
example.github.com/myproject/internal/monitoring.(*Monitoring).PublishResults(0xc000070600)
        /Users/myname/company/scripts/myproject/internal/monitoring/monitoring.go:25 +0xad
created by main.main
        /Users/myname/company/scripts/myproject/cmd/main.go:104 +0x835

goroutine 12 [IO wait]:
internal/poll.runtime_pollWait(0x4027be2758, 0x72)
        /opt/homebrew/Cellar/go/1.20.2/libexec/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc0004da080?, 0xc000469cb0?, 0x0)
        /opt/homebrew/Cellar/go/1.20.2/libexec/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.(*pollDesc).waitRead(...)
        /opt/homebrew/Cellar/go/1.20.2/libexec/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Accept(0xc0004da080)
        /opt/homebrew/Cellar/go/1.20.2/libexec/src/internal/poll/fd_unix.go:614 +0x2bd
net.(*netFD).accept(0xc0004da080)
        /opt/homebrew/Cellar/go/1.20.2/libexec/src/net/fd_unix.go:172 +0x35
net.(*TCPListener).accept(0xc001686048)
        /opt/homebrew/Cellar/go/1.20.2/libexec/src/net/tcpsock_posix.go:148 +0x25
net.(*TCPListener).Accept(0xc001686048)
        /opt/homebrew/Cellar/go/1.20.2/libexec/src/net/tcpsock.go:297 +0x3d
net/http.(*Server).Serve(0xc00168a000, {0xa736f0, 0xc001686048})
        /opt/homebrew/Cellar/go/1.20.2/libexec/src/net/http/server.go:3059 +0x385
net/http.(*Server).ListenAndServe(0xc00168a000)
        /opt/homebrew/Cellar/go/1.20.2/libexec/src/net/http/server.go:2988 +0x7d
net/http.ListenAndServe(...)
        /opt/homebrew/Cellar/go/1.20.2/libexec/src/net/http/server.go:3242
example.github.com/myproject/internal/api.(*API).Start(0xc000010978)
        /Users/myname/company/scripts/myproject/internal/api/api.go:48 +0x2f1
created by main.main
        /Users/myname/company/scripts/myproject/cmd/main.go:112 +0xa14

goroutine 11 [chan receive]:
example.github.com/myproject/internal/metrics.(*Metrics).Collect(0xc000012068)
        /Users/myname/company/scripts/myproject/internal/metrics/metrics.go:51 +0x111
created by main.main
        /Users/myname/company/scripts/myproject/cmd/main.go:111 +0x9aa

goroutine 20052 [sync.RWMutex.Lock]:
sync.runtime_SemacquireRWMutex(0x85b7e5?, 0xa8?, 0x98dc00?)
        /opt/homebrew/Cellar/go/1.20.2/libexec/src/runtime/sema.go:87 +0x26
sync.(*RWMutex).Lock(0x8?)
        /opt/homebrew/Cellar/go/1.20.2/libexec/src/sync/rwmutex.go:152 +0x71
github.com/go-co-op/gocron.(*Scheduler).getCurrentJob(0xc000140c60)
        /Users/myname/go/pkg/mod/github.com/go-co-op/gocron@v1.35.0/scheduler.go:1315 +0xd6
github.com/go-co-op/gocron.(*Scheduler).EveryRandom(0x85e06b?, 0x0?, 0x94b000?)
        /Users/myname/go/pkg/mod/github.com/go-co-op/gocron@v1.35.0/scheduler.go:539 +0x28
example.github.com/myproject/internal/executor.(*Executor).scheduleTask(0xc000024240, {{{0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, ...}, ...})
        /Users/myname/company/scripts/myproject/internal/executor/tasks.go:70 +0x47
example.github.com/myproject/internal/executor.(*Executor).runTask.func2(0xc000024240, 0xc003ce2ee8, {0xa70500, 0xc0034d43f0}, 0x3?, 0xc003ce2c78, 0x0, 0xc003ce2e20, 0xc003ce2c20)
        /Users/myname/company/scripts/myproject/internal/executor/tasks.go:131 +0x5a5
example.github.com/myproject/internal/executor.(*Executor).runTask(_, {{{0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, {0x0, 0x0, 0x0}, ...}, ...}, ...)
        /Users/myname/company/scripts/myproject/internal/executor/tasks.go:170 +0x311
reflect.Value.call({0x903040?, 0xc000022d40?, 0x4027f86450?}, {0x9ad087, 0x4}, {0xc0034d40f0, 0x2, 0x0?})
        /opt/homebrew/Cellar/go/1.20.2/libexec/src/reflect/value.go:586 +0xb07
reflect.Value.Call({0x903040?, 0xc000022d40?, 0x10?}, {0xc0034d40f0?, 0x2?, 0x4027fbd5b8?})
        /opt/homebrew/Cellar/go/1.20.2/libexec/src/reflect/value.go:370 +0xbc
github.com/go-co-op/gocron.callJobFuncWithParams({0x903040?, 0xc000022d40?}, {0xc0031f11a0, 0x2, 0xc003a4065c?})
        /Users/myname/go/pkg/mod/github.com/go-co-op/gocron@v1.35.0/gocron.go:118 +0x28a
github.com/go-co-op/gocron.runJob({{0x54, 0xc, 0x6d, 0xc2, 0x60, 0x59, 0x44, 0x57, 0xbc, 0xd1, ...}, ...})
        /Users/myname/go/pkg/mod/github.com/go-co-op/gocron@v1.35.0/executor.go:88 +0x1a5
github.com/go-co-op/gocron.(*jobFunction).singletonRunner(0xc003e13680)
        /Users/myname/go/pkg/mod/github.com/go-co-op/gocron@v1.35.0/executor.go:117 +0x1b8
created by github.com/go-co-op/gocron.(*executor).runJob
        /Users/myname/go/pkg/mod/github.com/go-co-op/gocron@v1.35.0/executor.go:219 +0x3ed
make: *** [run] Error 2
```

### To Reproduce

Unfortunately cannot share company code for this.

However, there is no issue when using `RemoveByTags` instead of `RemoveByID`

### Version

v1.35.0

### Expected behavior

There should be no `fatal error: concurrent map read and map write`. Somewhere Lock() is missing.



Discussion: To answer my own question, I think this is a bug and it hints at a deeper problem.

Looking at the source, I think I understand what goes wrong: in the end, these finished jobs are removed by removeByReference(), which rebuilds the jobs map without the one job that's being removed, and replaces the main, live jobs map with it. The problem is that when two jobs finish running at the same time, there is an implicit race condition here: job A gets the map that still contains jobs A and B, removes A, and installs a map that contains B. At the same time, job B gets the same map that contains A and B, removes B, and installs a map that contains A, thus trampling over the other job's work.

And the underlying reason is that jobsMap() is dangerous and doesn't do what it appears to do at first glance: it doesn't protect the map at all, it merely locks and unlocks uselessly for a moment, and returns the map itself, then the caller is free to iterate over it at will. In Go, concurrent access to a map is fine as long as every access to it is read-only, however, if there is a possibility that somebody can modify the map, then all accesses (reads and writes) must be protected.

This trick of rebuilding and replacing the map means that you aren't really modifying the map itself concurrently, so you're safe - but this comes at the cost of bugs like this one.

For a proper fix, you'd have to rewrite at least some parts of the scheduler: when removing a job that's just finished and doesn't have to run anymore, you don't need the generic, user-visible RemoveByReference() function, you can just directly delete the job by id (you don't have to test for a generic condition, because you know that the job belongs to the scheduler, you just ran it). But then you'd be modifying the map itself, so you'd have to put all read accesses behind the mutex, so perhaps you should replace jobsMap() with a call that just returns the slice with the jobs, and iterate over that, in every place where you're range-ing over the map now. I haven't looked over the entire source, so this may introduce other problems, but I imagine in the end this would be a safer design than the current one that concurrently iterates over (potentially stale copies) of an unprotected map.





Fix (patch):
diff --git a/scheduler.go b/scheduler.go
index e9e6507d..72f5c0c3 100644
--- a/scheduler.go
+++ b/scheduler.go
@@ -104,11 +104,13 @@ func (s *Scheduler) StartAsync() {
 func (s *Scheduler) start() {
 	s.executor.start()
 	s.setRunning(true)
-	s.runJobs(s.jobsMap())
+	s.runJobs()
 }
 
-func (s *Scheduler) runJobs(jobs map[uuid.UUID]*Job) {
-	for _, job := range jobs {
+func (s *Scheduler) runJobs() {
+	s.jobsMutex.RLock()
+	defer s.jobsMutex.RUnlock()
+	for _, job := range s.jobs {
 		ctx, cancel := context.WithCancel(context.Background())
 		job.mu.Lock()
 		job.ctx = ctx
@@ -151,12 +153,6 @@ func (s *Scheduler) JobsMap() map[uuid.UUID]*Job {
 	return jobs
 }
 
-func (s *Scheduler) jobsMap() map[uuid.UUID]*Job {
-	s.jobsMutex.RLock()
-	defer s.jobsMutex.RUnlock()
-	return s.jobs
-}
-
 // Name sets the name of the current job.
 //
 // If the scheduler is running using WithDistributedLocker(), the job name is used
@@ -167,12 +163,6 @@ func (s *Scheduler) Name(name string) *Scheduler {
 	return s
 }
 
-func (s *Scheduler) setJobs(jobs map[uuid.UUID]*Job) {
-	s.jobsMutex.Lock()
-	defer s.jobsMutex.Unlock()
-	s.jobs = jobs
-}
-
 // Len returns the number of Jobs in the Scheduler
 func (s *Scheduler) Len() int {
 	s.jobsMutex.RLock()
@@ -231,7 +221,7 @@ func (s *Scheduler) scheduleNextRun(job *Job) (bool, nextRun) {
 	}
 
 	if !job.shouldRun() {
-		s.RemoveByReference(job)
+		_ = s.RemoveByID(job)
 		return false, nextRun{}
 	}
 
@@ -510,13 +500,15 @@ func (s *Scheduler) roundToMidnightAndAddDSTAware(t time.Time, d time.Duration)
 
 // NextRun datetime when the next Job should run.
 func (s *Scheduler) NextRun() (*Job, time.Time) {
-	if len(s.jobsMap()) <= 0 {
+	s.jobsMutex.RLock()
+	defer s.jobsMutex.RUnlock()
+	if len(s.jobs) <= 0 {
 		return nil, time.Time{}
 	}
 
 	var jobID uuid.UUID
 	var nearestRun time.Time
-	for _, job := range s.jobsMap() {
+	for _, job := range s.jobs {
 		nr := job.NextRun()
 		if (nr.Before(nearestRun) || nearestRun.IsZero()) && s.now().Before(nr) {
 			nearestRun = nr
@@ -524,7 +516,7 @@ func (s *Scheduler) NextRun() (*Job, time.Time) {
 		}
 	}
 
-	return s.jobsMap()[jobID], nearestRun
+	return s.jobs[jobID], nearestRun
 }
 
 // EveryRandom schedules a new period Job that runs at random intervals
@@ -650,7 +642,9 @@ func (s *Scheduler) RunAll() {
 
 // RunAllWithDelay runs all Jobs with the provided delay in between each Job
 func (s *Scheduler) RunAllWithDelay(d time.Duration) {
-	for _, job := range s.jobsMap() {
+	s.jobsMutex.RLock()
+	defer s.jobsMutex.RUnlock()
+	for _, job := range s.jobs {
 		s.run(job)
 		s.time.Sleep(d)
 	}
@@ -698,16 +692,13 @@ func (s *Scheduler) Remove(job interface{}) {
 
 // RemoveByReference removes specific Job by reference
 func (s *Scheduler) RemoveByReference(job *Job) {
-	s.removeJobsUniqueTags(job)
-	s.removeByCondition(func(someJob *Job) bool {
-		job.mu.RLock()
-		defer job.mu.RUnlock()
-		return someJob == job
-	})
+	_ = s.RemoveByID(job)
 }
 
 func (s *Scheduler) findJobByTaskName(name string) *Job {
-	for _, job := range s.jobsMap() {
+	s.jobsMutex.RLock()
+	defer s.jobsMutex.RUnlock()
+	for _, job := range s.jobs {
 		if job.funcName == name {
 			return job
 		}
@@ -727,15 +718,14 @@ func (s *Scheduler) removeJobsUniqueTags(job *Job) {
 }
 
 func (s *Scheduler) removeByCondition(shouldRemove func(*Job) bool) {
-	retainedJobs := make(map[uuid.UUID]*Job, 0)
-	for _, job := range s.jobsMap() {
-		if !shouldRemove(job) {
-			retainedJobs[job.id] = job
-		} else {
+	s.jobsMutex.Lock()
+	defer s.jobsMutex.Unlock()
+	for _, job := range s.jobs {
+		if shouldRemove(job) {
 			s.stopJob(job)
+			delete(s.jobs, job.id)
 		}
 	}
-	s.setJobs(retainedJobs)
 }
 
 func (s *Scheduler) stopJob(job *Job) {
@@ -760,7 +750,7 @@ func (s *Scheduler) RemoveByTags(tags ...string) error {
 	}
 
 	for _, job := range jobs {
-		s.RemoveByReference(job)
+		_ = s.RemoveByID(job)
 	}
 	return nil
 }
@@ -780,7 +770,7 @@ func (s *Scheduler) RemoveByTagsAny(tags ...string) error {
 	}
 
 	for job := range mJob {
-		s.RemoveByReference(job)
+		_ = s.RemoveByID(job)
 	}
 
 	return errs
@@ -788,9 +778,12 @@ func (s *Scheduler) RemoveByTagsAny(tags ...string) error {
 
 // RemoveByID removes the job from the scheduler looking up by id
 func (s *Scheduler) RemoveByID(job *Job) error {
-	if _, ok := s.jobsMap()[job.id]; ok {
+	s.jobsMutex.Lock()
+	defer s.jobsMutex.Unlock()
+	if _, ok := s.jobs[job.id]; ok {
+		s.removeJobsUniqueTags(job)
 		s.stopJob(job)
-		delete(s.jobsMap(), job.id)
+		delete(s.jobs, job.id)
 		return nil
 	}
 	return ErrJobNotFound
@@ -800,8 +793,10 @@ func (s *Scheduler) RemoveByID(job *Job) error {
 func (s *Scheduler) FindJobsByTag(tags ...string) ([]*Job, error) {
 	var jobs []*Job
 
+	s.jobsMutex.RLock()
+	defer s.jobsMutex.RUnlock()
 Jobs:
-	for _, job := range s.jobsMap() {
+	for _, job := range s.jobs {
 		if job.hasTags(tags...) {
 			jobs = append(jobs, job)
 			continue Jobs
@@ -872,7 +867,9 @@ func (s *Scheduler) SingletonModeAll() {
 
 // TaskPresent checks if specific job's function was added to the scheduler.
 func (s *Scheduler) TaskPresent(j interface{}) bool {
-	for _, job := range s.jobsMap() {
+	s.jobsMutex.RLock()
+	defer s.jobsMutex.RUnlock()
+	for _, job := range s.jobs {
 		if job.funcName == getFunctionName(j) {
 			return true
 		}
@@ -880,25 +877,21 @@ func (s *Scheduler) TaskPresent(j interface{}) bool {
 	return false
 }
 
-// To avoid the recursive read lock on s.jobsMap() and this function,
-// creating this new function and distributing the lock between jobPresent, _jobPresent
-func (s *Scheduler) _jobPresent(j *Job, jobs map[uuid.UUID]*Job) bool {
+func (s *Scheduler) jobPresent(j *Job) bool {
 	s.jobsMutex.RLock()
 	defer s.jobsMutex.RUnlock()
-	if _, ok := jobs[j.id]; ok {
+	if _, ok := s.jobs[j.id]; ok {
 		return true
 	}
 	return false
 }
 
-func (s *Scheduler) jobPresent(j *Job) bool {
-	return s._jobPresent(j, s.jobsMap())
-}
-
 // Clear clears all Jobs from this scheduler
 func (s *Scheduler) Clear() {
 	s.stopJobs()
-	s.setJobs(make(map[uuid.UUID]*Job, 0))
+	s.jobsMutex.Lock()
+	defer s.jobsMutex.Unlock()
+	s.jobs = make(map[uuid.UUID]*Job)
 	// If unique tags was enabled, delete all the tags loaded in the tags sync.Map
 	if s.tagsUnique {
 		s.tags.Range(func(key interface{}, value interface{}) bool {
@@ -954,7 +947,7 @@ func (s *Scheduler) doCommon(jobFun interface{}, params ...interface{}) (*Job, e
 	if job.error != nil {
 		// delete the job from the scheduler as this job
 		// cannot be executed
-		s.RemoveByReference(job)
+		_ = s.RemoveByID(job)
 		return nil, job.error
 	}
 
@@ -965,7 +958,7 @@ func (s *Scheduler) doCommon(jobFun interface{}, params ...interface{}) (*Job, e
 
 	if val.Kind() != reflect.Func {
 		// delete the job for the same reason as above
-		s.RemoveByReference(job)
+		_ = s.RemoveByID(job)
 		return nil, ErrNotAFunction
 	}
 
@@ -992,13 +985,13 @@ func (s *Scheduler) doCommon(jobFun interface{}, params ...interface{}) (*Job, e
 	}
 
 	if len(params) != expectedParamLength {
-		s.RemoveByReference(job)
+		_ = s.RemoveByID(job)
 		job.error = wrapOrError(job.error, ErrWrongParams)
 		return nil, job.error
 	}
 
 	if job.runWithDetails && val.Type().In(len(params)).Kind() != reflect.ValueOf(*job).Kind() {
-		s.RemoveByReference(job)
+		_ = s.RemoveByID(job)
 		job.error = wrapOrError(job.error, ErrDoWithJobDetails)
 		return nil, job.error
 	}
@@ -1078,7 +1071,9 @@ func (s *Scheduler) Tag(t ...string) *Scheduler {
 // GetAllTags returns all tags.
 func (s *Scheduler) GetAllTags() []string {
 	var tags []string
-	for _, job := range s.jobsMap() {
+	s.jobsMutex.RLock()
+	defer s.jobsMutex.RUnlock()
+	for _, job := range s.jobs {
 		tags = append(tags, job.Tags()...)
 	}
 	return tags
@@ -1514,7 +1509,9 @@ func (s *Scheduler) WithDistributedElector(e Elector) {
 // If a new job is added, an additional call to this method, or the job specific
 // version must be executed in order for the new job to trigger event listeners.
 func (s *Scheduler) RegisterEventListeners(eventListeners ...EventListener) {
-	for _, job := range s.jobsMap() {
+	s.jobsMutex.RLock()
+	defer s.jobsMutex.RUnlock()
+	for _, job := range s.jobs {
 		job.RegisterEventListeners(eventListeners...)
 	}
 }