Repository: bacalhau-project/bacalhau (language: python)

Issue:
Sort executions in jobstore
The orchestrator's jobExecutions endpoint currently performs sorting of the executions within the API method, and we should move this to the jobstore.


Discussion / hints:

Fix patch:
diff --git a/cmd/cli/job/executions.go b/cmd/cli/job/executions.go
index 3a337299bf..94b9d25483 100644
--- a/cmd/cli/job/executions.go
+++ b/cmd/cli/job/executions.go
@@ -20,7 +20,7 @@ import (
 	"github.com/bacalhau-project/bacalhau/pkg/util/templates"
 )
 
-var executionsOrderByFields = []string{"modify_time", "create_time", "id", "state"}
+var executionsOrderByFields = []string{"modified_at", "created_at"}
 
 var (
 	executionShort = `List executions for a job by id.`
diff --git a/pkg/jobstore/boltdb/store.go b/pkg/jobstore/boltdb/store.go
index 481bc7bd99..f1cd3845d7 100644
--- a/pkg/jobstore/boltdb/store.go
+++ b/pkg/jobstore/boltdb/store.go
@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"slices"
 	"sort"
 	"strings"
 	"time"
@@ -20,6 +21,7 @@ import (
 	"github.com/bacalhau-project/bacalhau/pkg/lib/marshaller"
 	"github.com/bacalhau-project/bacalhau/pkg/lib/math"
 	"github.com/bacalhau-project/bacalhau/pkg/models"
+	"github.com/bacalhau-project/bacalhau/pkg/util"
 	"github.com/bacalhau-project/bacalhau/pkg/util/idgen"
 )
 
@@ -145,7 +147,8 @@ func (b *BoltJobStore) BeginTx(ctx context.Context) (jobstore.TxContext, error)
 func (b *BoltJobStore) Watch(ctx context.Context,
 	types jobstore.StoreWatcherType,
 	events jobstore.StoreEventType,
-	options ...jobstore.WatcherOption) *jobstore.Watcher {
+	options ...jobstore.WatcherOption,
+) *jobstore.Watcher {
 	return b.watchersManager.NewWatcher(ctx, types, events, options...)
 }
 
@@ -266,6 +269,33 @@ func (b *BoltJobStore) getExecutions(tx *bolt.Tx, options jobstore.GetExecutions
 		job = &j
 	}
 
+	// Sort By Given Order By
+	var sortFnc func(a, b models.Execution) int
+	switch options.OrderBy {
+	// create_time will eventually be deprectated. It is being used for backward compatibility.
+	case "create_time", "created_at", "": //nolint: goconst
+		sortFnc = func(a, b models.Execution) int { return util.Compare[int64]{}.Cmp(a.CreateTime, b.CreateTime) }
+	// modify_time will eventually be deprecated. It is being used for backward compatibility.
+	case "modify_time", "modified_at":
+		sortFnc = func(a, b models.Execution) int { return util.Compare[int64]{}.Cmp(a.ModifyTime, b.ModifyTime) }
+	default:
+		return nil, fmt.Errorf("OrderBy %s not supported for getExecutions", options.OrderBy)
+	}
+
+	if options.Reverse {
+		baseSortFnc := sortFnc
+		sortFnc = func(a, b models.Execution) int {
+			r := baseSortFnc(a, b)
+			if r == -1 {
+				return 1
+			}
+			if r == 1 {
+				return -1
+			}
+			return 0
+		}
+	}
+
 	bkt, err := NewBucketPath(BucketJobs, jobID, BucketJobExecutions).Get(tx, false)
 	if err != nil {
 		return nil, err
@@ -285,6 +315,14 @@ func (b *BoltJobStore) getExecutions(tx *bolt.Tx, options jobstore.GetExecutions
 		return nil
 	})
 
+	// sort executions
+	slices.SortFunc(execs, sortFnc)
+
+	// apply limit
+	if options.Limit > 0 && len(execs) > options.Limit {
+		execs = execs[:options.Limit]
+	}
+
 	return execs, err
 }
 
@@ -325,26 +363,30 @@ func (b *BoltJobStore) getJobs(tx *bolt.Tx, query jobstore.JobQuery) (*jobstore.
 	}
 
 	// Sort the jobs according to the query.SortBy and query.SortOrder
-	listSorter := func(i, j int) bool {
-		switch query.SortBy {
-		case "modified_at":
-			if query.SortReverse {
-				return result[i].ModifyTime > result[j].ModifyTime
-			} else {
-				return result[i].ModifyTime < result[j].ModifyTime
+	var sortFunc func(a, b models.Job) int
+	switch query.SortBy {
+	case "created_at", "":
+		sortFunc = func(a, b models.Job) int { return util.Compare[int64]{}.Cmp(a.CreateTime, b.CreateTime) }
+	case "modified_at":
+		sortFunc = func(a, b models.Job) int { return util.Compare[int64]{}.Cmp(a.ModifyTime, b.ModifyTime) }
+	default:
+		return nil, fmt.Errorf("OrderBy %s not supported for listJobs", query.SortBy)
+	}
+	if query.SortReverse {
+		baseSortFnc := sortFunc
+		sortFunc = func(a, b models.Job) int {
+			r := baseSortFnc(a, b)
+			if r == -1 {
+				return 1
 			}
-		default:
-			// We apply created_at as a default sort so that we can use it for pagination.
-			// Without a known default we won't have a stable sort that makes sense for
-			// offsets/limits.
-			if query.SortReverse {
-				return result[i].CreateTime > result[j].CreateTime
-			} else {
-				return result[i].CreateTime < result[j].CreateTime
+			if r == 1 {
+				return -1
 			}
+			return 0
 		}
 	}
-	sort.Slice(result, listSorter)
+
+	slices.SortFunc(result, sortFunc)
 
 	// If we have a selector, filter the results to only those that match
 	if query.Selector != nil {
@@ -587,7 +629,8 @@ func createInProgressIndexKey(job *models.Job) string {
 // GetJobHistory returns the job (and execution) history for the provided options
 func (b *BoltJobStore) GetJobHistory(ctx context.Context,
 	jobID string,
-	options jobstore.JobHistoryFilterOptions) ([]models.JobHistory, error) {
+	options jobstore.JobHistoryFilterOptions,
+) ([]models.JobHistory, error) {
 	var history []models.JobHistory
 	err := b.view(ctx, func(tx *bolt.Tx) (err error) {
 		history, err = b.getJobHistory(tx, jobID, options)
@@ -598,7 +641,8 @@ func (b *BoltJobStore) GetJobHistory(ctx context.Context,
 }
 
 func (b *BoltJobStore) getJobHistory(tx *bolt.Tx, jobID string,
-	options jobstore.JobHistoryFilterOptions) ([]models.JobHistory, error) {
+	options jobstore.JobHistoryFilterOptions,
+) ([]models.JobHistory, error) {
 	var history []models.JobHistory
 
 	jobID, err := b.reifyJobID(tx, jobID)
@@ -621,7 +665,6 @@ func (b *BoltJobStore) getJobHistory(tx *bolt.Tx, jobID string,
 				history = append(history, item)
 				return nil
 			})
-
 			if err != nil {
 				return nil, err
 			}
@@ -644,7 +687,6 @@ func (b *BoltJobStore) getJobHistory(tx *bolt.Tx, jobID string,
 				history = append(history, item)
 				return nil
 			})
-
 			if err != nil {
 				return nil, err
 			}
@@ -1096,7 +1138,8 @@ func (b *BoltJobStore) updateExecution(tx *bolt.Tx, request jobstore.UpdateExecu
 }
 
 func (b *BoltJobStore) appendExecutionHistory(tx *bolt.Tx, updated models.Execution,
-	previous models.ExecutionStateType, event models.Event) error {
+	previous models.ExecutionStateType, event models.Event,
+) error {
 	historyEntry := models.JobHistory{
 		Type:        models.JobHistoryTypeExecutionLevel,
 		JobID:       updated.JobID,
diff --git a/pkg/jobstore/types.go b/pkg/jobstore/types.go
index 2e62dce45b..b1f5753ac4 100644
--- a/pkg/jobstore/types.go
+++ b/pkg/jobstore/types.go
@@ -192,4 +192,7 @@ type JobHistoryFilterOptions struct {
 type GetExecutionsOptions struct {
 	JobID      string `json:"job_id"`
 	IncludeJob bool   `json:"include_job"`
+	OrderBy    string `json:"order_by"`
+	Reverse    bool   `json:"reverse"`
+	Limit      int    `json:"limit"`
 }
diff --git a/pkg/publicapi/endpoint/orchestrator/job.go b/pkg/publicapi/endpoint/orchestrator/job.go
index 2c87b93806..2fe906f391 100644
--- a/pkg/publicapi/endpoint/orchestrator/job.go
+++ b/pkg/publicapi/endpoint/orchestrator/job.go
@@ -9,7 +9,6 @@ import (
 	"github.com/labstack/echo/v4"
 	"github.com/rs/zerolog/log"
 	"github.com/samber/lo"
-	"golang.org/x/exp/slices"
 
 	"github.com/bacalhau-project/bacalhau/pkg/jobstore"
 	"github.com/bacalhau-project/bacalhau/pkg/lib/concurrency"
@@ -17,7 +16,6 @@ import (
 	"github.com/bacalhau-project/bacalhau/pkg/orchestrator"
 	"github.com/bacalhau-project/bacalhau/pkg/publicapi"
 	"github.com/bacalhau-project/bacalhau/pkg/publicapi/apimodels"
-	"github.com/bacalhau-project/bacalhau/pkg/util"
 )
 
 // godoc for Orchestrator PutJob
@@ -339,53 +337,17 @@ func (e *Endpoint) jobExecutions(c echo.Context) error {
 		return err
 	}
 
-	// TODO: move ordering to jobstore
-	// parse order_by
-	var sortFnc func(a, b models.Execution) int
-	switch args.OrderBy {
-	case "modify_time", "":
-		sortFnc = func(a, b models.Execution) int { return util.Compare[int64]{}.Cmp(a.ModifyTime, b.ModifyTime) }
-	case "create_time":
-		sortFnc = func(a, b models.Execution) int { return util.Compare[int64]{}.Cmp(a.CreateTime, b.CreateTime) }
-	case "id":
-		sortFnc = func(a, b models.Execution) int { return util.Compare[string]{}.Cmp(a.ID, b.ID) }
-	case "state":
-		sortFnc = func(a, b models.Execution) int {
-			return util.Compare[models.ExecutionStateType]{}.Cmp(a.ComputeState.StateType, b.ComputeState.StateType)
-		}
-	default:
-		return echo.NewHTTPError(http.StatusBadRequest, "invalid order_by")
-	}
-	if args.Reverse {
-		baseSortFnc := sortFnc
-		sortFnc = func(a, b models.Execution) int {
-			r := baseSortFnc(a, b)
-			if r == -1 {
-				return 1
-			}
-			if r == 1 {
-				return -1
-			}
-			return 0
-		}
-	}
-
 	// query executions
 	executions, err := e.store.GetExecutions(ctx, jobstore.GetExecutionsOptions{
-		JobID: jobID,
+		JobID:   jobID,
+		OrderBy: args.OrderBy,
+		Reverse: args.Reverse,
+		Limit:   int(args.Limit),
 	})
 	if err != nil {
 		return err
 	}
 
-	// sort executions
-	slices.SortFunc(executions, sortFnc)
-
-	// apply limit
-	if args.Limit > 0 && len(executions) > int(args.Limit) {
-		executions = executions[:args.Limit]
-	}
-
 	// prepare result
 	res := &apimodels.ListJobExecutionsResponse{
 		Items: make([]*models.Execution, len(executions)),


Test patch:
diff --git a/pkg/jobstore/boltdb/store_test.go b/pkg/jobstore/boltdb/store_test.go
index 4d7f412d61..6607b3f3f8 100644
--- a/pkg/jobstore/boltdb/store_test.go
+++ b/pkg/jobstore/boltdb/store_test.go
@@ -43,52 +43,73 @@ func (s *BoltJobstoreTestSuite) SetupTest() {
 	s.ctx = context.Background()
 
 	jobFixtures := []struct {
-		id              string
-		jobType         string
-		client          string
-		tags            map[string]string
-		jobStates       []models.JobStateType
-		executionStates []models.ExecutionStateType
+		id         string
+		jobType    string
+		client     string
+		tags       map[string]string
+		jobStates  []models.JobStateType
+		executions map[int][]models.ExecutionStateType
 	}{
 		{
-			id:              "110",
-			client:          "client1",
-			jobType:         "batch",
-			tags:            map[string]string{"gpu": "true", "fast": "true"},
-			jobStates:       []models.JobStateType{models.JobStateTypePending, models.JobStateTypeRunning, models.JobStateTypeStopped},
-			executionStates: []models.ExecutionStateType{models.ExecutionStateAskForBid, models.ExecutionStateAskForBidAccepted, models.ExecutionStateCancelled},
+			id:        "110",
+			client:    "client1",
+			jobType:   "batch",
+			tags:      map[string]string{"gpu": "true", "fast": "true"},
+			jobStates: []models.JobStateType{models.JobStateTypePending, models.JobStateTypeRunning, models.JobStateTypeStopped},
+			executions: map[int][]models.ExecutionStateType{
+				1: {models.ExecutionStateAskForBid, models.ExecutionStateAskForBidAccepted, models.ExecutionStateCancelled},
+			},
 		},
 		{
-			id:              "120",
-			client:          "client2",
-			jobType:         "batch",
-			tags:            map[string]string{},
-			jobStates:       []models.JobStateType{models.JobStateTypePending, models.JobStateTypeRunning, models.JobStateTypeStopped},
-			executionStates: []models.ExecutionStateType{models.ExecutionStateAskForBid, models.ExecutionStateAskForBidAccepted, models.ExecutionStateCancelled},
+			id:        "120",
+			client:    "client2",
+			jobType:   "batch",
+			tags:      map[string]string{},
+			jobStates: []models.JobStateType{models.JobStateTypePending, models.JobStateTypeRunning, models.JobStateTypeStopped},
+			executions: map[int][]models.ExecutionStateType{
+				1: {models.ExecutionStateAskForBid, models.ExecutionStateAskForBidAccepted, models.ExecutionStateCancelled},
+			},
+		},
+		{
+			id:        "130",
+			client:    "client3",
+			jobType:   "batch",
+			tags:      map[string]string{"slow": "true", "max": "10"},
+			jobStates: []models.JobStateType{models.JobStateTypePending, models.JobStateTypeRunning},
+			executions: map[int][]models.ExecutionStateType{
+				1: {models.ExecutionStateAskForBid, models.ExecutionStateAskForBidAccepted},
+			},
 		},
 		{
-			id:              "130",
-			client:          "client3",
-			jobType:         "batch",
-			tags:            map[string]string{"slow": "true", "max": "10"},
-			jobStates:       []models.JobStateType{models.JobStateTypePending, models.JobStateTypeRunning},
-			executionStates: []models.ExecutionStateType{models.ExecutionStateAskForBid, models.ExecutionStateAskForBidAccepted},
+			id:        "140",
+			client:    "client4",
+			jobType:   "batch",
+			tags:      map[string]string{"max": "10"},
+			jobStates: []models.JobStateType{models.JobStateTypePending, models.JobStateTypeRunning},
+			executions: map[int][]models.ExecutionStateType{
+				1: {models.ExecutionStateAskForBid, models.ExecutionStateAskForBidAccepted},
+			},
 		},
 		{
-			id:              "140",
-			client:          "client4",
-			jobType:         "batch",
-			tags:            map[string]string{"max": "10"},
-			jobStates:       []models.JobStateType{models.JobStateTypePending, models.JobStateTypeRunning},
-			executionStates: []models.ExecutionStateType{models.ExecutionStateAskForBid, models.ExecutionStateAskForBidAccepted},
+			id:        "150",
+			client:    "client5",
+			jobType:   "daemon",
+			tags:      map[string]string{"max": "10"},
+			jobStates: []models.JobStateType{models.JobStateTypePending, models.JobStateTypeRunning},
+			executions: map[int][]models.ExecutionStateType{
+				1: {models.ExecutionStateAskForBid, models.ExecutionStateAskForBidAccepted},
+			},
 		},
 		{
-			id:              "150",
-			client:          "client5",
-			jobType:         "daemon",
-			tags:            map[string]string{"max": "10"},
-			jobStates:       []models.JobStateType{models.JobStateTypePending, models.JobStateTypeRunning},
-			executionStates: []models.ExecutionStateType{models.ExecutionStateAskForBid, models.ExecutionStateAskForBidAccepted},
+			id:        "160",
+			client:    "client6",
+			jobType:   "batch",
+			tags:      map[string]string{"max": "10"},
+			jobStates: []models.JobStateType{models.JobStateTypePending, models.JobStateTypeRunning},
+			executions: map[int][]models.ExecutionStateType{
+				1: {models.ExecutionStateAskForBid, models.ExecutionStateAskForBidAccepted, models.ExecutionStateFailed},
+				2: {models.ExecutionStateAskForBid, models.ExecutionStateAskForBidAccepted, models.ExecutionStateCompleted},
+			},
 		},
 	}
 
@@ -104,15 +125,6 @@ func (s *BoltJobstoreTestSuite) SetupTest() {
 		err := s.store.CreateJob(s.ctx, *job, models.Event{})
 		s.Require().NoError(err)
 
-		s.clock.Add(1 * time.Second)
-		execution := mock.ExecutionForJob(job)
-		execution.ComputeState.StateType = models.ExecutionStateNew
-		// clear out CreateTime and ModifyTime from the mocked execution to let the job store fill those
-		execution.CreateTime = 0
-		execution.ModifyTime = 0
-		err = s.store.CreateExecution(s.ctx, *execution, models.Event{})
-		s.Require().NoError(err)
-
 		for i, state := range fixture.jobStates {
 			s.clock.Add(1 * time.Second)
 
@@ -134,30 +146,41 @@ func (s *BoltJobstoreTestSuite) SetupTest() {
 			s.Require().NoError(err)
 		}
 
-		for i, state := range fixture.executionStates {
+		for _, executionStates := range fixture.executions {
 			s.clock.Add(1 * time.Second)
-
-			oldState := models.ExecutionStateNew
-			if i > 0 {
-				oldState = fixture.executionStates[i-1]
-			}
-
-			// We are pretending this is a new execution struct
-			execution.ComputeState.StateType = state
-			execution.ModifyTime = s.clock.Now().UTC().UnixNano()
-
-			request := jobstore.UpdateExecutionRequest{
-				ExecutionID: execution.ID,
-				Condition: jobstore.UpdateExecutionCondition{
-					ExpectedStates:   []models.ExecutionStateType{oldState},
-					ExpectedRevision: uint64(i + 1),
-				},
-				NewValues: *execution,
-				Event:     models.Event{},
-			}
-
-			err = s.store.UpdateExecution(s.ctx, request)
+			execution := mock.ExecutionForJob(job)
+			execution.ComputeState.StateType = models.ExecutionStateNew
+			// clear out CreateTime and ModifyTime from the mocked execution to let the job store fill those
+			execution.CreateTime = 0
+			execution.ModifyTime = 0
+			err = s.store.CreateExecution(s.ctx, *execution, models.Event{})
 			s.Require().NoError(err)
+			for i, state := range executionStates {
+
+				s.clock.Add(1 * time.Second)
+
+				oldState := models.ExecutionStateNew
+				if i > 0 {
+					oldState = executionStates[i-1]
+				}
+
+				// We are pretending this is a new execution struct
+				execution.ComputeState.StateType = state
+				execution.ModifyTime = s.clock.Now().UTC().UnixNano()
+
+				request := jobstore.UpdateExecutionRequest{
+					ExecutionID: execution.ID,
+					Condition: jobstore.UpdateExecutionCondition{
+						ExpectedStates:   []models.ExecutionStateType{oldState},
+						ExpectedRevision: uint64(i + 1),
+					},
+					NewValues: *execution,
+					Event:     models.Event{},
+				}
+
+				err = s.store.UpdateExecution(s.ctx, request)
+				s.Require().NoError(err)
+			}
 		}
 
 	}
@@ -344,7 +367,7 @@ func (s *BoltJobstoreTestSuite) TestSearchJobs() {
 		})
 
 		require.NoError(t, err)
-		require.Equal(t, 1, len(response.Jobs))
+		require.Equal(t, 2, len(response.Jobs))
 	})
 
 	s.T().Run("everything sorted by created_at", func(t *testing.T) {
@@ -353,11 +376,11 @@ func (s *BoltJobstoreTestSuite) TestSearchJobs() {
 		})
 		require.NoError(t, err)
 		jobs := response.Jobs
-		require.Equal(t, 5, len(jobs))
+		require.Equal(t, 6, len(jobs))
 		ids := lo.Map(jobs, func(item models.Job, _ int) string {
 			return item.ID
 		})
-		require.EqualValues(t, []string{"110", "120", "130", "140", "150"}, ids)
+		require.EqualValues(t, []string{"110", "120", "130", "140", "150", "160"}, ids)
 
 		response, err = s.store.GetJobs(s.ctx, jobstore.JobQuery{
 			ReturnAll:   true,
@@ -365,11 +388,11 @@ func (s *BoltJobstoreTestSuite) TestSearchJobs() {
 		})
 		require.NoError(t, err)
 		jobs = response.Jobs
-		require.Equal(t, 5, len(jobs))
+		require.Equal(t, 6, len(jobs))
 		ids = lo.Map(jobs, func(item models.Job, _ int) string {
 			return item.ID
 		})
-		require.EqualValues(t, []string{"150", "140", "130", "120", "110"}, ids)
+		require.EqualValues(t, []string{"160", "150", "140", "130", "120", "110"}, ids)
 	})
 
 	s.T().Run("everything", func(t *testing.T) {
@@ -377,7 +400,7 @@ func (s *BoltJobstoreTestSuite) TestSearchJobs() {
 			ReturnAll: true,
 		})
 		require.NoError(t, err)
-		require.Equal(t, 5, len(response.Jobs))
+		require.Equal(t, 6, len(response.Jobs))
 	})
 
 	s.T().Run("everything offset", func(t *testing.T) {
@@ -386,7 +409,7 @@ func (s *BoltJobstoreTestSuite) TestSearchJobs() {
 			Offset:    1,
 		})
 		require.NoError(t, err)
-		require.Equal(t, 4, len(response.Jobs))
+		require.Equal(t, 5, len(response.Jobs))
 		require.Equal(t, uint32(1), response.Offset)
 	})
 
@@ -424,7 +447,7 @@ func (s *BoltJobstoreTestSuite) TestSearchJobs() {
 			ExcludeTags: []string{"fast"},
 		})
 		require.NoError(t, err)
-		require.Equal(t, 4, len(response.Jobs))
+		require.Equal(t, 5, len(response.Jobs))
 	})
 
 	s.T().Run("include/exclude same tag", func(t *testing.T) {
@@ -525,17 +548,110 @@ func (s *BoltJobstoreTestSuite) TestGetExecutions() {
 	s.Require().IsType(err, &bacerrors.MultipleJobsFound{})
 	s.Require().Nil(state)
 
+	// Created At Ascending Order Sort
+	state, err = s.store.GetExecutions(s.ctx, jobstore.GetExecutionsOptions{
+		JobID:   "160",
+		OrderBy: "created_at",
+	})
+	s.Require().NoError(err)
+	s.NotNil(state)
+	s.Equal(2, len(state))
+	s.Equal(state[0].GetCreateTime().Before(state[1].GetCreateTime()), true)
+
+	// Created At Descending Order Sort
+	state, err = s.store.GetExecutions(s.ctx, jobstore.GetExecutionsOptions{
+		JobID:   "160",
+		OrderBy: "created_at",
+		Reverse: true,
+	})
+	s.Require().NoError(err)
+	s.NotNil(state)
+	s.Equal(2, len(state))
+	s.Equal(state[0].GetCreateTime().After(state[1].GetCreateTime()), true)
+
+	// Created Time Backward Compatibility Ascending Order Sort
+	state, err = s.store.GetExecutions(s.ctx, jobstore.GetExecutionsOptions{
+		JobID:   "160",
+		OrderBy: "create_time",
+	})
+	s.Require().NoError(err)
+	s.NotNil(state)
+	s.Equal(2, len(state))
+	s.Equal(state[0].GetCreateTime().Before(state[1].GetCreateTime()), true)
+
+	// Create Time Backward Compatibility Descending Order Sort
+	state, err = s.store.GetExecutions(s.ctx, jobstore.GetExecutionsOptions{
+		JobID:   "160",
+		OrderBy: "create_time",
+		Reverse: true,
+	})
+	s.Require().NoError(err)
+	s.NotNil(state)
+	s.Equal(2, len(state))
+	s.Equal(state[0].GetCreateTime().After(state[1].GetCreateTime()), true)
+
+	// When OrderBy Empty, Created At Used as Default
+	state, err = s.store.GetExecutions(s.ctx, jobstore.GetExecutionsOptions{
+		JobID: "160",
+	})
+	s.Require().NoError(err)
+	s.NotNil(state)
+	s.Equal(2, len(state))
+	s.Equal(state[0].GetCreateTime().Before(state[1].GetCreateTime()), true)
+
+	// When OrderBy is set to Modified At
+	state, err = s.store.GetExecutions(s.ctx, jobstore.GetExecutionsOptions{
+		JobID:   "160",
+		OrderBy: "modified_at",
+	})
+	s.Require().NoError(err)
+	s.NotNil(state)
+	s.Equal(2, len(state))
+	s.Equal(state[0].GetModifyTime().Before(state[1].GetModifyTime()), true)
+
+	// When OrderBy is set to Modified At With Reverese
+	state, err = s.store.GetExecutions(s.ctx, jobstore.GetExecutionsOptions{
+		JobID:   "160",
+		OrderBy: "modified_at",
+		Reverse: true,
+	})
+	s.Require().NoError(err)
+	s.NotNil(state)
+	s.Equal(2, len(state))
+	s.Equal(state[0].GetModifyTime().After(state[1].GetModifyTime()), true)
+
+	// When OrderBy is set to Modify Time (Backward Compatibility)
+	state, err = s.store.GetExecutions(s.ctx, jobstore.GetExecutionsOptions{
+		JobID:   "160",
+		OrderBy: "modify_time",
+	})
+	s.Require().NoError(err)
+	s.NotNil(state)
+	s.Equal(2, len(state))
+	s.Equal(state[0].GetModifyTime().Before(state[1].GetModifyTime()), true)
+
+	// When OrderBy is set to Modify Time (Backward Compatibility)
+	state, err = s.store.GetExecutions(s.ctx, jobstore.GetExecutionsOptions{
+		JobID:   "160",
+		OrderBy: "modify_time",
+		Reverse: true,
+	})
+	s.Require().NoError(err)
+	s.NotNil(state)
+	s.Equal(2, len(state))
+	s.Equal(state[0].GetModifyTime().After(state[1].GetModifyTime()), true)
+
 }
 
 func (s *BoltJobstoreTestSuite) TestInProgressJobs() {
 	infos, err := s.store.GetInProgressJobs(s.ctx, "")
 	s.Require().NoError(err)
-	s.Require().Equal(3, len(infos))
+	s.Require().Equal(4, len(infos))
 	s.Require().Equal("130", infos[0].ID)
 
 	infos, err = s.store.GetInProgressJobs(s.ctx, "batch")
 	s.Require().NoError(err)
-	s.Require().Equal(2, len(infos))
+	s.Require().Equal(3, len(infos))
 	s.Require().Equal("130", infos[0].ID)
 
 	infos, err = s.store.GetInProgressJobs(s.ctx, "daemon")