Repository: bacalhau-project/bacalhau (language: python)

Issue:
When describing a job that doesn't exist bacalhau returns a 500 error instead of expected 404
### Bug Description
Run the command with a jobID that doesn't exist.
```
$ bacalhau job describe j-abcdefgh-ijkl-mnop-qrstu-vwzwzabdefg
```
observe the output:
```
06:41:13.987 | INF pkg/repo/fs.go:99 > Initializing repo at '/home/frrist/.bacalhau' for environment 'production'
Error: could not get job j-86588a34-9abf-414d-9148-94e21ddcb71b: Unexpected response code: 500 ({
  "error": "Job not found. ID: j-86588a34-9abf-414d-9148-94e21ddcb71b",
  "message": "Internal Server Error"
``` 

### Expected Behavior
This should produce a 404 error instead of 500.


### Bacalhau Versions
- **Agent Version**:  v1.4.0
- **CLI Client Version**: v1.4.0



Discussion / hints:
well.... TECHNICALLY the api should produce the 404 error with a message, but the UI should produce a 200 (because it was a valid exchange with the requester)


Fix patch:
diff --git a/cmd/cli/job/describe.go b/cmd/cli/job/describe.go
index c31cf1f363..0a45ce7c9b 100644
--- a/cmd/cli/job/describe.go
+++ b/cmd/cli/job/describe.go
@@ -82,13 +82,14 @@ func NewDescribeCmd() *cobra.Command {
 func (o *DescribeOptions) run(cmd *cobra.Command, args []string, api client.API) error {
 	ctx := cmd.Context()
 	jobID := args[0]
+
 	response, err := api.Jobs().Get(ctx, &apimodels.GetJobRequest{
 		JobID:   jobID,
 		Include: "executions,history",
 	})
 
 	if err != nil {
-		return fmt.Errorf("could not get job %s: %w", jobID, err)
+		return err
 	}
 
 	if o.OutputOpts.Format != "" {
diff --git a/cmd/cli/job/executions.go b/cmd/cli/job/executions.go
index 94b9d25483..23b5ff61ed 100644
--- a/cmd/cli/job/executions.go
+++ b/cmd/cli/job/executions.go
@@ -1,6 +1,7 @@
 package job
 
 import (
+	"errors"
 	"fmt"
 	"strconv"
 	"time"
@@ -147,7 +148,7 @@ func (o *ExecutionOptions) run(cmd *cobra.Command, args []string, api client.API
 		},
 	})
 	if err != nil {
-		return err
+		return errors.New(err.Error())
 	}
 
 	if err = output.Output(cmd, executionColumns, o.OutputOptions, response.Items); err != nil {
diff --git a/cmd/cli/job/get.go b/cmd/cli/job/get.go
index 411b5bcd4d..c38ed8f000 100644
--- a/cmd/cli/job/get.go
+++ b/cmd/cli/job/get.go
@@ -108,7 +108,7 @@ func get(cmd *cobra.Command, cmdArgs []string, api client.API, cfg types.Bacalha
 		jobID,
 		OG.DownloadSettings,
 	); err != nil {
-		return fmt.Errorf("downloading job: %w", err)
+		return err
 	}
 
 	return nil
diff --git a/cmd/cli/job/history.go b/cmd/cli/job/history.go
index 2fd5da17ab..2cc2178a0d 100644
--- a/cmd/cli/job/history.go
+++ b/cmd/cli/job/history.go
@@ -1,6 +1,7 @@
 package job
 
 import (
+	"errors"
 	"fmt"
 	"time"
 
@@ -160,7 +161,7 @@ func (o *HistoryOptions) run(cmd *cobra.Command, args []string, api client.API)
 		},
 	})
 	if err != nil {
-		return err
+		return errors.New(err.Error())
 	}
 
 	if err = output.Output(cmd, historyColumns, o.OutputOptions, response.Items); err != nil {
diff --git a/cmd/util/download.go b/cmd/util/download.go
index 804cedcbc6..696a258453 100644
--- a/cmd/util/download.go
+++ b/cmd/util/download.go
@@ -3,6 +3,7 @@ package util
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -33,7 +34,7 @@ func DownloadResultsHandler(
 		JobID: jobID,
 	})
 	if err != nil {
-		Fatal(cmd, fmt.Errorf("could not get results for job %s: %w", jobID, err), 1)
+		return errors.New(err.Error())
 	}
 
 	if len(response.Items) == 0 {
diff --git a/pkg/jobstore/boltdb/errors.go b/pkg/jobstore/boltdb/errors.go
new file mode 100644
index 0000000000..21f2dddc37
--- /dev/null
+++ b/pkg/jobstore/boltdb/errors.go
@@ -0,0 +1,31 @@
+package boltjobstore
+
+import (
+	"errors"
+
+	"github.com/bacalhau-project/bacalhau/pkg/models"
+	"go.etcd.io/bbolt"
+)
+
+const BOLTDB_COMPONENT = "BDB"
+
+func NewBoltDbError(err error) *models.BaseError {
+	switch {
+	case errors.Is(err, bbolt.ErrBucketNotFound):
+		return models.NewBaseError(err.Error()).WithCode(models.NewErrorCode(BOLTDB_COMPONENT, 404))
+	case errors.Is(err, bbolt.ErrBucketExists):
+		return models.NewBaseError(err.Error()).WithCode(models.NewErrorCode(BOLTDB_COMPONENT, 409))
+	case errors.Is(err, bbolt.ErrTxNotWritable):
+		return models.NewBaseError(err.Error()).WithCode(models.NewErrorCode(BOLTDB_COMPONENT, 500))
+	case errors.Is(err, bbolt.ErrIncompatibleValue):
+		return models.NewBaseError(err.Error()).WithCode(models.NewErrorCode(BOLTDB_COMPONENT, 500))
+	case errors.Is(err, bbolt.ErrKeyRequired):
+		return models.NewBaseError(err.Error()).WithCode(models.NewErrorCode(BOLTDB_COMPONENT, 500))
+	case errors.Is(err, bbolt.ErrKeyTooLarge):
+		return models.NewBaseError(err.Error()).WithCode(models.NewErrorCode(BOLTDB_COMPONENT, 500))
+	case errors.Is(err, bbolt.ErrValueTooLarge):
+		return models.NewBaseError(err.Error()).WithCode(models.NewErrorCode(BOLTDB_COMPONENT, 500))
+	default:
+		return models.NewBaseError(err.Error()).WithCode(models.NewErrorCode(BOLTDB_COMPONENT, 500))
+	}
+}
diff --git a/pkg/jobstore/boltdb/store.go b/pkg/jobstore/boltdb/store.go
index 18d6b3f297..e330969516 100644
--- a/pkg/jobstore/boltdb/store.go
+++ b/pkg/jobstore/boltdb/store.go
@@ -3,6 +3,7 @@ package boltjobstore
 import (
 	"bytes"
 	"context"
+	"errors"
 	"fmt"
 	"slices"
 	"sort"
@@ -20,6 +21,7 @@ import (
 	"github.com/bacalhau-project/bacalhau/pkg/lib/marshaller"
 	"github.com/bacalhau-project/bacalhau/pkg/lib/math"
 	"github.com/bacalhau-project/bacalhau/pkg/models"
+	"github.com/bacalhau-project/bacalhau/pkg/publicapi/apimodels"
 	"github.com/bacalhau-project/bacalhau/pkg/util"
 	"github.com/bacalhau-project/bacalhau/pkg/util/idgen"
 )
@@ -167,6 +169,7 @@ func (b *BoltJobStore) GetJob(ctx context.Context, id string) (models.Job, error
 }
 
 func (b *BoltJobStore) getJob(tx *bolt.Tx, jobID string) (models.Job, error) {
+
 	var job models.Job
 
 	jobID, err := b.reifyJobID(tx, jobID)
@@ -176,7 +179,7 @@ func (b *BoltJobStore) getJob(tx *bolt.Tx, jobID string) (models.Job, error) {
 
 	data := GetBucketData(tx, NewBucketPath(BucketJobs, jobID), SpecKey)
 	if data == nil {
-		return job, bacerrors.NewJobNotFound(jobID)
+		return job, apimodels.NewJobNotFound(jobID)
 	}
 
 	err = b.marshaller.Unmarshal(data, &job)
@@ -189,7 +192,7 @@ func (b *BoltJobStore) reifyJobID(tx *bolt.Tx, jobID string) (string, error) {
 	if idgen.ShortUUID(jobID) == jobID {
 		bktJobs, err := NewBucketPath(BucketJobs).Get(tx, false)
 		if err != nil {
-			return "", err
+			return "", NewBoltDbError(err)
 		}
 
 		found := make([]string, 0, 1)
@@ -202,11 +205,11 @@ func (b *BoltJobStore) reifyJobID(tx *bolt.Tx, jobID string) (string, error) {
 
 		switch len(found) {
 		case 0:
-			return "", bacerrors.NewJobNotFound(jobID)
+			return "", jobstore.NewErrJobNotFound(jobID)
 		case 1:
 			return found[0], nil
 		default:
-			return "", bacerrors.NewMultipleJobsFound(jobID, found)
+			return "", jobstore.NewErrMultipleJobsFound(jobID)
 		}
 	}
 
@@ -223,7 +226,7 @@ func (b *BoltJobStore) getExecution(tx *bolt.Tx, id string) (models.Execution, e
 	}
 
 	if bkt, err := NewBucketPath(BucketJobs, key, BucketJobExecutions).Get(tx, false); err != nil {
-		return exec, err
+		return exec, NewBoltDbError(err)
 	} else {
 		data := bkt.Get([]byte(id))
 		if data == nil {
@@ -242,11 +245,11 @@ func (b *BoltJobStore) getExecution(tx *bolt.Tx, id string) (models.Execution, e
 func (b *BoltJobStore) getExecutionJobID(tx *bolt.Tx, id string) (string, error) {
 	keys, err := b.executionsIndex.List(tx, []byte(id))
 	if err != nil {
-		return "", err
+		return "", NewBoltDbError(err)
 	}
 
 	if len(keys) != 1 {
-		return "", fmt.Errorf("too many leaf nodes in execution index")
+		return "", jobstore.NewJobStoreError("too many leaf nodes in execution index")
 	}
 
 	return string(keys[0]), nil
@@ -297,7 +300,7 @@ func (b *BoltJobStore) getExecutions(tx *bolt.Tx, options jobstore.GetExecutions
 
 	bkt, err := NewBucketPath(BucketJobs, jobID, BucketJobExecutions).Get(tx, false)
 	if err != nil {
-		return nil, err
+		return nil, NewBoltDbError(err)
 	}
 
 	var execs []models.Execution
@@ -421,7 +424,7 @@ func (b *BoltJobStore) getJobsInitialSet(tx *bolt.Tx, query jobstore.JobQuery) (
 	if query.ReturnAll || query.Namespace == "" {
 		bkt, err := NewBucketPath(BucketJobs).Get(tx, false)
 		if err != nil {
-			return nil, err
+			return nil, NewBoltDbError(err)
 		}
 
 		err = bkt.ForEachBucket(func(k []byte) error {
@@ -429,12 +432,12 @@ func (b *BoltJobStore) getJobsInitialSet(tx *bolt.Tx, query jobstore.JobQuery) (
 			return nil
 		})
 		if err != nil {
-			return nil, err
+			return nil, NewBoltDbError(err)
 		}
 	} else {
 		ids, err := b.namespacesIndex.List(tx, []byte(query.Namespace))
 		if err != nil {
-			return nil, err
+			return nil, NewBoltDbError(err)
 		}
 
 		for _, k := range ids {
@@ -455,7 +458,7 @@ func (b *BoltJobStore) getJobsIncludeTags(tx *bolt.Tx, jobSet map[string]struct{
 		tagLabel := []byte(strings.ToLower(tag))
 		ids, err := b.tagsIndex.List(tx, tagLabel)
 		if err != nil {
-			return nil, err
+			return nil, NewBoltDbError(err)
 		}
 
 		for _, k := range ids {
@@ -483,7 +486,7 @@ func (b *BoltJobStore) getJobsExcludeTags(tx *bolt.Tx, jobSet map[string]struct{
 		tagLabel := []byte(strings.ToLower(tag))
 		ids, err := b.tagsIndex.List(tx, tagLabel)
 		if err != nil {
-			return nil, err
+			return nil, NewBoltDbError(err)
 		}
 
 		for _, k := range ids {
@@ -584,7 +587,7 @@ func (b *BoltJobStore) getInProgressJobs(tx *bolt.Tx, jobType string) ([]models.
 
 	keys, err := b.inProgressIndex.List(tx)
 	if err != nil {
-		return nil, err
+		return nil, NewBoltDbError(err)
 	}
 
 	for _, jobIDKey := range keys {
@@ -758,7 +761,7 @@ func (b *BoltJobStore) CreateJob(ctx context.Context, job models.Job) error {
 	job.Normalize()
 	err := job.Validate()
 	if err != nil {
-		return err
+		return jobstore.NewJobStoreError(err.Error())
 	}
 	return b.update(ctx, func(tx *bolt.Tx) (err error) {
 		return b.createJob(tx, job)
@@ -777,7 +780,7 @@ func (b *BoltJobStore) update(ctx context.Context, update func(tx *bolt.Tx) erro
 	tx, externalTx = txFromContext(ctx)
 	if externalTx {
 		if !tx.Writable() {
-			return fmt.Errorf("readonly transaction provided in context for update operation")
+			return NewBoltDbError(errors.New("readonly transaction provided in context for update operation"))
 		}
 	} else {
 		tx, err = b.database.Begin(true)
@@ -817,7 +820,7 @@ func (b *BoltJobStore) view(ctx context.Context, view func(tx *bolt.Tx) error) e
 	if !externalTx {
 		tx, err = b.database.Begin(false)
 		if err != nil {
-			return err
+			return NewBoltDbError(err)
 		}
 	}
 
@@ -844,21 +847,21 @@ func (b *BoltJobStore) createJob(tx *bolt.Tx, job models.Job) error {
 
 	jobIDKey := []byte(job.ID)
 	if bkt, err := NewBucketPath(BucketJobs, job.ID).Get(tx, true); err != nil {
-		return err
+		return NewBoltDbError(err)
 	} else {
 		// Create the evaluations and executions buckets and so forth
 		if _, err := bkt.CreateBucketIfNotExists([]byte(BucketJobExecutions)); err != nil {
 			return err
 		}
 		if _, err := bkt.CreateBucketIfNotExists([]byte(BucketJobEvaluations)); err != nil {
-			return err
+			return NewBoltDbError(err)
 		}
 		if _, err := bkt.CreateBucketIfNotExists([]byte(BucketJobHistory)); err != nil {
-			return err
+			return NewBoltDbError(err)
 		}
 
 		if _, err := bkt.CreateBucketIfNotExists([]byte(BucketExecutionHistory)); err != nil {
-			return err
+			return NewBoltDbError(err)
 		}
 	}
 
@@ -869,7 +872,7 @@ func (b *BoltJobStore) createJob(tx *bolt.Tx, job models.Job) error {
 	}
 
 	if bkt, err := NewBucketPath(BucketJobs, job.ID).Get(tx, false); err != nil {
-		return err
+		return NewBoltDbError(err)
 	} else {
 		if err = bkt.Put(SpecKey, jobData); err != nil {
 			return err
@@ -879,11 +882,11 @@ func (b *BoltJobStore) createJob(tx *bolt.Tx, job models.Job) error {
 	// Create a composite key for the in progress index
 	jobkey := createInProgressIndexKey(&job)
 	if err = b.inProgressIndex.Add(tx, []byte(jobkey)); err != nil {
-		return err
+		return NewBoltDbError(err)
 	}
 
 	if err = b.namespacesIndex.Add(tx, jobIDKey, []byte(job.Namespace)); err != nil {
-		return err
+		return NewBoltDbError(err)
 	}
 
 	// Write sentinels keys for specific tags
@@ -909,7 +912,7 @@ func (b *BoltJobStore) deleteJob(tx *bolt.Tx, jobID string) error {
 
 	job, err := b.getJob(tx, jobID)
 	if err != nil {
-		return bacerrors.NewJobNotFound(jobID)
+		apimodels.NewJobNotFound(jobID)
 	}
 
 	tx.OnCommit(func() {
@@ -918,7 +921,7 @@ func (b *BoltJobStore) deleteJob(tx *bolt.Tx, jobID string) error {
 
 	// Delete the Job bucket (and everything within it)
 	if bkt, err := NewBucketPath(BucketJobs).Get(tx, false); err != nil {
-		return err
+		return NewBoltDbError(err)
 	} else {
 		if err = bkt.DeleteBucket([]byte(jobID)); err != nil {
 			return err
diff --git a/pkg/jobstore/errors.go b/pkg/jobstore/errors.go
index 321d473844..f0b2d52db3 100644
--- a/pkg/jobstore/errors.go
+++ b/pkg/jobstore/errors.go
@@ -6,154 +6,70 @@ import (
 	"github.com/bacalhau-project/bacalhau/pkg/models"
 )
 
-// ErrJobNotFound is returned when the job is not found
-type ErrJobNotFound struct {
-	JobID string
-}
-
-func NewErrJobNotFound(id string) ErrJobNotFound {
-	return ErrJobNotFound{JobID: id}
-}
-
-func (e ErrJobNotFound) Error() string {
-	return "job not found: " + e.JobID
-}
-
-// ErrJobAlreadyExists is returned when an job already exists
-type ErrJobAlreadyExists struct {
-	JobID string
-}
-
-func NewErrJobAlreadyExists(id string) ErrJobAlreadyExists {
-	return ErrJobAlreadyExists{JobID: id}
-}
+const (
+	JOB_STORE_COMPONENT = "JBS"
+)
 
-func (e ErrJobAlreadyExists) Error() string {
-	return "job already exists: " + e.JobID
+func NewErrJobNotFound(id string) *models.BaseError {
+	return models.NewBaseError("job not found: %s", id).WithCode(models.NewErrorCode(JOB_STORE_COMPONENT, 404))
 }
 
-// ErrInvalidJobState is returned when an job is in an invalid state.
-type ErrInvalidJobState struct {
-	JobID    string
-	Actual   models.JobStateType
-	Expected models.JobStateType
+func NewErrMultipleJobsFound(id string) *models.BaseError {
+	return models.NewBaseError("multiple jobs found for id %s", id).WithCode(models.NewErrorCode(JOB_STORE_COMPONENT, 400))
 }
 
-func NewErrInvalidJobState(id string, actual models.JobStateType, expected models.JobStateType) ErrInvalidJobState {
-	return ErrInvalidJobState{JobID: id, Actual: actual, Expected: expected}
+func NewErrJobAlreadyExists(id string) *models.BaseError {
+	return models.NewBaseError("job already exists: %s", id)
 }
 
-func (e ErrInvalidJobState) Error() string {
-	if e.Expected.IsUndefined() {
-		return fmt.Sprintf("job %s is in unexpected state %s", e.JobID, e.Actual)
+func NewErrInvalidJobState(id string, actual models.JobStateType, expected models.JobStateType) *models.BaseError {
+	var errorFormat string
+	if expected.IsUndefined() {
+		errorFormat = "job %s is in unexpected state %s"
+	} else {
+		errorFormat = "job %s is in state %s but expected %s"
 	}
-	return fmt.Sprintf("job %s is in state %s but expected %s", e.JobID, e.Actual, e.Expected)
-}
-
-// ErrInvalidJobVersion is returned when an job has an invalid version.
-type ErrInvalidJobVersion struct {
-	JobID    string
-	Actual   uint64
-	Expected uint64
-}
-
-func NewErrInvalidJobVersion(id string, actual, expected uint64) ErrInvalidJobVersion {
-	return ErrInvalidJobVersion{JobID: id, Actual: actual, Expected: expected}
-}
-
-func (e ErrInvalidJobVersion) Error() string {
-	return fmt.Sprintf("job %s has version %d but expected %d", e.JobID, e.Actual, e.Expected)
-}
 
-// ErrJobAlreadyTerminal is returned when an job is already in terminal state and cannot be updated.
-type ErrJobAlreadyTerminal struct {
-	JobID    string
-	Actual   models.JobStateType
-	NewState models.JobStateType
+	return models.NewBaseError(errorFormat, id, actual)
 }
 
-func NewErrJobAlreadyTerminal(id string, actual models.JobStateType, newState models.JobStateType) ErrJobAlreadyTerminal {
-	return ErrJobAlreadyTerminal{JobID: id, Actual: actual, NewState: newState}
+func NewErrInvalidJobVersion(id string, actual, expected uint64) *models.BaseError {
+	errorMessage := fmt.Sprintf("job %s has version %d but expected %d", id, actual, expected)
+	return models.NewBaseError(errorMessage)
 }
 
-func (e ErrJobAlreadyTerminal) Error() string {
-	return fmt.Sprintf("job %s is in terminal state %s and cannot transition to %s",
-		e.JobID, e.Actual, e.NewState)
+func NewErrJobAlreadyTerminal(id string, actual models.JobStateType, newState models.JobStateType) *models.BaseError {
+	errorMessage := fmt.Sprintf("job %s is in terminal state %s and connt transition to %s", id, actual, newState)
+	return models.NewBaseError(errorMessage)
 }
 
-// ErrExecutionNotFound is returned when an job already exists
-type ErrExecutionNotFound struct {
-	ExecutionID string
+func NewErrExecutionNotFound(id string) *models.BaseError {
+	return models.NewBaseError("execution not found: %s", id)
 }
 
-func NewErrExecutionNotFound(id string) ErrExecutionNotFound {
-	return ErrExecutionNotFound{ExecutionID: id}
+func NewErrExecutionAlreadyExists(id string) *models.BaseError {
+	return models.NewBaseError("execution already exists %s", id)
 }
 
-func (e ErrExecutionNotFound) Error() string {
-	return "execution not found: " + e.ExecutionID
-}
-
-// ErrExecutionAlreadyExists is returned when an job already exists
-type ErrExecutionAlreadyExists struct {
-	ExecutionID string
-}
-
-func NewErrExecutionAlreadyExists(id string) ErrExecutionAlreadyExists {
-	return ErrExecutionAlreadyExists{ExecutionID: id}
-}
-
-func (e ErrExecutionAlreadyExists) Error() string {
-	return "execution already exists: " + e.ExecutionID
-}
-
-// ErrInvalidExecutionState is returned when an execution is in an invalid state.
-type ErrInvalidExecutionState struct {
-	ExecutionID string
-	Actual      models.ExecutionStateType
-	Expected    []models.ExecutionStateType
-}
-
-func NewErrInvalidExecutionState(
-	id string, actual models.ExecutionStateType, expected ...models.ExecutionStateType) ErrInvalidExecutionState {
-	return ErrInvalidExecutionState{ExecutionID: id, Actual: actual, Expected: expected}
-}
-
-func (e ErrInvalidExecutionState) Error() string {
-	if len(e.Expected) > 0 {
-		return fmt.Sprintf("execution %s is in unexpected state %s", e.ExecutionID, e.Actual)
+func NewErrInvalidExecutionState(id string, actual models.ExecutionStateType, expected ...models.ExecutionStateType) *models.BaseError {
+	var errorMessage string
+	if len(expected) > 0 {
+		errorMessage = fmt.Sprintf("execution %s is in unexpected state %s", id, actual)
+	} else {
+		errorMessage = fmt.Sprintf("execution %s is in state %s, but expected %s", id, actual, expected)
 	}
-	return fmt.Sprintf("execution %s is in state %s, but expected %s", e.ExecutionID, e.Actual, e.Expected)
-}
-
-// ErrInvalidExecutionVersion is returned when an execution has an invalid version.
-type ErrInvalidExecutionVersion struct {
-	ExecutionID string
-	Actual      uint64
-	Expected    uint64
-}
-
-func NewErrInvalidExecutionVersion(id string, actual, expected uint64) ErrInvalidExecutionVersion {
-	return ErrInvalidExecutionVersion{ExecutionID: id, Actual: actual, Expected: expected}
+	return models.NewBaseError(errorMessage)
 }
 
-func (e ErrInvalidExecutionVersion) Error() string {
-	return fmt.Sprintf("execution %s has version %d but expected %d", e.ExecutionID, e.Actual, e.Expected)
-}
+func NewErrInvalidExecutionVersion(id string, actual, expected uint64) *models.BaseError {
+	return models.NewBaseError("execution %s has version %d but expected %d", id, actual, expected)
 
-// ErrExecutionAlreadyTerminal is returned when an execution is already in terminal state and cannot be updated.
-type ErrExecutionAlreadyTerminal struct {
-	ExecutionID string
-	Actual      models.ExecutionStateType
-	NewState    models.ExecutionStateType
 }
 
-func NewErrExecutionAlreadyTerminal(
-	id string, actual models.ExecutionStateType, newState models.ExecutionStateType) ErrExecutionAlreadyTerminal {
-	return ErrExecutionAlreadyTerminal{ExecutionID: id, Actual: actual, NewState: newState}
+func NewErrExecutionAlreadyTerminal(id string, actual models.ExecutionStateType, newState models.ExecutionStateType) *models.BaseError {
+	return models.NewBaseError("execution %s is in terminal state %s and cannot transition to %s", id, actual, newState)
 }
 
-func (e ErrExecutionAlreadyTerminal) Error() string {
-	return fmt.Sprintf("execution %s is in terminal state %s and cannot transition to %s",
-		e.ExecutionID, e.Actual, e.NewState)
+func NewJobStoreError(message string) *models.BaseError {
+	return models.NewBaseError(message)
 }
diff --git a/pkg/models/error.go b/pkg/models/error.go
index a4289c0b5e..90db5cf7b0 100644
--- a/pkg/models/error.go
+++ b/pkg/models/error.go
@@ -41,6 +41,38 @@ type HasDetails interface {
 	Details() map[string]string
 }
 
+type HasCode interface {
+	// Details a code
+	Code() int
+}
+
+type ErrorCode struct {
+	component      string
+	httpStatusCode int
+}
+
+func NewErrorCode(component string, httpStatusCode int) ErrorCode {
+	return ErrorCode{component: component, httpStatusCode: httpStatusCode}
+}
+
+func (ec ErrorCode) HTTPStatusCode() int {
+	if ec.httpStatusCode == 0 {
+		return 500
+	}
+	return ec.httpStatusCode
+}
+
+func (ec ErrorCode) Component() string {
+	return ec.component
+}
+
+func (ec ErrorCode) String() string {
+	if ec.httpStatusCode == 0 {
+		return fmt.Sprintf("%s-500", ec.component)
+	}
+	return fmt.Sprintf("%s-%d", ec.component, ec.httpStatusCode)
+}
+
 // BaseError is a custom error type in Go that provides additional fields
 // and methods for more detailed error handling. It implements the error
 // interface, as well as additional interfaces for providing a hint,
@@ -52,6 +84,7 @@ type BaseError struct {
 	retryable      bool
 	failsExecution bool
 	details        map[string]string
+	code           ErrorCode
 }
 
 // NewBaseError is a constructor function that creates a new BaseError with
@@ -88,6 +121,13 @@ func (e *BaseError) WithDetails(details map[string]string) *BaseError {
 	return e
 }
 
+// WithCode is a method that sets the code field of BaseError and
+// returns the BaseError itself for chaining
+func (e *BaseError) WithCode(code ErrorCode) *BaseError {
+	e.code = code
+	return e
+}
+
 // Error is a method that returns the message field of BaseError. This
 // method makes BaseError satisfy the error interface.
 func (e *BaseError) Error() string {
@@ -113,3 +153,8 @@ func (e *BaseError) FailsExecution() bool {
 func (e *BaseError) Details() map[string]string {
 	return e.details
 }
+
+// Details a Unique Code to identify the error
+func (e *BaseError) Code() ErrorCode {
+	return e.code
+}
diff --git a/pkg/orchestrator/endpoint.go b/pkg/orchestrator/endpoint.go
index 60e5f7d17a..9a2bb9b2cb 100644
--- a/pkg/orchestrator/endpoint.go
+++ b/pkg/orchestrator/endpoint.go
@@ -148,14 +148,14 @@ func (e *BaseEndpoint) StopJob(ctx context.Context, request *StopJobRequest) (St
 		// no need to stop a job that is already stopped
 		return StopJobResponse{}, nil
 	case models.JobStateTypeCompleted:
-		return StopJobResponse{}, fmt.Errorf("cannot stop job in state %s", job.State.StateType)
+		return StopJobResponse{}, models.NewBaseError("cannot stop job in state %s", job.State.StateType)
 	default:
 		// continue
 	}
 
 	txContext, err := e.store.BeginTx(ctx)
 	if err != nil {
-		return StopJobResponse{}, fmt.Errorf("failed to begin transaction: %w", err)
+		return StopJobResponse{}, jobstore.NewJobStoreError(err.Error())
 	}
 
 	defer func() {
diff --git a/pkg/publicapi/apimodels/error.go b/pkg/publicapi/apimodels/error.go
new file mode 100644
index 0000000000..236523a006
--- /dev/null
+++ b/pkg/publicapi/apimodels/error.go
@@ -0,0 +1,134 @@
+package apimodels
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+)
+
+// apierror represents a standardized error response for the api.
+//
+// it encapsulates:
+//   - an http status code
+//   - a human-readable error message
+//
+// purpose:
+//   - primarily used to send synchronous errors from the orchestrator endpoint
+//   - provides a structured json error response to http clients
+//
+// usage:
+//   - when the cli interacts with an orchestrator node via an http client:
+//     1. the http client receives this structured json error
+//     2. the human-readable message is output to stdout
+//     3. the http status code allows for programmatic handling of different error types
+//
+// benefits:
+//   - ensures consistent error formatting across the api
+//   - facilitates both user-friendly error messages and machine-readable error codes
+type APIError struct {
+	// httpstatuscode is the http status code associated with this error.
+	// it should correspond to standard http status codes (e.g., 400, 404, 500).
+	HTTPStatusCode int `json:"-"`
+
+	// message is a short, human-readable description of the error.
+	// it should be concise and provide a clear indication of what went wrong.
+	Message string `json:"message"`
+
+	// RequestID is the request ID of the request that caused the error.
+	RequestID string `json:"-"`
+}
+
+// NewAPIError creates a new APIError with the given HTTP status code and message.
+func NewAPIError(statusCode int, message string) *APIError {
+	return &APIError{
+		HTTPStatusCode: statusCode,
+		Message:        message,
+	}
+}
+
+// NewBadRequestError creates an APIError for Bad Request (400) errors.
+func NewBadRequestError(message string) *APIError {
+	return NewAPIError(http.StatusBadRequest, message)
+}
+
+// NewUnauthorizedError creates an APIError for Unauthorized (401) errors.
+func NewUnauthorizedError(message string) *APIError {
+	return NewAPIError(http.StatusUnauthorized, message)
+}
+
+// NewForbiddenError creates an APIError for Forbidden (403) errors.
+func NewForbiddenError(message string) *APIError {
+	return NewAPIError(http.StatusForbidden, message)
+}
+
+// NewNotFoundError creates an APIError for Not Found (404) errors.
+func NewNotFoundError(message string) *APIError {
+	return NewAPIError(http.StatusNotFound, message)
+}
+
+// NewConflictError creates an APIError for Conflict (409) errors.
+func NewConflictError(message string) *APIError {
+	return NewAPIError(http.StatusConflict, message)
+}
+
+// NewInternalServerError creates an APIError for Internal Server Error (500) errors.
+func NewInternalServerError(message string) *APIError {
+	return NewAPIError(http.StatusInternalServerError, message)
+}
+
+func NewJobNotFound(jobID string) *APIError {
+	return NewAPIError(http.StatusNotFound, fmt.Sprintf("job id %s not found", jobID))
+}
+
+// IsNotFound checks if the error is an APIError with a Not Found status.
+func IsNotFound(err error) bool {
+	apiErr, ok := err.(*APIError)
+	return ok && apiErr.HTTPStatusCode == http.StatusNotFound
+}
+
+// IsBadRequest checks if the error is an APIError with a Bad Request status.
+func IsBadRequest(err error) bool {
+	apiErr, ok := err.(*APIError)
+	return ok && apiErr.HTTPStatusCode == http.StatusBadRequest
+}
+
+// IsInternalServerError checks if the error is an APIError with an Internal Server Error status.
+func IsInternalServerError(err error) bool {
+	apiErr, ok := err.(*APIError)
+	return ok && apiErr.HTTPStatusCode == http.StatusInternalServerError
+}
+
+// Error implements the error interface, allowing APIError to be used as a standard Go error.
+func (e *APIError) Error() string {
+	return e.Message
+}
+
+// Parse HTTP Resposne to APIError
+func FromHttpResponse(resp *http.Response) (*APIError, error) {
+
+	if resp == nil {
+		return nil, errors.New("response is nil, cannot be unmarsheld to APIError")
+	}
+
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("error reading response body: %w", err)
+	}
+
+	var apiErr APIError
+	err = json.Unmarshal(body, &apiErr)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing response body: %w", err)
+	}
+
+	// If the JSON didn't include a status code, use the HTTP Status
+	if apiErr.HTTPStatusCode == 0 {
+		apiErr.HTTPStatusCode = resp.StatusCode
+	}
+
+	return &apiErr, nil
+}
diff --git a/pkg/publicapi/client/v2/client.go b/pkg/publicapi/client/v2/client.go
index 56c0fe43c1..6e0b9d377d 100644
--- a/pkg/publicapi/client/v2/client.go
+++ b/pkg/publicapi/client/v2/client.go
@@ -55,12 +55,28 @@ type httpClient struct {
 // and deserialize the response into a response object
 func (c *httpClient) Get(ctx context.Context, endpoint string, in apimodels.GetRequest, out apimodels.GetResponse) error {
 	r := in.ToHTTPRequest()
-	_, resp, err := requireOK(c.doRequest(ctx, http.MethodGet, endpoint, r)) //nolint:bodyclose // this is being closed
-	if err != nil && resp != nil && resp.StatusCode == http.StatusUnauthorized {
-		return apimodels.ErrInvalidToken
-	} else if err != nil {
+
+	_, resp, err := c.doRequest(ctx, http.MethodGet, endpoint, r) //nolint:bodyclose // this is being closed
+	if err != nil {
 		return err
 	}
+
+	if resp.StatusCode == http.StatusUnauthorized {
+		return apimodels.NewUnauthorizedError("invalid token")
+	}
+
+	var apiError *apimodels.APIError
+	if resp.StatusCode != http.StatusOK {
+		apiError, err = apimodels.FromHttpResponse(resp)
+		if err != nil {
+			return err
+		}
+	}
+
+	if apiError != nil {
+		return apiError
+	}
+
 	defer resp.Body.Close()
 
 	if out != nil {
@@ -80,13 +96,28 @@ func (c *httpClient) write(ctx context.Context, verb, endpoint string, in apimod
 	if r.BodyObj == nil && r.Body == nil {
 		r.BodyObj = in
 	}
-	_, resp, err := requireOK(c.doRequest(ctx, verb, endpoint, r)) //nolint:bodyclose // this is being closed
-	if err != nil && resp != nil && resp.StatusCode == http.StatusUnauthorized {
-		return apimodels.ErrInvalidToken
-	} else if err != nil {
+
+	_, resp, err := c.doRequest(ctx, verb, endpoint, r) //nolint:bodyclose // this is being closed
+	defer resp.Body.Close()
+	if err != nil {
 		return err
 	}
-	defer resp.Body.Close()
+
+	if resp.StatusCode == http.StatusUnauthorized {
+		return apimodels.ErrInvalidToken
+	}
+
+	var apiError *apimodels.APIError
+	if resp.StatusCode != http.StatusOK {
+		apiError, err = apimodels.FromHttpResponse(resp)
+		if err != nil {
+			return err
+		}
+	}
+
+	if apiError != nil {
+		return apiError
+	}
 
 	if out != nil {
 		if err := decodeBody(resp, &out); err != nil {
@@ -94,6 +125,7 @@ func (c *httpClient) write(ctx context.Context, verb, endpoint string, in apimod
 		}
 		out.Normalize()
 	}
+
 	return nil
 }
 
diff --git a/pkg/publicapi/client/v2/errors.go b/pkg/publicapi/client/v2/errors.go
index 529555f588..54d1dee929 100644
--- a/pkg/publicapi/client/v2/errors.go
+++ b/pkg/publicapi/client/v2/errors.go
@@ -165,13 +165,6 @@ func requireOK(d time.Duration, resp *http.Response, e error) (time.Duration, *h
 // response codes and validates that the received response code is among them
 func requireStatusIn(statuses ...int) doRequestWrapper {
 	return func(d time.Duration, resp *http.Response, e error) (time.Duration, *http.Response, error) {
-		if e != nil {
-			if resp != nil {
-				_ = resp.Body.Close()
-			}
-			return d, nil, e
-		}
-
 		for _, status := range statuses {
 			if resp.StatusCode == status {
 				return d, resp, nil
diff --git a/pkg/publicapi/middleware/error_handler.go b/pkg/publicapi/middleware/error_handler.go
new file mode 100644
index 0000000000..6631f54312
--- /dev/null
+++ b/pkg/publicapi/middleware/error_handler.go
@@ -0,0 +1,63 @@
+package middleware
+
+import (
+	"net/http"
+
+	"github.com/bacalhau-project/bacalhau/pkg/models"
+	"github.com/bacalhau-project/bacalhau/pkg/publicapi/apimodels"
+	"github.com/labstack/echo/v4"
+	"github.com/rs/zerolog/log"
+)
+
+func CustomHTTPErrorHandler(err error, c echo.Context) {
+
+	var code int
+	var message string
+
+	switch e := err.(type) {
+
+	case *models.BaseError:
+		// If it is already our custom APIError, use its code and message
+		code = e.Code().HTTPStatusCode()
+		message = e.Error()
+
+	case *echo.HTTPError:
+		// This is needed, in case any other middleware throws an error. In
+		// such a scenario we just use it as the error code and the message.
+		// One such example being when request body size is larger then the max
+		// size accepted
+		code = e.Code
+		message = e.Message.(string)
+
+	default:
+		// In an ideal world this should never happen. We should always have are errors
+		// from server as APIError. If output is this generic string, one should evaluate
+		// and map it to APIError and send in appropriate message.= http.StatusInternalServerError
+		message = "internal server error"
+		code = c.Response().Status
+
+		if c.Echo().Debug {
+			message = err.Error()
+		}
+	}
+
+	requestID := c.Request().Header.Get(echo.HeaderXRequestID)
+
+	// Don't override the status code if it is already been set.
+	// This is something that is advised by ECHO framework.
+	if !c.Response().Committed {
+		if c.Request().Method == http.MethodHead {
+			err = c.NoContent(code)
+		} else {
+			err = c.JSON(code, apimodels.APIError{
+				HTTPStatusCode: code,
+				Message:        message,
+				RequestID:      requestID,
+			})
+		}
+		if err != nil {
+			log.Info().Msg("unable to send json response while handling error.")
+		}
+	}
+
+}
diff --git a/pkg/publicapi/server.go b/pkg/publicapi/server.go
index e4ebcc4fdd..f6fa6a5433 100644
--- a/pkg/publicapi/server.go
+++ b/pkg/publicapi/server.go
@@ -90,12 +90,13 @@ func NewAPIServer(params ServerParams) (*Server, error) {
 	server.Router.Binder = NewNormalizeBinder()
 	server.Router.Validator = NewCustomValidator()
 
-	// enable debug mode to get clearer error messages
-	// TODO: disable debug mode after we implement our own error handler
-	server.Router.Debug = true
-
 	// set middleware
 	logLevel, err := zerolog.ParseLevel(params.Config.LogLevel)
+	if logLevel == zerolog.DebugLevel {
+		// enable debug mode to get clearer error messages
+		server.Router.Debug = true
+	}
+
 	if err != nil {
 		return nil, err
 	}
@@ -140,6 +141,10 @@ func NewAPIServer(params ServerParams) (*Server, error) {
 		middleware.VersionNotifyLogger(middlewareLogger, *serverVersion),
 	)
 
+	// Add custom http error handler. This is a centralized error handler for
+	// the server
+	server.Router.HTTPErrorHandler = middleware.CustomHTTPErrorHandler
+
 	var tlsConfig *tls.Config
 	if params.AutoCertDomain != "" {
 		log.Ctx(context.TODO()).Debug().Msgf("Setting up auto-cert for %s", params.AutoCertDomain)


Test patch:
diff --git a/cmd/cli/job/get_test.go b/cmd/cli/job/get_test.go
index 4d56d35ade..793d5c61f7 100644
--- a/cmd/cli/job/get_test.go
+++ b/cmd/cli/job/get_test.go
@@ -59,7 +59,7 @@ func (s *GetSuite) TestGetSingleFileFromOutputBadChoice() {
 	)
 
 	require.Error(s.T(), err, "expected error but it wasn't returned")
-	require.Contains(s.T(), getoutput, "Error: downloading job")
+	require.Contains(s.T(), getoutput, "Error: failed to find cid for missing")
 }
 
 func (s *GetSuite) TestGetSingleFileFromOutput() {
diff --git a/pkg/jobstore/boltdb/store_test.go b/pkg/jobstore/boltdb/store_test.go
index 2fa13cb94f..77b14f701e 100644
--- a/pkg/jobstore/boltdb/store_test.go
+++ b/pkg/jobstore/boltdb/store_test.go
@@ -15,7 +15,6 @@ import (
 	"github.com/stretchr/testify/suite"
 	"k8s.io/apimachinery/pkg/labels"
 
-	"github.com/bacalhau-project/bacalhau/pkg/bacerrors"
 	"github.com/bacalhau-project/bacalhau/pkg/jobstore"
 	"github.com/bacalhau-project/bacalhau/pkg/models"
 	"github.com/bacalhau-project/bacalhau/pkg/test/mock"
@@ -202,7 +201,7 @@ func (s *BoltJobstoreTestSuite) TestUnfilteredJobHistory() {
 
 	jobHistoryQueryResponse, err = s.store.GetJobHistory(s.ctx, "1", jobstore.JobHistoryQuery{})
 	s.Require().Error(err)
-	s.Require().IsType(err, &bacerrors.MultipleJobsFound{})
+	s.Require().IsType(err, &models.BaseError{})
 	s.Require().Nil(jobHistoryQueryResponse)
 }
 
@@ -543,7 +542,7 @@ func (s *BoltJobstoreTestSuite) TestGetExecutions() {
 		JobID: "100",
 	})
 	s.Require().Error(err)
-	s.Require().IsType(err, &bacerrors.JobNotFound{})
+	s.Require().IsType(err, &models.BaseError{})
 	s.Require().Nil(state)
 
 	state, err = s.store.GetExecutions(s.ctx, jobstore.GetExecutionsOptions{
@@ -557,7 +556,7 @@ func (s *BoltJobstoreTestSuite) TestGetExecutions() {
 		JobID: "1",
 	})
 	s.Require().Error(err)
-	s.Require().IsType(err, &bacerrors.MultipleJobsFound{})
+	s.Require().IsType(err, &models.BaseError{})
 	s.Require().Nil(state)
 
 	// Created At Ascending Order Sort
@@ -684,7 +683,7 @@ func (s *BoltJobstoreTestSuite) TestShortIDs() {
 	// No matches
 	_, err := s.store.GetJob(s.ctx, shortString)
 	s.Require().Error(err)
-	s.Require().IsType(err, &bacerrors.JobNotFound{})
+	s.Require().IsType(err, &models.BaseError{})
 
 	// Create and fetch the single entry
 	err = s.store.CreateJob(s.ctx, *job)
@@ -701,7 +700,7 @@ func (s *BoltJobstoreTestSuite) TestShortIDs() {
 
 	_, err = s.store.GetJob(s.ctx, shortString)
 	s.Require().Error(err)
-	s.Require().IsType(err, &bacerrors.MultipleJobsFound{})
+	s.Require().IsType(err, &models.BaseError{})
 }
 
 func (s *BoltJobstoreTestSuite) TestEvents() {