Repository: bacalhau-project/bacalhau (language: python)

Issue:
Cannot stop a job using its short ID
### Bug Description
```
frrist@cypress ~ [1]> bacalhau version
 CLIENT  SERVER  LATEST  UPDATE MESSAGE 
 v1.5.0  v1.5.0  1.5.0                  

frrist@cypress ~> bacalhau job list --limit 1000
 CREATED   ID          JOB     TYPE   STATE     
 21:50:40  j-95798596  docker  batch  Completed 
 21:55:07  j-c181116a  docker  batch  Stopped   
 21:57:18  j-48c496be  docker  batch  Queued    

frrist@cypress ~> bacalhau job stop j-48c496be
Checking job status

	Connecting to network  ................  done ✅  0.0s
	  Verifying job state  ................  done ✅  0.0s
	         Stopping job  ................  err  ❌  0.0s

Error: multiple jobs found for id j-48c496be
Hint:  to resolve this, either:
1. Ensure that the server is running and reachable at http://127.0.0.1:1234
2. Update the configuration to use a different host and port using:
   a. The '--api-host=<new_address> --api-port=<new_port>' flags with your command
   b. The '-c API.Host=<new_host> -c API.Port=<new_port>' flags with your command
   c. Set the host in a configuration file with 'bacalhau config set API.Host=<new_address>' and port with 'bacalhau config set API.Port=<new_port>'
3. If you are trying to reach the demo network, use '--api-host=bootstrap.demo.bacalhau.org' to call the network
frrist@cypress ~ [1]> bacalhau job describe j-48c496be
ID            = j-48c496be-e0e8-49a0-baef-67a2af5efaa7
Name          = Default Job
Namespace     = default
Type          = batch
State         = Queued
Message       = Job queued. not enough nodes to run job. requested: 1, available: 0, suitable: 0.
Count         = 1
Created Time  = 2024-10-21 21:57:18
Modified Time = 2024-10-21 21:57:18
Version       = 0

Summary

Job History
 TIME                 TOPIC       EVENT                                                                             
 2024-10-21 14:57:18  Submission  Job submitted                                                                     
 2024-10-21 14:57:18  Queueing    Job queued. not enough nodes to run job. requested: 1, available: 0, suitable: 0. 

Executions
 ID  NODE ID  STATE  DESIRED  REV.  CREATED  MODIFIED  COMMENT 

frrist@cypress ~> bacalhau job stop j-48c496be
Checking job status

	Connecting to network  ................  done ✅  0.0s
	  Verifying job state  ................  done ✅  0.0s
	         Stopping job  ................  err  ❌  0.0s

Error: multiple jobs found for id j-48c496be
Hint:  to resolve this, either:
1. Ensure that the server is running and reachable at http://127.0.0.1:1234
2. Update the configuration to use a different host and port using:
   a. The '--api-host=<new_address> --api-port=<new_port>' flags with your command
   b. The '-c API.Host=<new_host> -c API.Port=<new_port>' flags with your command
   c. Set the host in a configuration file with 'bacalhau config set API.Host=<new_address>' and port with 'bacalhau config set API.Port=<new_port>'
3. If you are trying to reach the demo network, use '--api-host=bootstrap.demo.bacalhau.org' to call the network

```

Strange that it can be described but not stopped.

### Bacalhau Versions
- **Agent Version**: v1.5.0
- **CLI Client Version**: v1.5.0


Discussion / hints:
<p><a href="https://linear.app/expanso/issue/ENG-290/cannot-stop-a-job-using-its-short-id">ENG-290 Cannot stop a job using its short ID</a></p>

Fix patch:
diff --git a/pkg/orchestrator/endpoint.go b/pkg/orchestrator/endpoint.go
index fd5a02fbf4..2d399480c6 100644
--- a/pkg/orchestrator/endpoint.go
+++ b/pkg/orchestrator/endpoint.go
@@ -110,7 +110,18 @@ func (e *BaseEndpoint) SubmitJob(ctx context.Context, request *SubmitJobRequest)
 }
 
 func (e *BaseEndpoint) StopJob(ctx context.Context, request *StopJobRequest) (StopJobResponse, error) {
-	job, err := e.store.GetJob(ctx, request.JobID)
+	txContext, err := e.store.BeginTx(ctx)
+	if err != nil {
+		return StopJobResponse{}, jobstore.NewJobStoreError(err.Error())
+	}
+
+	defer func() {
+		if err != nil {
+			_ = txContext.Rollback()
+		}
+	}()
+
+	job, err := e.store.GetJob(txContext, request.JobID)
 	if err != nil {
 		return StopJobResponse{}, err
 	}
@@ -124,21 +135,10 @@ func (e *BaseEndpoint) StopJob(ctx context.Context, request *StopJobRequest) (St
 		// continue
 	}
 
-	txContext, err := e.store.BeginTx(ctx)
-	if err != nil {
-		return StopJobResponse{}, jobstore.NewJobStoreError(err.Error())
-	}
-
-	defer func() {
-		if err != nil {
-			_ = txContext.Rollback()
-		}
-	}()
-
 	// update the job state, except if the job is already completed
 	// we allow marking a failed job as canceled
 	if err = e.store.UpdateJobState(txContext, jobstore.UpdateJobStateRequest{
-		JobID: request.JobID,
+		JobID: job.ID, // use the job ID from the store in case the request had a short ID
 		Condition: jobstore.UpdateJobCondition{
 			UnexpectedStates: []models.JobStateType{
 				models.JobStateTypeCompleted,
@@ -161,7 +161,7 @@ func (e *BaseEndpoint) StopJob(ctx context.Context, request *StopJobRequest) (St
 		now := time.Now().UTC().UnixNano()
 		eval := &models.Evaluation{
 			ID:          uuid.NewString(),
-			JobID:       request.JobID,
+			JobID:       job.ID,
 			TriggeredBy: models.EvalTriggerJobCancel,
 			Type:        job.Type,
 			Status:      models.EvalStatusPending,
diff --git a/pkg/publicapi/apimodels/error.go b/pkg/publicapi/apimodels/error.go
index 81845fdad9..07d62e0222 100644
--- a/pkg/publicapi/apimodels/error.go
+++ b/pkg/publicapi/apimodels/error.go
@@ -45,6 +45,12 @@ type APIError struct {
 
 	// Component is the component that caused the error.
 	Component string `json:"Component"`
+
+	// Hint is a string providing additional context or suggestions related to the error.
+	Hint string `json:"Hint,omitempty"`
+
+	// Details is a map of string key-value pairs providing additional error context.
+	Details map[string]string `json:"Details,omitempty"`
 }
 
 // NewAPIError creates a new APIError with the given HTTP status code and message.
@@ -52,6 +58,7 @@ func NewAPIError(statusCode int, message string) *APIError {
 	return &APIError{
 		HTTPStatusCode: statusCode,
 		Message:        message,
+		Details:        make(map[string]string),
 	}
 }
 
@@ -106,14 +113,22 @@ func FromBacError(err bacerrors.Error) *APIError {
 		Message:        err.Error(),
 		Code:           string(err.Code()),
 		Component:      err.Component(),
+		Hint:           err.Hint(),
+		Details:        err.Details(),
 	}
 }
 
 // ToBacError converts an APIError to a bacerror.Error
 func (e *APIError) ToBacError() bacerrors.Error {
+	details := e.Details
+	if details == nil {
+		details = make(map[string]string)
+	}
+	details["request_id"] = e.RequestID
 	return bacerrors.New(e.Error()).
 		WithHTTPStatusCode(e.HTTPStatusCode).
 		WithCode(bacerrors.Code(e.Code)).
 		WithComponent(e.Component).
-		WithDetails(map[string]string{"request_id": e.RequestID})
+		WithHint(e.Hint).
+		WithDetails(details)
 }
diff --git a/pkg/publicapi/client/v2/client.go b/pkg/publicapi/client/v2/client.go
index 95979028be..c4872876b8 100644
--- a/pkg/publicapi/client/v2/client.go
+++ b/pkg/publicapi/client/v2/client.go
@@ -309,9 +309,13 @@ func (c *httpClient) toHTTP(ctx context.Context, method, endpoint string, r *api
 }
 
 func (c *httpClient) interceptError(ctx context.Context, err error, resp *http.Response, method, endpoint string, r *apimodels.HTTPRequest) (bacErr bacerrors.Error) {
+	// Avoid adding common attributes if the error is an API error
+	var isAPIError bool
+
 	// Defer the addition of common attributes
+	// Only applied if the error is not derived from an API error
 	defer func() {
-		if bacErr != nil {
+		if bacErr != nil && !isAPIError {
 			bacErr = bacErr.
 				WithComponent(errorComponent).
 				WithDetails(map[string]string{
@@ -352,6 +356,7 @@ func (c *httpClient) interceptError(ctx context.Context, err error, resp *http.R
 
 		apiError := apimodels.GenerateAPIErrorFromHTTPResponse(resp)
 		if apiError != nil {
+			isAPIError = true
 			return apiError.ToBacError()
 		}
 
diff --git a/pkg/publicapi/middleware/error_handler.go b/pkg/publicapi/middleware/error_handler.go
index 840dc92538..7cc45a2266 100644
--- a/pkg/publicapi/middleware/error_handler.go
+++ b/pkg/publicapi/middleware/error_handler.go
@@ -11,70 +11,58 @@ import (
 )
 
 func CustomHTTPErrorHandler(err error, c echo.Context) {
-	var (
-		code      int
-		message   string
-		errorCode string
-		component string
-	)
+	if c.Response().Committed {
+		c.Echo().Logger.Warn("error handler skipped as response already committed")
+		return
+	}
+
+	var apiError *apimodels.APIError
 
 	switch e := err.(type) {
 
 	case bacerrors.Error:
-		// If it is already our custom APIError, use its code and message
-		code = e.HTTPStatusCode()
-		message = e.Error()
-		errorCode = string(e.Code())
-		component = e.Component()
+		apiError = apimodels.FromBacError(e)
 
 	case *echo.HTTPError:
 		// This is needed, in case any other middleware throws an error. In
 		// such a scenario we just use it as the error code and the message.
 		// One such example being when request body size is larger then the max
 		// size accepted
-		code = e.Code
-		message, _ = e.Message.(string)
-		errorCode = string(bacerrors.InternalError)
-		component = "APIServer"
+		apiError = &apimodels.APIError{
+			HTTPStatusCode: e.Code,
+			Code:           string(bacerrors.InternalError),
+			Message:        e.Message.(string),
+			Component:      "APIServer",
+		}
 		if c.Echo().Debug && e.Internal != nil {
-			message += ". " + e.Internal.Error()
+			apiError.Message += ". " + e.Internal.Error()
 		}
 
 	default:
 		// In an ideal world this should never happen. We should always have are errors
 		// from server as APIError. If output is this generic string, one should evaluate
 		// and map it to APIError and send in appropriate message.= http.StatusInternalServerError
-		code = http.StatusInternalServerError
-		message = "Internal server error"
-		errorCode = string(bacerrors.InternalError)
-		component = "Unknown"
-
+		apiError = &apimodels.APIError{
+			HTTPStatusCode: http.StatusInternalServerError,
+			Code:           string(bacerrors.InternalError),
+			Message:        "Internal server error",
+			Component:      "Unknown",
+		}
 		if c.Echo().Debug {
-			message += ". " + err.Error()
+			apiError.Message += ". " + err.Error()
 		}
 	}
 
-	// Don't override the status code if it is already been set.
-	// This is something that is advised by ECHO framework.
-	if !c.Response().Committed {
-		apiError := apimodels.APIError{
-			HTTPStatusCode: code,
-			Message:        message,
-			RequestID:      c.Request().Header.Get(echo.HeaderXRequestID),
-			Code:           errorCode,
-			Component:      component,
-		}
-		var responseErr error
-		if c.Request().Method == http.MethodHead {
-			responseErr = c.NoContent(code)
-		} else {
-			responseErr = c.JSON(code, apiError)
-		}
-		if responseErr != nil {
-			log.Error().Err(responseErr).
-				Str("original_error", err.Error()).
-				Msg("Failed to send error response")
-		}
+	apiError.RequestID = c.Request().Header.Get(echo.HeaderXRequestID)
+	var responseErr error
+	if c.Request().Method == http.MethodHead {
+		responseErr = c.NoContent(apiError.HTTPStatusCode)
+	} else {
+		responseErr = c.JSON(apiError.HTTPStatusCode, apiError)
+	}
+	if responseErr != nil {
+		log.Error().Err(responseErr).
+			Str("original_error", err.Error()).
+			Msg("Failed to send error response")
 	}
-
 }


Test patch:
diff --git a/pkg/publicapi/apimodels/error_test.go b/pkg/publicapi/apimodels/error_test.go
new file mode 100644
index 0000000000..a9b38b71fc
--- /dev/null
+++ b/pkg/publicapi/apimodels/error_test.go
@@ -0,0 +1,342 @@
+//go:build unit || !integration
+
+package apimodels
+
+import (
+	"bytes"
+	"encoding/json"
+	"io"
+	"net/http"
+	"testing"
+
+	"github.com/stretchr/testify/suite"
+
+	"github.com/bacalhau-project/bacalhau/pkg/bacerrors"
+)
+
+type APIErrorTestSuite struct {
+	suite.Suite
+}
+
+func TestAPIErrorSuite(t *testing.T) {
+	suite.Run(t, new(APIErrorTestSuite))
+}
+
+func (suite *APIErrorTestSuite) TestNewAPIError() {
+	tests := []struct {
+		name       string
+		statusCode int
+		message    string
+		wantErr    *APIError
+	}{
+		{
+			name:       "basic error",
+			statusCode: http.StatusBadRequest,
+			message:    "invalid request",
+			wantErr: &APIError{
+				HTTPStatusCode: http.StatusBadRequest,
+				Message:        "invalid request",
+				Details:        make(map[string]string),
+			},
+		},
+		{
+			name:       "internal server error",
+			statusCode: http.StatusInternalServerError,
+			message:    "internal server error",
+			wantErr: &APIError{
+				HTTPStatusCode: http.StatusInternalServerError,
+				Message:        "internal server error",
+				Details:        make(map[string]string),
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		suite.Run(tt.name, func() {
+			err := NewAPIError(tt.statusCode, tt.message)
+			suite.Equal(tt.wantErr.HTTPStatusCode, err.HTTPStatusCode)
+			suite.Equal(tt.wantErr.Message, err.Message)
+			suite.NotNil(err.Details)
+			suite.Empty(err.Details)
+		})
+	}
+}
+
+func (suite *APIErrorTestSuite) TestNewUnauthorizedError() {
+	tests := []struct {
+		name    string
+		message string
+	}{
+		{
+			name:    "basic unauthorized",
+			message: "unauthorized access",
+		},
+		{
+			name:    "empty message",
+			message: "",
+		},
+	}
+
+	for _, tt := range tests {
+		suite.Run(tt.name, func() {
+			err := NewUnauthorizedError(tt.message)
+			suite.Equal(http.StatusUnauthorized, err.HTTPStatusCode)
+			suite.Equal(tt.message, err.Message)
+			suite.NotNil(err.Details)
+		})
+	}
+}
+
+func (suite *APIErrorTestSuite) TestGenerateAPIErrorFromHTTPResponse() {
+	tests := []struct {
+		name          string
+		response      *http.Response
+		responseBody  interface{}
+		expectedError *APIError
+	}{
+		{
+			name: "valid error response",
+			responseBody: &APIError{
+				HTTPStatusCode: http.StatusBadRequest,
+				Message:        "invalid input",
+				RequestID:      "test-request-id",
+				Code:           "BAD_REQUEST",
+				Component:      "TestComponent",
+				Hint:           "Please check input format",
+				Details:        map[string]string{"key": "value"},
+			},
+		},
+		{
+			name:     "nil response",
+			response: nil,
+			expectedError: &APIError{
+				HTTPStatusCode: 0,
+				Message:        "API call error, invalid response",
+				Details:        make(map[string]string),
+			},
+		},
+		{
+			name:         "invalid json response",
+			responseBody: "invalid json",
+			expectedError: &APIError{
+				HTTPStatusCode: http.StatusBadRequest,
+				Message:        `Unable to parse API call response body. Error: "invalid character 'i' looking for beginning of value". Body received: "invalid json"`,
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		suite.Run(tt.name, func() {
+			var resp *http.Response
+			if tt.response == nil && tt.responseBody != nil {
+				var body []byte
+				var err error
+
+				switch v := tt.responseBody.(type) {
+				case string:
+					body = []byte(v)
+				default:
+					body, err = json.Marshal(tt.responseBody)
+					suite.Require().NoError(err)
+				}
+
+				resp = &http.Response{
+					StatusCode: http.StatusBadRequest,
+					Body:       io.NopCloser(bytes.NewReader(body)),
+				}
+			} else {
+				resp = tt.response
+			}
+
+			result := GenerateAPIErrorFromHTTPResponse(resp)
+
+			if tt.expectedError != nil {
+				suite.Equal(tt.expectedError.HTTPStatusCode, result.HTTPStatusCode)
+				suite.Equal(tt.expectedError.Message, result.Message)
+			} else {
+				originalError := tt.responseBody.(*APIError)
+				suite.Equal(originalError.HTTPStatusCode, result.HTTPStatusCode)
+				suite.Equal(originalError.Message, result.Message)
+				suite.Equal(originalError.RequestID, result.RequestID)
+				suite.Equal(originalError.Code, result.Code)
+				suite.Equal(originalError.Component, result.Component)
+				suite.Equal(originalError.Hint, result.Hint)
+				suite.Equal(originalError.Details, result.Details)
+			}
+		})
+	}
+}
+
+func (suite *APIErrorTestSuite) TestBacErrorConversion() {
+	// Test FromBacError
+	suite.Run("FromBacError", func() {
+		testCases := []struct {
+			name   string
+			bacErr bacerrors.Error
+			check  func(*APIError)
+		}{
+			{
+				name: "full error conversion",
+				bacErr: bacerrors.New("test error").
+					WithHTTPStatusCode(http.StatusBadRequest).
+					WithCode("TEST_ERROR").
+					WithComponent("test-component").
+					WithHint("test hint").
+					WithDetails(map[string]string{"key": "value"}),
+				check: func(apiErr *APIError) {
+					suite.Equal(http.StatusBadRequest, apiErr.HTTPStatusCode)
+					suite.Equal("test error", apiErr.Message)
+					suite.Equal("TEST_ERROR", apiErr.Code)
+					suite.Equal("test-component", apiErr.Component)
+					suite.Equal("test hint", apiErr.Hint)
+					suite.Equal(map[string]string{"key": "value"}, apiErr.Details)
+				},
+			},
+			{
+				name:   "minimal error conversion",
+				bacErr: bacerrors.New("minimal error"),
+				check: func(apiErr *APIError) {
+					suite.Equal(http.StatusInternalServerError, apiErr.HTTPStatusCode)
+					suite.Equal("minimal error", apiErr.Message)
+					suite.Empty(apiErr.Details)
+				},
+			},
+		}
+
+		for _, tc := range testCases {
+			suite.Run(tc.name, func() {
+				apiErr := FromBacError(tc.bacErr)
+				tc.check(apiErr)
+			})
+		}
+	})
+
+	// Test ToBacError
+	suite.Run("ToBacError", func() {
+		testCases := []struct {
+			name   string
+			apiErr *APIError
+			check  func(bacerrors.Error)
+		}{
+			{
+				name: "full error conversion",
+				apiErr: &APIError{
+					HTTPStatusCode: http.StatusBadRequest,
+					Message:        "test error",
+					Code:           "TEST_ERROR",
+					Component:      "test-component",
+					Hint:           "test hint",
+					RequestID:      "test-request-id",
+					Details:        map[string]string{"key": "value"},
+				},
+				check: func(bacErr bacerrors.Error) {
+					suite.Equal(http.StatusBadRequest, bacErr.HTTPStatusCode())
+					suite.Equal("test error", bacErr.Error())
+					suite.Equal(bacerrors.Code("TEST_ERROR"), bacErr.Code())
+					suite.Equal("test-component", bacErr.Component())
+					suite.Equal("test hint", bacErr.Hint())
+					details := bacErr.Details()
+					suite.Equal("test-request-id", details["request_id"])
+					suite.Equal("value", details["key"])
+				},
+			},
+			{
+				name: "nil details conversion",
+				apiErr: &APIError{
+					HTTPStatusCode: http.StatusBadRequest,
+					Message:        "test error",
+					RequestID:      "test-request-id",
+				},
+				check: func(bacErr bacerrors.Error) {
+					suite.Equal("test-request-id", bacErr.Details()["request_id"])
+				},
+			},
+		}
+
+		for _, tc := range testCases {
+			suite.Run(tc.name, func() {
+				bacErr := tc.apiErr.ToBacError()
+				tc.check(bacErr)
+			})
+		}
+	})
+}
+
+func (suite *APIErrorTestSuite) TestErrorInterface() {
+	tests := []struct {
+		name string
+		err  *APIError
+		want string
+	}{
+		{
+			name: "basic error message",
+			err:  NewAPIError(http.StatusBadRequest, "test error"),
+			want: "test error",
+		},
+		{
+			name: "empty error message",
+			err:  NewAPIError(http.StatusBadRequest, ""),
+			want: "",
+		},
+	}
+
+	for _, tt := range tests {
+		suite.Run(tt.name, func() {
+			suite.Equal(tt.want, tt.err.Error())
+		})
+	}
+}
+
+func (suite *APIErrorTestSuite) TestJSONSerialization() {
+	tests := []struct {
+		name  string
+		err   *APIError
+		check func(*APIError, *APIError)
+	}{
+		{
+			name: "full error serialization",
+			err: &APIError{
+				HTTPStatusCode: http.StatusBadRequest,
+				Message:        "test error",
+				RequestID:      "test-request-id",
+				Code:           "TEST_ERROR",
+				Component:      "test-component",
+				Hint:           "test hint",
+				Details:        map[string]string{"key": "value"},
+			},
+			check: func(original, decoded *APIError) {
+				suite.Equal(original.HTTPStatusCode, decoded.HTTPStatusCode)
+				suite.Equal(original.Message, decoded.Message)
+				suite.Equal(original.RequestID, decoded.RequestID)
+				suite.Equal(original.Code, decoded.Code)
+				suite.Equal(original.Component, decoded.Component)
+				suite.Equal(original.Hint, decoded.Hint)
+				suite.Equal(original.Details, decoded.Details)
+			},
+		},
+		{
+			name: "minimal error serialization",
+			err: &APIError{
+				HTTPStatusCode: http.StatusBadRequest,
+				Message:        "test error",
+			},
+			check: func(original, decoded *APIError) {
+				suite.Equal(original.HTTPStatusCode, decoded.HTTPStatusCode)
+				suite.Equal(original.Message, decoded.Message)
+				suite.Nil(decoded.Details)
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		suite.Run(tt.name, func() {
+			data, err := json.Marshal(tt.err)
+			suite.Require().NoError(err)
+
+			var decoded APIError
+			suite.Require().NoError(json.Unmarshal(data, &decoded))
+
+			tt.check(tt.err, &decoded)
+		})
+	}
+}
diff --git a/pkg/publicapi/middleware/error_handler_test.go b/pkg/publicapi/middleware/error_handler_test.go
index c59f382980..e864a30d5b 100644
--- a/pkg/publicapi/middleware/error_handler_test.go
+++ b/pkg/publicapi/middleware/error_handler_test.go
@@ -1,5 +1,3 @@
-//go:build unit || !integration
-
 package middleware
 
 import (
@@ -27,62 +25,220 @@ func (suite *CustomHTTPErrorHandlerTestSuite) SetupTest() {
 }
 
 func (suite *CustomHTTPErrorHandlerTestSuite) TestBaseError() {
-	req := httptest.NewRequest(http.MethodGet, "/", nil)
-	rec := httptest.NewRecorder()
-	c := suite.echo.NewContext(req, rec)
-
-	err := bacerrors.New("test base error").
-		WithHTTPStatusCode(http.StatusBadRequest).
-		WithCode("TEST_ERROR").
-		WithComponent("TEST_COMPONENT")
-	CustomHTTPErrorHandler(err, c)
-
-	suite.Equal(http.StatusBadRequest, rec.Result().StatusCode)
-
-	var apiError apimodels.APIError
-	suite.Require().NoError(json.NewDecoder(rec.Body).Decode(&apiError))
-
-	suite.Equal("test base error", apiError.Message)
-	suite.Equal("TEST_ERROR", apiError.Code)
-	suite.Equal("TEST_COMPONENT", apiError.Component)
+	tests := []struct {
+		name string
+		err  bacerrors.Error
+		want apimodels.APIError
+	}{
+		{
+			name: "basic error",
+			err: bacerrors.New("test base error").
+				WithHTTPStatusCode(http.StatusBadRequest).
+				WithCode("TEST_ERROR").
+				WithComponent("TEST_COMPONENT"),
+			want: apimodels.APIError{
+				HTTPStatusCode: http.StatusBadRequest,
+				Message:        "test base error",
+				Code:           "TEST_ERROR",
+				Component:      "TEST_COMPONENT",
+			},
+		},
+		{
+			name: "error with hint",
+			err: bacerrors.New("test error with hint").
+				WithHTTPStatusCode(http.StatusBadRequest).
+				WithCode("TEST_ERROR").
+				WithComponent("TEST_COMPONENT").
+				WithHint("test hint"),
+			want: apimodels.APIError{
+				HTTPStatusCode: http.StatusBadRequest,
+				Message:        "test error with hint",
+				Code:           "TEST_ERROR",
+				Component:      "TEST_COMPONENT",
+				Hint:           "test hint",
+			},
+		},
+		{
+			name: "error with details",
+			err: bacerrors.New("test error with details").
+				WithHTTPStatusCode(http.StatusBadRequest).
+				WithCode("TEST_ERROR").
+				WithComponent("TEST_COMPONENT").
+				WithDetails(map[string]string{"key": "value"}),
+			want: apimodels.APIError{
+				HTTPStatusCode: http.StatusBadRequest,
+				Message:        "test error with details",
+				Code:           "TEST_ERROR",
+				Component:      "TEST_COMPONENT",
+				Details:        map[string]string{"key": "value"},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		suite.Run(tt.name, func() {
+			req := httptest.NewRequest(http.MethodGet, "/", nil)
+			rec := httptest.NewRecorder()
+			c := suite.echo.NewContext(req, rec)
+
+			CustomHTTPErrorHandler(tt.err, c)
+
+			suite.Equal(tt.want.HTTPStatusCode, rec.Result().StatusCode)
+
+			var apiError apimodels.APIError
+			suite.Require().NoError(json.NewDecoder(rec.Body).Decode(&apiError))
+
+			suite.Equal(tt.want.Message, apiError.Message)
+			suite.Equal(tt.want.Code, apiError.Code)
+			suite.Equal(tt.want.Component, apiError.Component)
+			suite.Equal(tt.want.Hint, apiError.Hint)
+			if tt.want.Details != nil {
+				suite.Equal(tt.want.Details, apiError.Details)
+			}
+		})
+	}
 }
 
 func (suite *CustomHTTPErrorHandlerTestSuite) TestEchoHTTPError() {
-	req := httptest.NewRequest(http.MethodGet, "/", nil)
-	rec := httptest.NewRecorder()
-	c := suite.echo.NewContext(req, rec)
-
-	err := echo.NewHTTPError(http.StatusUnauthorized, "unauthorized access")
-
-	CustomHTTPErrorHandler(err, c)
-
-	suite.Equal(http.StatusUnauthorized, rec.Result().StatusCode)
-
-	var apiError apimodels.APIError
-	suite.Require().NoError(json.NewDecoder(rec.Body).Decode(&apiError))
-
-	suite.Equal("unauthorized access", apiError.Message)
-	suite.Equal(string(bacerrors.InternalError), apiError.Code)
-	suite.Equal("APIServer", apiError.Component)
+	tests := []struct {
+		name      string
+		err       *echo.HTTPError
+		debugMode bool
+		want      apimodels.APIError
+	}{
+		{
+			name:      "basic echo error",
+			err:       echo.NewHTTPError(http.StatusUnauthorized, "unauthorized access"),
+			debugMode: false,
+			want: apimodels.APIError{
+				HTTPStatusCode: http.StatusUnauthorized,
+				Message:        "unauthorized access",
+				Code:           string(bacerrors.InternalError),
+				Component:      "APIServer",
+			},
+		},
+		{
+			name: "echo error with internal error in debug mode",
+			err: &echo.HTTPError{
+				Code:     http.StatusBadRequest,
+				Message:  "bad request",
+				Internal: errors.New("internal error details"),
+			},
+			debugMode: true,
+			want: apimodels.APIError{
+				HTTPStatusCode: http.StatusBadRequest,
+				Message:        "bad request. internal error details",
+				Code:           string(bacerrors.InternalError),
+				Component:      "APIServer",
+			},
+		},
+		{
+			name: "echo error with internal error in non-debug mode",
+			err: &echo.HTTPError{
+				Code:     http.StatusBadRequest,
+				Message:  "bad request",
+				Internal: errors.New("internal error details"),
+			},
+			debugMode: false,
+			want: apimodels.APIError{
+				HTTPStatusCode: http.StatusBadRequest,
+				Message:        "bad request",
+				Code:           string(bacerrors.InternalError),
+				Component:      "APIServer",
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		suite.Run(tt.name, func() {
+			req := httptest.NewRequest(http.MethodGet, "/", nil)
+			rec := httptest.NewRecorder()
+			e := echo.New()
+			e.Debug = tt.debugMode
+			c := e.NewContext(req, rec)
+
+			CustomHTTPErrorHandler(tt.err, c)
+
+			suite.Equal(tt.want.HTTPStatusCode, rec.Result().StatusCode)
+
+			var apiError apimodels.APIError
+			suite.Require().NoError(json.NewDecoder(rec.Body).Decode(&apiError))
+
+			suite.Equal(tt.want.Message, apiError.Message)
+			suite.Equal(tt.want.Code, apiError.Code)
+			suite.Equal(tt.want.Component, apiError.Component)
+		})
+	}
 }
 
 func (suite *CustomHTTPErrorHandlerTestSuite) TestDefaultError() {
+	tests := []struct {
+		name      string
+		err       error
+		debugMode bool
+		want      apimodels.APIError
+	}{
+		{
+			name:      "unknown error in non-debug mode",
+			err:       errors.New("unknown error"),
+			debugMode: false,
+			want: apimodels.APIError{
+				HTTPStatusCode: http.StatusInternalServerError,
+				Message:        "Internal server error",
+				Code:           string(bacerrors.InternalError),
+				Component:      "Unknown",
+			},
+		},
+		{
+			name:      "unknown error in debug mode",
+			err:       errors.New("detailed error info"),
+			debugMode: true,
+			want: apimodels.APIError{
+				HTTPStatusCode: http.StatusInternalServerError,
+				Message:        "Internal server error. detailed error info",
+				Code:           string(bacerrors.InternalError),
+				Component:      "Unknown",
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		suite.Run(tt.name, func() {
+			req := httptest.NewRequest(http.MethodGet, "/", nil)
+			rec := httptest.NewRecorder()
+			e := echo.New()
+			e.Debug = tt.debugMode
+			c := e.NewContext(req, rec)
+
+			CustomHTTPErrorHandler(tt.err, c)
+
+			suite.Equal(tt.want.HTTPStatusCode, rec.Result().StatusCode)
+
+			var apiError apimodels.APIError
+			suite.Require().NoError(json.NewDecoder(rec.Body).Decode(&apiError))
+
+			suite.Equal(tt.want.Message, apiError.Message)
+			suite.Equal(tt.want.Code, apiError.Code)
+			suite.Equal(tt.want.Component, apiError.Component)
+		})
+	}
+}
+
+func (suite *CustomHTTPErrorHandlerTestSuite) TestCommittedResponse() {
 	req := httptest.NewRequest(http.MethodGet, "/", nil)
 	rec := httptest.NewRecorder()
 	c := suite.echo.NewContext(req, rec)
 
-	err := errors.New("unknown error")
+	// Simulate a committed response
+	c.Response().WriteHeader(http.StatusOK)
+	suite.True(c.Response().Committed)
 
+	err := errors.New("test error")
 	CustomHTTPErrorHandler(err, c)
 
-	suite.Equal(http.StatusInternalServerError, rec.Result().StatusCode)
-
-	var apiError apimodels.APIError
-	suite.Require().NoError(json.NewDecoder(rec.Body).Decode(&apiError))
-
-	suite.Equal("Internal server error", apiError.Message)
-	suite.Equal(string(bacerrors.InternalError), apiError.Code)
-	suite.Equal("Unknown", apiError.Component)
+	// Should maintain the original status code
+	suite.Equal(http.StatusOK, rec.Result().StatusCode)
+	suite.Empty(rec.Body.String())
 }
 
 func (suite *CustomHTTPErrorHandlerTestSuite) TestHeadRequest() {
@@ -99,21 +255,40 @@ func (suite *CustomHTTPErrorHandlerTestSuite) TestHeadRequest() {
 }
 
 func (suite *CustomHTTPErrorHandlerTestSuite) TestRequestIDPropagation() {
-	req := httptest.NewRequest(http.MethodGet, "/", nil)
-	req.Header.Set(echo.HeaderXRequestID, "test-request-id")
-	rec := httptest.NewRecorder()
-	c := suite.echo.NewContext(req, rec)
-
-	err := errors.New("test error")
-
-	CustomHTTPErrorHandler(err, c)
-
-	suite.Equal(http.StatusInternalServerError, rec.Result().StatusCode)
-
-	var apiError apimodels.APIError
-	suite.Require().NoError(json.NewDecoder(rec.Body).Decode(&apiError))
-
-	suite.Equal("test-request-id", apiError.RequestID)
+	tests := []struct {
+		name      string
+		requestID string
+		err       error
+	}{
+		{
+			name:      "with request ID",
+			requestID: "test-request-id",
+			err:       errors.New("test error"),
+		},
+		{
+			name:      "without request ID",
+			requestID: "",
+			err:       errors.New("test error"),
+		},
+	}
+
+	for _, tt := range tests {
+		suite.Run(tt.name, func() {
+			req := httptest.NewRequest(http.MethodGet, "/", nil)
+			if tt.requestID != "" {
+				req.Header.Set(echo.HeaderXRequestID, tt.requestID)
+			}
+			rec := httptest.NewRecorder()
+			c := suite.echo.NewContext(req, rec)
+
+			CustomHTTPErrorHandler(tt.err, c)
+
+			var apiError apimodels.APIError
+			suite.Require().NoError(json.NewDecoder(rec.Body).Decode(&apiError))
+
+			suite.Equal(tt.requestID, apiError.RequestID)
+		})
+	}
 }
 
 func TestCustomHTTPErrorHandlerTestSuite(t *testing.T) {
diff --git a/pkg/test/devstack/stop_test.go b/pkg/test/devstack/stop_test.go
new file mode 100644
index 0000000000..5b69308cf3
--- /dev/null
+++ b/pkg/test/devstack/stop_test.go
@@ -0,0 +1,194 @@
+//go:build integration || !unit
+
+package devstack
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/suite"
+
+	"github.com/bacalhau-project/bacalhau/pkg/devstack"
+	"github.com/bacalhau-project/bacalhau/pkg/docker"
+	dockmodels "github.com/bacalhau-project/bacalhau/pkg/executor/docker/models"
+	"github.com/bacalhau-project/bacalhau/pkg/models"
+	"github.com/bacalhau-project/bacalhau/pkg/node"
+	"github.com/bacalhau-project/bacalhau/pkg/publicapi/apimodels"
+	clientv2 "github.com/bacalhau-project/bacalhau/pkg/publicapi/client/v2"
+	"github.com/bacalhau-project/bacalhau/pkg/test/scenario"
+	"github.com/bacalhau-project/bacalhau/pkg/test/teststack"
+	"github.com/bacalhau-project/bacalhau/pkg/util/idgen"
+
+	"github.com/bacalhau-project/bacalhau/pkg/logger"
+)
+
+type StopSuite struct {
+	suite.Suite
+	requester     *node.Node
+	client        clientv2.API
+	stateResolver *scenario.StateResolver
+}
+
+func TestStopSuite(t *testing.T) {
+	suite.Run(t, new(StopSuite))
+}
+
+func (s *StopSuite) SetupSuite() {
+	logger.ConfigureTestLogging(s.T())
+	docker.MustHaveDocker(s.T())
+	ctx := context.Background()
+	stack := teststack.Setup(ctx, s.T(),
+		devstack.WithNumberOfHybridNodes(1),
+	)
+	s.requester = stack.Nodes[0]
+	s.client = clientv2.New(fmt.Sprintf("http://%s:%d", s.requester.APIServer.Address, s.requester.APIServer.Port))
+	s.stateResolver = scenario.NewStateResolverFromStore(s.requester.RequesterNode.JobStore)
+}
+
+func (s *StopSuite) TearDownSuite() {
+	if s.requester != nil {
+		s.requester.CleanupManager.Cleanup(context.Background())
+	}
+}
+
+func (s *StopSuite) TestStop_HappyPath() {
+	ctx := context.Background()
+	jobID, err := s.submitJob(10)
+	s.Require().NoError(err)
+
+	s.Require().NoError(s.stateResolver.Wait(ctx, jobID, scenario.WaitForRunningState()))
+
+	evalID, err := s.stopJob(jobID, "test stop")
+	s.Require().NoError(err)
+	s.Require().NotEmpty(evalID)
+
+	s.verifyJobState(jobID, models.JobStateTypeStopped, "test stop")
+}
+
+func (s *StopSuite) TestStop_ShortID() {
+	ctx := context.Background()
+	jobID, err := s.submitJob(10)
+	s.Require().NoError(err)
+
+	s.Require().NoError(s.stateResolver.Wait(ctx, jobID, scenario.WaitForRunningState()))
+
+	evalID, err := s.stopJob(idgen.ShortUUID(jobID), "test stop")
+	s.Require().NoError(err)
+	s.Require().NotEmpty(evalID)
+
+	s.verifyJobState(jobID, models.JobStateTypeStopped, "test stop")
+}
+
+func (s *StopSuite) TestStop_AlreadyCompleted() {
+	ctx := context.Background()
+	jobID, err := s.submitJob(0) // Short sleep so job completes quickly
+	s.Require().NoError(err)
+
+	s.Require().NoError(s.stateResolver.Wait(ctx, jobID, scenario.WaitForSuccessfulCompletion()))
+
+	_, err = s.stopJob(jobID, "test stop")
+	s.Require().Error(err)
+
+	s.verifyJobState(jobID, models.JobStateTypeCompleted, "")
+}
+
+func (s *StopSuite) TestStop_AlreadyStopped() {
+	ctx := context.Background()
+	jobID, err := s.submitJob(10)
+	s.Require().NoError(err)
+
+	s.Require().NoError(s.stateResolver.Wait(ctx, jobID, scenario.WaitForRunningState()))
+
+	evalID, err := s.stopJob(jobID, "first stop")
+	s.Require().NoError(err)
+	s.Require().NotEmpty(evalID)
+
+	_, err = s.stopJob(jobID, "second stop")
+	s.Require().NoError(err)
+
+	s.verifyJobState(jobID, models.JobStateTypeStopped, "first stop")
+}
+
+func (s *StopSuite) TestStop_NotFound() {
+	_, err := s.stopJob("j-nonexistent", "test stop")
+	s.Require().Error(err)
+	s.Require().Contains(err.Error(), "not found")
+}
+
+func (s *StopSuite) TestStop_MultipleMatches() {
+	ctx := context.Background()
+
+	// Submit two jobs that will have a common prefix (usually just 'j-')
+	jobID1, err := s.submitJob(10)
+	s.Require().NoError(err)
+	s.Require().NoError(s.stateResolver.Wait(ctx, jobID1, scenario.WaitForRunningState()))
+
+	jobID2, err := s.submitJob(10)
+	s.Require().NoError(err)
+	s.Require().NoError(s.stateResolver.Wait(ctx, jobID2, scenario.WaitForRunningState()))
+
+	// Extract common prefix (should be at least 'j-')
+	commonPrefix := jobID1[:2]
+	s.Require().True(strings.HasPrefix(jobID2, commonPrefix), "Jobs should share a common prefix")
+
+	// Try to stop using the common prefix
+	_, err = s.stopJob(commonPrefix, "test stop")
+	s.Require().Error(err)
+	s.Require().Contains(err.Error(), "multiple jobs")
+
+	// Verify neither job was stopped
+	s.verifyJobState(jobID1, models.JobStateTypeRunning, "")
+	s.verifyJobState(jobID2, models.JobStateTypeRunning, "")
+}
+
+// Helper function to create and submit a job
+func (s *StopSuite) submitJob(sleepTime int) (string, error) {
+	ctx := context.Background()
+	j := &models.Job{
+		Type:  models.JobTypeBatch,
+		Count: 1,
+		Tasks: []*models.Task{
+			{
+				Name: "main",
+				Engine: dockmodels.NewDockerEngineBuilder("busybox:latest").
+					WithEntrypoint("sh", "-c", fmt.Sprintf("sleep %d", sleepTime)).
+					MustBuild(),
+			},
+		},
+	}
+	submitResp, err := s.client.Jobs().Put(ctx, &apimodels.PutJobRequest{
+		Job: j,
+	})
+	if err != nil {
+		return "", err
+	}
+	return submitResp.JobID, nil
+}
+
+// Helper function to verify job state
+func (s *StopSuite) verifyJobState(jobID string, expectedState models.JobStateType, expectedMessage string) {
+	ctx := context.Background()
+	getResp, err := s.client.Jobs().Get(ctx, &apimodels.GetJobRequest{
+		JobID: jobID,
+	})
+	s.Require().NoError(err)
+	s.Require().Equal(expectedState, getResp.Job.State.StateType)
+	if expectedMessage != "" {
+		s.Require().Equal(expectedMessage, getResp.Job.State.Message)
+	}
+}
+
+// Helper function to stop a job and verify the response
+func (s *StopSuite) stopJob(jobID, reason string) (string, error) {
+	ctx := context.Background()
+	stopResp, err := s.client.Jobs().Stop(ctx, &apimodels.StopJobRequest{
+		JobID:  jobID,
+		Reason: reason,
+	})
+	if err != nil {
+		return "", err
+	}
+	return stopResp.EvaluationID, nil
+}
diff --git a/pkg/test/scenario/resolver.go b/pkg/test/scenario/resolver.go
index 0ca4dc75d4..82063b292f 100644
--- a/pkg/test/scenario/resolver.go
+++ b/pkg/test/scenario/resolver.go
@@ -191,6 +191,16 @@ func WaitForTerminalStates() StateChecks {
 	}
 }
 
+// WaitForRunningState waits until the job is in a running state, or fails if in a terminal state
+func WaitForRunningState() StateChecks {
+	return func(jobState *JobState) (bool, error) {
+		if jobState.State.StateType.IsTerminal() {
+			return false, fmt.Errorf("job is in terminal state %s", jobState.State.StateType)
+		}
+		return jobState.State.StateType == models.JobStateTypeRunning, nil
+	}
+}
+
 func WaitForSuccessfulCompletion() StateChecks {
 	return func(jobState *JobState) (bool, error) {
 		if jobState.State.StateType.IsTerminal() {