Repository: amp-labs/connectors
Pull request #1029 (2024-09-10 15:02:30)

test: Salesforce Bulk Read+Query
# Description

Using real requests recorded Provider API output to create mock tests.

These cover 2 related Salesforce features **BulkRead** and **BulkQuery**.

Cobalt0s: > [!WARNING]
> <b>This pull request is not mergeable via GitHub because a downstack PR is open. Once all requirements are satisfied, merge this PR as a stack <a href="https://app.graphite.dev/github/pr/amp-labs/connectors/1029?utm_source=stack-comment-downstack-mergeability-warning" >on Graphite</a>.</b>
> <a href="https://graphite.dev/docs/merge-pull-requests">Learn more</a>

* **#1032** <a href="https://app.graphite.dev/github/pr/amp-labs/connectors/1032?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a>
* **#1031** <a href="https://app.graphite.dev/github/pr/amp-labs/connectors/1031?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a>
* **#1030** <a href="https://app.graphite.dev/github/pr/amp-labs/connectors/1030?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a>
* **#1029** <a href="https://app.graphite.dev/github/pr/amp-labs/connectors/1029?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a> 👈
* **#1028** <a href="https://app.graphite.dev/github/pr/amp-labs/connectors/1028?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a>
* **#1017** <a href="https://app.graphite.dev/github/pr/amp-labs/connectors/1017?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a>: 1 other dependent PR ([#1018](https://github.com/amp-labs/connectors/pull/1018) <a href="https://app.graphite.dev/github/pr/amp-labs/connectors/1018?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a>)
* **#1010** <a href="https://app.graphite.dev/github/pr/amp-labs/connectors/1010?utm_source=stack-comment-icon" target="_blank"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="10px" height="10px"/></a>
* `main`

This stack of pull requests is managed by Graphite. <a href="https://stacking.dev/?utm_source=stack-comment">Learn more about stacking.</a>
<h2></h2>

 Join @Cobalt0s and the rest of your teammates on <a href="https://graphite.dev?utm-source=stack-comment"><img src="https://static.graphite.dev/graphite-32x32-black.png" alt="Graphite" width="11px" height="11px"/> <b>Graphite</b></a>
  <!-- Current dependencies on/for this PR: -->

Patch:
diff --git a/providers/salesforce/bulk-read.go b/providers/salesforce/bulk-read.go
index 71d54521..29e24a86 100644
--- a/providers/salesforce/bulk-read.go
+++ b/providers/salesforce/bulk-read.go
@@ -17,6 +17,10 @@ import (
 //		Deleted: true,
 //	})
 func (c *Connector) BulkRead(ctx context.Context, params common.ReadParams) (*GetJobInfoResult, error) {
+	if err := params.ValidateParams(true); err != nil {
+		return nil, err
+	}
+
 	soql := makeSOQL(params)
 	// Note: if params.Deleted is set to true query will return only removed items.
 


Test patch:
diff --git a/providers/salesforce/bulk-query_test.go b/providers/salesforce/bulk-query_test.go
new file mode 100644
index 00000000..cb894d0a
--- /dev/null
+++ b/providers/salesforce/bulk-query_test.go
@@ -0,0 +1,142 @@
+package salesforce
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/amp-labs/connectors/common"
+	"github.com/amp-labs/connectors/common/interpreter"
+	"github.com/amp-labs/connectors/test/utils/mockutils"
+	"github.com/amp-labs/connectors/test/utils/mockutils/mockserver"
+	"github.com/amp-labs/connectors/test/utils/testroutines"
+	"github.com/amp-labs/connectors/test/utils/testutils"
+)
+
+func TestBulkQuery(t *testing.T) { //nolint:funlen,gocognit,cyclop,maintidx
+	t.Parallel()
+
+	responseUnknownObject := testutils.DataFromFile(t, "unknown-object.json")
+	responseAccount := testutils.DataFromFile(t, "bulk/read-launch-job-account.json")
+
+	account := &GetJobInfoResult{
+		Id:              "750ak000009AVi5AAG",
+		Operation:       "query",
+		Object:          "Account",
+		CreatedById:     "005ak000005hvjJAAQ",
+		CreatedDate:     "2024-09-09T13:08:34.000+0000",
+		SystemModstamp:  "2024-09-09T13:08:34.000+0000",
+		State:           "UploadComplete",
+		ConcurrencyMode: "Parallel",
+		ContentType:     "CSV",
+		ApiVersion:      59.0,
+		LineEnding:      "LF",
+		ColumnDelimiter: "COMMA",
+	}
+
+	tests := []bulkQueryTestCase{
+		{
+			Name: "Mime response header expected",
+			Input: bulkQueryInput{
+				query:          "",
+				includeDeleted: false,
+			},
+			Server:       mockserver.Dummy(),
+			ExpectedErrs: []error{interpreter.ErrMissingContentType},
+		},
+		{
+			Name: "Correct error message is understood from JSON response",
+			Input: bulkQueryInput{
+				query:          "",
+				includeDeleted: false,
+			},
+			Server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				w.Header().Set("Content-Type", "application/json")
+				w.WriteHeader(http.StatusBadRequest)
+				_, _ = w.Write(responseUnknownObject)
+			})),
+			ExpectedErrs: []error{
+				common.ErrBadRequest, errors.New("sObject type 'Accout' is not supported"), // nolint:goerr113
+			},
+		},
+		{
+			Name: "Launch bulk job with SOQL query",
+			Input: bulkQueryInput{
+				query:          "SELECT Id,Name,BillingCity FROM Account",
+				includeDeleted: false,
+			},
+			Server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { //nolint:varnamelen
+				w.Header().Set("Content-Type", "application/json")
+				w.WriteHeader(http.StatusOK)
+				switch path := r.URL.Path; {
+				case strings.HasSuffix(path, "/services/data/v59.0/jobs/query"):
+					mockutils.RespondToBody(w, r, `{
+						"operation":"query",
+						"query":"SELECT Id,Name,BillingCity FROM Account"}`,
+						func() {
+							_, _ = w.Write(responseAccount)
+						})
+				default:
+					_, _ = w.Write([]byte{})
+				}
+			})),
+			Expected:     account,
+			ExpectedErrs: nil,
+		},
+		{
+			Name: "Include deleted items using Query All",
+			Input: bulkQueryInput{
+				query:          "SELECT Id,Name,BillingCity FROM Account",
+				includeDeleted: true,
+			},
+			Server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { //nolint:varnamelen
+				w.Header().Set("Content-Type", "application/json")
+				w.WriteHeader(http.StatusOK)
+				switch path := r.URL.Path; {
+				case strings.HasSuffix(path, "/services/data/v59.0/jobs/query"):
+					mockutils.RespondToBody(w, r, `{
+						"operation":"queryAll",
+						"query":"SELECT Id,Name,BillingCity FROM Account"}`,
+						func() {
+							_, _ = w.Write(responseAccount)
+						})
+				default:
+					_, _ = w.Write([]byte{})
+				}
+			})),
+			Expected:     account,
+			ExpectedErrs: nil,
+		},
+	}
+
+	for _, tt := range tests { // nolint:dupl
+		// nolint:varnamelen
+		tt := tt // rebind, omit loop side effects for parallel goroutine
+		t.Run(tt.Name, func(t *testing.T) {
+			t.Parallel()
+
+			tt.Run(t, func() (*Connector, error) {
+				return constructTestConnector(tt.Server.URL)
+			})
+		})
+	}
+}
+
+type (
+	bulkQueryInput struct {
+		query          string
+		includeDeleted bool
+	}
+	bulkQueryTestCaseType = testroutines.TestCase[bulkQueryInput, *GetJobInfoResult]
+	bulkQueryTestCase     bulkQueryTestCaseType
+)
+
+func (c bulkQueryTestCase) Run(t *testing.T, builder testroutines.ConnectorBuilder[*Connector]) {
+	t.Helper()
+	conn := builder.Build(t, c.Name)
+	output, err := conn.BulkQuery(context.Background(), c.Input.query, c.Input.includeDeleted)
+	bulkQueryTestCaseType(c).Validate(t, err, output)
+}
diff --git a/providers/salesforce/bulk-read_test.go b/providers/salesforce/bulk-read_test.go
new file mode 100644
index 00000000..7a874226
--- /dev/null
+++ b/providers/salesforce/bulk-read_test.go
@@ -0,0 +1,118 @@
+package salesforce
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/amp-labs/connectors"
+	"github.com/amp-labs/connectors/common"
+	"github.com/amp-labs/connectors/common/interpreter"
+	"github.com/amp-labs/connectors/test/utils/mockutils"
+	"github.com/amp-labs/connectors/test/utils/mockutils/mockserver"
+	"github.com/amp-labs/connectors/test/utils/testroutines"
+	"github.com/amp-labs/connectors/test/utils/testutils"
+)
+
+func TestBulkRead(t *testing.T) { //nolint:funlen,gocognit,cyclop,maintidx
+	t.Parallel()
+
+	responseUnknownObject := testutils.DataFromFile(t, "unknown-object.json")
+	responseAccount := testutils.DataFromFile(t, "bulk/read-launch-job-account.json")
+
+	tests := []bulkReadTestCase{
+		{
+			Name:         "Read object must be included",
+			Server:       mockserver.Dummy(),
+			ExpectedErrs: []error{common.ErrMissingObjects},
+		},
+		{
+			Name:         "At least one field is requested",
+			Input:        common.ReadParams{ObjectName: "Orders"},
+			Server:       mockserver.Dummy(),
+			ExpectedErrs: []error{common.ErrMissingFields},
+		},
+		{
+			Name:         "Mime response header expected",
+			Input:        common.ReadParams{ObjectName: "Orders", Fields: connectors.Fields("id")},
+			Server:       mockserver.Dummy(),
+			ExpectedErrs: []error{interpreter.ErrMissingContentType},
+		},
+		{
+			Name:  "Correct error message is understood from JSON response",
+			Input: common.ReadParams{ObjectName: "Accout", Fields: connectors.Fields("id")},
+			Server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				w.Header().Set("Content-Type", "application/json")
+				w.WriteHeader(http.StatusBadRequest)
+				_, _ = w.Write(responseUnknownObject)
+			})),
+			ExpectedErrs: []error{
+				common.ErrBadRequest, errors.New("sObject type 'Accout' is not supported"), // nolint:goerr113
+			},
+		},
+		{
+			Name: "Launch bulk job with SOQL query",
+			Input: common.ReadParams{
+				ObjectName: "Account",
+				Fields:     connectors.Fields("Id", "Name"),
+			},
+			Server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { //nolint:varnamelen
+				w.Header().Set("Content-Type", "application/json")
+				w.WriteHeader(http.StatusOK)
+				switch path := r.URL.Path; {
+				case strings.HasSuffix(path, "/services/data/v59.0/jobs/query"):
+					mockutils.RespondToBody(w, r, `{
+						"operation":"query",
+						"query":"SELECT Id,Name FROM Account"}`,
+						func() {
+							_, _ = w.Write(responseAccount)
+						})
+				default:
+					_, _ = w.Write([]byte{})
+				}
+			})),
+			Expected: &GetJobInfoResult{
+				Id:              "750ak000009AVi5AAG",
+				Operation:       "query",
+				Object:          "Account",
+				CreatedById:     "005ak000005hvjJAAQ",
+				CreatedDate:     "2024-09-09T13:08:34.000+0000",
+				SystemModstamp:  "2024-09-09T13:08:34.000+0000",
+				State:           "UploadComplete",
+				ConcurrencyMode: "Parallel",
+				ContentType:     "CSV",
+				ApiVersion:      59.0,
+				LineEnding:      "LF",
+				ColumnDelimiter: "COMMA",
+			},
+			ExpectedErrs: nil,
+		},
+	}
+
+	for _, tt := range tests { // nolint:dupl
+		// nolint:varnamelen
+		tt := tt // rebind, omit loop side effects for parallel goroutine
+		t.Run(tt.Name, func(t *testing.T) {
+			t.Parallel()
+
+			tt.Run(t, func() (*Connector, error) {
+				return constructTestConnector(tt.Server.URL)
+			})
+		})
+	}
+}
+
+type (
+	bulkReadTestCaseType = testroutines.TestCase[common.ReadParams, *GetJobInfoResult]
+	bulkReadTestCase     bulkReadTestCaseType
+)
+
+func (c bulkReadTestCase) Run(t *testing.T, builder testroutines.ConnectorBuilder[*Connector]) {
+	t.Helper()
+	conn := builder.Build(t, c.Name)
+	output, err := conn.BulkRead(context.Background(), c.Input)
+	bulkReadTestCaseType(c).Validate(t, err, output)
+}
diff --git a/providers/salesforce/test/bulk/read-launch-job-account.json b/providers/salesforce/test/bulk/read-launch-job-account.json
new file mode 100644
index 00000000..c20dd419
--- /dev/null
+++ b/providers/salesforce/test/bulk/read-launch-job-account.json
@@ -0,0 +1,14 @@
+{
+  "id" : "750ak000009AVi5AAG",
+  "operation" : "query",
+  "object" : "Account",
+  "createdById" : "005ak000005hvjJAAQ",
+  "createdDate" : "2024-09-09T13:08:34.000+0000",
+  "systemModstamp" : "2024-09-09T13:08:34.000+0000",
+  "state" : "UploadComplete",
+  "concurrencyMode" : "Parallel",
+  "contentType" : "CSV",
+  "apiVersion" : 59.0,
+  "lineEnding" : "LF",
+  "columnDelimiter" : "COMMA"
+}