Repository: opensearch-project/opensearch-go (go)

Issue:
[BUG] Querying Data Streams Stats with AWS Auth without a name returns a signing error 
### What is the bug?
When querying the Data Stream Stats API with AWS Authentication and without setting an index name the AWS Signature is invalid.

Error:
```
The request signature we calculated does not match the signature you provided. 
Check your AWS Secret Access Key and signing method.
```

### How can one reproduce the bug?
Setup an AWS Signer and OpenSearch Client and make a request for `/_data_streams/_stats` without setting an index name.
```go
awsCfg, err := config.LoadDefaultConfig(context.Background())
if err != nil {
    return err
}

awsSigner, err := awsv2.NewSigner(awsCfg)
if err != nil {
    return err
}

url := "https://my-opensearch-cluster"
osCfg := opensearch.Config{
    Addresses: []string{url},
    Signer:    awsSigner,
}

client, err := opensearch.NewClient(osCfg)
if err != nil {
    return err
}

resp, err := client.Indices.GetDataStreamStats()
if err != nil {
    return err
}
```

### What is the expected behavior?
The list of data streams and their stats is returned. The OpenSearch API supports making this request with no name and should return all of the data streams instead.

### What is your host/environment?
Linux 6.2.6
Go 1.20.5

### Do you have any screenshots?
No

### Do you have any additional context?
It looks like the GetDataStreams function always adds the name and thus ends up adding an additional `/` to the path. This `/` is dropped on the server side but does get factored into the signature calculation on the client side causing the mismatch.


Pull request:
Data stream stats: Add name only if present
### Description
This makes the GetDataStreamStats request only add the `Name` and path separator (`/`) to the path if Name is not empty.

As it stands when using this function without setting a name the path will be:
```
/_data_stream//_stats
```

This path is then used by the AWS Signer to calculate a signature. When the request is sent the server recalculates the signature but the additional `/` has been dropped by now and we get a signature mismatch. See #337 for details.

I have tested this fix in our own codebase and it resolves the issue as described.

### Issues Resolved
Closes #337 

By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin).


Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index c574d2e..13588e4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -51,7 +51,8 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 
 - Corrects curl logging to emit the correct URL destination ([#101](https://github.com/opensearch-project/opensearch-go/pull/101))
 - Corrects handling of errors without an error response body ([#286](https://github.com/opensearch-project/opensearch-go/pull/286))
+- Corrects AWSv4 signature on DataStream Stats with no index name specified ([#338](https://github.com/opensearch-project/opensearch-go/pull/338))
 
 ### Security
 
-[Unreleased]: https://github.com/opensearch-project/opensearch-go/compare/v2.2.0...HEAD
\ No newline at end of file
+[Unreleased]: https://github.com/opensearch-project/opensearch-go/compare/v2.2.0...HEAD
diff --git a/go.mod b/go.mod
index 8fdf09c..a6084d4 100644
--- a/go.mod
+++ b/go.mod
@@ -6,6 +6,7 @@ require (
 	github.com/aws/aws-sdk-go v1.44.298
 	github.com/aws/aws-sdk-go-v2 v1.19.0
 	github.com/aws/aws-sdk-go-v2/config v1.18.28
+	github.com/kinbiko/jsonassert v1.1.1
 	github.com/stretchr/testify v1.8.4
 	golang.org/x/net v0.7.0 // indirect
 )
diff --git a/go.sum b/go.sum
index cc74045..d70cdd2 100644
--- a/go.sum
+++ b/go.sum
@@ -33,6 +33,8 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/kinbiko/jsonassert v1.1.1 h1:DB12divY+YB+cVpHULLuKePSi6+ui4M/shHSzJISkSE=
+github.com/kinbiko/jsonassert v1.1.1/go.mod h1:NO4lzrogohtIdNUNzx8sdzB55M4R4Q1bsrWVdqQ7C+A=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
diff --git a/opensearchapi/api.indices.get_datastream_stats.go b/opensearchapi/api.indices.get_datastream_stats.go
index 4bc2e93..fe7dc31 100644
--- a/opensearchapi/api.indices.get_datastream_stats.go
+++ b/opensearchapi/api.indices.get_datastream_stats.go
@@ -58,8 +58,11 @@ func (r IndicesGetDataStreamStatsRequest) Do(ctx context.Context, transport Tran
 	method = "GET"
 
 	path.Grow(1 + len("_data_stream") + 1 + len(r.Name) + 1 + len("_stats"))
-	path.WriteString("/_data_stream/")
-	path.WriteString(r.Name)
+	path.WriteString("/_data_stream")
+	if r.Name != "" {
+		path.WriteString("/")
+		path.WriteString(r.Name)
+	}
 	path.WriteString("/_stats")
 
 	params = make(map[string]string)


Test patch:
diff --git a/opensearchapi/api.indices.datastream_test.go b/opensearchapi/api.indices.datastream_test.go
index 39f98e3..a6aeb12 100644
--- a/opensearchapi/api.indices.datastream_test.go
+++ b/opensearchapi/api.indices.datastream_test.go
@@ -13,18 +13,18 @@
 package opensearchapi_test
 
 import (
-	"bytes"
 	"context"
-	"encoding/json"
 	"fmt"
-	"github.com/opensearch-project/opensearch-go/v2"
-	"github.com/opensearch-project/opensearch-go/v2/opensearchapi"
-	"github.com/stretchr/testify/require"
 	"io/ioutil"
 	"net/http"
 	"strings"
 	"testing"
 	"time"
+
+	"github.com/kinbiko/jsonassert"
+	"github.com/opensearch-project/opensearch-go/v2"
+	"github.com/opensearch-project/opensearch-go/v2/opensearchapi"
+	"github.com/stretchr/testify/require"
 )
 
 type DataStreamRequest interface {
@@ -32,7 +32,9 @@ type DataStreamRequest interface {
 }
 
 func TestIndicesDataStreams_Do(t *testing.T) {
-	name := fmt.Sprintf("demo-%s", time.Now().Format("2006-01-02-15-04-05"))
+	// We need two datastreams to ensure endpoints that fetch both are tested appropriately
+	dataStream1 := fmt.Sprintf("demo-1-%s", time.Now().Format("2006-01-02-15-04-05"))
+	dataStream2 := fmt.Sprintf("demo-2-%s", time.Now().Format("2006-01-02-15-04-05"))
 
 	tests := []struct {
 		name     string
@@ -44,7 +46,27 @@ func TestIndicesDataStreams_Do(t *testing.T) {
 		{
 			name: "TestIndicesCreateDataStreamRequest_Do",
 			r: opensearchapi.IndicesCreateDataStreamRequest{
-				Name:       name,
+				Name:       dataStream1,
+				Pretty:     true,
+				Human:      true,
+				ErrorTrace: true,
+				Header: map[string][]string{
+					"Content-Type": {"application/json"},
+				},
+			},
+			want: &opensearchapi.Response{
+				StatusCode: 200,
+				Header: http.Header{
+					"Content-Type": []string{"application/json; charset=UTF-8"},
+				},
+			},
+			wantBody: `{"acknowledged":true}`,
+			wantErr:  false,
+		},
+		{
+			name: "TestIndicesCreateDataStream2Request_Do",
+			r: opensearchapi.IndicesCreateDataStreamRequest{
+				Name:       dataStream2,
 				Pretty:     true,
 				Human:      true,
 				ErrorTrace: true,
@@ -64,7 +86,7 @@ func TestIndicesDataStreams_Do(t *testing.T) {
 		{
 			name: "TestIndicesGetDataStreamRequest_Do",
 			r: opensearchapi.IndicesGetDataStreamRequest{
-				Name:       name,
+				Name:       dataStream1,
 				Pretty:     true,
 				Human:      true,
 				ErrorTrace: true,
@@ -101,7 +123,26 @@ func TestIndicesDataStreams_Do(t *testing.T) {
 		{
 			name: "TestIndicesGetStatsDataStreamRequest_Do",
 			r: opensearchapi.IndicesGetDataStreamStatsRequest{
-				Name:       name,
+				Name:       dataStream1,
+				Pretty:     true,
+				Human:      true,
+				ErrorTrace: true,
+				Header: map[string][]string{
+					"Content-Type": {"application/json"},
+				},
+			},
+			want: &opensearchapi.Response{
+				StatusCode: 200,
+				Header: http.Header{
+					"Content-Type": []string{"application/json; charset=UTF-8"},
+				},
+			},
+			wantBody: fmt.Sprintf(`{"_shards":{"total":2,"successful":1,"failed":0},"data_stream_count":1,"backing_indices":1,"total_store_size":"208b","total_store_size_bytes":208,"data_streams":[{"data_stream":"%s","backing_indices":1,"store_size":"208b","store_size_bytes":208,"maximum_timestamp":0}]}`, dataStream1),
+			wantErr:  false,
+		},
+		{
+			name: "TestIndicesGetAllStatsDataStreamRequest_Do",
+			r: opensearchapi.IndicesGetDataStreamStatsRequest{
 				Pretty:     true,
 				Human:      true,
 				ErrorTrace: true,
@@ -115,13 +156,33 @@ func TestIndicesDataStreams_Do(t *testing.T) {
 					"Content-Type": []string{"application/json; charset=UTF-8"},
 				},
 			},
-			wantBody: fmt.Sprintf(`{"_shards":{"total":2,"successful":1,"failed":0},"data_stream_count":1,"backing_indices":1,"total_store_size":"208b","total_store_size_bytes":208,"data_streams":[{"data_stream":"%s","backing_indices":1,"store_size":"208b","store_size_bytes":208,"maximum_timestamp":0}]}`, name),
+			wantBody: fmt.Sprintf(`{"_shards":{"total":4,"successful":2,"failed":0},"data_stream_count":2,"backing_indices":2,"total_store_size":"416b","total_store_size_bytes":416,"data_streams":["<<UNORDERED>>",{"data_stream":"%s","backing_indices":1,"store_size":"208b","store_size_bytes":208,"maximum_timestamp":0},{"data_stream":"%s","backing_indices":1,"store_size":"208b","store_size_bytes":208,"maximum_timestamp":0}]}`, dataStream2, dataStream1),
 			wantErr:  false,
 		},
 		{
 			name: "TestIndicesDeleteDataStreamRequest_Do",
 			r: opensearchapi.IndicesDeleteDataStreamRequest{
-				Name:       name,
+				Name:       dataStream1,
+				Pretty:     true,
+				Human:      true,
+				ErrorTrace: true,
+				Header: map[string][]string{
+					"Content-Type": {"application/json"},
+				},
+			},
+			want: &opensearchapi.Response{
+				StatusCode: 200,
+				Header: http.Header{
+					"Content-Type": []string{"application/json; charset=UTF-8"},
+				},
+			},
+			wantBody: `{"acknowledged":true}`,
+			wantErr:  false,
+		},
+		{
+			name: "TestIndicesDeleteDataStream2Request_Do",
+			r: opensearchapi.IndicesDeleteDataStreamRequest{
+				Name:       dataStream2,
 				Pretty:     true,
 				Human:      true,
 				ErrorTrace: true,
@@ -177,12 +238,8 @@ func TestIndicesDataStreams_Do(t *testing.T) {
 				body, err := ioutil.ReadAll(got.Body)
 				require.NoError(t, err)
 
-				buffer := new(bytes.Buffer)
-				if err := json.Compact(buffer, body); err != nil {
-					fmt.Println(err)
-				}
-
-				require.Equalf(t, buffer.String(), tt.wantBody, "Do() got = %v, want %v", got.String(), tt.wantBody)
+				ja := jsonassert.New(t)
+				ja.Assertf(string(body), tt.wantBody)
 			}
 		})
 	}
diff --git a/opensearchapi/api.indices.get_datastream_stats_test.go b/opensearchapi/api.indices.get_datastream_stats_test.go
new file mode 100644
index 0000000..1285c6c
--- /dev/null
+++ b/opensearchapi/api.indices.get_datastream_stats_test.go
@@ -0,0 +1,58 @@
+// SPDX-License-Identifier: Apache-2.0
+//
+// The OpenSearch Contributors require contributions made to
+// this file be licensed under the Apache-2.0 license or a
+// compatible open source license.
+//
+// Modifications Copyright OpenSearch Contributors. See
+// GitHub history for details.
+
+package opensearchapi
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+type stubTransport struct {
+	req *http.Request
+}
+
+func (t *stubTransport) Perform(req *http.Request) (*http.Response, error) {
+	t.req = req
+	return &http.Response{}, nil
+}
+
+func TestIndicesGetDataStreamStatsRequest(t *testing.T) {
+	tt := stubTransport{}
+	req := IndicesGetDataStreamStatsRequest{}
+
+	expectedPath := "/_data_stream/_stats"
+
+	_, err := req.Do(context.Background(), &tt)
+	if err != nil {
+		t.Fatalf("Error getting response: %s", err)
+	}
+
+	require.Equal(t, expectedPath, tt.req.URL.Path)
+}
+
+func TestIndicesGetDataStreamStatsRequestOne(t *testing.T) {
+	tt := stubTransport{}
+	req := IndicesGetDataStreamStatsRequest{
+		Name: "demo-1",
+	}
+
+	expectedPath := fmt.Sprintf("/_data_stream/%s/_stats", req.Name)
+
+	_, err := req.Do(context.Background(), &tt)
+	if err != nil {
+		t.Fatalf("Error getting response: %s", err)
+	}
+
+	require.Equal(t, expectedPath, tt.req.URL.Path)
+}