Repository: stellar/go
Pull request #5753 (2025-07-15 17:56:56)

Merge master into protocol-23
<!-- If you're making a doc PR or something tiny where the below is irrelevant, delete this
template and use a short description, but in your description aim to include both what the
change is, and why it is being made, with enough context for anyone to understand. -->

<details>
  <summary>PR Checklist</summary>
  
### PR Structure

* [ ] This PR has reasonably narrow scope (if not, break it down into smaller PRs).
* [ ] This PR avoids mixing refactoring changes with feature changes (split into two PRs
  otherwise).
* [ ] This PR's title starts with name of package that is most changed in the PR, ex.
  `services/friendbot`, or `all` or `doc` if the changes are broad or impact many
  packages.

### Thoroughness

* [ ] This PR adds tests for the most critical parts of the new functionality or fixes.
* [ ] I've updated any docs ([developer docs](https://developers.stellar.org/api/), `.md`
  files, etc... affected by this change). Take a look in the `docs` folder for a given service,
  like [this one](https://github.com/stellar/go/tree/master/services/horizon/internal/docs).

### Release planning

* [ ] I've reviewed the changes in this PR and if I consider them worthwhile for being mentioned on release notes then I have updated the relevant `CHANGELOG.md` within the  component folder structure. For example, if I changed horizon, then I updated ([services/horizon/CHANGELOG.md](services/horizon/CHANGELOG.md). I add a new line item describing the change and reference to this PR. If I don't update a CHANGELOG, I acknowledge this PR's change may not be mentioned in future release notes.  
* [ ] I've decided if this PR requires a new major/minor version according to
  [semver](https://semver.org/), or if it's mainly a patch change. The PR is targeted at the next
  release branch if it's not a patch change.
</details>

### What
Merge master into protocol-23



Patch:
diff --git a/support/datastore/s3_datastore.go b/support/datastore/s3_datastore.go
index 99125192..b6c9d38e 100644
--- a/support/datastore/s3_datastore.go
+++ b/support/datastore/s3_datastore.go
@@ -6,16 +6,19 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"net/http"
 	"os"
 	"path"
 	"strings"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
+	awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http"
 	"github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/aws/smithy-go"
+
 	"github.com/stellar/go/support/log"
 	"github.com/stellar/go/support/url"
 )
@@ -50,6 +53,15 @@ func NewS3DataStore(ctx context.Context, datastoreConfig DataStoreConfig) (DataS
 		if endpointUrl != "" {
 			o.BaseEndpoint = aws.String(endpointUrl)
 		}
+
+		// Check if default credentials were successfully retrieved from the chain.
+		// If not, fall back to anonymous credentials for public S3 bucket access.
+		_, err := cfg.Credentials.Retrieve(ctx)
+		if err != nil {
+			log.Infof("No default AWS credentials found, configuring S3 client for anonymous access")
+			o.Credentials = aws.AnonymousCredentials{}
+		}
+
 		o.Region = region
 		o.UsePathStyle = true
 	})
@@ -68,10 +80,37 @@ func FromS3Client(ctx context.Context, client *s3.Client, bucketPath string, sch
 	bucketName := parsed.Host
 	uploader := manager.NewUploader(client)
 
-	input := &s3.HeadBucketInput{Bucket: aws.String(bucketName)}
-	_, err = client.HeadBucket(ctx, input)
+	log.Infof("Creating S3 client for bucket: %s, prefix: %s", bucketName, prefix)
+
+	listInput := &s3.ListObjectsV2Input{
+		Bucket:  aws.String(bucketName),
+		Prefix:  aws.String(prefix),
+		MaxKeys: aws.Int32(1),
+	}
+
+	_, err = client.ListObjectsV2(ctx, listInput)
+
 	if err != nil {
-		return nil, fmt.Errorf("failed to head bucket, the bucket may not exist or you may not have access: %w", err)
+		// check for http redirect specifically as it implies a region issue.
+		var responseError *awshttp.ResponseError
+		if errors.As(err, &responseError) {
+			if responseError.HTTPStatusCode() == http.StatusMovedPermanently {
+				return nil, fmt.Errorf("bucket '%s' requires a different endpoint (301 PermanentRedirect)."+
+					" Please ensure the correct region is configured: %w", bucketName, err)
+			}
+		}
+
+		var apiError smithy.APIError
+		if errors.As(err, &apiError) {
+			switch apiError.ErrorCode() {
+			case "NoSuchBucket":
+				return nil, fmt.Errorf("bucket '%s' does not exist (NoSuchBucket): %w", bucketName, err)
+			case "AccessDenied":
+				return nil, fmt.Errorf("access denied to bucket '%s' (AccessDenied): %w", bucketName, err)
+			default:
+			}
+		}
+		return nil, fmt.Errorf("failed to list objects in bucket '%s': %w", bucketName, err)
 	}
 
 	return S3DataStore{client: client, uploader: uploader, bucket: bucketName, prefix: prefix, schema: schema}, nil
@@ -107,12 +146,14 @@ func (b S3DataStore) GetFile(ctx context.Context, filePath string) (io.ReadClose
 
 	output, err := b.client.GetObject(ctx, input)
 	if err != nil {
+		log.Errorf("Error retrieving file '%s': %v", filePath, err)
 		if isNotFoundError(err) {
 			return nil, os.ErrNotExist
 		}
-		return nil, err
+		return nil, fmt.Errorf("error retrieving file %s: %w", filePath, err)
 	}
 
+	log.Infof("File retrieved successfully: %s", filePath)
 	return output.Body, nil
 }
 


Test patch:
diff --git a/support/datastore/s3_datastore_test.go b/support/datastore/s3_datastore_test.go
index b19e558e..ab3ad24e 100644
--- a/support/datastore/s3_datastore_test.go
+++ b/support/datastore/s3_datastore_test.go
@@ -41,12 +41,11 @@ func (s *mockS3Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 	switch r.Method {
 	case http.MethodHead:
-		// See https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadBucket.html
 		// See https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadObject.html
 		s.handleHeadRequest(w, pathParts)
 	case http.MethodGet:
 		// See https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html
-		s.handleGetRequest(w, pathParts)
+		s.handleGetRequest(w, r, pathParts)
 	case http.MethodPut:
 		// See https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html
 		s.handlePutRequest(w, r, pathParts)
@@ -56,13 +55,6 @@ func (s *mockS3Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 }
 
 func (s *mockS3Server) handleHeadRequest(w http.ResponseWriter, pathParts []string) {
-	// Handle HeadBucket: A request with no key part in the path.
-	// We assume the bucket always exists in the mock.
-	if len(pathParts) < 2 || pathParts[1] == "" {
-		w.WriteHeader(http.StatusOK)
-		return
-	}
-
 	// Handle HeadObject: A request with a key.
 	key := pathParts[1]
 	obj, exists := s.objects[key]
@@ -75,7 +67,13 @@ func (s *mockS3Server) handleHeadRequest(w http.ResponseWriter, pathParts []stri
 	w.WriteHeader(http.StatusOK)
 }
 
-func (s *mockS3Server) handleGetRequest(w http.ResponseWriter, pathParts []string) {
+func (s *mockS3Server) handleGetRequest(w http.ResponseWriter, r *http.Request, pathParts []string) {
+	// Check for query param for ListObjectsV2
+	if r.Method == http.MethodGet && r.URL.Query().Get("list-type") == "2" {
+		w.WriteHeader(http.StatusOK)
+		return
+	}
+
 	if len(pathParts) < 2 {
 		http.Error(w, "Invalid path: Key is required for GET", http.StatusBadRequest)
 		return