Repository: deislabs/ratify (go)

Issue:
AWS ECR Auth needs to support cross region
### What happened in your environment?

Cross region support is not supported in current implementation. The current implementation loads creds from the region from the cluster, but not other regions from specific images. Non-cluster region image verification failed with wrong creds.

### What did you expect to happen?

Should allowed for pre-load of specific regions, and cached map of registries, for operation in same account.

### What version of Kubernetes are you running?

1.24

### What version of Ratify are you running?

RC1

### Anything else you would like to add?

I am working on a PR

### Are you willing to submit PRs to contribute to this bug fix?

- [X] Yes, I am willing to implement it.

Pull request:
feat: xregion aws ecr auth
# Description
Cross-region ECR auth provider

## What this PR does / why we need it:
Fixes an issue where region-specific creds are not used. Adds region to the ECR client cfg for call to get AuthZ tokens. Maps retrieved creds to ECR registry host.

```
├── common
│   ├── oras
│   │   └── authprovider
│   │       ├── aws
│   │       │   ├── awsecrbasic.go
│   │       │   ├── awsecrbasic_test.go
│   │       │   └── ecrauthtypes.go
├── utils
│   ├── awsauth
│   │   ├── utils.go
│   │   └── utils_test.go
```

## Which issue(s) this PR fixes *(optional, using `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when the PR gets merged)*:
Fixes #678 

## Type of change

Please delete options that are not relevant.

- [X] Bug fix (non-breaking change which fixes an issue)
- [ ] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected)
- [ ] Helm Chart Change (any edit/addition/update that is necessary for changes merged to the `main` branch)
- [ ] This change requires a documentation update

# How Has This Been Tested?

Tested locally against latest Ratify distribution.
```
time="2023-04-11T13:31:38Z" level=info msg="received request POST /ratify/gatekeeper/v1/mutate "
time="2023-04-11T13:31:38Z" level=info msg="start request POST /ratify/gatekeeper/v1/mutate"
time="2023-04-11T13:31:38Z" level=info msg="mutating image <AWS_ACCOUNT_ID>.dkr.ecr.us-west-2.amazonaws.com/pause:3.1"
time="2023-04-11T13:31:38Z" level=debug msg="no digest provided for reference <AWS_ACCOUNT_ID>.dkr.ecr.us-west-2.amazonaws.com/pause:3.1. attempting to resolve..."
time="2023-04-11T13:31:38Z" level=info msg="mutating image <AWS_ACCOUNT_ID>.dkr.ecr.us-east-2.amazonaws.com/pause:3.1"
time="2023-04-11T13:31:38Z" level=debug msg="no digest provided for reference <AWS_ACCOUNT_ID>.dkr.ecr.us-east-2.amazonaws.com/pause:3.1. attempting to resolve..."
time="2023-04-11T13:31:38Z" level=debug msg="mutation: execution time for image <AWS_ACCOUNT_ID>.dkr.ecr.us-east-2.amazonaws.com/pause:3.1: 69ms"
time="2023-04-11T13:31:38Z" level=debug msg="mutation: execution time for image <AWS_ACCOUNT_ID>.dkr.ecr.us-west-2.amazonaws.com/pause:3.1: 145ms"
time="2023-04-11T13:31:38Z" level=debug msg="mutation: execution time for request: 145ms"
time="2023-04-11T13:31:38Z" level=info msg="received request POST /ratify/gatekeeper/v1/verify "
time="2023-04-11T13:31:38Z" level=info msg="start request POST /ratify/gatekeeper/v1/verify"
time="2023-04-11T13:31:38Z" level=info msg="verifying subject <AWS_ACCOUNT_ID>.dkr.ecr.us-west-2.amazonaws.com/pause@sha256:fcaff905397ba63fd376d0c3019f1f1cb6e7506131389edbcb3d22719f1ae54d"
time="2023-04-11T13:31:38Z" level=debug msg="cache miss for subject <AWS_ACCOUNT_ID>.dkr.ecr.us-west-2.amazonaws.com/pause@sha256:fcaff905397ba63fd376d0c3019f1f1cb6e7506131389edbcb3d22719f1ae54d"
time="2023-04-11T13:31:38Z" level=info msg="Resolve of the image completed successfully the digest is sha256:fcaff905397ba63fd376d0c3019f1f1cb6e7506131389edbcb3d22719f1ae54d"
time="2023-04-11T13:31:38Z" level=info msg="verifying subject <AWS_ACCOUNT_ID>.dkr.ecr.us-east-2.amazonaws.com/pause@sha256:fcaff905397ba63fd376d0c3019f1f1cb6e7506131389edbcb3d22719f1ae54d"
time="2023-04-11T13:31:38Z" level=debug msg="cache miss for subject <AWS_ACCOUNT_ID>.dkr.ecr.us-east-2.amazonaws.com/pause@sha256:fcaff905397ba63fd376d0c3019f1f1cb6e7506131389edbcb3d22719f1ae54d"
time="2023-04-11T13:31:38Z" level=info msg="Resolve of the image completed successfully the digest is sha256:fcaff905397ba63fd376d0c3019f1f1cb6e7506131389edbcb3d22719f1ae54d"
{
  "isSuccess": false,
  "verifierReports": [
    {
      "isSuccess": false,
      "name": "notaryv2",
      "message": "an error thrown by the verifier: failed to verify signature, err: plugin unsupported due to nil verifier.pluginManager",
      "artifactType": "application/vnd.cncf.notary.signature"
    }
  ]
}
time="2023-04-11T13:31:38Z" level=debug msg="verification: execution time for image <AWS_ACCOUNT_ID>.dkr.ecr.us-east-2.amazonaws.com/pause@sha256:fcaff905397ba63fd376d0c3019f1f1cb6e7506131389edbcb3d22719f1ae54d: 99ms"
time="2023-04-11T13:31:38Z" level=debug msg="verification: execution time for image <AWS_ACCOUNT_ID>.dkr.ecr.us-west-2.amazonaws.com/pause@sha256:fcaff905397ba63fd376d0c3019f1f1cb6e7506131389edbcb3d22719f1ae54d: 239ms"
{
  "isSuccess": false,
time="2023-04-11T13:31:38Z" level=debug msg="verification: execution time for request: 239ms"
  "verifierReports": [
    {
      "isSuccess": false,
      "name": "notaryv2",
      "message": "an error thrown by the verifier: failed to verify signature, err: plugin unsupported due to nil verifier.pluginManager",
      "artifactType": "application/vnd.cncf.notary.signature"
    }
  ]
}
```

# Checklist:

- [X] Does the affected code have corresponding tests?
- [ ] Are the changes documented, not just with inline documentation, but also with conceptual documentation such as an overview of a new feature, or task-based documentation like a tutorial? Consider if this change should be announced on your project blog. N/A
- [ ] Does this introduce breaking changes that would require an announcement or bumping the major version?
- [X] Do all new files have appropriate license header?

# Post Merge Requirements
- [ ] MAINTAINERS: manually trigger the "Publish Package" workflow after merging any PR that indicates `Helm Chart Change`


Patch:
diff --git a/pkg/common/oras/authprovider/aws/awsecrbasic.go b/pkg/common/oras/authprovider/aws/awsecrbasic.go
index 7972f32..7e868e9 100644
--- a/pkg/common/oras/authprovider/aws/awsecrbasic.go
+++ b/pkg/common/oras/authprovider/aws/awsecrbasic.go
@@ -26,7 +26,9 @@ import (
 	"github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/credentials/stscreds"
 	"github.com/aws/aws-sdk-go-v2/service/ecr"
+	"github.com/aws/aws-sdk-go-v2/service/ecr/types"
 	provider "github.com/deislabs/ratify/pkg/common/oras/authprovider"
+	"github.com/deislabs/ratify/pkg/utils/awsauth"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
@@ -52,7 +54,7 @@ func init() {
 }
 
 // Get ECR auth token from IRSA config
-func getEcrAuthToken() (EcrAuthToken, error) {
+func (d *awsEcrBasicAuthProvider) getEcrAuthToken(artifact string) (EcrAuthToken, error) {
 	region := os.Getenv("AWS_REGION")
 	roleArn := os.Getenv("AWS_ROLE_ARN")
 	tokenFilePath := os.Getenv("AWS_WEB_IDENTITY_TOKEN_FILE")
@@ -92,13 +94,27 @@ func getEcrAuthToken() (EcrAuthToken, error) {
 		return EcrAuthToken{}, fmt.Errorf("failed to load default AWS basic auth config: %w", err)
 	}
 
+	// registry/region from image
+	registry, err := awsauth.RegistryFromImage(artifact)
+	if err != nil {
+		return EcrAuthToken{}, fmt.Errorf("failed to get registry from image: %w", err)
+	}
+	region = awsauth.RegionFromRegistry(registry)
+	if err != nil {
+		return EcrAuthToken{}, fmt.Errorf("failed to get region from image: %w", err)
+	}
+	logrus.Debugf("AWS ECR basic artifact=%s, registry=%s, region=%s", artifact, registry, region)
+	cfg.Region = region
+
 	ecrClient := ecr.NewFromConfig(cfg)
 	authOutput, err := ecrClient.GetAuthorizationToken(ctx, nil)
 	if err != nil {
 		return EcrAuthToken{}, fmt.Errorf("could not retrieve ECR auth token collection: %w", err)
 	}
 
-	return EcrAuthToken{AuthData: authOutput.AuthorizationData[0]}, nil
+	d.ecrAuthToken.AuthData[registry] = authOutput.AuthorizationData[0]
+
+	return d.ecrAuthToken, nil
 }
 
 // Create returns an AwsEcrBasicProvider
@@ -109,82 +125,74 @@ func (s *AwsEcrBasicProviderFactory) Create(authProviderConfig provider.AuthProv
 		return nil, err
 	}
 
-	if err := json.Unmarshal(authProviderConfigBytes, &conf); err != nil {
+	if err = json.Unmarshal(authProviderConfigBytes, &conf); err != nil {
 		return nil, fmt.Errorf("failed to parse auth provider configuration: %w", err)
 	}
 
-	// Build auth provider from AWS IRSA and ECR auth token
-	authData, err := getEcrAuthToken()
-	if err != nil {
-		return nil, fmt.Errorf("failed to get ECR auth data: %w", err)
-	}
+	ecrAuthToken := EcrAuthToken{}
+	ecrAuthToken.AuthData = make(map[string]types.AuthorizationData)
 
 	return &awsEcrBasicAuthProvider{
-		ecrAuthToken: authData,
+		ecrAuthToken: ecrAuthToken,
 		providerName: awsEcrAuthProviderName,
 	}, nil
 }
 
 // Enabled checks for non-empty AWS IAM creds
 func (d *awsEcrBasicAuthProvider) Enabled(ctx context.Context) bool {
-	creds, err := d.ecrAuthToken.BasicAuthCreds()
-	if creds == nil || err != nil {
-		logrus.Errorf("error getting basic ECR creds: %v", err)
-		return false
-	}
-
-	if len(creds) < 2 {
-		logrus.Error("basic ECR creds array had incorrect length")
-		return false
-	}
-
-	if creds[0] == "" || creds[1] == "" {
-		logrus.Error("basic ECR creds were empty")
-		return false
-	}
-
 	if d.providerName == "" {
 		logrus.Error("basic ECR providerName was empty")
 		return false
 	}
 
-	if d.ecrAuthToken.AuthData.ExpiresAt == nil {
-		logrus.Error("basic ECR expiry was nil")
-		return false
-	}
-
 	return true
 }
 
 // Provide returns the credentials for a specified artifact.
 // Uses AWS IRSA to retrieve creds from IRSA credential chain
 func (d *awsEcrBasicAuthProvider) Provide(ctx context.Context, artifact string) (provider.AuthConfig, error) {
+	logrus.Debugf("artifact = %s", artifact)
+
 	if !d.Enabled(ctx) {
 		return provider.AuthConfig{}, fmt.Errorf("AWS IRSA basic auth provider is not properly enabled")
 	}
 
+	registry, err := awsauth.RegistryFromImage(artifact)
+	if err != nil {
+		return provider.AuthConfig{}, errors.Wrapf(err, "could not get ECR registry from %s", artifact)
+	}
+
+	if !d.ecrAuthToken.exists(registry) {
+		logrus.Debugf("ecrAuthToken for %s does not exist", registry)
+		_, err = d.getEcrAuthToken(artifact)
+		if err != nil {
+			return provider.AuthConfig{}, errors.Wrapf(err, "could not get ECR auth token for %s", artifact)
+		}
+	}
+
 	// need to refresh AWS ECR credentials
 	t := time.Now().Add(time.Minute * 5)
-	if t.After(d.ecrAuthToken.Expiry()) || time.Now().After(d.ecrAuthToken.Expiry()) {
-		authToken, err := getEcrAuthToken()
+	if t.After(d.ecrAuthToken.Expiry(registry)) || time.Now().After(d.ecrAuthToken.Expiry(registry)) {
+		_, err = d.getEcrAuthToken(artifact)
 		if err != nil {
-			return provider.AuthConfig{}, errors.Wrap(err, "could not refresh ECR auth token")
+			return provider.AuthConfig{}, errors.Wrapf(err, "could not refresh ECR auth token for %s", artifact)
 		}
-		d.ecrAuthToken = authToken
-		logrus.Info("successfully refreshed ECR auth token")
+
+		logrus.Debugf("successfully refreshed ECR auth token for %s", artifact)
 	}
 
 	// Get ECR basic auth creds from auth data token
-	creds, err := d.ecrAuthToken.BasicAuthCreds()
+	var creds []string
+	creds, err = d.ecrAuthToken.BasicAuthCreds(registry)
 	if err != nil {
-		return provider.AuthConfig{}, errors.Wrap(err, "could not retrieve ECR credentials")
+		return provider.AuthConfig{}, errors.Wrapf(err, "could not retrieve ECR credentials for %s", artifact)
 	}
 
 	authConfig := provider.AuthConfig{
 		Username:  creds[0],
 		Password:  creds[1],
 		Provider:  d,
-		ExpiresOn: d.ecrAuthToken.Expiry(),
+		ExpiresOn: d.ecrAuthToken.Expiry(registry),
 	}
 
 	return authConfig, nil
diff --git a/pkg/common/oras/authprovider/aws/ecrauthtypes.go b/pkg/common/oras/authprovider/aws/ecrauthtypes.go
index 9e5b1ef..59dd438 100644
--- a/pkg/common/oras/authprovider/aws/ecrauthtypes.go
+++ b/pkg/common/oras/authprovider/aws/ecrauthtypes.go
@@ -26,22 +26,39 @@ import (
 
 // EcrAuthToken provides helper functions for ECR auth token data
 type EcrAuthToken struct {
-	AuthData types.AuthorizationData
+	AuthData map[string]types.AuthorizationData
 }
 
-// Expiry
-func (e EcrAuthToken) Expiry() time.Time {
-	return *e.AuthData.ExpiresAt
+// exists checks if authdata entries exist
+func (e EcrAuthToken) exists(key string) bool {
+	if len(e.AuthData) < 1 {
+		return false
+	}
+
+	if _, ok := e.AuthData[key]; !ok {
+		return false
+	}
+
+	if *e.AuthData[key].AuthorizationToken == "" {
+		return false
+	}
+
+	return true
+}
+
+// Expiry returns the expiry time
+func (e EcrAuthToken) Expiry(key string) time.Time {
+	return *e.AuthData[key].ExpiresAt
 }
 
-// ProxyEndpoint
-func (e EcrAuthToken) ProxyEndpoint() string {
-	return *e.AuthData.ProxyEndpoint
+// ProxyEndpoint returns the authdata proxy endpoint
+func (e EcrAuthToken) ProxyEndpoint(key string) string {
+	return *e.AuthData[key].ProxyEndpoint
 }
 
-// BasicAuthCreds
-func (e EcrAuthToken) BasicAuthCreds() ([]string, error) {
-	rawDecodedToken, err := base64.StdEncoding.DecodeString(*e.AuthData.AuthorizationToken)
+// BasicAuthCreds returns a string array of the basic creds
+func (e EcrAuthToken) BasicAuthCreds(key string) ([]string, error) {
+	rawDecodedToken, err := base64.StdEncoding.DecodeString(*e.AuthData[key].AuthorizationToken)
 	if err != nil {
 		return nil, fmt.Errorf("could not decode ECR auth token: %w", err)
 	}
diff --git a/pkg/utils/awsauth/utils.go b/pkg/utils/awsauth/utils.go
new file mode 100644
index 0000000..733cb2a
--- /dev/null
+++ b/pkg/utils/awsauth/utils.go
@@ -0,0 +1,43 @@
+/*
+Copyright The Ratify Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package awsauth
+
+import (
+	"net/url"
+	"strings"
+)
+
+// RegionFromRegistry parses AWS region ID from registry url
+func RegionFromRegistry(registry string) string {
+	a := strings.Split(registry, ".")
+	if len(a) >= 6 {
+		return a[3]
+	}
+	return ""
+}
+
+// RegistryFromImage parses registry host from image url
+func RegistryFromImage(image string) (string, error) {
+	if strings.Contains(image, "https://") {
+		u, err := url.Parse(image)
+		if err != nil {
+			return "", err
+		}
+		return u.Host, nil
+	}
+
+	return image[:strings.IndexByte(image, '/')], nil
+}


Test patch:
diff --git a/pkg/common/oras/authprovider/aws/awsecrbasic_test.go b/pkg/common/oras/authprovider/aws/awsecrbasic_test.go
index ab881c6..5e2bfa1 100644
--- a/pkg/common/oras/authprovider/aws/awsecrbasic_test.go
+++ b/pkg/common/oras/authprovider/aws/awsecrbasic_test.go
@@ -31,6 +31,7 @@ const (
 	// #nosec G101 (Ref: https://github.com/securego/gosec/issues/295)
 	testPassword = "eyJwYXlsb2FkIjoiOThPNTFqemhaUmZWVG"
 	testProxy    = "PROXY_ENDPOINT"
+	testHost     = "123456789012.dkr.ecr.us-east-2.amazonaws.com"
 )
 
 // Verifies that awsEcrBasicAuthProvider is properly constructed by mocking EcrAuthToken
@@ -40,16 +41,34 @@ func mockAuthData() types.AuthorizationData {
 	creds := []string{testUsername, testPassword}
 	encoded := base64.StdEncoding.EncodeToString([]byte(strings.Join(creds, ":")))
 
+	expiry := aws.Time(time.Now().Add(time.Minute * 10))
+
 	return types.AuthorizationData{
 		AuthorizationToken: aws.String(encoded),
-		ExpiresAt:          aws.Time(time.Now().Add(time.Minute * 10)),
+		ExpiresAt:          expiry,
 		ProxyEndpoint:      aws.String(testProxy),
 	}
 }
 
+func TestAwsEcrBasicAuthProvider_Create(t *testing.T) {
+	authProviderConfig := map[string]interface{}{
+		"name": "awsEcrBasic",
+	}
+
+	factory := AwsEcrBasicProviderFactory{}
+	_, err := factory.Create(authProviderConfig)
+
+	if err != nil {
+		t.Fatalf("create failed %v", err)
+	}
+}
+
 func TestAwsEcrBasicAuthProvider_Enabled(t *testing.T) {
+	ecrAuthToken := EcrAuthToken{}
+	ecrAuthToken.AuthData = make(map[string]types.AuthorizationData)
+	ecrAuthToken.AuthData[testHost] = mockAuthData()
 	authProvider := awsEcrBasicAuthProvider{
-		ecrAuthToken: EcrAuthToken{mockAuthData()},
+		ecrAuthToken: ecrAuthToken,
 		providerName: awsEcrAuthProviderName,
 	}
 
@@ -63,24 +82,4 @@ func TestAwsEcrBasicAuthProvider_Enabled(t *testing.T) {
 	if authProvider.Enabled(ctx) {
 		t.Fatal("enabled should have returned false but returned true")
 	}
-
-	authProvider = awsEcrBasicAuthProvider{
-		ecrAuthToken: EcrAuthToken{mockAuthData()},
-		providerName: awsEcrAuthProviderName,
-	}
-
-	authProvider.ecrAuthToken.AuthData.AuthorizationToken = aws.String("")
-	if authProvider.Enabled(ctx) {
-		t.Fatal("enabled should have returned false but returned true")
-	}
-
-	authProvider = awsEcrBasicAuthProvider{
-		ecrAuthToken: EcrAuthToken{mockAuthData()},
-		providerName: awsEcrAuthProviderName,
-	}
-
-	authProvider.ecrAuthToken.AuthData.ExpiresAt = nil
-	if authProvider.Enabled(ctx) {
-		t.Fatal("enabled should have returned false but returned true")
-	}
 }
diff --git a/pkg/utils/awsauth/utils_test.go b/pkg/utils/awsauth/utils_test.go
new file mode 100644
index 0000000..c9576bf
--- /dev/null
+++ b/pkg/utils/awsauth/utils_test.go
@@ -0,0 +1,43 @@
+/*
+Copyright The Ratify Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package awsauth
+
+import "testing"
+
+const (
+	image    = "123456789012.dkr.ecr.us-east-2.amazonaws.com/pause:3.1"
+	registry = "123456789012.dkr.ecr.us-east-2.amazonaws.com"
+	region   = "us-east-2"
+)
+
+func TestRegistryFromImage_ReturnsExpected(t *testing.T) {
+	reg, err := RegistryFromImage(image)
+
+	if reg == "" || err != nil {
+		t.Fatalf("registry parsing failed, expected registry but returned error %v", err)
+	}
+
+	if reg != registry {
+		t.Fatalf("incorrect registry returned, expected %s, but received %s", registry, reg)
+	}
+}
+
+func TestRegionFromRegistry_ReturnsExpected(t *testing.T) {
+	reg := RegionFromRegistry(registry)
+	if reg != region {
+		t.Fatalf("incorrect region returned, expected %s, but received %s", region, reg)
+	}
+}