Repository: minio/minio (language: go)

Issue:
(Unable to initialize IAM sub-system, some users may not be available: signature is invalid) with large session policy 
Hi ! 
We are facing the exact same issue as #16934 since we upgraded from RELEASE.2023-01-31T02-24-19Z to RELEASE.2023-04-13T03-08-07Z (we had to upgrade due to being vulnerable to Cve-2023-28432).
Now, each time there is a cluster restart (e.g after any update) the IAM subsystem gets corrupted with error `Unable to initialize IAM sub-system, some users may not be available: signature is invalid` and nothing related to IAM / STS works anymore.  
Wiping .minio.sys folder gets it running again (with side-effect of losing every service account and policy) ... until next restart / upgrade.
We tried upgrading to the latest version `RELEASE.2023-05-04T21-44-30Z` and it corrupted the IAM subsystem once again ...
So this bug is definitely not fixed  

Configuration :  
Operator `v5.0.4`  
Minio `RELEASE.2023-05-04T21-44-30Z`  

Tenant configuration :  
```yaml
apiVersion: v1
items:
- apiVersion: minio.min.io/v2
  kind: Tenant
  metadata:
    annotations:
      prometheus.io/path: /minio/v2/metrics/cluster
      prometheus.io/port: "9000"
      prometheus.io/scrape: "true"
    creationTimestamp: "2022-02-10T16:39:24Z"
    generation: 13
    labels:
      app: minio-datanode
    name: minio-datanode
    namespace: minio-datanode
    resourceVersion: "881002015"
    uid: 249e3083-6152-4d37-a226-527db3c81d94
  scheduler:
    name: ""
  spec:
    certConfig:
      commonName: minio-datanode
      dnsNames:
      - minio-datanode-pool-0-{0...3}.minio-datanode-hl.minio-datanode.svc.cluster.local
      - minio-datanode-pool-1-{0...3}.minio-datanode-hl.minio-datanode.svc.cluster.local
      organizationName:
      - system:nodes
    configuration:
      name: minio-datanode-configuration
    env:
    - name: MINIO_IDENTITY_OPENID_CONFIG_URL
      value: https://auth.mydomain.fr/auth/realms/myrealm/.well-known/openid-configuration
    - name: MINIO_IDENTITY_OPENID_CLIENT_ID
      value: minio-datanode
    - name: MINIO_IDENTITY_OPENID_CLIENT_SECRET
      value: <redacted>
    - name: MINIO_IDENTITY_OPENID_REDIRECT_URI
      value: https://minio-console.mydomain.fr/oauth_callback
    - name: MINIO_IDENTITY_OPENID_SCOPES
      value: openid, profile, email, roles
    - name: MINIO_IDENTITY_OPENID_CLAIM_NAME
      value: policy
    - name: MINIO_DOMAIN
      value: minio.mydomain.fr
    - name: MINIO_BROWSER
      value: "on"
    - name: MINIO_BROWSER_REDIRECT_URL
      value: https://minio-console.mydomain.fr
    - name: MINIO_SERVER_URL
      value: https://minio.mydomain.fr
    image: minio/minio:RELEASE.2023-05-04T21-44-30Z
    imagePullPolicy: IfNotPresent
    imagePullSecret: {}
    mountPath: /export
    podManagementPolicy: Parallel
    pools:
    - affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
          - labelSelector:
              matchExpressions:
              - key: v1.min.io/tenant
                operator: In
                values:
                - minio
              - key: v1.min.io/pool
                operator: In
                values:
                - pool-0
            topologyKey: kubernetes.io/hostname
      name: pool-0
      nodeSelector:
        datanode: "true"
      resources:
        limits:
          cpu: "30"
          memory: 128Gi
        requests:
          cpu: "5"
          memory: 32Gi
      securityContext:
        fsGroup: 0
        runAsGroup: 0
        runAsNonRoot: false
        runAsUser: 0
      servers: 4
      volumeClaimTemplate:
        metadata:
          name: data
        spec:
          accessModes:
          - ReadWriteOnce
          resources:
            requests:
              storage: 1117Gi
          storageClassName: minio-disks-datanode
        status: {}
      volumesPerServer: 18
    - affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
          - labelSelector:
              matchExpressions:
              - key: v1.min.io/tenant
                operator: In
                values:
                - minio
              - key: v1.min.io/pool
                operator: In
                values:
                - pool-1
            topologyKey: kubernetes.io/hostname
      name: pool-1
      nodeSelector:
        minio-pool1: "true"
      resources:
        limits:
          cpu: "30"
          memory: 128Gi
        requests:
          cpu: "5"
          memory: 32Gi
      securityContext:
        fsGroup: 0
        runAsGroup: 0
        runAsNonRoot: false
        runAsUser: 0
      servers: 4
      volumeClaimTemplate:
        metadata:
          name: data
        spec:
          accessModes:
          - ReadWriteOnce
          resources:
            requests:
              storage: 2233Gi
          storageClassName: minio-disks-pool1
        status: {}
      volumesPerServer: 18
    requestAutoCert: true
    serviceMetadata:
      consoleServiceAnnotations:
        v2.min.io: console-svc
      consoleServiceLabels:
        label: console-svc
      minioServiceAnnotations:
        v2.min.io: minio-svc
      minioServiceLabels:
        label: minio-svc
  status:
    availableReplicas: 8
    certificates:
      autoCertEnabled: true
      customCertificates: {}
    currentState: Initialized
    drivesOnline: 144
    healthStatus: green
    pools:
    - legacySecurityContext: false
      ssName: minio-datanode-pool-0
      state: PoolInitialized
    - legacySecurityContext: false
      ssName: minio-datanode-pool-1
      state: PoolInitialized
    revision: 0
    syncVersion: v5.0.0
    usage:
      capacity: 172718343979008
      rawCapacity: 258986527948800
      rawUsage: 29487540011008
      usage: 15240031941494
    writeQuorum: 8
kind: List
metadata:
  resourceVersion: ""
```  

Logs :  
```
Waiting for all MinIO sub-systems to be initialize...
Automatically configured API requests per node based on available memory on the system: 5540
All MinIO sub-systems initialized successfully in 66.239391ms

API: SYSTEM()
Time: 14:04:04 UTC 05/07/2023
DeploymentID: 6b4a7600-1bf7-461a-9768-9518443b6de9
Error: Unable to initialize IAM sub-system, some users may not be available: signature is invalid (*fmt.wrapError)
       3: internal/logger/logger.go:258:logger.LogIf()
       2: cmd/iam.go:311:cmd.(*IAMSys).Init()
       1: cmd/server-main.go:652:cmd.serverMain.func3()
MinIO Object Storage Server
Copyright: 2015-2023 MinIO, Inc.
License: GNU AGPLv3 <https://www.gnu.org/licenses/agpl-3.0.html>
Version: RELEASE.2023-05-04T21-44-30Z (go1.20.3 linux/amd64)

Status:         144 Online, 0 Offline. 
S3-API: https://minio.mydomain.fr 
Console: https://minio-console.mydomain.fr 

Documentation: https://min.io/docs/minio/linux/index.html

API: SYSTEM()
Time: 14:14:05 UTC 05/07/2023
DeploymentID: 6b4a7600-1bf7-461a-9768-9518443b6de9
Error: Failure in periodic refresh for IAM (took 0.95s): signature is invalid (*errors.errorString)
       2: internal/logger/logger.go:258:logger.LogIf()
       1: cmd/iam.go:451:cmd.(*IAMSys).watch()
```


Discussion / hints:
@olevitt this is not a bug it looks like you are missing to provide some key piece of information here.

It almost looks like you have changed something in your deployment. Please describe when did you deploy this first? Did you ever have KMS configured?

Please share the configuration secret also, and what values do you have there.
Thanks for your reply.  
Not sure what key piece of information are missing, I think most information are contained in the tenant configuration ?  
This tenant was first deployed on `2022-02-10T16:39:24Z` and has been upgraded ever since using `kubectl minio tenant upgrade --image x`. We added a second pool at some point but the cluster was running fine for a few months after that.  
The bug first occured when upgrading from `RELEASE.2023-01-31T02-24-19Z` to `RELEASE.2023-04-13T03-08-07Z` and is now occuring every time we upgrade. Note that minIO itself is working just fine, data was never lost but STS / IAM gets broken after every upgrade. We followed suggestion from https://github.com/minio/minio/pull/16812#issuecomment-1497574424 after having encountering the bug for the first time but it only got it working until the next upgrade.  

Here are the additional info you requested :  
We have never configured KMS.  
Here is the configuration secret `minio-datanode-configuration`, which appears to me to be in line with the tenant configuration :  
```
export MINIO_IDENTITY_OPENID_REDIRECT_URI="https://minio-console.mydomain.fr/oauth_callback"
export MINIO_BROWSER_REDIRECT_URL="https://minio-console.mydomain.fr"
export MINIO_SERVER_URL="https://minio.mydomain.fr"
export MINIO_IDENTITY_OPENID_SCOPES="openid, profile, email, roles"
export MINIO_DOMAIN="minio.mydomain.fr"
export MINIO_ROOT_USER="minio"
export MINIO_IDENTITY_OPENID_CLIENT_ID="minio-datanode"
export MINIO_IDENTITY_OPENID_CLAIM_NAME="policy"
export MINIO_BROWSER="on"
export MINIO_ROOT_PASSWORD="redacted"
export MINIO_IDENTITY_OPENID_CONFIG_URL="https://auth.mydomain.fr/auth/realms/myrealm/.well-known/openid-configuration"
```
This doesn't look like true at all from what you are saying @olevitt 

The signature invalid error can only happen if the persisted data was encrypted.

In your case the IAM data is not encrypted. Unless you have something that corrupts the on disk data somehow.

We also do not heal any IAM content upon upgrades anymore.

I also went through the steps mentioned, I simply cannot reproduce this. So it is very much something in the environment that is wrong, you have definitely changed something in your setup when you upgrade.
Do you change root credentials everytime you upgrade? Does these STS and Service accounts based on the external OIDC ?
I think the problem is because of the JWT session token that is saved on the disk that is invalid for some reason few things are possible 

- you have changed your root credentials in all the steps here in between
- there is some timing issue in your set-up that is NTP is not configured well on all the nodes and an upgrade is basically trying to validate these session tokens.
- or we have an error where we are not ignoring this error properly and moving on.
Thanks for those possible causes, I will look into it.  
I assure you we have changed the root password only once in this deployment lifetime and we don't do it when we upgrade version, we simply `kubectl minio tenant upgrade --image`.
I will look deeper onto the JWT validation lead. Do you have a way in mind that I could use to enable verbose logging to try to narrow down the sub-cause that make the validation fail ?
Using `mc admin trace sspcloud --all  | grep -i BOOTSTRAP` I was able to confirm that error occurs at the "Loading service accounts" step (https://github.com/minio/minio/blob/477230c82e3c8f07c9c0ef21107d33240d7ef1cb/cmd/iam-object-store.go#L443) as previous loadings are present in logs but not the "loading STS users" which is supposed to follow.  Note that this occurs in all servers (2 pools of 4 servers) which leads me to believe this is not related to a single disk corruption or something in that vein.
I currently tend to think that the `or we have an error where we are not ignoring this error properly and moving on.` on JWT validation of service accounts may well be the issue here.
Sadly I have currently no easy idea on how to debug this more. I can't read the service account data as it's erasure coded. I may try to setup a standalone deployment with the same IAM configuration and try to reproduce this and check the content of the service account data. But if anyone has a better idea on how I can further debug this please let me know, I would gladly do it.  

After a few hours of debug, I finally managed to reproduce the error in a fresh new, simple, standalone deployment and track down the cause !  
**The error appears to be due to having a user policy too large (but still under the 20kb limit) and is triggered by creating an access key / service account as a user with a very large policy**.  
As the user policy is added to the `sessionPolicy` claim of the service account session token, the resulting token is too large and appears to make the JWT validation fail.  
Once a such service account is created then any IAM initiatialisation will fail, effectively making the IAM subsystem corrupted and unable to start. Any pod restart (for any reason, not only version upgrade contrary to what I first thought) will make the IAM susbsystem unusable.  
I think it's a regression in a version after `RELEASE.2023-01-31T02-24-19Z` as we had this one in production for quite some time with huge policies without any issue at all. Also, we had the bug appear just after upgrading which is in line with the bug being somewhere in the JWT validation. Service accounts created on `RELEASE.2023-01-31T02-24-19Z` were validated just fine in `RELEASE.2023-01-31T02-24-19Z` but not anymore on `RELEASE.2023-04-13T03-08-07Z`.

Here is how to reproduce :  
1. Create a new tenant with openid configured. Here is the one I used (basically created a fresh one from the helm tenant chart and added the openid env variables)  :  
```yaml
---
# Source: tenant/templates/tenant-configuration.yaml
apiVersion: v1
kind: Secret
metadata:
  name: minio-test-env-configuration
type: Opaque
stringData:
  config.env: |-
    export MINIO_ROOT_USER="minio"
    export MINIO_ROOT_PASSWORD="minio123"
---
# Source: tenant/templates/tenant.yaml
apiVersion: minio.min.io/v2
kind: Tenant
metadata:
  name: minio-test
  ## Optionally pass labels to be applied to the statefulset pods
  labels:
    app: minio-test
spec:
  image: quay.io/minio/minio:RELEASE.2023-05-04T21-44-30Z
  imagePullPolicy: IfNotPresent
  ## Secret with default environment variable configurations
  configuration:
    name: minio-test-env-configuration
  pools:
    - servers: 1
      name: pool-1
      volumesPerServer: 1
      volumeClaimTemplate:
        metadata:
          name: data
        spec:
          accessModes:
            - ReadWriteOnce
          resources:
            requests:
              storage: 10Gi
  env:
    - name: MINIO_IDENTITY_OPENID_CONFIG_URL
      value: "https://auth.mydomain.fr/auth/realms/myrealm/.well-known/openid-configuration"
    - name: MINIO_IDENTITY_OPENID_CLIENT_ID
      value: "minio-datanode"
    - name: MINIO_IDENTITY_OPENID_CLIENT_SECRET
      value: "redacted"
    - name: MINIO_IDENTITY_OPENID_REDIRECT_URI
      value: https://minio-test-console.mydomain.fr/oauth_callback
    - name: MINIO_IDENTITY_OPENID_SCOPES
      value: openid, profile, email, roles
    - name: MINIO_IDENTITY_OPENID_CLAIM_NAME
      value: policy
    - name: MINIO_DOMAIN
      value: "minio-test.mydomain.fr"
    - name: MINIO_BROWSER
      value: "on" # to turn-off browser
    - name: MINIO_BROWSER_REDIRECT_URL
      value: "https://minio-test-console.mydomain.fr"
    - name: MINIO_SERVER_URL
      value: "https://minio-test.mydomain.fr"
  mountPath: /export
  subPath: /data
  requestAutoCert: true
  features:
    bucketDNS: false
  podManagementPolicy: Parallel
  prometheusOperator: false
  logging:
    anonymous: true
    json: true
    quiet: true
```  

2. Make sure your `idp` fills the `policy` claim, I chose `stsonly` as the value of this claim  
3. Create a very large policy (tested with a 12kb policy, in my testing the cut-off seems to be somewhere between 5kb and 10kb) with the name corresponding to the `policy` claim value : `mc admin policy create test stsonly stsonly.json`. 
4. Now you should be able to access the console as an openid user and use the minio server as you please
5. Now let's trigger the bug. Inside the console, logged as an OIDC user, create a new access key for this user and **click on "restrict policy" but don't actually restrict the policy**, just wait until the textarea is filled and click create. Notice that once you dismiss the credentials popup, you are redirected to the service account list but it's still empty (this is a related bug I think). If you check the data inside the minio pod (`export/data/.minio.sys/config/iam/service-accounts/`) you will notice that the service account has been created but has a very very large sessionToken. This file will lead to IAM subsystem initialisation failure.
6. Kill / restart the pod (or do a version upgrade). Wait for the pod to restart : the IAM subsystem fails to start and will fail as long as the service account is there

If you prefer, I can recreate an issue with those findings, let me know !


Wonderful will have this fixed, we shouldn't allow this to be possible at all.

This is because we have the sessionToken validated prior to loading the credentials, in newer releases.

I will have to take a look why it is allowing such a token to be created in the first place.

Fix patch:
diff --git a/cmd/admin-handler-utils.go b/cmd/admin-handler-utils.go
index 706a52463f4b9..b0a098c0c9d22 100644
--- a/cmd/admin-handler-utils.go
+++ b/cmd/admin-handler-utils.go
@@ -168,10 +168,16 @@ func toAdminAPIErr(ctx context.Context, err error) APIError {
 			}
 		case errors.Is(err, errPolicyInUse):
 			apiErr = APIError{
-				Code:           "XMinioAdminPolicyInUse",
+				Code:           "XMinioIAMPolicyInUse",
 				Description:    "The policy cannot be removed, as it is in use",
 				HTTPStatusCode: http.StatusBadRequest,
 			}
+		case errors.Is(err, errSessionPolicyTooLarge):
+			apiErr = APIError{
+				Code:           "XMinioIAMServiceAccountSessionPolicyTooLarge",
+				Description:    err.Error(),
+				HTTPStatusCode: http.StatusBadRequest,
+			}
 		case errors.Is(err, kes.ErrKeyExists):
 			apiErr = APIError{
 				Code:           "XMinioKMSKeyExists",
diff --git a/cmd/iam-store.go b/cmd/iam-store.go
index a0a032d62b837..57184f9f305c5 100644
--- a/cmd/iam-store.go
+++ b/cmd/iam-store.go
@@ -27,7 +27,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/dustin/go-humanize"
 	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/madmin-go/v2"
 	"github.com/minio/minio-go/v7/pkg/set"
@@ -2221,13 +2220,13 @@ func (store *IAMStoreSys) UpdateServiceAccount(ctx context.Context, accessKey st
 	delete(m, sessionPolicyNameExtracted)
 
 	// sessionPolicy is nil and there is embedded policy attached we remove
-	// rembedded policy at that point.
+	// embedded policy at that point.
 	if _, ok := m[iampolicy.SessionPolicyName]; ok && opts.sessionPolicy == nil {
 		delete(m, iampolicy.SessionPolicyName)
 		m[iamPolicyClaimNameSA()] = inheritedPolicyType
 	}
 
-	if opts.sessionPolicy != nil {
+	if opts.sessionPolicy != nil { // session policies is being updated
 		if err := opts.sessionPolicy.Validate(); err != nil {
 			return updatedAt, err
 		}
@@ -2237,8 +2236,8 @@ func (store *IAMStoreSys) UpdateServiceAccount(ctx context.Context, accessKey st
 			return updatedAt, err
 		}
 
-		if len(policyBuf) > 16*humanize.KiByte {
-			return updatedAt, fmt.Errorf("Session policy should not exceed 16 KiB characters")
+		if len(policyBuf) > 2048 {
+			return updatedAt, errSessionPolicyTooLarge
 		}
 
 		// Overwrite session policy claims.
diff --git a/cmd/iam.go b/cmd/iam.go
index 4f5b2b0a20cbb..05351accf56bd 100644
--- a/cmd/iam.go
+++ b/cmd/iam.go
@@ -32,7 +32,6 @@ import (
 	"sync/atomic"
 	"time"
 
-	humanize "github.com/dustin/go-humanize"
 	"github.com/minio/madmin-go/v2"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/arn"
@@ -944,8 +943,8 @@ func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser string, gro
 		if err != nil {
 			return auth.Credentials{}, time.Time{}, err
 		}
-		if len(policyBuf) > 16*humanize.KiByte {
-			return auth.Credentials{}, time.Time{}, fmt.Errorf("Session policy should not exceed 16 KiB characters")
+		if len(policyBuf) > 2048 {
+			return auth.Credentials{}, time.Time{}, errSessionPolicyTooLarge
 		}
 	}
 
diff --git a/cmd/sts-handlers.go b/cmd/sts-handlers.go
index 5b71263724c26..10e188b57b798 100644
--- a/cmd/sts-handlers.go
+++ b/cmd/sts-handlers.go
@@ -238,7 +238,7 @@ func (sts *stsAPIHandlers) AssumeRole(w http.ResponseWriter, r *http.Request) {
 	// The plain text that you use for both inline and managed session
 	// policies shouldn't exceed 2048 characters.
 	if len(sessionPolicyStr) > 2048 {
-		writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, fmt.Errorf("Session policy shouldn't exceed 2048 characters"))
+		writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, errSessionPolicyTooLarge)
 		return
 	}
 
diff --git a/cmd/typed-errors.go b/cmd/typed-errors.go
index c125d47f81c2e..6b9d4391accdb 100644
--- a/cmd/typed-errors.go
+++ b/cmd/typed-errors.go
@@ -115,3 +115,6 @@ var errUploadIDNotFound = errors.New("Specified Upload ID is not found")
 
 // error returned when PartNumber is greater than the maximum allowed 10000 parts
 var errInvalidMaxParts = errors.New("Part number is greater than the maximum allowed 10000 parts")
+
+// error returned for session policies > 2048
+var errSessionPolicyTooLarge = errors.New("Session policy should not exceed 2048 characters")
diff --git a/internal/jwt/parser.go b/internal/jwt/parser.go
index 62e4138b12da9..44c29c284597a 100644
--- a/internal/jwt/parser.go
+++ b/internal/jwt/parser.go
@@ -34,6 +34,7 @@ import (
 	"time"
 
 	"github.com/buger/jsonparser"
+	"github.com/dustin/go-humanize"
 	jwtgo "github.com/golang-jwt/jwt/v4"
 	jsoniter "github.com/json-iterator/go"
 )
@@ -53,7 +54,7 @@ var (
 	SigningMethodHS512 *SigningMethodHMAC
 )
 
-const base64BufferSize = 8192
+const base64BufferSize = 64 * humanize.KiByte
 
 var (
 	base64BufPool sync.Pool


Test patch: