Repository: kubernetes-sigs/kueue

Issue:
Inconsistent limit on the number of resources and flavors in a ClusterQueue
**What happened**:

In PR [#6906](https://github.com/kubernetes-sigs/kueue/pull/6906), the coveredResource and Flavors limits where increased to 64, however some fields still use the old limit of 16, including the [Resources under FlavorQuotas](https://github.com/kubernetes-sigs/kueue/blob/c05bfbd980952fd47b9957c9c5952dada41f0996/apis/kueue/v1beta1/clusterqueue_types.go#L215-L216) and [FlavorsReservation and FlavorsUsage under ClusterQueueStatus](https://github.com/kubernetes-sigs/kueue/blob/c05bfbd980952fd47b9957c9c5952dada41f0996/apis/kueue/v1beta1/clusterqueue_types.go#L277-L291).

So when creating ClusterQueues with more than 16 resources per flavor, we will get an error:
```
* spec.resourceGroups[0].flavors[0].resources: Too many: 33: must have at most 16 items
```

**What you expected to happen**:

The above fields should have MaxItems of 64 to match the limit on coveredResources and Flavors in order to successfully create ClusterQueue Flavors with up to 64 resources.

**How to reproduce it (as minimally and precisely as possible)**:

Create a ClusterQueue with more than 16 resources in a resourceGroup.


**Anything else we need to know?**:

**Environment**: 
- Kubernetes version (use `kubectl version`): v1.33.4
- Kueue version (use `git describe --tags --dirty --always`): da94f0576
- Cloud provider or hardware configuration:
- OS (e.g: `cat /etc/os-release`):
- Kernel (e.g. `uname -a`):
- Install tools:
- Others:

Discussion:
@mimowo: @oyangz please submit a PR if you want to get this in 0.14

Solution patch:
diff --git a/apis/kueue/v1beta1/clusterqueue_types.go b/apis/kueue/v1beta1/clusterqueue_types.go
index f35d7e019..82823f9e4 100644
--- a/apis/kueue/v1beta1/clusterqueue_types.go
+++ b/apis/kueue/v1beta1/clusterqueue_types.go
@@ -208,11 +208,11 @@ type FlavorQuotas struct {
 	Name ResourceFlavorReference `json:"name"`
 
 	// resources is the list of quotas for this flavor per resource.
-	// There could be up to 16 resources.
+	// There could be up to 64 resources.
 	// +listType=map
 	// +listMapKey=name
 	// +kubebuilder:validation:MinItems=1
-	// +kubebuilder:validation:MaxItems=16
+	// +kubebuilder:validation:MaxItems=64
 	Resources []ResourceQuota `json:"resources"`
 }
 
@@ -278,7 +278,7 @@ type ClusterQueueStatus struct {
 	// workloads assigned to this ClusterQueue.
 	// +listType=map
 	// +listMapKey=name
-	// +kubebuilder:validation:MaxItems=16
+	// +kubebuilder:validation:MaxItems=64
 	// +optional
 	FlavorsReservation []FlavorUsage `json:"flavorsReservation"`
 
@@ -286,7 +286,7 @@ type ClusterQueueStatus struct {
 	// workloads admitted in this ClusterQueue.
 	// +listType=map
 	// +listMapKey=name
-	// +kubebuilder:validation:MaxItems=16
+	// +kubebuilder:validation:MaxItems=64
 	// +optional
 	FlavorsUsage []FlavorUsage `json:"flavorsUsage"`
 
@@ -348,7 +348,7 @@ type FlavorUsage struct {
 	// resources lists the quota usage for the resources in this flavor.
 	// +listType=map
 	// +listMapKey=name
-	// +kubebuilder:validation:MaxItems=16
+	// +kubebuilder:validation:MaxItems=64
 	Resources []ResourceUsage `json:"resources"`
 }
 
diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_clusterqueues.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_clusterqueues.yaml
index e7192274b..153f9fd41 100644
--- a/charts/kueue/templates/crd/kueue.x-k8s.io_clusterqueues.yaml
+++ b/charts/kueue/templates/crd/kueue.x-k8s.io_clusterqueues.yaml
@@ -395,7 +395,7 @@ spec:
                             resources:
                               description: |-
                                 resources is the list of quotas for this flavor per resource.
-                                There could be up to 16 resources.
+                                There could be up to 64 resources.
                               items:
                                 properties:
                                   borrowingLimit:
@@ -456,7 +456,7 @@ spec:
                                   - name
                                   - nominalQuota
                                 type: object
-                              maxItems: 16
+                              maxItems: 64
                               minItems: 1
                               type: array
                               x-kubernetes-list-map-keys:
@@ -654,7 +654,7 @@ spec:
                           required:
                             - name
                           type: object
-                        maxItems: 16
+                        maxItems: 64
                         type: array
                         x-kubernetes-list-map-keys:
                           - name
@@ -663,7 +663,7 @@ spec:
                       - name
                       - resources
                     type: object
-                  maxItems: 16
+                  maxItems: 64
                   type: array
                   x-kubernetes-list-map-keys:
                     - name
@@ -707,7 +707,7 @@ spec:
                           required:
                             - name
                           type: object
-                        maxItems: 16
+                        maxItems: 64
                         type: array
                         x-kubernetes-list-map-keys:
                           - name
@@ -716,7 +716,7 @@ spec:
                       - name
                       - resources
                     type: object
-                  maxItems: 16
+                  maxItems: 64
                   type: array
                   x-kubernetes-list-map-keys:
                     - name
diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_cohorts.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_cohorts.yaml
index ac459dd50..04907d2ca 100644
--- a/charts/kueue/templates/crd/kueue.x-k8s.io_cohorts.yaml
+++ b/charts/kueue/templates/crd/kueue.x-k8s.io_cohorts.yaml
@@ -164,7 +164,7 @@ spec:
                             resources:
                               description: |-
                                 resources is the list of quotas for this flavor per resource.
-                                There could be up to 16 resources.
+                                There could be up to 64 resources.
                               items:
                                 properties:
                                   borrowingLimit:
@@ -225,7 +225,7 @@ spec:
                                   - name
                                   - nominalQuota
                                 type: object
-                              maxItems: 16
+                              maxItems: 64
                               minItems: 1
                               type: array
                               x-kubernetes-list-map-keys:
diff --git a/config/components/crd/bases/kueue.x-k8s.io_clusterqueues.yaml b/config/components/crd/bases/kueue.x-k8s.io_clusterqueues.yaml
index ea1f7a22a..f66477a9c 100644
--- a/config/components/crd/bases/kueue.x-k8s.io_clusterqueues.yaml
+++ b/config/components/crd/bases/kueue.x-k8s.io_clusterqueues.yaml
@@ -386,7 +386,7 @@ spec:
                           resources:
                             description: |-
                               resources is the list of quotas for this flavor per resource.
-                              There could be up to 16 resources.
+                              There could be up to 64 resources.
                             items:
                               properties:
                                 borrowingLimit:
@@ -447,7 +447,7 @@ spec:
                               - name
                               - nominalQuota
                               type: object
-                            maxItems: 16
+                            maxItems: 64
                             minItems: 1
                             type: array
                             x-kubernetes-list-map-keys:
@@ -651,7 +651,7 @@ spec:
                         required:
                         - name
                         type: object
-                      maxItems: 16
+                      maxItems: 64
                       type: array
                       x-kubernetes-list-map-keys:
                       - name
@@ -660,7 +660,7 @@ spec:
                   - name
                   - resources
                   type: object
-                maxItems: 16
+                maxItems: 64
                 type: array
                 x-kubernetes-list-map-keys:
                 - name
@@ -705,7 +705,7 @@ spec:
                         required:
                         - name
                         type: object
-                      maxItems: 16
+                      maxItems: 64
                       type: array
                       x-kubernetes-list-map-keys:
                       - name
@@ -714,7 +714,7 @@ spec:
                   - name
                   - resources
                   type: object
-                maxItems: 16
+                maxItems: 64
                 type: array
                 x-kubernetes-list-map-keys:
                 - name
diff --git a/config/components/crd/bases/kueue.x-k8s.io_cohorts.yaml b/config/components/crd/bases/kueue.x-k8s.io_cohorts.yaml
index 743212a57..d6c959b82 100644
--- a/config/components/crd/bases/kueue.x-k8s.io_cohorts.yaml
+++ b/config/components/crd/bases/kueue.x-k8s.io_cohorts.yaml
@@ -149,7 +149,7 @@ spec:
                           resources:
                             description: |-
                               resources is the list of quotas for this flavor per resource.
-                              There could be up to 16 resources.
+                              There could be up to 64 resources.
                             items:
                               properties:
                                 borrowingLimit:
@@ -210,7 +210,7 @@ spec:
                               - name
                               - nominalQuota
                               type: object
-                            maxItems: 16
+                            maxItems: 64
                             minItems: 1
                             type: array
                             x-kubernetes-list-map-keys:
diff --git a/site/content/en/docs/reference/kueue.v1beta1.md b/site/content/en/docs/reference/kueue.v1beta1.md
index 961898022..618e50567 100644
--- a/site/content/en/docs/reference/kueue.v1beta1.md
+++ b/site/content/en/docs/reference/kueue.v1beta1.md
@@ -1512,7 +1512,7 @@ ClusterQueue will have an Active condition set to False.</p>
 </td>
 <td>
    <p>resources is the list of quotas for this flavor per resource.
-There could be up to 16 resources.</p>
+There could be up to 64 resources.</p>
 </td>
 </tr>
 </tbody>



Test patch:
diff --git a/test/integration/singlecluster/webhook/core/clusterqueue_test.go b/test/integration/singlecluster/webhook/core/clusterqueue_test.go
index b42e378ba..33e52d7aa 100644
--- a/test/integration/singlecluster/webhook/core/clusterqueue_test.go
+++ b/test/integration/singlecluster/webhook/core/clusterqueue_test.go
@@ -38,7 +38,7 @@ import (
 )
 
 const (
-	resourcesMaxItems = 16
+	resourcesMaxItems = 64
 	flavorsMaxItems   = 64
 )
 
diff --git a/test/integration/singlecluster/webhook/core/cohort_test.go b/test/integration/singlecluster/webhook/core/cohort_test.go
index 89ee16dc2..2c86ca3b7 100644
--- a/test/integration/singlecluster/webhook/core/cohort_test.go
+++ b/test/integration/singlecluster/webhook/core/cohort_test.go
@@ -102,7 +102,7 @@ var _ = ginkgo.Describe("Cohort Webhook", ginkgo.Ordered, func() {
 			ginkgo.Entry("Should reject too many resources in resource group",
 				func() *kueue.Cohort {
 					fq := testing.MakeFlavorQuotas("flavor")
-					for i := range 17 {
+					for i := range 65 {
 						fq = fq.Resource(corev1.ResourceName(fmt.Sprintf("cpu%d", i)))
 					}
 					return testing.MakeCohort("cohort").