Repository: kubernetes/kops (go)

Issue:
Unable to pull image from private ECR from master node
/kind bug

**1. What `kops` version are you running? The command `kops version`, will display
 this information.**
Version 1.23.0 (git-a067cd7742a497a5c512762b9880664d865289f1)

root@xxxxxxx:~/.aws# docker-credential-ecr-login version
0.6.0


**2. What Kubernetes version are you running? `kubectl version` will print the
 version if a cluster is running or provide the Kubernetes version specified as
 a `kops` flag.**
1.21
**3. What cloud provider are you using?**
aws
**4. What commands did you run?  What is the simplest way to reproduce this issue?**
docker pull xxxxxx.dkr.ecr.xxx.amazonaws.com/xxx:1.1
**5. What happened after the commands executed?**
```
cat /root/.ecr/log/ecr-login.log
```

time="2022-04-15T20:43:19Z" level=debug msg="Calling ECR.GetAuthorizationToken" registry=xxxxxx
time="2022-04-15T20:43:19Z" level=error msg="Error retrieving credentials" error="ecr: Failed to get authorization token: EC2RoleRequestError: no EC2 instance role found\ncaused by: EC2MetadataError: failed to make EC2Metadata request\ncaused by: "
time="2022-04-15T20:47:17Z" level=debug msg="Could not fetch credentials for cache prefix, disabling cache" error="EC2RoleRequestError: no EC2 instance role found\ncaused by: EC2MetadataError: failed to make EC2Metadata request\ncaused by: "
time="2022-04-15T20:47:17Z" level=debug msg="Retrieving credentials" region=xxxxx registry=xxxxxx serverURL=xxxxxx.dkr.ecr.xxxxxx.amazonaws.com.cn

**6. What did you expect to happen?**
docker pull from private ECR inside master node should be working well
**7. Please provide your cluster manifest. Execute
  `kops get --name my.example.com -o yaml` to display your cluster manifest.
  You may want to remove your cluster name and other sensitive information.**

```yaml
apiVersion: kops.k8s.io/v1alpha2
kind: Cluster
metadata:
  creationTimestamp: "2022-04-11T20:26:16Z"
  generation: 3
  name: xxxxx.k8s.local
spec:
  api:
    loadBalancer:
      class: Network
      type: Public
  assets:
    containerRegistry: xxxxxxx.dkr.ecr.xxxxxxx.amazonaws.com.cn
    fileRepository: https://s3.xxxx.amazonaws.com.cn/xxxxxxx-file-repository
  authorization:
    rbac: {}
  channel: stable
  cloudProvider: aws
  configBase: s3://xxxxxxxxx-state-store/xxxxxxxx.k8s.local
  etcdClusters:
  - cpuRequest: 200m
    etcdMembers:
    - encryptedVolume: true
      instanceGroup: master-xxxxxxxa
      name: a
    - encryptedVolume: true
      instanceGroup: master-xxxxxxxb
      name: b
    - encryptedVolume: true
      instanceGroup: master-xxxxxxxd
      name: d
    memoryRequest: 100Mi
    name: main
  - cpuRequest: 100m
    etcdMembers:
    - encryptedVolume: true
      instanceGroup: master-xxxxxxxa
      name: a
    - encryptedVolume: true
      instanceGroup: master-xxxxxxxb
      name: b
    - encryptedVolume: true
      instanceGroup: master-xxxxxxxd
      name: d
    memoryRequest: 100Mi
    name: events
  iam:
    allowContainerRegistry: true
    legacy: false
  kubelet:
    anonymousAuth: false
  kubernetesApiAccess:
  - 0.0.0.0/0
  - ::/0
  kubernetesVersion: 1.23.5
  masterInternalName: api.internal.xxxxxxx.k8s.local
  masterPublicName: api.xxxxxxx.k8s.local
  networkCIDR: 10.100.0.0/16
  networking:
    calico: {}
  nonMasqueradeCIDR: 100.64.0.0/10
  sshAccess:
  - 0.0.0.0/0
  - ::/0
  subnets:
  - cidr: 10.100.32.0/19
    name: xxxxxxxa
    type: Private
    zone: xxxxxxxa
  - cidr: 10.100.64.0/19
    name: xxxxxxxb
    type: Private
    zone: xxxxxxxb
  - cidr: 10.100.96.0/19
    name: xxxxxxxd
    type: Private
    zone: xxxxxxxd
  - cidr: 10.100.0.0/22
    name: utility-xxxxxxxa
    type: Utility
    zone: xxxxxxxa
  - cidr: 10.100.4.0/22
    name: utility-xxxxxxxb
    type: Utility
    zone: xxxxxxxb
  - cidr: 10.100.8.0/22
    name: utility-xxxxxxxd
    type: Utility
    zone: xxxxxxxd
  topology:
    dns:
      type: Public
    masters: private
    nodes: private

---

apiVersion: kops.k8s.io/v1alpha2
kind: InstanceGroup
metadata:
  creationTimestamp: "2022-04-11T20:26:18Z"
  labels:
    kops.k8s.io/cluster: xxxxxxx.k8s.local
  name: bastions
spec:
  image: ami-xxxxxxx
  instanceMetadata:
    httpPutResponseHopLimit: 1
    httpTokens: required
  machineType: t3.micro
  maxSize: 1
  minSize: 1
  nodeLabels:
    kops.k8s.io/instancegroup: bastions
  role: Bastion
  subnets:
  - xxxxxxxa
  - xxxxxxxb
  - xxxxxxxd

---

apiVersion: kops.k8s.io/v1alpha2
kind: InstanceGroup
metadata:
  creationTimestamp: "2022-04-11T20:26:16Z"
  labels:
    kops.k8s.io/cluster: xxxxxxx.k8s.local
  name: master-xxxxxxxa
spec:
  image: ami-xxxxxxx
  instanceMetadata:
    httpPutResponseHopLimit: 3
    httpTokens: required
  machineType: t3.small
  maxSize: 1
  minSize: 1
  nodeLabels:
    kops.k8s.io/instancegroup: master-xxxxxxxa
  role: Master
  subnets:
  - xxxxxxxa

---

apiVersion: kops.k8s.io/v1alpha2
kind: InstanceGroup
metadata:
  creationTimestamp: "2022-04-11T20:26:17Z"
  labels:
    kops.k8s.io/cluster: xxxxxxx.k8s.local
  name: master-xxxxxxxb
spec:
  image: ami-xxxxxxx
  instanceMetadata:
    httpPutResponseHopLimit: 3
    httpTokens: required
  machineType: t3.small
  maxSize: 1
  minSize: 1
  nodeLabels:
    kops.k8s.io/instancegroup: master-xxxxxxxb
  role: Master
  subnets:
  - xxxxxxxb

---

apiVersion: kops.k8s.io/v1alpha2
kind: InstanceGroup
metadata:
  creationTimestamp: "2022-04-11T20:26:17Z"
  labels:
    kops.k8s.io/cluster: xxxxxxx.k8s.local
  name: master-xxxxxxxd
spec:
  image: ami-xxxxxxx
  instanceMetadata:
    httpPutResponseHopLimit: 3
    httpTokens: required
  machineType: t3.small
  maxSize: 1
  minSize: 1
  nodeLabels:
    kops.k8s.io/instancegroup: master-xxxxxxxd
  role: Master
  subnets:
  - xxxxxxxd

---

apiVersion: kops.k8s.io/v1alpha2
kind: InstanceGroup
metadata:
  creationTimestamp: "2022-04-11T20:26:17Z"
  labels:
    kops.k8s.io/cluster: xxxxxxx.k8s.local
  name: nodes-xxxxxxxa
spec:
  image: ami-xxxxxxx
  instanceMetadata:
    httpPutResponseHopLimit: 1
    httpTokens: required
  machineType: c5d.large
  maxSize: 1
  minSize: 1
  nodeLabels:
    kops.k8s.io/instancegroup: nodes-xxxxxxxa
  role: Node
  subnets:
  - xxxxxxxa

---

apiVersion: kops.k8s.io/v1alpha2
kind: InstanceGroup
metadata:
  creationTimestamp: "2022-04-11T20:26:17Z"
  labels:
    kops.k8s.io/cluster: xxxxxxx.k8s.local
  name: nodes-xxxxxxxb
spec:
  image: ami-xxxxxxx
  instanceMetadata:
    httpPutResponseHopLimit: 1
    httpTokens: required
  machineType: c5d.large
  maxSize: 1
  minSize: 1
  nodeLabels:
    kops.k8s.io/instancegroup: nodes-xxxxxxxb
  role: Node
  subnets:
  - xxxxxxxb

---

apiVersion: kops.k8s.io/v1alpha2
kind: InstanceGroup
metadata:
  creationTimestamp: "2022-04-11T20:26:18Z"
  labels:
    kops.k8s.io/cluster: xxxxxxx.k8s.local
  name: nodes-xxxxxxxd
spec:
  image: ami-xxxxxxx
  instanceMetadata:
    httpPutResponseHopLimit: 1
    httpTokens: required
  machineType: c5d.large
  maxSize: 0
  minSize: 0
  nodeLabels:
    kops.k8s.io/instancegroup: nodes-xxxxxxxd
  role: Node
  subnets:
  - xxxxxxxd
```

**8. Please run the commands with most verbose logging by adding the `-v 10` flag.
  Paste the logs into this report, or in a gist and provide the gist link here.**

**9. Anything else do we need to know?**
```
root@ip-xxxxxxxx:~/.aws# TOKEN=`curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600"` && curl -H "X-aws-ec2-metadata-token: $TOKEN" -v http://169.254.169.254/latest/meta-data/iam/security-credentials/xxxxxxxx
```
we can see the credentials properly:
{
  "Code" : "Success",
  "LastUpdated" : "2022-04-15T20:47:16Z",
  "Type" : "AWS-HMAC",
  "AccessKeyId" : "xxxxxxxxxxxx",
  "SecretAccessKey" : "xxxxxxxxxxxxxxxxxx",
  "Token" : "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
  "Expiration" : "2022-04-16T02:52:56Z"
}



Pull request:
Fix Amazon ECR endpoint in China
We were trying to set up a Kubernetes cluster in AWS China and kept getting Unauthorized errors when pulling images. After some digging, we found that the current hard-coded Amazon ECR endpoint in China is not correct (see the [offical docs](https://docs.aws.amazon.com/general/latest/gr/ecr.html)).

With this fix, we can now pull images from our private ECR normally in China. This should also fix #13494

Patch:
diff --git a/k8s/crds/kops.k8s.io_clusters.yaml b/k8s/crds/kops.k8s.io_clusters.yaml
index 7dc2cca116..260176e010 100644
--- a/k8s/crds/kops.k8s.io_clusters.yaml
+++ b/k8s/crds/kops.k8s.io_clusters.yaml
@@ -3224,11 +3224,6 @@ spec:
                     description: NodeLocalDNS specifies the configuration for the
                       node-local-dns addon
                     properties:
-                      additionalConfig:
-                        description: AdditionalConfig is used to provide additional
-                          config for node local dns by the user - it will include
-                          the original CoreFile made by kOps.
-                        type: string
                       cpuRequest:
                         anyOf:
                         - type: integer
diff --git a/nodeup/pkg/model/kubelet.go b/nodeup/pkg/model/kubelet.go
index abfe0c280a..cd95af9fff 100644
--- a/nodeup/pkg/model/kubelet.go
+++ b/nodeup/pkg/model/kubelet.go
@@ -463,7 +463,7 @@ providers:
   - name: ecr-credential-provider
     matchImages:
       - "*.dkr.ecr.*.amazonaws.com"
-      - "*.dkr.ecr.*.amazonaws.cn"
+      - "*.dkr.ecr.*.amazonaws.com.cn"
       - "*.dkr.ecr-fips.*.amazonaws.com"
       - "*.dkr.ecr.us-iso-east-1.c2s.ic.gov"
       - "*.dkr.ecr.us-isob-east-1.sc2s.sgov.gov"
diff --git a/pkg/apis/kops/cluster.go b/pkg/apis/kops/cluster.go
index 450d93457e..70af056f80 100644
--- a/pkg/apis/kops/cluster.go
+++ b/pkg/apis/kops/cluster.go
@@ -585,8 +585,6 @@ type NodeLocalDNSConfig struct {
 	Enabled *bool `json:"enabled,omitempty"`
 	// ExternalCoreFile is used to provide a complete NodeLocalDNS CoreFile by the user - ignores other provided flags which modify the CoreFile.
 	ExternalCoreFile string `json:"externalCoreFile,omitempty"`
-	// AdditionalConfig is used to provide additional config for node local dns by the user - it will include the original CoreFile made by kOps.
-	AdditionalConfig string `json:"additionalConfig,omitempty"`
 	// Image overrides the default docker image used for node-local-dns addon.
 	Image *string `json:"image,omitempty"`
 	// Local listen IP address. It can be any IP in the 169.254.20.0/16 space or any other IP address that can be guaranteed to not collide with any existing IP.
diff --git a/pkg/apis/kops/v1alpha2/cluster.go b/pkg/apis/kops/v1alpha2/cluster.go
index ac37cbf89e..3d4d5c6470 100644
--- a/pkg/apis/kops/v1alpha2/cluster.go
+++ b/pkg/apis/kops/v1alpha2/cluster.go
@@ -572,8 +572,6 @@ type NodeLocalDNSConfig struct {
 	Enabled *bool `json:"enabled,omitempty"`
 	// ExternalCoreFile is used to provide a complete NodeLocalDNS CoreFile by the user - ignores other provided flags which modify the CoreFile.
 	ExternalCoreFile string `json:"externalCoreFile,omitempty"`
-	// AdditionalConfig is used to provide additional config for node local dns by the user - it will include the original CoreFile made by kOps.
-	AdditionalConfig string `json:"additionalConfig,omitempty"`
 	// Image overrides the default docker image used for node-local-dns addon.
 	Image *string `json:"image,omitempty"`
 	// Local listen IP address. It can be any IP in the 169.254.20.0/16 space or any other IP address that can be guaranteed to not collide with any existing IP.
diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go
index 16c26c283e..b8fbaa7bd0 100644
--- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go
+++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go
@@ -6277,7 +6277,6 @@ func Convert_kops_NodeAuthorizerSpec_To_v1alpha2_NodeAuthorizerSpec(in *kops.Nod
 func autoConvert_v1alpha2_NodeLocalDNSConfig_To_kops_NodeLocalDNSConfig(in *NodeLocalDNSConfig, out *kops.NodeLocalDNSConfig, s conversion.Scope) error {
 	out.Enabled = in.Enabled
 	out.ExternalCoreFile = in.ExternalCoreFile
-	out.AdditionalConfig = in.AdditionalConfig
 	out.Image = in.Image
 	out.LocalIP = in.LocalIP
 	out.ForwardToKubeDNS = in.ForwardToKubeDNS
@@ -6295,7 +6294,6 @@ func Convert_v1alpha2_NodeLocalDNSConfig_To_kops_NodeLocalDNSConfig(in *NodeLoca
 func autoConvert_kops_NodeLocalDNSConfig_To_v1alpha2_NodeLocalDNSConfig(in *kops.NodeLocalDNSConfig, out *NodeLocalDNSConfig, s conversion.Scope) error {
 	out.Enabled = in.Enabled
 	out.ExternalCoreFile = in.ExternalCoreFile
-	out.AdditionalConfig = in.AdditionalConfig
 	out.Image = in.Image
 	out.LocalIP = in.LocalIP
 	out.ForwardToKubeDNS = in.ForwardToKubeDNS
diff --git a/pkg/apis/kops/v1alpha3/cluster.go b/pkg/apis/kops/v1alpha3/cluster.go
index 3de7374bee..d9e029fdd9 100644
--- a/pkg/apis/kops/v1alpha3/cluster.go
+++ b/pkg/apis/kops/v1alpha3/cluster.go
@@ -554,8 +554,6 @@ type NodeLocalDNSConfig struct {
 	Enabled *bool `json:"enabled,omitempty"`
 	// ExternalCoreFile is used to provide a complete NodeLocalDNS CoreFile by the user - ignores other provided flags which modify the CoreFile.
 	ExternalCoreFile string `json:"externalCoreFile,omitempty"`
-	// AdditionalConfig is used to provide additional config for node local dns by the user - it will include the original CoreFile made by kOps.
-	AdditionalConfig string `json:"additionalConfig,omitempty"`
 	// Image overrides the default docker image used for node-local-dns addon.
 	Image *string `json:"image,omitempty"`
 	// Local listen IP address. It can be any IP in the 169.254.20.0/16 space or any other IP address that can be guaranteed to not collide with any existing IP.
diff --git a/pkg/apis/kops/v1alpha3/zz_generated.conversion.go b/pkg/apis/kops/v1alpha3/zz_generated.conversion.go
index eae3dd680f..abfcefc706 100644
--- a/pkg/apis/kops/v1alpha3/zz_generated.conversion.go
+++ b/pkg/apis/kops/v1alpha3/zz_generated.conversion.go
@@ -6496,7 +6496,6 @@ func Convert_kops_NetworkingSpec_To_v1alpha3_NetworkingSpec(in *kops.NetworkingS
 func autoConvert_v1alpha3_NodeLocalDNSConfig_To_kops_NodeLocalDNSConfig(in *NodeLocalDNSConfig, out *kops.NodeLocalDNSConfig, s conversion.Scope) error {
 	out.Enabled = in.Enabled
 	out.ExternalCoreFile = in.ExternalCoreFile
-	out.AdditionalConfig = in.AdditionalConfig
 	out.Image = in.Image
 	out.LocalIP = in.LocalIP
 	out.ForwardToKubeDNS = in.ForwardToKubeDNS
@@ -6514,7 +6513,6 @@ func Convert_v1alpha3_NodeLocalDNSConfig_To_kops_NodeLocalDNSConfig(in *NodeLoca
 func autoConvert_kops_NodeLocalDNSConfig_To_v1alpha3_NodeLocalDNSConfig(in *kops.NodeLocalDNSConfig, out *NodeLocalDNSConfig, s conversion.Scope) error {
 	out.Enabled = in.Enabled
 	out.ExternalCoreFile = in.ExternalCoreFile
-	out.AdditionalConfig = in.AdditionalConfig
 	out.Image = in.Image
 	out.LocalIP = in.LocalIP
 	out.ForwardToKubeDNS = in.ForwardToKubeDNS
diff --git a/upup/models/cloudup/resources/addons/nodelocaldns.addons.k8s.io/k8s-1.12.yaml.template b/upup/models/cloudup/resources/addons/nodelocaldns.addons.k8s.io/k8s-1.12.yaml.template
index a53d1cbf48..e84d2fac69 100644
--- a/upup/models/cloudup/resources/addons/nodelocaldns.addons.k8s.io/k8s-1.12.yaml.template
+++ b/upup/models/cloudup/resources/addons/nodelocaldns.addons.k8s.io/k8s-1.12.yaml.template
@@ -107,7 +107,6 @@ data:
         {{- end }}
     }
     {{- end }}
-{{ KubeDNS.NodeLocalDNS.AdditionalConfig | indent 4 }}
   {{- end }}
 ---
 apiVersion: apps/v1


Test patch:
diff --git a/tests/integration/update_cluster/many-addons-ccm-irsa26/data/aws_s3_object_cluster-completed.spec_content b/tests/integration/update_cluster/many-addons-ccm-irsa26/data/aws_s3_object_cluster-completed.spec_content
index 4a66467730..e3b33489f6 100644
--- a/tests/integration/update_cluster/many-addons-ccm-irsa26/data/aws_s3_object_cluster-completed.spec_content
+++ b/tests/integration/update_cluster/many-addons-ccm-irsa26/data/aws_s3_object_cluster-completed.spec_content
@@ -155,11 +155,6 @@ spec:
     memoryLimit: 170Mi
     memoryRequest: 70Mi
     nodeLocalDNS:
-      additionalConfig: |
-        googleapis.com:53 {
-            bind 169.254.20.10
-            forward . 10.238.1.130
-        }
       cpuRequest: 25m
       enabled: true
       forwardToKubeDNS: false
diff --git a/tests/integration/update_cluster/many-addons-ccm-irsa26/data/aws_s3_object_minimal.example.com-addons-bootstrap_content b/tests/integration/update_cluster/many-addons-ccm-irsa26/data/aws_s3_object_minimal.example.com-addons-bootstrap_content
index 55e437a775..85461cc6cb 100644
--- a/tests/integration/update_cluster/many-addons-ccm-irsa26/data/aws_s3_object_minimal.example.com-addons-bootstrap_content
+++ b/tests/integration/update_cluster/many-addons-ccm-irsa26/data/aws_s3_object_minimal.example.com-addons-bootstrap_content
@@ -41,7 +41,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.12
     manifest: nodelocaldns.addons.k8s.io/k8s-1.12.yaml
-    manifestHash: 0d1c36564e406cde717c45ea59f8a89f2990a6f1c105d97295d0e1612e570114
+    manifestHash: fabeb460a56eda2aa329fe1f932a98f60de82f54bb4baedcdf9d5b2393557d73
     name: nodelocaldns.addons.k8s.io
     needsRollingUpdate: all
     selector:
diff --git a/tests/integration/update_cluster/many-addons-ccm-irsa26/data/aws_s3_object_minimal.example.com-addons-nodelocaldns.addons.k8s.io-k8s-1.12_content b/tests/integration/update_cluster/many-addons-ccm-irsa26/data/aws_s3_object_minimal.example.com-addons-nodelocaldns.addons.k8s.io-k8s-1.12_content
index f96fe93cf2..4a5b842fc9 100644
--- a/tests/integration/update_cluster/many-addons-ccm-irsa26/data/aws_s3_object_minimal.example.com-addons-nodelocaldns.addons.k8s.io-k8s-1.12_content
+++ b/tests/integration/update_cluster/many-addons-ccm-irsa26/data/aws_s3_object_minimal.example.com-addons-nodelocaldns.addons.k8s.io-k8s-1.12_content
@@ -44,7 +44,7 @@ spec:
 
 apiVersion: v1
 data:
-  Corefile: |
+  Corefile: |-
     cluster.local:53 {
         errors
         cache {
@@ -91,10 +91,6 @@ data:
         forward . __PILLAR__UPSTREAM__SERVERS__
         prometheus :9253
     }
-    googleapis.com:53 {
-        bind 169.254.20.10
-        forward . 10.238.1.130
-    }
 kind: ConfigMap
 metadata:
   creationTimestamp: null
diff --git a/tests/integration/update_cluster/many-addons-ccm-irsa26/in-v1alpha2.yaml b/tests/integration/update_cluster/many-addons-ccm-irsa26/in-v1alpha2.yaml
index 0f9382ed86..618ead34aa 100644
--- a/tests/integration/update_cluster/many-addons-ccm-irsa26/in-v1alpha2.yaml
+++ b/tests/integration/update_cluster/many-addons-ccm-irsa26/in-v1alpha2.yaml
@@ -38,11 +38,6 @@ spec:
   kubeDNS:
     nodeLocalDNS:
       enabled: true
-      additionalConfig: |
-        googleapis.com:53 {
-            bind 169.254.20.10
-            forward . 10.238.1.130
-        }
   kubernetesVersion: v1.26.0-alpha.0
   masterPublicName: api.minimal.example.com
   networkCIDR: 172.20.0.0/16