Change:
echo "== Finished upgrading nodes to ${KUBE_VERSION}. ==" >&2 } <ins> function update-coredns-config() { # Get the current CoreDNS version local -r coredns_addon_path="/etc/kubernetes/addons/0-dns/coredns" local -r tmpdir=/tmp local -r download_dir=$(mktemp --tmpdir=${tmpdir} -d coredns-migration.XXXXXXXXXX) || exit 1 # clean up cleanup() { rm -rf "${download_dir}" } trap cleanup EXIT # Get the new installed CoreDNS version echo "Waiting for CoreDNS to update" until [[ $(${KUBE_ROOT}/cluster/kubectl.sh -n kube-system get deployment coredns -o=jsonpath='{$.metadata.resourceVersion}') -ne ${COREDNS_DEPLOY_RESOURCE_VERSION} ]]; do sleep 1 done echo "Fetching the latest installed CoreDNS version" NEW_COREDNS_VERSION=$(${KUBE_ROOT}/cluster/kubectl.sh -n kube-system get deployment coredns -o=jsonpath='{$.spec.template.spec.containers[:1].image}' | cut -d ":" -f 2) case "$(uname -m)" in x86_64*) host_arch=amd64 corefile_tool_SHA="fd4d8a42d8a1c38cb49b75cca3c7c82677b97c0c6e5ee2a7d5fb02314ccfbb59" ;; i?86_64*) host_arch=amd64 corefile_tool_SHA="fd4d8a42d8a1c38cb49b75cca3c7c82677b97c0c6e5ee2a7d5fb02314ccfbb59" ;; amd64*) host_arch=amd64 corefile_tool_SHA="fd4d8a42d8a1c38cb49b75cca3c7c82677b97c0c6e5ee2a7d5fb02314ccfbb59" ;; aarch64*) host_arch=arm64 corefile_tool_SHA="05503f379eaaa703034c50da7ce7c273d7a7b3569eddb55afe300bd6d6c40988" ;; arm64*) host_arch=arm64 corefile_tool_SHA="05503f379eaaa703034c50da7ce7c273d7a7b3569eddb55afe300bd6d6c40988" ;; arm*) host_arch=arm corefile_tool_SHA="bc826bde6662c11cbb6e6e215397d07d4fedb754c1a6e208271d7d784eb28600" ;; s390x*) host_arch=s390x corefile_tool_SHA="4ed6b7067f65dc8f147a4dd116242495fbec5e6057bb68e1868ef1fb25e07993" ;; ppc64le*) host_arch=ppc64le corefile_tool_SHA="7bce38ed762a2607e158c65b378e6f23e2b80fc4e93dcf50a55f986c7ea2db43" ;; *) echo "Unsupported host arch. Must be x86_64, 386, arm, arm64, s390x or ppc64le." >&2 exit 1 ;; esac # Download the CoreDNS migration tool echo "== Downloading the CoreDNS migration tool ==" wget -P ${download_dir} "https://github.com/coredns/corefile-migration/releases/download/v1.0.2/corefile-tool-${host_arch}" >/dev/null 2>&1 local -r checkSHA=$(sha256sum ${download_dir}/corefile-tool-${host_arch} | cut -d " " -f 1) if [[ "${checkSHA}" != "${corefile_tool_SHA}" ]]; then echo "!!! CheckSum for the CoreDNS migration tool did not match !!!" >&2 exit 1 fi chmod +x ${download_dir}/corefile-tool-${host_arch} # Migrate the CoreDNS ConfigMap depending on whether it is being downgraded or upgraded. ${KUBE_ROOT}/cluster/kubectl.sh -n kube-system get cm coredns -o jsonpath='{.data.Corefile}' > ${download_dir}/Corefile-old if test "$(printf '%sn' ${CURRENT_COREDNS_VERSION} ${NEW_COREDNS_VERSION} | sort -V | head -n 1)" != ${NEW_COREDNS_VERSION}; then echo "== Upgrading the CoreDNS ConfigMap ==" ${download_dir}/corefile-tool-${host_arch} migrate --from ${CURRENT_COREDNS_VERSION} --to ${NEW_COREDNS_VERSION} --corefile ${download_dir}/Corefile-old > ${download_dir}/Corefile ${KUBE_ROOT}/cluster/kubectl.sh -n kube-system create configmap coredns --from-file ${download_dir}/Corefile -o yaml --dry-run | ${KUBE_ROOT}/cluster/kubectl.sh apply -f - else # In case of a downgrade, a custom CoreDNS Corefile will be overwritten by a default Corefile. In that case, # the user will need to manually modify the resulting (default) Corefile after the downgrade is complete. echo "== Applying the latest default CoreDNS configuration ==" gcloud compute --project ${PROJECT}  scp --zone ${ZONE} ${MASTER_NAME}:${coredns_addon_path}/coredns.yaml ${download_dir}/coredns-manifest.yaml > /dev/null ${KUBE_ROOT}/cluster/kubectl.sh apply -f ${download_dir}/coredns-manifest.yaml fi echo "== The CoreDNS Config has been updated ==" } echo "Fetching the previously installed CoreDNS version" CURRENT_COREDNS_VERSION=$(${KUBE_ROOT}/cluster/kubectl.sh -n kube-system get deployment coredns -o=jsonpath='{$.spec.template.spec.containers[:1].image}' | cut -d ":" -f 2) COREDNS_DEPLOY_RESOURCE_VERSION=$(${KUBE_ROOT}/cluster/kubectl.sh -n kube-system get deployment coredns -o=jsonpath='{$.metadata.resourceVersion}') </ins> master_upgrade=true node_upgrade=true node_prereqs=false

Discussion:
Which jobs are failing: Multiple jobs on . Which test(s) are failing: on the following jobs: Since when has it been failing: Since 5/17. Testgrid links: See above. Reason for failure: All the failures mentioned above seem to have to have the same root cause, namely failure while waiting for CoreDNS to be ready. Example: /sig testing /priority critical-urgent /kind failing test /milestone v1.15 /cc
Some of this (possible all of it) is due to , which is dependent on being merged.
/sig cluster-lifecycle
Not the gce jobs thoght. Only the kinder job. So the gce jobs are failing with a problem outside of kubeadm/kinder but possibly still related to the corefile plugin change
/assign (At kubecon this week)
The gce failures appear to show the same behavior, but with kube-up instead of kubeadm. I.e. it appears that kube-up may also be retaining old configs when upgrading/downgrading. If thats true, then we may need to integrate the migration tools into kube-up as well.
I have opened which aims to fix all the failing tests except , which will be fixed via
Great job so far and Would it be possible to ask for the open PRs to be merged and/or a solution found soon? We have these tests failing for quite some time and code freeze coming up in the next days. Thank you in advance for your efforts!
thanks for the heads up. let me know if you don't have the time for the refactor that i requested and i can try to take over there. the alternative is to rollback the coredns version which i assume is not ideal, due to a variety of fixes in the new version.
is there any more work planned for this issue? I see that some of the kinder jobs are still failing after merged:
merging and will resolve and close this issue.
Thank you for the update !
updates: kinder tests were fixed via
/unassign i was here for: it's a bit flaky (which is due to the tooling), but green.
Since and we're moved to 1.16, and we're waiting for to merge, can we move this issue to 1.16?
? is that the right PR?

Good catch, I meant
The GCE test failures were fixed by and /close
Closing this issue. <detailsIn response to : The GCE test failures were fixed by and /close Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details