Change:
return err } <del> // MigRollingUpdate starts a MIG rolling update, upgrading the nodes to a new // instance template named tmpl, and waits up to nt times the number of nodes // for it to complete. func MigRollingUpdate(tmpl string, nt time.Duration) error { Logf(fmt.Sprintf("starting the MIG rolling update to %s", tmpl)) id, err := migRollingUpdateStart(tmpl, nt) if err != nil { return fmt.Errorf("couldn't start the MIG rolling update: %v", err) } Logf(fmt.Sprintf("polling the MIG rolling update (%s) until it completes", id)) if err := migRollingUpdatePoll(id, nt); err != nil { return fmt.Errorf("err waiting until update completed: %v", err) } return nil } // migRollingUpdateStart (GCE/GKE-only) starts a MIG rolling update using templ // as the new template, waiting up to nt per node, and returns the ID of that // update. func migRollingUpdateStart(templ string, nt time.Duration) (string, error) { var errLast error var id string prefix, suffix := "Started [", "]." if err := wait.Poll(Poll, SingleCallTimeout, func() (bool, error) { // TODO(mikedanese): make this hit the compute API directly instead of //                 shelling out to gcloud. // NOTE(mikedanese): If you are changing this gcloud command, update //                 cluster/gce/upgrade.sh to match this EXACTLY. // A `rolling-updates start` call outputs what we want to stderr. _, output, err := retryCmd("gcloud", "alpha", "compute", "rolling-updates", fmt.Sprintf("--project=%s", TestContext.CloudConfig.ProjectID), fmt.Sprintf("--zone=%s", TestContext.CloudConfig.Zone), "start", // Required args. fmt.Sprintf("--group=%s", TestContext.CloudConfig.NodeInstanceGroup), fmt.Sprintf("--template=%s", templ), // Optional args to fine-tune behavior. fmt.Sprintf("--instance-startup-timeout=%ds", int(nt.Seconds())), // NOTE: We can speed up this process by increasing //       --max-num-concurrent-instances. fmt.Sprintf("--max-num-concurrent-instances=%d", 1), fmt.Sprintf("--max-num-failed-instances=%d", 0), fmt.Sprintf("--min-instance-update-time=%ds", 0)) if err != nil { errLast = fmt.Errorf("rolling-updates call failed with err: %v", err) return false, nil } // The 'start' call probably succeeded; parse the output and try to find // the line that looks like "Started [url/to/<id>]." and return <id>. for _, line := range strings.Split(output, "n") { // As a sanity check, ensure the line starts with prefix and ends // with suffix. if strings.Index(line, prefix) != 0 || strings.Index(line, suffix) != len(line)-len(suffix) { continue } url := strings.Split(strings.TrimSuffix(strings.TrimPrefix(line, prefix), suffix), "/") id = url[len(url)-1] Logf("Started MIG rolling update; ID: %s", id) return true, nil } errLast = fmt.Errorf("couldn't find line like '%s ... %s' in output to MIG rolling-update start. Output: %s", prefix, suffix, output) return false, nil }); err != nil { return "", fmt.Errorf("migRollingUpdateStart() failed with last error: %v", errLast) } return id, nil } // migRollingUpdatePoll (CKE/GKE-only) polls the progress of the MIG rolling // update with ID id until it is complete. It returns an error if this takes // longer than nt times the number of nodes. func migRollingUpdatePoll(id string, nt time.Duration) error { // Two keys and a val. status, progress, done := "status", "statusMessage", "ROLLED_OUT" start, timeout := time.Now(), nt*time.Duration(TestContext.CloudConfig.NumNodes) var errLast error Logf("Waiting up to %v for MIG rolling update to complete.", timeout) if wait.Poll(RestartPoll, timeout, func() (bool, error) { // A `rolling-updates describe` call outputs what we want to stdout. output, _, err := retryCmd("gcloud", "alpha", "compute", "rolling-updates", fmt.Sprintf("--project=%s", TestContext.CloudConfig.ProjectID), fmt.Sprintf("--zone=%s", TestContext.CloudConfig.Zone), "describe", id) if err != nil { errLast = fmt.Errorf("Error calling rolling-updates describe %s: %v", id, err) Logf("%v", errLast) return false, nil } // The 'describe' call probably succeeded; parse the output and try to // find the line that looks like "status: <status>" and see whether it's // done. Logf("Waiting for MIG rolling update: %s (%v elapsed)", ParseKVLines(output, progress), time.Since(start)) if st := ParseKVLines(output, status); st == done { return true, nil } return false, nil }) != nil { return fmt.Errorf("timeout waiting %v for MIG rolling update to complete. Last error: %v", timeout, errLast) } Logf("MIG rolling update complete after %v", time.Since(start)) return nil } </del> func cleanupNodeUpgradeGCE(tmplBefore string) { Logf("Cleaning up any unused node templates") tmplAfter, err := MigTemplate()

Discussion:
This test wants to restart all nodes and make sure that kubernetes survives the process. That's a neat test, but it does the actual restart in a poor way. It does a MIG rolling-update to its own image. If a rolling-update already exists in some sort of error state then all future rolling-updates in that project on an instance group with the same name will fail until someone manually cancels the borked update. We can restart in other less-confusing ways. I suggest one of or . Both seem reasonable to me. cc
ref:
ref/