jiebi/CodeConvo huggingface dataset:
</text> </g> <g> <del> <rect style="fill: #ffffff" x="924.748" y="303" width="57.1" height="38"/> <rect style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x="924.748" y="303" width="57.1" height="38"/> <text font-size="12.8" style="fill: #000000;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="953.298" y="325.9"> <tspan x="953.298" y="325.9">Proxy</tspan> </del> <ins> <rect style="fill: #ffffff" x="949.748" y="228" width="57.1" height="38"/> <rect style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x="949.748" y="228" width="57.1" height="38"/> <text font-size="12.8" style="fill: #000000;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="978.298" y="250.9"> <tspan x="978.298" y="250.9">Proxy</tspan> </ins> </text> </g> <g>
<rect style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x="-30" y="416.75" width="364" height="146"/> </g> <g> <del> <rect style="fill: #ffffff" x="418.774" y="551" width="176.225" height="121"/> <rect style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x="418.774" y="551" width="176.225" height="121"/> <text font-size="12.8" style="fill: #000000;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="506.886" y="583.4"> <tspan x="506.886" y="583.4">Distributed</tspan> <tspan x="506.886" y="599.4">Watchable</tspan> <tspan x="506.886" y="615.4">Storage</tspan> <tspan x="506.886" y="631.4"></tspan> <tspan x="506.886" y="647.4">(implemented via etcd)</tspan> </text> </g> <g> </del> <rect style="fill: #ffffff" x="201.314" y="594.318" width="154.6" height="38"/> <rect style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x="201.314" y="594.318" width="154.6" height="38"/> <text font-size="12.8" style="fill: #000000;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="278.614" y="617.218">
<polygon style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="222.862,466.265 218.126,456.137 223.059,458.767 228.122,456.399 "/> </g> <g> <del> <line style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x1="308.892" y1="503.5" x2="362.289" y2="504.346"/> <polygon style="fill: #000000" points="369.788,504.465 359.71,509.306 362.289,504.346 359.869,499.307 "/> <polygon style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="369.788,504.465 359.71,509.306 362.289,504.346 359.869,499.307 "/> </g> <g> </del> <line style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x1="318.054" y1="544.587" x2="410.664" y2="606.112"/> <polygon style="fill: #000000" points="320.821,540.422 309.725,539.053 315.288,548.752 "/> <polygon style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="320.821,540.422 309.725,539.053 315.288,548.752 "/>
var lastEndpoints []api.Endpoints sleep := 5 * time.Second <ins> // Used to avoid spamming the error log file, makes error logging edge triggered. hadSuccess := true </ins> for { data, err := ioutil.ReadFile(s.filename) if err != nil { <del> glog.Errorf("Couldn't read file: %s : %v", s.filename, err) </del> <ins> msg := fmt.Sprintf("Couldn't read file: %s : %v", s.filename, err) if hadSuccess { glog.Error(msg) } else { glog.V(1).Info(msg) } hadSuccess = false </ins> time.Sleep(sleep) continue } <ins> hadSuccess = true </ins> if bytes.Equal(lastData, data) { time.Sleep(sleep)
glog.Infof("Version test passed") } <ins> func runSelfLinkTest(c *client.Client) { var svc api.Service err := c.Post().Path("services").Body( &api.Service{ ObjectMeta: api.ObjectMeta{ Name: "selflinktest", Labels: map[string]string{ "name": "selflinktest", }, }, Port: 12345, // This is here because validation requires it. Selector: map[string]string{ "foo": "bar", }, }, ).Do().Into(&svc) if err != nil { glog.Fatalf("Failed creating selflinktest service: %v", err) } err = c.Get().AbsPath(svc.SelfLink).Do().Into(&svc) if err != nil { glog.Fatalf("Failed listing service with supplied self link '%v': %v", svc.SelfLink, err) } var svcList api.ServiceList err = c.Get().Path("services").Do().Into(&svcList) if err != nil { glog.Fatalf("Failed listing services: %v", err) } err = c.Get().AbsPath(svcList.SelfLink).Do().Into(&svcList) if err != nil { glog.Fatalf("Failed listing services with supplied self link '%v': %v", svcList.SelfLink, err) } found := false for i := range svcList.Items { item := &svcList.Items[i] if item.Name != "selflinktest" { continue } found = true err = c.Get().AbsPath(item.SelfLink).Do().Into(&svc) if err != nil { glog.Fatalf("Failed listing service with supplied self link '%v': %v", item.SelfLink, err) } break } if !found { glog.Fatalf("never found selflinktest service") } glog.Infof("Self link test passed") // TODO: Should test PUT at some point, too. } </ins> func runAtomicPutTest(c *client.Client) { var svc api.Service err := c.Post().Path("services").Body(
newURL.Path = path.Join(h.canonicalPrefix, req.URL.Path) newURL.RawQuery = "" newURL.Fragment = "" <del> return h.selfLinker.SetSelfLink(obj, newURL.String()) </del> <ins> err := h.selfLinker.SetSelfLink(obj, newURL.String()) if err != nil { return err } if !runtime.IsListType(obj) { return nil } // Set self-link of objects in the list. items, err := runtime.ExtractList(obj) if err != nil { return err } for i := range items { if err := h.setSelfLinkAddName(items[i], req); err != nil { return err } } return runtime.SetList(obj, items) </ins> } // Like setSelfLink, but appends the object's name.
"github.com/google/gofuzz" ) <ins> func TestIsList(t *testing.T) { tests := []struct { obj    runtime.Object isList bool }{ {&api.PodList{}, true}, {&api.Pod{}, false}, } for _, item := range tests { if e, a := item.isList, runtime.IsListType(item.obj); e != a { t.Errorf("%v: Expected %v, got %v", reflect.TypeOf(item.obj), e, a) } } } </ins> func TestExtractList(t *testing.T) { pl := &api.PodList{ Items: []api.Pod{
func TestGetEtcd(t *testing.T) { fakeClient := tools.MakeFakeEtcdClient(t) ch := make(chan interface{}, 1) <ins> manifest := api.ContainerManifest{ID: "foo", Version: "v1beta1", Containers: []api.Container{{Name: "1", Image: "foo"}}} </ins> fakeClient.Data["/registry/hosts/machine/kubelet"] = tools.EtcdResponseWithError{ R: &etcd.Response{ Node: &etcd.Node{ Value: api.EncodeOrDie(&api.ContainerManifestList{ <del> Items: []api.ContainerManifest{{ID: "foo"}}, </del> <ins> Items: []api.ContainerManifest{manifest}, </ins> }), ModifiedIndex: 1, },
t.Errorf("Expected %#v, Got %#v", 2, lastIndex) } update := (<-ch).(kubelet.PodUpdate) <del> expected := CreatePodUpdate(kubelet.SET, kubelet.Pod{Name: "foo", Manifest: api.ContainerManifest{ID: "foo"}}) </del> <ins> expected := CreatePodUpdate(kubelet.SET, kubelet.Pod{Name: "foo", Manifest: manifest}) </ins> if !reflect.DeepEqual(expected, update) { t.Errorf("Expected %#v, Got %#v", expected, update) } <ins> for i := range update.Pods { if errs := kubelet.ValidatePod(&update.Pods[i]); len(errs) != 0 { t.Errorf("Expected no validation errors on %#v, Got %#v", update.Pods[i], errs) } } </ins> } func TestWatchEtcd(t *testing.T) {
return pod, nil } <del> var simpleSubdomainSafeEncoding = base64.NewEncoding("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ012345678900") </del> <ins> var simpleSubdomainSafeEncoding = base32.NewEncoding("0123456789abcdefghijklmnopqrstuv") var unsafeDNSLabelReplacement = regexp.MustCompile("[^a-z0-9]+") </ins> <del> // simpleSubdomainSafeHash generates a compact hash of the input that uses characters // only in the range a-zA-Z0-9, making it suitable for DNS subdomain labels func simpleSubdomainSafeHash(s string) string { </del> <ins> // simpleSubdomainSafeHash generates a pod name for the given path that is // suitable as a subdomain label. func simpleSubdomainSafeHash(path string) string { name := strings.ToLower(filepath.Base(path)) name = unsafeDNSLabelReplacement.ReplaceAllString(name, "") </ins> hasher := sha1.New() <del> hasher.Write([]byte(s)) </del> <ins> hasher.Write([]byte(path)) </ins> sha := simpleSubdomainSafeEncoding.EncodeToString(hasher.Sum(nil)) <del> if len(sha) > 20 { sha = sha[:20] } return sha </del> <ins> return fmt.Sprintf("%.15s%.30s", name, sha) </ins> }
func TestExtractFromDir(t *testing.T) { manifests := []api.ContainerManifest{ <del> {ID: "", Containers: []api.Container{{Image: "foo"}}}, {ID: "", Containers: []api.Container{{Image: "bar"}}}, </del> <ins> {Version: "v1beta1", Containers: []api.Container{{Name: "1", Image: "foo"}}}, {Version: "v1beta1", Containers: []api.Container{{Name: "2", Image: "bar"}}}, </ins> } files := make([]*os.File, len(manifests))
if !reflect.DeepEqual(expected, update) { t.Errorf("Expected %#v, Got %#v", expected, update) } <ins> for i := range update.Pods { if errs := kubelet.ValidatePod(&update.Pods[i]); len(errs) != 0 { t.Errorf("Expected no validation errors on %#v, Got %#v", update.Pods[i], errs) } } } func TestSubdomainSafeName(t *testing.T) { type Case struct { Input    string Expected string } testCases := []Case{ {"/some/path/invalidUPPERCASE", "invaliduppercasa6hlenc0vpqbbdtt26ghneqsq3pvud"}, {"/some/path/_-!%$#&@^&*(){}", "nvhc03p016m60huaiv3avts372rl2p"}, } for _, testCase := range testCases { value := simpleSubdomainSafeHash(testCase.Input) if value != testCase.Expected { t.Errorf("Expected %s, Got %s", testCase.Expected, value) } value2 := simpleSubdomainSafeHash(testCase.Input) if value != value2 { t.Errorf("Value for %s was not stable across runs: %s %s", testCase.Input, value, value2) } } </ins> } // These are used for testing extract json (below)
func TestExtractFromHttpMultiple(t *testing.T) { manifests := []api.ContainerManifest{ <del> {Version: "v1beta1", ID: ""}, {Version: "v1beta1", ID: "bar"}, </del> <ins> {Version: "v1beta1", ID: "", Containers: []api.Container{{Name: "1", Image: "foo"}}}, {Version: "v1beta1", ID: "bar", Containers: []api.Container{{Name: "1", Image: "foo"}}}, </ins> } data, err := json.Marshal(manifests) if err != nil {
if !reflect.DeepEqual(expected, update) { t.Errorf("Expected: %#v, Got: %#v", expected, update) } <ins> for i := range update.Pods { if errs := kubelet.ValidatePod(&update.Pods[i]); len(errs) != 0 { t.Errorf("Expected no validation errors on %#v, Got %#v", update.Pods[i], errs) } } </ins> } func TestExtractFromHttpEmptyArray(t *testing.T) {
* **Kubernetes Web Interface** ([ui.md](ui.md)): Accessing the Kubernetes web user interface. <del> * **Kubecfg Command Line Interface** ([cli.md](cli.md)): The `kubecfg` command line reference. </del> <ins> * **Kubectl Command Line Interface** ([kubectl.md](kubectl.md)): The `kubectl` command line reference. </ins> * **Roadmap** ([roadmap.md](roadmap.md)): The set of supported use cases, features, docs, and patterns that are required before Kubernetes 1.0.
The replication controller is forever constrained to this narrow responsibility. It itself will not perform readiness nor liveness probes. Rather than performing auto-scaling, it is intended to be controlled by an external auto-scaler (as discussed in [#492](https://github.com/GoogleCloudPlatform/kubernetes/issues/492)), which would change its `replicas` field. We will not add scheduling policies (e.g., [spreading](https://github.com/GoogleCloudPlatform/kubernetes/issues/367#issuecomment-48428019)) to replication controller. Nor should it verify that the pods controlled match the currently specified template, as that would obstruct auto-sizing and other automated processes. Similarly, completion deadlines, ordering dependencies, configuration expansion, and other features belong elsehwere. We even plan to factor out the mechanism for bulk pod creation ([#170](https://github.com/GoogleCloudPlatform/kubernetes/issues/170)). <del> The replication controller is intended to be a composable building-block primitive. We expect higher-level APIs and/or tools to be built on top of it and other complementary primitives for user convenience in the future. The "macro" operations currently supported by kubecfg (run, stop, resize, rollingupdate) are proof-of-concept examples of this. For instance, we could imagine something like [Asgard](http://techblog.netflix.com/2012/06/asgard-web-based-cloud-management-and.html) managing replication controllers, auto-scalers, services, scheduling policies, canaries, etc. </del> <ins> The replication controller is intended to be a composable building-block primitive. We expect higher-level APIs and/or tools to be built on top of it and other complementary primitives for user convenience in the future. The "macro" operations currently supported by kubectl (run-container, stop, resize, rollingupdate) are proof-of-concept examples of this. For instance, we could imagine something like [Asgard](http://techblog.netflix.com/2012/06/asgard-web-based-cloud-management-and.html) managing replication controllers, auto-scalers, services, scheduling policies, canaries, etc. </ins> ## Common usage patterns
# install.  See https://github.com/saltstack/salt-bootstrap/issues/270 # # -M installs the master <del> # FIXME: The following line should be replaced with: # curl -L http://bootstrap.saltstack.com | sh -s -- -M # when the merged salt-api service is included in the fedora salt-master rpm # Merge is here: https://github.com/saltstack/salt/pull/13554 # Fedora git repository is here: http://pkgs.fedoraproject.org/cgit/salt.git/ # (a new service file needs to be added for salt-api) curl -sS -L https://raw.githubusercontent.com/saltstack/salt-bootstrap/v2014.06.30/bootstrap-salt.sh | sh -s -- -M </del> <ins> curl -sS -L --connect-timeout 20 --retry 6 --retry-delay 10 https://bootstrap.saltstack.com | sh -s -- -M </ins> fi # Build release
# # We specify -X to avoid a race condition that can cause minion failure to # install.  See https://github.com/saltstack/salt-bootstrap/issues/270 <del> curl -sS -L http://bootstrap.saltstack.com | sh -s -- -X </del> <ins> curl -sS -L --connect-timeout 20 --retry 6 --retry-delay 10 https://bootstrap.saltstack.com | sh -s -- -X </ins> ## TODO this only works on systemd distros, need to find a work-around as removing -X above fails to start the services installed systemctl enable salt-minion
myKubelet := kubelet.NewIntegrationTestKubelet(machineList[0], &fakeDocker1) go util.Forever(func() { myKubelet.Run(cfg1.Updates()) }, 0) go util.Forever(func() { <del> kubelet.ListenAndServeKubeletServer(myKubelet, cfg1.Channel("http"), http.DefaultServeMux, "localhost", 10250) </del> <ins> kubelet.ListenAndServeKubeletServer(myKubelet, cfg1.Channel("http"), "localhost", 10250) </ins> }, 0) // Kubelet (machine)
otherKubelet := kubelet.NewIntegrationTestKubelet(machineList[1], &fakeDocker2) go util.Forever(func() { otherKubelet.Run(cfg2.Updates()) }, 0) go util.Forever(func() { <del> kubelet.ListenAndServeKubeletServer(otherKubelet, cfg2.Channel("http"), http.DefaultServeMux, "localhost", 10251) </del> <ins> kubelet.ListenAndServeKubeletServer(otherKubelet, cfg2.Channel("http"), "localhost", 10251) </ins> }, 0) return apiServer.URL
type Server struct { host    HostInterface updates chan<- interface{} <del> handler http.Handler </del> <ins> mux     *http.ServeMux </ins> } <del> func ListenAndServeKubeletServer(host HostInterface, updates chan<- interface{}, delegate http.Handler, address string, port uint) { </del> <ins> // ListenAndServeKubeletServer initializes a server to respond to HTTP network requests on the Kubelet func ListenAndServeKubeletServer(host HostInterface, updates chan<- interface{}, address string, port uint) { </ins> glog.Infof("Starting to listen on %s:%d", address, port) <del> handler := Server{ host:    host, updates: updates, handler: delegate, } </del> <ins> handler := NewServer(host, updates) </ins> s := &http.Server{ Addr:           net.JoinHostPort(address, strconv.FormatUint(uint64(port), 10)), Handler:        &handler,
} fw.updateReader = startReading(fw.updateChan) fw.fakeKubelet = &fakeKubelet{} <del> fw.serverUnderTest = &Server{ host:    fw.fakeKubelet, updates: fw.updateChan, } </del> <ins> server := NewServer(fw.fakeKubelet, fw.updateChan) fw.serverUnderTest = &server </ins> fw.testHTTPServer = httptest.NewServer(fw.serverUnderTest) return fw }
MASTER_NAME="${INSTANCE_PREFIX}-master" MASTER_TAG="${INSTANCE_PREFIX}-master" MINION_TAG="${INSTANCE_PREFIX}-minion" <del> MINION_NAMES=($(eval echo ${INSTANCE_PREFIX}-minion-{1..${NUM_MINIONS}})) </del> <ins> # Unable to use hostnames yet because DNS is not in cluster, so we revert external look-up name to use the minion IP #MINION_NAMES=($(eval echo ${INSTANCE_PREFIX}-minion-{1..${NUM_MINIONS}})) </ins> # IP LOCATIONS FOR INTERACTING WITH THE MINIONS MINION_IP_BASE="10.245.2." for (( i=0; i <${NUM_MINIONS}; i++)) do KUBE_MINION_IP_ADDRESSES[$i]="${MINION_IP_BASE}$[$i+2]" MINION_IP[$i]="${MINION_IP_BASE}$[$i+2]" <ins> MINION_NAMES[$i]="${MINION_IP[$i]}" </ins> VAGRANT_MINION_NAMES[$i]="minion-$[$i+1]" done
} filteredMinions := v.saltMinionsByRole(minions, "kubernetes-pool") for _, minion := range filteredMinions { <del> fmt.Println("Minion: ", minion.Host, " , ", instance, " IP: ", minion.IP) if minion.Host == instance { </del> <ins> // Due to vagrant not running with a dedicated DNS setup, we return the IP address of a minion as its hostname at this time if minion.IP == instance { </ins> return net.ParseIP(minion.IP), nil } }
t.Fatalf("Incorrect number of instances returned") } <del> if instances[0] != "kubernetes-minion-1" { </del> <ins> // no DNS in vagrant cluster, so we return IP as hostname expectedInstanceHost := "10.245.2.2" expectedInstanceIP := "10.245.2.2" if instances[0] != expectedInstanceHost { </ins> t.Fatalf("Invalid instance returned") }
package main <del> import "syscall" </del> <ins> import ( "os" "os/signal" "syscall" ) </ins> func main() { <del> // Halts execution, waiting on signal. syscall.Pause() </del> <ins> c := make(chan os.Signal, 1) signal.Notify(c, os.Interrupt, os.Kill, syscall.SIGTERM) // Block until a signal is received. <-c </ins> }
} } <del> // And create a configuration source that reads from a local file config.NewConfigSourceFile(*configFile, serviceConfig.Channel("file"), endpointsConfig.Channel("file")) glog.Infof("Using configuration file %s", *configFile) </del> loadBalancer := proxy.NewLoadBalancerRR() proxier := proxy.NewProxier(loadBalancer, net.IP(bindAddress)) // Wire proxier to handle changes to services
* [Example of dynamic updates](examples/update-demo/README.md) * [Cluster monitoring with heapster and cAdvisor](https://github.com/GoogleCloudPlatform/heapster) * [Community projects](https://github.com/GoogleCloudPlatform/kubernetes/wiki/Kubernetes-Community) <del> * [Development guide](docs/development.md) </del> <ins> * [Development guide](docs/devel/development.md) </ins> Or fork and start hacking!
<ins> # On Collaborative Development Kubernetes is open source, but many of the people working on it do so as their day job.  In order to avoid forcing people to be "at work" effectively 24/7, we want to establish some semi-formal protocols around development.  Hopefully these rules make things go more smoothly.  If you find that this is not the case, please complain loudly. ## Patches welcome First and foremost: as a potential contributor, your changes and ideas are welcome at any hour of the day or night, weekdays, weekends, and holidays.  Please do not ever hesitate to ask a question or send a PR. ## Timezones and calendars For the time being, most of the people working on this project are in the US and on Pacific time.  Any times mentioned henceforth will refer to this timezone.  Any references to "work days" will refer to the US calendar. ## Code reviews All changes must be code reviewed.  For non-maintainers this is obvious, since you can't commit anyway.  But even for maintainers, we want all changes to get at least one review, preferably from someone who knows the areas the change touches.  For non-trivial changes we may want two reviewers.  The primary reviewer will make this decision and nominate a second reviewer, if needed.  Except for trivial changes, PRs should sit for at least a 2 hours to allow for wider review. Most PRs will find reviewers organically.  If a maintainer intends to be the primary reviewer of a PR they should set themselves as the assignee on GitHub and say so in a reply to the PR.  Only the primary reviewer of a change should actually do the merge, except in rare cases (e.g. they are unavailable in a reasonable timeframe). If a PR has gone 2 work days without an owner emerging, please poke the PR thread and ask for a reviewer to be assigned. Except for rare cases, such as trivial changes (e.g. typos, comments) or emergencies (e.g. broken builds), maintainers should not merge their own changes. Expect reviewers to request that you avoid [common go style mistakes](https://code.google.com/p/go-wiki/wiki/CodeReviewComments) in your PRs. ## Assigned reviews Maintainers can assign reviews to other maintainers, when appropriate. The assignee becomes the shepherd for that PR and is responsible for merging the PR once they are satisfied with it or else closing it.  The assignee might request reviews from non-maintainers. ## Merge hours Maintainers will do merges between the hours of 7:00 am Monday and 7:00 pm (19:00h) Friday.  PRs that arrive over the weekend or on holidays will only be merged if there is a very good reason for it and if the code review requirements have been met. There may be discussion an even approvals granted outside of the above hours, but merges will generally be deferred. ## Holds Any maintainer or core contributor who wants to review a PR but does not have time immediately may put a hold on a PR simply by saying so on the PR discussion and offering an ETA measured in single-digit days at most.  Any PR that has a hold shall not be merged until the person who requested the hold acks the review, withdraws their hold, or is overruled by a preponderance of maintainers. </ins>
<ins> # Hunting flaky tests in Kubernetes Sometimes unit tests are flaky.  This means that due to (usually) race conditions, they will occasionally fail, even though most of the time they pass. We have a goal of 99.9% flake free tests.  This means that there is only one flake in one thousand runs of a test. Running a test 1000 times on your own machine can be tedious and time consuming.  Fortunately, there is a better way to achieve this using Kubernetes. _Note: these instructions are mildly hacky for now, as we get run once semantics and logging they will get better_ There is a testing image ```brendanburns/flake``` up on the docker hub.  We will use this image to test our fix. Create a replication controller with the following config: ```yaml id: flakeController desiredState: replicas: 24 replicaSelector: name: flake podTemplate: desiredState: manifest: version: v1beta1 id: "" volumes: [] containers: - name: flake image: brendanburns/flake env: - name: TEST_PACKAGE value: pkg/tools - name: REPO_SPEC value: https://github.com/GoogleCloudPlatform/kubernetes restartpolicy: {} labels: name: flake labels: name: flake ``` ```./cluster/kubecfg.sh -c controller.yaml create replicaControllers``` This will spin up 100 instances of the test.  They will run to completion, then exit, the kubelet will restart them, eventually you will have sufficient runs for your purposes, and you can stop the replication controller: ```sh ./cluster/kubecfg.sh stop flakeController ./cluster/kubecfg.sh rm flakeController ``` Now examine the machines with ```docker ps -a``` and look for tasks that exited with non-zero exit codes (ignore those that exited -1, since that's what happens when you stop the replica controller) Happy flake hunting! </ins>