jiebi/CodeConvo huggingface dataset:
# KUBE_RACE="-race" KUBE_GOFLAGS="-tags 'integration no-docker' "  KUBE_RACE=""  <ins> KUBE_TIMEOUT="${KUBE_TIMEOUT}"  </ins> KUBE_TEST_API_VERSIONS="$1"  KUBE_API_VERSIONS="v1,experimental/v1alpha1"  "${KUBE_ROOT}/hack/test-go.sh" test/integration
func rcByNameContainer(name string, replicas int, image string, labels map[string]string, c api.Container) *api.ReplicationController { // Add "name": name to the labels, overwriting if it exists. labels["name"] = name <ins> gracePeriod := int64(0) </ins> return &api.ReplicationController{ TypeMeta: unversioned.TypeMeta{ Kind:       "ReplicationController",
// In case of failure or too long waiting time, an error is returned. func waitForRCPodToDisappear(c *client.Client, ns, rcName, podName string) error { label := labels.SelectorFromSet(labels.Set(map[string]string{"name": rcName})) <del> return waitForPodToDisappear(c, ns, podName, label, 20*time.Second, 5*time.Minute) </del> <ins> // NodeController evicts pod after 5 minutes, so we need timeout greater than that. // Additionally, there can be non-zero grace period, so we are setting 10 minutes // to be on the safe size. return waitForPodToDisappear(c, ns, podName, label, 20*time.Second, 10*time.Minute) </ins> } // waitForService waits until the service appears (exist == true), or disappears (exist == false)
return fmt.Errorf("pod %s is not running and cannot be attached to; current phase is %s", p.PodName, pod.Status.Phase) } <del> containerName := p.ContainerName if len(containerName) == 0 { glog.V(4).Infof("defaulting container name to %s", pod.Spec.Containers[0].Name) containerName = pod.Spec.Containers[0].Name } </del> // TODO: refactor with terminal helpers from the edit utility once that is merged var stdin io.Reader tty := p.TTY
Name(pod.Name). Namespace(pod.Namespace). SubResource("attach"). <del> Param("container", containerName) </del> <ins> Param("container", p.GetContainerName(pod)) </ins> return p.Attach.Attach(req, p.Config, stdin, p.Out, p.Err, tty) } <ins> // GetContainerName returns the name of the container to attach to, with a fallback. func (p *AttachOptions) GetContainerName(pod *api.Pod) string { if len(p.ContainerName) > 0 { return p.ContainerName } glog.V(4).Infof("defaulting container name to %s", pod.Spec.Containers[0].Name) return pod.Spec.Containers[0].Name } </ins>
return err } if status == api.PodSucceeded || status == api.PodFailed { <del> return handleLog(c, pod.Namespace, pod.Name, &api.PodLogOptions{Container: pod.Spec.Containers[0].Name}, opts.Out) </del> <ins> return handleLog(c, pod.Namespace, pod.Name, &api.PodLogOptions{Container: opts.GetContainerName(pod)}, opts.Out) </ins> } opts.Client = c opts.PodName = pod.Name opts.Namespace = pod.Namespace <del> return opts.Run() </del> <ins> if err := opts.Run(); err != nil { fmt.Fprintf(opts.Out, "Error attaching, falling back to logs: %vn", err) return handleLog(c, pod.Namespace, pod.Name, &api.PodLogOptions{Container: opts.GetContainerName(pod)}, opts.Out) } return nil </ins> } func getRestartPolicy(cmd *cobra.Command, interactive bool) (api.RestartPolicy, error) {
if containerChanges.StartInfraContainer && (len(containerChanges.ContainersToStart) > 0) { glog.V(4).Infof("Creating pod infra container for %q", podFullName) podInfraContainerID, err = dm.createPodInfraContainer(pod) <ins> if err != nil { glog.Errorf("Failed to create pod infra container: %v; Skipping pod %q", err, podFullName) return err } </ins> // Call the networking plugin <del> if err == nil { err = dm.networkPlugin.SetUpPod(pod.Namespace, pod.Name, podInfraContainerID) } </del> <ins> err = dm.networkPlugin.SetUpPod(pod.Namespace, pod.Name, podInfraContainerID) </ins> if err != nil { glog.Errorf("Failed to create pod infra container: %v; Skipping pod %q", err, podFullName) <ins> // Delete infra container if delErr := dm.KillContainerInPod(kubecontainer.ContainerID{ ID:   string(podInfraContainerID), Type: "docker"}, nil, pod); delErr != nil { glog.Warningf("Clear infra container failed for pod %q: %v", podFullName, delErr) } </ins> return err }
const pluginName = "kubernetes.io/flocker" <del> func newInitializedVolumePlugMgr() volume.VolumePluginMgr { </del> <ins> func newInitializedVolumePlugMgr(t *testing.T) (volume.VolumePluginMgr, string) { </ins> plugMgr := volume.VolumePluginMgr{} <del> plugMgr.InitPlugins(ProbeVolumePlugins(), volume.NewFakeVolumeHost("/foo/bar", nil, nil)) return plugMgr </del> <ins> dir, err := ioutil.TempDir("", "flocker") assert.NoError(t, err) plugMgr.InitPlugins(ProbeVolumePlugins(), volume.NewFakeVolumeHost(dir, nil, nil)) return plugMgr, dir </ins> } func TestGetByName(t *testing.T) { assert := assert.New(t) <del> plugMgr := newInitializedVolumePlugMgr() </del> <ins> plugMgr, _ := newInitializedVolumePlugMgr(t) </ins> plug, err := plugMgr.FindPluginByName(pluginName) assert.NotNil(plug, "Can't find the plugin by name")
assert := assert.New(t) <del> plugMgr := newInitializedVolumePlugMgr() </del> <ins> plugMgr, rootDir := newInitializedVolumePlugMgr(t) if rootDir != "" { defer os.RemoveAll(rootDir) } </ins> plug, err := plugMgr.FindPluginByName(flockerPluginName) assert.NoError(err)
kube::test::get_object_assert 'rc mock2' "{{${labels_field}.status}}" 'replaced' fi fi <ins> # Command: kubectl edit multiple resources temp_editor="${KUBE_TEMP}/tmp-editor.sh" echo -e '#!/bin/bashnsed -i "s/status: replaced/status: edited/g" $@' > "${temp_editor}" chmod +x "${temp_editor}" EDITOR="${temp_editor}" kubectl edit "${kube_flags[@]}" -f "${file}" # Post-condition: mock service (and mock2) and mock rc (and mock2) are edited if [ "$has_svc" = true ]; then kube::test::get_object_assert 'services mock' "{{${labels_field}.status}}" 'edited' if [ "$two_svcs" = true ]; then kube::test::get_object_assert 'services mock2' "{{${labels_field}.status}}" 'edited' fi fi if [ "$has_rc" = true ]; then kube::test::get_object_assert 'rc mock' "{{${labels_field}.status}}" 'edited' if [ "$two_rcs" = true ]; then kube::test::get_object_assert 'rc mock2' "{{${labels_field}.status}}" 'edited' fi fi # cleaning rm "${temp_editor}" </ins> # Command # We need to set --overwrite, because otherwise, if the first attempt to run "kubectl label" # fails on some, but not all, of the resources, retries will fail because it tries to modify
} return false, nil } <ins> // stripComments will transform a YAML file into JSON, thus dropping any comments // in it. Note that if the given file has a syntax error, the transformation will // fail and we will manually drop all comments from the file. func stripComments(file []byte) []byte { stripped, err := yaml.ToJSON(file) if err != nil { stripped = manualStrip(file) } return stripped } // manualStrip is used for dropping comments from a YAML file func manualStrip(file []byte) []byte { stripped := []byte{} for _, line := range bytes.Split(file, []byte("n")) { if bytes.HasPrefix(bytes.TrimSpace(line), []byte("#")) { continue } stripped = append(stripped, line...) stripped = append(stripped, 'n') } return stripped } </ins>
command := exec.Command("mount", mountArgs...) output, err := command.CombinedOutput() if err != nil { <del> glog.Errorf("Mount failed: %vnMounting arguments: %s %s %s %vnOutput: %sn", </del> <ins> return fmt.Errorf("Mount failed: %vnMounting arguments: %s %s %s %vnOutput: %sn", </ins> err, source, target, fstype, options, string(output)) } return err
command := exec.Command("umount", target) output, err := command.CombinedOutput() if err != nil { <del> glog.Errorf("Unmount failed: %vnUnmounting arguments: %snOutput: %sn", err, target, string(output)) return err </del> <ins> return fmt.Errorf("Unmount failed: %vnUnmounting arguments: %snOutput: %sn", err, target, string(output)) </ins> } return nil }
#   google         - Heapster, Google Cloud Monitoring, and Google Cloud Logging #   googleinfluxdb - Enable influxdb and google (except GCM) #   standalone     - Heapster only. Metrics available via Heapster REST API. <del> ENABLE_CLUSTER_MONITORING="${KUBE_ENABLE_CLUSTER_MONITORING:-googleinfluxdb}" </del> <ins> ENABLE_CLUSTER_MONITORING="${KUBE_ENABLE_CLUSTER_MONITORING:-influxdb}" </ins> # Optional: Enable node logging. ENABLE_NODE_LOGGING="${KUBE_ENABLE_NODE_LOGGING:-true}"
AUTOSCALER_MIN_NODES="${KUBE_AUTOSCALER_MIN_NODES:-1}" AUTOSCALER_MAX_NODES="${KUBE_AUTOSCALER_MAX_NODES:-${NUM_MINIONS}}" TARGET_NODE_UTILIZATION="${KUBE_TARGET_NODE_UTILIZATION:-0.7}" <ins> ENABLE_CLUSTER_MONITORING=googleinfluxdb </ins> fi # Optional: Enable deployment experimental feature, not ready for production use.
# Kubernetes Cluster Admin Guide: Cluster Components <ins> **Table of Contents** <!-- BEGIN MUNGE: GENERATED_TOC --> - [Kubernetes Cluster Admin Guide: Cluster Components](#kubernetes-cluster-admin-guide-cluster-components) - [Master Components](#master-components) - [kube-apiserver](#kube-apiserver) - [etcd](#etcd) - [kube-controller-manager](#kube-controller-manager) - [kube-scheduler](#kube-scheduler) - [addons](#addons) - [DNS](#dns) - [User interface](#user-interface) - [Container Resource Monitoring](#container-resource-monitoring) - [Cluster-level Logging](#cluster-level-logging) - [Node components](#node-components) - [kubelet](#kubelet) - [kube-proxy](#kube-proxy) - [docker](#docker) - [rkt](#rkt) - [monit](#monit) - [fluentd](#fluentd) <!-- END MUNGE: GENERATED_TOC --> </ins> This document outlines the various binary components that need to run to deliver a functioning Kubernetes cluster.
Addon objects are created in the "kube-system" namespace. <del> Example addons are: * [DNS](http://releases.k8s.io/HEAD/cluster/addons/dns/) provides cluster local DNS. * [kube-ui](http://releases.k8s.io/HEAD/cluster/addons/kube-ui/) provides a graphical UI for the cluster. * [fluentd-elasticsearch](http://releases.k8s.io/HEAD/cluster/addons/fluentd-elasticsearch/) provides log storage. Also see the [gcp version](http://releases.k8s.io/HEAD/cluster/addons/fluentd-gcp/). * [cluster-monitoring](http://releases.k8s.io/HEAD/cluster/addons/cluster-monitoring/) provides monitoring for the cluster. </del> <ins> #### DNS While the other addons are not strictly required, all Kubernetes clusters should have [cluster DNS](dns.md), as many examples rely on it. Cluster DNS is a DNS server, in addition to the other DNS server(s) in your environment, which serves DNS records for Kubernetes services. Containers started by Kubernetes automatically include this DNS server in their DNS searches. #### User interface The kube-ui provides a read-only overview of the cluster state.  Access [the UI using kubectl proxy](../user-guide/connecting-to-applications-proxy.md#connecting-to-the-kube-ui-service-from-your-local-workstation) #### Container Resource Monitoring [Container Resource Monitoring](../user-guide/monitoring.md) records generic time-series metrics about containers in a central database, and provides a UI for browsing that data. #### Cluster-level Logging [Container Logging](../user-guide/monitoring.md) saves container logs to a central log store with search/browsing interface.  There are two implementations: * [Cluster-level logging to Google Cloud Logging]( docs/user-guide/logging.md#cluster-level-logging-to-google-cloud-logging) * [Cluster-level Logging with Elasticsearch and Kibana]( docs/user-guide/logging.md#cluster-level-logging-with-elasticsearch-and-kibana) </ins> ## Node components
`monit` is a lightweight process babysitting system for keeping kubelet and docker running. <ins> ### fluentd `fluentd` is a daemon which helps provide [cluster-level logging](#cluster-level-logging). </ins> <!-- BEGIN MUNGE: GENERATED_ANALYTICS --> [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/docs/admin/cluster-components.md?pixel)]()
- [Scheduler pod template](#scheduler-pod-template) - [Controller Manager Template](#controller-manager-template) - [Starting and Verifying Apiserver, Scheduler, and Controller Manager](#starting-and-verifying-apiserver-scheduler-and-controller-manager) <del> - [Logging](#logging) - [Monitoring](#monitoring) - [DNS](#dns) </del> <ins> - [Starting Cluster Services](#starting-cluster-services) </ins> - [Troubleshooting](#troubleshooting) - [Running validate-cluster](#running-validate-cluster) - [Inspect pods and services](#inspect-pods-and-services)
- Otherwise, if taking the firewall-based security approach - `--api-servers=http://$MASTER_IP` - `--config=/etc/kubernetes/manifests` <del> - `--cluster-dns=` to the address of the DNS server you will setup (see [Starting Addons](#starting-addons).) </del> <ins> - `--cluster-dns=` to the address of the DNS server you will setup (see [Starting Cluster Services](#starting-cluster-services).) </ins> - `--cluster-domain=` to the dns domain prefix to use for cluster DNS addresses. - `--docker-root=` - `--root-dir=`
You should soon be able to see all your nodes by running the `kubectl get nodes` command. Otherwise, you will need to manually create node objects. <del> ### Logging **TODO** talk about starting Logging. ### Monitoring **TODO** talk about starting Monitoring. ### DNS **TODO** talk about starting DNS. </del> <ins> ### Starting Cluster Services You will want to complete your Kubernetes clusters by adding cluster-wide services.  These are sometimes called *addons*, and [an overview of their purpose is in the admin guide]( ../../docs/admin/cluster-components.md#addons). Notes for setting up each cluster service are given below: * Cluster DNS: * required for many kubernetes examples * [Setup instructions](http://releases.k8s.io/HEAD/cluster/addons/dns/) * [Admin Guide](../admin/dns.md) * Cluster-level Logging * Multiple implementations with different storage backends and UIs. * [Elasticsearch Backend Setup Instructions](http://releases.k8s.io/HEAD/cluster/addons/fluentd-elasticsearch/) * [Google Cloud Logging Backend Setup Instructions](http://releases.k8s.io/HEAD/cluster/addons/fluentd-gcp/). * Both require running fluentd on each node. * [User Guide](../user-guide/logging.md) * Container Resource Monitoring * [Setup instructions](http://releases.k8s.io/HEAD/cluster/addons/cluster-monitoring/) * GUI * [Setup instructions](http://releases.k8s.io/HEAD/cluster/addons/kube-ui/) cluster. </ins> ## Troubleshooting
# Connecting to applications: kubectl proxy and apiserver proxy <ins> <!-- BEGIN MUNGE: GENERATED_TOC --> - [Connecting to applications: kubectl proxy and apiserver proxy](#connecting-to-applications-kubectl-proxy-and-apiserver-proxy) - [Getting the apiserver proxy URL of kube-ui](#getting-the-apiserver-proxy-url-of-kube-ui) - [Connecting to the kube-ui service from your local workstation](#connecting-to-the-kube-ui-service-from-your-local-workstation) <!-- END MUNGE: GENERATED_TOC --> </ins> You have seen the [basics](accessing-the-cluster.md) about `kubectl proxy` and `apiserver proxy`. This guide shows how to use them together to access a service([kube-ui](ui.md)) running on the Kubernetes cluster from your workstation.
# Logging <ins> **Table of Contents** <!-- BEGIN MUNGE: GENERATED_TOC --> - [Logging](#logging) - [Logging by Kubernetes Components](#logging-by-kubernetes-components) - [Examining the logs of running containers](#examining-the-logs-of-running-containers) - [Cluster level logging to Google Cloud Logging](#cluster-level-logging-to-google-cloud-logging) - [Cluster level logging with Elasticsearch and Kibana](#cluster-level-logging-with-elasticsearch-and-kibana) - [Ingesting Application Log Files](#ingesting-application-log-files) - [Known issues](#known-issues) <!-- END MUNGE: GENERATED_TOC --> </ins> ## Logging by Kubernetes Components Kubernetes components, such as kubelet and apiserver, use the [glog](https://godoc.org/github.com/golang/glog) logging library.  Developer conventions for logging severity are described in [docs/devel/logging.md](../devel/logging.md).
-o -path './test/e2e/*'  -o -path './test/e2e_node/*'  -o -path './test/integration/*'  <ins> -o -path './test/component/scheduler/perf/*'  </ins> ) -prune  ) -name '*_test.go' -print0 | xargs -0n1 dirname | sed 's|^./||' | sort -u )
<ins> <!-- BEGIN MUNGE: UNVERSIONED_WARNING --> <!-- BEGIN STRIP_FOR_RELEASE --> <img src="http://kubernetes.io/img/warning.png" alt="WARNING" width="25" height="25"> <img src="http://kubernetes.io/img/warning.png" alt="WARNING" width="25" height="25"> <img src="http://kubernetes.io/img/warning.png" alt="WARNING" width="25" height="25"> <img src="http://kubernetes.io/img/warning.png" alt="WARNING" width="25" height="25"> <img src="http://kubernetes.io/img/warning.png" alt="WARNING" width="25" height="25"> <h2>PLEASE NOTE: This document applies to the HEAD of the source tree</h2> If you are using a released version of Kubernetes, you should refer to the docs that go with that version. <strong> The latest release of this document can be found [here](http://releases.k8s.io/release-1.1/docs/proposals/choosing-scheduler.md). Documentation for other releases can be found at [releases.k8s.io](http://releases.k8s.io). </strong> -- <!-- END STRIP_FOR_RELEASE --> <!-- END MUNGE: UNVERSIONED_WARNING --> Scheduler Performance Test ====== Motivation ------ We already have a performance testing system -- Kubemark. However, Kubemark requires setting up and bootstrapping a whole cluster, which takes a lot of time. We want to have a standard way to reproduce scheduling latency metrics result and benchmark scheduler as simple and fast as possible. We have the following goals: - Save time on testing - The test and benchmark can be run in a single box. We only set up components necessary to scheduling without booting up a cluster. - Profiling runtime metrics to find out bottleneck - Write scheduler integration test but focus on performance measurement. Take advantage of go profiling tools and collect fine-grained metrics, like cpu-profiling, memory-profiling and block-profiling. - Reproduce test result easily - We want to have a known place to do the performance related test for scheduler. Developers should just run one script to collect all the information they need. Currently the test suite has the following: - density test (by adding a new Go test) - schedule 30k pods on 1000 (fake) nodes and 3k pods on 100 (fake) nodes - print out scheduling rate every second - let you learn the rate changes vs number of scheduled pods - benchmark - make use of `go test -bench` and report nanosecond/op. - schedule b.N pods when the cluster has N nodes and P scheduled pods. Since it takes relatively long time to finish one round, b.N is small: 10 - 100. How To Run ------ ``` cd kubernetes/test/component/scheduler/perf ./test-performance.sh ``` <!-- BEGIN MUNGE: GENERATED_ANALYTICS --> [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/test/component/scheduler/perf/README.md?pixel)]() <!-- END MUNGE: GENERATED_ANALYTICS --> </ins>
<ins> /* Copyright 2015 The Kubernetes Authors All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package benchmark import ( "testing" "time" ) // BenchmarkScheduling100Nodes0Pods benchmarks the scheduling rate // when the cluster has 100 nodes and 0 scheduled pods func BenchmarkScheduling100Nodes0Pods(b *testing.B) { benchmarkScheduling(100, 0, b) } // BenchmarkScheduling100Nodes1000Pods benchmarks the scheduling rate // when the cluster has 100 nodes and 1000 scheduled pods func BenchmarkScheduling100Nodes1000Pods(b *testing.B) { benchmarkScheduling(100, 1000, b) } // BenchmarkScheduling1000Nodes0Pods benchmarks the scheduling rate // when the cluster has 1000 nodes and 0 scheduled pods func BenchmarkScheduling1000Nodes0Pods(b *testing.B) { benchmarkScheduling(1000, 0, b) } // BenchmarkScheduling1000Nodes1000Pods benchmarks the scheduling rate // when the cluster has 1000 nodes and 1000 scheduled pods func BenchmarkScheduling1000Nodes1000Pods(b *testing.B) { benchmarkScheduling(1000, 1000, b) } // benchmarkScheduling benchmarks scheduling rate with specific number of nodes // and specific number of pods already scheduled. Since an operation takes relatively // long time, b.N should be small: 10 - 100. func benchmarkScheduling(numNodes, numScheduledPods int, b *testing.B) { schedulerConfigFactory, finalFunc := mustSetupScheduler() defer finalFunc() c := schedulerConfigFactory.Client makeNodes(c, numNodes) makePods(c, numScheduledPods) for { scheduled := schedulerConfigFactory.ScheduledPodLister.Store.List() if len(scheduled) >= numScheduledPods { break } time.Sleep(1 * time.Second) } // start benchmark b.ResetTimer() makePods(c, b.N) for { // This can potentially affect performance of scheduler, since List() is done under mutex. // TODO: Setup watch on apiserver and wait until all pods scheduled. scheduled := schedulerConfigFactory.ScheduledPodLister.Store.List() if len(scheduled) >= numScheduledPods+b.N { break } // Note: This might introduce slight deviation in accuracy of benchmark results. // Since the total amount of time is relatively large, it might not be a concern. time.Sleep(100 * time.Millisecond) } } </ins>
<ins> /* Copyright 2015 The Kubernetes Authors All rights reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package benchmark import ( "fmt" "testing" "time" ) // TestSchedule100Node3KPods schedules 3k pods on 100 nodes. func TestSchedule100Node3KPods(t *testing.T) { schedulePods(100, 3000) } // TestSchedule1000Node30KPods schedules 30k pods on 1000 nodes. func TestSchedule1000Node30KPods(t *testing.T) { schedulePods(1000, 30000) } // schedulePods schedules specific number of pods on specific number of nodes. // This is used to learn the scheduling throughput on various // sizes of cluster and changes as more and more pods are scheduled. // It won't stop until all pods are scheduled. func schedulePods(numNodes, numPods int) { schedulerConfigFactory, destroyFunc := mustSetupScheduler() defer destroyFunc() c := schedulerConfigFactory.Client makeNodes(c, numNodes) makePods(c, numPods) prev := 0 start := time.Now() for { // This can potentially affect performance of scheduler, since List() is done under mutex. // Listing 10000 pods is an expensive operation, so running it frequently may impact scheduler. // TODO: Setup watch on apiserver and wait until all pods scheduled. scheduled := schedulerConfigFactory.ScheduledPodLister.Store.List() fmt.Printf("%dstrate: %dttotal: %dn", time.Since(start)/time.Second, len(scheduled)-prev, len(scheduled)) if len(scheduled) >= numPods { return } prev = len(scheduled) time.Sleep(1 * time.Second) } } </ins>