Change:
What would you like to be : We'd like to have the Prometheus stack () running in our scalability tests. Why is this needed: There are couple of reasons: will simplify writing SLI assertions/tests that are based on metrics not necessarily exported on master. Some examples here are the new In-Cluster Network Programming Latency SLI (exported in kube-proxy) or soon to be implemented RTT SLI where the metric will be exported in regular pods (probes). With Prometheus stack, collecting and aggregating such metrics is very simple. -prometheus provides many useful metrics about the state of the cluster, such metrics might be invaluable when debugging scalability test failures. mimics what customers do in their cluster, making the test setup more realistic.
/sig scalability
/assign
The GCE 5K Node Performance test with enabled Prometheus completed successfully. We can enable Prometheus in all tests.
Looks like kubemark-500 has some issue, test fails often with Kubemark-5000 also failed once with Will disable Prometheus in kubemark, will work on fixing it when I'm back from my vacation.
For the first kind of the failure that kubemark-500 suffered from, i.e. the root cause is Looks like the token for the "prometheus-operator" service account was not ready, as we can see in the logs it was created after the POST to create the servicemonitor was called.
It looks like the second failure (kubemark-5000) was caused by the prometheus-k8s pods not scheduled by the kube-scheduler. This is strange but may be related to other issues we had. Hard to investigate what really happened there, will leave it for now. Will re-enable prometheus tomorrow and will see if it happens again.
Closing, as this is basically done in a minimal version. Other improvements will be tracked in separate issues. /close
Closing this issue. <detailsIn response to : Closing, as this is basically done in a minimal version. Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details

Discussion:
package api import ( <del> "github.com/fsouza/go-dockerclient" </del> "github.com/GoogleCloudPlatform/kubernetes/pkg/util" <ins> "github.com/fsouza/go-dockerclient" </ins> ) // Common string formats

Selector                   map[string]string `json:"selector,omitempty" yaml:"selector,omitempty"` CreateExternalLoadBalancer bool              `json:"createExternalLoadBalancer,omitempty" yaml:"createExternalLoadBalancer,omitempty"` <del> // Container port to connect to, either a name or a port number </del> <ins> // ContainerPort is the name of the port on the container to direct traffic to. // Optional, if unspecified use the first port on the container. </ins> ContainerPort util.IntOrString `json:"containerPort,omitempty" yaml:"containerPort,omitempty"` }

package registry import ( <ins> "fmt" </ins> "net" "strconv" "github.com/GoogleCloudPlatform/kubernetes/pkg/api" "github.com/GoogleCloudPlatform/kubernetes/pkg/labels" <ins> "github.com/GoogleCloudPlatform/kubernetes/pkg/util" </ins> "github.com/golang/glog" )

podRegistry     PodRegistry } <ins> func findPort(manifest *api.ContainerManifest, portName util.IntOrString) (int, error) { if ((portName.Kind == util.IntstrString && len(portName.StrVal) == 0) || (portName.Kind == util.IntstrInt && portName.IntVal == 0)) && len(manifest.Containers[0].Ports) > 0 { return manifest.Containers[0].Ports[0].ContainerPort, nil } if portName.Kind == util.IntstrInt { return portName.IntVal, nil } name := portName.StrVal for _, container := range manifest.Containers { for _, port := range container.Ports { if port.Name == name { return port.ContainerPort, nil } } } return -1, fmt.Errorf("no suitable port for manifest: %s", manifest.ID) } </ins> func (e *EndpointController) SyncServiceEndpoints() error { services, err := e.serviceRegistry.ListServices() if err != nil {

} endpoints := make([]string, len(pods)) for ix, pod := range pods { <del> // TODO: Use port names in the service object, don't just use port #0 endpoints[ix] = net.JoinHostPort( pod.CurrentState.Host, strconv.Itoa(pod.DesiredState.Manifest.Containers[0].Ports[0].HostPort), ) </del> <ins> port, err := findPort(&pod.DesiredState.Manifest, service.ContainerPort) if err != nil { glog.Errorf("Failed to find port for service: %v, %v", service, err) continue } endpoints[ix] = net.JoinHostPort(pod.CurrentState.PodIP, strconv.Itoa(port)) </ins> } err = e.serviceRegistry.UpdateEndpoints(api.Endpoints{ Name:      service.ID,

"testing" "github.com/GoogleCloudPlatform/kubernetes/pkg/api" <ins> "github.com/GoogleCloudPlatform/kubernetes/pkg/util" </ins> ) <ins> func TestFindPort(t *testing.T) { manifest := api.ContainerManifest{ Containers: []api.Container{ { Ports: []api.Port{ { Name:          "foo", ContainerPort: 8080, HostPort:      9090, }, { Name:          "bar", ContainerPort: 8000, HostPort:      9000, }, }, }, }, } port, err := findPort(&manifest, util.IntOrString{Kind: util.IntstrString, StrVal: "foo"}) expectNoError(t, err) if port != 8080 { t.Errorf("Expected 8080, Got %d", port) } port, err = findPort(&manifest, util.IntOrString{Kind: util.IntstrString, StrVal: "bar"}) expectNoError(t, err) if port != 8000 { t.Errorf("Expected 8000, Got %d", port) } port, err = findPort(&manifest, util.IntOrString{Kind: util.IntstrInt, IntVal: 8000}) if port != 8000 { t.Errorf("Expected 8000, Got %d", port) } port, err = findPort(&manifest, util.IntOrString{Kind: util.IntstrInt, IntVal: 7000}) if port != 7000 { t.Errorf("Expected 7000, Got %d", port) } port, err = findPort(&manifest, util.IntOrString{Kind: util.IntstrString, StrVal: "baz"}) if err == nil { t.Error("unexpected non-error") } port, err = findPort(&manifest, util.IntOrString{Kind: util.IntstrString, StrVal: ""}) expectNoError(t, err) if port != 8080 { t.Errorf("Expected 8080, Got %d", port) } port, err = findPort(&manifest, util.IntOrString{}) expectNoError(t, err) if port != 8080 { t.Errorf("Expected 8080, Got %d", port) } } </ins> func TestSyncEndpointsEmpty(t *testing.T) { serviceRegistry := MockServiceRegistry{} podRegistry := MockPodRegistry{}