Issue 1:
kube-error.txt

Is this a request for help? (If yes, you should use our troubleshooting guide and community support channels, see http://kubernetes.io/docs/troubleshooting/.):

What keywords did you search in Kubernetes issues before filing this one? (If you have found any duplicates, you should instead reply there.):

Is this a BUG REPORT or FEATURE REQUEST? (choose one):

Kubernetes version (use kubectl version):

1.4.1

Environment:

Cloud provider or hardware configuration:

OS (e.g. from /etc/os-release):

Kernel (e.g. uname -a):

Install tools:

Others:

What happened:

[root@k8s-master ~]# kubectl version

Client Version: version.Info{Major:"1", Minor:"4", GitVersion:"v1.4.1", GitCommit:"33cf7b9acbb2cb7c9c72a10d6636321fb180b159", GitTreeState:"clean", BuildDate:"2016-10-10T18:19:49Z", GoVersion:"go1.6.3", Compiler:"gc", Platform:"linux/amd64"}

Server Version: version.Info{Major:"1", Minor:"4", GitVersion:"v1.4.1", GitCommit:"33cf7b9acbb2cb7c9c72a10d6636321fb180b159", GitTreeState:"clean", BuildDate:"2016-10-10T18:13:36Z", GoVersion:"go1.6.3", Compiler:"gc", Platform:"linux/amd64"}

[root@k8s-master ~]# kubectl get nodes

NAME STATUS AGE

10.37.116.32 Ready 1h

[root@k8s-master ~]# ps -ef|grep kube

root 1767 1708 0 11:32 pts/1 00:00:04 kube-apiserver --logtostderr=true --v=5 --etcd-servers=https://10.37.116.31:2379,https://10.37.116.32:2379 --etcd-cafile=/ssl/ca.pem --etcd-certfile=/ssl/server.pem --etcd-keyfile=/ssl/server-key.pem --bind-address=10.37.116.31 --secure-port=6443 --advertise-address=10.37.116.31 --allow-privileged=false --service-cluster-ip-range=172.16.0.0/16 --admission-control=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota --client-ca-file=/ssl/ca.pem --tls-cert-file=/ssl/server.pem --tls-private-key-file=/ssl/server-key.pem

root 1781 1726 0 11:33 pts/2 00:00:01 kube-controller-manager --logtostderr=true --v=5 --master=127.0.0.1:8080 --allocate-node-cidrs=true --cluster-cidr=172.16.0.0/16 --root-ca-file=/ssl/ca.pem --service-account-private-key-file=/ssl/server-key.pem

root 1795 1040 0 11:33 pts/0 00:00:00 kube-scheduler --logtostderr=true --v=5 --master=127.0.0.1:8080

[root@k8s-node-1 ~]# ps -ef|grep kube

root 26089 1223 1 11:34 pts/0 00:00:06 kube-proxy --logtostderr=true --v=5 --hostname-override=10.37.116.32 --kubeconfig=/etc/kubernetes/kubecfg --proxy-mode=iptables

root 26529 24882 0 11:34 pts/1 00:00:04 kubelet --logtostderr=true --v=5 --address=10.37.116.32 --port=10250 --hostname-override=10.37.116.32 --kubeconfig=/etc/kubernetes/kubecfg --require-kubeconfig=true --pod-infra-container-image=10.37.116.11:8443/pause:latest --allow-privileged=false --tls-cert-file=/ssl/server.pem --tls-private-key-file=/ssl/server-key.pem

[root@k8s-node-1 ~]# cat /etc/kubernetes/kubecfg

apiVersion: v1

kind: Config

users:

name: kubelet

user:

client-certificate: /ssl/server.pem

client-key: /ssl/server-key.pem

clusters:

name: dev

cluster:

certificate-authority: /ssl/ca.pem

server: https://10.37.116.31:6443

contexts:

context:

cluster: dev

user: kubelet

name: service-account-context

current-context: service-account-context

[root@k8s-master ~]# kubectl get sa --all-namespaces -o yaml

apiVersion: v1

items:

apiVersion: v1

kind: ServiceAccount

metadata:

creationTimestamp: 2016-11-10T03:24:12Z

name: default

namespace: default

resourceVersion: "2732"

selfLink: /api/v1/namespaces/default/serviceaccounts/default

uid: 26a7b030-a6f5-11e6-84f0-080027acd9e1

secrets:

name: default-token-g9xgv

apiVersion: v1

kind: ServiceAccount

metadata:

creationTimestamp: 2016-11-10T03:24:26Z

name: default

namespace: kube-system

resourceVersion: "2754"

selfLink: /api/v1/namespaces/kube-system/serviceaccounts/default

uid: 2edaf5ed-a6f5-11e6-84f0-080027acd9e1

secrets:

name: default-token-rsqoz

kind: List

metadata: {}

[root@k8s-master ~]# kubectl get secrets --all-namespaces -o yaml

apiVersion: v1

items:

apiVersion: v1

data:

ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURtakNDQW9LZ0F3SUJBZ0lVY1puRlRXd29UMWxpeXVtT3JGYXcxaVBzQTVrd0RRWUpLb1pJaHZjTkFRRUwKQlFBd1pURUxNQWtHQTFVRUJoTUNRMDR4RGpBTUJnTlZCQWdUQlVoMWJtRnVNUkV3RHdZRFZRUUhFd2hEYUdGdQpaM05vWVRFU01CQUdBMVVFQ2hNSlN6aHpMQ0JKYm1NdU1SRXdEd1lEVlFRTEV3aE5hVzRnVkdWamFERU1NQW9HCkExVUVBeE1EU3poVE1CNFhEVEUyTVRFeE1EQXlNemt3TUZvWERUSXhNVEV3T1RBeU16a3dNRm93WlRFTE1Ba0cKQTFVRUJoTUNRMDR4RGpBTUJnTlZCQWdUQlVoMWJtRnVNUkV3RHdZRFZRUUhFd2hEYUdGdVozTm9ZVEVTTUJBRwpBMVVFQ2hNSlN6aHpMQ0JKYm1NdU1SRXdEd1lEVlFRTEV3aE5hVzRnVkdWamFERU1NQW9HQTFVRUF4TURTemhUCk1JSUJJakFOQmdrcWhraUc5dzBCQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBd1c1R0dXR0JCR0ZMUG53cHVraFoKYk4zeEV2cUQzWkdkWHZWNzI2NTRSeHM1VGREcWoxZXN6ZFgvVUxpd3VEZEQ5cFNpYWlVb1IzMytCZ1dFRHRGSApoRmlMUzhBdW5maXkrd0xlam0wVktrdHRhcTA2bjd0RDNyWll0cGc1c1BCNlhHVGtqVHlsZWdpNFhCRE55ajUvCmtyRjliVXpmZmI5VVF2TGhVYUpveWIwQWsvRXNFdUQ2Unc5OVpFempGQS9jbHdMWFA5MmowTlk1MEc5N3ZwN0wKMVNsZ1ZudHdveUNYaE1ua2R3LysyMVdkQVprdUtjbERSSUY2eW45bFU0Y05ZOE96ZVlIQ3lnazFqMVN5VHRhNgpRUCsxZDI2QXRnaHpGZkhzRGxidkFyUmt1ei85RWhxRmRoenVuUm8wSWVFdXQvWkx3OGZHQ2MzSGY3NmgyS2JlClhRSURBUUFCbzBJd1FEQU9CZ05WSFE4QkFmOEVCQU1DQVFZd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBZEJnTlYKSFE0RUZnUVVUaXoxekdYR1ZZMXpIT3dZNHZDOXhuc2RxdFV3RFFZSktvWklodmNOQVFFTEJRQURnZ0VCQUpNZwp1ZVg3V0thRllGc2srUmVYaitjRXp3aStUZjBiZ09abFVaVVJVaUlvWTc1NlA3WW9ZUkZUNHJZL1kyb1BUUHBQCjdIOUVLdGkvRzFFTE5hbVBCZ3ovejV2aDJTVzlRZ25ZcGVjeGpDM0tqODFKWVlWOTV6NEJLbE9Tb2JEMlpiYnEKb25KL1NPK3ZSWGhraGI2c3VpdHdvNy9WNGlWWG54MUdNTU5NVldKUWFDYWpZaEx6ZWZIemdQNG0wamNEUnBudwp3d09UZklSYVdSRUtGT3JvcTlBTFdSdFBBcEFTV0c4d2ZINVVMYk40NzczbHJvZ0V4d1ltVnVqNDYwY1hhVzdwCkVvZHdWVG04dXE2MzNsQ3RNS0YyVVpCbzZ5V2tMellBclU1NHk5MmZSMmdiampLckJhcGVKYmZwN1NFUmhZNHIKb0dsOWdpWFZLZlVRMVhxUzZlZz0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=

namespace: ZGVmYXVsdA==

token: ZXlKaGJHY2lPaUpTVXpJMU5pSXNJblI1Y0NJNklrcFhWQ0o5LmV5SnBjM01pT2lKcmRXSmxjbTVsZEdWekwzTmxjblpwWTJWaFkyTnZkVzUwSWl3aWEzVmlaWEp1WlhSbGN5NXBieTl6WlhKMmFXTmxZV05qYjNWdWRDOXVZVzFsYzNCaFkyVWlPaUprWldaaGRXeDBJaXdpYTNWaVpYSnVaWFJsY3k1cGJ5OXpaWEoyYVdObFlXTmpiM1Z1ZEM5elpXTnlaWFF1Ym1GdFpTSTZJbVJsWm1GMWJIUXRkRzlyWlc0dFp6bDRaM1lpTENKcmRXSmxjbTVsZEdWekxtbHZMM05sY25acFkyVmhZMk52ZFc1MEwzTmxjblpwWTJVdFlXTmpiM1Z1ZEM1dVlXMWxJam9pWkdWbVlYVnNkQ0lzSW10MVltVnlibVYwWlhNdWFXOHZjMlZ5ZG1salpXRmpZMjkxYm5RdmMyVnlkbWxqWlMxaFkyTnZkVzUwTG5WcFpDSTZJakkyWVRkaU1ETXdMV0UyWmpVdE1URmxOaTA0TkdZd0xUQTRNREF5TjJGalpEbGxNU0lzSW5OMVlpSTZJbk41YzNSbGJUcHpaWEoyYVdObFlXTmpiM1Z1ZERwa1pXWmhkV3gwT21SbFptRjFiSFFpZlEuYU1kS0RfN2lVVXZsVG5LSVNUTFRqVk5QcTE4TEktSGVPMnR0dlhJZHlQSVFFaTNrQ3dtYnh2ZVJCbEdPbjBkN3lEeUxYX09ZZldmSm9VcE1sYVRfeV9XcFRWMml0dUlRbUVfd3N0X05iOW91ckVRZEFHbmJnS0RwX281eWJlS09pU2UzN0h6RFE3ekVHNHh4QWdxZTR1M1NTNl9PQmM0OHdsTWtsUF9LSW4tSTA2c3EzMXRwTV9TMUY1VFpsVnA2U0dnbXU5WWF0TVpySmx1VXEzMllaazQ2R0RtVFdGWW05WERXU2RlQlVPbkxWWnE4UEREZjE4TGJJZVhwUVZ5NWUyem1HRm0tem9uU0xQOV9DclUzeTBCRGRJb1lnWlJvcFNFUkNJSVRyN1Rlay1PSVJwWU91bjZSR3dTNVhtb0ozZzg5a0hHSUpTRGNoUldaOGl5cGJB

kind: Secret

metadata:

annotations:

kubernetes.io/service-account.name: default

kubernetes.io/service-account.uid: 26a7b030-a6f5-11e6-84f0-080027acd9e1

creationTimestamp: 2016-11-10T03:24:12Z

name: default-token-g9xgv

namespace: default

resourceVersion: "2731"

selfLink: /api/v1/namespaces/default/secrets/default-token-g9xgv

uid: 26af0a29-a6f5-11e6-84f0-080027acd9e1

type: kubernetes.io/service-account-token

apiVersion: v1

data:

ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURtakNDQW9LZ0F3SUJBZ0lVY1puRlRXd29UMWxpeXVtT3JGYXcxaVBzQTVrd0RRWUpLb1pJaHZjTkFRRUwKQlFBd1pURUxNQWtHQTFVRUJoTUNRMDR4RGpBTUJnTlZCQWdUQlVoMWJtRnVNUkV3RHdZRFZRUUhFd2hEYUdGdQpaM05vWVRFU01CQUdBMVVFQ2hNSlN6aHpMQ0JKYm1NdU1SRXdEd1lEVlFRTEV3aE5hVzRnVkdWamFERU1NQW9HCkExVUVBeE1EU3poVE1CNFhEVEUyTVRFeE1EQXlNemt3TUZvWERUSXhNVEV3T1RBeU16a3dNRm93WlRFTE1Ba0cKQTFVRUJoTUNRMDR4RGpBTUJnTlZCQWdUQlVoMWJtRnVNUkV3RHdZRFZRUUhFd2hEYUdGdVozTm9ZVEVTTUJBRwpBMVVFQ2hNSlN6aHpMQ0JKYm1NdU1SRXdEd1lEVlFRTEV3aE5hVzRnVkdWamFERU1NQW9HQTFVRUF4TURTemhUCk1JSUJJakFOQmdrcWhraUc5dzBCQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBd1c1R0dXR0JCR0ZMUG53cHVraFoKYk4zeEV2cUQzWkdkWHZWNzI2NTRSeHM1VGREcWoxZXN6ZFgvVUxpd3VEZEQ5cFNpYWlVb1IzMytCZ1dFRHRGSApoRmlMUzhBdW5maXkrd0xlam0wVktrdHRhcTA2bjd0RDNyWll0cGc1c1BCNlhHVGtqVHlsZWdpNFhCRE55ajUvCmtyRjliVXpmZmI5VVF2TGhVYUpveWIwQWsvRXNFdUQ2Unc5OVpFempGQS9jbHdMWFA5MmowTlk1MEc5N3ZwN0wKMVNsZ1ZudHdveUNYaE1ua2R3LysyMVdkQVprdUtjbERSSUY2eW45bFU0Y05ZOE96ZVlIQ3lnazFqMVN5VHRhNgpRUCsxZDI2QXRnaHpGZkhzRGxidkFyUmt1ei85RWhxRmRoenVuUm8wSWVFdXQvWkx3OGZHQ2MzSGY3NmgyS2JlClhRSURBUUFCbzBJd1FEQU9CZ05WSFE4QkFmOEVCQU1DQVFZd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBZEJnTlYKSFE0RUZnUVVUaXoxekdYR1ZZMXpIT3dZNHZDOXhuc2RxdFV3RFFZSktvWklodmNOQVFFTEJRQURnZ0VCQUpNZwp1ZVg3V0thRllGc2srUmVYaitjRXp3aStUZjBiZ09abFVaVVJVaUlvWTc1NlA3WW9ZUkZUNHJZL1kyb1BUUHBQCjdIOUVLdGkvRzFFTE5hbVBCZ3ovejV2aDJTVzlRZ25ZcGVjeGpDM0tqODFKWVlWOTV6NEJLbE9Tb2JEMlpiYnEKb25KL1NPK3ZSWGhraGI2c3VpdHdvNy9WNGlWWG54MUdNTU5NVldKUWFDYWpZaEx6ZWZIemdQNG0wamNEUnBudwp3d09UZklSYVdSRUtGT3JvcTlBTFdSdFBBcEFTV0c4d2ZINVVMYk40NzczbHJvZ0V4d1ltVnVqNDYwY1hhVzdwCkVvZHdWVG04dXE2MzNsQ3RNS0YyVVpCbzZ5V2tMellBclU1NHk5MmZSMmdiampLckJhcGVKYmZwN1NFUmhZNHIKb0dsOWdpWFZLZlVRMVhxUzZlZz0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=

namespace: a3ViZS1zeXN0ZW0=

token: ZXlKaGJHY2lPaUpTVXpJMU5pSXNJblI1Y0NJNklrcFhWQ0o5LmV5SnBjM01pT2lKcmRXSmxjbTVsZEdWekwzTmxjblpwWTJWaFkyTnZkVzUwSWl3aWEzVmlaWEp1WlhSbGN5NXBieTl6WlhKMmFXTmxZV05qYjNWdWRDOXVZVzFsYzNCaFkyVWlPaUpyZFdKbExYTjVjM1JsYlNJc0ltdDFZbVZ5Ym1WMFpYTXVhVzh2YzJWeWRtbGpaV0ZqWTI5MWJuUXZjMlZqY21WMExtNWhiV1VpT2lKa1pXWmhkV3gwTFhSdmEyVnVMWEp6Y1c5Nklpd2lhM1ZpWlhKdVpYUmxjeTVwYnk5elpYSjJhV05sWVdOamIzVnVkQzl6WlhKMmFXTmxMV0ZqWTI5MWJuUXVibUZ0WlNJNkltUmxabUYxYkhRaUxDSnJkV0psY201bGRHVnpMbWx2TDNObGNuWnBZMlZoWTJOdmRXNTBMM05sY25acFkyVXRZV05qYjNWdWRDNTFhV1FpT2lJeVpXUmhaalZsWkMxaE5tWTFMVEV4WlRZdE9EUm1NQzB3T0RBd01qZGhZMlE1WlRFaUxDSnpkV0lpT2lKemVYTjBaVzA2YzJWeWRtbGpaV0ZqWTI5MWJuUTZhM1ZpWlMxemVYTjBaVzA2WkdWbVlYVnNkQ0o5LlRBSTdoNVBLLWRYX1QyN0poaGkzblpsaEMwYjNfT21TWHI1ZUw4b0RlbVZhMHk5ZHotQ3NqVlVZam9YMkJzeFRuc0QwU2JHc1YwM1NEVHJOakVJZGZRNDg4NGVRejJGRkxvckVJQkJ6N1hDM3d1RkNMMnNkcExhcnltY2U2cFBRX1hFVDR3TzZpYjVLRHBJODhkUmxMOUt4QzNoUG5UTDh6VkEyU3hkdUU2MU9qZlQ4NGM3MkdKMTh4cXdzRmZnWFhwaEVvODVkV19Tb3Qza0l0b2hEMEtFcDVNLUoxc3dlMm5Sdk1DRkxBcHpYaC1PaUU0QlA1RkNUTEFocE0xMnlrUUNqLTZ0Y1dkMmNpanVKWjZHSXVrOTZFRHlTNWdxZXJBOHVEdWV4X3lLdnBiMUctSHFyVlBpRmZXRnM5Wk1yLXBVeVoyOG1JV01NcHFKTzdiX0JZUQ==

kind: Secret

metadata:

annotations:

kubernetes.io/service-account.name: default

kubernetes.io/service-account.uid: 2edaf5ed-a6f5-11e6-84f0-080027acd9e1

creationTimestamp: 2016-11-10T03:24:26Z

name: default-token-rsqoz

namespace: kube-system

resourceVersion: "2753"

selfLink: /api/v1/namespaces/kube-system/secrets/default-token-rsqoz

uid: 2edf140b-a6f5-11e6-84f0-080027acd9e1

type: kubernetes.io/service-account-token

kind: List

metadata: {}

[root@k8s-master ~]# kubectl get svc,po --all-namespaces

NAMESPACE NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE

default svc/kubernetes 172.16.0.1 443/TCP 54m

kube-system svc/kubernetes-dashboard 172.16.58.226 80/TCP 45m

NAMESPACE NAME READY STATUS RESTARTS AGE

kube-system po/kubernetes-dashboard-v1.4.1-6s1qb 0/1 CrashLoopBackOff 6 7m

[root@k8s-master ~]# kubectl -n kube-system get po kubernetes-dashboard-v1.4.1-6s1qb -o yaml

apiVersion: v1

kind: Pod

metadata:

annotations:

kubernetes.io/created-by: |

{"kind":"SerializedReference","apiVersion":"v1","reference":{"kind":"ReplicationController","namespace":"kube-system","name":"kubernetes-dashboard-v1.4.1","uid":"deb11313-a6f1-11e6-84f0-080027acd9e1","apiVersion":"v1","resourceVersion":"3214"}}

creationTimestamp: 2016-11-10T03:38:02Z

generateName: kubernetes-dashboard-v1.4.1-

labels:

k8s-app: kubernetes-dashboard

kubernetes.io/cluster-service: "true"

version: v1.4.1

name: kubernetes-dashboard-v1.4.1-6s1qb

namespace: kube-system

ownerReferences:

apiVersion: v1

controller: true

kind: ReplicationController

name: kubernetes-dashboard-v1.4.1

uid: deb11313-a6f1-11e6-84f0-080027acd9e1

resourceVersion: "4813"

selfLink: /api/v1/namespaces/kube-system/pods/kubernetes-dashboard-v1.4.1-6s1qb

uid: 1513fc5e-a6f7-11e6-b259-080027acd9e1

spec:

containers:

args:

--apiserver-host=https://10.37.116.31:6443

env:

name: KUBECONFIG

value: /etc/kubernetes/kubecfg

image: 10.37.116.11:8443/kubernetes-dashboard-amd64:v1.4.1

imagePullPolicy: IfNotPresent

livenessProbe:

failureThreshold: 3

httpGet:

path: /

port: 9090

scheme: HTTP

initialDelaySeconds: 30

periodSeconds: 10

successThreshold: 1

timeoutSeconds: 30

name: kubernetes-dashboard

ports:

containerPort: 9090

protocol: TCP

resources:

limits:

cpu: 100m

memory: 50Mi

requests:

cpu: 100m

memory: 50Mi

terminationMessagePath: /dev/termination-log

volumeMounts:

mountPath: /etc/pki

name: etcpki

readOnly: true

mountPath: /etc/kubernetes

name: config

readOnly: true

mountPath: /var/run/secrets/kubernetes.io/serviceaccount

name: default-token-rsqoz

readOnly: true

dnsPolicy: ClusterFirst

nodeName: 10.37.116.32

restartPolicy: Always

securityContext: {}

serviceAccount: default

serviceAccountName: default

terminationGracePeriodSeconds: 30

volumes:

hostPath:

path: /etc/pki

name: etcpki

hostPath:

path: /etc/kubernetes

name: config

name: default-token-rsqoz

secret:

defaultMode: 420

secretName: default-token-rsqoz

status:

conditions:

lastProbeTime: null

lastTransitionTime: 2016-11-10T03:38:02Z

status: "True"

type: Initialized

lastProbeTime: null

lastTransitionTime: 2016-11-10T03:43:45Z

message: 'containers with unready status: [kubernetes-dashboard]'

reason: ContainersNotReady

status: "False"

type: Ready

lastProbeTime: null

lastTransitionTime: 2016-11-10T03:38:02Z

status: "True"

type: PodScheduled

containerStatuses:

containerID: docker://c196c881084ada28a001b207879e1317c0fd9aa01fc8343ee6ce7f8a42b35666

image: 10.37.116.11:8443/kubernetes-dashboard-amd64:v1.4.1

imageID: docker://sha256:1dda73f463b239955d4cf94a9cd525ab5306bee0eb53c17534d3282bc50ae7aa

lastState:

terminated:

containerID: docker://c196c881084ada28a001b207879e1317c0fd9aa01fc8343ee6ce7f8a42b35666

exitCode: 1

finishedAt: 2016-11-10T03:48:49Z

reason: Error

startedAt: 2016-11-10T03:48:48Z

name: kubernetes-dashboard

ready: false

restartCount: 7

state:

waiting:

message: Back-off 5m0s restarting failed container=kubernetes-dashboard pod=kubernetes-dashboard-v1.4.1-6s1qb_kube-system(1513fc5e-a6f7-11e6-b259-080027acd9e1)

reason: CrashLoopBackOff

hostIP: 10.37.116.32

phase: Running

podIP: 172.16.22.130

startTime: 2016-11-10T03:38:02Z

[root@k8s-master ~]# kubectl -n kube-system logs po/kubernetes-dashboard-v1.4.1-6s1qb

Using HTTP port: 9090

Using apiserver-host location: https://10.37.116.31:6443

Creating API server client for https://10.37.116.31:6443

Error while initializing connection to Kubernetes apiserver. This most likely means that the cluster is misconfigured (e.g., it has invalid apiserver certificates or service accounts configuration) or the --apiserver-host param points to a server that does not exist. Reason: the server has asked for the client to provide credentials

What you expected to happen:

How to reproduce it (as minimally and precisely as possible):

Anything else do we need to know:

Issue 2:
Is this a request for help? (If yes, you should use our troubleshooting guide and community support channels, see http://kubernetes.io/docs/troubleshooting/.):

No.

What keywords did you search in Kubernetes issues before filing this one? (If you have found any duplicates, you should instead reply there.):

upgrade volume

mount timeout

ContainerCreating

Is this a BUG REPORT or FEATURE REQUEST? (choose one):

BUG REPORT

Kubernetes version (use kubectl version):

Server Version: version.Info{Major:"1", Minor:"5+", GitVersion:"v1.5.0-beta.2", GitCommit:"0776eab45fe28f02bbeac0f05ae1a203051a21eb", GitTreeState:"clean", BuildDate:"2016-11-24T22:30:23Z", GoVersion:"go1.7.3", Compiler:"gc", Platform:"linux/amd64"}

Environment:

Cloud provider or hardware configuration: GCE

OS (e.g. from /etc/os-release):

master:

BUILD_ID=8977.0.0
NAME="Container-VM Image"
GOOGLE_CRASH_ID=Lakitu
VERSION_ID=56
BUG_REPORT_URL=https://crbug.com/new
PRETTY_NAME="Google Container-VM Image"
VERSION=56
GOOGLE_METRICS_PRODUCT_ID=26
HOME_URL="https://cloud.google.com/compute/docs/containers/vm-image/"
ID=gci

minion:

PRETTY_NAME="Debian GNU/Linux 7 (wheezy)"
NAME="Debian GNU/Linux"
VERSION_ID="7"
VERSION="7 (wheezy)"
ID=debian
ANSI_COLOR="1;31"
HOME_URL="http://www.debian.org/"
SUPPORT_URL="http://www.debian.org/support/"
BUG_REPORT_URL="http://bugs.debian.org/"

Kernel (e.g. uname -a):

master: Linux kubernetes-master 4.4.21+ #1 SMP Thu Nov 10 02:50:15 PST 2016 x86_64 Intel(R) Xeon(R) CPU @ 2.30GHz GenuineIntel GNU/Linux

minion: Linux kubernetes-minion-group-4klk 3.16.0-4-amd64 #1 SMP Debian 3.16.36-1+deb8u2 (2016-10-19) x86_64 GNU/Linux

Install tools: cluster/gce/upgrade.sh

Others:

What happened:

Pods with PVCs get stuck in ContainerCreating after Node gets upgraded. The error is timeout expired waiting for volumes to attach/mount as shown below:

$ kubectl describe pod web-1
Name: web-1
Namespace: default
Node: kubernetes-minion-group-54hw/10.128.0.4
Start Time: Mon, 28 Nov 2016 11:52:11 -0800
Labels: app=nginx
Status: Pending
IP:
Controllers: PetSet/web
Containers:
nginx:
Container ID:
Image: gcr.io/google_containers/nginx-slim:0.8
Image ID:
Port: 80/TCP
Requests:
cpu: 100m
State: Waiting
Reason: ContainerCreating
Ready: False
Restart Count: 0
Volume Mounts:
/usr/share/nginx/html from www (rw)
/var/run/secrets/kubernetes.io/serviceaccount from default-token-xim26 (ro)
Environment Variables: <none>
Conditions:
Type Status
Initialized True
Ready False
PodScheduled True
Volumes:
www:
Type: PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
ClaimName: www-web-1
ReadOnly: false
default-token-xim26:
Type: Secret (a volume populated by a Secret)
SecretName: default-token-xim26
QoS Class: Burstable
Tolerations: <none>
Events:
FirstSeen LastSeen Count From SubObjectPath Type Reason Message
--------- -------- ----- ---- ------------- -------- ------ -------
40m 40m 1 {default-scheduler } Normal Scheduled Successfully assigned web-1 to kubernetes-minion-group-54hw
40m 40m 1 {kubelet kubernetes-minion-group-54hw} spec.containers{nginx} Normal Pulling pulling image "gcr.io/google_containers/nginx-slim:0.8"
40m 40m 1 {kubelet kubernetes-minion-group-54hw} spec.containers{nginx} Normal Pulled Successfully pulled image "gcr.io/google_containers/nginx-slim:0.8"
40m 40m 1 {kubelet kubernetes-minion-group-54hw} spec.containers{nginx} Normal Created Created container with docker id d2d15140a74a; Security:[seccomp=unconfined]
40m 40m 1 {kubelet kubernetes-minion-group-54hw} spec.containers{nginx} Normal Started Started container with docker id d2d15140a74a
16m 16m 4 {kubelet kubernetes-minion-group-54hw} Warning FailedMount MountVolume.SetUp failed for volume "kubernetes.io/secret/270e4f19-b5a4-11e6-8922-42010a800002-default-token-xim26" (spec.Name: "default-token-xim26") pod "270e4f19-b5a4-11e6-8922-42010a800002" (UID: "270e4f19-b5a4-11e6-8922-42010a800002") with: Get https://kubernetes-master/api/v1/namespaces/default/secrets/default-token-xim26: dial tcp: lookup kubernetes-master on 169.254.169.254:53: no such host
16m 16m 1 {kubelet kubernetes-minion-group-54hw} Warning FailedMount MountVolume.SetUp failed for volume "kubernetes.io/secret/270e4f19-b5a4-11e6-8922-42010a800002-default-token-xim26" (spec.Name: "default-token-xim26") pod "270e4f19-b5a4-11e6-8922-42010a800002" (UID: "270e4f19-b5a4-11e6-8922-42010a800002") with: Get https://kubernetes-master/api/v1/namespaces/default/secrets/default-token-xim26: dial tcp 10.128.0.2:443: i/o timeout
15m 15m 3 {kubelet kubernetes-minion-group-54hw} Warning FailedMount MountVolume.SetUp failed for volume "kubernetes.io/secret/270e4f19-b5a4-11e6-8922-42010a800002-default-token-xim26" (spec.Name: "default-token-xim26") pod "270e4f19-b5a4-11e6-8922-42010a800002" (UID: "270e4f19-b5a4-11e6-8922-42010a800002") with: Get https://kubernetes-master/api/v1/namespaces/default/secrets/default-token-xim26: dial tcp 10.128.0.2:443: getsockopt: connection refused
9m 9m 1 {kubelet kubernetes-minion-group-54hw} Warning FailedSync Error syncing pod, skipping: network is not ready: [Kubenet does not have netConfig. This is most likely due to lack of PodCIDR]
9m 9m 3 {kubelet kubernetes-minion-group-54hw} Warning FailedSync Error syncing pod, skipping: network is not ready: [Kubenet does not have netConfig. This is most likely due to lack of PodCIDR]
6m 14s 4 {kubelet kubernetes-minion-group-54hw} Warning FailedMount Unable to mount volumes for pod "web-1_default(270e4f19-b5a4-11e6-8922-42010a800002)": timeout expired waiting for volumes to attach/mount for pod "web-1"/"default". list of unattached/unmounted volumes=[www]
6m 14s 4 {kubelet kubernetes-minion-group-54hw} Warning FailedSync Error syncing pod, skipping: timeout expired waiting for volumes to attach/mount for pod "web-1"/"default". list of unattached/unmounted volumes=[www]

What you expected to happen:

Upon Node upgrade, Pods should either be terminated (so a controller can recreate them) or they should come back to Running on their own.

How to reproduce it (as minimally and precisely as possible):

Start a v1.4 cluster on GCE.

git checkout release-1.4
git pull upstream release-1.4
make clean quick-release
cluster/kube-up.sh

Create the following example PetSet and wait for all Pods to be Running.

kubectl create -f petset.yaml

# A headless service to create DNS records
apiVersion: v1
kind: Service
metadata:
name: nginx
labels:
app: nginx
spec:
ports:
- port: 80
name: web
# *.nginx.default.svc.cluster.local
clusterIP: None
selector:
app: nginx
---
apiVersion: apps/v1alpha1
kind: PetSet
metadata:
name: web
spec:
serviceName: "nginx"
replicas: 5
template:
metadata:
labels:
app: nginx
annotations:
pod.alpha.kubernetes.io/initialized: "true"
spec:
containers:
- name: nginx
image: gcr.io/google_containers/nginx-slim:0.8
ports:
- containerPort: 80
name: web
volumeMounts:
- name: www
mountPath: /usr/share/nginx/html
volumeClaimTemplates:
- metadata:
name: www
annotations:
volume.alpha.kubernetes.io/storage-class: anything
spec:
accessModes: [ "ReadWriteOnce" ]
resources:
requests:
storage: 1Gi

Delete PetSet without cascading (keep the Pods).

kubectl delete petset web --cascade=false

Upgrade master and nodes to v1.5.0-beta.2 with cluster/gce/upgrade.sh.

git checkout release-1.5
git pull upstream release-1.5
cluster/gce/upgrade.sh v1.5.0-beta.2

Check for Pods stuck in ContainerCreating.

NAME READY STATUS RESTARTS AGE
web-0 0/1 ContainerCreating 0 42m
web-4 0/1 ContainerCreating 0 42m

Anything else do we need to know:

After the above steps, some Pods may end up terminating. It seems to be random which ones terminate and which ones get stuck in ContainerCreating.

Manually deleting and recreating all the stuck Pods after the Nodes are upgraded allows them to correctly attach to the preexisting PVCs and become Running.