Issue 1:
Env : Kubernetes in AWS

We tried the new cluster on kubernetes version 1.3.3 on AWS. While group the logs we get into the kibana and searched the index. Surprised that apart from ".kibana" none of the index are there in Elasticsearch.

I check the fluentd pods . It is continuously shows the error like below

2016-07-28T00:17:03.664465814Z 2016-07-28 00:17:03 +0000 [warn]: /var/log/containers/fluentd-elasticsearch-ip-172-20-0-52.us-west-2.compute.internal_kube-system_fluentd-elasticsearch-6a49f2af0d0714085d386fb931edaf541c8c4fa9b1f0da091a4b300e3a11e003.log unreadable. It is excluded and would be examined next time.
2016-07-28T00:17:03.664559691Z 2016-07-28 00:17:03 +0000 [warn]: /var/log/containers/kibana-logging-v1-nxjuy_kube-system_kibana-logging-ab427edc0c0edd25fc55edf646e1537e49cc38ff2eb051317d5fcf170131fc71.log unreadable. It is excluded and would be examined next time.
2016-07-28T00:17:03.664593370Z 2016-07-28 00:17:03 +0000 [warn]: /var/log/containers/kube-dns-v17-69s8t_kube-system_POD-3d1e77939c1fab4db8cf041ba3d1a459b88abeabb277e5440f7796ff0dc78812.log unreadable. It is excluded and would be examined next time.
2016-07-28T00:17:03.664642183Z 2016-07-28 00:17:03 +0000 [warn]: /var/log/containers/kibana-logging-v1-nxjuy_kube-system_POD-87721bfa3323e44fb503551b17c9a94ae47901dd5cf11b17cf804a8200907d1b.log unreadable. It is excluded and would be examined next time.
2016-07-28T00:17:03.664678048Z 2016-07-28 00:17:03 +0000 [warn]: /var/log/containers/elasticsearch-logging-v1-bxa9m_kube-system_POD-ace752fa24ccb772f71200faef350c712f4a08496eb1db5b7d5836a3a409a644.log unreadable. It is excluded and would be examined next time.

I get into the fluentd-pods and get the mount points

root@fluentd-elasticsearch-ip-172-20-0-52:/mnt# df -h
Filesystem Size Used Avail Use% Mounted on
none 30G 4.8G 24G 18% /
tmpfs 3.7G 0 3.7G 0% /dev
tmpfs 3.7G 0 3.7G 0% /sys/fs/cgroup
/dev/xvda1 32G 3.3G 27G 11% /var/log
/dev/mapper/vg--ephemeral-ephemeral 30G 4.8G 24G 18% /etc/hosts
shm 64M 0 64M 0% /dev/shm
shm 64M 0 64M 0% /var/lib/docker/containers/43922ac6ea9ee45b59b51ad29586d94db0e5f59860cd2e185913d5cddd25d370/shm
shm 64M 0 64M 0% /var/lib/docker/containers/7424dae6b429fe1a6bf19cc40c42c236cf6918b6cf61ec44eae8b1b106e31f04/shm
shm 64M 0 64M 0% /var/lib/docker/containers/87721bfa3323e44fb503551b17c9a94ae47901dd5cf11b17cf804a8200907d1b/shm
shm 64M 0 64M 0% /var/lib/docker/containers/b920173a0caf9d79827673b6d4d4173e0dd3277d70acdb6202d970c68930304c/shm
shm 64M 0 64M 0% /var/lib/docker/containers/b97ab84116826a2c941a7272b26c442a724a1287c40030a90d6f182ccb871c51/shm
shm 64M 0 64M 0% /var/lib/docker/containers/3d1e77939c1fab4db8cf041ba3d1a459b88abeabb277e5440f7796ff0dc78812/shm
shm 64M 0 64M 0% /var/lib/docker/containers/6b6f5b8fcbc37826e58d51a3b922547cd918ff85697b69696d134c875878e352/shm
shm 64M 0 64M 0% /var/lib/docker/containers/ace752fa24ccb772f71200faef350c712f4a08496eb1db5b7d5836a3a409a644/shm

oot@fluentd-elasticsearch-ip-172-20-0-52:/mnt# cd /var/log/containers/
root@fluentd-elasticsearch-ip-172-20-0-52:/var/log/containers# ls -l
total 100
lrwxrwxrwx 1 root root 171 Jul 28 07:11 cpeservice-7tpvp_default_POD-f28f2a2d9f1f7575d03533571ea6465b8b508de0c03ebd6a7b29eb1bcea88b5b.log -> /mnt/ephemeral/docker/containers/f28f2a2d9f1f7575d03533571ea6465b8b508de0c03ebd6a7b29eb1bcea88b5b/f28f2a2d9f1f7575d03533571ea6465b8b508de0c03ebd6a7b29eb1bcea88b5b-json.log
lrwxrwxrwx 1 root root 171 Jul 28 07:11 cpeservice-7tpvp_default_cpeservice-da3bcfa83d377d2b846c92bc66a545044a134eb0d3787847d92f576abd276215.log -> /mnt/ephemeral/docker/containers/da3bcfa83d377d2b846c92bc66a545044a134eb0d3787847d92f576abd276215/da3bcfa83d377d2b846c92bc66a545044a134eb0d3787847d92f576abd276215-json.log
lrwxrwxrwx 1 root root 171 Jul 27 09:17 elasticsearch-logging-v1-bxa9m_kube-system_POD-ace752fa24ccb772f71200faef350c712f4a08496eb1db5b7d5836a3a409a644.log -> /mnt/ephemeral/docker/containers/ace752fa24ccb772f71200faef350c712f4a08496eb1db5b7d5836a3a409a644/ace752fa24ccb772f71200faef350c712f4a08496eb1db5b7d5836a3a409a644-json.log
lrwxrwxrwx 1 root root 171 Jul 27 09:19 elasticsearch-logging-v1-bxa9m_kube-system_elasticsearch-logging-46fd3a1293523b7097f6712b27b274376983744dd39bf9167c6f5da8629f3814.log -> /mnt/ephemeral/docker/containers/46fd3a1293523b7097f6712b27b274376983744dd39bf9167c6f5da8629f3814/46fd3a1293523b7097f6712b27b274376983744dd39bf9167c6f5da8629f3814-json.log
lrwxrwxrwx 1 root root 171 Jul 27 09:17 elasticsearch-logging-v1-oc1sv_kube-system_POD-6b6f5b8fcbc37826e58d51a3b922547cd918ff85697b69696d134c875878e352.log -> /mnt/ephemeral/docker/containers/6b6f5b8fcbc37826e58d51a3b922547cd918ff85697b69696d134c875878e352/6b6f5b8fcbc37826e58d51a3b922547cd918ff85697b69696d134c875878e352-json.log
lrwxrwxrwx 1 root root 171 Jul 27 09:19 elasticsearch-logging-v1-oc1sv_kube-system_elasticsearch-logging-f8c218eb4d0184609856e432db14fa809b9ea57e2f393f2a980afc4596e90526.log -> /mnt/ephemeral/docker/containers/f8c218eb4d0184609856e432db14fa809b9ea57e2f393f2a980afc4596e90526/f8c218eb4d0184609856e432db14fa809b9ea57e2f393f2a980afc4596e90526-json.log
lrwxrwxrwx 1 root root 171 Jul 27 09:17 fluentd-elasticsearch-ip-172-20-0-52.us-west-2.compute.internal_kube-system_POD-de76e2c8d8a17a2cd463f84c927cb76d1d755165dfcb4420aef0c23f8d6acc42.log -> /mnt/ephemeral/docker/containers/de76e2c8d8a17a2cd463f84c927cb76d1d755165dfcb4420aef0c23f8d6acc42/de76e2c8d8a17a2cd463f84c927cb76d1d755165dfcb4420aef0c23f8d6acc42-json.log
lrwxrwxrwx 1 root root 171 Jul 27 09:18 fluentd-elasticsearch-ip-172-20-0-52.us-west-2.compute.internal_kube-system_fluentd-elasticsearch-6a49f2af0d0714085d386fb931edaf541c8c4fa9b1f0da091a4b300e3a11e003.log -> /mnt/ephemeral/docker/containers/6a49f2af0d0714085d386fb931edaf541c8c4fa9b1f0da091a4b300e3a11e003/6a49f2af0d0714085d386fb931edaf541c8c4fa9b1f0da091a4b300e3a11e003-json.log
lrwxrwxrwx 1 root root 171 Jul 27 09:17 kibana-logging-v1-nxjuy_kube-system_POD-87721bfa3323e44fb503551b17c9a94ae47901dd5cf11b17cf804a8200907d1b.log -> /mnt/ephemeral/docker/containers/87721bfa3323e44fb503551b17c9a94ae47901dd5cf11b17cf804a8200907d1b/87721bfa3323e44fb503551b17c9a94ae47901dd5cf11b17cf804a8200907d1b-json.log
lrwxrwxrwx 1 root root 171 Jul 27 09:18 kibana-logging-v1-nxjuy_kube-system_kibana-logging-50b7c41b3b3854715adb5711c9e3143c0218fb84583c235761c6bbcb8a01cc3b.log -> /mnt/ephemeral/docker/containers/50b7c41b3b3854715adb5711c9e3143c0218fb84583c235761c6bbcb8a01cc3b/50b7c41b3b3854715adb5711c9e3143c0218fb84583c235761c6bbcb8a01cc3b-json.log
lrwxrwxrwx 1 root root 171 Jul 27 09:19 kibana-logging-v1-nxjuy_kube-system_kibana-logging-a97a3ede7063c17cc60e809688be9a47c17889f0aece6871aff647c779a1461b.log -> /mnt/ephemeral/docker/containers/a97a3ede7063c17cc60e809688be9a47c17889f0aece6871aff647c779a1461b/a97a3ede7063c17cc60e809688be9a47c17889f0aece6871aff647c779a1461b-json.log
lrwxrwxrwx 1 root root 171 Jul 27 09:18 kibana-logging-v1-nxjuy_kube-system_kibana-logging-ab427edc0c0edd25fc55edf646e1537e49cc38ff2eb051317d5fcf170131fc71.log -> /mnt/ephemeral/docker/containers/ab427edc0c0edd25fc55edf646e1537e49cc38ff2eb051317d5fcf170131fc71/ab427edc0c0edd25fc55edf646e1537e49cc38ff2eb051317d5fcf170131fc71-json.log
lrwxrwxrwx 1 root root 171 Jul 27 09:17 kube-dns-v17-69s8t_kube-system_POD-3d1e77939c1fab4db8cf041ba3d1a459b88abeabb277e5440f7796ff0dc78812.log -> /mnt/ephemeral/docker/containers/3d1e77939c1fab4db8cf041ba3d1a459b88abeabb277e5440f7796ff0dc78812/3d1e77939c1fab4db8cf041ba3d1a459b88abeabb277e5440f7796ff0dc78812-json.log
lrwxrwxrwx 1 root root 171 Jul 27 09:19 kube-dns-v17-69s8t_kube-system_dnsmasq-06c56f6f57980e2d54314d520a2a835848ddf75ec771bd3e89cf723f52eb7205.log -> /mnt/ephemeral/docker/containers/06c56f6f57980e2d54314d520a2a835848ddf75ec771bd3e89cf723f52eb7205/06c56f6f57980e2d54314d520a2a835848ddf75ec771bd3e89cf723f52eb7205-json.log
lrwxrwxrwx 1 root root 171 Jul 27 09:19 kube-dns-v17-69s8t_kube-system_healthz-c26dfa9225b75d070ad1d1519a81fe65e6fc04f7c6ebd37518f3f9c3fd5e9b63.log -> /mnt/ephemeral/docker/containers/c26dfa9225b75d070ad1d1519a81fe65e6fc04f7c6ebd37518f3f9c3fd5e9b63/c26dfa9225b75d070ad1d1519a81fe65e6fc04f7c6ebd37518f3f9c3fd5e9b63-json.log
lrwxrwxrwx 1 root root 171 Jul 27 09:18 kube-dns-v17-69s8t_kube-system_kubedns-f4780a45e8afbe7ce9bf7bb10728aa0bf935d969c41e5c4cd7f250db0436e97a.log -> /mnt/ephemeral/docker/containers/f4780a45e8afbe7ce9bf7bb10728aa0bf935d969c41e5c4cd7f250db0436e97a/f4780a45e8afbe7ce9bf7bb10728aa0bf935d969c41e5c4cd7f250db0436e97a-json.log
lrwxrwxrwx 1 root root 171 Jul 27 09:17 kube-proxy-ip-172-20-0-52.us-west-2.compute.internal_kube-system_POD-43922ac6ea9ee45b59b51ad29586d94db0e5f59860cd2e185913d5cddd25d370.log -> /mnt/ephemeral/docker/containers/43922ac6ea9ee45b59b51ad29586d94db0e5f59860cd2e185913d5cddd25d370/43922ac6ea9ee45b59b51ad29586d94db0e5f59860cd2e185913d5cddd25d370-json.log
lrwxrwxrwx 1 root root 171 Jul 27 09:18 kube-proxy-ip-172-20-0-52.us-west-2.compute.internal_kube-system_kube-proxy-036676e9cda81f67c40e1e6465bd48716f4d4ab4c1cdcb7b542aa6ee9ad72255.log -> /mnt/ephemeral/docker/containers/036676e9cda81f67c40e1e6465bd48716f4d4ab4c1cdcb7b542aa6ee9ad72255/036676e9cda81f67c40e1e6465bd48716f4d4ab4c1cdcb7b542aa6ee9ad72255-json.log
lrwxrwxrwx 1 root root 171 Jul 27 09:17 kubernetes-dashboard-v1.1.0-cvbw0_kube-system_POD-b920173a0caf9d79827673b6d4d4173e0dd3277d70acdb6202d970c68930304c.log -> /mnt/ephemeral/docker/containers/b920173a0caf9d79827673b6d4d4173e0dd3277d70acdb6202d970c68930304c/b920173a0caf9d79827673b6d4d4173e0dd3277d70acdb6202d970c68930304c-json.log
lrwxrwxrwx 1 root root 171 Jul 27 09:18 kubernetes-dashboard-v1.1.0-cvbw0_kube-system_kubernetes-dashboard-0b5687b60e4dd031926865f6cfbe4b5874cfb38a85bfd15662297593178ee102.log -> /mnt/ephemeral/docker/containers/0b5687b60e4dd031926865f6cfbe4b5874cfb38a85bfd15662297593178ee102/0b5687b60e4dd031926865f6cfbe4b5874cfb38a85bfd15662297593178ee102-json.log
lrwxrwxrwx 1 root root 171 Jul 27 09:17 monitoring-influxdb-grafana-v3-pj1oj_kube-system_POD-b97ab84116826a2c941a7272b26c442a724a1287c40030a90d6f182ccb871c51.log -> /mnt/ephemeral/docker/containers/b97ab84116826a2c941a7272b26c442a724a1287c40030a90d6f182ccb871c51/b97ab84116826a2c941a7272b26c442a724a1287c40030a90d6f182ccb871c51-json.log
lrwxrwxrwx 1 root root 171 Jul 27 09:19 monitoring-influxdb-grafana-v3-pj1oj_kube-system_grafana-5a086e9362906b23ab5d02d8b1fbf93949fa6ca2429a391cdebd2ab9e013b13f.log -> /mnt/ephemeral/docker/containers/5a086e9362906b23ab5d02d8b1fbf93949fa6ca2429a391cdebd2ab9e013b13f/5a086e9362906b23ab5d02d8b1fbf93949fa6ca2429a391cdebd2ab9e013b13f-json.log
lrwxrwxrwx 1 root root 171 Jul 27 09:18 monitoring-influxdb-grafana-v3-pj1oj_kube-system_influxdb-a688685d00d81b7475226024a2ea714cdd3a9def0788172c3c8ec53df002e32c.log -> /mnt/ephemeral/docker/containers/a688685d00d81b7475226024a2ea714cdd3a9def0788172c3c8ec53df002e32c/a688685d00d81b7475226024a2ea714cdd3a9def0788172c3c8ec53df002e32c-json.log
lrwxrwxrwx 1 root root 171 Jul 28 07:11 userservice-tgcxu_default_POD-9c1849c0381c96614cf706f469069667ed249e3dd652310301e315983bb3ff72.log -> /mnt/ephemeral/docker/containers/9c1849c0381c96614cf706f469069667ed249e3dd652310301e315983bb3ff72/9c1849c0381c96614cf706f469069667ed249e3dd652310301e315983bb3ff72-json.log
lrwxrwxrwx 1 root root 171 Jul 28 07:11 userservice-tgcxu_default_userservice-50f78bda3dd8c946b86a9aca741d2225f3581e47c22ba076b548910ca4298e37.log -> /mnt/ephemeral/docker/containers/50f78bda3dd8c946b86a9aca741d2225f3581e47c22ba076b548910ca4298e37/50f78bda3dd8c946b86a9aca741d2225f3581e47c22ba076b548910ca4298e37-json.log

cat: cpeservice-7tpvp_default_POD-f28f2a2d9f1f7575d03533571ea6465b8b508de0c03ebd6a7b29eb1bcea88b5b.log: No such file or directory

There is something wrong in mount the volumes on AWS. Is there any workaround for this issues .

Issue 2:
It seems that there are potentially two separate issues here:

The inconsistent docker root due to a potential race between kubelet and docker.

The misconfiguration (on AWS) of fluentd to mount /var/lib/docker instead of /mnt/ephemeral/docker.

For the first issue, perhaps the kubelet should fail if it is not able to retrieve the docker root from docker info?

An alternative solution would be to mount our volume for AWS to /var/lib/docker instead of /mnt/ephemeral/docker. This would solve both issues. But, it seems that the kubelet is providing the ability to dynamically locate the docker root for some reason.

Details:

I'm having intermittent issues with fluentd pushing logs to elasticsearch. It seems that fluentd is unable to see the log files, even though it is able list the containers directory. This is happening on an AWS cluster.

From inside the fluentd I see the following:

root@fluentd-elasticsearch-ip-172-20-0-194:/# cat /varlog/containers/vapor-web-v1-x894w_vapor-alpha_nginx-521e74369e30312a32806b018e62ef81d6a606dd87c0772aaacc2461ae84e491.log
cat: /varlog/containers/vapor-web-v1-x894w_vapor-alpha_nginx-521e74369e30312a32806b018e62ef81d6a606dd87c0772aaacc2461ae84e491.log: No such file or directory

On AWS the log file symlinks look like this:

root@fluentd-elasticsearch-ip-172-20-0-194:/# ls -al /varlog/containers/vapor-web-v1-x894w_*
lrwxrwxrwx 1 root root 171 Aug 27 18:03 /varlog/containers/vapor-web-v1-x894w_vapor-alpha_POD-99cd13b12f2b45275f5a3bcb7537d08ce722c48d66e3e21b57ad0563d5f2d8c0.log -> /mnt/ephemeral/docker/containers/99cd13b12f2b45275f5a3bcb7537d08ce722c48d66e3e21b57ad0563d5f2d8c0/99cd13b12f2b45275f5a3bcb7537d08ce722c48d66e3e21b57ad0563d5f2d8c0-json.log
lrwxrwxrwx 1 root root 171 Aug 27 18:03 /varlog/containers/vapor-web-v1-x894w_vapor-alpha_nginx-521e74369e30312a32806b018e62ef81d6a606dd87c0772aaacc2461ae84e491.log -> /mnt/ephemeral/docker/containers/521e74369e30312a32806b018e62ef81d6a606dd87c0772aaacc2461ae84e491/521e74369e30312a32806b018e62ef81d6a606dd87c0772aaacc2461ae84e491-json.log
lrwxrwxrwx 1 root root 171 Aug 27 18:03 /varlog/containers/vapor-web-v1-x894w_vapor-alpha_vapor-unicorn-1e5233b3d6d2e8c92c8534dd61d46b4aadc7766d54423eb17425682d678bad9a.log -> /mnt/ephemeral/docker/containers/1e5233b3d6d2e8c92c8534dd61d46b4aadc7766d54423eb17425682d678bad9a/1e5233b3d6d2e8c92c8534dd61d46b4aadc7766d54423eb17425682d678bad9a-json.log

Fluentd seems to expect that the symlinks will point to /var/lib/docker/containers but on AWS we use /mnt/ephemeral/docker/containers.

From the node I see:

root@ip-172-20-0-194:/home/ubuntu# ls -al /var/log/containers/vapor-web-v1-x894w_vapor-alpha_*
lrwxrwxrwx 1 root root 171 Aug 27 18:03 /var/log/containers/vapor-web-v1-x894w_vapor-alpha_nginx-521e74369e30312a32806b018e62ef81d6a606dd87c0772aaacc2461ae84e491.log -> /mnt/ephemeral/docker/containers/521e74369e30312a32806b018e62ef81d6a606dd87c0772aaacc2461ae84e491/521e74369e30312a32806b018e62ef81d6a606dd87c0772aaacc2461ae84e491-json.log
lrwxrwxrwx 1 root root 171 Aug 27 18:03 /var/log/containers/vapor-web-v1-x894w_vapor-alpha_POD-99cd13b12f2b45275f5a3bcb7537d08ce722c48d66e3e21b57ad0563d5f2d8c0.log -> /mnt/ephemeral/docker/containers/99cd13b12f2b45275f5a3bcb7537d08ce722c48d66e3e21b57ad0563d5f2d8c0/99cd13b12f2b45275f5a3bcb7537d08ce722c48d66e3e21b57ad0563d5f2d8c0-json.log
lrwxrwxrwx 1 root root 171 Aug 27 18:03 /var/log/containers/vapor-web-v1-x894w_vapor-alpha_vapor-unicorn-1e5233b3d6d2e8c92c8534dd61d46b4aadc7766d54423eb17425682d678bad9a.log -> /mnt/ephemeral/docker/containers/1e5233b3d6d2e8c92c8534dd61d46b4aadc7766d54423eb17425682d678bad9a/1e5233b3d6d2e8c92c8534dd61d46b4aadc7766d54423eb17425682d678bad9a-json.log

root@ip-172-20-0-194:/home/ubuntu# cat /mnt/ephemeral/docker/containers/521e74369e30312a32806b018e62ef81d6a606dd87c0772aaacc2461ae84e491/521e74369e30312a32806b018e62ef81d6a606dd87c0772aaacc2461ae84e491-json.log | tail -n 5
{"log":"172.20.0.70 - - [28/Aug/2015:17:29:26 +0000] \"GET / HTTP/1.1\" 302 116 \"-\" \"Pingdom.com_bot_version_1.4_(http://www.pingdom.com/)\"\n","stream":"stdout","time":"2015-08-28T17:29:26.242591558Z"}
{"log":"172.20.0.70 - - [28/Aug/2015:17:29:26 +0000] \"GET /users/sign_in HTTP/1.1\" 200 1954 \"-\" \"Pingdom.com_bot_version_1.4_(http://www.pingdom.com/)\"\n","stream":"stdout","time":"2015-08-28T17:29:26.265429648Z"}
{"log":"2015/08/28 17:29:26 [info] 5#5: *58423 client 172.20.0.70 closed keepalive connection\n","stream":"stdout","time":"2015-08-28T17:29:26.271201715Z"}
{"log":"2015/08/28 17:29:38 [info] 5#5: *58426 client closed connection while waiting for request, client: 172.20.0.171, server: 0.0.0.0:80\n","stream":"stdout","time":"2015-08-28T17:29:38.549273837Z"}
{"log":"2015/08/28 17:29:46 [info] 5#5: *58427 client closed connection while waiting for request, client: 172.20.0.33, server: 0.0.0.0:80\n","stream":"stdout","time":"2015-08-28T17:29:46.549263618Z"}