Repository: apache/airflow

Issue:
KubernetesPodOperator stops tailing logs in long living task
KubernetesPodOperator stops tailing logs after 4 hours in long living task. When log tailing is interrupted, task log is flooded by `Event: pod had an event of type Running` messages. Messages are generated every 2 seconds.  
Log tail interruption occurs every time after 4 hours. Interruption can be related to timeout on AKS control plane. Nevertheless, KubernetesPodOperator should continue to tail logs after interruption. I'm going to submit fix. 

**Apache Airflow version**: 1.10.12
**Kubernetes version (if you are using kubernetes)** (use `kubectl version`): 1.17.7
**Environment**: Azure Kubernetes Service

```
[2020-09-23 13:35:43,077] {pod_launcher.py:156} INFO - b'\x1b[2K\ron 600: percentage: 2.00\n'
[2020-09-23 13:51:01,477] {pod_launcher.py:156} INFO - b'\x1b[2K\ron 900: percentage: 3.00\n'
[2020-09-23 14:05:50,749] {pod_launcher.py:156} INFO - b'\x1b[2K\ron 1200: percentage: 4.00\n'
[2020-09-23 14:20:42,323] {pod_launcher.py:156} INFO - b'\x1b[2K\ron 1500: percentage: 5.00\n'
[2020-09-23 14:35:42,412] {pod_launcher.py:156} INFO - b'\x1b[2K\ron 1800: percentage: 6.00\n'
[2020-09-23 14:50:52,880] {pod_launcher.py:156} INFO - b'\x1b[2K\ron 2100: percentage: 7.00\n'
[2020-09-23 15:06:05,138] {pod_launcher.py:156} INFO - b'\x1b[2K\ron 2400: percentage: 8.00\n'
[2020-09-23 15:21:24,700] {pod_launcher.py:156} INFO - b'\x1b[2K\ron 2700: percentage: 9.00\n'
[2020-09-23 15:36:52,563] {pod_launcher.py:156} INFO - b'\x1b[2K\ron 3000: percentage: 10.00\n'
[2020-09-23 15:52:32,510] {pod_launcher.py:156} INFO - b'\x1b[2K\ron 3300: percentage: 11.00\n'
[2020-09-23 16:08:53,489] {pod_launcher.py:156} INFO - b'\x1b[2K\ron 3600: percentage: 12.00\n'
[2020-09-23 16:24:13,732] {pod_launcher.py:156} INFO - b'\x1b[2K\ron 3900: percentage: 13.00\n'
[2020-09-23 16:24:14,633] {pod_launcher.py:156} INFO - b'\x1b[2K\ron 3900: percentage: 13.00\n'
[2020-09-23 16:24:14,690] {pod_launcher.py:156} INFO - b'\x1b[2K\ron 3900: percentage: 13.00\n'
[2020-09-23 16:24:14,883] {pod_launcher.py:156} INFO - b'\x1b[2K\ron 3900: percentage: 13.00\n'
[2020-09-23 16:39:40,816] {pod_launcher.py:156} INFO - b'\x1b[2K\ron 4200: percentage: 14.00\n'
[2020-09-23 16:55:21,781] {pod_launcher.py:156} INFO - b'\x1b[2K\ron 4500: percentage: 15.00\n'
[2020-09-23 17:10:58,901] {pod_launcher.py:171} INFO - Event: some-pod had an event of type Running
[2020-09-23 17:10:58,927] {pod_launcher.py:166} INFO - Pod some-pod has state running
[2020-09-23 17:11:01,148] {pod_launcher.py:171} INFO - Event: some-pod had an event of type Running
[2020-09-23 17:11:01,154] {pod_launcher.py:166} INFO - Pod some-pod has state running
[2020-09-23 17:11:01,154] {pod_launcher.py:166} INFO - Pod some-pod has state running
...
the same message for next 10 hours every 2 seconds
```


Discussion:

Resolving patch:
diff --git a/airflow/kubernetes/pod_launcher.py b/airflow/kubernetes/pod_launcher.py
--- a/airflow/kubernetes/pod_launcher.py
+++ b/airflow/kubernetes/pod_launcher.py
@@ -16,10 +16,12 @@
 # under the License.
 """Launches PODs"""
 import json
+import math
 import time
 from datetime import datetime as dt
 from typing import Optional, Tuple
 
+import pendulum
 import tenacity
 from kubernetes import client, watch
 from kubernetes.client.models.v1_pod import V1Pod
@@ -124,9 +126,23 @@ def monitor_pod(self, pod: V1Pod, get_logs: bool) -> Tuple[State, Optional[str]]
         :return:  Tuple[State, Optional[str]]
         """
         if get_logs:
-            logs = self.read_pod_logs(pod)
-            for line in logs:
-                self.log.info(line)
+            read_logs_since_sec = None
+            last_log_time = None
+            while True:
+                logs = self.read_pod_logs(pod, timestamps=True, since_seconds=read_logs_since_sec)
+                for line in logs:
+                    timestamp, message = self.parse_log_line(line.decode('utf-8'))
+                    last_log_time = pendulum.parse(timestamp)
+                    self.log.info(message)
+                time.sleep(1)
+
+                if not self.base_container_is_running(pod):
+                    break
+
+                self.log.warning('Pod %s log read interrupted', pod.metadata.name)
+                delta = pendulum.now() - last_log_time
+                # Prefer logs duplication rather than loss
+                read_logs_since_sec = math.ceil(delta.total_seconds())
         result = None
         if self.extract_xcom:
             while self.base_container_is_running(pod):
@@ -140,6 +156,22 @@ def monitor_pod(self, pod: V1Pod, get_logs: bool) -> Tuple[State, Optional[str]]
             time.sleep(2)
         return self._task_status(self.read_pod(pod)), result
 
+    def parse_log_line(self, line: str) -> Tuple[str, str]:
+        """
+        Parse K8s log line and returns the final state
+
+        :param line: k8s log line
+        :type line: str
+        :return: timestamp and log message
+        :rtype: Tuple[str, str]
+        """
+        split_at = line.find(' ')
+        if split_at == -1:
+            raise Exception('Log not in "{{timestamp}} {{log}}" format. Got: {}'.format(line))
+        timestamp = line[:split_at]
+        message = line[split_at + 1:].rstrip()
+        return timestamp, message
+
     def _task_status(self, event):
         self.log.info(
             'Event: %s had an event of type %s',
@@ -171,16 +203,28 @@ def base_container_is_running(self, pod: V1Pod):
         wait=tenacity.wait_exponential(),
         reraise=True
     )
-    def read_pod_logs(self, pod: V1Pod, tail_lines: int = 10):
+    def read_pod_logs(self,
+                      pod: V1Pod,
+                      tail_lines: Optional[int] = None,
+                      timestamps: bool = False,
+                      since_seconds: Optional[int] = None):
         """Reads log from the POD"""
+        additional_kwargs = {}
+        if since_seconds:
+            additional_kwargs['since_seconds'] = since_seconds
+
+        if tail_lines:
+            additional_kwargs['tail_lines'] = tail_lines
+
         try:
             return self._client.read_namespaced_pod_log(
                 name=pod.metadata.name,
                 namespace=pod.metadata.namespace,
                 container='base',
                 follow=True,
-                tail_lines=tail_lines,
-                _preload_content=False
+                timestamps=timestamps,
+                _preload_content=False,
+                **additional_kwargs
             )
         except BaseHTTPError as e:
             raise AirflowException(


Test patch:
diff --git a/kubernetes_tests/test_kubernetes_pod_operator.py b/kubernetes_tests/test_kubernetes_pod_operator.py
--- a/kubernetes_tests/test_kubernetes_pod_operator.py
+++ b/kubernetes_tests/test_kubernetes_pod_operator.py
@@ -404,7 +404,7 @@ def test_volume_mount(self):
             )
             context = create_context(k)
             k.execute(context=context)
-            mock_logger.info.assert_any_call(b"retrieved from mount\n")
+            mock_logger.info.assert_any_call('retrieved from mount')
             actual_pod = self.api_client.sanitize_for_serialization(k.pod)
             self.expected_pod['spec']['containers'][0]['args'] = args
             self.expected_pod['spec']['containers'][0]['volumeMounts'] = [{
diff --git a/tests/kubernetes/test_pod_launcher.py b/tests/kubernetes/test_pod_launcher.py
--- a/tests/kubernetes/test_pod_launcher.py
+++ b/tests/kubernetes/test_pod_launcher.py
@@ -48,17 +48,17 @@ def test_read_pod_logs_retries_successfully(self):
                 _preload_content=False,
                 container='base',
                 follow=True,
+                timestamps=False,
                 name=mock.sentinel.metadata.name,
-                namespace=mock.sentinel.metadata.namespace,
-                tail_lines=10
+                namespace=mock.sentinel.metadata.namespace
             ),
             mock.call(
                 _preload_content=False,
                 container='base',
                 follow=True,
+                timestamps=False,
                 name=mock.sentinel.metadata.name,
-                namespace=mock.sentinel.metadata.namespace,
-                tail_lines=10
+                namespace=mock.sentinel.metadata.namespace
             )
         ])
 
@@ -80,19 +80,39 @@ def test_read_pod_logs_successfully_with_tail_lines(self):
         self.mock_kube_client.read_namespaced_pod_log.side_effect = [
             mock.sentinel.logs
         ]
-        logs = self.pod_launcher.read_pod_logs(mock.sentinel, 100)
+        logs = self.pod_launcher.read_pod_logs(mock.sentinel, tail_lines=100)
         self.assertEqual(mock.sentinel.logs, logs)
         self.mock_kube_client.read_namespaced_pod_log.assert_has_calls([
             mock.call(
                 _preload_content=False,
                 container='base',
                 follow=True,
+                timestamps=False,
                 name=mock.sentinel.metadata.name,
                 namespace=mock.sentinel.metadata.namespace,
                 tail_lines=100
             ),
         ])
 
+    def test_read_pod_logs_successfully_with_since_seconds(self):
+        mock.sentinel.metadata = mock.MagicMock()
+        self.mock_kube_client.read_namespaced_pod_log.side_effect = [
+            mock.sentinel.logs
+        ]
+        logs = self.pod_launcher.read_pod_logs(mock.sentinel, since_seconds=2)
+        self.assertEqual(mock.sentinel.logs, logs)
+        self.mock_kube_client.read_namespaced_pod_log.assert_has_calls([
+            mock.call(
+                _preload_content=False,
+                container='base',
+                follow=True,
+                timestamps=False,
+                name=mock.sentinel.metadata.name,
+                namespace=mock.sentinel.metadata.namespace,
+                since_seconds=2
+            ),
+        ])
+
     def test_read_pod_events_successfully_returns_events(self):
         mock.sentinel.metadata = mock.MagicMock()
         self.mock_kube_client.list_namespaced_event.return_value = mock.sentinel.events
@@ -162,3 +182,15 @@ def test_read_pod_retries_fails(self):
             self.pod_launcher.read_pod,
             mock.sentinel
         )
+
+    def test_parse_log_line(self):
+        timestamp, message = \
+            self.pod_launcher.parse_log_line('2020-10-08T14:16:17.793417674Z Valid message\n')
+
+        self.assertEqual(timestamp, '2020-10-08T14:16:17.793417674Z')
+        self.assertEqual(message, 'Valid message')
+
+        self.assertRaises(
+            Exception,
+            self.pod_launcher.parse_log_line('2020-10-08T14:16:17.793417674ZInvalid message\n'),
+        )