Repository: cloud-custodian/cloud-custodian

Issue:
c7n_kube - tests spin up threads without closing them
which causes test runner to hang if executing without xdist


Discussion:
ie run `pytest tools/c7n_kube` will hang.
also seems to orphan directories in the cwd, re
```
test-validator/         
     test-admission/         
   test-admission-pod/    
```

Resolving patch:
diff --git a/tools/c7n_kube/c7n_kube/server.py b/tools/c7n_kube/c7n_kube/server.py
--- a/tools/c7n_kube/c7n_kube/server.py
+++ b/tools/c7n_kube/c7n_kube/server.py
@@ -1,9 +1,10 @@
 # Copyright The Cloud Custodian Authors.
 # SPDX-License-Identifier: Apache-2.0
 import base64
+import http.server
 import json
 import os
-import http.server
+import tempfile
 
 from c7n.config import Config
 from c7n.loader import DirectoryLoader
@@ -25,7 +26,8 @@ class AdmissionControllerServer(http.server.HTTPServer):
     def __init__(self, policy_dir, on_exception="warn", *args, **kwargs):
         self.policy_dir = policy_dir
         self.on_exception = on_exception
-        self.directory_loader = DirectoryLoader(Config.empty())
+        temp_dir = tempfile.TemporaryDirectory()
+        self.directory_loader = DirectoryLoader(Config.empty(output_dir=temp_dir.name))
         policy_collection = self.directory_loader.load_directory(os.path.abspath(self.policy_dir))
         self.policy_collection = policy_collection.filter(modes=["k8s-admission"])
         log.info(f"Loaded {len(self.policy_collection)} policies")


Test patch:
diff --git a/tools/c7n_kube/tests/test_k8s_server.py b/tools/c7n_kube/tests/test_k8s_server.py
--- a/tools/c7n_kube/tests/test_k8s_server.py
+++ b/tools/c7n_kube/tests/test_k8s_server.py
@@ -6,6 +6,8 @@
 import threading
 import time
 
+from contextlib import contextmanager
+
 import requests
 
 from unittest.mock import patch, MagicMock
@@ -15,7 +17,7 @@
 from common_kube import KubeTest
 
 
-class TestAdmissionControllerServer(AdmissionControllerServer):
+class MockAdmissionControllerServer(AdmissionControllerServer):
     def __init__(self, bind_and_activate=False, *args, **kwargs):
         super().__init__(bind_and_activate=bind_and_activate, *args, **kwargs)
 
@@ -24,25 +26,30 @@ class TestServer(KubeTest):
     def find_port(self):
         sock = socket.socket()
         sock.bind(("", 0))
-        _, port = sock.getsockname()
+        ((server, port)) = sock.getsockname()
         return port
 
-    def _server(self, policies, on_exception="warn"):
+    @contextmanager
+    def _server(self, policies, on_exception="warn", timeout=1):
         port = self.find_port()
         with tempfile.TemporaryDirectory() as temp_dir:
             with open(f"{temp_dir}/policy.yaml", "w+") as f:
                 json.dump(policies, f)
-            server = TestAdmissionControllerServer(
+            server = MockAdmissionControllerServer(
                 server_address=("localhost", port),
                 RequestHandlerClass=AdmissionControllerHandler,
                 policy_dir=temp_dir,
                 bind_and_activate=True,
                 on_exception=on_exception,
             )
-            server_thread = threading.Thread(target=server.handle_request)
+            server_thread = threading.Thread(target=server.serve_forever)
             server_thread.start()
-            time.sleep(1)
-        return server, port
+            time.sleep(timeout)
+        try:
+            yield server, port
+        finally:
+            server.shutdown()
+            server_thread.join(timeout=timeout)
 
     def test_server_load_non_k8s_policies(self):
         with tempfile.TemporaryDirectory() as temp_dir:
@@ -52,7 +59,7 @@ def test_server_load_non_k8s_policies(self):
                 json.dump({"policies": [{"name": "test2", "resource": "ec2"}]}, f)
             with open(f"{temp_dir}/policy3.yaml", "w+") as f:
                 json.dump({"policies": [{"name": "test3", "resource": "ebs"}]}, f)
-            server = TestAdmissionControllerServer(
+            server = MockAdmissionControllerServer(
                 server_address=("localhost", 8080),
                 RequestHandlerClass=AdmissionControllerHandler,
                 policy_dir=temp_dir,
@@ -68,7 +75,7 @@ def test_server_load_k8s_policies_no_mode(self):
                 json.dump({"policies": [{"name": "test2", "resource": "k8s.deployment"}]}, f)
             with open(f"{temp_dir}/policy3.yaml", "w+") as f:
                 json.dump({"policies": [{"name": "test3", "resource": "k8s.service"}]}, f)
-            server = TestAdmissionControllerServer(
+            server = MockAdmissionControllerServer(
                 server_address=("localhost", 8082),
                 RequestHandlerClass=AdmissionControllerHandler,
                 policy_dir=temp_dir,
@@ -112,7 +119,7 @@ def test_server_load_k8s_policies_proper_mode(self):
                 )
             with open(f"{temp_dir}/policy3.yaml", "w+") as f:
                 json.dump({"policies": [{"name": "test3", "resource": "k8s.service"}]}, f)
-            server = TestAdmissionControllerServer(
+            server = MockAdmissionControllerServer(
                 server_address=("localhost", 8080),
                 RequestHandlerClass=AdmissionControllerHandler,
                 policy_dir=temp_dir,
@@ -123,10 +130,10 @@ def test_server_load_k8s_policies_proper_mode(self):
 
     def test_server_handle_get_empty_policies(self):
         policies = {"policies": []}
-        server, port = self._server(policies)
-        res = requests.get(f"http://localhost:{port}")
-        self.assertEqual(res.json(), [])
-        self.assertEqual(res.status_code, 200)
+        with self._server(policies) as ((server, port)):
+            res = requests.get(f"http://localhost:{port}")
+            self.assertEqual(res.json(), [])
+            self.assertEqual(res.status_code, 200)
 
     def test_server_handle_get_policies(self):
         policies = {
@@ -144,30 +151,30 @@ def test_server_handle_get_policies(self):
                 }
             ]
         }
-        server, port = self._server(policies)
-        res = requests.get(f"http://localhost:{port}")
-        self.assertEqual(res.json(), policies["policies"])
-        self.assertEqual(res.status_code, 200)
+        with self._server(policies) as ((server, port)):
+            res = requests.get(f"http://localhost:{port}")
+            self.assertEqual(res.json(), policies["policies"])
+            self.assertEqual(res.status_code, 200)
 
     def test_server_handle_post_no_policies(self):
         policies = {"policies": []}
-        server, port = self._server(policies)
-        event = self.get_event("create_pod")
-        res = requests.post(f"http://localhost:{port}", json=event)
-        self.assertEqual(res.status_code, 200)
-        self.assertEqual(
-            {
-                "apiVersion": "admission.k8s.io/v1",
-                "kind": "AdmissionReview",
-                "response": {
-                    "allowed": True,
-                    "uid": "662c3df2-ade6-4165-b395-770857bc17b7",
-                    "warnings": [],
-                    "status": {"code": 200, "message": "OK"},
+        with self._server(policies) as ((server, port)):
+            event = self.get_event("create_pod")
+            res = requests.post(f"http://localhost:{port}", json=event)
+            self.assertEqual(res.status_code, 200)
+            self.assertEqual(
+                {
+                    "apiVersion": "admission.k8s.io/v1",
+                    "kind": "AdmissionReview",
+                    "response": {
+                        "allowed": True,
+                        "uid": "662c3df2-ade6-4165-b395-770857bc17b7",
+                        "warnings": [],
+                        "status": {"code": 200, "message": "OK"},
+                    },
                 },
-            },
-            res.json(),
-        )
+                res.json(),
+            )
 
     def test_server_handle_post_policies_deny_on_match(self):
         policies = {
@@ -185,11 +192,11 @@ def test_server_handle_post_policies_deny_on_match(self):
                 }
             ]
         }
-        server, port = self._server(policies)
-        event = self.get_event("create_pod")
-        res = requests.post(f"http://localhost:{port}", json=event)
-        self.assertEqual(res.status_code, 200)
-        self.assertFalse(res.json()["response"]["allowed"])
+        with self._server(policies) as ((server, port)):
+            event = self.get_event("create_pod")
+            res = requests.post(f"http://localhost:{port}", json=event)
+            self.assertEqual(res.status_code, 200)
+            self.assertFalse(res.json()["response"]["allowed"])
 
     def test_server_handle_post_policies_allow_on_match(self):
         policies = {
@@ -207,11 +214,11 @@ def test_server_handle_post_policies_allow_on_match(self):
                 }
             ]
         }
-        server, port = self._server(policies)
-        event = self.get_event("create_pod")
-        res = requests.post(f"http://localhost:{port}", json=event)
-        self.assertEqual(res.status_code, 200)
-        self.assertTrue(res.json()["response"]["allowed"])
+        with self._server(policies) as ((server, port)):
+            event = self.get_event("create_pod")
+            res = requests.post(f"http://localhost:{port}", json=event)
+            self.assertEqual(res.status_code, 200)
+            self.assertTrue(res.json()["response"]["allowed"])
 
     def test_server_handle_post_policies_deny_on_match_multiple(self):
         policies = {
@@ -254,15 +261,20 @@ def test_server_handle_post_policies_deny_on_match_multiple(self):
                 },
             ]
         }
-        server, port = self._server(policies)
-        event = self.get_event("create_pod")
-        res = requests.post(f"http://localhost:{port}", json=event)
-        self.assertEqual(res.status_code, 200)
-        self.assertFalse(res.json()["response"]["allowed"])
-        failures = json.loads(res.json()["response"]["status"]["message"].split(":", 1)[-1])
-        self.assertEqual(len(failures), 2)
-        self.assertEqual(failures[0], {"name": "test-admission", "description": "description 1"})
-        self.assertEqual(failures[1], {"name": "test-admission-2", "description": "description 2"})
+        with self._server(policies) as (server, port):
+            event = self.get_event("create_pod")
+            res = requests.post(f"http://localhost:{port}", json=event)
+            self.assertEqual(res.status_code, 200)
+            self.assertFalse(res.json()["response"]["allowed"])
+            failures = json.loads(res.json()["response"]["status"]["message"].split(":", 1)[-1])
+            self.assertEqual(len(failures), 2)
+            self.assertEqual(
+                failures[0], {"name": "test-admission", "description": "description 1"}
+            )
+            self.assertEqual(
+                failures[1],
+                {"name": "test-admission-2", "description": "description 2"},
+            )
 
     def test_server_onmatch_warn(self):
         policies = {
@@ -281,36 +293,41 @@ def test_server_onmatch_warn(self):
                 },
             ]
         }
-        server, port = self._server(policies)
-        event = self.get_event("create_pod")
-        res = requests.post(f"http://localhost:{port}", json=event)
-        self.assertEqual(res.status_code, 200)
-        self.assertTrue(res.json()["response"]["allowed"])
-        self.assertEqual(
-            res.json()["response"]["warnings"],
-            ["test-admission-pod:description deployment"],
-        )
+        with self._server(policies) as (server, port):
+            event = self.get_event("create_pod")
+            res = requests.post(f"http://localhost:{port}", json=event)
+            self.assertEqual(res.status_code, 200)
+            self.assertTrue(res.json()["response"]["allowed"])
+            self.assertEqual(
+                res.json()["response"]["warnings"],
+                ["test-admission-pod:description deployment"],
+            )
 
     def test_server_init(self):
         policies = {"policies": []}
         with patch("c7n_kube.server.AdmissionControllerServer") as patched:
-            server, port = self._server(policies)
-            init(host="0.0.0.0", port=port, policy_dir="policies", serve_forever=False)
-            patched.assert_called_once()
-            patched.assert_called_with(
-                server_address=("0.0.0.0", port),
-                RequestHandlerClass=AdmissionControllerHandler,
-                policy_dir="policies",
-                on_exception="warn",
-            )
-            patched.return_value.serve_forever.assert_called_once()
+            with self._server(policies) as (server, port):
+                init(
+                    host="0.0.0.0",
+                    port=port,
+                    policy_dir="policies",
+                    serve_forever=False,
+                )
+                patched.assert_called_once()
+                patched.assert_called_with(
+                    server_address=("0.0.0.0", port),
+                    RequestHandlerClass=AdmissionControllerHandler,
+                    policy_dir="policies",
+                    on_exception="warn",
+                )
+                patched.return_value.serve_forever.assert_called_once()
 
     def test_server_bad_post(self):
         policies = {"policies": []}
-        server, port = self._server(policies)
-        res = requests.post(f"http://localhost:{port}", data="bad data")
-        self.assertEqual(res.status_code, 400)
-        self.assertEqual(res.json(), {"error": "Expecting value: line 1 column 1 (char 0)"})
+        with self._server(policies) as (server, port):
+            res = requests.post(f"http://localhost:{port}", data="bad data")
+            self.assertEqual(res.status_code, 400)
+            self.assertEqual(res.json(), {"error": "Expecting value: line 1 column 1 (char 0)"})
 
     def test_server_response_with_patch(self):
         policies = {
@@ -337,17 +354,17 @@ def test_server_response_with_patch(self):
                 },
             ]
         }
-        server, port = self._server(policies)
-        event = self.get_event("create_pod")
-        res = requests.post(f"http://localhost:{port}", json=event)
-        self.assertEqual(res.status_code, 200)
-        self.assertTrue(res.json()["response"]["allowed"])
-        self.assertEqual(res.json()["response"]["warnings"], ["label-pod:warning goes here"])
-        self.assertEqual(res.json()["response"]["patchType"], "JSONPatch")
-        self.assertEqual(
-            res.json()["response"]["patch"],
-            "W3sib3AiOiAicmVtb3ZlIiwgInBhdGgiOiAiL21ldGFkYXRhL2xhYmVscy90ZXN0In0sIHsib3AiOiAiYWRkIiwgInBhdGgiOiAiL21ldGFkYXRhL2xhYmVscy9mb28iLCAidmFsdWUiOiAiYmFyIn0sIHsib3AiOiAicmVwbGFjZSIsICJwYXRoIjogIi9tZXRhZGF0YS9sYWJlbHMvcm9sZSIsICJ2YWx1ZSI6ICJkaWZmZXJlbnQgcm9sZSJ9XQ==",
-        )  # noqa
+        with self._server(policies) as (server, port):
+            event = self.get_event("create_pod")
+            res = requests.post(f"http://localhost:{port}", json=event)
+            self.assertEqual(res.status_code, 200)
+            self.assertTrue(res.json()["response"]["allowed"])
+            self.assertEqual(res.json()["response"]["warnings"], ["label-pod:warning goes here"])
+            self.assertEqual(res.json()["response"]["patchType"], "JSONPatch")
+            self.assertEqual(
+                res.json()["response"]["patch"],
+                "W3sib3AiOiAicmVtb3ZlIiwgInBhdGgiOiAiL21ldGFkYXRhL2xhYmVscy90ZXN0In0sIHsib3AiOiAiYWRkIiwgInBhdGgiOiAiL21ldGFkYXRhL2xhYmVscy9mb28iLCAidmFsdWUiOiAiYmFyIn0sIHsib3AiOiAicmVwbGFjZSIsICJwYXRoIjogIi9tZXRhZGF0YS9sYWJlbHMvcm9sZSIsICJ2YWx1ZSI6ICJkaWZmZXJlbnQgcm9sZSJ9XQ==",
+            )  # noqa
 
     def test_server_bad_policy_execution_warn(self):
         policies = {
@@ -366,32 +383,31 @@ def test_server_bad_policy_execution_warn(self):
                 },
             ]
         }
-        server, port = self._server(policies)
-
-        server.policy_collection = MagicMock()
-        server.policy_collection.policies = []
-
-        mock_policy_1 = MagicMock()
-        mock_policy_1.name = "test-admission-pod"
-        mock_policy_1.push.side_effect = Exception("foo")
-
-        mock_policy_2 = MagicMock()
-        mock_policy_2.name = "test-admission-pod-2"
-        mock_policy_2.push.side_effect = Exception("bar")
-
-        server.policy_collection.policies.append(mock_policy_1)
-        server.policy_collection.policies.append(mock_policy_2)
-
-        event = self.get_event("create_pod")
-        res = requests.post(f"http://localhost:{port}", json=event)
-        self.assertEqual(res.status_code, 200)
-        self.assertEqual(
-            res.json()["response"]["warnings"],
-            [
-                "test-admission-pod:Error in executing policy: foo",
-                "test-admission-pod-2:Error in executing policy: bar",
-            ],
-        )
+        with self._server(policies) as (server, port):
+            server.policy_collection = MagicMock()
+            server.policy_collection.policies = []
+
+            mock_policy_1 = MagicMock()
+            mock_policy_1.name = "test-admission-pod"
+            mock_policy_1.push.side_effect = Exception("foo")
+
+            mock_policy_2 = MagicMock()
+            mock_policy_2.name = "test-admission-pod-2"
+            mock_policy_2.push.side_effect = Exception("bar")
+
+            server.policy_collection.policies.append(mock_policy_1)
+            server.policy_collection.policies.append(mock_policy_2)
+
+            event = self.get_event("create_pod")
+            res = requests.post(f"http://localhost:{port}", json=event)
+            self.assertEqual(res.status_code, 200)
+            self.assertEqual(
+                res.json()["response"]["warnings"],
+                [
+                    "test-admission-pod:Error in executing policy: foo",
+                    "test-admission-pod-2:Error in executing policy: bar",
+                ],
+            )
 
     def test_server_bad_policy_execution_deny(self):
         policies = {
@@ -410,37 +426,36 @@ def test_server_bad_policy_execution_deny(self):
                 },
             ]
         }
-        server, port = self._server(policies, on_exception="deny")
-
-        server.policy_collection = MagicMock()
-        server.policy_collection.policies = []
-
-        mock_policy_1 = MagicMock()
-        mock_policy_1.name = "test-admission-pod"
-        mock_policy_1.push.side_effect = Exception("foo")
-
-        mock_policy_2 = MagicMock()
-        mock_policy_2.name = "test-admission-pod-2"
-        mock_policy_2.push.side_effect = Exception("bar")
-
-        server.policy_collection.policies.append(mock_policy_1)
-        server.policy_collection.policies.append(mock_policy_2)
-
-        event = self.get_event("create_pod")
-        res = requests.post(f"http://localhost:{port}", json=event)
-        self.assertEqual(res.status_code, 200)
-        self.assertFalse(res.json()["response"]["allowed"])
-        failures = json.loads(res.json()["response"]["status"]["message"].split(":", 1)[-1])
-        self.assertEqual(
-            failures,
-            [
-                {
-                    "name": "test-admission-pod",
-                    "description": "Error in executing policy: foo",
-                },
-                {
-                    "name": "test-admission-pod-2",
-                    "description": "Error in executing policy: bar",
-                },
-            ],
-        )
+        with self._server(policies, on_exception="deny") as (server, port):
+            server.policy_collection = MagicMock()
+            server.policy_collection.policies = []
+
+            mock_policy_1 = MagicMock()
+            mock_policy_1.name = "test-admission-pod"
+            mock_policy_1.push.side_effect = Exception("foo")
+
+            mock_policy_2 = MagicMock()
+            mock_policy_2.name = "test-admission-pod-2"
+            mock_policy_2.push.side_effect = Exception("bar")
+
+            server.policy_collection.policies.append(mock_policy_1)
+            server.policy_collection.policies.append(mock_policy_2)
+
+            event = self.get_event("create_pod")
+            res = requests.post(f"http://localhost:{port}", json=event)
+            self.assertEqual(res.status_code, 200)
+            self.assertFalse(res.json()["response"]["allowed"])
+            failures = json.loads(res.json()["response"]["status"]["message"].split(":", 1)[-1])
+            self.assertEqual(
+                failures,
+                [
+                    {
+                        "name": "test-admission-pod",
+                        "description": "Error in executing policy: foo",
+                    },
+                    {
+                        "name": "test-admission-pod-2",
+                        "description": "Error in executing policy: bar",
+                    },
+                ],
+            )