Repository: Azure/WALinuxAgent
Pull request #3361 (2025-04-29 18:47:13)

clean old agent cgroup setup
<!-- DO NOT DELETE THIS TEMPLATE -->

## Description

fixing https://github.com/Azure/WALinuxAgent/issues/3355 https://github.com/actions/runner-images/issues/11790

Issue # <!-- if any -->
<!--
Please add an informative description that covers that changes made by the pull request. 
This checklist is used to make sure that common issues in a pull request are addressed.
This will expedite the process of getting your pull request merged and avoid extra work on your part to fix issues discovered during the review process.
-->

---

### PR information
- [ ] Ensure development PR is based on the `develop` branch.
- [ ] The title of the PR is clear and informative.
- [ ] There are a small number of commits, each of which has an informative message. This means that previously merged commits do not appear in the history of the PR. For information on cleaning up the commits in your pull request, [see this page](https://github.com/Azure/azure-powershell/blob/master/documentation/development-docs/cleaning-up-commits.md).
- [ ] If applicable, the PR references the bug/issue that it fixes in the description.
- [ ] New Unit tests were added for the changes made

### Quality of Code and Contribution Guidelines
- [ ] I have read the [contribution guidelines](https://github.com/Azure/WALinuxAgent/blob/master/.github/CONTRIBUTING.md).

narrieta: @nagworld9 can you add our github issue to the description?
nagworld9: > @nagworld9 can you add our github issue to the description?

I already added link

Patch:
diff --git a/azurelinuxagent/ga/cgroupconfigurator.py b/azurelinuxagent/ga/cgroupconfigurator.py
index ba3abe93..00411576 100644
--- a/azurelinuxagent/ga/cgroupconfigurator.py
+++ b/azurelinuxagent/ga/cgroupconfigurator.py
@@ -140,9 +140,8 @@ class CGroupConfigurator(object):
                     # If a distro is not supported, attempt to clean up any existing drop in files in case it was
                     # previously supported. It is necessary to cleanup in this scenario in case the OS hits any bugs on
                     # the kernel related to cgroups.
-                    if not self.using_cgroup_v2():
-                        log_cgroup_info("Agent will reset the quotas in case cgroup usage went from enabled to disabled")
-                        self._reset_agent_cgroup_setup()
+                    log_cgroup_info("Agent will reset the quotas in case cgroup usage went from enabled to disabled")
+                    self._reset_agent_cgroup_setup()
                     return
 
                 # We check the agent unit 'Slice' property before setting up azure.slice. This check is done first
@@ -156,6 +155,7 @@ class CGroupConfigurator(object):
                     return
 
                 # Before agent setup, cleanup the old agent setup (drop-in files) since new agent uses different approach(systemctl) to setup cgroups.
+                log_cgroup_info("Cleaning up old agent setup (drop-in files), if any")
                 self._cleanup_old_agent_setup()
 
                 # Notes about slice setup:
@@ -198,6 +198,11 @@ class CGroupConfigurator(object):
                 log_cgroup_info('Agent cgroups enabled: {0}'.format(self._agent_cgroups_enabled))
                 self._initialized = True
 
+                if self._cgroups_api is not None and not self._cgroups_api.can_enforce_cpu():
+                    # If agent cgroups are not enabled or quotas not enabled, reset the quota for the agent unit
+                    log_cgroup_info("Reset CPU quota if agent cgroups were not enabled for enforcement")
+                    self._reset_cpu_quota(systemd.get_agent_unit_name())
+
         def _check_cgroups_supported(self):
             distro_supported = CGroupUtil.distro_supported()
             if not distro_supported:
@@ -484,13 +489,12 @@ class CGroupConfigurator(object):
             NOTE: This resets the quota on the agent's default dropin file; any local overrides on the VM will take precedence
             over this setting.
             """
-            if self._cgroups_api.can_enforce_cpu():
-                log_cgroup_info("Resetting {0}'s CPUQuota".format(unit_name), send_event=False)
-                if CGroupConfigurator._Impl._try_set_cpu_quota(unit_name, "infinity"): # systemd expresses no-quota as infinity, following the same convention
-                    try:
-                        log_cgroup_info('Current CPUQuota: {0}'.format(systemd.get_unit_property(unit_name, "CPUQuotaPerSecUSec")))
-                    except Exception as e:
-                        log_cgroup_warning('Failed to get current CPUQuotaPerSecUSec after reset: {0}'.format(ustr(e)))
+            log_cgroup_info("Resetting {0}'s CPUQuota".format(unit_name), send_event=False)
+            if CGroupConfigurator._Impl._try_set_cpu_quota(unit_name, "infinity"): # systemd expresses no-quota as infinity, following the same convention
+                try:
+                    log_cgroup_info('Current CPUQuota: {0}'.format(systemd.get_unit_property(unit_name, "CPUQuotaPerSecUSec")))
+                except Exception as e:
+                    log_cgroup_warning('Failed to get current CPUQuotaPerSecUSec after reset: {0}'.format(ustr(e)))
 
         # W0238: Unused private member `_Impl.__try_set_cpu_quota(quota)` (unused-private-member)
         @staticmethod


Test patch:
diff --git a/tests/data/init/12-CPUQuota.conf b/tests/data/init/12-CPUQuota.conf
new file mode 100644
index 00000000..0ed39762
--- /dev/null
+++ b/tests/data/init/12-CPUQuota.conf
@@ -0,0 +1,2 @@
+[Service]
+CPUQuota=50%
\ No newline at end of file
diff --git a/tests/ga/test_cgroupconfigurator.py b/tests/ga/test_cgroupconfigurator.py
index ba6f1ce5..61dd6f29 100644
--- a/tests/ga/test_cgroupconfigurator.py
+++ b/tests/ga/test_cgroupconfigurator.py
@@ -100,6 +100,27 @@ class CGroupConfiguratorSystemdTestCase(AgentTestCase):
             with self._get_cgroup_configurator() as configurator:
                 self.assertFalse(configurator.enabled(), "cgroups were enabled")
 
+    def test_should_cleanup_and_reset_cpu_quota_if_agent_cgroups_not_enabled_for_enforcement(self):
+        command_mocks = [MockCommand(r"^systemctl show walinuxagent.service --property CPUQuotaPerSecUSec",
+                                     '''CPUQuotaPerSecUSec=5ms
+                                     ''')]
+        with self._get_cgroup_configurator_v2(initialize=False, mock_commands=command_mocks) as configurator:
+            agent_drop_in_file_cpu_quota = configurator.mocks.get_mapped_path(UnitFilePaths.cpu_quota)
+
+            # The mock creates the drop-in file
+            configurator.mocks.add_data_file(os.path.join(data_dir, 'init', "12-CPUQuota.conf"),
+                                             UnitFilePaths.cpu_quota)
+            self.assertTrue(os.path.exists(agent_drop_in_file_cpu_quota),
+                            "{0} was not created".format(agent_drop_in_file_cpu_quota))
+
+            configurator.initialize()
+
+            self.assertFalse(os.path.exists(agent_drop_in_file_cpu_quota),
+                             "{0} was not cleaned up".format(agent_drop_in_file_cpu_quota))
+            cmd = 'systemctl set-property walinuxagent.service CPUQuota= --runtime'
+            self.assertIn(cmd, configurator.mocks.commands_call_list,
+                          "The command to reset the CPU quota was not called")
+
     def test_initialize_should_start_tracking_the_agent_cgroups(self):
         with self._get_cgroup_configurator() as configurator:
             tracked = CGroupsTelemetry._tracked
diff --git a/tests_e2e/tests/cgroup_v2_enabled/check_enforcement.py b/tests_e2e/tests/cgroup_v2_enabled/check_enforcement.py
index 3c83ab8b..da3feb80 100644
--- a/tests_e2e/tests/cgroup_v2_enabled/check_enforcement.py
+++ b/tests_e2e/tests/cgroup_v2_enabled/check_enforcement.py
@@ -16,7 +16,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-from assertpy import assert_that
 
 from tests_e2e.tests.lib.agent_test import AgentVmTest
 from tests_e2e.tests.lib.logging import log
@@ -25,15 +24,12 @@ from tests_e2e.tests.lib.ssh_client import SshClient
 
 class CheckEnforcement(AgentVmTest):
     """
-    This test verifies that the agent does not enforce limits in cgroupv2 distros.
+    This test verifies that the agent has no CPU limits in cgroupv2 distros.
     """
 
     def run(self):
         ssh_client: SshClient = self._context.create_ssh_client()
-        #
-        # 2025-04-18T21:15:37.336480Z INFO ExtHandler ExtHandler [CGI] Setting azuremonitoragent's CPUQuota to 25%
-        log.info("Parsing agent log(Settings CPUQuota) on the test VM")
-        output = [line for line in ssh_client.run_command("grep -E 'INFO ExtHandler.*(Setting|Resetting).*CPUQuota' /var/log/waagent.log | head").strip().split('\n') if line]
 
-        assert_that(len(output) == 0).described_as("The agent log should not contain matching records of cpu quota being set. Output from the agent log:\n\t\t{0}".format('\n\t\t'.join(output))).is_true()
-        log.info("The agent log indicates that agent does not enforce limits in cgroupv2 distros")
+        log.info("Verifying CPU quota not set in the agent cgroup")
+        self._run_remote_test(ssh_client, "cgroup_v2_enabled-check_quota.py", use_sudo=True)
+        log.info("Verified CPU quota not set in the agent cgroup")
diff --git a/tests_e2e/tests/scripts/cgroup_v2_enabled-check_quota.py b/tests_e2e/tests/scripts/cgroup_v2_enabled-check_quota.py
new file mode 100755
index 00000000..942443a9
--- /dev/null
+++ b/tests_e2e/tests/scripts/cgroup_v2_enabled-check_quota.py
@@ -0,0 +1,46 @@
+#!/usr/bin/env pypy3
+
+# Microsoft Azure Linux Agent
+#
+# Copyright 2018 Microsoft Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Script verifies that no cpu quota in the agent cgroup
+
+from assertpy import fail, assert_that
+
+from tests_e2e.tests.lib.cgroup_helpers import check_agent_quota_disabled, get_agent_cpu_quota, check_log_message
+from tests_e2e.tests.lib.logging import log
+from tests_e2e.tests.lib.retry import retry_if_false
+
+
+def main():
+    """
+    Verifies that no cpu quota in the agent cgroup
+    """
+    log.info("Querying the agent CPU quota")
+    disabled: bool = retry_if_false(check_agent_quota_disabled)
+    if not disabled:
+        fail("The agent failed to disable its CPUQuota when cgroups were not enabled. Current CPUQuota: {0}".format(get_agent_cpu_quota()))
+    log.info("The agent cgroup CPU quota is not set as expected")
+
+    #
+    # 2025-04-18T21:15:37.336480Z INFO ExtHandler ExtHandler [CGI] Setting azuremonitoragent's CPUQuota to 25%
+    log.info("Parsing agent log to look for CPU quota set messages")
+    found: bool = check_log_message("INFO ExtHandler.*Setting.*CPUQuota")
+    assert_that(found).described_as("The agent log should not contain matching records of cpu quota being set").is_false()
+    log.info("The agent log indicates that agent does not enforce limits in cgroupv2 distros")
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file