Repository: Azure/WALinuxAgent
Pull request #1325 (2018-08-29 16:44:39)

Enable cgroups by default
<!-- DO NOT DELETE THIS TEMPLATE -->

## Description

Issue #1316 

This is the first commit for #1316, which enables cgroups by default for all distros. It maintains the ability to override distro-specific functionality as part of the OS factory mechanism. 

- move cgroups implementation to the default osutil class
- cleanup the cgroups setup and logging
- update the agent version to 2.2.31.2

---

### PR information
- [x] The title of the PR is clear and informative.
- [x] There are a small number of commits, each of which has an informative message. This means that previously merged commits do not appear in the history of the PR. For information on cleaning up the commits in your pull request, [see this page](https://github.com/Azure/azure-powershell/blob/master/documentation/development-docs/cleaning-up-commits.md).
- [x] Except for special cases involving multiple contributors, the PR is started from a fork of the main repository, not a branch.
- [x] If applicable, the PR references the bug/issue that it fixes in the description.
- [ ] New Unit tests were added for the changes made and Travis.CI is passing.

### Quality of Code and Contribution Guidelines
- [x] I have read the [contribution guidelines](https://github.com/Azure/WALinuxAgent/blob/master/.github/CONTRIBUTING.md).


Patch:
diff --git a/azurelinuxagent/common/cgroups.py b/azurelinuxagent/common/cgroups.py
index b936f4f9..5cd68703 100644
--- a/azurelinuxagent/common/cgroups.py
+++ b/azurelinuxagent/common/cgroups.py
@@ -22,11 +22,11 @@ import time
 from azurelinuxagent.common import logger
 from azurelinuxagent.common.future import ustr
 from azurelinuxagent.common.osutil import get_osutil
+from azurelinuxagent.common.osutil.default import BASE_CGROUPS
 from azurelinuxagent.common.utils import fileutil
 from azurelinuxagent.common.version import AGENT_NAME, CURRENT_VERSION
 
 
-BASE_CGROUPS = '/sys/fs/cgroup'
 WRAPPER_CGROUP_NAME = "Agent+Extensions"
 METRIC_HIERARCHIES = ['cpu', 'memory']
 MEMORY_DEFAULT = -1
@@ -505,7 +505,6 @@ class CGroups(object):
             Add this process to the "agent" cgroup, if required
         Actual collection of metrics from cgroups happens in the -run-exthandlers instance
         """
-        cgroups_enabled = True
         if CGroups.enabled():
             try:
                 CGroups._osutil.mount_cgroups()
@@ -517,21 +516,26 @@ class CGroups(object):
                         cg.add(pid)
                         logger.info("Add daemon process pid {0} to {1} cgroup".format(pid, cg.name))
                     else:
-                        logger.info("Daemon process pid {0} cgroup managed by systemd".format(pid))
-                status = "OK"
+                        logger.info("Add daemon process pid {0} to systemd cgroup".format(pid))
+                status = "ok"
             except CGroupsException as cge:
                 status = cge.msg
-                cgroups_enabled = False
+                CGroups.disable()
+            except Exception as ge:
+                status = ustr(ge)
                 CGroups.disable()
         else:
-            status = "Cgroups not supported by platform"
+            status = "not supported by platform"
+            CGroups.disable()
+
+        logger.info("CGroups: {0}".format(status))
 
         from azurelinuxagent.common.event import add_event, WALAEventOperation
         add_event(
             AGENT_NAME,
             version=CURRENT_VERSION,
             op=WALAEventOperation.InitializeCGroups,
-            is_success=cgroups_enabled,
+            is_success=CGroups.enabled(),
             message=status,
             log_event=False)
 
diff --git a/azurelinuxagent/common/osutil/default.py b/azurelinuxagent/common/osutil/default.py
index b61f37d3..a10e9aa0 100644
--- a/azurelinuxagent/common/osutil/default.py
+++ b/azurelinuxagent/common/osutil/default.py
@@ -95,6 +95,9 @@ IFNAMSIZ = 16
 
 IP_COMMAND_OUTPUT = re.compile('^\d+:\s+(\w+):\s+(.*)$')
 
+BASE_CGROUPS = '/sys/fs/cgroup'
+
+
 class DefaultOSUtil(object):
     def __init__(self):
         self.agent_conf_file_path = '/etc/waagent.conf'
@@ -290,11 +293,54 @@ class DefaultOSUtil(object):
         return id_that == id_this or \
             id_that == self._correct_instance_id(id_this)
 
-    def is_cgroups_supported(self):
-        return False
+    @staticmethod
+    def is_cgroups_supported():
+        """
+        Enabled by default; disabled in WSL/Travis
+        """
+        is_wsl = '-Microsoft-' in platform.platform()
+        is_travis = 'TRAVIS' in os.environ and os.environ['TRAVIS'] == 'true'
+        base_fs_exists = os.path.exists(BASE_CGROUPS)
+        return not is_wsl and not is_travis and base_fs_exists
+
+    @staticmethod
+    def _cgroup_path(tail=""):
+        return os.path.join(BASE_CGROUPS, tail).rstrip(os.path.sep)
 
     def mount_cgroups(self):
-        pass
+        try:
+            path = self._cgroup_path()
+            if not os.path.exists(path):
+                fileutil.mkdir(path)
+                self.mount(device='cgroup_root',
+                           mount_point=path,
+                           option="-t tmpfs",
+                           chk_err=False)
+            elif not os.path.isdir(self._cgroup_path()):
+                logger.error("Could not mount cgroups: ordinary file at {0}".format(path))
+                return
+
+            for metric_hierarchy in ['cpu,cpuacct', 'memory']:
+                target_path = self._cgroup_path(metric_hierarchy)
+                if not os.path.exists(target_path):
+                    fileutil.mkdir(target_path)
+                    self.mount(device=metric_hierarchy,
+                               mount_point=target_path,
+                               option="-t cgroup -o {0}".format(metric_hierarchy),
+                               chk_err=False)
+
+            for metric_hierarchy in ['cpu', 'cpuacct']:
+                target_path = self._cgroup_path(metric_hierarchy)
+                if not os.path.exists(target_path):
+                    os.symlink(self._cgroup_path('cpu,cpuacct'), target_path)
+
+        except OSError as oe:
+            # log a warning for read-only file systems
+            logger.warn("Could not mount cgroups: {0}", ustr(oe))
+            raise
+        except Exception as e:
+            logger.error("Could not mount cgroups: {0}", ustr(e))
+            raise
 
     def get_agent_conf_file_path(self):
         return self.agent_conf_file_path
@@ -658,7 +704,6 @@ class DefaultOSUtil(object):
         shellutil.run("iptables -I INPUT -p udp --dport 68 -j ACCEPT",
                       chk_err=False)
 
-
     def remove_rules_files(self, rules_files=__RULES_FILES__):
         lib_dir = conf.get_lib_dir()
         for src in rules_files:
@@ -744,7 +789,6 @@ class DefaultOSUtil(object):
                     ifaces[iface_name] = socket.inet_ntoa(ifconf_buff[i+20:i+24])
         return ifaces
 
-
     def get_first_if(self):
         """
         Return the interface name, and IPv4 addr of the "primary" interface or,
diff --git a/azurelinuxagent/common/osutil/factory.py b/azurelinuxagent/common/osutil/factory.py
index 480f2a81..0ef37f93 100644
--- a/azurelinuxagent/common/osutil/factory.py
+++ b/azurelinuxagent/common/osutil/factory.py
@@ -32,6 +32,7 @@ from .ubuntu import UbuntuOSUtil, Ubuntu12OSUtil, Ubuntu14OSUtil, \
 from .alpine import AlpineOSUtil
 from .bigip import BigIpOSUtil
 from .gaia import GaiaOSUtil
+from .iosxe import IosxeOSUtil
 
 from distutils.version import LooseVersion as Version
 
diff --git a/azurelinuxagent/common/osutil/ubuntu.py b/azurelinuxagent/common/osutil/ubuntu.py
index a85ff399..db8dd08c 100644
--- a/azurelinuxagent/common/osutil/ubuntu.py
+++ b/azurelinuxagent/common/osutil/ubuntu.py
@@ -16,22 +16,14 @@
 # Requires Python 2.6+ and Openssl 1.0+
 #
 
-import os
-import platform
 import time
 
 import azurelinuxagent.common.logger as logger
-import azurelinuxagent.common.utils.fileutil as fileutil
 import azurelinuxagent.common.utils.shellutil as shellutil
 
-from azurelinuxagent.common.future import ustr
 from azurelinuxagent.common.osutil.default import DefaultOSUtil
 
 
-def _cgroup_path(tail=""):
-    return os.path.join('/sys/fs/cgroup/', tail).rstrip(os.path.sep)
-
-
 class Ubuntu14OSUtil(DefaultOSUtil):
 
     def __init__(self):
@@ -56,40 +48,6 @@ class Ubuntu14OSUtil(DefaultOSUtil):
     def get_dhcp_lease_endpoint(self):
         return self.get_endpoint_from_leases_path('/var/lib/dhcp/dhclient.*.leases')
 
-    def is_cgroups_supported(self):
-        is_wsl = '-Microsoft-' in platform.platform()
-        is_travis = 'TRAVIS' in os.environ and os.environ['TRAVIS'] == 'true'
-        return not is_wsl and not is_travis
-
-    def mount_cgroups(self):
-        try:
-            if not os.path.exists(_cgroup_path()):
-                fileutil.mkdir(_cgroup_path())
-                self.mount(device='cgroup_root',
-                           mount_point=_cgroup_path(),
-                           option="-t tmpfs",
-                           chk_err=False)
-            elif not os.path.isdir(_cgroup_path()):
-                logger.error("Could not mount cgroups: ordinary file at {0}".format(_cgroup_path()))
-                return
-
-            for metric_hierarchy in ['cpu,cpuacct', 'memory']:
-                target_path = _cgroup_path(metric_hierarchy)
-                if not os.path.exists(target_path):
-                    fileutil.mkdir(target_path)
-                self.mount(device=metric_hierarchy,
-                           mount_point=target_path,
-                           option="-t cgroup -o {0}".format(metric_hierarchy),
-                           chk_err=False)
-
-            for metric_hierarchy in ['cpu', 'cpuacct']:
-                target_path = _cgroup_path(metric_hierarchy)
-                if not os.path.exists(target_path):
-                    os.symlink(_cgroup_path('cpu,cpuacct'), target_path)
-
-        except Exception as e:
-            logger.error("Could not mount cgroups: {0}", ustr(e))
-
 
 class Ubuntu12OSUtil(Ubuntu14OSUtil):
     def __init__(self):
@@ -103,6 +61,7 @@ class Ubuntu12OSUtil(Ubuntu14OSUtil):
     def mount_cgroups(self):
         pass
 
+
 class Ubuntu16OSUtil(Ubuntu14OSUtil):
     """
     Ubuntu 16.04, 16.10, and 17.04.
@@ -179,9 +138,3 @@ class UbuntuSnappyOSUtil(Ubuntu14OSUtil):
     def __init__(self):
         super(UbuntuSnappyOSUtil, self).__init__()
         self.conf_file_path = '/apps/walinuxagent/current/waagent.conf'
-
-    def mount_cgroups(self):
-        """
-        Already mounted in Snappy
-        """
-        pass
diff --git a/azurelinuxagent/common/version.py b/azurelinuxagent/common/version.py
index 170bf8d7..f394df1d 100644
--- a/azurelinuxagent/common/version.py
+++ b/azurelinuxagent/common/version.py
@@ -116,7 +116,7 @@ def get_distro():
 
 AGENT_NAME = "WALinuxAgent"
 AGENT_LONG_NAME = "Azure Linux Agent"
-AGENT_VERSION = '2.2.31.1'
+AGENT_VERSION = '2.2.31.2'
 AGENT_LONG_VERSION = "{0}-{1}".format(AGENT_NAME, AGENT_VERSION)
 AGENT_DESCRIPTION = """
 The Azure Linux Agent supports the provisioning and running of Linux


Test patch:
diff --git a/tests/ga/test_update.py b/tests/ga/test_update.py
index ac5039f6..8ed4cee6 100644
--- a/tests/ga/test_update.py
+++ b/tests/ga/test_update.py
@@ -1000,8 +1000,8 @@ class TestUpdate(UpdateTestCase):
         self.assertTrue(2 < len(self.update_handler.agents))
 
         # Purge every other agent
-        kept_agents = self.update_handler.agents[1::2]
-        purged_agents = self.update_handler.agents[::2]
+        kept_agents = self.update_handler.agents[::2]
+        purged_agents = self.update_handler.agents[1::2]
 
         # Reload and assert only the kept agents remain on disk
         self.update_handler.agents = kept_agents