Repository: Azure/WALinuxAgent
Pull request #1538 (2019-05-28 06:25:28)

Making cgroup telemetry work end to end, with new spec for metrics
<!-- DO NOT DELETE THIS TEMPLATE -->

## Description
Enabling Telemetry E2E for Cgroup Perf counters for extensions.

<!--
Please add an informative description that covers that changes made by the pull request. 
This checklist is used to make sure that common issues in a pull request are addressed.
This will expedite the process of getting your pull request merged and avoid extra work on your part to fix issues discovered during the review process.
-->

---

### PR information
- [x] The title of the PR is clear and informative.
- [x] There are a small number of commits, each of which has an informative message. This means that previously merged commits do not appear in the history of the PR. For information on cleaning up the commits in your pull request, [see this page](https://github.com/Azure/azure-powershell/blob/master/documentation/development-docs/cleaning-up-commits.md).
- [x] Except for special cases involving multiple contributors, the PR is started from a fork of the main repository, not a branch.
- [ ] If applicable, the PR references the bug/issue that it fixes in the description.
- [x] New Unit tests were added for the changes made and Travis.CI is passing.

### Quality of Code and Contribution Guidelines
- [x] I have read the [contribution guidelines](https://github.com/Azure/WALinuxAgent/blob/master/.github/CONTRIBUTING.md).

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/azure/walinuxagent/1538)
<!-- Reviewable:end -->



Patch:
diff --git a/.gitignore b/.gitignore
index 3e547318..563e6e15 100644
--- a/.gitignore
+++ b/.gitignore
@@ -69,3 +69,6 @@ bin/waagent2.0c
 
 # rope project
 .ropeproject/
+
+# mac osx specific files
+.DS_Store
diff --git a/azurelinuxagent/common/cgroup.py b/azurelinuxagent/common/cgroup.py
index 5d61299f..17fe2afd 100644
--- a/azurelinuxagent/common/cgroup.py
+++ b/azurelinuxagent/common/cgroup.py
@@ -14,94 +14,118 @@
 #
 # Requires Python 2.6+ and Openssl 1.0+
 
+import os
 import re
 
+from azurelinuxagent.common import logger
 from azurelinuxagent.common.exception import CGroupsException
+from azurelinuxagent.common.future import ustr
 from azurelinuxagent.common.osutil import get_osutil
+from azurelinuxagent.common.utils import fileutil
 
-re_user_system_times = re.compile('user (\d+)\nsystem (\d+)\n')
-
-# The metric classes (Cpu, Memory, etc) can all assume that CGroups is enabled, as the CGroupTelemetry
-# class is very careful not to call them if CGroups isn't enabled. Any tests should be disabled if the osutil
-# is_cgroups_support() method returns false.
+re_user_system_times = re.compile(r'user (\d+)\nsystem (\d+)\n')
 
 
 class CGroup(object):
-    def __init__(self):
-        pass
+    @staticmethod
+    def create(cgroup_path, controller, extension_name):
+        """
+        Factory method to create the correct CGroup.
+        """
+        if controller is "cpu":
+            return CpuCgroup(extension_name, cgroup_path)
+        elif controller is "memory":
+            return MemoryCgroup(extension_name, cgroup_path)
 
-    def _get_cgroup_file(self, hierarchy, file_name):
-        return os.path.join(self.cgroups[hierarchy], file_name)
+    def __init__(self, name, cgroup_path, controller_type):
+        """
+        Initialize _data collection for the Memory controller
+        :param: name: Name of the CGroup
+        :param: cgroup_path: Path of the controller
+        :param: controller_type:
+        :return:
+        """
+        self.name = name
+        self.path = cgroup_path
+        self.controller = controller_type
 
+    def _get_cgroup_file(self, file_name):
+        return os.path.join(self.path, file_name)
 
-    def get_file_contents(self, hierarchy, file_name):
+    def _get_file_contents(self, file_name):
         """
-        Retrieve the value of a parameter from a hierarchy.
+        Retrieve the contents to file.
 
-        :param str hierarchy: Name of cgroup metric hierarchy
-        :param str file_name: Name of file within that metric hierarchy
+        :param str file_name: Name of file within that metric controller
         :return: Entire contents of the file
         :rtype: str
         """
-        if hierarchy in self.cgroups:
-            parameter_file = self._get_cgroup_file(hierarchy, file_name)
 
-            try:
-                return fileutil.read_file(parameter_file)
-            except Exception:
-                raise CGroupsException("Could not retrieve cgroup file {0}/{1}".format(hierarchy, file_name))
-        else:
-            raise CGroupsException("{0} subsystem not available in cgroup {1}. cgroup paths: {2}".format(
-                hierarchy, self.name, self.cgroups))
+        parameter_file = self._get_cgroup_file(file_name)
+
+        try:
+            return fileutil.read_file(parameter_file)
+        except Exception:
+            raise CGroupsException("Could not retrieve cgroup file {0}/{1}".
+                                   format(self.path, file_name))
 
-    def get_parameter(self, hierarchy, parameter_name):
+    def _get_parameters(self, parameter_name, first_line_only=False):
         """
-        Retrieve the value of a parameter from a hierarchy.
-        Assumes the parameter is the sole line of the file.
+        Retrieve the values of a parameter from a controller.
+        Returns a list of values in the file.
 
-        :param str hierarchy: Name of cgroup metric hierarchy
-        :param str parameter_name: Name of file within that metric hierarchy
+        :param first_line_only: return only the first line.
+        :param str parameter_name: Name of file within that metric controller
         :return: The first line of the file, without line terminator
-        :rtype: str
+        :rtype: [str]
         """
-        result = ""
+        result = []
         try:
-            values = self.get_file_contents(hierarchy, parameter_name).splitlines()
-            result = values[0]
+            values = self._get_file_contents(parameter_name).splitlines()
+            result = values[0] if first_line_only else values
         except IndexError:
-            parameter_filename = self._get_cgroup_file(hierarchy, parameter_name)
+            parameter_filename = self._get_cgroup_file(parameter_name)
             logger.error("File {0} is empty but should not be".format(parameter_filename))
-        except CGroupsException as e:
-            # ignore if the file does not exist yet
-            pass
+        except CGroupsException:
+            return None
         except Exception as e:
-            parameter_filename = self._get_cgroup_file(hierarchy, parameter_name)
+            parameter_filename = self._get_cgroup_file(parameter_name)
             logger.error("Exception while attempting to read {0}: {1}".format(parameter_filename, ustr(e)))
         return result
 
+    def collect(self):
+        raise NotImplementedError()
+
+    def is_active(self):
+        tasks = self._get_parameters("tasks")
+        if tasks:
+            return len(tasks) != 0
+        else:
+            return False
+
 
 class CpuCgroup(CGroup):
-    def __init__(self, cgt):
+    def __init__(self, name, cgroup_path):
         """
-        Initialize data collection for the Cpu hierarchy. User must call update() before attempting to get
+        Initialize _data collection for the Cpu controller. User must call update() before attempting to get
         any useful metrics.
 
-        :param cgt: CGroupsTelemetry
-        :return:
+        :return: CpuCgroup
         """
-        self.cgt = cgt
-        self.osutil = get_osutil()
-        self.current_cpu_total = self.get_current_cpu_total()
-        self.previous_cpu_total = 0
-        self.current_system_cpu = self.osutil.get_total_cpu_ticks_since_boot()
-        self.previous_system_cpu = 0
+        super(CpuCgroup, self).__init__(name, cgroup_path, "cpu")
+
+        self._osutil = get_osutil()
+        self._current_cpu_total = self._get_current_cpu_total()
+        self._previous_cpu_total = 0
+        self._current_system_cpu = self._osutil.get_total_cpu_ticks_since_boot()
+        self._previous_system_cpu = 0
 
     def __str__(self):
-        return "Cgroup: Current {0}, previous {1}; System: Current {2}, previous {3}".format(
-            self.current_cpu_total, self.previous_cpu_total, self.current_system_cpu, self.previous_system_cpu
+        return "cgroup: Name: {0}, cgroup_path: {1}; Controller: {2}".format(
+            self.name, self.path, self.controller
         )
 
-    def get_current_cpu_total(self):
+    def _get_current_cpu_total(self):
         """
         Compute the number of USER_HZ of CPU time (user and system) consumed by this cgroup since boot.
 
@@ -109,8 +133,7 @@ class CpuCgroup(CGroup):
         """
         cpu_total = 0
         try:
-            cpu_stat = self.cgt.cgroup.\
-                get_file_contents('cpu', 'cpuacct.stat')
+            cpu_stat = self._get_file_contents('cpuacct.stat')
             if cpu_stat is not None:
                 m = re_user_system_times.match(cpu_stat)
                 if m:
@@ -123,17 +146,17 @@ class CpuCgroup(CGroup):
             pass
         return cpu_total
 
-    def update(self):
+    def _update_cpu_data(self):
         """
-        Update all raw data required to compute metrics of interest. The intent is to call update() once, then
-        call the various get_*() methods which use this data, which we've collected exactly once.
+        Update all raw _data required to compute metrics of interest. The intent is to call update() once, then
+        call the various get_*() methods which use this _data, which we've collected exactly once.
         """
-        self.previous_cpu_total = self.current_cpu_total
-        self.previous_system_cpu = self.current_system_cpu
-        self.current_cpu_total = self.get_current_cpu_total()
-        self.current_system_cpu = self.osutil.get_total_cpu_ticks_since_boot()
+        self._previous_cpu_total = self._current_cpu_total
+        self._previous_system_cpu = self._current_system_cpu
+        self._current_cpu_total = self._get_current_cpu_total()
+        self._current_system_cpu = self._osutil.get_total_cpu_ticks_since_boot()
 
-    def get_cpu_percent(self):
+    def _get_cpu_percent(self):
         """
         Compute the percent CPU time used by this cgroup over the elapsed time since the last time this instance was
         update()ed.  If the cgroup fully consumed 2 cores on a 4 core system, return 200.
@@ -141,10 +164,10 @@ class CpuCgroup(CGroup):
         :return: CPU usage in percent of a single core
         :rtype: float
         """
-        cpu_delta = self.current_cpu_total - self.previous_cpu_total
-        system_delta = max(1, self.current_system_cpu - self.previous_system_cpu)
+        cpu_delta = self._current_cpu_total - self._previous_cpu_total
+        system_delta = max(1, self._current_system_cpu - self._previous_system_cpu)
 
-        return round(float(cpu_delta * self.cgt.cpu_count * 100) / float(system_delta), 3)
+        return round(float(cpu_delta * self._osutil.get_processor_cores() * 100) / float(system_delta), 3)
 
     def collect(self):
         """
@@ -152,29 +175,45 @@ class CpuCgroup(CGroup):
 
         :rtype: [(str, str, float)]
         """
-        self.update()
-        usage = self.get_cpu_percent()
-        return [("Process", "% Processor Time", usage)]
+        self._update_cpu_data()
+        usage = self._get_cpu_percent()
+        return [CollectedMetrics("cpu", "% Processor Time", usage)]
 
 
-class MemoryCGroup(CGroup):
-    def __init__(self, cgt):
+class MemoryCgroup(CGroup):
+    def __init__(self, name, cgroup_path):
         """
-        Initialize data collection for the Memory hierarchy
+        Initialize _data collection for the Memory controller
 
-        :param CGroupsTelemetry cgt: The telemetry object for which memory metrics should be collected
-        :return:
+        :return: MemoryCgroup
         """
-        self.cgt = cgt
+        super(MemoryCgroup, self).__init__(name, cgroup_path, "memory")
+
+    def __str__(self):
+        return "cgroup: Name: {0}, cgroup_path: {1}; Controller: {2}".format(
+            self.name, self.path, self.controller
+        )
 
-    def get_memory_usage(self):
+    def _get_memory_usage(self):
         """
         Collect memory.usage_in_bytes from the cgroup.
 
         :return: Memory usage in bytes
         :rtype: int
         """
-        usage = self.cgt.cgroup.get_parameter('memory', 'memory.usage_in_bytes')
+        usage = self._get_parameters('memory.usage_in_bytes', first_line_only=True)
+        if not usage:
+            usage = "0"
+        return int(usage)
+
+    def _get_memory_max_usage(self):
+        """
+        Collect memory.usage_in_bytes from the cgroup.
+
+        :return: Memory usage in bytes
+        :rtype: int
+        """
+        usage = self._get_parameters('memory.max_usage_in_bytes', first_line_only=True)
         if not usage:
             usage = "0"
         return int(usage)
@@ -185,11 +224,20 @@ class MemoryCGroup(CGroup):
 
         :rtype: [(str, str, float)]
         """
-        usage = self.get_memory_usage()
-        return [("Memory", "Total Memory Usage", usage)]
+        usage = self._get_memory_usage()
+        max_usage = self._get_memory_max_usage()
+        return [CollectedMetrics("memory", "Total Memory Usage", usage),
+                CollectedMetrics("memory", "Max Memory Usage", max_usage)]
+
+
+class CollectedMetrics(object):
+    def __init__(self, controller, metric_name, value):
+        self.controller = controller
+        self.metric_name = metric_name
+        self.value = value
 
 #
-# TODO: Do we need this code?
+# TODO: Do we need this code? - For not we'll keep this code. Will remove in the next round.
 #
 #
 # MEMORY_DEFAULT = -1
@@ -241,7 +289,7 @@ class MemoryCGroup(CGroup):
 #         logger.verbose("writing {0} to {1}".format(limit_units, cpu_shares_file))
 #         fileutil.write_file(cpu_shares_file, '{0}\n'.format(limit_units))
 #     else:
-#         raise CGroupsException("CPU hierarchy not available in this cgroup")
+#         raise CGroupsException("CPU controller not available in this cgroup")
 #
 # @staticmethod
 # def get_num_cores():
@@ -275,7 +323,7 @@ class MemoryCGroup(CGroup):
 #         logger.verbose("writing {0} to {1}".format(value, memory_limit_file))
 #         fileutil.write_file(memory_limit_file, '{0}\n'.format(value))
 #     else:
-#         raise CGroupsException("Memory hierarchy not available in this cgroup")
+#         raise CGroupsException("Memory controller not available in this cgroup")
 #
 # class CGroupsLimits(object):
 #     @staticmethod
diff --git a/azurelinuxagent/common/cgroupapi.py b/azurelinuxagent/common/cgroupapi.py
index 4d9e4647..0a6f5197 100644
--- a/azurelinuxagent/common/cgroupapi.py
+++ b/azurelinuxagent/common/cgroupapi.py
@@ -20,6 +20,7 @@ import shutil
 import subprocess
 
 from azurelinuxagent.common import logger
+from azurelinuxagent.common.cgroup import CpuCgroup, MemoryCgroup, CGroup
 from azurelinuxagent.common.exception import CGroupsException
 from azurelinuxagent.common.future import ustr
 from azurelinuxagent.common.utils import fileutil, shellutil
@@ -94,7 +95,7 @@ class CGroupsApi(object):
             fields = entry.split(':')
             if fields[0] == controller_id:
                 return fields[2].lstrip(os.path.sep)
-        raise CGroupsException("This process belongs to no cgroup for hierarchy ID {0}".format(controller_id))
+        raise CGroupsException("This process belongs to no cgroup for controller ID {0}".format(controller_id))
 
 
     @staticmethod
@@ -164,6 +165,10 @@ class FileSystemCgroupsApi(CGroupsApi):
 
         return os.path.join(extensions_root, cgroup_name)
 
+    def _create_extension_cgroup(self, controller, extension_name):
+        return CGroup.create(self._get_extension_cgroup_path(controller, extension_name),
+                             controller, extension_name)
+
     @staticmethod
     def _add_process_to_cgroup(pid, cgroup_path):
         tasks_file = os.path.join(cgroup_path, 'cgroup.procs')
@@ -174,7 +179,7 @@ class FileSystemCgroupsApi(CGroupsApi):
         """
         Creates a cgroup for the VM Agent in each of the controllers we are tracking; returns the created cgroups.
         """
-        cgroup_paths = []
+        cgroups = []
 
         pid = int(os.getpid())
 
@@ -188,17 +193,17 @@ class FileSystemCgroupsApi(CGroupsApi):
 
                 self._add_process_to_cgroup(pid, path)
 
-                cgroup_paths.append(path)
+                cgroups.append(CGroup.create(path, controller, VM_AGENT_CGROUP_NAME))
 
             except Exception as e:
                 logger.warn('Cannot create "{0}" cgroup for the agent. Error: {1}'.format(controller, ustr(e)))
 
         self._foreach_controller(create_cgroup, 'Will not create a cgroup for the VM Agent')
 
-        if len(cgroup_paths) == 0:
+        if len(cgroups) == 0:
             raise CGroupsException("Failed to create any cgroup for the VM Agent")
 
-        return cgroup_paths
+        return cgroups
 
     def create_extension_cgroups_root(self):
         """
@@ -217,20 +222,20 @@ class FileSystemCgroupsApi(CGroupsApi):
         """
         Creates a cgroup for the given extension in each of the controllers we are tracking; returns the created cgroups.
         """
-        cgroup_paths = []
+        cgroups = []
 
         def create_cgroup(controller):
-            path = self._get_extension_cgroup_path(controller, extension_name)
+            cgroup = self._create_extension_cgroup(controller, extension_name)
 
-            if not os.path.isdir(path):
-                FileSystemCgroupsApi._try_mkdir(path)
-                logger.info("Created cgroup {0}".format(path))
+            if not os.path.isdir(cgroup.path):
+                FileSystemCgroupsApi._try_mkdir(cgroup.path)
+                logger.info("Created cgroup {0}".format(cgroup.path))
 
-            cgroup_paths.append(path)
+            cgroups.append(cgroup)
 
         self._foreach_controller(create_cgroup, 'Will not create a cgroup for extension {0}'.format(extension_name))
 
-        return cgroup_paths
+        return cgroups
 
     def remove_extension_cgroups(self, extension_name):
         """
@@ -247,15 +252,16 @@ class FileSystemCgroupsApi(CGroupsApi):
         """
         Returns the cgroups for the given extension.
         """
-        cgroup_paths = []
+
+        cgroups = []
 
         def get_cgroup(controller):
-            path = self._get_extension_cgroup_path(controller, extension_name)
-            cgroup_paths.append(path)
+            cgroup = self._create_extension_cgroup(controller, extension_name)
+            cgroups.append(cgroup)
 
         self._foreach_controller(get_cgroup, 'Failed to retrieve cgroups for extension {0}'.format(extension_name))
 
-        return cgroup_paths
+        return cgroups
 
     def start_extension_command(self, extension_name, command, shell, cwd, env, stdout, stderr):
         """
@@ -328,7 +334,8 @@ class SystemdCgroupsApi(CGroupsApi):
             cpu_cgroup_path = os.path.join(CGROUPS_FILE_SYSTEM_ROOT, 'cpu', cgroup_unit)
             memory_cgroup_path = os.path.join(CGROUPS_FILE_SYSTEM_ROOT, 'memory', cgroup_unit)
 
-            return [cpu_cgroup_path, memory_cgroup_path]
+            return [CGroup.create(cpu_cgroup_path, 'cpu', cgroup_unit),
+                    CGroup.create(memory_cgroup_path, 'memory', cgroup_unit)]
         except Exception as e:
             raise CGroupsException("Failed to get paths of agent's cgroups. Error: {0}".format(ustr(e)))
 
@@ -383,7 +390,8 @@ After=system-{1}.slice""".format(extension_name, EXTENSIONS_ROOT_CGROUP_NAME)
         cpu_cgroup_path = os.path.join(CGROUPS_FILE_SYSTEM_ROOT, 'cpu', 'system.slice', scope_name)
         memory_cgroup_path = os.path.join(CGROUPS_FILE_SYSTEM_ROOT, 'memory', 'system.slice', scope_name)
 
-        return [cpu_cgroup_path, memory_cgroup_path]
+        return [CGroup.create(cpu_cgroup_path, 'cpu', extension_name),
+                CGroup.create(memory_cgroup_path, 'cpu', extension_name)]
 
     def start_extension_command(self, extension_name, command, shell, cwd, env, stdout, stderr):
         def pre_exec_function():
diff --git a/azurelinuxagent/common/cgroupconfigurator.py b/azurelinuxagent/common/cgroupconfigurator.py
index 5cb60426..173b9419 100644
--- a/azurelinuxagent/common/cgroupconfigurator.py
+++ b/azurelinuxagent/common/cgroupconfigurator.py
@@ -19,6 +19,7 @@ import subprocess
 
 from azurelinuxagent.common import logger
 from azurelinuxagent.common.cgroupapi import CGroupsApi
+from azurelinuxagent.common.cgroupstelemetry import CGroupsTelemetry
 from azurelinuxagent.common.exception import CGroupsException
 from azurelinuxagent.common.future import ustr
 from azurelinuxagent.common.osutil import get_osutil
@@ -97,8 +98,8 @@ class CGroupConfigurator(object):
                 cgroups = self._cgroups_api.create_agent_cgroups()
 
                 if track_cgroups:
-                    # TODO: Add to tracking list
-                    pass
+                    for cgroup in cgroups:
+                        CGroupsTelemetry.track_cgroup(cgroup)
 
                 return cgroups
 
@@ -128,9 +129,6 @@ class CGroupConfigurator(object):
             """
             def __impl():
                 cgroups = self._cgroups_api.remove_extension_cgroups(name)
-
-                # TODO: Remove from tracking list
-
                 return cgroups
 
             self._invoke_cgroup_operation(__impl, "Failed to delete cgroups for extension '{0}'.".format(name))
@@ -167,9 +165,15 @@ class CGroupConfigurator(object):
                 def track_cgroups():
                     cgroups = self._cgroups_api.get_extension_cgroups(extension_name)
 
-                    # TODO: Add to tracking list
+                    try:
+                        for cgroup in cgroups:
+                            CGroupsTelemetry.track_cgroup(cgroup)
+                    except Exception as e:
+                        logger.info("Cannot add CGroup into tracking list. Error: {0}".format(ustr(e)))
 
-                self._invoke_cgroup_operation(track_cgroups, "Failed to add cgroups for extension '{0}' to the tracking list; resource usage will not be tracked".format(extension_name))
+                self._invoke_cgroup_operation(track_cgroups,
+                                              "Failed to add cgroups for extension '{0}' to the tracking list; "
+                                              "resource usage will not be tracked".format(extension_name))
 
             return process
 
diff --git a/azurelinuxagent/common/cgroupstelemetry.py b/azurelinuxagent/common/cgroupstelemetry.py
index 0b2fad32..a00ad0fc 100644
--- a/azurelinuxagent/common/cgroupstelemetry.py
+++ b/azurelinuxagent/common/cgroupstelemetry.py
@@ -13,47 +13,180 @@
 # limitations under the License.
 #
 # Requires Python 2.6+ and Openssl 1.0+
-
-import time
+import threading
+from datetime import datetime as dt
 
 from azurelinuxagent.common import logger
-from azurelinuxagent.common.cgroup import CpuCgroup, MemoryCGroup
+
 
 class CGroupsTelemetry(object):
     """
     """
-    _tracked = {}
+    _tracked = []
+    _cgroup_metrics = {}
+    _rlock = threading.RLock()
 
     @staticmethod
-    def metrics_hierarchies():
-        return CGroupsTelemetry._hierarchies
+    def _get_metrics_list(metric):
+        return [metric.average(), metric.min(), metric.max(), metric.median(), metric.count(),
+                metric.first_poll_time(), metric.last_poll_time()]
+
+    @staticmethod
+    def _process_cgroup_metric(cgroup_metrics):
+        current_memory_usage, max_memory_levels, current_cpu_usage = cgroup_metrics.get_metrics()
+        processed_extension = {
+            "memory": {
+                "cur_mem": CGroupsTelemetry._get_metrics_list(current_memory_usage),
+                "max_mem": CGroupsTelemetry._get_metrics_list(max_memory_levels)
+            },
+            "cpu": {
+                "cur_cpu": CGroupsTelemetry._get_metrics_list(current_cpu_usage)
+            }
+        }
+        return processed_extension
 
     @staticmethod
     def track_cgroup(cgroup):
         """
-        Adds the given item to the list of tracked cgroups
+        Adds the given item to the dictionary of tracked cgroups
         """
-        pass  # TODO
+        with CGroupsTelemetry._rlock:
+            CGroupsTelemetry._tracked.append(cgroup)
+            logger.info("Started tracking new cgroup: {0}, path: {1}".format(cgroup.name, cgroup.path))
 
     @staticmethod
-    def is_tracked(name):
+    def is_tracked(name, controller):
         """
         Returns true if the given item is in the list of tracked items
+        O(n) operation. But limited to few cgroup objects we have.
         """
-        pass  # TODO
+        with CGroupsTelemetry._rlock:
+            for cgroup in CGroupsTelemetry._tracked:
+                if name == cgroup.name and controller == cgroup.controller:
+                    return True
+
+        return False
 
     @staticmethod
-    def stop_tracking(name):
+    def stop_tracking(cgroup):
         """
         Stop tracking the cgroups for the given name
         """
-        if CGroupsTelemetry.is_tracked(name):
-            del (CGroupsTelemetry._tracked[name])  # TODO - thread-safeness; review implementation
+        with CGroupsTelemetry._rlock:
+            CGroupsTelemetry._tracked.remove(cgroup)
+            logger.info("Stopped tracking cgroup: {0}, path: {1}".format(cgroup.name, cgroup.path))
 
     @staticmethod
-    def collect_all_tracked():
-        pass  # TODO
+    def report_all_tracked():
+        collected_metrics = {}
 
+        for name, cgroup_metrics in CGroupsTelemetry._cgroup_metrics.items():
+            collected_metrics[name] = CGroupsTelemetry._process_cgroup_metric(cgroup_metrics)
+            cgroup_metrics.clear()
+
+        # Doing cleanup after the metrics have already been collected.
+        for key in [key for key in CGroupsTelemetry._cgroup_metrics if
+                    CGroupsTelemetry._cgroup_metrics[key].marked_for_delete]:
+            del CGroupsTelemetry._cgroup_metrics[key]
+
+        return collected_metrics
+
+    @staticmethod
+    def poll_all_tracked():
+        with CGroupsTelemetry._rlock:
+            for cgroup in CGroupsTelemetry._tracked[:]:
+                if cgroup.name not in CGroupsTelemetry._cgroup_metrics:
+                    CGroupsTelemetry._cgroup_metrics[cgroup.name] = CgroupMetrics()
+                CGroupsTelemetry._cgroup_metrics[cgroup.name].add_new_data(cgroup.controller, cgroup.collect())
+
+                if not cgroup.is_active():
+                    CGroupsTelemetry.stop_tracking(cgroup)
+                    CGroupsTelemetry._cgroup_metrics[cgroup.name].marked_for_delete = True
+
+    @staticmethod
+    def prune_all_tracked():
+        with CGroupsTelemetry._rlock:
+            for cgroup in CGroupsTelemetry._tracked[:]:
+                if not cgroup.is_active():
+                    CGroupsTelemetry.stop_tracking(cgroup)
+
+    @staticmethod
+    def cleanup():
+        with CGroupsTelemetry._rlock:
+            CGroupsTelemetry._tracked *= 0  # emptying the list
+
+
+class CgroupMetrics(object):
     def __init__(self):
-        pass  # TODO
+        self._current_memory_usage = Metric()
+        self._max_memory_levels = Metric()
+        self._current_cpu_usage = Metric()
+        self.marked_for_delete = False
+
+    def _add_memory_usage(self, metric):
+        self._current_memory_usage.append(metric[0].value)
+        self._max_memory_levels.append(metric[1].value)
+
+    def _add_cpu_usage(self, metric):
+        self._current_cpu_usage.append(metric[0].value)
+
+    def add_new_data(self, controller, metric):
+        if controller is "cpu":
+            self._add_cpu_usage(metric)
+        elif controller is "memory":
+            self._add_memory_usage(metric)
+
+    def get_metrics(self):
+        return self._current_memory_usage, self._max_memory_levels, self._current_cpu_usage
+
+    def clear(self):
+        self._current_memory_usage.clear()
+        self._max_memory_levels.clear()
+        self._current_cpu_usage.clear()
+
+
+class Metric(object):
+    def __init__(self):
+        self._data = []
+        self._first_poll_time = None
+        self._last_poll_time = None
+
+    def append(self, metric):
+        if not self._first_poll_time:
+            #  We only want to do it first time.
+            self._first_poll_time = dt.utcnow()
+
+        self._data.append(metric)
+        self._last_poll_time = dt.utcnow()
+
+    def clear(self):
+        self._data *= 0
+
+    def average(self):
+        return float(sum(self._data)
+                     ) / float(len(self._data)) if self._data else None
+
+    def max(self):
+        return max(self._data) if self._data else None
+
+    def min(self):
+        return min(self._data) if self._data else None
+
+    def median(self):
+        data = sorted(self._data)
+        l_len = len(data)
+        if l_len < 1:
+            return None
+        if l_len % 2 == 0:
+            return (data[int((l_len - 1) / 2)] + data[int((l_len + 1) / 2)]) / 2.0
+        else:
+            return data[int((l_len - 1) / 2)]
+
+    def count(self):
+        return len(self._data)
+
+    def first_poll_time(self):
+        return str(self._first_poll_time)
 
+    def last_poll_time(self):
+        return str(self._last_poll_time)
diff --git a/azurelinuxagent/common/event.py b/azurelinuxagent/common/event.py
index 4ad441e6..82e2ad9f 100644
--- a/azurelinuxagent/common/event.py
+++ b/azurelinuxagent/common/event.py
@@ -47,6 +47,7 @@ class WALAEventOperation:
     AutoUpdate = "AutoUpdate"
     CustomData = "CustomData"
     CGroupsLimitsCrossed = "CGroupsLimitsCrossed"
+    ExtensionMetricsData = "ExtensionMetricsData"
     Deploy = "Deploy"
     Disable = "Disable"
     Downgrade = "Downgrade"
diff --git a/azurelinuxagent/common/protocol/wire.py b/azurelinuxagent/common/protocol/wire.py
index 130d9598..c80e6991 100644
--- a/azurelinuxagent/common/protocol/wire.py
+++ b/azurelinuxagent/common/protocol/wire.py
@@ -370,7 +370,7 @@ def vm_status_to_v1(vm_status, ext_statuses):
         'version': '1.1',
         'timestampUTC': timestamp,
         'aggregateStatus': v1_agg_status,
-        'guestOSInfo' : v1_ga_guest_info
+        'guestOSInfo': v1_ga_guest_info
     }
     return v1_vm_status
 
diff --git a/azurelinuxagent/ga/exthandlers.py b/azurelinuxagent/ga/exthandlers.py
index ed28ff00..5d0839be 100644
--- a/azurelinuxagent/ga/exthandlers.py
+++ b/azurelinuxagent/ga/exthandlers.py
@@ -25,23 +25,21 @@ import os
 import random
 import re
 import shutil
-import stat
 import signal
-import subprocess
-import time
+import stat
+import sys
 import tempfile
+import time
 import traceback
 import zipfile
-import sys
 
 import azurelinuxagent.common.conf as conf
 import azurelinuxagent.common.logger as logger
 import azurelinuxagent.common.utils.fileutil as fileutil
 import azurelinuxagent.common.version as version
 from azurelinuxagent.common.cgroupconfigurator import CGroupConfigurator
-from azurelinuxagent.common.cgroupstelemetry import CGroupsTelemetry
 from azurelinuxagent.common.errorstate import ErrorState, ERROR_STATE_DELTA_INSTALL
-from azurelinuxagent.common.event import add_event, WALAEventOperation, elapsed_milliseconds, report_event
+from azurelinuxagent.common.event import add_event, WALAEventOperation, elapsed_milliseconds
 from azurelinuxagent.common.exception import ExtensionError, ProtocolError, ProtocolNotFoundError, \
     ExtensionDownloadError, ExtensionOperationError, ExtensionErrorCodes
 from azurelinuxagent.common.future import ustr
@@ -1120,7 +1118,8 @@ class ExtHandlerInstance(object):
         last_update = int(time.time() - os.stat(heartbeat_file).st_mtime)
         return last_update <= 600
 
-    def launch_command(self, cmd, timeout=300, extension_error_code=ExtensionErrorCodes.PluginProcessingError, env=None):
+    def launch_command(self, cmd, timeout=300, extension_error_code=ExtensionErrorCodes.PluginProcessingError,
+                       env=None):
         begin_utc = datetime.datetime.utcnow()
         self.logger.verbose("Launch command: [{0}]", cmd)
 
@@ -1147,9 +1146,11 @@ class ExtHandlerInstance(object):
                         stderr=stderr)
 
                 except OSError as e:
-                    raise ExtensionOperationError("Failed to launch '{0}': {1}".format(full_path, e.strerror), code=extension_error_code)
+                    raise ExtensionOperationError("Failed to launch '{0}': {1}".format(full_path, e.strerror),
+                                                  code=extension_error_code)
 
-                msg = ExtHandlerInstance._capture_process_output(process, stdout, stderr, cmd, timeout, extension_error_code)
+                msg = ExtHandlerInstance._capture_process_output(process, stdout, stderr, cmd, timeout,
+                                                                 extension_error_code)
 
                 duration = elapsed_milliseconds(begin_utc)
                 log_msg = "{0}\n{1}".format(cmd, "\n".join([line for line in msg.split('\n') if line != ""]))
diff --git a/azurelinuxagent/ga/monitor.py b/azurelinuxagent/ga/monitor.py
index 106d92a4..cbd68a55 100644
--- a/azurelinuxagent/ga/monitor.py
+++ b/azurelinuxagent/ga/monitor.py
@@ -19,18 +19,17 @@ import datetime
 import json
 import os
 import platform
-import time
 import threading
-import traceback
+import time
 import uuid
 
 import azurelinuxagent.common.conf as conf
 import azurelinuxagent.common.logger as logger
-from azurelinuxagent.common.errorstate import ErrorState
+import azurelinuxagent.common.utils.networkutil as networkutil
 
-from azurelinuxagent.common.cgroupconfigurator import CGroupConfigurator
 from azurelinuxagent.common.cgroupstelemetry import CGroupsTelemetry
-from azurelinuxagent.common.event import add_event, report_metric, WALAEventOperation
+from azurelinuxagent.common.errorstate import ErrorState
+from azurelinuxagent.common.event import add_event, WALAEventOperation
 from azurelinuxagent.common.exception import EventError, ProtocolError, OSUtilError, HttpError
 from azurelinuxagent.common.future import ustr
 from azurelinuxagent.common.osutil import get_osutil
@@ -38,15 +37,11 @@ from azurelinuxagent.common.protocol import get_protocol_util
 from azurelinuxagent.common.protocol.healthservice import HealthService
 from azurelinuxagent.common.protocol.imds import get_imds_client
 from azurelinuxagent.common.protocol.restapi import TelemetryEventParam, \
-    TelemetryEventList, \
-    TelemetryEvent, \
-    set_properties
-import azurelinuxagent.common.utils.networkutil as networkutil
+    TelemetryEventList, TelemetryEvent, set_properties
 from azurelinuxagent.common.utils.restutil import IOErrorCounter
 from azurelinuxagent.common.utils.textutil import parse_doc, findall, find, getattrib, hash_strings
 from azurelinuxagent.common.version import DISTRO_NAME, DISTRO_VERSION, \
-    DISTRO_CODE_NAME, AGENT_LONG_VERSION, \
-    AGENT_NAME, CURRENT_AGENT, CURRENT_VERSION
+    DISTRO_CODE_NAME, AGENT_NAME, CURRENT_AGENT, CURRENT_VERSION
 
 
 def parse_event(data_str):
@@ -91,6 +86,17 @@ def parse_json_event(data_str):
     return event
 
 
+def generate_extension_metrics_telemetry_dictionary(schema_version=1.0,
+                                                    performance_metrics=None):
+    if schema_version == 1.0:
+        telemetry_dict = {"SchemaVersion": 1.0}
+        if performance_metrics:
+            telemetry_dict["PerfMetrics"] = performance_metrics
+        return telemetry_dict
+    else:
+        return None
+
+
 def get_monitor_handler():
     return MonitorHandler()
 
@@ -98,7 +104,9 @@ def get_monitor_handler():
 class MonitorHandler(object):
     EVENT_COLLECTION_PERIOD = datetime.timedelta(minutes=1)
     TELEMETRY_HEARTBEAT_PERIOD = datetime.timedelta(minutes=30)
-    CGROUP_TELEMETRY_PERIOD = datetime.timedelta(minutes=5)
+    # extension metrics period
+    CGROUP_TELEMETRY_POLLING_PERIOD = datetime.timedelta(minutes=5)
+    CGROUP_TELEMETRY_REPORTING_PERIOD = datetime.timedelta(minutes=30)
     # host plugin
     HOST_PLUGIN_HEARTBEAT_PERIOD = datetime.timedelta(minutes=1)
     HOST_PLUGIN_HEALTH_PERIOD = datetime.timedelta(minutes=5)
@@ -114,7 +122,8 @@ class MonitorHandler(object):
         self.event_thread = None
         self.last_event_collection = None
         self.last_telemetry_heartbeat = None
-        self.last_cgroup_telemetry = None
+        self.last_cgroup_polling_telemetry = None
+        self.last_cgroup_report_telemetry = None
         self.last_host_plugin_heartbeat = None
         self.last_imds_heartbeat = None
         self.protocol = None
@@ -253,13 +262,15 @@ class MonitorHandler(object):
 
     def daemon(self):
         min_delta = min(MonitorHandler.TELEMETRY_HEARTBEAT_PERIOD,
-                        MonitorHandler.CGROUP_TELEMETRY_PERIOD,
+                        MonitorHandler.CGROUP_TELEMETRY_POLLING_PERIOD,
+                        MonitorHandler.CGROUP_TELEMETRY_REPORTING_PERIOD,
                         MonitorHandler.EVENT_COLLECTION_PERIOD,
                         MonitorHandler.HOST_PLUGIN_HEARTBEAT_PERIOD,
                         MonitorHandler.IMDS_HEARTBEAT_PERIOD).seconds
         while self.should_run:
             self.send_telemetry_heartbeat()
-            self.send_cgroup_telemetry()
+            self.poll_telemetry_metrics()
+            self.send_telemetry_metrics()
             self.collect_and_send_events()
             self.send_host_plugin_heartbeat()
             self.send_imds_heartbeat()
@@ -270,9 +281,7 @@ class MonitorHandler(object):
         sysinfo_names = [v.name for v in self.sysinfo]
         for param in event.parameters:
             if param.name in sysinfo_names:
-                logger.verbose("Remove existing event parameter: [{0}:{1}]",
-                               param.name,
-                               param.value)
+                logger.verbose("Remove existing event parameter: [{0}:{1}]", param.name, param.value)
                 event.parameters.remove(param)
         event.parameters.extend(self.sysinfo)
 
@@ -399,54 +408,40 @@ class MonitorHandler(object):
 
             self.last_telemetry_heartbeat = datetime.datetime.utcnow()
 
-    def send_cgroup_telemetry(self):
-        if self.last_cgroup_telemetry is None:
-            self.last_cgroup_telemetry = datetime.datetime.utcnow()
-
-        if datetime.datetime.utcnow() >= (self.last_telemetry_heartbeat + MonitorHandler.CGROUP_TELEMETRY_PERIOD):
-            try:
-                metric_reported, metric_threshold = CGroupsTelemetry.collect_all_tracked()
-                for cgroup_name, metrics in metric_reported.items():
-                    thresholds = metric_threshold[cgroup_name]
-
-                    for metric_group, metric_name, value in metrics:
-                        if value > 0:
-                            report_metric(metric_group, metric_name, cgroup_name, value)
-
-                        if metric_group == "Memory":
-                            # Memory is collected in bytes, and limit is set in megabytes.
-                            if value >= CGroupConfigurator._format_memory_value('megabytes', thresholds.memory_limit):
-                                msg = "CGroup {0}: Crossed the Memory Threshold. " \
-                                      "Current Value: {1} bytes, Threshold: {2} megabytes." \
-                                       .format(cgroup_name, value, thresholds.memory_limit)
-
-                                logger.warn(msg)
-                                add_event(name=AGENT_NAME,
-                                          version=CURRENT_VERSION,
-                                          op=WALAEventOperation.CGroupsLimitsCrossed,
-                                          is_success=True,
-                                          message=msg,
-                                          log_event=True)
-
-                        if metric_group == "Process":
-                            if value >= thresholds.cpu_limit:
-                                msg = "CGroup {0}: Crossed the Processor Threshold. " \
-                                      "Current Value: {1}, Threshold: {2}." \
-                                       .format(cgroup_name, value, thresholds.cpu_limit)
-
-                                logger.warn(msg)
-                                add_event(name=AGENT_NAME,
-                                          version=CURRENT_VERSION,
-                                          op=WALAEventOperation.CGroupsLimitsCrossed,
-                                          is_success=True,
-                                          message=msg,
-                                          log_event=True)
-
-            except Exception as e:
-                logger.warn("Monitor: failed to collect cgroups performance metrics: {0}", ustr(e))
-                logger.verbose(traceback.format_exc())
-
-            self.last_cgroup_telemetry = datetime.datetime.utcnow()
+    def poll_telemetry_metrics(self):
+        time_now = datetime.datetime.utcnow()
+        if not self.last_cgroup_polling_telemetry:
+            self.last_cgroup_polling_telemetry = time_now
+
+        if time_now >= (self.last_cgroup_polling_telemetry +
+                        MonitorHandler.CGROUP_TELEMETRY_POLLING_PERIOD):
+            CGroupsTelemetry.poll_all_tracked()
+            self.last_cgroup_polling_telemetry = time_now
+
+    def send_telemetry_metrics(self):
+        to_send = False
+        time_now = datetime.datetime.utcnow()
+        performance_metrics = {}
+
+        if not self.last_cgroup_report_telemetry:
+            self.last_cgroup_report_telemetry = time_now
+
+        if time_now >= (self.last_cgroup_report_telemetry +
+                        MonitorHandler.CGROUP_TELEMETRY_REPORTING_PERIOD):
+            performance_metrics = CGroupsTelemetry.report_all_tracked()
+            self.last_cgroup_report_telemetry = time_now
+            to_send = True
+
+        if to_send:
+            message = generate_extension_metrics_telemetry_dictionary(schema_version=1.0,
+                                                                      performance_metrics=performance_metrics)
+
+            add_event(name=AGENT_NAME,
+                      version=CURRENT_VERSION,
+                      op=WALAEventOperation.ExtensionMetricsData,
+                      is_success=True,
+                      message=json.dumps(message, separators=(',', ':')),
+                      log_event=False)
 
     def log_altered_network_configuration(self):
         """


Test patch:
diff --git a/tests/common/test_cgroupapi.py b/tests/common/test_cgroupapi.py
index f8b09661..f7f7ea6c 100644
--- a/tests/common/test_cgroupapi.py
+++ b/tests/common/test_cgroupapi.py
@@ -146,7 +146,7 @@ class FileSystemCgroupsApiTestCase(AgentTestCase):
 
         def assert_cgroup_created(controller):
             cgroup_path = os.path.join(self.cgroups_file_system_root, controller, "walinuxagent.service")
-            self.assertIn(cgroup_path, agent_cgroups)
+            self.assertTrue(any(cgroups.path == cgroup_path for cgroups in agent_cgroups))
             self.assertTrue(os.path.exists(cgroup_path))
             cgroup_task = int(fileutil.read_file(os.path.join(cgroup_path, "cgroup.procs")))
             current_process = os.getpid()
@@ -170,8 +170,10 @@ class FileSystemCgroupsApiTestCase(AgentTestCase):
         extension_cgroups = api.create_extension_cgroups("Microsoft.Compute.TestExtension-1.2.3")
 
         def assert_cgroup_created(controller):
-            cgroup_path = os.path.join(self.cgroups_file_system_root, controller, "walinuxagent.extensions", "Microsoft.Compute.TestExtension_1.2.3")
-            self.assertIn(cgroup_path, extension_cgroups)
+            cgroup_path = os.path.join(self.cgroups_file_system_root, controller, "walinuxagent.extensions",
+                                       "Microsoft.Compute.TestExtension_1.2.3")
+
+            self.assertTrue(any(cgroups.path == cgroup_path for cgroups in extension_cgroups))
             self.assertTrue(os.path.exists(cgroup_path))
 
         assert_cgroup_created("cpu")
@@ -185,7 +187,8 @@ class FileSystemCgroupsApiTestCase(AgentTestCase):
         api.remove_extension_cgroups("Microsoft.Compute.TestExtension-1.2.3")
 
         for cgroup in extension_cgroups:
-            self.assertFalse(os.path.exists(cgroup))
+            self.assertFalse(os.path.exists(cgroup.path))
+
 
     def test_get_extension_cgroups_should_return_all_cgroups(self):
         api = FileSystemCgroupsApi()
@@ -197,7 +200,7 @@ class FileSystemCgroupsApiTestCase(AgentTestCase):
         self.assertEqual(len(retrieved), len(created))
 
         for cgroup in created:
-            self.assertTrue(cgroup in retrieved)
+            self.assertTrue(any(retrieved_cgroup.path == cgroup.path for retrieved_cgroup in retrieved))
 
     def test_start_extension_command_should_add_the_child_process_to_the_extension_cgroup(self):
         api = FileSystemCgroupsApi()
@@ -216,7 +219,7 @@ class FileSystemCgroupsApiTestCase(AgentTestCase):
         process.wait()
 
         for cgroup in extension_cgroups:
-            cgroups_procs_path = os.path.join(cgroup, "cgroup.procs")
+            cgroups_procs_path = os.path.join(cgroup.path, "cgroup.procs")
             with open(cgroups_procs_path, "r") as f:
                 contents = f.read()
             pid = int(contents)
diff --git a/tests/common/test_cgroups.py b/tests/common/test_cgroups.py
index d7a4eb6d..26a75a0e 100644
--- a/tests/common/test_cgroups.py
+++ b/tests/common/test_cgroups.py
@@ -17,17 +17,15 @@
 
 from __future__ import print_function
 
-from azurelinuxagent.common.cgroup import CpuCgroup, MemoryCGroup
+import random
+
+from azurelinuxagent.common.cgroup import CpuCgroup, MemoryCgroup
 from azurelinuxagent.common.cgroupconfigurator import CGroupConfigurator
 from azurelinuxagent.common.cgroupstelemetry import CGroupsTelemetry
 from azurelinuxagent.common.exception import CGroupsException
 from azurelinuxagent.common.version import AGENT_NAME
 from tests.tools import *
 
-import os
-import random
-import time
-
 
 def consume_cpu_time():
     waste = 0
@@ -104,15 +102,15 @@ class TestCGroups(AgentTestCase):
         self.assertIn('memory', cg.cgroups)
         ct = CGroupsTelemetry("test", cg)
         cpu = CpuCgroup(ct)
-        self.assertGreater(cpu.current_system_cpu, 0)
+        self.assertGreater(cpu._current_system_cpu, 0)
 
         consume_cpu_time()  # Eat some CPU
-        cpu.update()
+        cpu._update_cpu_data()
 
-        self.assertGreater(cpu.current_cpu_total, cpu.previous_cpu_total)
-        self.assertGreater(cpu.current_system_cpu, cpu.previous_system_cpu)
+        self.assertGreater(cpu._current_cpu_total, cpu._previous_cpu_total)
+        self.assertGreater(cpu._current_system_cpu, cpu._previous_system_cpu)
 
-        percent_used = cpu.get_cpu_percent()
+        percent_used = cpu._get_cpu_percent()
         self.assertGreater(percent_used, 0)
 
     def test_telemetry_in_place_leaf_cgroup(self):
@@ -126,12 +124,12 @@ class TestCGroups(AgentTestCase):
         if cg.cgroups['cpu'] != root.cgroups['cpu']:
             ct = CGroupsTelemetry("test", cg)
             cpu = CpuCgroup(ct)
-            self.assertLess(cpu.current_cpu_total, cpu.current_system_cpu)
+            self.assertLess(cpu._current_cpu_total, cpu._current_system_cpu)
 
             consume_cpu_time()  # Eat some CPU
             time.sleep(1)  # Generate some idle time
-            cpu.update()
-            self.assertLess(cpu.current_cpu_total, cpu.current_system_cpu)
+            cpu._update_cpu_data()
+            self.assertLess(cpu._current_cpu_total, cpu._current_system_cpu)
 
     def exercise_telemetry_instantiation(self, test_cgroup):
         test_extension_name = test_cgroup.name
@@ -141,7 +139,7 @@ class TestCGroups(AgentTestCase):
         self.assertTrue(CGroupsTelemetry.is_tracked(test_extension_name))
         consume_cpu_time()
         time.sleep(1)
-        metrics, limits = CGroupsTelemetry.collect_all_tracked()
+        metrics, limits = CGroupsTelemetry.report_all_tracked()
         my_metrics = metrics[test_extension_name]
         self.assertEqual(len(my_metrics), 2)
         for item in my_metrics:
@@ -213,13 +211,13 @@ class TestCGroups(AgentTestCase):
         self.assertIs(cg, ct.cgroup)
         cpu = CpuCgroup(ct)
         self.assertIs(cg, cpu.cgt.cgroup)
-        ticks_before = cpu.current_cpu_total
+        ticks_before = cpu._current_cpu_total
         consume_cpu_time()
         time.sleep(1)
-        cpu.update()
-        ticks_after = cpu.current_cpu_total
+        cpu._update_cpu_data()
+        ticks_after = cpu._current_cpu_total
         self.assertGreater(ticks_after, ticks_before)
-        p2 = cpu.get_cpu_percent()
+        p2 = cpu._get_cpu_percent()
         self.assertGreater(p2, 0)
         # when running under PyCharm, this is often > 100
         # on a multi-core machine
@@ -236,8 +234,8 @@ class TestCGroups(AgentTestCase):
         self.assertIn('memory', cg.cgroups)
         ct = CGroupsTelemetry('test', cg)
         self.assertIs(cg, ct.cgroup)
-        memory = MemoryCGroup(ct)
-        usage_in_bytes = memory.get_memory_usage()
+        memory = MemoryCgroup(ct)
+        usage_in_bytes = memory._get_memory_usage()
         self.assertGreater(usage_in_bytes, 100000)
 
     def test_format_memory_value(self):
diff --git a/tests/common/test_cgroupstelemetry.py b/tests/common/test_cgroupstelemetry.py
new file mode 100644
index 00000000..2df72268
--- /dev/null
+++ b/tests/common/test_cgroupstelemetry.py
@@ -0,0 +1,419 @@
+# Copyright 2019 Microsoft Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Requires Python 2.4+ and Openssl 1.0+
+#
+
+
+import random
+
+from mock import patch
+
+from azurelinuxagent.common.cgroup import CGroup
+from azurelinuxagent.common.cgroupstelemetry import CGroupsTelemetry, Metric
+from tests.tools import AgentTestCase
+
+
+def median(lst):
+    data = sorted(lst)
+    l_len = len(data)
+    if l_len < 1:
+        return None
+    if l_len % 2 == 0:
+        return (data[int((l_len - 1) / 2)] + data[int((l_len + 1) / 2)]) / 2.0
+    else:
+        return data[int((l_len - 1) / 2)]
+
+
+def generate_metric_list(lst):
+    return [float(sum(lst)) / float(len(lst)),
+            min(lst),
+            max(lst),
+            median(lst),
+            len(lst)]
+
+
+class TestCGroupsTelemetry(AgentTestCase):
+    @staticmethod
+    def cleanup_cgroup_telemetry():
+        CGroupsTelemetry.cleanup()
+
+    @patch("azurelinuxagent.common.cgroup.CpuCgroup._get_current_cpu_total")
+    @patch("azurelinuxagent.common.osutil.default.DefaultOSUtil.get_total_cpu_ticks_since_boot")
+    def test_telemetry_polling_with_active_cgroups(self, *args):
+        self.cleanup_cgroup_telemetry()
+
+        num_extensions = 5
+        for i in range(num_extensions):
+            dummy_cpu_cgroup = CGroup.create("dummy_path", "cpu", "dummy_extension_{0}".format(i))
+            CGroupsTelemetry.track_cgroup(dummy_cpu_cgroup)
+
+            dummy_memory_cgroup = CGroup.create("dummy_path", "memory", "dummy_extension_{0}".format(i))
+            CGroupsTelemetry.track_cgroup(dummy_memory_cgroup)
+
+        with patch("azurelinuxagent.common.cgroup.MemoryCgroup._get_memory_max_usage") as patch_get_memory_max_usage:
+            with patch("azurelinuxagent.common.cgroup.MemoryCgroup._get_memory_usage") as patch_get_memory_usage:
+                with patch("azurelinuxagent.common.cgroup.CpuCgroup._get_cpu_percent") as patch_get_cpu_percent:
+                    with patch("azurelinuxagent.common.cgroup.CGroup.is_active") as patch_is_active:
+                        patch_is_active.return_value = True
+
+                        current_cpu = 30
+                        current_memory = 209715200
+                        current_max_memory = 471859200
+
+                        patch_get_cpu_percent.return_value = current_cpu
+                        patch_get_memory_usage.return_value = current_memory            # example 200 MB
+                        patch_get_memory_max_usage.return_value = current_max_memory    # example 450 MB
+
+                        poll_count = 1
+
+                        for data_count in range(poll_count, 10):
+                            CGroupsTelemetry.poll_all_tracked()
+                            self.assertEqual(CGroupsTelemetry._cgroup_metrics.__len__(), num_extensions)
+                            for cgroup_name, cgroup_metric in CGroupsTelemetry._cgroup_metrics.items():
+                                current_memory_usage, max_memory_levels, current_cpu_usage = cgroup_metric.get_metrics()
+                                self.assertEqual(len(current_memory_usage._data), data_count)
+                                self.assertListEqual(current_memory_usage._data, [current_memory] * data_count)
+                                self.assertEqual(len(max_memory_levels._data), data_count)
+                                self.assertListEqual(max_memory_levels._data, [current_max_memory] * data_count)
+                                self.assertEqual(len(current_cpu_usage._data), data_count)
+                                self.assertListEqual(current_cpu_usage._data, [current_cpu] * data_count)
+
+                        CGroupsTelemetry.report_all_tracked()
+
+                        self.assertEqual(CGroupsTelemetry._cgroup_metrics.__len__(), num_extensions)
+                        for cgroup_name, cgroup_metric in CGroupsTelemetry._cgroup_metrics.items():
+                            current_memory_usage, max_memory_levels, current_cpu_usage = cgroup_metric.get_metrics()
+                            self.assertEqual(len(current_memory_usage._data), 0)
+                            self.assertEqual(len(max_memory_levels._data), 0)
+                            self.assertEqual(len(current_cpu_usage._data), 0)
+
+        with patch("azurelinuxagent.common.cgroup.MemoryCgroup._get_memory_max_usage") as patch_get_memory_max_usage:
+            with patch("azurelinuxagent.common.cgroup.MemoryCgroup._get_memory_usage") as patch_get_memory_usage:
+                with patch("azurelinuxagent.common.cgroup.CpuCgroup._get_cpu_percent") as patch_get_cpu_percent:
+                    with patch("azurelinuxagent.common.cgroup.CGroup.is_active") as patch_is_active:
+                        patch_is_active.return_value = True
+
+                        current_cpu = 30
+                        current_memory = 209715200
+                        current_max_memory = 471859200
+
+                        patch_get_cpu_percent.return_value = current_cpu
+                        patch_get_memory_usage.return_value = current_memory            # example 200 MB
+                        patch_get_memory_max_usage.return_value = current_max_memory    # example 450 MB
+
+                        poll_count = 1
+
+                        for data_count in range(poll_count, 10):
+                            CGroupsTelemetry.poll_all_tracked()
+                            self.assertEqual(CGroupsTelemetry._cgroup_metrics.__len__(), num_extensions)
+                            for cgroup_name, cgroup_metric in CGroupsTelemetry._cgroup_metrics.items():
+                                current_memory_usage, max_memory_levels, current_cpu_usage = cgroup_metric.get_metrics()
+                                self.assertEqual(len(current_memory_usage._data), data_count)
+                                self.assertListEqual(current_memory_usage._data, [current_memory] * data_count)
+                                self.assertEqual(len(max_memory_levels._data), data_count)
+                                self.assertListEqual(max_memory_levels._data, [current_max_memory] * data_count)
+                                self.assertEqual(len(current_cpu_usage._data), data_count)
+                                self.assertListEqual(current_cpu_usage._data, [current_cpu] * data_count)
+
+                        CGroupsTelemetry.report_all_tracked()
+
+                        self.assertEqual(CGroupsTelemetry._cgroup_metrics.__len__(), num_extensions)
+                        for cgroup_name, cgroup_metric in CGroupsTelemetry._cgroup_metrics.items():
+                            current_memory_usage, max_memory_levels, current_cpu_usage = cgroup_metric.get_metrics()
+                            self.assertEqual(len(current_memory_usage._data), 0)
+                            self.assertEqual(len(max_memory_levels._data), 0)
+                            self.assertEqual(len(current_cpu_usage._data), 0)
+
+        self.cleanup_cgroup_telemetry()
+
+    @patch("azurelinuxagent.common.cgroup.CpuCgroup._get_current_cpu_total")
+    @patch("azurelinuxagent.common.osutil.default.DefaultOSUtil.get_total_cpu_ticks_since_boot")
+    def test_telemetry_polling_with_inactive_cgroups(self, *args):
+        self.cleanup_cgroup_telemetry()
+
+        num_extensions = 5
+        for i in range(num_extensions):
+            dummy_cpu_cgroup = CGroup.create("dummy_path", "cpu", "dummy_extension_{0}".format(i))
+            CGroupsTelemetry.track_cgroup(dummy_cpu_cgroup)
+
+            dummy_memory_cgroup = CGroup.create("dummy_path", "memory", "dummy_extension_{0}".format(i))
+            CGroupsTelemetry.track_cgroup(dummy_memory_cgroup)
+
+        with patch("azurelinuxagent.common.cgroup.MemoryCgroup._get_memory_max_usage") as patch_get_memory_max_usage:
+            with patch("azurelinuxagent.common.cgroup.MemoryCgroup._get_memory_usage") as patch_get_memory_usage:
+                with patch("azurelinuxagent.common.cgroup.CpuCgroup._get_cpu_percent") as patch_get_cpu_percent:
+                    with patch("azurelinuxagent.common.cgroup.CGroup.is_active") as patch_is_active:
+                        patch_is_active.return_value = False
+
+                        no_extensions_expected = 0
+                        data_count = 1
+                        current_cpu = 30
+                        current_memory = 209715200
+                        current_max_memory = 471859200
+
+                        patch_get_cpu_percent.return_value = current_cpu
+                        patch_get_memory_usage.return_value = current_memory  # example 200 MB
+                        patch_get_memory_max_usage.return_value = current_max_memory  # example 450 MB
+
+                        for i in range(num_extensions):
+                            self.assertTrue(CGroupsTelemetry.is_tracked("dummy_extension_{0}".format(i), "cpu"))
+                            self.assertTrue(CGroupsTelemetry.is_tracked("dummy_extension_{0}".format(i), "memory"))
+
+                        CGroupsTelemetry.poll_all_tracked()
+
+                        for i in range(num_extensions):
+                            self.assertFalse(CGroupsTelemetry.is_tracked("dummy_extension_{0}".format(i), "cpu"))
+                            self.assertFalse(CGroupsTelemetry.is_tracked("dummy_extension_{0}".format(i), "memory"))
+
+                        self.assertEqual(CGroupsTelemetry._cgroup_metrics.__len__(), num_extensions)
+
+                        for cgroup_name, cgroup_metric in CGroupsTelemetry._cgroup_metrics.items():
+                            current_memory_usage, max_memory_levels, current_cpu_usage = cgroup_metric.get_metrics()
+                            self.assertEqual(len(current_memory_usage._data), data_count)
+                            self.assertListEqual(current_memory_usage._data, [current_memory] * data_count)
+                            self.assertEqual(len(max_memory_levels._data), data_count)
+                            self.assertListEqual(max_memory_levels._data, [current_max_memory] * data_count)
+                            self.assertEqual(len(current_cpu_usage._data), data_count)
+                            self.assertListEqual(current_cpu_usage._data, [current_cpu] * data_count)
+
+                        CGroupsTelemetry.report_all_tracked()
+
+                        self.assertEqual(CGroupsTelemetry._cgroup_metrics.__len__(), no_extensions_expected)
+                        for cgroup_name, cgroup_metric in CGroupsTelemetry._cgroup_metrics.items():
+                            current_memory_usage, max_memory_levels, current_cpu_usage = cgroup_metric.get_metrics()
+                            self.assertEqual(len(current_memory_usage._data), 0)
+                            self.assertEqual(len(max_memory_levels._data), 0)
+                            self.assertEqual(len(current_cpu_usage._data), 0)
+
+        self.cleanup_cgroup_telemetry()
+
+    @patch("azurelinuxagent.common.cgroup.CpuCgroup._get_current_cpu_total")
+    @patch("azurelinuxagent.common.osutil.default.DefaultOSUtil.get_total_cpu_ticks_since_boot")
+    def test_telemetry_polling_with_changing_cgroups_state(self, *args):
+        self.cleanup_cgroup_telemetry()
+
+        num_extensions = 5
+        for i in range(num_extensions):
+            dummy_cpu_cgroup = CGroup.create("dummy_path", "cpu", "dummy_extension_{0}".format(i))
+            CGroupsTelemetry.track_cgroup(dummy_cpu_cgroup)
+
+            dummy_memory_cgroup = CGroup.create("dummy_path", "memory", "dummy_extension_{0}".format(i))
+            CGroupsTelemetry.track_cgroup(dummy_memory_cgroup)
+
+        with patch("azurelinuxagent.common.cgroup.MemoryCgroup._get_memory_max_usage") as patch_get_memory_max_usage:
+            with patch("azurelinuxagent.common.cgroup.MemoryCgroup._get_memory_usage") as patch_get_memory_usage:
+                with patch("azurelinuxagent.common.cgroup.CpuCgroup._get_cpu_percent") as patch_get_cpu_percent:
+                    with patch("azurelinuxagent.common.cgroup.CGroup.is_active") as patch_is_active:
+                        patch_is_active.return_value = True
+
+                        no_extensions_expected = 0
+                        expected_data_count = 2
+
+                        current_cpu = 30
+                        current_memory = 209715200
+                        current_max_memory = 471859200
+
+                        patch_get_cpu_percent.return_value = current_cpu
+                        patch_get_memory_usage.return_value = current_memory  # example 200 MB
+                        patch_get_memory_max_usage.return_value = current_max_memory  # example 450 MB
+
+                        for i in range(num_extensions):
+                            self.assertTrue(CGroupsTelemetry.is_tracked("dummy_extension_{0}".format(i), "cpu"))
+                            self.assertTrue(CGroupsTelemetry.is_tracked("dummy_extension_{0}".format(i), "memory"))
+
+                        CGroupsTelemetry.poll_all_tracked()
+
+                        for i in range(num_extensions):
+                            self.assertTrue(CGroupsTelemetry.is_tracked("dummy_extension_{0}".format(i), "cpu"))
+                            self.assertTrue(CGroupsTelemetry.is_tracked("dummy_extension_{0}".format(i), "memory"))
+
+                        self.assertEqual(CGroupsTelemetry._cgroup_metrics.__len__(), num_extensions)
+
+                        patch_is_active.return_value = False
+                        CGroupsTelemetry.poll_all_tracked()
+
+                        for i in range(num_extensions):
+                            self.assertFalse(CGroupsTelemetry.is_tracked("dummy_extension_{0}".format(i), "cpu"))
+                            self.assertFalse(CGroupsTelemetry.is_tracked("dummy_extension_{0}".format(i), "memory"))
+
+                        self.assertEqual(CGroupsTelemetry._cgroup_metrics.__len__(), num_extensions)
+
+                        for cgroup_name, cgroup_metric in CGroupsTelemetry._cgroup_metrics.items():
+                            current_memory_usage, max_memory_levels, current_cpu_usage = cgroup_metric.get_metrics()
+                            self.assertEqual(len(current_memory_usage._data), expected_data_count)
+                            self.assertListEqual(current_memory_usage._data, [current_memory] * expected_data_count)
+                            self.assertEqual(len(max_memory_levels._data), expected_data_count)
+                            self.assertListEqual(max_memory_levels._data, [current_max_memory] * expected_data_count)
+                            self.assertEqual(len(current_cpu_usage._data), expected_data_count)
+                            self.assertListEqual(current_cpu_usage._data, [current_cpu] * expected_data_count)
+
+                        CGroupsTelemetry.report_all_tracked()
+
+                        self.assertEqual(CGroupsTelemetry._cgroup_metrics.__len__(), no_extensions_expected)
+                        for cgroup_name, cgroup_metric in CGroupsTelemetry._cgroup_metrics.items():
+                            current_memory_usage, max_memory_levels, current_cpu_usage = cgroup_metric.get_metrics()
+                            self.assertEqual(len(current_memory_usage._data), 0)
+                            self.assertEqual(len(max_memory_levels._data), 0)
+                            self.assertEqual(len(current_cpu_usage._data), 0)
+
+        self.cleanup_cgroup_telemetry()
+
+    @patch("azurelinuxagent.common.osutil.default.DefaultOSUtil.get_total_cpu_ticks_since_boot")
+    @patch("azurelinuxagent.common.cgroup.CpuCgroup._get_current_cpu_total")
+    @patch("azurelinuxagent.common.cgroup.CpuCgroup._update_cpu_data")
+    def test_telemetry_calculations(self, *args):
+        self.cleanup_cgroup_telemetry()
+
+        num_polls = 10
+        num_extensions = 1
+        num_summarization_values = 7
+
+        cpu_percent_values = [random.randint(0, 100) for _ in range(num_polls)]
+
+        # only verifying calculations and not validity of the values.
+        memory_usage_values = [random.randint(0, 8 * 1024 ** 3) for _ in range(num_polls)]
+        max_memory_usage_values = [random.randint(0, 8 * 1024 ** 3) for _ in range(num_polls)]
+
+        for i in range(num_extensions):
+            dummy_cpu_cgroup = CGroup.create("dummy_path", "cpu", "dummy_extension_{0}".format(i))
+            CGroupsTelemetry.track_cgroup(dummy_cpu_cgroup)
+
+            dummy_memory_cgroup = CGroup.create("dummy_path", "memory", "dummy_extension_{0}".format(i))
+            CGroupsTelemetry.track_cgroup(dummy_memory_cgroup)
+
+        self.assertEqual(2 * num_extensions, len(CGroupsTelemetry._tracked))
+
+        with patch("azurelinuxagent.common.cgroup.MemoryCgroup._get_memory_max_usage") as patch_get_memory_max_usage:
+            with patch("azurelinuxagent.common.cgroup.MemoryCgroup._get_memory_usage") as patch_get_memory_usage:
+                with patch("azurelinuxagent.common.cgroup.CpuCgroup._get_cpu_percent") as patch_get_cpu_percent:
+                    with patch("azurelinuxagent.common.cgroup.CGroup.is_active") as patch_is_active:
+                        for i in range(num_polls):
+                            patch_is_active.return_value = True
+                            patch_get_cpu_percent.return_value = cpu_percent_values[i]
+                            patch_get_memory_usage.return_value = memory_usage_values[i]  # example 200 MB
+                            patch_get_memory_max_usage.return_value = max_memory_usage_values[i]  # example 450 MB
+                            CGroupsTelemetry.poll_all_tracked()
+
+        collected_metrics = CGroupsTelemetry.report_all_tracked()
+        for i in range(num_extensions):
+            name = "dummy_extension_{0}".format(i)
+
+            self.assertIn(name, collected_metrics)
+            self.assertIn("memory", collected_metrics[name])
+            self.assertIn("cur_mem", collected_metrics[name]["memory"])
+            self.assertIn("max_mem", collected_metrics[name]["memory"])
+            self.assertEqual(num_summarization_values, len(collected_metrics[name]["memory"]["cur_mem"]))
+            self.assertEqual(num_summarization_values, len(collected_metrics[name]["memory"]["max_mem"]))
+
+            self.assertListEqual(generate_metric_list(memory_usage_values),
+                                 collected_metrics[name]["memory"]["cur_mem"][0:5])
+            self.assertListEqual(generate_metric_list(max_memory_usage_values),
+                                 collected_metrics[name]["memory"]["max_mem"][0:5])
+
+            self.assertIn("cpu", collected_metrics[name])
+            self.assertIn("cur_cpu", collected_metrics[name]["cpu"])
+            self.assertEqual(num_summarization_values, len(collected_metrics[name]["cpu"]["cur_cpu"]))
+            self.assertListEqual(generate_metric_list(cpu_percent_values),
+                                 collected_metrics[name]["cpu"]["cur_cpu"][0:5])
+
+        self.cleanup_cgroup_telemetry()
+
+    def test_cgroup_tracking(self):
+        self.cleanup_cgroup_telemetry()
+
+        num_extensions = 5
+        num_controllers = 2
+        for i in range(num_extensions):
+            dummy_cpu_cgroup = CGroup.create("dummy_path", "cpu", "dummy_extension_{0}".format(i))
+            CGroupsTelemetry.track_cgroup(dummy_cpu_cgroup)
+
+            dummy_memory_cgroup = CGroup.create("dummy_path", "memory", "dummy_extension_{0}".format(i))
+            CGroupsTelemetry.track_cgroup(dummy_memory_cgroup)
+
+        for i in range(num_extensions):
+            self.assertTrue(CGroupsTelemetry.is_tracked("dummy_extension_{0}".format(i), "cpu"))
+            self.assertTrue(CGroupsTelemetry.is_tracked("dummy_extension_{0}".format(i), "memory"))
+
+        self.assertEqual(num_extensions * num_controllers, len(CGroupsTelemetry._tracked))
+
+    def test_cgroup_pruning(self):
+        self.cleanup_cgroup_telemetry()
+
+        num_extensions = 5
+        num_controllers = 2
+        for i in range(num_extensions):
+            dummy_cpu_cgroup = CGroup.create("dummy_path", "cpu", "dummy_extension_{0}".format(i))
+            CGroupsTelemetry.track_cgroup(dummy_cpu_cgroup)
+
+            dummy_memory_cgroup = CGroup.create("dummy_path", "memory", "dummy_extension_{0}".format(i))
+            CGroupsTelemetry.track_cgroup(dummy_memory_cgroup)
+
+        for i in range(num_extensions):
+            self.assertTrue(CGroupsTelemetry.is_tracked("dummy_extension_{0}".format(i), "cpu"))
+            self.assertTrue(CGroupsTelemetry.is_tracked("dummy_extension_{0}".format(i), "memory"))
+
+        self.assertEqual(num_extensions * num_controllers, len(CGroupsTelemetry._tracked))
+
+        CGroupsTelemetry.prune_all_tracked()
+
+        for i in range(num_extensions):
+            self.assertFalse(CGroupsTelemetry.is_tracked("dummy_extension_{0}".format(i), "cpu"))
+            self.assertFalse(CGroupsTelemetry.is_tracked("dummy_extension_{0}".format(i), "memory"))
+
+        self.assertEqual(0, len(CGroupsTelemetry._tracked))
+
+    def test_cgroup_is_tracked(self):
+        self.cleanup_cgroup_telemetry()
+
+        num_extensions = 5
+        for i in range(num_extensions):
+            dummy_cpu_cgroup = CGroup.create("dummy_path", "cpu", "dummy_extension_{0}".format(i))
+            CGroupsTelemetry.track_cgroup(dummy_cpu_cgroup)
+
+            dummy_memory_cgroup = CGroup.create("dummy_path", "memory", "dummy_extension_{0}".format(i))
+            CGroupsTelemetry.track_cgroup(dummy_memory_cgroup)
+
+        for i in range(num_extensions):
+            self.assertTrue(CGroupsTelemetry.is_tracked("dummy_extension_{0}".format(i), 'cpu'))
+            self.assertTrue(CGroupsTelemetry.is_tracked("dummy_extension_{0}".format(i), 'memory'))
+
+        self.assertFalse(CGroupsTelemetry.is_tracked("not_present_dummy_extension_{0}".format(i), 'cpu'))
+        self.assertFalse(CGroupsTelemetry.is_tracked("not_present_dummy_extension_{0}".format(i), 'memory'))
+
+
+class TestMetric(AgentTestCase):
+    def test_empty_metrics(self):
+        test_metric = Metric()
+        self.assertEqual("None", test_metric.first_poll_time())
+        self.assertEqual("None", test_metric.last_poll_time())
+        self.assertEqual(0, test_metric.count())
+        self.assertEqual(None, test_metric.median())
+        self.assertEqual(None, test_metric.max())
+        self.assertEqual(None, test_metric.min())
+        self.assertEqual(None, test_metric.average())
+        # self.assertEqual("None", test_metric.append())
+
+    def test_metrics(self):
+        num_polls = 10
+
+        test_values = [random.randint(0, 100) for _ in range(num_polls)]
+
+        test_metric = Metric()
+        for value in test_values:
+            test_metric.append(value)
+
+        self.assertListEqual(generate_metric_list(test_values), [test_metric.average(), test_metric.min(),
+                                                                 test_metric.max(), test_metric.median(),
+                                                                 test_metric.count()])
diff --git a/tests/ga/test_exthandlers.py b/tests/ga/test_exthandlers.py
index 423aec3b..1d59f786 100644
--- a/tests/ga/test_exthandlers.py
+++ b/tests/ga/test_exthandlers.py
@@ -594,4 +594,3 @@ sys.stderr.write("STDERR")
             output = self.ext_handler_instance.launch_command(command)
 
         self.assertIn("[stderr]\nCannot read stdout/stderr:", output)
-
diff --git a/tests/ga/test_exthandlers_download_extension.py b/tests/ga/test_exthandlers_download_extension.py
index ddb2f5ce..cbaa1890 100644
--- a/tests/ga/test_exthandlers_download_extension.py
+++ b/tests/ga/test_exthandlers_download_extension.py
@@ -9,6 +9,7 @@ from azurelinuxagent.ga.exthandlers import ExtHandlerInstance, NUMBER_OF_DOWNLOA
 from azurelinuxagent.common.exception import ExtensionDownloadError, ExtensionErrorCodes
 from tests.tools import *
 
+
 class DownloadExtensionTestCase(AgentTestCase):
     """
     Test cases for launch_command
@@ -19,12 +20,8 @@ class DownloadExtensionTestCase(AgentTestCase):
         cls.mock_cgroups = patch("azurelinuxagent.ga.exthandlers.CGroupConfigurator")
         cls.mock_cgroups.start()
 
-        cls.mock_cgroups_telemetry = patch("azurelinuxagent.ga.exthandlers.CGroupsTelemetry")
-        cls.mock_cgroups_telemetry.start()
-
     @classmethod
     def tearDownClass(cls):
-        cls.mock_cgroups_telemetry.stop()
         cls.mock_cgroups.stop()
 
         AgentTestCase.tearDownClass()
diff --git a/tests/ga/test_monitor.py b/tests/ga/test_monitor.py
index 54bd00f1..5dc1bab4 100644
--- a/tests/ga/test_monitor.py
+++ b/tests/ga/test_monitor.py
@@ -17,8 +17,8 @@
 from datetime import timedelta
 
 from azurelinuxagent.common.protocol.wire import WireProtocol
-from tests.tools import *
 from azurelinuxagent.ga.monitor import *
+from tests.tools import *
 
 
 @patch('azurelinuxagent.common.event.EventLogger.add_event')
@@ -87,11 +87,13 @@ class TestMonitor(AgentTestCase):
     @patch("azurelinuxagent.ga.monitor.MonitorHandler.send_telemetry_heartbeat")
     @patch("azurelinuxagent.ga.monitor.MonitorHandler.collect_and_send_events")
     @patch("azurelinuxagent.ga.monitor.MonitorHandler.send_host_plugin_heartbeat")
-    @patch("azurelinuxagent.ga.monitor.MonitorHandler.send_cgroup_telemetry")
+    @patch("azurelinuxagent.ga.monitor.MonitorHandler.poll_telemetry_metrics")
+    @patch("azurelinuxagent.ga.monitor.MonitorHandler.send_telemetry_metrics")
     @patch("azurelinuxagent.ga.monitor.MonitorHandler.send_imds_heartbeat")
     def test_heartbeats(self,
                         patch_imds_heartbeat,
-                        patch_cgroup_telemetry,
+                        patch_send_telemetry_metrics,
+                        patch_poll_telemetry_metrics,
                         patch_hostplugin_heartbeat,
                         patch_send_events,
                         patch_telemetry_heartbeat,
@@ -107,7 +109,8 @@ class TestMonitor(AgentTestCase):
         self.assertEqual(0, patch_send_events.call_count)
         self.assertEqual(0, patch_telemetry_heartbeat.call_count)
         self.assertEqual(0, patch_imds_heartbeat.call_count)
-        self.assertEqual(0, patch_cgroup_telemetry.call_count)
+        self.assertEqual(0, patch_send_telemetry_metrics.call_count)
+        self.assertEqual(0, patch_poll_telemetry_metrics.call_count)
 
         monitor_handler.start()
         time.sleep(1)
@@ -117,11 +120,13 @@ class TestMonitor(AgentTestCase):
         self.assertNotEqual(0, patch_send_events.call_count)
         self.assertNotEqual(0, patch_telemetry_heartbeat.call_count)
         self.assertNotEqual(0, patch_imds_heartbeat.call_count)
-        self.assertNotEqual(0, patch_cgroup_telemetry.call_count)
+        self.assertNotEqual(0, patch_send_telemetry_metrics.call_count)
+        self.assertNotEqual(0, patch_poll_telemetry_metrics.call_count)
 
         monitor_handler.stop()
 
-    @patch("azurelinuxagent.ga.monitor.MonitorHandler.send_cgroup_telemetry")
+    @patch("azurelinuxagent.ga.monitor.MonitorHandler.send_telemetry_metrics")
+    @patch("azurelinuxagent.ga.monitor.MonitorHandler.poll_telemetry_metrics")
     def test_heartbeat_timings_updates_after_window(self, *args):
         monitor_handler = get_monitor_handler()
 
@@ -158,7 +163,8 @@ class TestMonitor(AgentTestCase):
 
         monitor_handler.stop()
 
-    @patch("azurelinuxagent.ga.monitor.MonitorHandler.send_cgroup_telemetry")
+    @patch("azurelinuxagent.ga.monitor.MonitorHandler.send_telemetry_metrics")
+    @patch("azurelinuxagent.ga.monitor.MonitorHandler.poll_telemetry_metrics")
     def test_heartbeat_timings_no_updates_within_window(self, *args):
         monitor_handler = get_monitor_handler()
 
@@ -218,6 +224,30 @@ class TestMonitor(AgentTestCase):
         self.assertEqual(False, args[5].call_args[1]['is_success'])
         monitor_handler.stop()
 
+    @patch("azurelinuxagent.common.event.add_event")
+    @patch("azurelinuxagent.common.cgroupstelemetry.CGroupsTelemetry.poll_all_tracked")
+    @patch("azurelinuxagent.common.cgroupstelemetry.CGroupsTelemetry.report_all_tracked")
+    def test_send_extension_metrics_telemetry(self, patch_report_all_tracked, patch_poll_all_tracked, *args):
+        patch_report_all_tracked.return_value = {
+            "memory": {
+                "cur_mem": [1, 1, 1, 1, 1, str(datetime.datetime.utcnow()), str(datetime.datetime.utcnow())],
+                "max_mem": [1, 1, 1, 1, 1, str(datetime.datetime.utcnow()), str(datetime.datetime.utcnow())]
+            },
+            "cpu": {
+                "cur_cpu": [1, 1, 1, 1, 1, str(datetime.datetime.utcnow()), str(datetime.datetime.utcnow())]
+            }
+        }
+
+        monitor_handler = get_monitor_handler()
+        monitor_handler.init_protocols()
+        monitor_handler.last_cgroup_polling_telemetry = datetime.datetime.utcnow() - timedelta(hours=1)
+        monitor_handler.last_cgroup_report_telemetry = datetime.datetime.utcnow() - timedelta(hours=1)
+        monitor_handler.poll_telemetry_metrics()
+        monitor_handler.send_telemetry_metrics()
+        self.assertEqual(1, patch_poll_all_tracked.call_count)
+        self.assertEqual(1, patch_report_all_tracked.call_count)
+        monitor_handler.stop()
+
 
 @patch('azurelinuxagent.common.event.EventLogger.add_event')
 @patch("azurelinuxagent.common.utils.restutil.http_post")