Repository: ros-tooling/cross_compile

Issue:
Report if an artifact is created per stage
## Description

Getting the number of artifacts (Docker images) created per stage allows us to see which stages contribute to the total storage footprint of the tool, and why certain artifacts may contribute more than others.

## Related Issues

Part of #205 

## Completion Criteria

For each stage, log if an artifact is created and what its name is.

## Implementation Notes / Suggestions

None

## Testing Notes / Suggestions

For each stage, the accurate number of artifacts is output.


Discussion:

Patch:
diff --git a/ros_cross_compile/builders.py b/ros_cross_compile/builders.py
index 92e029f..b3c72e7 100644
--- a/ros_cross_compile/builders.py
+++ b/ros_cross_compile/builders.py
@@ -15,6 +15,7 @@ import logging
 import os
 from pathlib import Path
 
+from ros_cross_compile.data_collector import DataCollector
 from ros_cross_compile.docker_client import DockerClient
 from ros_cross_compile.pipeline_stages import PipelineStage
 from ros_cross_compile.pipeline_stages import PipelineStageConfigOptions
@@ -60,5 +61,6 @@ class DockerBuildStage(PipelineStage):
         super().__init__('run_emulated_docker_build')
 
     def __call__(self, platform: Platform, docker_client: DockerClient, ros_workspace_dir: Path,
-                 pipeline_stage_config_options: PipelineStageConfigOptions):
+                 pipeline_stage_config_options: PipelineStageConfigOptions,
+                 data_collector: DataCollector):
         run_emulated_docker_build(docker_client, platform, ros_workspace_dir)
diff --git a/ros_cross_compile/data_collector.py b/ros_cross_compile/data_collector.py
index 4732419..1589656 100644
--- a/ros_cross_compile/data_collector.py
+++ b/ros_cross_compile/data_collector.py
@@ -21,7 +21,8 @@ from pathlib import Path
 import time
 from typing import Dict, List, NamedTuple, Union
 
-from ros_cross_compile.sysroot_creator import INTERNALS_DIR
+
+INTERNALS_DIR = 'cc_internals'
 
 
 Datum = NamedTuple('Datum', [('name', str),
@@ -58,10 +59,16 @@ class DataCollector:
             complete = True
         finally:
             elapsed = time.monotonic() - start
-            time_metric = Datum(name + '-time', elapsed,
+            time_metric = Datum('{}-time'.format(name), elapsed,
                                 Units.Seconds.value, time.monotonic(), complete)
             self.add_datum(time_metric)
 
+    def add_size(self, name: str, size: int):
+        """Provide an interface to add collected Docker image sizes."""
+        size_metric = Datum('{}-size'.format(name), size,
+                            Units.Bytes.value, time.monotonic(), True)
+        self.add_datum(size_metric)
+
 
 class DataWriter:
     """Provides an interface to write collected data to a file."""
diff --git a/ros_cross_compile/dependencies.py b/ros_cross_compile/dependencies.py
index bee727d..9e18969 100644
--- a/ros_cross_compile/dependencies.py
+++ b/ros_cross_compile/dependencies.py
@@ -18,6 +18,7 @@ from pathlib import Path
 from typing import List
 from typing import Optional
 
+from ros_cross_compile.data_collector import DataCollector
 from ros_cross_compile.docker_client import DockerClient
 from ros_cross_compile.pipeline_stages import PipelineStage
 from ros_cross_compile.pipeline_stages import PipelineStageConfigOptions
@@ -30,6 +31,7 @@ logger = logging.getLogger('Rosdep Gatherer')
 
 CUSTOM_SETUP = '/usercustom/rosdep_setup'
 CUSTOM_DATA = '/usercustom/custom-data'
+_IMG_NAME = 'ros_cross_compile:rosdep'
 
 
 def rosdep_install_script(platform: Platform) -> Path:
@@ -57,11 +59,10 @@ def gather_rosdeps(
     """
     out_path = rosdep_install_script(platform)
 
-    image_name = 'ros_cross_compile:rosdep'
-    logger.info('Building rosdep collector image: %s', image_name)
+    logger.info('Building rosdep collector image: %s', _IMG_NAME)
     docker_client.build_image(
         dockerfile_name='rosdep.Dockerfile',
-        tag=image_name,
+        tag=_IMG_NAME,
     )
 
     logger.info('Running rosdep collector image on workspace {}'.format(workspace))
@@ -74,7 +75,7 @@ def gather_rosdeps(
         volumes[custom_data_dir] = CUSTOM_DATA
 
     docker_client.run_container(
-        image_name=image_name,
+        image_name=_IMG_NAME,
         environment={
             'CUSTOM_SETUP': CUSTOM_SETUP,
             'OUT_PATH': str(out_path),
@@ -111,10 +112,13 @@ class DependenciesStage(PipelineStage):
         super().__init__('gather_rosdeps')
 
     def __call__(self, platform: Platform, docker_client: DockerClient, ros_workspace_dir: Path,
-                 pipeline_stage_config_options: PipelineStageConfigOptions):
+                 pipeline_stage_config_options: PipelineStageConfigOptions,
+                 data_collector: DataCollector):
         """
         Run the inspection and output the dependency installation script.
 
+        Also recovers the size of the docker image generated.
+
         :raises RuntimeError if the step was skipped when no dependency script has been
         previously generated
         """
@@ -129,3 +133,6 @@ class DependenciesStage(PipelineStage):
                 custom_script=pipeline_stage_config_options.custom_script,
                 custom_data_dir=pipeline_stage_config_options.custom_data_dir)
         assert_install_rosdep_script_exists(ros_workspace_dir, platform)
+
+        img_size = docker_client.get_image_size(_IMG_NAME)
+        data_collector.add_size(self.name, img_size)
diff --git a/ros_cross_compile/docker_client.py b/ros_cross_compile/docker_client.py
index 5578142..846ba42 100644
--- a/ros_cross_compile/docker_client.py
+++ b/ros_cross_compile/docker_client.py
@@ -142,3 +142,6 @@ class DockerClient:
         if exit_code:
             raise docker.errors.ContainerError(
                 image_name, exit_code, '', image_name, 'See above ^')
+
+    def get_image_size(self, img_name: str) -> int:
+        return self._client.images.get(img_name).attrs['Size']
diff --git a/ros_cross_compile/pipeline_stages.py b/ros_cross_compile/pipeline_stages.py
index 4078e58..1aa429c 100644
--- a/ros_cross_compile/pipeline_stages.py
+++ b/ros_cross_compile/pipeline_stages.py
@@ -16,6 +16,7 @@ from abc import ABC, abstractmethod
 from pathlib import Path
 from typing import List, NamedTuple, Optional
 
+from ros_cross_compile.data_collector import DataCollector
 from ros_cross_compile.docker_client import DockerClient
 from ros_cross_compile.platform import Platform
 
@@ -47,5 +48,6 @@ class PipelineStage(ABC):
 
     @abstractmethod
     def __call__(self, platform: Platform, docker_client: DockerClient, ros_workspace_dir: Path,
-                 customizations: PipelineStageConfigOptions):
+                 pipeline_stage_config_options: PipelineStageConfigOptions,
+                 data_collector: DataCollector):
         raise NotImplementedError
diff --git a/ros_cross_compile/ros_cross_compile.py b/ros_cross_compile/ros_cross_compile.py
index 8fdd0da..d823cd7 100644
--- a/ros_cross_compile/ros_cross_compile.py
+++ b/ros_cross_compile/ros_cross_compile.py
@@ -186,7 +186,7 @@ def cross_compile_pipeline(
 
     for stage in stages:
         with data_collector.timer('cross_compile_{}'.format(stage.name)):
-            stage(platform, docker_client, ros_workspace_dir, customizations)
+            stage(platform, docker_client, ros_workspace_dir, customizations, data_collector)
 
 
 def main():
diff --git a/ros_cross_compile/sysroot_creator.py b/ros_cross_compile/sysroot_creator.py
index a03cff0..d8d772d 100755
--- a/ros_cross_compile/sysroot_creator.py
+++ b/ros_cross_compile/sysroot_creator.py
@@ -19,6 +19,8 @@ import platform as py_platform
 import shutil
 from typing import Optional
 
+from ros_cross_compile.data_collector import DataCollector
+from ros_cross_compile.data_collector import INTERNALS_DIR
 from ros_cross_compile.docker_client import DockerClient
 from ros_cross_compile.pipeline_stages import PipelineStage
 from ros_cross_compile.pipeline_stages import PipelineStageConfigOptions
@@ -27,8 +29,6 @@ from ros_cross_compile.platform import Platform
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
-INTERNALS_DIR = 'cc_internals'
-
 
 def build_internals_dir(platform: Platform) -> Path:
     """Construct a relative path for this build, for storing intermediate artifacts."""
@@ -135,5 +135,9 @@ class CreateSysrootStage(PipelineStage):
         super().__init__('create_workspace_sysroot_image')
 
     def __call__(self, platform: Platform, docker_client: DockerClient, ros_workspace_dir: Path,
-                 pipeline_stage_config_options: PipelineStageConfigOptions):
+                 pipeline_stage_config_options: PipelineStageConfigOptions,
+                 data_collector: DataCollector):
         create_workspace_sysroot_image(docker_client, platform)
+
+        img_size = docker_client.get_image_size(platform.sysroot_image_tag)
+        data_collector.add_size(self.name, img_size)


Test patch:
diff --git a/test/test_builders.py b/test/test_builders.py
index 4292c96..1e51cc8 100644
--- a/test/test_builders.py
+++ b/test/test_builders.py
@@ -23,13 +23,15 @@ from ros_cross_compile.platform import Platform
 def test_emulated_docker_build():
     # Very simple smoke test to validate that all internal syntax is correct
     mock_docker_client = Mock()
+    mock_data_collector = Mock()
     platform = Platform('aarch64', 'ubuntu', 'eloquent')
 
     # a default set of customizations for the docker build stage
     customizations = PipelineStageConfigOptions(False, [], None, None, None)
     temp_stage = DockerBuildStage()
 
-    temp_stage(platform, mock_docker_client, Path('dummy_path'), customizations)
+    temp_stage(platform, mock_docker_client,
+               Path('dummy_path'), customizations, mock_data_collector)
 
     assert mock_docker_client.run_container.call_count == 1
 
diff --git a/test/test_dependencies.py b/test/test_dependencies.py
index 5e35ded..e29394e 100644
--- a/test/test_dependencies.py
+++ b/test/test_dependencies.py
@@ -17,6 +17,7 @@ import shutil
 import docker
 import pytest
 
+from ros_cross_compile.data_collector import DataCollector
 from ros_cross_compile.dependencies import DependenciesStage
 from ros_cross_compile.dependencies import gather_rosdeps
 from ros_cross_compile.dependencies import rosdep_install_script
@@ -59,12 +60,13 @@ def test_dummy_ros2_pkg(tmpdir):
     client = DockerClient()
     platform = Platform(arch='aarch64', os_name='ubuntu', ros_distro='dashing')
     out_script = ws / rosdep_install_script(platform)
+    test_collector = DataCollector()
 
     # a default set of customizations for the dependencies stage
     customizations = PipelineStageConfigOptions(False, [], None, None, None)
     temp_stage = DependenciesStage()
 
-    temp_stage(platform, client, ws, customizations)
+    temp_stage(platform, client, ws, customizations, test_collector)
 
     result = out_script.read_text()
     assert 'ros-dashing-ament-cmake' in result
diff --git a/test/test_sysroot_creator.py b/test/test_sysroot_creator.py
index ecd0ae8..854ff9f 100644
--- a/test/test_sysroot_creator.py
+++ b/test/test_sysroot_creator.py
@@ -85,13 +85,15 @@ def test_basic_sysroot_creation(tmpdir):
     # Very simple smoke test to validate that all internal syntax is correct
 
     mock_docker_client = Mock()
+    mock_data_collector = Mock()
     platform = Platform('aarch64', 'ubuntu', 'eloquent')
 
     # a default set of customizations for the docker build stage
     customizations = PipelineStageConfigOptions(False, [], None, None, None)
     temp_stage = CreateSysrootStage()
 
-    temp_stage(platform, mock_docker_client, Path('dummy_path'), customizations)
+    temp_stage(platform, mock_docker_client,
+               Path('dummy_path'), customizations, mock_data_collector)
     assert mock_docker_client.build_image.call_count == 1