Repository: avocado-framework/avocado

Issue:
Cache: purge by overall cache size
Besides the existing features, we need to trim the cache to a defined size, so that it matches the storage limitations of users (and CI systems).

For instance it may be the case that a GitLab cache limit is 4 GiB, in that case we can sort by last access, and remove all that exceeds 4 GiB (that is, keep the last accessed 4 GiB worth of cached files).

This would ensure that the cache is automatically being removed of files that were used last (and possibly not used anymore).

This can be available to users as a command such as:

```
avocado assets purge --by-limit=4G
```

This is one last use case of #4311


Discussion:

Resolving patch:
diff --git a/avocado/plugins/assets.py b/avocado/plugins/assets.py
--- a/avocado/plugins/assets.py
+++ b/avocado/plugins/assets.py
@@ -28,6 +28,7 @@
 from avocado.utils import data_structures
 from avocado.utils.asset import SUPPORTED_OPERATORS, Asset
 from avocado.utils.astring import iter_tabular_output
+from avocado.utils.data_structures import DataSize, InvalidDataSize
 from avocado.utils.output import display_data_size
 
 
@@ -268,6 +269,14 @@ class Assets(CLICmd):
     name = 'assets'
     description = 'Manage assets'
 
+    @staticmethod
+    def _count_filter_args(config):
+        sub_command = config.get('assets_subcommand')
+        args = [config.get("assets.{}.days".format(sub_command)),
+                config.get("assets.{}.size_filter".format(sub_command)),
+                config.get("assets.{}.overall_limit".format(sub_command))]
+        return len([a for a in args if a is not None])
+
     def configure(self, parser):
         """
         Add the subparser for the assets action.
@@ -299,6 +308,20 @@ def register_filter_options(subparser, section):
                                      long_arg='--by-days',
                                      parser=subparser)
 
+            help_msg = ("Filter will be based on a overall system limit "
+                        "threshold in bytes (with assets orded by last "
+                        "access) or with a suffix unit. Valid suffixes are: ")
+            help_msg += ','.join(DataSize.MULTIPLIERS.keys())
+
+            settings.register_option(section=section,
+                                     key='overall_limit',
+                                     help_msg=help_msg,
+                                     default=None,
+                                     key_type=str,
+                                     metavar="LIMIT",
+                                     long_arg='--by-overall-limit',
+                                     parser=subparser)
+
         parser = super(Assets, self).configure(parser)
 
         subcommands = parser.add_subparsers(dest='assets_subcommand')
@@ -374,8 +397,9 @@ def register_filter_options(subparser, section):
     def handle_purge(self, config):
         days = config.get('assets.purge.days')
         size_filter = config.get('assets.purge.size_filter')
-        if (days is None and size_filter is None) \
-                or (days is not None and size_filter is not None):
+        overall_limit = config.get('assets.purge.overall_limit')
+
+        if self._count_filter_args(config) != 1:
             msg = ("You should choose --by-size-filter or --by-days. "
                    "For help, run: avocado assets purge --help")
             LOG_UI.error(msg)
@@ -387,13 +411,24 @@ def handle_purge(self, config):
                 Asset.remove_assets_by_unused_for_days(days, cache_dirs)
             elif size_filter is not None:
                 Asset.remove_assets_by_size(size_filter, cache_dirs)
+            elif overall_limit is not None:
+                try:
+                    size = DataSize(overall_limit).b
+                    Asset.remove_assets_by_overall_limit(size, cache_dirs)
+                except InvalidDataSize:
+                    error_msg = "You are using an invalid suffix. "
+                    error_msg += "Use one of the following values: "
+                    error_msg += ",".join(DataSize.MULTIPLIERS.keys())
+                    LOG_UI.error(error_msg)
+                    return
+
         except (FileNotFoundError, OSError) as e:
             LOG_UI.error("Could not remove asset: %s", e)
 
     def handle_list(self, config):
         days = config.get('assets.list.days')
         size_filter = config.get('assets.list.size_filter')
-        if (days is not None and size_filter is not None):
+        if self._count_filter_args(config) == 2:
             msg = ("You should choose --by-size-filter or --by-days. "
                    "For help, run: avocado assets list --help")
             LOG_UI.error(msg)
diff --git a/avocado/utils/asset.py b/avocado/utils/asset.py
--- a/avocado/utils/asset.py
+++ b/avocado/utils/asset.py
@@ -431,7 +431,7 @@ def asset_name(self):
         return os.path.basename(self.parsed_name.path)
 
     @classmethod
-    def get_all_assets(cls, cache_dirs):
+    def get_all_assets(cls, cache_dirs, sort=True):
         """Returns all assets stored in all cache dirs."""
         assets = []
         for cache_dir in cache_dirs:
@@ -441,6 +441,11 @@ def get_all_assets(cls, cache_dirs):
                     if not f.endswith('-CHECKSUM') and \
                        not f.endswith('_metadata.json'):
                         assets.append(os.path.join(root, f))
+        if sort:
+            assets = {a: os.stat(a).st_atime for a in assets}
+            return [a[0] for a in sorted(assets.items(),
+                                         key=lambda x: x[1],
+                                         reverse=True)]
         return assets
 
     @classmethod
@@ -533,6 +538,27 @@ def get_assets_by_size(cls, size_filter, cache_dirs):
                 result.append(file_path)
         return result
 
+    @classmethod
+    def remove_assets_by_overall_limit(cls, limit, cache_dirs):
+        """This will remove assets based on overall limit.
+
+        We are going to sort the assets based on the access time first.
+        For instance it may be the case that a GitLab cache limit is 4
+        GiB, in that case we can sort by last access, and remove all
+        that exceeds 4 GiB (that is, keep the last accessed 4 GiB worth
+        of cached files).
+
+        Note: during the usage of this method, you should use bytes as limit.
+
+        :param limit: a integer limit in bytes.
+        :param cache_dirs: list of directories to use during the search.
+        """
+        size_sum = 0
+        for asset in cls.get_all_assets(cache_dirs):
+            size_sum += os.stat(asset).st_size
+            if size_sum >= limit:
+                cls.remove_asset_by_path(asset)
+
     @classmethod
     def remove_assets_by_size(cls, size_filter, cache_dirs):
         for file_path in cls.get_assets_by_size(size_filter, cache_dirs):
diff --git a/selftests/check.py b/selftests/check.py
--- a/selftests/check.py
+++ b/selftests/check.py
@@ -502,7 +502,8 @@ def create_suites(args):
                            'selftests/unit/',
                            'selftests/functional/'],
         'run.test_runner': 'nrunner',
-        'run.ignore_missing_references': True
+        'run.ignore_missing_references': True,
+        'job.output.testlogs.statuses': ['FAIL']
     }
 
     if not args.disable_static_checks:


Test patch:
diff --git a/selftests/functional/plugin/test_assets.py b/selftests/functional/plugin/test_assets.py
--- a/selftests/functional/plugin/test_assets.py
+++ b/selftests/functional/plugin/test_assets.py
@@ -10,6 +10,7 @@
 
 from avocado.core import exit_codes
 from avocado.utils import process
+from avocado.utils.asset import Asset
 
 from ... import (AVOCADO, TestCaseTmpDir, get_temporary_config,
                  skipUnlessPathExists)
@@ -158,6 +159,38 @@ def test_asset_list(self):
         result = process.run(cmd_line)
         self.assertIn(name, result.stdout_text)
 
+    def test_asset_purge_by_overall_cache_size(self):
+        """Make sure that we can set cache limits."""
+        # creates a single byte asset
+        asset_file = tempfile.NamedTemporaryFile(delete=False)
+        asset_file.write(b'\xff')
+        asset_file.close()
+
+        config = self.config_file.name
+        url = asset_file.name
+        name = "should-be-removed"
+        cmd_line = "%s --config %s assets register %s %s" % (AVOCADO,
+                                                             config,
+                                                             name,
+                                                             url)
+        result = process.run(cmd_line)
+        self.assertIn("Now you can reference it by name {}".format(name),
+                      result.stdout_text)
+
+        cmd_line = "%s --config %s assets purge --by-overall-limit 2" % (AVOCADO,
+                                                                         config)
+        process.run(cmd_line)
+
+        cmd_line = "%s --config %s assets list" % (AVOCADO, config)
+        result = process.run(cmd_line)
+        self.assertIn(name, result.stdout_text)
+
+        # Double check that the sum of assets is not bigger than 2bytes
+        size_sum = 0
+        for asset in Asset.get_all_assets([self.mapping['cache_dir']]):
+            size_sum += os.stat(asset).st_size
+        self.assertLessEqual(size_sum, 2)
+
     def tearDown(self):
         self.base_dir.cleanup()