Repository: avocado-framework/avocado

Issue:
Add timeout parameter to `avocado assets fetch` command
Today, `avocado assets command` does not have a timeout parameter. If it runs on a CI environment, it will timeout the job.

Adding a timeout parameter allows one specific fetch to be canceled in case of a timeout is reached while others can happen and the job in the CI can flow.


Discussion:

Resolving patch:
diff --git a/avocado/plugins/assets.py b/avocado/plugins/assets.py
--- a/avocado/plugins/assets.py
+++ b/avocado/plugins/assets.py
@@ -45,7 +45,7 @@ def __init__(self, file_name, klass=None, method=None):
         self.klass = klass
         # we need to make sure we cover the setUp method when fetching
         # assets for a specific test
-        self.method = [method, 'setUp']
+        self.methods = [method, 'setUp']
         self.asgmts = {}
         self.calls = []
 
@@ -67,8 +67,8 @@ def __init__(self, file_name, klass=None, method=None):
         self.visit(self.tree)
 
     def _parse_args(self, node):
-        """
-        Parse the AST fetch_asset node and build the arguments dictionary.
+        """Parse the AST fetch_asset node and build the arguments dictionary.
+
         :param node: AST node to be evaluated
         :type node: ast.Attribute
         :returns: keywords and arguments from a fetch_asset call.
@@ -127,8 +127,8 @@ def _parse_args(self, node):
         return fetch_args
 
     def visit_ClassDef(self, node):  # pylint: disable=C0103
-        """
-        Visit ClassDef on AST and save current Class.
+        """Visit ClassDef on AST and save current Class.
+
         :param node: AST node to be evaluated
         :type node: ast.*
         """
@@ -143,14 +143,14 @@ def visit_ClassDef(self, node):  # pylint: disable=C0103
             self.generic_visit(node)
 
     def visit_FunctionDef(self, node):  # pylint: disable=C0103
-        """
-        Visit FunctionDef on AST and save current method.
+        """Visit FunctionDef on AST and save current method.
+
         :param node: AST node to be evaluated
         :type node: ast.*
         """
         # make sure we are into a class method and not a function
         if self.current_klass:
-            if self.method[0] and node.name not in self.method:
+            if self.methods[0] and node.name not in self.methods:
                 return
 
             self.current_method = node.name
@@ -166,9 +166,11 @@ def _ast_list_to_list(node):
         return result
 
     def visit_Assign(self, node):  # pylint: disable=C0103
-        """
-        Visit Assign on AST and build list of assignments that matches the
+        """Visit Assign on AST and build assignments.
+
+        This method will visit and build list of assignments that matches the
         pattern pattern `name = string`.
+
         :param node: AST node to be evaluated
         :type node: ast.*
         """
@@ -194,8 +196,8 @@ def visit_Assign(self, node):  # pylint: disable=C0103
         self.generic_visit(node)
 
     def visit_Call(self, node):  # pylint: disable=C0103
-        """
-        Visit Calls on AST and build list of calls that matches the pattern.
+        """Visit Calls on AST and build list of calls that matches the pattern.
+
         :param node: AST node to be evaluated
         :type node: ast.*
         """
@@ -209,14 +211,15 @@ def visit_Call(self, node):  # pylint: disable=C0103
 
 
 def fetch_assets(test_file, klass=None, method=None, logger=None):
-    """
-    Fetches the assets based on keywords listed on FetchAssetHandler.calls.
+    """Fetches the assets based on keywords listed on FetchAssetHandler.calls.
+
     :param test_file: File name of instrumented test to be evaluated
-    :type test_file: str
+                      :type test_file: str
     :returns: list of names that were successfully fetched and list of
-    fails.
+              fails.
     """
     cache_dirs = settings.as_dict().get('datadir.paths.cache_dirs')
+    timeout = settings.as_dict().get('assets.fetch.timeout')
     success = []
     fail = []
     handler = FetchAssetHandler(test_file, klass, method)
@@ -225,18 +228,12 @@ def fetch_assets(test_file, klass=None, method=None, logger=None):
         if expire is not None:
             expire = data_structures.time_to_seconds(str(expire))
 
-        # make dictionary unpacking compatible with python 3.4 as it does
-        # not support constructions like:
-        # Asset(**call, cache_dirs=cache_dirs, expire=expire)
-        call['cache_dirs'] = cache_dirs
-        call['expire'] = expire
-
         try:
-            asset_obj = Asset(**call)
+            asset_obj = Asset(**call, cache_dirs=cache_dirs, expire=expire)
             if logger is not None:
                 logger.info('Fetching asset from %s:%s.%s',
                             test_file, klass, method)
-            asset_obj.fetch()
+            asset_obj.fetch(timeout)
             success.append(call['name'])
         except (OSError, ValueError) as failed:
             fail.append(failed)
@@ -244,10 +241,11 @@ def fetch_assets(test_file, klass=None, method=None, logger=None):
 
 
 class FetchAssetJob(JobPreTests):  # pylint: disable=R0903
-    """
-    Implements the assets fetch job pre tests. This has the same effect of
-    running the 'avocado assets fetch INSTRUMENTED', but it runs during the
-    test execution, before the actual test starts.
+    """Implements the assets fetch job pre tests.
+
+    This has the same effect of running the 'avocado assets fetch
+    INSTRUMENTED', but it runs during the test execution, before the actual
+    test starts.
     """
     name = "fetchasset"
     description = "Fetch assets before the test run"
@@ -297,8 +295,7 @@ def _count_filter_args(config):
         return len([a for a in args if a is not None])
 
     def configure(self, parser):
-        """
-        Add the subparser for the assets action.
+        """Add the subparser for the assets action.
 
         :param parser: The Avocado command line application parser
         :type parser: :class:`avocado.core.parser.ArgumentParser`
@@ -370,6 +367,16 @@ def register_filter_options(subparser, section):
                                  parser=fetch_subcommand_parser,
                                  long_arg='--ignore-errors')
 
+        help_msg = "Timeout to be used when download an asset."
+        settings.register_option(section='assets.fetch',
+                                 key='timeout',
+                                 help_msg=help_msg,
+                                 default=300,
+                                 key_type=int,
+                                 metavar="TIMEOUT",
+                                 parser=fetch_subcommand_parser,
+                                 long_arg='--timeout')
+
         register_subcommand_parser = subcommands.add_parser(
                 'register',
                 help='Register an asset directly to the cacche')
diff --git a/avocado/utils/asset.py b/avocado/utils/asset.py
--- a/avocado/utils/asset.py
+++ b/avocado/utils/asset.py
@@ -40,6 +40,9 @@
 #: The default hash algorithm to use on asset cache operations
 DEFAULT_HASH_ALGORITHM = 'sha1'
 
+#: The default timeout for the downloading of assets
+DOWNLOAD_TIMEOUT = 300
+
 SUPPORTED_OPERATORS = {'==': operator.eq,
                        '<': operator.lt,
                        '>': operator.gt,
@@ -114,15 +117,18 @@ def _create_metadata_file(self, asset_file):
             with open(metadata_path, "w") as metadata_file:
                 json.dump(self.metadata, metadata_file)
 
-    def _download(self, url_obj, asset_path):
+    def _download(self, url_obj, asset_path, timeout=None):
         """
         Download the asset from an uri.
 
         :param url_obj: object from urlparse.
         :param asset_path: full path of the asset file.
+        :param timeout: timeout in seconds. Default is
+                        :data:`avocado.utils.asset.DOWNLOAD_TIMEOUT`.
         :returns: if the downloaded file matches the hash.
         :rtype: bool
         """
+        timeout = timeout or DOWNLOAD_TIMEOUT
         try:
             # Temporary unique name to use while downloading
             temp = '%s.%s' % (asset_path, str(uuid.uuid4()))
@@ -137,7 +143,7 @@ def _download(self, url_obj, asset_path):
                 except OSError:
                     LOG.info("Asset not in cache after lock, fetching it.")
 
-                url_download(url_obj.geturl(), temp)
+                url_download(url_obj.geturl(), temp, timeout=timeout)
                 shutil.copy(temp, asset_path)
                 self._create_hash_file(asset_path)
                 if not self._verify_hash(asset_path):
@@ -198,7 +204,7 @@ def read_hash_from_file(cls, filename):
             LOG.error('%s: %s', exc_type.__name__, exc_value)
             return [None, None]
 
-    def _get_local_file(self, url_obj, asset_path):
+    def _get_local_file(self, url_obj, asset_path, _):
         """
         Create a symlink for a local file into the cache.
 
@@ -327,13 +333,15 @@ def _verify_hash(self, asset_path):
         """
         return self._has_valid_hash(asset_path, self.asset_hash)
 
-    def fetch(self):
+    def fetch(self, timeout=None):
         """Try to fetch the current asset.
 
         First tries to find the asset on the provided cache_dirs list.
         Then tries to download the asset from the locations list
         provided.
 
+        :param timeout: timeout in seconds. Default is
+                        :data:`avocado.utils.asset.DOWNLOAD_TIMEOUT`.
         :raise OSError: When it fails to fetch the asset
         :returns: The path for the file on the cache directory.
         :rtype: str
@@ -341,6 +349,7 @@ def fetch(self):
         # First let's search for the file in each one of the cache locations
         asset_file = None
         error = "unknown"
+        timeout = timeout or DOWNLOAD_TIMEOUT
         try:
             return self.find_asset_file(create_metadata=True)
         except OSError:
@@ -372,7 +381,7 @@ def fetch(self):
             if not os.path.isdir(dirname):
                 os.makedirs(dirname, exist_ok=True)
             try:
-                if fetch(urlobj, asset_file):
+                if fetch(urlobj, asset_file, timeout):
                     if self.metadata is not None:
                         self._create_metadata_file(asset_file)
                     return asset_file
diff --git a/avocado/utils/download.py b/avocado/utils/download.py
--- a/avocado/utils/download.py
+++ b/avocado/utils/download.py
@@ -20,7 +20,7 @@
 import logging
 import os
 import shutil
-import socket
+from multiprocessing import Process
 from urllib.request import urlopen
 
 from . import aurl, crypto, output
@@ -34,24 +34,21 @@ def url_open(url, data=None, timeout=5):
 
     :param url: URL to open.
     :param data: (optional) data to post.
-    :param timeout: (optional) default timeout in seconds.
+    :param timeout: (optional) default timeout in seconds. Please, be aware
+                    that timeout here is just for blocking operations during
+                    the connection setup, since this method doesn't read the
+                    file from the url.
     :return: file-like object.
     :raises: `URLError`.
     """
-    # Save old timeout
-    old_timeout = socket.getdefaulttimeout()
-    socket.setdefaulttimeout(timeout)
-    try:
-        result = urlopen(url, data=data)
-        msg = ('Retrieved URL "%s": content-length %s, date: "%s", '
-               'last-modified: "%s"')
-        log.debug(msg, url,
-                  result.headers.get('Content-Length', 'UNKNOWN'),
-                  result.headers.get('Date', 'UNKNOWN'),
-                  result.headers.get('Last-Modified', 'UNKNOWN'))
-        return result
-    finally:
-        socket.setdefaulttimeout(old_timeout)
+    result = urlopen(url, data=data, timeout=timeout)
+    msg = ('Retrieved URL "%s": content-length %s, date: "%s", '
+           'last-modified: "%s"')
+    log.debug(msg, url,
+              result.headers.get('Content-Length', 'UNKNOWN'),
+              result.headers.get('Date', 'UNKNOWN'),
+              result.headers.get('Last-Modified', 'UNKNOWN'))
+    return result
 
 
 def url_download(url, filename, data=None, timeout=300):
@@ -64,14 +61,23 @@ def url_download(url, filename, data=None, timeout=300):
     :param timeout: (optional) default timeout in seconds.
     :return: `None`.
     """
-    log.info('Fetching %s -> %s', url, filename)
-
-    src_file = url_open(url, data=data, timeout=timeout)
-    try:
-        with open(filename, 'wb') as dest_file:
-            shutil.copyfileobj(src_file, dest_file)
-    finally:
-        src_file.close()
+    def download():
+        log.info('Fetching %s -> %s', url, filename)
+
+        src_file = url_open(url, data=data)
+        try:
+            with open(filename, 'wb') as dest_file:
+                shutil.copyfileobj(src_file, dest_file)
+        finally:
+            src_file.close()
+
+    process = Process(target=download)
+    process.start()
+    process.join(timeout)
+    if process.is_alive():
+        process.terminate()
+        process.join()
+        raise OSError("Aborting downloading. Timeout was reach.")
 
 
 def url_download_interactive(url, output_file, title='', chunk_size=102400):


Test patch: