Repository: spack/spack

Issue:
spack cannot show curl processor bar
It's strange that , recent `spack` version cannot show download url and process bar.
I checked the code and found that `spack` don't print the log in fetch stage now.

And I dumped the `curl_args` by this way ,and found `'-#'` option enabled for `curl`, which means it should print the `process bar` when curl some package.
```
index fb8737389..5d3e64568 100644
--- a/lib/spack/spack/fetch_strategy.py
+++ b/lib/spack/spack/fetch_strategy.py
@@ -342,6 +342,8 @@ def _fetch_from_url(self, url):
         else:
             curl_args.append('-sS')  # just errors when not.

+        tty.msg(curl_args)
+
         connect_timeout = spack.config.get('config:connect_timeout', 10)

         if self.extra_options:
```

But the log I tested is that:
```
# spack install -v --keep-stage nicstat
==> Installing nicstat
==> ['-C', '-', '-o', '/tmp/root/spack-stage/build_cache/linux-centos8-aarch64-gcc-8.3.1-nicstat-1.95-wykopqo35367xqfw5lmlotytldtj7ygd.spec.yaml.part', '-f', '-D', '-', '-L', 'https://spack-llnl-mirror.s3-us-west-2.amazonaws.com/build_cache/linux-centos8-aarch64-gcc-8.3.1-nicstat-1.95-wykopqo35367xqfw5lmlotytldtj7ygd.spec.yaml', '--stderr', '-', '-#']
==> No binary for nicstat found: installing from source
==> nicstat: Executing phase: 'edit'
==> [2020-08-12-19:57:01.463389] Copying Makefile.Linux to makefile
==> [2020-08-12-19:57:01.464149] FILTER FILE: makefile [replacing "CMODEL =\s+-m32"]
==> [2020-08-12-19:57:01.466012] FILTER FILE: makefile [replacing "sudo"]
==> nicstat: Executing phase: 'build'
==> [2020-08-12-19:57:01.469777] 'make' '-j8'
gcc -O3     nicstat.c   -o nicstat
mv nicstat `./nicstat.sh --bin-name`
==> nicstat: Executing phase: 'install'
==> [2020-08-12-19:57:02.797622] Installing . to /home/xiaojun/spack/opt/spack/linux-centos8-aarch64/gcc-8.3.1/nicstat-1.95-wykopqo35367xqfw5lmlotytldtj7ygd
[+] /home/xiaojun/spack/opt/spack/linux-centos8-aarch64/gcc-8.3.1/nicstat-1.95-wykopqo35367xqfw5lmlotytldtj7ygd
```

So, is that a bug or I missed something to enable curl log?


Discussion:
~@darmac Have you tried using the `-d` option?~
~A *lot* of build-related output was pushed to the debug level as of https://github.com/spack/spack/pull/17546.~

Nvm.  I see the issue.

Resolving patch:
diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py
--- a/lib/spack/spack/fetch_strategy.py
+++ b/lib/spack/spack/fetch_strategy.py
@@ -295,6 +295,9 @@ def fetch(self):
         url = None
         errors = []
         for url in self.candidate_urls:
+            if not self._existing_url(url):
+                continue
+
             try:
                 partial_file, save_file = self._fetch_from_url(url)
                 if save_file:
@@ -309,13 +312,22 @@ def fetch(self):
         if not self.archive_file:
             raise FailedDownloadError(url)
 
+    def _existing_url(self, url):
+        tty.debug('Checking existence of {0}'.format(url))
+        curl = self.curl
+        # Telling curl to fetch the first byte (-r 0-0) is supposed to be
+        # portable.
+        curl_args = ['--stderr', '-', '-s', '-f', '-r', '0-0', url]
+        _ = curl(*curl_args, fail_on_error=False, output=os.devnull)
+        return curl.returncode == 0
+
     def _fetch_from_url(self, url):
         save_file = None
         partial_file = None
         if self.stage.save_filename:
             save_file = self.stage.save_filename
             partial_file = self.stage.save_filename + '.part'
-        tty.debug('Fetching {0}'.format(url))
+        tty.msg('Fetching {0}'.format(url))
         if partial_file:
             save_args = ['-C',
                          '-',  # continue partial downloads
@@ -330,8 +342,6 @@ def _fetch_from_url(self, url):
             '-',  # print out HTML headers
             '-L',  # resolve 3xx redirects
             url,
-            '--stderr',  # redirect stderr output
-            '-',         # redirect to stdout
         ]
 
         if not spack.config.get('config:verify_ssl'):
@@ -340,7 +350,7 @@ def _fetch_from_url(self, url):
         if sys.stdout.isatty() and tty.msg_enabled():
             curl_args.append('-#')  # status bar when using a tty
         else:
-            curl_args.append('-sS')  # just errors when not.
+            curl_args.append('-sS')  # show errors if fail
 
         connect_timeout = spack.config.get('config:connect_timeout', 10)
 
@@ -569,7 +579,7 @@ def fetch(self):
                 raise
 
         # Notify the user how we fetched.
-        tty.debug('Using cached archive: {0}'.format(path))
+        tty.msg('Using cached archive: {0}'.format(path))
 
 
 class VCSFetchStrategy(FetchStrategy):


Test patch:
diff --git a/lib/spack/spack/test/url_fetch.py b/lib/spack/spack/test/url_fetch.py
--- a/lib/spack/spack/test/url_fetch.py
+++ b/lib/spack/spack/test/url_fetch.py
@@ -6,8 +6,10 @@
 import collections
 import os
 import pytest
+import sys
 
 from llnl.util.filesystem import working_dir, is_exe
+import llnl.util.tty as tty
 
 import spack.repo
 import spack.config
@@ -173,6 +175,25 @@ def test_unknown_hash(checksum_type):
         crypto.Checker('a')
 
 
+def test_url_with_status_bar(tmpdir, mock_archive, monkeypatch, capfd):
+    """Ensure fetch with status bar option succeeds."""
+    def is_true():
+        return True
+
+    testpath = str(tmpdir)
+
+    monkeypatch.setattr(sys.stdout, 'isatty', is_true)
+    monkeypatch.setattr(tty, 'msg_enabled', is_true)
+
+    fetcher = fs.URLFetchStrategy(mock_archive.url)
+    with Stage(fetcher, path=testpath) as stage:
+        assert fetcher.archive_file is None
+        stage.fetch()
+
+    status = capfd.readouterr()[1]
+    assert '##### 100.0%' in status
+
+
 def test_url_extra_fetch(tmpdir, mock_archive):
     """Ensure a fetch after downloading is effectively a no-op."""
     testpath = str(tmpdir)