Repository: conda/conda-build

Issue:
conda-build 3.17.1 fails with SSL error
### Actual Behavior

Package build fails with SSL error:

```
Attempting to finalize metadata for pyadsclientlib
INFO:conda_build.metadata:Attempting to finalize metadata for pyadsclientlib
Traceback (most recent call last):
  File "/usr/share/tomcat/miniconda_temp/1544092798/lib/python3.7/site-packages/urllib3/contrib/pyopenssl.py", line 444, in wrap_socket
    cnx.do_handshake()
  File "/usr/share/tomcat/miniconda_temp/1544092798/lib/python3.7/site-packages/OpenSSL/SSL.py", line 1907, in do_handshake
    self._raise_ssl_error(self._ssl, result)
  File "/usr/share/tomcat/miniconda_temp/1544092798/lib/python3.7/site-packages/OpenSSL/SSL.py", line 1639, in _raise_ssl_error
    _raise_current_error()
  File "/usr/share/tomcat/miniconda_temp/1544092798/lib/python3.7/site-packages/OpenSSL/_util.py", line 54, in exception_from_error_queue
    raise exception_type(errors)
OpenSSL.SSL.Error: [('SSL routines', 'tls_process_server_certificate', 'certificate verify failed')]

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/share/tomcat/miniconda_temp/1544092798/lib/python3.7/site-packages/urllib3/connectionpool.py", line 600, in urlopen
    chunked=chunked)
  File "/usr/share/tomcat/miniconda_temp/1544092798/lib/python3.7/site-packages/urllib3/connectionpool.py", line 343, in _make_request
    self._validate_conn(conn)
  File "/usr/share/tomcat/miniconda_temp/1544092798/lib/python3.7/site-packages/urllib3/connectionpool.py", line 849, in _validate_conn
    conn.connect()
  File "/usr/share/tomcat/miniconda_temp/1544092798/lib/python3.7/site-packages/urllib3/connection.py", line 356, in connect
    ssl_context=context)
  File "/usr/share/tomcat/miniconda_temp/1544092798/lib/python3.7/site-packages/urllib3/util/ssl_.py", line 359, in ssl_wrap_socket
    return context.wrap_socket(sock, server_hostname=server_hostname)
  File "/usr/share/tomcat/miniconda_temp/1544092798/lib/python3.7/site-packages/urllib3/contrib/pyopenssl.py", line 450, in wrap_socket
    raise ssl.SSLError('bad handshake: %r' % e)
ssl.SSLError: ("bad handshake: Error([('SSL routines', 'tls_process_server_certificate', 'certificate verify failed')])",)

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/share/tomcat/miniconda_temp/1544092798/lib/python3.7/site-packages/requests/adapters.py", line 445, in send
    timeout=timeout
  File "/usr/share/tomcat/miniconda_temp/1544092798/lib/python3.7/site-packages/urllib3/connectionpool.py", line 638, in urlopen
    _stacktrace=sys.exc_info()[2])
  File "/usr/share/tomcat/miniconda_temp/1544092798/lib/python3.7/site-packages/urllib3/util/retry.py", line 398, in increment
    raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='repo_host', port=443): Max retries exceeded with url: /channeldata.json (Caused by SSLError(SSLError("bad handshake: Error([('SSL routines', 'tls_process_server_certificate', 'certificate verify failed')])")))

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/share/tomcat/miniconda_temp/1544092798/bin/conda-build", line 11, in <module>
    sys.exit(main())
  File "/usr/share/tomcat/miniconda_temp/1544092798/lib/python3.7/site-packages/conda_build/cli/main_build.py", line 456, in main
    execute(sys.argv[1:])
  File "/usr/share/tomcat/miniconda_temp/1544092798/lib/python3.7/site-packages/conda_build/cli/main_build.py", line 447, in execute
    verify=args.verify, variants=args.variants)
  File "/usr/share/tomcat/miniconda_temp/1544092798/lib/python3.7/site-packages/conda_build/api.py", line 208, in build
    notest=notest, need_source_download=need_source_download, variants=variants)
  File "/usr/share/tomcat/miniconda_temp/1544092798/lib/python3.7/site-packages/conda_build/build.py", line 2311, in build_tree
    notest=notest,
  File "/usr/share/tomcat/miniconda_temp/1544092798/lib/python3.7/site-packages/conda_build/build.py", line 1324, in build
    output_metas = expand_outputs([(m, need_source_download, need_reparse_in_env)])
  File "/usr/share/tomcat/miniconda_temp/1544092798/lib/python3.7/site-packages/conda_build/render.py", line 735, in expand_outputs
    for (output_dict, m) in _m.copy().get_output_metadata_set(permit_unsatisfiable_variants=False):
  File "/usr/share/tomcat/miniconda_temp/1544092798/lib/python3.7/site-packages/conda_build/metadata.py", line 2034, in get_output_metadata_set
    bypass_env_check=bypass_env_check)
  File "/usr/share/tomcat/miniconda_temp/1544092798/lib/python3.7/site-packages/conda_build/metadata.py", line 721, in finalize_outputs_pass
    permit_unsatisfiable_variants=permit_unsatisfiable_variants)
  File "/usr/share/tomcat/miniconda_temp/1544092798/lib/python3.7/site-packages/conda_build/render.py", line 509, in finalize_metadata
    exclude_pattern)
  File "/usr/share/tomcat/miniconda_temp/1544092798/lib/python3.7/site-packages/conda_build/render.py", line 371, in add_upstream_pins
    permit_unsatisfiable_variants, exclude_pattern)
  File "/usr/share/tomcat/miniconda_temp/1544092798/lib/python3.7/site-packages/conda_build/render.py", line 359, in _read_upstream_pin_files
    permit_unsatisfiable_variants=permit_unsatisfiable_variants)
  File "/usr/share/tomcat/miniconda_temp/1544092798/lib/python3.7/site-packages/conda_build/render.py", line 143, in get_env_dependencies
    channel_urls=tuple(m.config.channel_urls))
  File "/usr/share/tomcat/miniconda_temp/1544092798/lib/python3.7/site-packages/conda_build/environ.py", line 736, in get_install_actions
    locking=locking, timeout=timeout)
  File "/usr/share/tomcat/miniconda_temp/1544092798/lib/python3.7/site-packages/conda_build/index.py", line 265, in get_build_index
    channel_content = _download_channeldata(channel.base_url + '/channeldata.json')
  File "/usr/share/tomcat/miniconda_temp/1544092798/lib/python3.7/site-packages/conda/exports.py", line 181, in __call__
    value = self.func(*args, **kw)
  File "/usr/share/tomcat/miniconda_temp/1544092798/lib/python3.7/site-packages/conda_build/index.py", line 168, in _download_channeldata
    return requests.get(channel_url)
  File "/usr/share/tomcat/miniconda_temp/1544092798/lib/python3.7/site-packages/requests/api.py", line 72, in get
    return request('get', url, params=params, **kwargs)
  File "/usr/share/tomcat/miniconda_temp/1544092798/lib/python3.7/site-packages/requests/api.py", line 58, in request
    return session.request(method=method, url=url, **kwargs)
  File "/usr/share/tomcat/miniconda_temp/1544092798/lib/python3.7/site-packages/requests/sessions.py", line 512, in request
    resp = self.send(prep, **send_kwargs)
  File "/usr/share/tomcat/miniconda_temp/1544092798/lib/python3.7/site-packages/requests/sessions.py", line 622, in send
    r = adapter.send(request, **kwargs)
  File "/usr/share/tomcat/miniconda_temp/1544092798/lib/python3.7/site-packages/requests/adapters.py", line 511, in send
    raise SSLError(e, request=request)
requests.exceptions.SSLError: HTTPSConnectionPool(host='repo_host', port=443): Max retries exceeded with url: /channeldata.json (Caused by SSLError(SSLError("bad handshake: Error([('SSL routines', 'tls_process_server_certificate', 'certificate verify failed')])")))
```

### Expected Behavior

Package should be build succesfully.

### Steps to Reproduce

IDK what causes this exactly and the environment is non-reproductible. Building on my machine succeeds but fails on Jenkins.

##### `Output of conda info`

I do not have shell access to this machine.

Repo host has self-signed certificate, conda is configured to not verify ssl certs with `conda config --set ssl_verify "false"`. Downgrading to 3.16.2 fixes this issue.


Discussion:
I'm seeing a similar issue where the `proxy_servers` settings in the .condarc file are being ignored in jenkins. Setting them manually on the command line before running `conda build` works as expected. 

It was similar enough to tack in on here, but let me know if it's different enough to warrant a ticket on its own. 

Downgrading to 3.16.3 worked for me, whereas all the 3.17.+ versions are broken in this regard. 

This is happening because the new approach used requests directly, rather than going via conda's download functionality.  We need to fix this so that conda's configuration is respected.

Resolving patch:
diff --git a/conda_build/cli/main_debug.py b/conda_build/cli/main_debug.py
--- a/conda_build/cli/main_debug.py
+++ b/conda_build/cli/main_debug.py
@@ -81,7 +81,7 @@ def execute(args):
                 test_file = "conda_test_runner.sh" if on_win else "conda_test_runner.sh"
                 print("To run your tests, you might want to start with running the {} file.".format(test_file))
             else:
-                build_file = "bld.bat" if on_win else "conda_build.sh"
+                build_file = "conda_build.bat" if on_win else "conda_build.sh"
                 print("To run your build, you might want to start with running the {} file.".format(build_file))
             print("#" * 80)
 
diff --git a/conda_build/index.py b/conda_build/index.py
--- a/conda_build/index.py
+++ b/conda_build/index.py
@@ -22,8 +22,6 @@
 from conda.common.compat import ensure_binary
 # from conda.resolve import dashlist
 
-import requests
-from requests.exceptions import InvalidSchema
 import pytz
 from jinja2 import Environment, PackageLoader
 from tqdm import tqdm
@@ -40,8 +38,9 @@
 
 
 from . import conda_interface, utils
-from .conda_interface import MatchSpec, VersionOrder, human_bytes, context, memoized
+from .conda_interface import MatchSpec, VersionOrder, human_bytes, context
 from .conda_interface import CondaError, CondaHTTPError, get_index, url_path
+from .conda_interface import download, TemporaryDirectory
 from .utils import glob, get_logger, FileNotFoundError, PermissionError
 
 try:
@@ -163,9 +162,16 @@ def submit(self, fn, *args, **kwargs):
     from conda._vendor.toolz.itertoolz import concat, concatv, groupby  # NOQA
 
 
-@memoized
 def _download_channeldata(channel_url):
-    return requests.get(channel_url)
+    with TemporaryDirectory() as td:
+        tf = os.path.join(td, "channeldata.json")
+        download(channel_url, tf)
+        try:
+            with open(tf) as f:
+                data = json.load(f)
+        except json.decoder.JSONDecodeError:
+            data = {}
+    return data
 
 
 def get_build_index(subdir, bldpkgs_dir, output_folder=None, clear_cache=False,
@@ -249,43 +255,30 @@ def get_build_index(subdir, bldpkgs_dir, output_folder=None, clear_cache=False,
             superchannel = {}
             # we need channeldata.json too, as it is a more reliable source of run_exports data
             for channel in expanded_channels:
-                err = None
                 if channel.scheme == "file":
                     location = channel.location
-                    if not os.path.isabs(channel.location) and os.path.exists(os.path.join('/', channel.location)):
-                        location = os.path.join('/', channel.location)
-                    with open(os.path.join(location, channel.name, 'channeldata.json')) as f:
-                        channel_data[channel.name] = json.load(f)
-                else:
+                    if utils.on_win:
+                        location = location.lstrip("/")
+                    elif (not os.path.isabs(channel.location) and
+                            os.path.exists(os.path.join(os.path.sep, channel.location))):
+                        location = os.path.join(os.path.sep, channel.location)
+                    channeldata_file = os.path.join(location, channel.name, 'channeldata.json')
                     retry = 0
-                    max_retries = 1
+                    max_retries = 10
                     while retry < max_retries:
-                        # download channeldata.json for url
                         try:
-                            channel_content = _download_channeldata(channel.base_url + '/channeldata.json')
-                            if channel_content.status_code == 200:
-                                try:
-                                    # load its JSON content
-                                    channel_data[channel.name] = channel_content.json()
-                                except ValueError:
-                                    # no JSON data; skip it
-                                    pass
-                                break
-                            elif channel_content.status_code == 404:
-                                break
-                        except InvalidSchema as e:
-                            # store exception for potential later use; retry
-                            err = e
-                            log.warn("Problem downloading channeldata for url: %s.  Retrying (%d of %d) in 2 sec" % (
-                                channel.name, retry + 1, max_retries))
-                            time.sleep(2)
+                            with open(channeldata_file, "r+") as f:
+                                channel_data[channel.name] = json.load(f)
+                            break
+                        except (IOError, json.decoder.JSONDecodeError):
+                            time.sleep(0.2)
                             retry += 1
-                            if retry == max_retries:
-                                log.warn("Problem downloading channeldata for url: %s.  Exception may follow this message. "
-                                        "Rerun in debug mode for more info" % channel.name)
-                                if err:
-                                    log.warn(str(err))
-
+                else:
+                    # download channeldata.json for url
+                    try:
+                        channel_data[channel.name] = _download_channeldata(channel.base_url + '/channeldata.json')
+                    except CondaHTTPError:
+                        continue
                 # collapse defaults metachannel back into one superchannel, merging channeldata
                 if channel.base_url in context.default_channels and channel_data.get(channel.name):
                     packages = superchannel.get('packages', {})
diff --git a/conda_build/windows.py b/conda_build/windows.py
--- a/conda_build/windows.py
+++ b/conda_build/windows.py
@@ -276,7 +276,7 @@ def write_build_scripts(m, env, bld_bat):
         if m.config.activate and m.name() != 'conda':
             _write_bat_activation_text(fo, m)
     # bld_bat may have been generated elsewhere with contents of build/script
-    work_script = join(m.config.work_dir, 'bld.bat')
+    work_script = join(m.config.work_dir, 'conda_build.bat')
     if os.path.isfile(bld_bat):
         with open(bld_bat) as fi:
             data = fi.read()
@@ -329,7 +329,7 @@ def build(m, bld_bat, stats, provision_only=False):
     work_script, env_script = write_build_scripts(m, env, bld_bat)
 
     if not provision_only and os.path.isfile(work_script):
-        cmd = ['cmd.exe', '/c', 'bld.bat']
+        cmd = ['cmd.exe', '/c', os.path.basename(work_script)]
         # rewrite long paths in stdout back to their env variables
         if m.config.debug:
             rewrite_env = None


Test patch: