Repository: conan-io/conan

Issue:
Issues with proxies
I've started Conan server in my organization's local network. For external sites (like github) proxy must be used, but not for computers in local network.
So, I have there env variables:
```
HTTP_PROXY=http://proxy.domain:3128
HTTP_PROXY=https://proxy.domain:3128
NO_PROXY=localhost,127.0.0.1,*.domain,192.168.*
```
With these variables I can create packages with recipes in my local cache (and this recipes can download external sources), but cannot download recipes and prebuilt packages from organization's Conan server.

I've tried to set same settings in conan.conf, but had no luck. Also I've tried to use Conan server domain name and/or IP address in NO_PROXY with same result:
```
> conan search -r <remotename> "OpenCV*"
ERROR: Permission denied for user: '<username>'. [Remote: <remotename>]
```

But if I disable all proxies (comment in conan.conf and unset env variables), it works just fine:
```
Existing package recipes:

OpenCV/3.3.1@lukyanets/testing
```

Conan v. 1.0.1, OS: Arch Linux, CentOS 6, Windows 10.

Discussion:

Patch:
diff --git a/conans/client/cmd/export.py b/conans/client/cmd/export.py
index ed8a2e3e2..ac240f57e 100644
--- a/conans/client/cmd/export.py
+++ b/conans/client/cmd/export.py
@@ -113,6 +113,9 @@ def _capture_export_scm_data(conanfile, src_path, destination_folder, output, pa
         origin = scm.get_remote_url()
         if not origin:
             raise ConanException("Repo origin cannot be deduced by 'auto'")
+        if os.path.exists(origin):
+            output.warn("Repo origin looks like a local path: %s" % origin)
+            origin = origin.replace("\\", "/")
         output.success("Repo origin deduced by 'auto': %s" % origin)
         scm.url = origin
     if scm.revision == "auto":
diff --git a/conans/client/conf/__init__.py b/conans/client/conf/__init__.py
index 2be13e19e..f210659e9 100644
--- a/conans/client/conf/__init__.py
+++ b/conans/client/conf/__init__.py
@@ -31,6 +31,7 @@ os:
         version: ["8.1", "10.0"]
     Linux:
     Macos:
+        version: [None, "10.6", "10.7", "10.8", "10.9", "10.10", "10.11", "10.12", "10.13", "10.14"]
     Android:
         api_level: ANY
     iOS:
@@ -66,7 +67,7 @@ compiler:
         version: ["3.3", "3.4", "3.5", "3.6", "3.7", "3.8", "3.9", "4.0", "5.0", "6.0", "7.0"]
         libcxx: [libstdc++, libstdc++11, libc++]
     apple-clang:
-        version: ["5.0", "5.1", "6.0", "6.1", "7.0", "7.3", "8.0", "8.1", "9.0", "9.1"]
+        version: ["5.0", "5.1", "6.0", "6.1", "7.0", "7.3", "8.0", "8.1", "9.0", "9.1", "10.0"]
         libcxx: [libstdc++, libc++]
 
 build_type: [None, Debug, Release, RelWithDebInfo, MinSizeRel]
diff --git a/conans/client/conf/detect.py b/conans/client/conf/detect.py
index a12341d72..5a39c0646 100644
--- a/conans/client/conf/detect.py
+++ b/conans/client/conf/detect.py
@@ -61,7 +61,7 @@ def _clang_compiler(output, compiler_exe="clang"):
             compiler = "apple-clang"
         elif "clang version" in out:
             compiler = "clang"
-        installed_version = re.search("([0-9]\.[0-9])", out).group()
+        installed_version = re.search("([0-9]+\.[0-9])", out).group()
         if installed_version:
             output.success("Found %s %s" % (compiler, installed_version))
             return compiler, installed_version
diff --git a/conans/client/migrations.py b/conans/client/migrations.py
index 9a7b586a8..6d132a89d 100644
--- a/conans/client/migrations.py
+++ b/conans/client/migrations.py
@@ -42,15 +42,17 @@ class ClientMigrator(Migrator):
         # VERSION 0.1
         if old_version is None:
             return
-        if old_version < Version("1.4.0"):
+        if old_version < Version("1.5.0"):
             old_settings = """
 # Only for cross building, 'os_build/arch_build' is the system that runs Conan
 os_build: [Windows, WindowsStore, Linux, Macos, FreeBSD, SunOS]
 arch_build: [x86, x86_64, ppc64le, ppc64, armv6, armv7, armv7hf, armv8, sparc, sparcv9, mips, mips64, avr, armv7s, armv7k]
+
 # Only for building cross compilation tools, 'os_target/arch_target' is the system for
 # which the tools generate code
 os_target: [Windows, Linux, Macos, Android, iOS, watchOS, tvOS, FreeBSD, SunOS, Arduino]
 arch_target: [x86, x86_64, ppc64le, ppc64, armv6, armv7, armv7hf, armv8, sparc, sparcv9, mips, mips64, avr, armv7s, armv7k]
+
 # Rest of the settings are "host" settings:
 # - For native building/cross building: Where the library/program will run.
 # - For building cross compilation tools: Where the cross compiler will run.
@@ -83,7 +85,8 @@ compiler:
         version: ["4.1", "4.4", "4.5", "4.6", "4.7", "4.8", "4.9",
                   "5", "5.1", "5.2", "5.3", "5.4", "5.5",
                   "6", "6.1", "6.2", "6.3", "6.4",
-                  "7", "7.1", "7.2", "7.3"]
+                  "7", "7.1", "7.2", "7.3",
+                  "8", "8.1"]
         libcxx: [libstdc++, libstdc++11]
         threads: [None, posix, win32] #  Windows MinGW
         exception: [None, dwarf2, sjlj, seh] # Windows MinGW
@@ -92,13 +95,14 @@ compiler:
         version: ["8", "9", "10", "11", "12", "14", "15"]
         toolset: [None, v90, v100, v110, v110_xp, v120, v120_xp, v140, v140_xp, v140_clang_c2, LLVM-vs2014, LLVM-vs2014_xp, v141, v141_xp, v141_clang_c2]
     clang:
-        version: ["3.3", "3.4", "3.5", "3.6", "3.7", "3.8", "3.9", "4.0", "5.0", "6.0"]
+        version: ["3.3", "3.4", "3.5", "3.6", "3.7", "3.8", "3.9", "4.0", "5.0", "6.0", "7.0"]
         libcxx: [libstdc++, libstdc++11, libc++]
     apple-clang:
         version: ["5.0", "5.1", "6.0", "6.1", "7.0", "7.3", "8.0", "8.1", "9.0", "9.1"]
         libcxx: [libstdc++, libc++]
+
 build_type: [None, Debug, Release, RelWithDebInfo, MinSizeRel]
-cppstd: [None, 98, gnu98, 11, gnu11, 14, gnu14, 17, gnu17]
+cppstd: [None, 98, gnu98, 11, gnu11, 14, gnu14, 17, gnu17, 20, gnu20]
 """
             self._update_settings_yml(old_settings)
 
diff --git a/conans/client/rest/conan_requester.py b/conans/client/rest/conan_requester.py
index 82686d110..384becba7 100644
--- a/conans/client/rest/conan_requester.py
+++ b/conans/client/rest/conan_requester.py
@@ -9,7 +9,7 @@ class ConanRequester(object):
     def __init__(self, requester, client_cache, timeout):
         self.proxies = client_cache.conan_config.proxies or {}
         self._no_proxy_match = [el.strip() for el in
-                                self.proxies.pop("no_proxy_match", "").split(",")]
+                                self.proxies.pop("no_proxy_match", "").split(",") if el]
         self._timeout_seconds = timeout
 
         # Retrocompatibility with deprecated no_proxy
@@ -58,14 +58,28 @@ class ConanRequester(object):
         return kwargs
 
     def get(self, url, **kwargs):
-        return self._requester.get(url, **self._add_kwargs(url, kwargs))
+        return self._call_method("get", url, **kwargs)
 
     def put(self, url, **kwargs):
-        return self._requester.put(url, **self._add_kwargs(url, kwargs))
+        return self._call_method("put", url, **kwargs)
 
     def delete(self, url, **kwargs):
-        return self._requester.delete(url, **self._add_kwargs(url, kwargs))
+        return self._call_method("delete", url, **kwargs)
 
     def post(self, url, **kwargs):
-        return self._requester.post(url, **self._add_kwargs(url, kwargs))
-
+        return self._call_method("post", url, **kwargs)
+
+    def _call_method(self, method, url, **kwargs):
+        popped = False
+        if self.proxies or self._no_proxy_match:
+            old_env = dict(os.environ)
+            # Clean the proxies from the environ and use the conan specified proxies
+            for var_name in ("http_proxy", "https_proxy", "no_proxy"):
+                popped = popped or os.environ.pop(var_name, None)
+                popped = popped or os.environ.pop(var_name.upper(), None)
+        try:
+            return getattr(self._requester, method)(url, **self._add_kwargs(url, kwargs))
+        finally:
+            if popped:
+                os.environ.clear()
+                os.environ.update(old_env)
diff --git a/conans/client/tools/win.py b/conans/client/tools/win.py
index 2cfcd3741..d02923871 100644
--- a/conans/client/tools/win.py
+++ b/conans/client/tools/win.py
@@ -5,6 +5,7 @@ import re
 import deprecation
 
 import subprocess
+
 from contextlib import contextmanager
 
 from conans.client.tools.env import environment_append
@@ -136,8 +137,6 @@ def vswhere(all_=False, prerelease=False, products=None, requires=None, version=
 
     program_files = os.environ.get("ProgramFiles(x86)", os.environ.get("ProgramFiles"))
 
-    vswhere_path = ""
-
     if program_files:
         vswhere_path = os.path.join(program_files, "Microsoft Visual Studio", "Installer",
                                     "vswhere.exe")
@@ -186,11 +185,16 @@ def vswhere(all_=False, prerelease=False, products=None, requires=None, version=
 
     try:
         output = subprocess.check_output(arguments)
-        vswhere_out = decode_text(output).strip()
+        output = decode_text(output).strip()
+        # Ignore the "description" field, that even decoded contains non valid charsets for json
+        # (ignored ones)
+        output = "\n".join([line for line in output.splitlines()
+                            if not line.strip().startswith('"description"')])
+
     except (ValueError, subprocess.CalledProcessError, UnicodeDecodeError) as e:
         raise ConanException("vswhere error: %s" % str(e))
 
-    return json.loads(vswhere_out)
+    return json.loads(output)
 
 
 def vs_comntools(compiler_version):


Test patch:
diff --git a/conans/test/build_helpers/cpp_std_flags_test.py b/conans/test/build_helpers/cpp_std_flags_test.py
index 6d80880d2..70f09e990 100644
--- a/conans/test/build_helpers/cpp_std_flags_test.py
+++ b/conans/test/build_helpers/cpp_std_flags_test.py
@@ -148,6 +148,8 @@ class CompilerFlagsTest(unittest.TestCase):
         self.assertEquals(cppstd_flag("apple-clang", "9.1", "14"), '-std=c++14')
         self.assertEquals(cppstd_flag("apple-clang", "9.1", "17"), "-std=c++17")
 
+        self.assertEquals(cppstd_flag("apple-clang", "10.0", "17"), "-std=c++17")
+
     def test_apple_clang_cppstd_defaults(self):
         self.assertEquals(cppstd_default("apple-clang", "2"), "gnu98")
         self.assertEquals(cppstd_default("apple-clang", "3"), "gnu98")
@@ -157,6 +159,7 @@ class CompilerFlagsTest(unittest.TestCase):
         self.assertEquals(cppstd_default("apple-clang", "7"), "gnu98")
         self.assertEquals(cppstd_default("apple-clang", "8"), "gnu98")
         self.assertEquals(cppstd_default("apple-clang", "9"), "gnu98")
+        self.assertEquals(cppstd_default("apple-clang", "10"), "gnu98")
 
     def test_visual_cppstd_flags(self):
         self.assertEquals(cppstd_flag("Visual Studio", "12", "11"), None)
diff --git a/conans/test/functional/proxies_conf_test.py b/conans/test/functional/proxies_conf_test.py
index c1b27c56d..c1ab870ae 100644
--- a/conans/test/functional/proxies_conf_test.py
+++ b/conans/test/functional/proxies_conf_test.py
@@ -1,6 +1,7 @@
 import unittest
 import os
 
+from conans import tools
 from conans.test.utils.tools import TestClient
 from conans.util.files import save
 from conans.client.conan_api import get_basic_requester
@@ -59,3 +60,44 @@ http=http://conan.url
         self.assertEquals(client.requester.get("**otherexcluded_one***"), "excluded!")
         self.assertEquals(client.requester.get("MyExcludedUrl***"), "excluded!")
         self.assertEquals(client.requester.get("**MyExcludedUrl***"), "not excluded!")
+
+    def test_environ_kept(self):
+        client = TestClient()
+        conf = """
+[proxies]
+        """
+        save(client.client_cache.conan_conf_path, conf)
+        client.client_cache.invalidate()
+        requester = get_basic_requester(client.client_cache)
+
+        def verify_env(url, **kwargs):
+            self.assertTrue("HTTP_PROXY" in os.environ)
+
+        with tools.environment_append({"HTTP_PROXY": "my_system_proxy"}):
+            requester._requester.get = verify_env
+            requester.get("MyUrl")
+
+    def test_environ_removed(self):
+
+        client = TestClient()
+        conf = """
+[proxies]
+no_proxy_match=MyExcludedUrl*
+"""
+        save(client.client_cache.conan_conf_path, conf)
+        client.client_cache.invalidate()
+        requester = get_basic_requester(client.client_cache)
+
+        def verify_env(url, **kwargs):
+            self.assertFalse("HTTP_PROXY" in os.environ)
+            self.assertFalse("http_proxy" in os.environ)
+
+        with tools.environment_append({"http_proxy": "my_system_proxy"}):
+            requester._requester.get = verify_env
+            requester.get("MyUrl")
+            self.assertEqual(os.environ["http_proxy"], "my_system_proxy")
+
+        with tools.environment_append({"HTTP_PROXY": "my_system_proxy"}):
+            requester._requester.get = verify_env
+            requester.get("MyUrl")
+            self.assertEqual(os.environ["HTTP_PROXY"], "my_system_proxy")
diff --git a/conans/test/functional/scm_test.py b/conans/test/functional/scm_test.py
index 2420e5b44..e510fce5c 100644
--- a/conans/test/functional/scm_test.py
+++ b/conans/test/functional/scm_test.py
@@ -79,6 +79,23 @@ class ConanLib(ConanFile):
         folder = self.client.client_cache.source(ConanFileReference.loads("lib/0.1@user/channel"))
         self.assertIn("othersubfolder", os.listdir(folder))
 
+    def test_auto_filesystem_remote_git(self):
+        # https://github.com/conan-io/conan/issues/3109
+        conanfile = base.format(directory="None", url="auto", revision="auto")
+        repo = temp_folder()
+        self.client.save({"conanfile.py": conanfile, "myfile.txt": "My file is copied"}, repo)
+        self.client.runner("git init .", cwd=repo)
+        self.client.runner('git config user.email "you@example.com"', cwd=repo)
+        self.client.runner('git config user.name "Your Name"', cwd=repo)
+        self.client.runner("git add .", cwd=repo)
+        self.client.runner('git commit -m  "commiting"', cwd=repo)
+        self.client.runner('git clone "%s" .' % repo, cwd=self.client.current_folder)
+        self.client.run("export . user/channel")
+        self.assertIn("WARN: Repo origin looks like a local path", self.client.out)
+        os.remove(self.client.client_cache.scm_folder(ConanFileReference.loads("lib/0.1@user/channel")))
+        self.client.run("install lib/0.1@user/channel --build")
+        self.assertIn("lib/0.1@user/channel: Getting sources from url:", self.client.out)
+
     def test_auto_git(self):
         curdir = self.client.current_folder.replace("\\", "/")
         conanfile = base.format(directory="None", url="auto", revision="auto")
diff --git a/conans/test/util/tools_test.py b/conans/test/util/tools_test.py
index fff78866e..eff17920c 100644
--- a/conans/test/util/tools_test.py
+++ b/conans/test/util/tools_test.py
@@ -6,6 +6,9 @@ import platform
 import unittest
 
 from collections import namedtuple
+
+import six
+from mock.mock import patch, mock_open
 from six import StringIO
 
 from conans.client.client_cache import CONAN_CONF
@@ -14,7 +17,7 @@ from conans import tools
 from conans.client.conan_api import ConanAPIV1
 from conans.client.conf import default_settings_yml, default_client_conf
 from conans.client.output import ConanOutput
-from conans.client.tools.win import vcvars_dict
+from conans.client.tools.win import vcvars_dict, vswhere
 from conans.client.tools.scm import Git
 
 from conans.errors import ConanException, NotFoundException
@@ -516,9 +519,8 @@ class HelloConan(ConanFile):
         # This package hopefully doesn't exist
         self.assertFalse(spt._tool.installed("oidfjgesiouhrgioeurhgielurhgaeiorhgioearhgoaeirhg"))
 
+    @unittest.skipUnless(platform.system() == "Windows", "Requires vswhere")
     def msvc_build_command_test(self):
-        if platform.system() != "Windows":
-            return
         settings = Settings.loads(default_settings_yml)
         settings.os = "Windows"
         settings.compiler = "Visual Studio"
@@ -535,12 +537,77 @@ class HelloConan(ConanFile):
         with self.assertRaisesRegexp(ConanException, "Cannot build_sln_command"):
             tools.msvc_build_command(settings, "project.sln")
 
-        # succesful definition via settings
+        # successful definition via settings
         settings.build_type = "Debug"
         cmd = tools.msvc_build_command(settings, "project.sln")
         self.assertIn('msbuild project.sln /p:Configuration=Debug /p:Platform="x86"', cmd)
         self.assertIn('vcvarsall.bat', cmd)
 
+    @unittest.skipUnless(platform.system() == "Windows", "Requires vswhere")
+    def vswhere_description_strip_test(self):
+        myoutput = """
+[
+  {
+    "instanceId": "17609d7c",
+    "installDate": "2018-06-11T02:15:04Z",
+    "installationName": "VisualStudio/15.7.3+27703.2026",
+    "installationPath": "",
+    "installationVersion": "15.7.27703.2026",
+    "productId": "Microsoft.VisualStudio.Product.Enterprise",
+    "productPath": "",
+    "isPrerelease": false,
+    "displayName": "Visual Studio Enterprise 2017",
+    "description": "生産性向上と、さまざまな規模のチーム間の調整のための Microsoft DevOps ソリューション",
+    "channelId": "VisualStudio.15.Release",
+    "channelUri": "https://aka.ms/vs/15/release/channel",
+    "enginePath": "",
+    "releaseNotes": "https://go.microsoft.com/fwlink/?LinkId=660692#15.7.3",
+    "thirdPartyNotices": "https://go.microsoft.com/fwlink/?LinkId=660708",
+    "updateDate": "2018-06-11T02:15:04.7009868Z",
+    "catalog": {
+      "buildBranch": "d15.7",
+      "buildVersion": "15.7.27703.2026",
+      "id": "VisualStudio/15.7.3+27703.2026",
+      "localBuild": "build-lab",
+      "manifestName": "VisualStudio",
+      "manifestType": "installer",
+      "productDisplayVersion": "15.7.3",
+      "productLine": "Dev15",
+      "productLineVersion": "2017",
+      "productMilestone": "RTW",
+      "productMilestoneIsPreRelease": "False",
+      "productName": "Visual Studio",
+      "productPatchVersion": "3",
+      "productPreReleaseMilestoneSuffix": "1.0",
+      "productRelease": "RTW",
+      "productSemanticVersion": "15.7.3+27703.2026",
+      "requiredEngineVersion": "1.16.1187.57215"
+    },
+    "properties": {
+      "campaignId": "",
+      "canceled": "0",
+      "channelManifestId": "VisualStudio.15.Release/15.7.3+27703.2026",
+      "nickname": "",
+      "setupEngineFilePath": ""
+    }
+  },
+  {
+    "instanceId": "VisualStudio.12.0",
+    "installationPath": "",
+    "installationVersion": "12.0"
+  }
+]
+
+"""
+        if six.PY3:
+            # In python3 the output from subprocess.check_output are bytes, not str
+            myoutput = myoutput.encode()
+        myrunner = mock_open()
+        myrunner.check_output = lambda x: myoutput
+        with patch('conans.client.tools.win.subprocess', myrunner):
+            json = vswhere()
+            self.assertNotIn("descripton", json)
+
     def vcvars_echo_test(self):
         if platform.system() != "Windows":
             return