Repository: conan-io/conan
Pull request #11936 (2022-08-23 09:12:47)

Support remotes.json in 1.X
Changelog: Feature: Support remotes.json in Conan 1.X
Docs: TODO

Closes: https://github.com/conan-io/conan/issues/11777

**Important:** we should **clearly** document for this that if you have both txt and json in the folder, repo, etc. that you are config installing from, it can have undefined behaviour as the order of the install is not deterministic. One time the txt could prevail and others the json...



Patch:
diff --git a/conans/client/conf/config_installer.py b/conans/client/conf/config_installer.py
index 4e54c6b9b..434868531 100644
--- a/conans/client/conf/config_installer.py
+++ b/conans/client/conf/config_installer.py
@@ -108,9 +108,10 @@ def _process_file(directory, filename, config, cache, output, folder):
         finally:
             _filecopy(directory, filename, cache.cache_folder)
             migrate_registry_file(cache, output)
+    # FIXME: remove in 2.0
     elif filename == "remotes.json":
-        # Fix for Conan 2.0
-        raise ConanException("remotes.json install is not supported yet. Use 'remotes.txt'")
+        output.info("Defining remotes from remotes.json")
+        _filecopy(directory, filename, cache.cache_folder)
     else:
         # This is ugly, should be removed in Conan 2.0
         if filename in ("README.md", "LICENSE.txt") and not directory.startswith("templates"):
@@ -137,6 +138,7 @@ def _process_folder(config, folder, cache, output):
         raise ConanException("No such directory: '%s'" % str(folder))
     if config.source_folder:
         folder = os.path.join(folder, config.source_folder)
+
     for root, dirs, files in walk(folder):
         dirs[:] = [d for d in dirs if d != ".git"]
         for f in files:
diff --git a/conans/client/loader.py b/conans/client/loader.py
index d1d89130d..804d73b50 100644
--- a/conans/client/loader.py
+++ b/conans/client/loader.py
@@ -2,6 +2,7 @@ import fnmatch
 import imp
 import inspect
 import os
+import re
 import sys
 import types
 import uuid
@@ -436,13 +437,20 @@ def _parse_conanfile(conan_file_path):
         old_modules = list(sys.modules.keys())
         with chdir(current_dir):
             old_dont_write_bytecode = sys.dont_write_bytecode
-            sys.dont_write_bytecode = True
-            loaded = imp.load_source(module_id, conan_file_path)
-            sys.dont_write_bytecode = old_dont_write_bytecode
-
-        required_conan_version = getattr(loaded, "required_conan_version", None)
-        if required_conan_version:
-            validate_conan_version(required_conan_version)
+            try:
+                sys.dont_write_bytecode = True
+                # FIXME: imp is deprecated in favour of implib
+                loaded = imp.load_source(module_id, conan_file_path)
+                sys.dont_write_bytecode = old_dont_write_bytecode
+            except ImportError:
+                version_txt = _get_required_conan_version_without_loading(conan_file_path)
+                if version_txt:
+                    validate_conan_version(version_txt)
+                raise
+
+            required_conan_version = getattr(loaded, "required_conan_version", None)
+            if required_conan_version:
+                validate_conan_version(required_conan_version)
 
         # These lines are necessary, otherwise local conanfile imports with same name
         # collide, but no error, and overwrite other packages imports!!
@@ -475,3 +483,20 @@ def _parse_conanfile(conan_file_path):
         sys.path.pop(0)
 
     return loaded, module_id
+
+
+def _get_required_conan_version_without_loading(conan_file_path):
+    # First, try to detect the required_conan_version in "text" mode
+    # https://github.com/conan-io/conan/issues/11239
+    contents = load(conan_file_path)
+
+    txt_version = None
+
+    try:
+        found = re.search(r"required_conan_version\s*=\s*(.*)", contents)
+        if found:
+            txt_version = found.group(1).replace('"', "")
+    except:
+        pass
+
+    return txt_version
diff --git a/conans/model/manifest.py b/conans/model/manifest.py
index d323df770..1b260beaf 100644
--- a/conans/model/manifest.py
+++ b/conans/model/manifest.py
@@ -25,7 +25,8 @@ def gather_files(folder):
     symlinks = {}
     keep_python = get_env("CONAN_KEEP_PYTHON_FILES", False)
     for root, dirs, files in walk(folder):
-        dirs[:] = [d for d in dirs if d != "__pycache__"]  # Avoid recursing pycache
+        if not keep_python:
+            dirs[:] = [d for d in dirs if d != "__pycache__"]  # Avoid recursing pycache
         for d in dirs:
             abs_path = os.path.join(root, d)
             if os.path.islink(abs_path):


Test patch:
diff --git a/conans/test/functional/command/config_install_test.py b/conans/test/functional/command/config_install_test.py
index 7d438307e..400ea596d 100644
--- a/conans/test/functional/command/config_install_test.py
+++ b/conans/test/functional/command/config_install_test.py
@@ -333,13 +333,56 @@ class Pkg(ConanFile):
         self.client.run("remote list")
         self.assertEqual("conan.io: https://server.conan.io [Verify SSL: True]\n", self.client.out)
 
-    def test_install_remotes_json_error(self):
+    def test_install_remotes_json(self):
         folder = temp_folder()
-        save_files(folder, {"remotes.json": ""})
-        self.client.run('config install "%s"' % folder, assert_error=True)
-        self.assertIn("ERROR: Failed conan config install: "
-                      "remotes.json install is not supported yet. Use 'remotes.txt'",
-                      self.client.out)
+
+        remotes_json = textwrap.dedent("""
+            {
+                "remotes": [
+                    { "name": "repojson1", "url": "https://repojson1.net", "verify_ssl": false },
+                    { "name": "repojson2", "url": "https://repojson2.com", "verify_ssl": true }
+                ]
+            }
+        """)
+
+        remotes_txt = textwrap.dedent("""\
+            repotxt1 https://repotxt1.net False
+            repotxt2 https://repotxt2.com True
+        """)
+
+        # remotes.txt and json try to define both the remotes,
+        # could lead to unpredictable results
+        save_files(folder, {"remotes.json": remotes_json,
+                            "remotes.txt": remotes_txt})
+
+        self.client.run(f'config install "{folder}"')
+        assert "Defining remotes from remotes.json" in self.client.out
+        assert "Defining remotes from remotes.txt" in self.client.out
+
+        # If there's only a remotes.txt it's the one installed
+        folder = temp_folder()
+        save_files(folder, {"remotes.txt": remotes_txt})
+
+        self.client.run(f'config install "{folder}"')
+
+        assert "Defining remotes from remotes.txt" in self.client.out
+
+        self.client.run('remote list')
+
+        assert "repotxt1: https://repotxt1.net [Verify SSL: False]" in self.client.out
+        assert "repotxt2: https://repotxt2.com [Verify SSL: True]" in self.client.out
+
+        # If there's only a remotes.json it's the one installed
+        folder = temp_folder()
+        save_files(folder, {"remotes.json": remotes_json})
+
+        self.client.run(f'config install "{folder}"')
+        assert "Defining remotes from remotes.json" in self.client.out
+
+        self.client.run('remote list')
+
+        assert "repojson1: https://repojson1.net [Verify SSL: False]" in self.client.out
+        assert "repojson2: https://repojson2.com [Verify SSL: True]" in self.client.out
 
     def test_without_profile_folder(self):
         shutil.rmtree(self.client.cache.profiles_path)
diff --git a/conans/test/integration/conanfile/required_conan_version_test.py b/conans/test/integration/conanfile/required_conan_version_test.py
index 7f0c11f25..f0261c11b 100644
--- a/conans/test/integration/conanfile/required_conan_version_test.py
+++ b/conans/test/integration/conanfile/required_conan_version_test.py
@@ -35,3 +35,45 @@ class RequiredConanVersionTest(unittest.TestCase):
         client.run("install pkg/1.0@", assert_error=True)
         self.assertIn("Current Conan version (%s) does not satisfy the defined one (>=100.0)"
                       % __version__, client.out)
+
+    def test_required_conan_version_with_loading_issues(self):
+        # https://github.com/conan-io/conan/issues/11239
+        client = TestClient()
+        conanfile = textwrap.dedent("""
+                    from conan import missing_import
+
+                    required_conan_version = ">=100.0"
+
+                    class Lib(ConanFile):
+                        pass
+                    """)
+        client.save({"conanfile.py": conanfile})
+        client.run("export . pkg/1.0@", assert_error=True)
+        self.assertIn("Current Conan version (%s) does not satisfy the defined one (>=100.0)"
+                      % __version__, client.out)
+
+        # Assigning required_conan_version without spaces
+        conanfile = textwrap.dedent("""
+                            from conan import missing_import
+
+                            required_conan_version=">=100.0"
+
+                            class Lib(ConanFile):
+                                pass
+                            """)
+        client.save({"conanfile.py": conanfile})
+        client.run("export . pkg/1.0@", assert_error=True)
+        self.assertIn("Current Conan version (%s) does not satisfy the defined one (>=100.0)"
+                      % __version__, client.out)
+
+        # If the range is correct, everything works, of course
+        conanfile = textwrap.dedent("""
+                            from conan import ConanFile
+
+                            required_conan_version = ">1.0.0"
+
+                            class Lib(ConanFile):
+                                pass
+                            """)
+        client.save({"conanfile.py": conanfile})
+        client.run("export . pkg/1.0@")
diff --git a/conans/test/unittests/model/manifest_test.py b/conans/test/unittests/model/manifest_test.py
index 0945a0650..4a88b65b1 100644
--- a/conans/test/unittests/model/manifest_test.py
+++ b/conans/test/unittests/model/manifest_test.py
@@ -1,6 +1,6 @@
 import os
 
-
+from conans.client.tools import environment_append
 from conans.model.manifest import FileTreeManifest
 from conans.test.utils.test_files import temp_folder
 from conans.util.files import load, md5, save
@@ -49,3 +49,17 @@ class TestManifest:
         save(os.path.join(tmp_dir, "conanmanifest.txt"), "1478122267\nsome: file.py: 123\n")
         read_manifest = FileTreeManifest.load(tmp_dir)
         assert read_manifest.file_sums["some: file.py"] == "123"
+
+
+def test_pycache_included():
+    tmp_dir = temp_folder()
+    files = {"__pycache__/damn.py": "binarythings",
+             "pythonfile.pyc": "binarythings3"}
+    for filename, content in files.items():
+        save(os.path.join(tmp_dir, filename), content)
+
+    with environment_append({"CONAN_KEEP_PYTHON_FILES": "1"}):
+        manifest = FileTreeManifest.create(tmp_dir)
+    manifest = repr(manifest)
+    assert "pythonfile.pyc" in manifest
+    assert "__pycache__/damn.py" in manifest