Repository: conan-io/conan

Issue:
[SCM] Issue related to AST parsing with non-ascii characters
- [x] Conan >= 1.8

After introducing AST parsing to resolve ``auto`` keyword in SCM feature, people are having issues with py2 and unicode. A ``conanfile.py`` like [this](https://github.com/hanickadot/compile-time-regular-expressions/blob/bff10b5acc653c12a157abf5fd63cad0283ae69f/conanfile.py) should work:

```python
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from conans import ConanFile


class CtreConan(ConanFile):
    name = "CTRE"
    license = "MIT"
    url = "https://github.com/hanickadot/compile-time-regular-expressions"
    author = "Hana Dusíková (ctre@hanicka.net)"
    description = "Compile Time Regular Expression for C++17/20"
    homepage = "https://github.com/hanickadot/compile-time-regular-expressions"
    no_copy_source = True
    scm = {
        "type": "git",
        "url": "auto",
        "revision": "auto"
    }
    
    def package(self):
        self.copy("LICENSE", "licenses")
        self.copy("*.hpp")
    
    def package_id(self):
        self.info.header_only()
```

There are several possible issues to test:
- non-ascii chars (author name)
- shebang
- shebang with py2 or py3 and running with a different one
- encoding header string




Discussion:

Resolving patch:
diff --git a/conans/client/cmd/export.py b/conans/client/cmd/export.py
--- a/conans/client/cmd/export.py
+++ b/conans/client/cmd/export.py
@@ -1,6 +1,7 @@
 import ast
 import os
 import shutil
+import six
 
 from conans.client.cmd.export_linter import conan_linter
 from conans.client.file_copier import FileCopier
@@ -13,7 +14,6 @@
 from conans.util.files import save, rmdir, is_dirty, set_dirty, mkdir, load
 from conans.util.log import logger
 from conans.search.search import search_recipes
-from conans.client.plugin_manager import PluginManager
 
 
 def export_alias(reference, target_reference, client_cache):
@@ -99,6 +99,19 @@ def _capture_export_scm_data(conanfile, conanfile_dir, destination_folder, outpu
 def _replace_scm_data_in_conanfile(conanfile_path, scm_data):
     # Parsing and replacing the SCM field
     content = load(conanfile_path)
+    headers = []
+
+    if six.PY2:
+        # Workaround for https://bugs.python.org/issue22221
+        lines_without_headers = []
+        lines = content.splitlines(True)
+        for line in lines:
+            if not lines_without_headers and line.startswith("#"):
+                headers.append(line)
+            else:
+                lines_without_headers.append(line)
+        content = ''.join(lines_without_headers)
+
     lines = content.splitlines(True)
     tree = ast.parse(content)
     to_replace = []
@@ -127,6 +140,7 @@ def _replace_scm_data_in_conanfile(conanfile_path, scm_data):
 
     new_text = "scm = " + ",\n          ".join(str(scm_data).split(",")) + "\n"
     content = content.replace(to_replace[0], new_text)
+    content = content if not headers else ''.join(headers) + content
     save(conanfile_path, content)
 
 


Test patch:
diff --git a/conans/test/client/cmd/ast_replacement_scm_test.py b/conans/test/client/cmd/ast_replacement_scm_test.py
new file mode 100644
--- /dev/null
+++ b/conans/test/client/cmd/ast_replacement_scm_test.py
@@ -0,0 +1,80 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import os
+import unittest
+import tempfile
+import shutil
+import uuid
+import codecs
+import six
+
+from conans.util.files import load
+from conans.test.utils.tools import handleRemoveReadonly
+from conans.client.cmd.export import _replace_scm_data_in_conanfile
+
+
+class ASTReplacementTest(unittest.TestCase):
+    scm_data = {'type': 'git',
+                'url': 'this-is-the-url',
+                'revision': '42'}
+
+    conanfile = six.u("""{header}
+from conans import ConanFile
+
+class LibConan(ConanFile):
+    name = "Lib"
+    author = "{author}"
+    scm = {{"type": "git",
+           "url": "auto",
+           "revision": "auto"}}
+    """)
+
+    def run(self, *args, **kwargs):
+        self._tmp_folder = tempfile.mkdtemp(suffix='_conans')
+        try:
+            super(ASTReplacementTest, self).run(*args, **kwargs)
+        finally:
+            shutil.rmtree(self._tmp_folder, ignore_errors=False, onerror=handleRemoveReadonly)
+
+    def _get_conanfile(self, header='', author="jgsogo", encoding="ascii"):
+        tmp = os.path.join(self._tmp_folder, str(uuid.uuid4()))
+        with codecs.open(tmp, 'w', encoding=encoding) as f:
+            f.write(self.conanfile.format(header=header, author=author))
+        return tmp
+
+    def _check_result(self, conanfile):
+        content = load(conanfile)
+
+        self.assertEqual(content.count(self.scm_data['url']), 1)
+        self.assertEqual(content.count(self.scm_data['revision']), 1)
+        self.assertIn(self.scm_data['url'], content)
+        self.assertIn(self.scm_data['revision'], content)
+
+    def test_base(self):
+        conanfile = self._get_conanfile()
+        _replace_scm_data_in_conanfile(conanfile, self.scm_data)
+        self._check_result(conanfile)
+
+    def test_author_non_ascii(self):
+        conanfile = self._get_conanfile(author=six.u("¡ÑÁí!"), encoding='utf-8')
+        _replace_scm_data_in_conanfile(conanfile, self.scm_data)
+        self._check_result(conanfile)
+
+    def test_shebang_utf8(self):
+        header = "#!/usr/bin/env python2\n# -*- coding: utf-8 -*-"
+        conanfile = self._get_conanfile(author=six.u("¡Ñandú!"), header=header, encoding='utf-8')
+        _replace_scm_data_in_conanfile(conanfile, self.scm_data)
+        self._check_result(conanfile)
+
+    def test_shebang_ascii(self):
+        header = "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-"
+        conanfile = self._get_conanfile(author="jgsogo", header=header, encoding='ascii')
+        _replace_scm_data_in_conanfile(conanfile, self.scm_data)
+        self._check_result(conanfile)
+
+    def test_shebang_several(self):
+        header = "#!/usr/bin/env python2\n# -*- coding: utf-8 -*-\n# -*- coding: utf-8 -*-"
+        conanfile = self._get_conanfile(author=six.u("¡Ñandú!"), header=header, encoding='utf-8')
+        _replace_scm_data_in_conanfile(conanfile, self.scm_data)
+        self._check_result(conanfile)
\ No newline at end of file