Repository: conan-io/conan

Issue:
scm replacement fails if scm = statement contains or is followed by blank line
conan 1.11.2, Windows 7

In one of my conanfiles, I am using the scm features, with a snippet that reads like

```
10:    scm = {
11:        "type": "git",
12:        "url": "https://github.deere.com/FOCUS/conan-strawberryperl-PAR-Packer.git",
13:        "revision": "auto"
14:    }
15: 
16:    # short_paths is needed for builds (the pp tests have some long paths), even though the package doesn't end up including any deep paths
17:    short_paths=True
```

When this recipe is expocted, the revision = "auto" is *not* replaced, even though I do get the message
```
package/1.0@user/channel: Revision deduced by 'auto': 570c6fe032219a17cc3268b7cf1858635e76bde8
```

I've tracked this back to https://github.com/conan-io/conan/blob/6ec73ac2b4be98e1ab4e5ee797008b50ca2d7424/conans/client/cmd/export.py#L208-L209

Basically, it looks like this code takes everythign from the statement assigning scm (e.g. line 10), up to but not including the line containing the next item in the body (so 10..16). 

But when forming a string it then skips over blank lines (e.g. line 15 is skipped). The resulting string in to_replace is then missing the blank line, so the https://github.com/conan-io/conan/blob/6ec73ac2b4be98e1ab4e5ee797008b50ca2d7424/conans/client/cmd/export.py#L216 doesn't find a match and doesn't replace anything.

So, to propose a couple of fixes:

* There should be an error if content.replace doesn't actually match to_replace; we never want to leave any "auto" values in the exported recipe.

* rather than skipping blank lines entirely in the list comprehension to assign replace, it should just trim leading or trailing blank lines, but retain those in the middle. 

* Maybe it should trim comment-only lines too, so as not to eat comments between scm= and the next assignment?

So maybe something like:

```diff
-                        replace = [line for line in lines[(stmt.lineno-1):next_line]
-                                   if line.strip()]
+                       replace = lines[(stmt.lineno-1):next_line]
+                       while re.fullmatch(r'\s*(#.*)?',replace[-1],re.DOTALL): 
+                            del replace[-1]
```

To help us debug your issue please explain:

- [x] I've read the [CONTRIBUTING guide](https://github.com/conan-io/conan/blob/develop/.github/CONTRIBUTING.md).
- [x] I've specified the Conan version, operating system version and any tool that can be relevant.
- [x] I've explained the steps to reproduce the error or the motivation/use case of the question/suggestion.




Discussion:
Bug confirmed. Thanks for the detailed investigation.
I agree there should be an error when the replacement is supposed to happen but it doesn't.

Resolving patch:
diff --git a/conans/client/cmd/export.py b/conans/client/cmd/export.py
--- a/conans/client/cmd/export.py
+++ b/conans/client/cmd/export.py
@@ -189,6 +189,7 @@ def _replace_scm_data_in_conanfile(conanfile_path, scm_data):
     lines = content.splitlines(True)
     tree = ast.parse(content)
     to_replace = []
+    comments = []
     for i_body, item in enumerate(tree.body):
         if isinstance(item, ast.ClassDef):
             statements = item.body
@@ -203,16 +204,22 @@ def _replace_scm_data_in_conanfile(conanfile_path, scm_data):
                                     next_line = tree.body[i_body+1].lineno - 1
                             else:
                                 next_line = statements[i+1].lineno - 1
+                                next_line_content = lines[next_line].strip()
+                                if next_line_content.endswith('"""') or next_line_content.endswith("'''"):
+                                    next_line += 1
                         except IndexError:
                             next_line = stmt.lineno
-                        replace = [line for line in lines[(stmt.lineno-1):next_line]
-                                   if line.strip()]
+                        replace = [line for line in lines[(stmt.lineno-1):next_line]]
                         to_replace.append("".join(replace).lstrip())
+                        comments = [line.strip('\n') for line in replace
+                                    if line.strip().startswith("#") or not line.strip()]
                         break
     if len(to_replace) != 1:
         raise ConanException("The conanfile.py defines more than one class level 'scm' attribute")
 
     new_text = "scm = " + ",\n          ".join(str(scm_data).split(",")) + "\n"
+    if comments:
+        new_text += '\n'.join(comments) + "\n"
     content = content.replace(to_replace[0], new_text)
     content = content if not headers else ''.join(headers) + content
 


Test patch:
diff --git a/conans/test/unittests/client/cmd/ast_replacement_scm_test.py b/conans/test/unittests/client/cmd/ast_replacement_scm_test.py
--- a/conans/test/unittests/client/cmd/ast_replacement_scm_test.py
+++ b/conans/test/unittests/client/cmd/ast_replacement_scm_test.py
@@ -11,11 +11,14 @@
 import six
 
 from conans.client.cmd.export import _replace_scm_data_in_conanfile
+from conans.client.graph.python_requires import ConanPythonRequire
+from conans.client.loader import parse_conanfile
 from conans.test.utils.tools import try_remove_readonly
 from conans.util.files import load
 
 
 class ASTReplacementTest(unittest.TestCase):
+    python_requires = ConanPythonRequire(None, None)
     scm_data = {'type': 'git',
                 'url': 'this-is-the-url',
                 'revision': '42'}
@@ -29,6 +32,7 @@ class LibConan(ConanFile):
     scm = {{"type": "git",
            "url": "auto",
            "revision": "auto"}}
+{footer}
     """)
 
     def run(self, *args, **kwargs):
@@ -38,10 +42,10 @@ def run(self, *args, **kwargs):
         finally:
             shutil.rmtree(self._tmp_folder, ignore_errors=False, onerror=try_remove_readonly)
 
-    def _get_conanfile(self, header='', author="jgsogo", encoding="ascii"):
+    def _get_conanfile(self, header='', author="jgsogo", encoding="ascii", footer=""):
         tmp = os.path.join(self._tmp_folder, str(uuid.uuid4()))
         with codecs.open(tmp, 'w', encoding=encoding) as f:
-            f.write(self.conanfile.format(header=header, author=author))
+            f.write(self.conanfile.format(header=header, author=author, footer=footer))
         return tmp
 
     def _check_result(self, conanfile):
@@ -52,11 +56,20 @@ def _check_result(self, conanfile):
         self.assertIn(self.scm_data['url'], content)
         self.assertIn(self.scm_data['revision'], content)
 
+        try:
+            # Check it is loadable by Conan machinery
+            _, conanfile = parse_conanfile(conanfile, python_requires=self.python_requires)
+        except Exception as e:
+            self.fail("Invalid conanfile: {}".format(e))
+        else:
+            self.assertEqual(conanfile.scm, self.scm_data)
+
     def test_base(self):
         conanfile = self._get_conanfile()
         _replace_scm_data_in_conanfile(conanfile, self.scm_data)
         self._check_result(conanfile)
 
+    @unittest.skipUnless(six.PY3, "Works only in Py3 (assumes utf-8 for source files)")
     def test_author_non_ascii(self):
         conanfile = self._get_conanfile(author=six.u("¡ÑÁí!"), encoding='utf-8')
         _replace_scm_data_in_conanfile(conanfile, self.scm_data)
@@ -78,4 +91,65 @@ def test_shebang_several(self):
         header = "#!/usr/bin/env python2\n# -*- coding: utf-8 -*-\n# -*- coding: utf-8 -*-"
         conanfile = self._get_conanfile(author=six.u("¡Ñandú!"), header=header, encoding='utf-8')
         _replace_scm_data_in_conanfile(conanfile, self.scm_data)
-        self._check_result(conanfile)
\ No newline at end of file
+        self._check_result(conanfile)
+
+    def test_multiline_statement(self):
+        """ Statement with several lines below the scm attribute """
+        statement = "\n    long_list = 'a', 'b', 'c' \\\n        'd', 'e'"
+        conanfile = self._get_conanfile(footer=statement)
+        _replace_scm_data_in_conanfile(conanfile, self.scm_data)
+        self._check_result(conanfile)
+
+    # Add comments below the SCM
+    def test_comment_file_level(self):
+        comment = "# This is a comment, file level"
+        conanfile = self._get_conanfile(footer=comment)
+        self.assertIn(comment, load(conanfile))
+        _replace_scm_data_in_conanfile(conanfile, self.scm_data)
+        self._check_result(conanfile)
+        self.assertIn(comment, load(conanfile))
+
+    def test_comment_class_level(self):
+        comment = "    # This is a comment, file level"
+        conanfile = self._get_conanfile(footer=comment)
+        self.assertIn(comment, load(conanfile))
+        _replace_scm_data_in_conanfile(conanfile, self.scm_data)
+        self._check_result(conanfile)
+        self.assertIn(comment, load(conanfile))
+
+    def test_two_comments(self):
+        comment = "    # line1\n    # line2"
+        conanfile = self._get_conanfile(footer=comment)
+        self.assertIn(comment, load(conanfile))
+        _replace_scm_data_in_conanfile(conanfile, self.scm_data)
+        self._check_result(conanfile)
+        self.assertIn(comment, load(conanfile))
+
+    def test_multiline_comment(self):
+        comment = '    """\n    line1\n    line2\n    """'
+        conanfile = self._get_conanfile(footer=comment)
+        self.assertIn(comment, load(conanfile))
+        _replace_scm_data_in_conanfile(conanfile, self.scm_data)
+        self._check_result(conanfile)
+        # FIXME: We lost the multiline comment
+        # self.assertIn(comment, load(conanfile))
+
+    # Something below the comment
+    def test_comment_and_attribute(self):
+        comment = '    # line1\n    url=23'
+        conanfile = self._get_conanfile(footer=comment)
+        self.assertIn(comment, load(conanfile))
+        _replace_scm_data_in_conanfile(conanfile, self.scm_data)
+        self._check_result(conanfile)
+        self.assertIn(comment, load(conanfile))
+
+    def test_multiline_comment_and_attribute(self):
+        comment = '    """\n    line1\n    line2\n    """\n    url=23'
+        conanfile = self._get_conanfile(footer=comment)
+        self.assertIn(comment, load(conanfile))
+        _replace_scm_data_in_conanfile(conanfile, self.scm_data)
+        self._check_result(conanfile)
+        # FIXME: We lost the multiline comment
+        self.assertIn("    url=23", load(conanfile))
+        # self.assertIn(comment, load(conanfile))
+