Repository: datalad/datalad

Issue:
dochelpers `[PY: PY]` macro handling cannot deal with `[]` in the conditional docs
The datalad-next `credentials` command shows it in the CLI for its `spec` parameter. It fails to find the macro and leaves it in

Comments:

Patch:
diff --git a/.appveyor.yml b/.appveyor.yml
index 86566adfc..e6e00dcab 100644
--- a/.appveyor.yml
+++ b/.appveyor.yml
@@ -95,7 +95,7 @@ environment:
     - ID: MacP38core
       # ~40min
       DTS: datalad.core datalad.dataset datalad.runner datalad.support
-      APPVEYOR_BUILD_WORKER_IMAGE: macOS
+      APPVEYOR_BUILD_WORKER_IMAGE: macos-monterey
       PY: 3.8
       # does not give a functional installation
       # INSTALL_GITANNEX: git-annex -m snapshot
@@ -154,7 +154,7 @@ environment:
           datalad.interface
           datalad.tests
           datalad.ui
-      APPVEYOR_BUILD_WORKER_IMAGE: macOS
+      APPVEYOR_BUILD_WORKER_IMAGE: macos-monterey
       PY: 3.8
       INSTALL_GITANNEX: git-annex
       DATALAD_LOCATIONS_SOCKETS: /Users/appveyor/DLTMP/sockets
@@ -164,7 +164,7 @@ environment:
       DTS: >
           datalad.local
           datalad.distributed
-      APPVEYOR_BUILD_WORKER_IMAGE: macOS
+      APPVEYOR_BUILD_WORKER_IMAGE: macos-monterey
       PY: 3.8
       INSTALL_GITANNEX: git-annex
       DATALAD_LOCATIONS_SOCKETS: /Users/appveyor/DLTMP/sockets
diff --git a/changelog.d/pr-7226.md b/changelog.d/pr-7226.md
new file mode 100644
index 000000000..03f0d2171
--- /dev/null
+++ b/changelog.d/pr-7226.md
@@ -0,0 +1,6 @@
+### 🐛 Bug Fixes
+
+- Interface-specific (python vs CLI) doc generation for commands and their parameters was broken when brackets were used within the interface markups. 
+  Fixes [#7225](https://github.com/datalad/datalad/issues/7225) via
+  [PR #7226](https://github.com/datalad/datalad/pull/7226)
+  (by [@bpoldrack](https://github.com/bpoldrack))
diff --git a/datalad/cli/interface.py b/datalad/cli/interface.py
index 6a5d60212..bee96588a 100644
--- a/datalad/cli/interface.py
+++ b/datalad/cli/interface.py
@@ -60,15 +60,15 @@ def alter_interface_docs_for_cmdline(docs):
         flags=re.MULTILINE | re.DOTALL)
     # clean cmdline in-line bits
     docs = re.sub(
-        r'\[PY:\s[^\[\]]*\sPY\]',
+        r'\[PY:\s.*?\sPY\]',
         '',
         docs,
         flags=re.MULTILINE | re.DOTALL)
     docs = re.sub(
-        r'\[CMD:\s([^\[\]]*)\sCMD\]',
+        r'\[CMD:\s(.*?)\sCMD\]',
         lambda match: match.group(1),
         docs,
-        flags=re.MULTILINE)
+        flags=re.MULTILINE | re.DOTALL)
     docs = re.sub(
         r'\|\| CMDLINE \>\>(.*?)\<\< CMDLINE \|\|',
         lambda match: match.group(1),
diff --git a/datalad/interface/base.py b/datalad/interface/base.py
index 6dc512586..4bdbdd17e 100644
--- a/datalad/interface/base.py
+++ b/datalad/interface/base.py
@@ -184,15 +184,15 @@ def alter_interface_docs_for_api(docs):
         flags=re.MULTILINE | re.DOTALL)
     # clean cmdline in-line bits
     docs = re.sub(
-        r'\[CMD:\s[^\[\]]*\sCMD\]',
+        r'\[CMD:\s.*?\sCMD\]',
         '',
         docs,
         flags=re.MULTILINE | re.DOTALL)
     docs = re.sub(
-        r'\[PY:\s([^\[\]]*)\sPY\]',
+        r'\[PY:\s(.*?)\sPY\]',
         lambda match: match.group(1),
         docs,
-        flags=re.MULTILINE)
+        flags=re.MULTILINE | re.DOTALL)
     # select only the python alternative from argument specifications
     docs = re.sub(
         r'``([a-zA-Z0-9_,.]+)\|\|([a-zA-Z0-9-,.]+)``',


Test patch:
diff --git a/datalad/cli/tests/test_interface.py b/datalad/cli/tests/test_interface.py
index 7090988f6..9a00cbcb2 100644
--- a/datalad/cli/tests/test_interface.py
+++ b/datalad/cli/tests/test_interface.py
@@ -28,6 +28,12 @@ def test_alter_interface_docs_for_cmdline():
     assert_in('a b', alt)
     assert_in('not\n   reflowed', alt)
     assert_in("Something for the cmdline only Multiline!", alt)
+    assert_not_in("Some Python-only bits", alt)
+    assert_not_in("just for Python", alt)
+    assert_in("just for the command line", alt)
+    assert_in("multiline cli-only with [ brackets\n[] ]", alt)
+    assert_not_in("multiline\npython-only with [ brackets [] ]", alt)
+
     # args
     altarg = alter_interface_docs_for_cmdline(demo_argdoc)
     # RST role markup
@@ -42,10 +48,15 @@ def test_alter_interface_docs_for_cmdline():
         'one bla bla two bla')
 
     altpd = alter_interface_docs_for_cmdline(demo_paramdoc)
+    assert_not_in("PY", altpd)
+    assert_not_in("CMD", altpd)
     assert_not_in('python', altpd)
+    assert_not_in("python-only with [ some brackets []", altpd)
     assert_in('in between', altpd)
     assert_in('appended', altpd)
     assert_in('cmdline', altpd)
+    assert_in("multiline cli-only [\n  brackets included "
+              "[ can we also have || ?]", altpd)
 
 
 def test_name_generation():
diff --git a/datalad/interface/tests/test_docs.py b/datalad/interface/tests/test_docs.py
index fdf0e872d..7dda2fab5 100644
--- a/datalad/interface/tests/test_docs.py
+++ b/datalad/interface/tests/test_docs.py
@@ -57,7 +57,9 @@ demo_doc = """\
     << PYTHON ||
 
     And an example for in-line markup: [PY: just for Python PY] and
-    the other one [CMD: just for the command line CMD]. End of demo.
+    the other one [CMD: just for the command line CMD]. [PY: multiline
+    python-only with [ brackets [] ] PY][CMD: multiline cli-only with [ brackets
+    [] ] CMD]. End of demo.
 
     Generic appendix. Ding dong ding dong ding dong.  Ding dong ding dong ding
     dong.  Ding dong ding dong ding dong.  Ding dong ding dong ding dong.  Ding
@@ -70,7 +72,10 @@ demo_paramdoc = """\
     Parameters
     ----------
     dataset : Dataset or None, optional
-      something [PY: python only PY] in between [CMD: cmdline only CMD] appended [PY: more python PY]
+      something [PY: python only PY] in between [CMD: cmdline only CMD] appended
+      Brackets can also be within and we can deal with [PY: multiline
+      python-only with [ some brackets [] PY] [CMD: multiline cli-only [
+      brackets included [ can we also have || ?] CMD].
       dataset is given, an attempt is made to identify the dataset based
       Dataset (e.g. a path), or value must be `None`. [Default: None]
 """
@@ -102,9 +107,19 @@ def test_alter_interface_docs_for_api():
     assert_in('a b', alt)
     assert_in('not\n   reflowed', alt)
     assert_in("Some Python-only bits Multiline!", alt)
+    assert_in("Some Python-only bits", alt)
+    assert_in("just for Python", alt)
+    assert_not_in("just for the command line", alt)
+    assert_not_in("multiline cli-only with [ brackets\n[] ]", alt)
+    assert_in("multiline\npython-only with [ brackets [] ]", alt)
 
     altpd = alter_interface_docs_for_api(demo_paramdoc)
+    assert_not_in("PY", altpd)
+    assert_not_in("CMD", altpd)
     assert_in('python', altpd)
     assert_in('in between', altpd)
     assert_in('appended', altpd)
+    assert_in("multiline\n  python-only with [ some brackets []", altpd)
     assert_not_in('cmdline', altpd)
+    assert_not_in("multiline cli-only [\n  brackets included "
+                  "[ can we also have || ?]", altpd)