Repository: Python-Markdown/markdown (python)

Issue:
"Unterminated character set" exception when using extra extension
Hi,

When using the "extra" extensions, some invalid markdowns (i think?) are causing exceptions rather than returning plaintext line.

Example:

```
txt = """
*[^1^]: This is going to crash if extra extension is enabled
"""

import markdown
ok = markdown.markdown(txt, extensions=[])
not_ok = markdown.markdown(txt, extensions=['extra'])
```

Is this expected behavior?

Full exception stack trace:
```
Cell In[29], line 7
      5 import markdown
      6 ok = markdown.markdown(txt, extensions=[])
----> 7 not_ok = markdown.markdown(txt, extensions=['extra'])

File lib/python3.10/site-packages/markdown/core.py:482, in markdown(text, **kwargs)
    464 """
    465 Convert a markdown string to HTML and return HTML as a Unicode string.
    466 
   (...)
    479 
    480 """
    481 md = Markdown(**kwargs)
--> 482 return md.convert(text)

File lib/python3.10/site-packages/markdown/core.py:357, in Markdown.convert(self, source)
    354     self.lines = prep.run(self.lines)
    356 # Parse the high-level elements.
--> 357 root = self.parser.parseDocument(self.lines).getroot()
    359 # Run the tree-processors
    360 for treeprocessor in self.treeprocessors:

File lib/python3.10/site-packages/markdown/blockparser.py:117, in BlockParser.parseDocument(self, lines)
    115 # Create an `ElementTree` from the lines
    116 self.root = etree.Element(self.md.doc_tag)
--> 117 self.parseChunk(self.root, '\n'.join(lines))
    118 return etree.ElementTree(self.root)

File lib/python3.10/site-packages/markdown/blockparser.py:136, in BlockParser.parseChunk(self, parent, text)
    120 def parseChunk(self, parent: etree.Element, text: str) -> None:
    121     """ Parse a chunk of Markdown text and attach to given `etree` node.
    122 
    123     While the `text` argument is generally assumed to contain multiple
   (...)
    134 
    135     """
--> 136     self.parseBlocks(parent, text.split('\n\n'))

File lib/python3.10/site-packages/markdown/blockparser.py:158, in BlockParser.parseBlocks(self, parent, blocks)
    156 for processor in self.blockprocessors:
    157     if processor.test(parent, blocks[0]):
--> 158         if processor.run(parent, blocks) is not False:
    159             # run returns True or None
    160             break

File lib/python3.10/site-packages/markdown/extensions/abbr.py:61, in AbbrPreprocessor.run(self, parent, blocks)
     58 abbr = m.group('abbr').strip()
     59 title = m.group('title').strip()
     60 self.parser.md.inlinePatterns.register(
---> 61     AbbrInlineProcessor(self._generate_pattern(abbr), title), 'abbr-%s' % abbr, 2
     62 )
     63 if block[m.end():].strip():
     64     # Add any content after match back to blocks as separate block
     65     blocks.insert(0, block[m.end():].lstrip('\n'))

File lib/python3.10/site-packages/markdown/extensions/abbr.py:94, in AbbrInlineProcessor.__init__(self, pattern, title)
     93 def __init__(self, pattern: str, title: str):
---> 94     super().__init__(pattern)
     95     self.title = title

File lib/python3.10/site-packages/markdown/inlinepatterns.py:297, in InlineProcessor.__init__(self, pattern, md)
    287 """
    288 Create an instant of an inline processor.
    289 
   (...)
    294 
    295 """
    296 self.pattern = pattern
--> 297 self.compiled_re = re.compile(pattern, re.DOTALL | re.UNICODE)
    299 # API for Markdown to pass `safe_mode` into instance
    300 self.safe_mode = False

File lib/python3.10/re.py:251, in compile(pattern, flags)
    249 def compile(pattern, flags=0):
    250     "Compile a regular expression pattern, returning a Pattern object."
--> 251     return _compile(pattern, flags)

File lib/python3.10/re.py:303, in _compile(pattern, flags)
    301 if not sre_compile.isstring(pattern):
    302     raise TypeError("first argument must be string or compiled pattern")
--> 303 p = sre_compile.compile(pattern, flags)
    304 if not (flags & DEBUG):
    305     if len(_cache) >= _MAXCACHE:
    306         # Drop the oldest item

File lib/python3.10/sre_compile.py:788, in compile(p, flags)
    786 if isstring(p):
    787     pattern = p
--> 788     p = sre_parse.parse(p, flags)
    789 else:
    790     pattern = None

File lib/python3.10/sre_parse.py:955, in parse(str, flags, state)
    952 state.str = str
    954 try:
--> 955     p = _parse_sub(source, state, flags & SRE_FLAG_VERBOSE, 0)
    956 except Verbose:
    957     # the VERBOSE flag was switched on inside the pattern.  to be
    958     # on the safe side, we'll parse the whole thing again...
    959     state = State()

File lib/python3.10/sre_parse.py:444, in _parse_sub(source, state, verbose, nested)
    442 start = source.tell()
    443 while True:
--> 444     itemsappend(_parse(source, state, verbose, nested + 1,
    445                        not nested and not items))
    446     if not sourcematch("|"):
    447         break

File lib/python3.10/sre_parse.py:841, in _parse(source, state, verbose, nested, first)
    838         raise source.error(err.msg, len(name) + 1) from None
    839 sub_verbose = ((verbose or (add_flags & SRE_FLAG_VERBOSE)) and
    840                not (del_flags & SRE_FLAG_VERBOSE))
--> 841 p = _parse_sub(source, state, sub_verbose, nested + 1)
    842 if not source.match(")"):
    843     raise source.error("missing ), unterminated subpattern",
    844                        source.tell() - start)

File lib/python3.10/sre_parse.py:444, in _parse_sub(source, state, verbose, nested)
    442 start = source.tell()
    443 while True:
--> 444     itemsappend(_parse(source, state, verbose, nested + 1,
    445                        not nested and not items))
    446     if not sourcematch("|"):
    447         break

File lib/python3.10/sre_parse.py:550, in _parse(source, state, verbose, nested, first)
    548 this = sourceget()
    549 if this is None:
--> 550     raise source.error("unterminated character set",
    551                        source.tell() - here)
    552 if this == "]" and set:
    553     break

error: unterminated character set at position 17
```

Pull request:
Explicitly omit carot and backslash from abbr
Fixes #1444.

Patch:
diff --git a/docs/changelog.md b/docs/changelog.md
index 33f05ce..8deaefd 100644
--- a/docs/changelog.md
+++ b/docs/changelog.md
@@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * Include `scripts/*.py` in the generated source tarballs (#1430).
 * Ensure lines after heading in loose list are properly detabbed (#1443).
 * Give smarty tree processor higher priority than toc (#1440).
+* Explicitly omit carrot (`^`) and backslash (`\`) from abbreviations (#1444).
 
 ## [3.5.2] -- 2024-01-10
 
diff --git a/docs/extensions/abbreviations.md b/docs/extensions/abbreviations.md
index d03651f..9a98a91 100644
--- a/docs/extensions/abbreviations.md
+++ b/docs/extensions/abbreviations.md
@@ -36,6 +36,14 @@ will be rendered as:
 is maintained by the <abbr title="World Wide Web Consortium">W3C</abbr>.</p>
 ```
 
+The following three characters are not permitted in an abbreviation. Any
+abbreviation definitions which include one will not be recognized as an
+abbreviation definition.
+
+1. carrot (`^`)
+2. backslash (`\`)
+3. left square bracket (`]`)
+
 Usage
 -----
 
diff --git a/markdown/extensions/abbr.py b/markdown/extensions/abbr.py
index 738368a..46d3f35 100644
--- a/markdown/extensions/abbr.py
+++ b/markdown/extensions/abbr.py
@@ -41,7 +41,7 @@ class AbbrExtension(Extension):
 class AbbrPreprocessor(BlockProcessor):
     """ Abbreviation Preprocessor - parse text for abbr references. """
 
-    RE = re.compile(r'^[*]\[(?P<abbr>[^\]]*)\][ ]?:[ ]*\n?[ ]*(?P<title>.*)$', re.MULTILINE)
+    RE = re.compile(r'^[*]\[(?P<abbr>[^\]\^\\]*)\][ ]?:[ ]*\n?[ ]*(?P<title>.*)$', re.MULTILINE)
 
     def test(self, parent: etree.Element, block: str) -> bool:
         return True
@@ -73,18 +73,15 @@ class AbbrPreprocessor(BlockProcessor):
 
     def _generate_pattern(self, text: str) -> str:
         """
-        Given a string, returns an regex pattern to match that string.
+        Given a string, returns a regex pattern to match that string.
 
-        'HTML' -> r'(?P<abbr>[H][T][M][L])'
+        'HTML' -> r'(?P<abbr>\b[H][T][M][L]\b)'
 
-        Note: we force each char as a literal match (in brackets) as we don't
-        know what they will be beforehand.
+        Note: we force each char as a literal match via a character set (in brackets)
+        as we don't know what they will be beforehand.
 
         """
-        chars = list(text)
-        for i in range(len(chars)):
-            chars[i] = r'[%s]' % chars[i]
-        return r'(?P<abbr>\b%s\b)' % (r''.join(chars))
+        return f"(?P<abbr>\\b{ ''.join(f'[{ c }]' for c in text) }\\b)"
 
 
 class AbbrInlineProcessor(InlineProcessor):


Test patch:
diff --git a/tests/test_extensions.py b/tests/test_extensions.py
index a9e789f..c96772f 100644
--- a/tests/test_extensions.py
+++ b/tests/test_extensions.py
@@ -85,35 +85,6 @@ class TestExtensionClass(unittest.TestCase):
         self.assertEqual(ext.getConfigs(), {'foo': 'baz', 'bar': 'blah'})
 
 
-class TestAbbr(unittest.TestCase):
-    """ Test abbr extension. """
-
-    def setUp(self):
-        self.md = markdown.Markdown(extensions=['abbr'])
-
-    def testSimpleAbbr(self):
-        """ Test Abbreviations. """
-        text = 'Some text with an ABBR and a REF. Ignore REFERENCE and ref.' + \
-               '\n\n*[ABBR]: Abbreviation\n' + \
-               '*[REF]: Abbreviation Reference'
-        self.assertEqual(
-            self.md.convert(text),
-            '<p>Some text with an <abbr title="Abbreviation">ABBR</abbr> '
-            'and a <abbr title="Abbreviation Reference">REF</abbr>. Ignore '
-            'REFERENCE and ref.</p>'
-        )
-
-    def testNestedAbbr(self):
-        """ Test Nested Abbreviations. """
-        text = '[ABBR](/foo) and _ABBR_\n\n' + \
-               '*[ABBR]: Abbreviation'
-        self.assertEqual(
-            self.md.convert(text),
-            '<p><a href="/foo"><abbr title="Abbreviation">ABBR</abbr></a> '
-            'and <em><abbr title="Abbreviation">ABBR</abbr></em></p>'
-        )
-
-
 class TestMetaData(unittest.TestCase):
     """ Test `MetaData` extension. """
 
diff --git a/tests/test_syntax/extensions/test_abbr.py b/tests/test_syntax/extensions/test_abbr.py
index fbb25ff..708af51 100644
--- a/tests/test_syntax/extensions/test_abbr.py
+++ b/tests/test_syntax/extensions/test_abbr.py
@@ -95,6 +95,25 @@ class TestAbbr(TestCase):
             )
         )
 
+    def test_abbr_nested(self):
+        self.assertMarkdownRenders(
+            self.dedent(
+                """
+                [ABBR](/foo)
+
+                _ABBR_
+
+                *[ABBR]: Abbreviation
+                """
+            ),
+            self.dedent(
+                """
+                <p><a href="/foo"><abbr title="Abbreviation">ABBR</abbr></a></p>
+                <p><em><abbr title="Abbreviation">ABBR</abbr></em></p>
+                """
+            )
+        )
+
     def test_abbr_no_blank_Lines(self):
         self.assertMarkdownRenders(
             self.dedent(
@@ -240,3 +259,45 @@ class TestAbbr(TestCase):
                 """
             )
         )
+
+    def test_abbr_ignore_special_chars(self):
+        self.assertMarkdownRenders(
+            self.dedent(
+                r"""
+                [^] [\\] [\]] []]
+
+                *[^]: Not an abbreviation
+
+                *[\\]: Not an abbreviation
+
+                *[\]]: Not an abbreviation
+
+                *[]]: Not an abbreviation
+                """
+            ),
+            self.dedent(
+                r"""
+                <p>[^] [\] []] []]</p>
+                <p>*[^]: Not an abbreviation</p>
+                <p>*[\]: Not an abbreviation</p>
+                <p>*[]]: Not an abbreviation</p>
+                <p>*[]]: Not an abbreviation</p>
+                """
+            )
+        )
+
+    def test_abbr_hyphen(self):
+        self.assertMarkdownRenders(
+            self.dedent(
+                """
+                ABBR-abbr
+
+                *[ABBR-abbr]: Abbreviation
+                """
+            ),
+            self.dedent(
+                """
+                <p><abbr title="Abbreviation">ABBR-abbr</abbr></p>
+                """
+            )
+        )