Repository: freelawproject/eyecite

Issue:
Try to include style HTML tags in citation span
See parent issue for description

Discussion:

Solution patch:
diff --git a/eyecite/annotate.py b/eyecite/annotate.py
index 7f5ba46..d3558b6 100644
--- a/eyecite/annotate.py
+++ b/eyecite/annotate.py
@@ -1,11 +1,18 @@
 from bisect import bisect_left, bisect_right
 from difflib import SequenceMatcher
 from functools import partial
+from logging import getLogger
 from typing import Any, Callable, Iterable, Optional, Tuple
 
 import fast_diff_match_patch
 
-from eyecite.utils import is_balanced_html, wrap_html_tags
+from eyecite.utils import (
+    is_balanced_html,
+    maybe_balance_style_tags,
+    wrap_html_tags,
+)
+
+logger = getLogger("eyecite")
 
 
 def annotate_citations(
@@ -59,6 +66,9 @@ def annotate_citations(
     Returns:
         The annotated text.
     """
+    if unbalanced_tags not in ["unchecked", "skip", "wrap"]:
+        raise ValueError(f"Unknown option '{unbalanced_tags}")
+
     # set up offset_updater if we have to move annotations to source_text
     offset_updater = None
     if source_text and source_text != plain_text:
@@ -88,13 +98,20 @@ def annotate_citations(
         # handle HTML tags
         if unbalanced_tags == "unchecked":
             pass
-        elif unbalanced_tags in ("skip", "wrap"):
-            if not is_balanced_html(span_text):
-                if unbalanced_tags == "skip":
-                    continue
+        elif not is_balanced_html(span_text):
+            if unbalanced_tags == "wrap":
                 span_text = wrap_html_tags(span_text, after, before)
-        else:
-            raise ValueError(f"Unknown option '{unbalanced_tags}")
+            else:  # "skip" case
+                original_span_text = span_text
+                start, end, span_text = maybe_balance_style_tags(
+                    start, end, plain_text
+                )
+                if not is_balanced_html(span_text):
+                    logger.error(
+                        "Citation was not annotated due to unbalanced tags %s",
+                        original_span_text,
+                    )
+                    continue
 
         if annotator is not None:
             annotated_span = annotator(before, span_text, after)
diff --git a/eyecite/utils.py b/eyecite/utils.py
index b2bb66a..c606a32 100644
--- a/eyecite/utils.py
+++ b/eyecite/utils.py
@@ -130,3 +130,57 @@ def hash_sha256(dictionary: dict) -> int:
 
     # Calculate the hash of the bytes, convert to an int, and return
     return int.from_bytes(hashlib.sha256(json_bytes).digest(), byteorder="big")
+
+
+def maybe_balance_style_tags(
+    start: int, end: int, plain_text: str
+) -> tuple[int, int, str]:
+    """Try to include style tags at the edge of the span marked as invalid
+
+    In some HTML sources the citations are styled with tags like <i> or <em>
+    When the citation is found in a stripped-of-tags text, the span may
+    leave out the opening or closing tag. When this happens and we try to
+    annotate the HTML, it will render invalid HTML. This happens mostly with
+    IdCitation, ReferenceCitation, etc.
+
+    This function will try to find opening or closing tags inmediately
+    preceding or following the citation span. If it finds them, it will
+    return the new start, end and span. If not, it will return the old ones
+
+    :param start: the original start of the span
+    :param end: the origina end of the span
+    :param plain_text: the text to annotate
+    :return: a tuple (new start, new end, new span text)
+    """
+    span_text = plain_text[start:end]
+    style_tags = ["i", "em", "b"]
+    tolerance = 5  # tolerate at most this amount of whitespace
+
+    for tag in style_tags:
+        opening_tag = f"<{tag}>"
+        closing_tag = f"</{tag}>"
+        has_opening = opening_tag in span_text
+        has_closing = closing_tag in span_text
+        if has_opening and not has_closing:
+            # look for closing tag after the end
+            extended_end = max(
+                end + len(closing_tag) + tolerance, len(plain_text)
+            )
+            if end_match := re.search(
+                rf"{span_text}\s*{closing_tag}",
+                plain_text[start:extended_end],
+                flags=re.MULTILINE,
+            ):
+                end = start + end_match.end()
+
+        if not has_opening and has_closing:
+            # look for opening tag before the start
+            extended_start = min(start - len(opening_tag) - tolerance, 0)
+            if start_match := re.search(
+                rf"{opening_tag}\s*{span_text}",
+                plain_text[extended_start:end],
+                flags=re.MULTILINE,
+            ):
+                start = extended_start + start_match.start()
+
+    return start, end, plain_text[start:end]


Test patch:
diff --git a/tests/test_AnnotateTest.py b/tests/test_AnnotateTest.py
index e61c7af..7572b82 100644
--- a/tests/test_AnnotateTest.py
+++ b/tests/test_AnnotateTest.py
@@ -12,6 +12,7 @@ class AnnotateTest(TestCase):
         def lower_annotator(before, text, after):
             return before + text.lower() + after
 
+        self.maxDiff = None
         test_pairs = (
             # single cite
             ("1 U.S. 1", "<0>1 U.S. 1</0>", []),
@@ -59,10 +60,10 @@ class AnnotateTest(TestCase):
                 "<body>foo  <i><0>1   <b>U.S.</b></i>   1</0> bar</body>",
                 ["html", "inline_whitespace"],
             ),
-            # whitespace and html -- skip unbalanced tags
+            # whitespace and html -- unbalanced tags are repaired
             (
                 "foo  <i>1 U.S.</i> 1; 2 <i>U.S.</i> 2",
-                "foo  <i>1 U.S.</i> 1; <1>2 <i>U.S.</i> 2</1>",
+                "foo  <0><i>1 U.S.</i> 1</0>; <1>2 <i>U.S.</i> 2</1>",
                 ["html", "inline_whitespace"],
                 {"unbalanced_tags": "skip"},
             ),
@@ -101,6 +102,94 @@ class AnnotateTest(TestCase):
                 [],
                 {"annotator": lower_annotator},
             ),
+            # solvable unbalanced <em> tag. Need the FullCaseCitation first
+            # so the ReferenceCitation can be found
+            # from https://www.courtlistener.com/api/rest/v4/opinions/8496639/
+            # source: Opinion.xml_harvard
+            (
+                " partially secured by a debtor’s principal residence was not "
+                "con-firmable. <em>Nobelman v. Am. Sav. Bank, </em>"
+                "508 U.S. 324, 113 S.Ct. 2106, 124 L.Ed.2d 228 (1993). That "
+                "plan proposed to bifurcate the claim and... pay the unsecured"
+                "... only by a lien on the debtor’s principal residence.” "
+                "<em>Nobelman </em>at 332, 113 S.Ct. 2106. Section 1123(b)(5) "
+                "codifies the <em>Nobelman </em>decision in individual debtor "
+                "chapter 11 cases.",
+                " partially secured by a debtor’s principal residence was not"
+                " con-firmable. <em>Nobelman v. Am. Sav. Bank, </em>"
+                "<a href='something'>508 U.S. 324</a>, <a href='something'>"
+                "113 S.Ct. 2106</a>, <a href='something'>124 L.Ed.2d 228</a>"
+                " (1993). That plan proposed to bifurcate the claim and..."
+                " pay the unsecured... only by a lien on the debtor’s"
+                " principal residence.” <a href='something'><em>Nobelman </em>"
+                "at 332</a>, <a href='something'>113 S.Ct. 2106</a>. Section"
+                " 1123(b)(5) codifies the <em>Nobelman </em>decision in"
+                " individual debtor chapter 11 cases.",
+                ["html", "all_whitespace"],
+                {"annotate_anchors": True, "unbalanced_tags": "skip"},
+            ),
+            # solvable unbalanced <i> tag
+            # from https://www.courtlistener.com/api/rest/v4/opinions/2841253/
+            # source: Opinion.html
+            (
+                "he has not agreed so to submit.’”  <i>Howsam v. Dean"
+                " Witter Reynolds, Inc.</i>, 537 U.S. 79, 83, 123 S. Ct."
+                " 588, 591 (2002) (combined mandamus and"
+                " interlocutory appeal) (citing <i>Howsam</i> at 84, 123"
+                " S. Ct. at 592)",
+                "he has not agreed so to submit.’”  <i>Howsam v. Dean"
+                " Witter Reynolds, Inc.</i>, <a href='something'>537 U.S."
+                " 79</a>, 83, <a href='something'>123 S. Ct. 588</a>, 591"
+                " (2002) (combined mandamus and interlocutory appeal)"
+                " (citing <a href='something'><i>Howsam</i> at 84</a>, <a"
+                " href='something'>123 S. Ct. at 592</a>)",
+                ["html", "all_whitespace"],
+                {"annotate_anchors": True, "unbalanced_tags": "skip"},
+            ),
+            # The next 2 examples could be resolved if we increased the
+            # character tolerance or admitted the full case name instead of
+            # just one of the parties
+            (
+                # https://www.courtlistener.com/api/rest/v4/opinions/1535649/
+                # source: xml_harvard
+                "See also Styler v. Tall Oaks, Inc. (In re Hatch),"
+                " 93 B.R. 263, 267 (Bankr.D. Utah 1988),"
+                " <em> rev'd </em> 114 B.R. 747 (D.Utah 1989)."
+                "</p>...   The court makes no"
+                " determination as to whe Fifth Amendment to the"
+                " constitution of the United States.” <em> Styler v."
+                " Tall Oaks, Inc. (In re Hatch), </em> at 748."
+                "</p>",
+                "See also Styler v. Tall Oaks, Inc. (In re Hatch),"
+                " <a href='something'>93 B.R. 263</a>, 267"
+                " (Bankr.D. Utah 1988), <em> rev'd </em> <a"
+                " href='something'>114 B.R. 747</a> (D.Utah 1989)."
+                "</p>...   The court makes no"
+                " determination as to whe Fifth Amendment to the"
+                " constitution of the United States.” <em> Styler v."
+                " Tall Oaks, Inc. (In re Hatch), </em> at 748."
+                "</p>",
+                ["html", "all_whitespace"],
+                {"annotate_anchors": True, "unbalanced_tags": "skip"},
+            ),
+            (
+                # https://www.courtlistener.com/api/rest/v4/opinions/1985850/
+                # source: html_lawbox
+                "to act rationally. <i>See, e.g., </i><i>State v."
+                " Wingler,</i> 25 <i>N.J.</i> 161, 175, 135 <i>A.</i>2d"
+                " 468 (1957); <i>citing, ...  have been applied.'"
+                " [<i>State v. Wingler</i> at 175, 135 <i>A.</i>2d"
+                " 468, <i>citing, </i><i>Minnesota ex rel.</i>",
+                "to act rationally. <i>See, e.g., </i><i>State v."
+                " Wingler,</i> <a href='something'>25 <i>N.J.</i>"
+                " 161</a>, 175, <a href='something'>135 <i>A.</i>2d"
+                " 468</a> (1957); <i>citing, ...  have been applied.'"
+                " [<i>State v. Wingler</i> at 175, <a"
+                " href='something'>135 <i>A.</i>2d 468</a>, <i>citing,"
+                " </i><i>Minnesota ex rel.</i>",
+                ["html", "all_whitespace"],
+                {"annotate_anchors": True, "unbalanced_tags": "skip"},
+            ),
         )
         for source_text, expected, clean_steps, *annotate_kwargs in test_pairs:
             annotate_kwargs = annotate_kwargs[0] if annotate_kwargs else {}
@@ -115,6 +204,13 @@ class AnnotateTest(TestCase):
                     (c.span(), f"<{i}>", f"</{i}>")
                     for i, c in enumerate(cites)
                 ]
+
+                if annotate_kwargs.pop("annotate_anchors", False):
+                    annotations = [
+                        (c.span(), "<a href='something'>", "</a>")
+                        for c in cites
+                    ]
+
                 annotated = annotate_citations(
                     plain_text,
                     annotations,