Repository: adbar/trafilatura

Issue:
Code blocks using `pre` could be detected more effectively
A problem occurs, because some websites just use the `<pre>` tag to surround code examples. When extracting source code that is wrapped just using `<pre>`, the code is not recognized yet. For example, articles on medium.com, such as [this one](https://overcast.blog/mapping-legacy-code-techniques-for-better-architecture-understanding-4479e8e3d864), behave like this.

I suggest to use indicators, such as `("`, to detect code. If and indicator appears within the `pre` tag, the block is expected to be code.

Discussion:

Solution patch:
diff --git a/README.md b/README.md
index 28f5ab2..79cf369 100644
--- a/README.md
+++ b/README.md
@@ -38,7 +38,7 @@ The extractor strikes a balance between limiting noise (precision) and
 including all valid parts (recall). It is **robust and reasonably fast**.
 
 Trafilatura is [widely used](https://trafilatura.readthedocs.io/en/latest/used-by.html)
-and integrated into [thousands of projects](https://github.com/adbar/trafilatura/network/dependents>)
+and integrated into [thousands of projects](https://github.com/adbar/trafilatura/network/dependents)
 by companies like HuggingFace, IBM, and Microsoft Research as well as institutions like
 the Allen Institute, Stanford, the Tokyo Institute of Technology, and
 the University of Munich.
diff --git a/trafilatura/htmlprocessing.py b/trafilatura/htmlprocessing.py
index 0b8a0b0..f78ffbf 100644
--- a/trafilatura/htmlprocessing.py
+++ b/trafilatura/htmlprocessing.py
@@ -44,6 +44,8 @@ HTML_TAG_MAPPING = {v: k for k, v in REND_TAG_MAPPING.items()}
 
 PRESERVE_IMG_CLEANING = {"figure", "picture", "source"}
 
+CODE_INDICATORS = ["{", "(\"", "('", "\n    "]
+
 
 def tree_cleaning(tree: HtmlElement, options: Extractor) -> HtmlElement:
     "Prune the tree by discarding unwanted elements."
@@ -315,8 +317,18 @@ def convert_quotes(elem: _Element) -> None:
             code_flag = True
             for subelem in code_elems:
                 subelem.attrib.clear()
+        if _is_code_block(elem.text):
+            code_flag = True
     elem.tag = "code" if code_flag else "quote"
 
+def _is_code_block(text: Optional[str]) -> bool:
+    "Check if the element text is part of a code block."
+    if not text:
+        return False
+    for indicator in CODE_INDICATORS:
+        if indicator in text:
+            return True
+    return False
 
 def convert_headings(elem: _Element) -> None:
     "Add head tags and delete attributes."
diff --git a/trafilatura/xml.py b/trafilatura/xml.py
index a31e70d..a37fdcd 100644
--- a/trafilatura/xml.py
+++ b/trafilatura/xml.py
@@ -34,8 +34,8 @@ TEI_DIV_SIBLINGS = {"p", "list", "table", "quote", "ab"}
 
 CONTROL_PARSER = XMLParser(remove_blank_text=True)
 
-NEWLINE_ELEMS = {'code', 'graphic', 'head', 'lb', 'list', 'p', 'quote', 'row', 'table'}
-SPECIAL_FORMATTING = {'del', 'head', 'hi', 'ref'}
+NEWLINE_ELEMS = {'graphic', 'head', 'lb', 'list', 'p', 'quote', 'row', 'table'}
+SPECIAL_FORMATTING = {'code', 'del', 'head', 'hi', 'ref'}
 WITH_ATTRIBUTES = {'cell', 'row', 'del', 'graphic', 'head', 'hi', 'item', 'list', 'ref'}
 NESTING_WHITELIST = {"cell", "figure", "item", "note", "quote"}
 
@@ -251,8 +251,8 @@ def validate_tei(xmldoc: _Element) -> bool:
 
 
 def replace_element_text(element: _Element, include_formatting: bool) -> str:
-    "Determine element text based on just the text of the element. One must deal with the tail separately."
     elem_text = element.text or ""
+    "Determine element text based on just the text of the element. One must deal with the tail separately."
     # handle formatting: convert to markdown
     if include_formatting and element.text:
         if element.tag == "head":
@@ -268,8 +268,12 @@ def replace_element_text(element: _Element, include_formatting: bool) -> str:
             if rend in HI_FORMATTING:
                 elem_text = f"{HI_FORMATTING[rend]}{elem_text}{HI_FORMATTING[rend]}"
         elif element.tag == "code":
-            if "\n" in element.text:
-                elem_text = f"```\n{elem_text}\n```"
+            if "\n" in elem_text or element.xpath(".//lb"):  # Handle <br> inside <code>
+                # Convert <br> to \n within code blocks
+                for lb in element.xpath(".//lb"):
+                    elem_text = f"{elem_text}\n{lb.tail}"
+                    lb.getparent().remove(lb)
+                elem_text = f"```\n{elem_text}\n```\n"
             else:
                 elem_text = f"`{elem_text}`"
     # handle links


Test patch:
diff --git a/tests/unit_tests.py b/tests/unit_tests.py
index b43362c..4ee0be2 100644
--- a/tests/unit_tests.py
+++ b/tests/unit_tests.py
@@ -354,15 +354,27 @@ def test_formatting():
 Here is a code sample:
 
 `import trafilatura`"""
-    my_document = html.fromstring('<html><body><article><h3>Title</h3><p>Here is a code sample:</p><code>import trafilatura\ntrafilatura.extract("")</code></p></article></body></html>')
+    my_document = html.fromstring('<html><body><article><h3>Title</h3><p>Here is a code sample:</p><code><span>import</span> <span>something</span><br/>something.run("somewhere")</code><p>Sometimes code is wrapped using <code>pre</code> and <code>code</code>:</p><pre><code>import trafilatura\ntrafilatura.extract("")</code></pre><p>Less often code is wrapped using just <code>pre</code>:</p><pre>\n    trafilatura.extract("")</pre></article></body></html>')
     my_result = extract(my_document, output_format='txt', include_formatting=True, config=ZERO_CONFIG)
+    print(my_result)
     assert my_result == """### Title
 
 Here is a code sample:
 
+```
+import something
+something.run("somewhere")
+```
+Sometimes code is wrapped using `pre` and `code`:
+
 ```
 import trafilatura
 trafilatura.extract("")
+```
+Less often code is wrapped using just `pre`:
+
+```
+trafilatura.extract("")
 ```"""
 
     # nested