Repository: adbar/trafilatura

Issue:
"Chinese dot" breaks the fingerprint method
Hi everyone,

I noticed that if this "chinese dot" is included in the fingerprint hash "。" then the hash is always "ffffffffffffffff"

```
from trafilatura.deduplication import content_fingerprint, Simhash
content_a = Simhash("""行政長官岑浩""")


print(Simhash("""行政長官岑浩""").to_hex()) # 13dd8c82d4634a48
print(Simhash("""欢迎与我交流""").to_hex()) # 58429793861fa351
print(Simhash("""行政長官岑浩。""").to_hex()) #ffffffffffffffff
print(Simhash("""欢迎与我交流。""").to_hex()) #ffffffffffffffff
```

I hope the encoding is not all messed up.

Hints from issue comments:
@reinoldus: I did some more debugging and it seems like the sample_tokens function returns an empty list, because due to the "chinese dot" not being detected as punctuation the isalnum check fails:

```
        if token.isalnum():
            tokens.append(token)
```


We have a couple of options here:

Use the "regex" lib to remove unicode punctuation

```
    import regex  
    strip_punct = regex.compile(r'\p{P}+')
    token = strip_punct.sub('', token)
```

or this approach:

```
import unicodedata
import sys

tbl = dict.fromkeys(i for i in range(sys.maxunicode)
                      if unicodedata.category(chr(i)).startswith('P'))
def remove_punctuation(text):
    return text.translate(tbl)
```


Or for this particular case we could just append the whole text if no tokens have been found (probably the worst option)

```
    if not tokens:
        tokens.append(inputstring)
```


Let me know if any of those options would work for you and I will open a PR.@adbar: Hi @reinoldus, thanks for the suggestions, it actually amounts to a tokenization issue.

I believe the most efficient approach is to add the character to `string.punctuation` in a global variable and then to replace this line in the deduplication module (just above):
`token = token.strip(string.punctuation)`

Feel free to draft a pull request on it, I could review and integrate it.

Solution patch:
diff --git a/README.md b/README.md
index 28f5ab2..79cf369 100644
--- a/README.md
+++ b/README.md
@@ -38,7 +38,7 @@ The extractor strikes a balance between limiting noise (precision) and
 including all valid parts (recall). It is **robust and reasonably fast**.
 
 Trafilatura is [widely used](https://trafilatura.readthedocs.io/en/latest/used-by.html)
-and integrated into [thousands of projects](https://github.com/adbar/trafilatura/network/dependents>)
+and integrated into [thousands of projects](https://github.com/adbar/trafilatura/network/dependents)
 by companies like HuggingFace, IBM, and Microsoft Research as well as institutions like
 the Allen Institute, Stanford, the Tokyo Institute of Technology, and
 the University of Munich.
diff --git a/trafilatura/deduplication.py b/trafilatura/deduplication.py
index f73a852..29831c9 100644
--- a/trafilatura/deduplication.py
+++ b/trafilatura/deduplication.py
@@ -5,7 +5,6 @@
 
 import re
 import string
-
 from difflib import SequenceMatcher
 from functools import lru_cache
 from hashlib import blake2b
@@ -13,16 +12,18 @@ from operator import add
 from threading import RLock
 from typing import Any, Dict, List, Optional, Union
 
+import unicodedata
 from lxml.etree import _Element
 
 from .settings import LRU_SIZE
 from .utils import trim
 
-
 STRIP_EXTENSION = re.compile(r"\.[^/?#]{2,63}$")
 
 BIN_COUNT_FUNC = getattr(int, "bit_count", lambda x: bin(x).count("1"))
 
+PUNCT_TBL = str.maketrans({i: ' ' for i in range(0x10FFFF) if unicodedata.category(chr(i)).startswith('P')})
+
 
 @lru_cache(maxsize=1024)
 def is_similar_domain(reference: str, new_string: str, threshold: float = 0.5) -> bool:
@@ -32,6 +33,28 @@ def is_similar_domain(reference: str, new_string: str, threshold: float = 0.5) -
     return SequenceMatcher(None, reference, new_string).ratio() >= threshold
 
 
+def _get_sample_by_length(tokens: List[str], target_length: int) -> List[str]:
+    """Helper function to get a sample of tokens based on length criteria."""
+    sample = []
+    for i in range(4, -1, -1):
+        sample = [t for t in tokens if len(t) > i]
+        if len(sample) >= target_length / 2:
+            return sample
+    return sample
+
+
+def sample_tokens_fallback(inputstring: str, length: int = 64) -> List[str]:
+    """
+    This fallback implementation is used when the primary sample_tokens function
+    generates an empty token list. This is mostly relevant for languages like
+    mandarin where none latin-based punctuation is used e.g.: 。
+    """
+    # Replace all punctuation with spaces using translation table
+    clean_text = inputstring.translate(PUNCT_TBL)
+    tokens = [t for t in clean_text.split() if t.isalnum()]
+    return _get_sample_by_length(tokens, length)
+
+
 def sample_tokens(inputstring: str, length: int = 64) -> List[str]:
     """Split input into list of tokens and adjust length threshold to make sure
     there is enough data."""
@@ -40,11 +63,12 @@ def sample_tokens(inputstring: str, length: int = 64) -> List[str]:
         token = token.strip(string.punctuation)
         if token.isalnum():
             tokens.append(token)
-    sample = []
-    for i in range(4, -1, -1):
-        sample = [t for t in tokens if len(t) > i]
-        if len(sample) >= length / 2:
-            return sample
+
+    sample = _get_sample_by_length(tokens, length)
+
+    if len(sample) == 0:
+        return sample_tokens_fallback(inputstring, length)
+
     return sample
 
 
diff --git a/trafilatura/htmlprocessing.py b/trafilatura/htmlprocessing.py
index 0b8a0b0..f78ffbf 100644
--- a/trafilatura/htmlprocessing.py
+++ b/trafilatura/htmlprocessing.py
@@ -44,6 +44,8 @@ HTML_TAG_MAPPING = {v: k for k, v in REND_TAG_MAPPING.items()}
 
 PRESERVE_IMG_CLEANING = {"figure", "picture", "source"}
 
+CODE_INDICATORS = ["{", "(\"", "('", "\n    "]
+
 
 def tree_cleaning(tree: HtmlElement, options: Extractor) -> HtmlElement:
     "Prune the tree by discarding unwanted elements."
@@ -315,8 +317,18 @@ def convert_quotes(elem: _Element) -> None:
             code_flag = True
             for subelem in code_elems:
                 subelem.attrib.clear()
+        if _is_code_block(elem.text):
+            code_flag = True
     elem.tag = "code" if code_flag else "quote"
 
+def _is_code_block(text: Optional[str]) -> bool:
+    "Check if the element text is part of a code block."
+    if not text:
+        return False
+    for indicator in CODE_INDICATORS:
+        if indicator in text:
+            return True
+    return False
 
 def convert_headings(elem: _Element) -> None:
     "Add head tags and delete attributes."
diff --git a/trafilatura/xml.py b/trafilatura/xml.py
index a31e70d..a37fdcd 100644
--- a/trafilatura/xml.py
+++ b/trafilatura/xml.py
@@ -34,8 +34,8 @@ TEI_DIV_SIBLINGS = {"p", "list", "table", "quote", "ab"}
 
 CONTROL_PARSER = XMLParser(remove_blank_text=True)
 
-NEWLINE_ELEMS = {'code', 'graphic', 'head', 'lb', 'list', 'p', 'quote', 'row', 'table'}
-SPECIAL_FORMATTING = {'del', 'head', 'hi', 'ref'}
+NEWLINE_ELEMS = {'graphic', 'head', 'lb', 'list', 'p', 'quote', 'row', 'table'}
+SPECIAL_FORMATTING = {'code', 'del', 'head', 'hi', 'ref'}
 WITH_ATTRIBUTES = {'cell', 'row', 'del', 'graphic', 'head', 'hi', 'item', 'list', 'ref'}
 NESTING_WHITELIST = {"cell", "figure", "item", "note", "quote"}
 
@@ -251,8 +251,8 @@ def validate_tei(xmldoc: _Element) -> bool:
 
 
 def replace_element_text(element: _Element, include_formatting: bool) -> str:
-    "Determine element text based on just the text of the element. One must deal with the tail separately."
     elem_text = element.text or ""
+    "Determine element text based on just the text of the element. One must deal with the tail separately."
     # handle formatting: convert to markdown
     if include_formatting and element.text:
         if element.tag == "head":
@@ -268,8 +268,12 @@ def replace_element_text(element: _Element, include_formatting: bool) -> str:
             if rend in HI_FORMATTING:
                 elem_text = f"{HI_FORMATTING[rend]}{elem_text}{HI_FORMATTING[rend]}"
         elif element.tag == "code":
-            if "\n" in element.text:
-                elem_text = f"```\n{elem_text}\n```"
+            if "\n" in elem_text or element.xpath(".//lb"):  # Handle <br> inside <code>
+                # Convert <br> to \n within code blocks
+                for lb in element.xpath(".//lb"):
+                    elem_text = f"{elem_text}\n{lb.tail}"
+                    lb.getparent().remove(lb)
+                elem_text = f"```\n{elem_text}\n```\n"
             else:
                 elem_text = f"`{elem_text}`"
     # handle links


Test patch:
diff --git a/tests/deduplication_tests.py b/tests/deduplication_tests.py
index 4b8a00a..dd6a16d 100644
--- a/tests/deduplication_tests.py
+++ b/tests/deduplication_tests.py
@@ -24,6 +24,24 @@ def test_hashes():
     assert generate_hash_filename(content) == "42LNugG3Sc95646i"
 
 
+def test_content_fingerprint():
+    "Test content fingerprint generation for different types of text"
+    # Test regular Latin text
+    text = "Hello world! This is a test string with some numbers 123."
+    fingerprint = trafilatura.deduplication.content_fingerprint(text)
+    assert fingerprint == "5efdce9f2b554683"
+
+    # Test Chinese text
+    chinese_text = "这是一个测试。我们在测试中文。"
+    chinese_fingerprint = trafilatura.deduplication.content_fingerprint(chinese_text)
+    print(chinese_fingerprint)
+    assert chinese_fingerprint == "ff377edee6edfb78"
+
+    # Test mixed text
+    mixed_text = "Hello世界。This is混合文本!"
+    mixed_fingerprint = trafilatura.deduplication.content_fingerprint(mixed_text)
+    assert mixed_fingerprint == "24979dc6c8a26a5"
+
 
 def test_simhash():
     "Test similarity calculation based on Simhash class."
@@ -122,8 +140,47 @@ def test_dedup():
     assert trafilatura.htmlprocessing.process_node(my_p, options) is None
 
 
+def test_sample_tokens(monkeypatch):
+    "Test token sampling functions including fallback for non-latin text"
+
+    call_counter = {'fallback': 0, 'main': 0}
+    original_fallback = trafilatura.deduplication.sample_tokens_fallback
+
+    def spy_fallback(*args, **kwargs):
+        call_counter['fallback'] += 1
+        return original_fallback(*args, **kwargs)
+
+    monkeypatch.setattr(trafilatura.deduplication, 'sample_tokens_fallback', spy_fallback)
+
+    # Test regular text
+    text = "Hello world! This is a test string with some numbers 123."
+    tokens = trafilatura.deduplication.sample_tokens(text)
+    assert len(tokens) > 0
+    assert "Hello" in tokens
+    assert "world" in tokens
+    assert "123" in tokens
+    assert call_counter['fallback'] == 0, "Fallback shouldn't be called for Latin character text"
+
+    # Test Chinese text with Chinese punctuation
+    chinese_text = "这是一个测试。我们在测试中文。"
+    tokens = trafilatura.deduplication.sample_tokens(chinese_text)
+    assert len(tokens) == 2
+    assert "这是一个测试" in tokens
+    assert "我们在测试中文" in tokens
+    assert call_counter['fallback'] == 1, "Fallback should be called for Chinese text"
+
+    # Test mixed text using the default sample tokens method
+    mixed_text = "Hello世界。This is混合文本!"
+    tokens = trafilatura.deduplication.sample_tokens(mixed_text)
+    assert len(tokens) == 1
+    assert 'is混合文本' in tokens
+    assert call_counter['fallback'] == 1, "Fallback shouldn't be called due to blank"
+
+
 if __name__ == "__main__":
     test_hashes()
     test_simhash()
     test_lrucache()
     test_dedup()
+    test_sample_tokens()
+    test_content_fingerprint()
diff --git a/tests/unit_tests.py b/tests/unit_tests.py
index b43362c..4ee0be2 100644
--- a/tests/unit_tests.py
+++ b/tests/unit_tests.py
@@ -354,15 +354,27 @@ def test_formatting():
 Here is a code sample:
 
 `import trafilatura`"""
-    my_document = html.fromstring('<html><body><article><h3>Title</h3><p>Here is a code sample:</p><code>import trafilatura\ntrafilatura.extract("")</code></p></article></body></html>')
+    my_document = html.fromstring('<html><body><article><h3>Title</h3><p>Here is a code sample:</p><code><span>import</span> <span>something</span><br/>something.run("somewhere")</code><p>Sometimes code is wrapped using <code>pre</code> and <code>code</code>:</p><pre><code>import trafilatura\ntrafilatura.extract("")</code></pre><p>Less often code is wrapped using just <code>pre</code>:</p><pre>\n    trafilatura.extract("")</pre></article></body></html>')
     my_result = extract(my_document, output_format='txt', include_formatting=True, config=ZERO_CONFIG)
+    print(my_result)
     assert my_result == """### Title
 
 Here is a code sample:
 
+```
+import something
+something.run("somewhere")
+```
+Sometimes code is wrapped using `pre` and `code`:
+
 ```
 import trafilatura
 trafilatura.extract("")
+```
+Less often code is wrapped using just `pre`:
+
+```
+trafilatura.extract("")
 ```"""
 
     # nested