Repository: andytzeng/pyxurls

Issue:
require ASCII TLDs to be followed by a word break again
A commit from the author has fixed the word break for false positively parsing text as url.
https://github.com/mvdan/xurls/commit/cee2e13ff09deef34fcfa2b932bc99766a0b7b24

related issue https://github.com/mvdan/xurls/issues/49



Discussion:

Patch:
diff --git a/.travis.yml b/.travis.yml
index a402fc7..9d06a47 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -9,7 +9,7 @@ before_install:
   - sudo apt-get update
   - sudo apt-get install libre2-dev pybind11-dev
 install:
-  - pip install "google-re2==0.0.5" ./
+  - pip install "google-re2" ./
 script:
   - pytest
 deploy:
diff --git a/setup.py b/setup.py
index 2cad55b..864f97c 100644
--- a/setup.py
+++ b/setup.py
@@ -27,7 +27,7 @@ with open(path.join(this_directory, 'README.md'), encoding='utf-8') as f:
 
 setuptools.setup(
     name='pyxurls',
-    version='0.1.2',
+    version='0.1.3',
     author='Andy Tzeng',
     author_email='andytzeng@aol.tw',
     description='A regular expression based URL extractor which extracts URLs from text.',
diff --git a/xurls/__init__.py b/xurls/__init__.py
index 9f84d64..230d5ec 100644
--- a/xurls/__init__.py
+++ b/xurls/__init__.py
@@ -1,3 +1,3 @@
-# Copyright 2015 Andy Tzeng <andytzeng@aol.tw>. All Rights Reserved.
+# Copyright 2021 Andy Tzeng <andytzeng@aol.tw>. All Rights Reserved.
 # Licensed under BSD 3-Clause License.
 from .xurls import Relaxed, Strict, StrictScheme  # NOQA
diff --git a/xurls/express.py b/xurls/express.py
index 61e6f44..70e29a9 100644
--- a/xurls/express.py
+++ b/xurls/express.py
@@ -1,11 +1,11 @@
-# Copyright 2015 Andy Tzeng <andytzeng@aol.tw>. All Rights Reserved.
+# Copyright 2021 Andy Tzeng <andytzeng@aol.tw>. All Rights Reserved.
 # Licensed under BSD 3-Clause License.
 import enum
 
 from . import schemes
 from ._regex import re
-from .tlds import TLDs
-from .tlds_pseudo import PseudoTLDs
+from .tlds import ASCII_TLDS, NON_ASCII_TLDS
+from .tlds_pseudo import PSEUDO_TLDS
 from .unicode import otherPuncMinusDoubleQuote
 
 
@@ -53,8 +53,14 @@ def strict_exp():
 
 def relaxed_exp():
     punycode = r'xn--[a-z0-9-]+'
-    known_tlds = any_of(*(TLDs + PseudoTLDs))
-    site = rf'{URIPattern.domain.value}(?i:({punycode}|{known_tlds}))'
+
+    # Use \b to make sure ASCII TLDs are immediately followed by a word break.
+    # We can't do that with unicode TLDs, as they don't see following
+    # whitespace as a word break.
+    ascii_tlds = any_of(*(ASCII_TLDS + PSEUDO_TLDS))
+    non_ascii_tlds = any_of(*NON_ASCII_TLDS)
+    tlds = rf'(?i:({punycode}|{ascii_tlds}\b|{non_ascii_tlds}))'
+    site = URIPattern.domain.value + tlds
     hostname = rf'({site}|{URIPattern.ip_addr.value})'
     web_url = rf'{hostname}{URIPattern.port.value}(/|/{URIPattern.path_cont.value})?'
     return rf'{strict_exp()}|{web_url}'
diff --git a/xurls/schemes.py b/xurls/schemes.py
index e1f5866..2758fcc 100644
--- a/xurls/schemes.py
+++ b/xurls/schemes.py
@@ -1,4 +1,4 @@
-# Copyright 2015 Andy Tzeng <andytzeng@aol.tw>. All Rights Reserved.
+# Copyright 2021 Andy Tzeng <andytzeng@aol.tw>. All Rights Reserved.
 # Licensed under BSD 3-Clause License.
 
 # Refer the tool unicodegen from mvdan.cc/xurls/v2
@@ -74,6 +74,7 @@ OFFICIAL = [
     'dweb',
     'ed2k',
     'elsi',
+    'embedded',
     'ens',
     'ethereum',
     'example',
@@ -218,6 +219,7 @@ OFFICIAL = [
     'msrp',
     'msrps',
     'mss',
+    'mt',
     'mtqp',
     'mumble',
     'mupdate',
@@ -265,12 +267,14 @@ OFFICIAL = [
     'rtsp',
     'rtsps',
     'rtspu',
+    'sarif',
     'secondlife',
     'secret-token',
     'service',
     'session',
     'sftp',
     'sgn',
+    'shc',
     'shttp',
     'sieve',
     'simpleledger',
@@ -324,6 +328,7 @@ OFFICIAL = [
     'vscode-insiders',
     'vsls',
     'wais',
+    'wcr',
     'webcal',
     'wifi',
     'wpid',
diff --git a/xurls/tlds.py b/xurls/tlds.py
index 93c1aa7..0bb047e 100644
--- a/xurls/tlds.py
+++ b/xurls/tlds.py
@@ -1,4 +1,4 @@
-# Copyright 2015 Andy Tzeng <andytzeng@aol.tw>. All Rights Reserved.
+# Copyright 2021 Andy Tzeng <andytzeng@aol.tw>. All Rights Reserved.
 # Licensed under BSD 3-Clause License.
 
 # Refer the tool unicodegen from mvdan.cc/xurls/v2
@@ -8,7 +8,7 @@
 # Sources:
 #  * https://data.iana.org/TLD/tlds-alpha-by-domain.txt
 #  * https://publicsuffix.org/list/effective_tld_names.dat
-TLDs = [
+ASCII_TLDS = [
     'aaa',
     'aarp',
     'abarth',
@@ -222,7 +222,6 @@ TLDs = [
     'cars',
     'casa',
     'case',
-    'caseih',
     'cash',
     'casino',
     'cat',
@@ -234,7 +233,6 @@ TLDs = [
     'cbs',
     'cc',
     'cd',
-    'ceb',
     'center',
     'ceo',
     'cern',
@@ -472,7 +470,6 @@ TLDs = [
     'frontier',
     'ftr',
     'fujitsu',
-    'fujixerox',
     'fun',
     'fund',
     'furniture',
@@ -638,11 +635,9 @@ TLDs = [
     'it',
     'itau',
     'itv',
-    'iveco',
     'jaguar',
     'java',
     'jcb',
-    'jcp',
     'je',
     'jeep',
     'jetzt',
@@ -758,7 +753,6 @@ TLDs = [
     'ltda',
     'lu',
     'lundbeck',
-    'lupin',
     'luxe',
     'luxury',
     'lv',
@@ -837,6 +831,7 @@ TLDs = [
     'mtr',
     'mu',
     'museum',
+    'music',
     'mutual',
     'mv',
     'mw',
@@ -847,7 +842,6 @@ TLDs = [
     'nab',
     'nagoya',
     'name',
-    'nationwide',
     'natura',
     'navy',
     'nba',
@@ -860,7 +854,6 @@ TLDs = [
     'network',
     'neustar',
     'new',
-    'newholland',
     'news',
     'next',
     'nextdirect',
@@ -909,7 +902,6 @@ TLDs = [
     'onion',
     'onl',
     'online',
-    'onyourside',
     'ooo',
     'open',
     'oracle',
@@ -1109,7 +1101,6 @@ TLDs = [
     'shouji',
     'show',
     'showtime',
-    'shriram',
     'si',
     'silk',
     'sina',
@@ -1143,7 +1134,6 @@ TLDs = [
     'space',
     'sport',
     'spot',
-    'spreadbetting',
     'sr',
     'srl',
     'ss',
@@ -1367,6 +1357,9 @@ TLDs = [
     'zone',
     'zuerich',
     'zw',
+]
+
+NON_ASCII_TLDS = [
     'ελ',
     'ευ',
     'бг',
@@ -1392,6 +1385,7 @@ TLDs = [
     'اتصالات',
     'ارامكو',
     'الاردن',
+    'البحرين',
     'الجزائر',
     'السعودية',
     'السعوديه',
@@ -1448,6 +1442,7 @@ TLDs = [
     'ලංකා',
     'คอม',
     'ไทย',
+    'ລາວ',
     'გე',
     'みんな',
     'アマゾン',
diff --git a/xurls/tlds_pseudo.py b/xurls/tlds_pseudo.py
index 9b6c811..524f4a2 100644
--- a/xurls/tlds_pseudo.py
+++ b/xurls/tlds_pseudo.py
@@ -1,4 +1,4 @@
-# Copyright 2015 Andy Tzeng <andytzeng@aol.tw>. All Rights Reserved.
+# Copyright 2021 Andy Tzeng <andytzeng@aol.tw>. All Rights Reserved.
 # Licensed under BSD 3-Clause License.
 
 # Refer the tool unicodegen from mvdan.cc/xurls/v2
@@ -10,7 +10,7 @@
 #  * https://en.wikipedia.org/wiki/Category:Pseudo-top-level_domains
 #  * https://tools.ietf.org/html/draft-grothoff-iesg-special-use-p2p-names-00
 #  * https://www.iana.org/assignments/special-use-domain-names/special-use-domain-names.xhtml
-PseudoTLDs = [
+PSEUDO_TLDS = [
     'bit',  # Namecoin
     'example',  # Example domain
     'exit',  # Tor exit node
diff --git a/xurls/unicode.py b/xurls/unicode.py
index ab2021f..2133628 100644
--- a/xurls/unicode.py
+++ b/xurls/unicode.py
@@ -1,4 +1,4 @@
-# Copyright 2015 Andy Tzeng <andytzeng@aol.tw>. All Rights Reserved.
+# Copyright 2021 Andy Tzeng <andytzeng@aol.tw>. All Rights Reserved.
 # Licensed under BSD 3-Clause License.
 
 # Refer the tool unicodegen from mvdan.cc/xurls/v2
diff --git a/xurls/xurls.py b/xurls/xurls.py
index a959ed2..738f8ad 100644
--- a/xurls/xurls.py
+++ b/xurls/xurls.py
@@ -1,4 +1,4 @@
-# Copyright 2015 Andy Tzeng <andytzeng@aol.tw>. All Rights Reserved.
+# Copyright 2021 Andy Tzeng <andytzeng@aol.tw>. All Rights Reserved.
 # Licensed under BSD 3-Clause License.
 from . import express
 from ._regex import LONEST_MATCH, re


Test patch:
diff --git a/tests/test_xurls.py b/tests/test_xurls.py
index 2927b0c..695a71f 100644
--- a/tests/test_xurls.py
+++ b/tests/test_xurls.py
@@ -1,4 +1,4 @@
-# Copyright 2015 Andy Tzeng <andytzeng@aol.tw>. All Rights Reserved.
+# Copyright 2021 Andy Tzeng <andytzeng@aol.tw>. All Rights Reserved.
 # Licensed under BSD 3-Clause License.
 import pytest
 
@@ -183,9 +183,8 @@ def test_loosen_extra_cases():
             ('foo.com bar.com', 'foo.com'),
             ('foo.com-foo', 'foo.com'),
             ('foo.company', True),
-            # We can't use \b to not match foo.com here, because that
-            # does't work for non-ASCII TLDs.
-            ('foo.comrandom', 'foo.com'),
+            ('foo.comrandom', None),
+            ('some.guy', None),
             ('foo.example', True),
             ('foo.i2p', True),
             ('foo.local', True),