nebius/SWE-bench-extra huggingface dataset:
Repository: lipoja/URLExtract

Issue:
Does Not extract the URL that is leading special character
Hi,

I am trying to extract the URL from the following text,
But it is not grabbing the link due to the hyphen.

EncodedText="* test link -https://www.google.com";

extractor = URLExtract()
urls = extractor.find_urls(EncodedText)

print(URLs)

---output
[]

Do you have any idea to fix this?


 

Comments:

Patch:
diff --git a/urlextract/urlextract_core.py b/urlextract/urlextract_core.py
index 0f5bc0a..8f7d5c8 100644
--- a/urlextract/urlextract_core.py
+++ b/urlextract/urlextract_core.py
@@ -488,6 +488,9 @@ class URLExtract(CacheFile):
         # URL should not start with two backslashes
         if complete_url.startswith("//"):
             complete_url = complete_url[2:]
+        # URL should not start with unreserved characters
+        if complete_url.startswith(("-", ".", "~", "_")):
+            complete_url = complete_url[1:]
         if not self._is_domain_valid(
             complete_url, tld, check_dns=check_dns, with_schema_only=with_schema_only
         ):


Test patch:
diff --git a/tests/unit/test_find_urls.py b/tests/unit/test_find_urls.py
index 03b6c17..d776bbd 100644
--- a/tests/unit/test_find_urls.py
+++ b/tests/unit/test_find_urls.py
@@ -37,8 +37,8 @@ import pytest
             ["https://example.com/what.com"],
         ),
         (
-            "https://i2.wp.com/siliconfilter.com/2011/06/example.jpg",
-            ["https://i2.wp.com/siliconfilter.com/2011/06/example.jpg"],
+            "* test link -https://www.example.com",
+            ["https://www.example.com"],
         ),
         (
             "https://www.test.org/paper/apostrophe'in-url",
Repository: lipoja/URLExtract

Issue:
should not grab email fragments
```
>>> from urlextract import URLExtract
>>> extractor = URLExtract()
>>> extractor.find_urls("@gmail.com")
['@gmail.com']
>>> extractor.find_urls("bad.email @gmail.com")
['bad.email', '@gmail.com']
```

should not be grabbing `@gmail.com` at all

Comments:

Patch:
diff --git a/urlextract/urlextract_core.py b/urlextract/urlextract_core.py
index 8f7d5c8..dec043c 100644
--- a/urlextract/urlextract_core.py
+++ b/urlextract/urlextract_core.py
@@ -577,6 +577,10 @@ class URLExtract(CacheFile):
         url_parts = uritools.urisplit(url)
         # <scheme>://<authority>/<path>?<query>#<fragment>
 
+        # authority can't start with @
+        if url_parts.authority.startswith('@'):
+            return False
+
         # if URI contains user info and schema was automatically added
         # the url is probably an email
         if url_parts.getuserinfo() and added_schema:


Test patch:
diff --git a/tests/unit/test_extract_email.py b/tests/unit/test_extract_email.py
index d8c12e6..77352dd 100644
--- a/tests/unit/test_extract_email.py
+++ b/tests/unit/test_extract_email.py
@@ -34,6 +34,7 @@ def test_extract_email_disabled(urlextract, text, expected):
     [
         ("Do not extract emails by default jan@example.com", ["jan@example.com"]),
         ("<email@address.net>", ["email@address.net"]),
+        ("whitespace @address.net>", []),
         ("Given URIs are not mail jan@example.com/asdasd jan@example.com:1234", []),
         ("Given URIs are not mail jan@example.com?not jan@example.com#not", []),
     ],
diff --git a/tests/unit/test_find_urls.py b/tests/unit/test_find_urls.py
index d776bbd..783ca0c 100644
--- a/tests/unit/test_find_urls.py
+++ b/tests/unit/test_find_urls.py
@@ -57,6 +57,7 @@ import pytest
             "<script src='//www.example.com/somejsfile.js'>",
             ["www.example.com/somejsfile.js"],
         ),
+        ("bad.email @address.net>", ['bad.email']),
     ],
 )
 def test_find_urls(urlextract, text, expected):
Repository: lipoja/URLExtract

Issue:
Wrong indices with uppercase characters in domain name
I am getting wrong indices when the domain name of a URL contains uppercase characters.

To reproduce:
```
from urlextract import URLExtract
extractor = URLExtract()
urls = extractor.find_urls("www.Google.com", get_indices=True)
print(urls[0])
urls = extractor.find_urls("www.google.com", get_indices=True)
print(urls[0])
```
Output is:
```
('www.Google.com', (1, 15))
('www.google.com', (0, 14))
```


Comments:

Patch:
diff --git a/urlextract/urlextract_core.py b/urlextract/urlextract_core.py
index b6b3767..8283024 100644
--- a/urlextract/urlextract_core.py
+++ b/urlextract/urlextract_core.py
@@ -760,8 +760,11 @@ class URLExtract(CacheFile):
 
         url_parts = uritools.urisplit(tpm_url)
         host = str(url_parts.gethost())
-        offset = url.find(host)
-        return host.rfind(tld) + offset
+        # `host` is always returned in lowercase,
+        # so make sure `url` & `tld` must also be lowercase,
+        # otherwise the `find()` may fail.
+        offset = url.lower().find(host)
+        return host.rfind(tld.lower()) + offset
 
     # TODO: move type assertion to be Generator based
     # found https://stackoverflow.com/a/38423388/14669675


Test patch:
diff --git a/tests/unit/test_find_urls.py b/tests/unit/test_find_urls.py
index 90250e6..9ee46eb 100644
--- a/tests/unit/test_find_urls.py
+++ b/tests/unit/test_find_urls.py
@@ -112,10 +112,10 @@ def test_find_urls_unique(urlextract, text, expected):
     "text, expected",
     [
         (
-            "Let's have URL http://janlipovsky.cz and a second URL https://example.com/@eon01/asdsd-dummy it's over.",
+            "Let's have URL http://janlipovsky.cz and a second URL https://Example.Com/@eon01/asdsd-dummy it's over.",
             [
                 ("http://janlipovsky.cz", (15, 36)),
-                ("https://example.com/@eon01/asdsd-dummy", (54, 92)),
+                ("https://Example.Com/@eon01/asdsd-dummy", (54, 92)),
             ],
         ),
         (
diff --git a/tests/unit/test_get_tld_pos.py b/tests/unit/test_get_tld_pos.py
new file mode 100644
index 0000000..64997cd
--- /dev/null
+++ b/tests/unit/test_get_tld_pos.py
@@ -0,0 +1,30 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+This file contains pytests for _get_tld_pos() method of URLExtract
+
+.. Licence MIT
+.. codeauthor:: Jan Lipovský <janlipovsky@gmail.com>, janlipovsky.cz
+"""
+import pytest
+
+
+@pytest.mark.parametrize(
+    "url, tld, expected",
+    [
+        ("httpbin.org/status/200", ".org", 7),
+        ("https://httpbin.org/status/200", ".org", 15),
+        ("caseInsensitive.cOM", ".cOM", 15),
+        ("https://caseInsensitive.COM/status/200", ".COM", 23),
+    ],
+)
+def test_get_ltd_pos(urlextract, url, tld, expected):
+    """
+    Testing _get_tld_pos returning index
+
+    :param fixture urlextract: fixture holding URLExtract object
+    :param str url: URL in which tld should be located
+    :param str tld: TLD we want to find
+    :param int expected: index of tld that has be found in url
+    """
+    assert urlextract._get_tld_pos(url, tld) == expected
Repository: lucianopaz/compress_pickle

Issue:
File ending with `.pkl.gz` is stored as `.pklgz`
Hello @lucianopaz,

Thank you for making this library!

I wanted to ask why, if the provided path is `.pkl.gz`, the files are stored as `.pklgz`.
This feels like an undesired default behaviour that may be, if necessary, enabled with a flag, as it breaks libraries that depend on the files to be stored as the given path requires.

Ciao e Grazie,
Luca

Comments:

Patch:
diff --git a/compress_pickle/__init__.py b/compress_pickle/__init__.py
index a9221a9..1471ef4 100644
--- a/compress_pickle/__init__.py
+++ b/compress_pickle/__init__.py
@@ -4,4 +4,4 @@ from . import compressers
 from . import picklers
 from . import io
 
-__version__ = "2.0.0"
+__version__ = "2.0.1"
diff --git a/compress_pickle/utils.py b/compress_pickle/utils.py
index f172d63..4e226ad 100644
--- a/compress_pickle/utils.py
+++ b/compress_pickle/utils.py
@@ -135,4 +135,4 @@ def _infer_compression_from_path(path: PathType) -> Optional[str]:
 
 def _set_default_extension(path: PathType, compression: Optional[str]) -> str:
     root, current_ext = splitext(_stringyfy_path(path))
-    return root + get_default_compression_mapping()[compression]
+    return root + "." + get_default_compression_mapping()[compression]


Test patch:
diff --git a/tests/test_utils.py b/tests/test_utils.py
index 8b54c9e..6ec6fcc 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -1,6 +1,12 @@
 import pytest
 import io
-from compress_pickle.utils import _stringyfy_path, _infer_compression_from_path
+from os.path import splitext
+from compress_pickle.compressers.registry import get_default_compression_mapping
+from compress_pickle.utils import (
+    _stringyfy_path,
+    _infer_compression_from_path,
+    _set_default_extension,
+)
 
 
 def test_stringify_path():
@@ -10,6 +16,17 @@ def test_stringify_path():
         _stringyfy_path({"a"})
 
 
+@pytest.mark.usefixtures("compressions")
+def test_set_default_extension(compressions):
+    root = "somepath.someotherstuff"
+    path = root + ".ext"
+    new_path = _set_default_extension(path, compression=compressions)
+    assert splitext(new_path) == (
+        root,
+        "." + get_default_compression_mapping()[compressions],
+    )
+
+
 @pytest.mark.usefixtures("valid_extensions")
 def test_infer_compression_from_path(valid_extensions):
     extension, compression = valid_extensions
Repository: lucidsushi/gs_pip_install

Issue:
Path error when specifying extras
https://github.com/lucidsushi/gs_pip_install/blob/09668ce1fe393dc338b5b3f3181f76bd44891e74/gs_pip_install/gs_pip_install.py#L79

Currently a list of lists is created - will raise:

ERROR: Invalid requirement: 'gcs_packages/geotab_data_factory.tar.gz[[pipelines]]'
Hint: It looks like a path. File 'gcs_packages/geotab_data_factory.tar.gz[[pipelines]]' does not exist.

Replace to `install_path = f"{install_path}{extras[package]}`


Comments:

Patch:
diff --git a/gs_pip_install/gs_pip_install.py b/gs_pip_install/gs_pip_install.py
index 1313713..3bc6d66 100644
--- a/gs_pip_install/gs_pip_install.py
+++ b/gs_pip_install/gs_pip_install.py
@@ -76,7 +76,7 @@ def install_packages(
         package, *_ = gs_package_zip_file.split('.')
         install_path = f"{packages_download_dir}/{gs_package_zip_file}"
         if extras.get(package):
-            install_path = f"{install_path}[{extras[package]}]"
+            install_path = f"{install_path}{extras[package]}"
 
         if not target_dir:
             install_command = [


Test patch:
diff --git a/tests/test_gs_pip_install.py b/tests/test_gs_pip_install.py
index c468e7a..a03a95f 100644
--- a/tests/test_gs_pip_install.py
+++ b/tests/test_gs_pip_install.py
@@ -147,7 +147,7 @@ class TestInstall(unittest.TestCase):
         mock_list_dir.return_value = ['some_package.tar.gz']
         gs_pip_install.install_packages(
             packages_download_dir='some_download_dest',
-            extras={'some_package': 'extra_a, extra_b'},
+            extras={'some_package': '[extra_a, extra_b]'},
         )
         mock_subprocess.assert_called_once_with(
             [
@@ -160,3 +160,13 @@ class TestInstall(unittest.TestCase):
                 "some_download_dest/some_package.tar.gz[extra_a, extra_b]",
             ]
         )
+
+    def test_strip_extras(self):
+
+        # fmt: off
+        assert gs_pip_install._strip_extras(
+            'some_package.tar.gz[extra_a, extra_b]'
+        ) == (
+            'some_package.tar.gz', '[extra_a, extra_b]'
+        )
+        # fmt: on