Repository: RafaelWO/unparallel (language: python)

Issue:
Support requests to different URLs
## 🚀 Feature Request
Currently, one has to pass a base URL which will be used for all requests. Hence, you cannot make requests to different services, or URLs. 

Change the logic of the `up()` function to support various full URLs as paths. 

Note: This will make the parameter `base_url` optional which results in a **breaking change**.

## 🔈 Motivation

Others seem to have the requirement of making thousands of requests to different URLs (see [various](https://stackoverflow.com/questions/2632520/what-is-the-fastest-way-to-send-100-000-http-requests-in-python) [stack](https://stackoverflow.com/questions/72013184/how-to-request-data-from-hundreds-of-urls-in-python) [overflow](https://stackoverflow.com/questions/74567219/how-do-i-get-python-to-send-as-many-concurrent-http-requests-as-possible) [posts](https://stackoverflow.com/questions/57126286/fastest-parallel-requests-in-python)).

## 🛰 Alternatives

n/a

## 📎 Additional context

n/a



Discussion / hints:

Fix patch:
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
index c079cf1..05124c8 100644
--- a/.github/workflows/docs.yml
+++ b/.github/workflows/docs.yml
@@ -3,6 +3,9 @@ on:
   push:
     branches:
       - main
+    tags:
+      - '**'
+
 permissions:
   contents: write
 jobs:
diff --git a/unparallel/unparallel.py b/unparallel/unparallel.py
index 2e00cb9..873e67a 100644
--- a/unparallel/unparallel.py
+++ b/unparallel/unparallel.py
@@ -17,11 +17,11 @@
 class RequestError:
     """A dataclass wrapping an exception that was raised during a web request.
 
-    Besides the exception itself, this contains the path, method, and (optional) payload
+    Besides the exception itself, this contains the URL, method, and (optional) payload
     of the failed request.
     """
 
-    path: str
+    url: str
     method: str
     payload: Optional[Any]
     exception: Exception
@@ -42,17 +42,17 @@ def sort_by_idx(results: List[Tuple[int, Any]]) -> List[Any]:
 async def single_request(
     idx: int,
     client: httpx.AsyncClient,
-    path: str,
+    url: str,
     method: str,
     json: Optional[Any] = None,
     max_retries_on_timeout: int = 3,
 ) -> Tuple[int, Any]:
-    """Do a single web request for the given path, HTTP method, and playload.
+    """Do a single web request for the given URL, HTTP method, and playload.
 
     Args:
         idx (int): The index of the task (required for sorting afterwards).
         client (AsyncClient): The httpx client.
-        path (str): The path after the base URI.
+        url (str): The URL after the base URI.
         method (str): The HTTP method.
         json (Optional[Any], optional): The JSON payload. Defaults to None.
         max_retries_on_timeout (int): The maximum number retries if the requests fails
@@ -65,7 +65,7 @@ async def single_request(
     exception: Optional[Exception] = None
     for trial in range(1, max_retries_on_timeout + 1):
         try:
-            response = await client.request(method, path, json=json)
+            response = await client.request(method, url, json=json)
             response.raise_for_status()
             json_data = response.json()
             return idx, json_data
@@ -84,7 +84,7 @@ async def single_request(
     return (
         idx,
         RequestError(
-            path=path,
+            url=url,
             method=method,
             payload=json,
             exception=exception,
@@ -93,9 +93,9 @@ async def single_request(
 
 
 async def request_urls(
-    base_url: str,
-    paths: Union[str, List[str]],
+    urls: List[str],
     method: str,
+    base_url: Optional[str] = None,
     headers: Optional[Dict[str, Any]] = None,
     payloads: Optional[Any] = None,
     flatten_result: bool = False,
@@ -105,44 +105,43 @@ async def request_urls(
     progress: bool = True,
 ) -> List[Any]:
     """
-    Asynchronously issues requests to a URL at the specified path(s)
+    Asynchronously issues requests to the specified URL(s)
     via ``asyncio`` and ``httpx``.
 
     Args:
-        base_url: The base URL of the service, e.g. http://localhost:8000.
-        paths: One path or a list of paths, e.g. /foobar/. If one path but multiple
-            payloads are supplied, that path is used for all requests.
-        method: HTTP method to use, e.g. get, post, etc.
-        headers: A dictionary of headers to use.
-        payloads: A list of JSON payloads (dictionaries) for e.g. HTTP post requests.
-            Used together with paths.
-        flatten_result: If True and the response per request is a list, flatten that
+        urls (List[str]): A list of URLs for the HTTP requests.
+        method (str): HTTP method to use, e.g. get, post, etc.
+        base_url (Optional[str]): The base URL of the service, e.g. http://localhost:8000.
+        headers (Optional[Dict[str, Any]]): A dictionary of headers to use.
+        payloads (Optional[Any]): A list of JSON payloads (dictionaries) for e.g. HTTP post requests.
+            Used together with ``urls``.
+        flatten_result (bool): If True and the response per request is a list, flatten that
             list of lists. This is useful when using paging.
         max_retries_on_timeout (int): The maximum number retries if the requests fails
             due to a timeout (``httpx.TimeoutException``). Defauls to 3.
         limits (httpx.Limits): The limits configuration for ``httpx``.
         timeouts (httpx.Timeout): The timeout configuration for ``httpx``.
-        progress: If set to True, progress bar is shown
+        progress (bool): Whether to show a progress bar. Defaults to ``True``.
 
     Returns:
         A list of the response data per request in the same order as the input
-        (paths/payloads).
+        (URLs/payloads).
     """
     tasks = []
     results = []
 
     logging.debug(
-        f"Issuing {len(paths)} {method.upper()} request(s) to base URL '{base_url}' "
+        f"Issuing {len(urls)} {method.upper()} request(s) to base URL '{base_url}' "
         f"with {limits.max_connections} max connections..."
     )
     async with httpx.AsyncClient(
-        base_url=base_url, headers=headers, timeout=timeouts, limits=limits
+        base_url=base_url or "", headers=headers, timeout=timeouts, limits=limits
     ) as client:
-        for i, path in enumerate(paths):
+        for i, url in enumerate(urls):
             task = asyncio.create_task(
                 single_request(
                     idx=i,
-                    path=path,
+                    url=url,
                     client=client,
                     method=method,
                     json=payloads[i] if payloads else None,
@@ -164,9 +163,9 @@ async def request_urls(
 
 
 async def up(
-    base_url: str,
-    paths: Union[str, List[str]],
+    urls: Union[str, List[str]],
     method: str = "GET",
+    base_url: Optional[str] = None,
     headers: Optional[Dict[str, Any]] = None,
     payloads: Optional[Any] = None,
     flatten_result: bool = False,
@@ -177,21 +176,24 @@ async def up(
     timeouts: Optional[httpx.Timeout] = None,
     progress: bool = True,
 ) -> List[Any]:
-    """Creates async web requests to a URL at the specified path(s) via ``asyncio``
+    """Creates async web requests to the specified URL(s) using ``asyncio``
     and ``httpx``.
 
     Args:
-        base_url (str):  The base URL of the target API/service.
-        paths (Union[str, List[str]]): One path or a list of paths, e.g. /foobar/.
-            If one path but multiple payloads are supplied, that path is used for all
-            requests.
+        urls (Union[str, List[str]]): A list of URLs as the targets for the requests.
+            If only one URL but multiple payloads are supplied, that URL is used for
+            all requests.
+            If a ``base_url`` is supplied, this can also be a list of paths
+            (or one path).
         method (str): HTTP method to use - one of ``GET``, ``OPTIONS``, ``HEAD``,
             ``POST``, ``PUT``, ``PATCH``, or ``DELETE``. Defaults to ``GET``.
+        base_url (Optional[str]):  The base URL of the target API/service. Defaults to
+            None.
         headers (Optional[Dict[str, Any]], optional): A dictionary of headers to use.
             Defaults to None.
         payloads (Optional[Any], optional): A list of JSON payloads (dictionaries) e.g.
-            for HTTP post requests. Used together with paths. If one payload but
-            multiple paths are supplied, that payload is used for all requests.
+            for HTTP post requests. Used together with ``urls``. If one payload but
+            multiple URLs are supplied, that payload is used for all requests.
             Defaults to None.
         flatten_result (bool): If True and the response per request is a list,
             flatten that list of lists. This is useful when using paging.
@@ -211,12 +213,12 @@ async def up(
 
     Raises:
         ValueError: If the HTTP method is not valid.
-        ValueError: If the number of paths provided does not match the number of
-            payloads (except if there is only one path).
+        ValueError: If the number of URLs provided does not match the number of
+            payloads (except if there is only one URL).
 
     Returns:
         List[Any]:  A list of the response data per request in the same order as the
-        input (paths/payloads).
+        input (URLs/payloads).
     """
     # Check if method it valid
     if method.upper() not in VALID_HTTP_METHODS:
@@ -225,22 +227,24 @@ async def up(
             f"Supported methods: {VALID_HTTP_METHODS}"
         )
 
-    # Check if payloads align with paths
+    # Wrap single URL into list to check for alignment with payload
+    if isinstance(urls, str):
+        urls = [urls]
+
+    # Check if payloads align with URLs
     if payloads:
-        if isinstance(paths, str):
-            paths = [paths]
         if not isinstance(payloads, list):
             payloads = [payloads]
-        if len(paths) == 1 and len(payloads) > 1:
-            logging.info(f"Using path '{paths[0]}' for all {len(payloads)} payloads")
-            paths = paths * len(payloads)
-        if len(payloads) == 1 and len(paths) > 1:
-            logging.info(f"Using payload '{payloads[0]}' for all {len(paths)} paths")
-            payloads = payloads * len(paths)
-        if len(paths) != len(payloads):
+        if len(urls) == 1 and len(payloads) > 1:
+            logging.info(f"Using URL '{urls[0]}' for all {len(payloads)} payloads")
+            urls = urls * len(payloads)
+        if len(payloads) == 1 and len(urls) > 1:
+            logging.info(f"Using payload '{payloads[0]}' for all {len(urls)} URLs")
+            payloads = payloads * len(urls)
+        if len(urls) != len(payloads):
             raise ValueError(
-                f"The number of paths does not match the number of payloads: "
-                f"{len(paths)} != {len(payloads)}"
+                f"The number of URLs does not match the number of payloads: "
+                f"{len(urls)} != {len(payloads)}"
             )
 
     if timeouts is None:
@@ -253,9 +257,9 @@ async def up(
             limits = DEFAULT_LIMITS
 
     return await request_urls(
-        base_url=base_url,
-        paths=paths,
+        urls=urls,
         method=method,
+        base_url=base_url,
         headers=headers,
         payloads=payloads,
         flatten_result=flatten_result,


Test patch:
diff --git a/tests/test_unparallel.py b/tests/test_unparallel.py
index db290af..c422469 100644
--- a/tests/test_unparallel.py
+++ b/tests/test_unparallel.py
@@ -34,7 +34,7 @@ def test_order_by_idx():
 async def test_single_request(url, method, payload, respx_mock):
     getattr(respx_mock, method)(url).mock(return_value=Response(200, json=payload))
     session = AsyncClient()
-    result = await single_request(1, session, path=url, method=method)
+    result = await single_request(1, session, url=url, method=method)
     assert result == (1, payload)
     await session.aclose()
 
@@ -45,8 +45,12 @@ async def test_single_request_fail(status, respx_mock):
     url = "http://test.com/foo"
     respx_mock.get(url).mock(return_value=Response(status))
     session = AsyncClient()
-    result = await single_request(1, session, path=url, method="GET")
-    assert isinstance(result[1], RequestError)
+    idx, result = await single_request(1, session, url=url, method="GET")
+    assert idx == 1
+    assert isinstance(result, RequestError)
+    assert result.method == "GET"
+    assert result.url == url
+    assert isinstance(result.exception, Exception)
     await session.aclose()
 
 
@@ -58,7 +62,7 @@ async def test_single_request_timeout(respx_mock):
     start_time = time.time()
     retries = 2
     result = await single_request(
-        1, session, path=url, method="GET", max_retries_on_timeout=retries
+        1, session, url=url, method="GET", max_retries_on_timeout=retries
     )
     assert isinstance(result[1], RequestError)
     assert time.time() - start_time > retries
@@ -69,7 +73,7 @@ async def test_single_request_timeout(respx_mock):
 async def test_full_run_via_httpbin():
     url = "https://httpbin.org"
     paths = [f"/get?i={i}" for i in range(20)]
-    results = await up(url, paths, "get", max_connections=10)
+    results = await up(paths, method="get", base_url=url, max_connections=10)
     assert len(results) == 20
     assert all(res["args"]["i"] == str(i) for i, res in enumerate(results))
 
@@ -77,21 +81,23 @@ async def test_full_run_via_httpbin():
 @pytest.mark.asyncio
 async def test_up_get(caplog, respx_mock):
     caplog.set_level(logging.DEBUG)
-    base_url = "http://test.com"
-    paths = ["/get?foo=0", "/get?foo=1", "/get?foo=2"]
-    for path in paths:
-        key, val = path.split("?")[1].split("=")
-        respx_mock.get(base_url + path).mock(
-            return_value=Response(200, json={key: int(val)})
-        )
+    urls = [
+        "http://test1.com/get?foo=0",
+        "http://test2.com/get?foo=1",
+        "http://test3.com/get?foo=2",
+    ]
+    for url in urls:
+        key, val = url.split("?")[1].split("=")
+        respx_mock.get(url).mock(return_value=Response(200, json={key: int(val)}))
 
-    results = await up(base_url, paths, "GET")
+    results = await up(urls, "GET")
 
     my_log = next(rec for rec in caplog.records if rec.module == "unparallel")
     assert "Issuing 3 GET request(s)" in my_log.message
-    assert len(results) == len(paths)
-    for i, (res, path) in enumerate(zip(results, paths)):
+    assert len(results) == len(urls)
+    for i, (res, path) in enumerate(zip(results, urls)):
         assert res == {"foo": i}
+        assert path.endswith(str(i))
 
 
 @pytest.mark.asyncio
@@ -103,7 +109,7 @@ async def test_up_post_single_vs_multi_path(paths, respx_mock):
         side_effect=[Response(200, json=data) for data in payloads]
     )
 
-    results = await up(base_url, paths, "post", payloads=payloads)
+    results = await up(paths, "post", base_url=base_url, payloads=payloads)
     assert len(results) == len(payloads)
     for res, data in zip(results, payloads):
         assert res == data
@@ -119,7 +125,7 @@ async def test_up_post_single_vs_multi_payload(payloads, respx_mock):
     for path in paths:
         respx_mock.post(base_url + path).mock(return_value=Response(200))
 
-    results = await up(base_url, paths, "post", payloads=payloads)
+    results = await up(paths, "post", base_url=base_url, payloads=payloads)
     assert len(results) == len(paths)
 
 
@@ -134,7 +140,10 @@ async def test_up_post_single_vs_multi_payload(payloads, respx_mock):
 )
 async def test_request_urls_flat(patched_fetch, flatten, expected):
     results = await request_urls(
-        "http://test.com", paths=["/a", "/b"], method="get", flatten_result=flatten
+        urls=["/a", "/b"],
+        method="get",
+        base_url="http://test.com",
+        flatten_result=flatten,
     )
     assert results == expected
 
@@ -143,14 +152,17 @@ async def test_request_urls_flat(patched_fetch, flatten, expected):
 async def test_up_misaligned_paths_and_payloads():
     with pytest.raises(ValueError):
         await up(
-            "http://test.com", paths=["/a", "/b"], method="POST", payloads=[1, 2, 3]
+            urls=["/a", "/b"],
+            method="POST",
+            base_url="http://test.com",
+            payloads=[1, 2, 3],
         )
 
 
 @pytest.mark.asyncio
 async def test_up_wrong_method():
     with pytest.raises(ValueError):
-        await up("http://test.com", paths="/a", method="foobar")
+        await up("/a", method="foobar", base_url="http://test.com")
 
 
 @pytest.mark.parametrize(
@@ -184,6 +196,6 @@ def get_kwargs(*args, **kwargs):
 
     request_urls_mock.side_effect = get_kwargs
 
-    options = await up("foobar", "/bar", **up_kwargs)
+    options = await up("/bar", base_url="foobar", **up_kwargs)
     assert options["timeouts"] == expected_timeouts
     assert options["limits"] == expected_limits