Repository: deepset-ai/haystack

Issue:
Import utils fetch_archive_from_http url parsing URL format assumption
**Describe the bug**
The function `fetch_archive_from_http` from import utils makes the assumption that a URL ends with the file extension. This assumption prevents fetching an archive from a URL that contains a query after the file extension. See also [RFC 3986](http://tools.ietf.org/html/rfc3986#section-3).

Fetching an archive from an [Amazon S3 presigned URL](https://docs.aws.amazon.com/AmazonS3/latest/userguide/using-presigned-url.html) is one use case that results in this bug.

**Error message**
`Skipped url https:/<S3_BUCKET_NAME>.s3.<REGION>.amazonaws.com/filename.tar.xz?response-content-disposition=inline&X-Amz-Security-Token=<TOKEN>&X-Amz-Algorithm=<X-AMZ-ALGORITHM>&X-Amz-Date=<X-AMZ-DATE>&X-Amz-SignedHeaders=host&X-Amz-Expires=<X-AMZ-EXPIRES>&X-Amz-Credential=<CREDENTIAL>&X-Amz-Signature=<SIGNATURE> as file type is not supported here. See haystack documentation for support of more file types`

**Expected behavior**
I expect the function to fetch the archive without an error.

**Additional context**
Add any other context about the problem here, like document types / preprocessing steps / settings of reader etc.

**To Reproduce**
1. Create an Amazon S3 bucket.
2. Upload a tar.xz archive into your Amazon S3 bucket.
3. Create an [Amazon S3 presigned URL](https://docs.aws.amazon.com/AmazonS3/latest/userguide/ShareObjectPreSignedURL.html) for your archive.
4. Fetch your archive from the Amazon S# presigned URL
```python
from haystack.utils import fetch_archive_from_http

url = "<YOUR_PRESIGNED_URL_FROM_STEP_3>"
fetch_archive_from_http(url, output_dir="my_archive_dir")
```

**FAQ Check**
- [x] Have you had a look at [our new FAQ page](https://docs.haystack.deepset.ai/docs/faq)?

**System:**
 - OS: n/a
 - GPU/CPU: n/a
 - Haystack version (commit or version number): v1.17.x
 - DocumentStore: n/a
 - Reader: n/a
 - Retriever: n/a



Discussion:

Resolving patch:
diff --git a/haystack/utils/import_utils.py b/haystack/utils/import_utils.py
--- a/haystack/utils/import_utils.py
+++ b/haystack/utils/import_utils.py
@@ -7,6 +7,8 @@
 import importlib.util
 from pathlib import Path
 from typing import Optional, Dict, Union, Tuple, List
+from urllib.parse import urlparse, unquote
+from os.path import splitext, basename
 
 import requests
 
@@ -93,6 +95,20 @@ def load_documents_from_hf_datasets(dataset_name: str, split: Optional[str] = "t
     return documents
 
 
+def get_filename_extension_from_url(url: str) -> Tuple[str, str]:
+    """
+    Extracts the filename and file extension from an url.
+
+    :param url: http address
+    :return: Tuple (filename, file extension) of the file at the url.
+    """
+    parsed = urlparse(url)
+    root, extension = splitext(parsed.path)
+    archive_extension = extension[1:]
+    file_name = unquote(basename(root[1:]))
+    return file_name, archive_extension
+
+
 def fetch_archive_from_http(
     url: str,
     output_dir: str,
@@ -122,7 +138,7 @@ def fetch_archive_from_http(
     else:
         logger.info("Fetching from %s to '%s'", url, output_dir)
 
-        _, _, archive_extension = url.rpartition(".")
+        file_name, archive_extension = get_filename_extension_from_url(url)
         request_data = requests.get(url, proxies=proxies, timeout=timeout)
 
         if archive_extension == "zip":
@@ -131,7 +147,6 @@ def fetch_archive_from_http(
         elif archive_extension == "gz" and not "tar.gz" in url:
             gzip_archive = gzip.GzipFile(fileobj=io.BytesIO(request_data.content))
             file_content = gzip_archive.read()
-            file_name = url.split("/")[-1][: -(len(archive_extension) + 1)]
             with open(f"{output_dir}/{file_name}", "wb") as file:
                 file.write(file_content)
         elif archive_extension in ["gz", "bz2", "xz"]:


Test patch:
diff --git a/test/others/test_utils.py b/test/others/test_utils.py
--- a/test/others/test_utils.py
+++ b/test/others/test_utils.py
@@ -1,6 +1,5 @@
 import importlib
 import logging
-from random import random
 from typing import List
 from unittest import mock
 
@@ -15,6 +14,7 @@
 from haystack.schema import Answer, Document, Span, Label
 from haystack.utils import print_answers
 from haystack.utils.deepsetcloud import DeepsetCloud, DeepsetCloudExperiments
+from haystack.utils.import_utils import get_filename_extension_from_url
 from haystack.utils.labels import aggregate_labels
 from haystack.utils.preprocessing import convert_files_to_docs, tika_convert_files_to_docs
 from haystack.utils.cleaning import clean_wiki_text
@@ -168,6 +168,30 @@ def test_convert_files_to_docs(samples_path):
     assert documents and len(documents) > 0
 
 
+@pytest.mark.unit
+def test_get_filename_extension_from_url_without_params_zip():
+    url = "http://www.mysite.com/resources/myfile.zip"
+    file_name, extension = get_filename_extension_from_url(url)
+    assert extension == "zip"
+    assert file_name == "myfile"
+
+
+@pytest.mark.unit
+def test_get_filename_extension_from_url_with_params_zip():
+    url = "https:/<S3_BUCKET_NAME>.s3.<REGION>.amazonaws.com/filename.zip?response-content-disposition=inline&X-Amz-Security-Token=<TOKEN>&X-Amz-Algorithm=<X-AMZ-ALGORITHM>&X-Amz-Date=<X-AMZ-DATE>&X-Amz-SignedHeaders=host&X-Amz-Expires=<X-AMZ-EXPIRES>&X-Amz-Credential=<CREDENTIAL>&X-Amz-Signature=<SIGNATURE>"
+    file_name, extension = get_filename_extension_from_url(url)
+    assert extension == "zip"
+    assert file_name == "filename"
+
+
+@pytest.mark.unit
+def test_get_filename_extension_from_url_with_params_xz():
+    url = "https:/<S3_BUCKET_NAME>.s3.<REGION>.amazonaws.com/filename.tar.xz?response-content-disposition=inline&X-Amz-Security-Token=<TOKEN>&X-Amz-Algorithm=<X-AMZ-ALGORITHM>&X-Amz-Date=<X-AMZ-DATE>&X-Amz-SignedHeaders=host&X-Amz-Expires=<X-AMZ-EXPIRES>&X-Amz-Credential=<CREDENTIAL>&X-Amz-Signature=<SIGNATURE>"
+    file_name, extension = get_filename_extension_from_url(url)
+    assert extension == "xz"
+    assert file_name == "filename.tar"
+
+
 @pytest.mark.tika
 def test_tika_convert_files_to_docs(samples_path):
     documents = tika_convert_files_to_docs(dir_path=samples_path, clean_func=clean_wiki_text, split_paragraphs=True)