Repository: iterative/dvc

Issue:
import-url: does not work for URL that contains a query string
```
(.env) [ivan@ivan ~/Projects/test-gs]$ dvc version
DVC version: 0.86.5+1f4fbf.mod
Python version: 3.7.6
Platform: Darwin-18.2.0-x86_64-i386-64bit
Binary: False
Package: None
Cache: reflink - supported, hardlink - supported, symlink - supported
Filesystem type (cache directory): ('apfs', '/dev/disk1s1')
Filesystem type (workspace): ('apfs', '/dev/disk1s1')
```

`dvc import-url -v https://www.dropbox.com/?test data`

gets you:

```
2020-02-29 21:01:49,849 ERROR: unexpected error
------------------------------------------------------------
Traceback (most recent call last):
  File "/Users/ivan/Projects/dvc/dvc/main.py", line 49, in main
    ret = cmd.run()
  File "/Users/ivan/Projects/dvc/dvc/command/imp_url.py", line 16, in run
    self.args.url, out=self.args.out, fname=self.args.file
  File "/Users/ivan/Projects/dvc/dvc/repo/__init__.py", line 28, in wrapper
    ret = f(repo, *args, **kwargs)
  File "/Users/ivan/Projects/dvc/dvc/repo/scm_context.py", line 4, in run
    result = method(repo, *args, **kw)
  File "/Users/ivan/Projects/dvc/dvc/repo/imp_url.py", line 20, in imp_url
    accompany_outs=True,
  File "/Users/ivan/Projects/dvc/dvc/stage.py", line 534, in create
    stage, kwargs.get("deps", []), erepo=kwargs.get("erepo", None)
  File "/Users/ivan/Projects/dvc/dvc/dependency/__init__.py", line 77, in loads_from
    ret.append(_get(stage, s, info))
  File "/Users/ivan/Projects/dvc/dvc/dependency/__init__.py", line 61, in _get
    return d(stage, p, info)
  File "/Users/ivan/Projects/dvc/dvc/output/base.py", line 101, in __init__
    self.path_info = self._parse_path(remote, path)
  File "/Users/ivan/Projects/dvc/dvc/output/base.py", line 109, in _parse_path
    return self.REMOTE.path_cls(path)
  File "/Users/ivan/Projects/dvc/dvc/path_info.py", line 115, in __init__
    assert not p.query and not p.params and not p.fragment
AssertionError
------------------------------------------------------------
```

Context: https://discordapp.com/channels/485586884165107732/485596304961962003/683485273085968424




import-url: does not work for URL that contains a query string
```
(.env) [ivan@ivan ~/Projects/test-gs]$ dvc version
DVC version: 0.86.5+1f4fbf.mod
Python version: 3.7.6
Platform: Darwin-18.2.0-x86_64-i386-64bit
Binary: False
Package: None
Cache: reflink - supported, hardlink - supported, symlink - supported
Filesystem type (cache directory): ('apfs', '/dev/disk1s1')
Filesystem type (workspace): ('apfs', '/dev/disk1s1')
```

`dvc import-url -v https://www.dropbox.com/?test data`

gets you:

```
2020-02-29 21:01:49,849 ERROR: unexpected error
------------------------------------------------------------
Traceback (most recent call last):
  File "/Users/ivan/Projects/dvc/dvc/main.py", line 49, in main
    ret = cmd.run()
  File "/Users/ivan/Projects/dvc/dvc/command/imp_url.py", line 16, in run
    self.args.url, out=self.args.out, fname=self.args.file
  File "/Users/ivan/Projects/dvc/dvc/repo/__init__.py", line 28, in wrapper
    ret = f(repo, *args, **kwargs)
  File "/Users/ivan/Projects/dvc/dvc/repo/scm_context.py", line 4, in run
    result = method(repo, *args, **kw)
  File "/Users/ivan/Projects/dvc/dvc/repo/imp_url.py", line 20, in imp_url
    accompany_outs=True,
  File "/Users/ivan/Projects/dvc/dvc/stage.py", line 534, in create
    stage, kwargs.get("deps", []), erepo=kwargs.get("erepo", None)
  File "/Users/ivan/Projects/dvc/dvc/dependency/__init__.py", line 77, in loads_from
    ret.append(_get(stage, s, info))
  File "/Users/ivan/Projects/dvc/dvc/dependency/__init__.py", line 61, in _get
    return d(stage, p, info)
  File "/Users/ivan/Projects/dvc/dvc/output/base.py", line 101, in __init__
    self.path_info = self._parse_path(remote, path)
  File "/Users/ivan/Projects/dvc/dvc/output/base.py", line 109, in _parse_path
    return self.REMOTE.path_cls(path)
  File "/Users/ivan/Projects/dvc/dvc/path_info.py", line 115, in __init__
    assert not p.query and not p.params and not p.fragment
AssertionError
------------------------------------------------------------
```

Context: https://discordapp.com/channels/485586884165107732/485596304961962003/683485273085968424




import-url: allow queries in URL
Looks like disallowing URLs containing queries happened in b83564d (#2707), i.e. `dvc>0.66.1`... Not sure why. Seems fine to me to remove the `assert not p.query` restriction.

- [x] fix `URLInfo` for http URLs (via  subclass `HTTPURLInfo(URLInfo)`)
  + [x] add support for params
  + [x] add support for queries
    * fixes #3424 `dvc import-url -v https://www.dropbox.com/?test data`
  + [x] add support for fragments
- [x] add tests


Discussion:
Updated the description.
Hi. We have a user [on Discord](https://discordapp.com/channels/485586884165107732/485596304961962003/684064944492511292) asking for updates on this. Is it expected to be a quick fix @iterative/engineering? Thanks 🙂 
Updated the description.
Hi. We have a user [on Discord](https://discordapp.com/channels/485586884165107732/485596304961962003/684064944492511292) asking for updates on this. Is it expected to be a quick fix @iterative/engineering? Thanks 🙂 


Resolving patch:
diff --git a/dvc/path_info.py b/dvc/path_info.py
--- a/dvc/path_info.py
+++ b/dvc/path_info.py
@@ -241,3 +241,79 @@ class CloudURLInfo(URLInfo):
     @property
     def path(self):
         return self._spath.lstrip("/")
+
+
+class HTTPURLInfo(URLInfo):
+    def __init__(self, url):
+        p = urlparse(url)
+        stripped = p._replace(params=None, query=None, fragment=None)
+        super().__init__(stripped.geturl())
+        self.params = p.params
+        self.query = p.query
+        self.fragment = p.fragment
+
+    @classmethod
+    def from_parts(
+        cls,
+        scheme=None,
+        host=None,
+        user=None,
+        port=None,
+        path="",
+        netloc=None,
+        params=None,
+        query=None,
+        fragment=None,
+    ):
+        assert bool(host) ^ bool(netloc)
+
+        if netloc is not None:
+            return cls(
+                "{}://{}{}{}{}{}".format(
+                    scheme,
+                    netloc,
+                    path,
+                    (";" + params) if params else "",
+                    ("?" + query) if query else "",
+                    ("#" + fragment) if fragment else "",
+                )
+            )
+
+        obj = cls.__new__(cls)
+        obj.fill_parts(scheme, host, user, port, path)
+        obj.params = params
+        obj.query = query
+        obj.fragment = fragment
+        return obj
+
+    @property
+    def _extra_parts(self):
+        return (self.params, self.query, self.fragment)
+
+    @property
+    def parts(self):
+        return self._base_parts + self._path.parts + self._extra_parts
+
+    @cached_property
+    def url(self):
+        return "{}://{}{}{}{}{}".format(
+            self.scheme,
+            self.netloc,
+            self._spath,
+            (";" + self.params) if self.params else "",
+            ("?" + self.query) if self.query else "",
+            ("#" + self.fragment) if self.fragment else "",
+        )
+
+    def __eq__(self, other):
+        if isinstance(other, (str, bytes)):
+            other = self.__class__(other)
+        return (
+            self.__class__ == other.__class__
+            and self._base_parts == other._base_parts
+            and self._path == other._path
+            and self._extra_parts == other._extra_parts
+        )
+
+    def __hash__(self):
+        return hash(self.parts)
diff --git a/dvc/remote/http.py b/dvc/remote/http.py
--- a/dvc/remote/http.py
+++ b/dvc/remote/http.py
@@ -4,6 +4,7 @@
 
 from funcy import cached_property, memoize, wrap_prop, wrap_with
 
+from dvc.path_info import HTTPURLInfo
 import dvc.prompt as prompt
 from dvc.config import ConfigError
 from dvc.exceptions import DvcException, HTTPError
@@ -25,6 +26,7 @@ def ask_password(host, user):
 
 class RemoteHTTP(RemoteBASE):
     scheme = Schemes.HTTP
+    path_cls = HTTPURLInfo
     SESSION_RETRIES = 5
     SESSION_BACKOFF_FACTOR = 0.1
     REQUEST_TIMEOUT = 10


Test patch:
diff --git a/tests/unit/test_path_info.py b/tests/unit/test_path_info.py
--- a/tests/unit/test_path_info.py
+++ b/tests/unit/test_path_info.py
@@ -4,6 +4,7 @@
 import pytest
 
 from dvc.path_info import CloudURLInfo
+from dvc.path_info import HTTPURLInfo
 from dvc.path_info import PathInfo
 from dvc.path_info import URLInfo
 
@@ -44,13 +45,23 @@ def test_url_info_parents(cls):
     ]
 
 
-@pytest.mark.parametrize("cls", [URLInfo, CloudURLInfo])
+@pytest.mark.parametrize("cls", [URLInfo, CloudURLInfo, HTTPURLInfo])
 def test_url_info_deepcopy(cls):
     u1 = cls("ssh://user@test.com:/test1/test2/test3")
     u2 = copy.deepcopy(u1)
     assert u1 == u2
 
 
+def test_https_url_info_str():
+    url = "https://user@test.com/test1;p=par?q=quer#frag"
+    u = HTTPURLInfo(url)
+    assert u.url == url
+    assert str(u) == u.url
+    assert u.params == "p=par"
+    assert u.query == "q=quer"
+    assert u.fragment == "frag"
+
+
 @pytest.mark.parametrize(
     "path, as_posix, osname",
     [