Repository: datalad/datalad-next

Issue:
`AnnexKey` should be hashable
I would think that using this type as a key in a mapping would be a standard use case. It would require the following change:

```diff
diff --git a/datalad_next/types/annexkey.py b/datalad_next/types/annexkey.py
index 74bbcb6..5a85342 100644
--- a/datalad_next/types/annexkey.py
+++ b/datalad_next/types/annexkey.py
@@ -14,7 +14,7 @@ _annexkey_regex = re.compile(
 )
 
 
-@dataclass
+@dataclass(frozen=True)
 class AnnexKey:
     """Representation of a git-annex key
 
```

This would mean that post-init assignment of properties is not possible. I tend to think that this is OK.

Comments:

Patch:
diff --git a/datalad_next/types/annexkey.py b/datalad_next/types/annexkey.py
index 74bbcb6..037280b 100644
--- a/datalad_next/types/annexkey.py
+++ b/datalad_next/types/annexkey.py
@@ -1,3 +1,5 @@
+"""git-annex key representation"""
+
 from __future__ import annotations
 
 from dataclasses import dataclass
@@ -14,7 +16,7 @@ _annexkey_regex = re.compile(
 )
 
 
-@dataclass
+@dataclass(frozen=True)
 class AnnexKey:
     """Representation of a git-annex key
 
@@ -29,6 +31,7 @@ class AnnexKey:
 
     @classmethod
     def from_str(cls, key: str):
+        """Return an ``AnnexKey`` instance from a key string"""
         key_matched = _annexkey_regex.match(key)
         if not key_matched:
             # without a sensible key there is no hope
diff --git a/datalad_next/types/archivist.py b/datalad_next/types/archivist.py
new file mode 100644
index 0000000..12e9b2b
--- /dev/null
+++ b/datalad_next/types/archivist.py
@@ -0,0 +1,143 @@
+"""``dl+archive:`` archive member locator"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import PurePosixPath
+import re
+
+from .annexkey import AnnexKey
+from .enums import ArchiveType
+
+
+# be relatively permissive
+_recognized_urls = re.compile(r'^dl\+archive:(?P<key>.*)#(?P<props>.*)')
+# each archive member is identified by a (relative) path inside
+# the archive.
+_archive_member_props = re.compile(
+    # a path may contain any char but '&'
+    # TODO check that something in the machinery ensures proper
+    # quoting
+    'path=(?P<path>[^&]+)'
+    # size info (in bytes) is optional
+    '(&size=(?P<size>[0-9]+)|)'
+    # archive type label is optional
+    '(&atype=(?P<atype>[a-z0-9]+)|)'
+)
+
+
+@dataclass
+class ArchivistLocator:
+    """Representation of a ``dl+archive:`` archive member locator
+
+    These locators are used by the ``datalad-archives`` and ``archivist``
+    git-annex special remotes. They identify a member of a archive that is
+    itself identified by an annex key.
+
+    Each member is annotated with its size (in bytes). Optionally,
+    the file format type of the archive can be annotated too.
+
+    Syntax of ``dl+archives:`` locators
+    -----------------------------------
+
+    The locators the following minimal form::
+
+        dl+archive:<archive-key>#path=<path-in-archive>
+
+    where ``<archive-key>`` is a regular git-annex key of an archive file,
+    and ``<path-in-archive>`` is a POSIX-style relative path pointing to
+    a member within the archive.
+
+    Two optional, additional attributes ``size`` and ``atype`` are recognized
+    (only ``size`` is also understood by the ``datalad-archives``
+    special remote).
+
+    ``size`` declares the size of the (extracted) archive member in bytes::
+
+        dl+archive:<archive-key>#path=<path-in-archive>&size=<size-in-bytes>
+
+    ``atype`` declares the type of the containing archive using a label.
+    Currently recognized labels are ``tar`` (a TAR archive, compressed or not),
+    and ``zip`` (a ZIP archive). See
+    :class:`~datalad_next.types.enums.ArchiveType` for all recognized labels.
+
+    If no type information is given, :func:`ArchivistLocator.from_str()` will
+    try to determine the archive type from the archive key (via ``*E``-type
+    git-annex backends, such as DataLad's default ``MD5E``).
+
+    The order in the fragment part of the URL (after ``#``) is significant.
+    ``path`` must come first, followed by ``size`` or ``atype``. If both
+    ``size`` and ``atype`` are present, ``size`` must be declared first. A
+    complete example of a URL is::
+
+        dl+archive:MD5-s389--e9f624eb778e6f945771c543b6e9c7b2#path=dir/file.csv&size=234&atype=tar
+    """
+    akey: AnnexKey
+    member: PurePosixPath
+    size: int
+    # datalad-archives did not have the type info, we want to be
+    # able to handle those too, make optional
+    atype: ArchiveType | None = None
+
+    def __str__(self) -> str:
+        return 'dl+archive:{akey}#path={member}&size={size}{atype}'.format(
+            akey=self.akey,
+            # TODO needs quoting?
+            member=self.member,
+            size=self.size,
+            atype=f'&atype={self.atype.value}' if self.atype else '',
+        )
+
+    @classmethod
+    def from_str(cls, url: str):
+        """Return ``ArchivistLocator`` from ``str`` form"""
+        url_matched = _recognized_urls.match(url)
+        if not url_matched:
+            raise ValueError('Unrecognized dl+archives locator syntax')
+        url_matched = url_matched.groupdict()
+        # convert to desired type
+        akey = AnnexKey.from_str(url_matched['key'])
+
+        # archive member properties
+        props_matched = _archive_member_props.match(url_matched['props'])
+        if not props_matched:
+            # without at least a 'path' there is nothing we can do here
+            raise ValueError(
+                'dl+archives locator contains invalid archive member '
+                f'specification: {url_matched["props"]!r}')
+        props_matched = props_matched.groupdict()
+        amember_path = PurePosixPath(props_matched['path'])
+        if amember_path.is_absolute():
+            raise ValueError(
+                'dl+archives locator contains absolute archive member path')
+        if '..' in amember_path.parts:
+            raise ValueError(
+                'dl+archives locator archive member path contains ".."')
+
+        # size is optional, regex ensure that it is an int
+        size = props_matched.get('size')
+
+        # archive type, could be None
+        atype = props_matched.get('atype')
+        if atype is not None:
+            # if given, most be known type
+            try:
+                atype = getattr(ArchiveType, atype)
+            except AttributeError as e:
+                raise ValueError(
+                    'dl+archives locator archive type unrecognized') from e
+
+        if atype is None and akey.backend.endswith('E'):
+            # try by key name extension
+            suf = PurePosixPath(akey.name).suffixes
+            if '.zip' == suf[-1]:
+                atype = ArchiveType.zip
+            elif '.tar' in suf:
+                atype = ArchiveType.tar
+
+        return cls(
+            akey=akey,
+            member=amember_path,
+            size=size,
+            atype=atype,
+        )
diff --git a/datalad_next/types/enums.py b/datalad_next/types/enums.py
new file mode 100644
index 0000000..ad26721
--- /dev/null
+++ b/datalad_next/types/enums.py
@@ -0,0 +1,13 @@
+"""Type ENUMs"""
+
+from enum import Enum
+
+
+class ArchiveType(Enum):
+    """Enumeration of archive types
+
+    Each one should have an associated ArchiveOperations handler.
+    """
+    # TODO the values could also be handler classes ...
+    tar = 'tar'
+    zip = 'zip'
diff --git a/docs/source/pyutils.rst b/docs/source/pyutils.rst
index cff773d..d93505e 100644
--- a/docs/source/pyutils.rst
+++ b/docs/source/pyutils.rst
@@ -24,3 +24,5 @@ Python utilities
    utils.requests_auth
    tests.fixtures
    types.annexkey
+   types.archivist
+   types.enums


Test patch:
diff --git a/datalad_next/types/tests/test_annexkey.py b/datalad_next/types/tests/test_annexkey.py
index 80c68c0..a4e3878 100644
--- a/datalad_next/types/tests/test_annexkey.py
+++ b/datalad_next/types/tests/test_annexkey.py
@@ -1,9 +1,26 @@
+import pytest
+
 from ..annexkey import AnnexKey
 
 
 def test_annexkey():
     for key in (
         'MD5E-s792207360--985e680a221e47db05063a12b91d7d89.tar',
+        'SHA256E-s31390--f50d7ac4c6b9031379986bc362fcefb65f1e52621ce1708d537e740fefc59cc0.mp3',
+        'URL-s1899248--http&c%%ai.stanford.edu%,126nilsson%MLBOOK.pdf/URL-s1899248--http&c%%ai.stanford.edu%,126nilsson%MLBOOK.pdf',
     ):
         # round-tripping for any key must give same outcome
         assert key == str(AnnexKey.from_str(key))
+
+    # check that it can be used as a dict-key, i.e. is hashable
+    key = AnnexKey.from_str('MD5-s9--985e680a221e47db05063a12b91d7d89')
+    d = {key: 'some'}
+
+
+def test_annexkey_errors():
+    for wrong in (
+        'MD5E-985e680a221e47db05063a12b91d7d89.tar',
+        'MD5E-SUPRISE--985e680a221e47db05063a12b91d7d89.tar',
+    ):
+        with pytest.raises(ValueError):
+            AnnexKey.from_str(wrong)
diff --git a/datalad_next/types/tests/test_archivist.py b/datalad_next/types/tests/test_archivist.py
new file mode 100644
index 0000000..8f78163
--- /dev/null
+++ b/datalad_next/types/tests/test_archivist.py
@@ -0,0 +1,53 @@
+import pytest
+
+from ..annexkey import AnnexKey
+from ..archivist import ArchivistLocator
+from ..enums import ArchiveType
+
+some_key = 'MD5-s389--e9f624eb778e6f945771c543b6e9c7b2'
+
+
+def test_archivistlocator():
+    test_locator = \
+        f'dl+archive:{some_key}#path=dir/file.csv&size=234&atype=tar'
+
+    al = ArchivistLocator.from_str(test_locator)
+
+    assert al.akey == AnnexKey.from_str(some_key)
+    assert al.atype == ArchiveType.tar
+
+    # round trip
+    assert str(al) == test_locator
+
+    # type determination from key
+    assert ArchivistLocator.from_str(
+        'dl+archive:MD5E-s1--e9f624eb778e6f945771c543b6e9c7b2.tar#path=f.txt'
+    ).atype == ArchiveType.tar
+    assert ArchivistLocator.from_str(
+        'dl+archive:MD5E-s1--e9f624eb778e6f945771c543b6e9c7b2.zip#path=f.txt'
+    ).atype == ArchiveType.zip
+
+
+def test_archivistlocatori_errors():
+    for wrong in (
+        # no chance without prefix
+        'bogus',
+        # not just a prefix or some bogus properties
+        'dl+archive:',
+        'dl+archive:#',
+        'dl+archive:keything',
+        'dl+archive:#props',
+        'dl+archive:keything#props',
+        # a real key is required, but not sufficient
+        f'dl+archive:{some_key}#props',
+        # we require a member path, the rest is optional
+        f'dl+archive:{some_key}#size=123',
+        f'dl+archive:{some_key}#size=123&atype=tar',
+        # must be a proper POSIX path, relative, no ..
+        f'dl+archive:{some_key}#path=/dummy',
+        f'dl+archive:{some_key}#path=../dd',
+        # cannot work with unknown archive type
+        f'dl+archive:{some_key}#path=good&size=123&atype=eh!',
+    ):
+        with pytest.raises(ValueError):
+            ArchivistLocator.from_str(wrong)