Repository: datalad/datalad-next

Issue:
Enable HTTP header customization for URLOperations
this would be the patch:

```diff
diff --git a/datalad_next/url_operations/http.py b/datalad_next/url_operations/http.py
index 8700b7b..2d30c64 100644
--- a/datalad_next/url_operations/http.py
+++ b/datalad_next/url_operations/http.py
@@ -36,13 +36,24 @@ class HttpUrlOperations(UrlOperations):
     authentication challenges.
     """
 
-    _headers = {
-        'user-agent': user_agent('datalad', datalad.__version__),
-    }
+    def __init__(self, cfg=None, headers=None):
+        """
+        Parameters
+        ----------
+        cfg: ConfigManager, optional
+          A config manager instance that is consulted for any configuration
+          filesystem configuration individual handlers may support.
+        """
+        super().__init__(cfg=cfg)
+        self._headers = {
+            'user-agent': user_agent('datalad', datalad.__version__),
+        }
+        if headers:
+            self._headers.update(headers)
 
     def get_headers(self, headers: Dict | None = None) -> Dict:
         # start with the default
-        hdrs = dict(HttpUrlOperations._headers)
+        hdrs = dict(self._headers)
         if headers is not None:
             hdrs.update(headers)
         return hdrs
```

With the headers exposed in the constructor, we can use handler customization (that still needs to be documented #335) and supply custom headers like so:


```
datalad \
  -c datalad.url-handler.https://fz-juelich.sciebo.de/public.php/webdav.class=datalad_next.url_operations.http.HttpUrlOperations \
   -c 'datalad.url-handler.https://fz-juelich.sciebo.de/public.php/webdav.kwargs={"headers": {"X-Requested-With": "XMLHttpRequest"}}' \
   download \
   'https://fz-juelich.sciebo.de/public.php/webdav/king/hubert-neufeld-j-udI4zim2E-unsplash.jpg'
```

Comments:

Patch:
diff --git a/changelog.d/20230505_074522_michael.hanke_httphdrs.md b/changelog.d/20230505_074522_michael.hanke_httphdrs.md
new file mode 100644
index 0000000..60f9534
--- /dev/null
+++ b/changelog.d/20230505_074522_michael.hanke_httphdrs.md
@@ -0,0 +1,7 @@
+### 💫 Enhancements and new features
+
+- The `HttpUrlOperations` handler now supports custom HTTP headers.
+  This makes it possible to define custom handlers in configuration
+  that include such header customization, for example to send
+  custom secret or session IDs.
+  Fixes https://github.com/datalad/datalad-next/issues/336 (by @mih)
diff --git a/datalad_next/url_operations/http.py b/datalad_next/url_operations/http.py
index 8700b7b..6b964aa 100644
--- a/datalad_next/url_operations/http.py
+++ b/datalad_next/url_operations/http.py
@@ -36,13 +36,28 @@ class HttpUrlOperations(UrlOperations):
     authentication challenges.
     """
 
-    _headers = {
-        'user-agent': user_agent('datalad', datalad.__version__),
-    }
+    def __init__(self, cfg=None, headers: Dict | None = None):
+        """
+        Parameters
+        ----------
+        cfg: ConfigManager, optional
+          A config manager instance that is consulted for any configuration
+          filesystem configuration individual handlers may support.
+        headers: dict, optional
+          Additional or alternative headers to add to a request. The default
+          headers contain a ``user-agent`` declaration. Any headers provided
+          here override corresponding defaults.
+        """
+        super().__init__(cfg=cfg)
+        self._headers = {
+            'user-agent': user_agent('datalad', datalad.__version__),
+        }
+        if headers:
+            self._headers.update(headers)
 
     def get_headers(self, headers: Dict | None = None) -> Dict:
         # start with the default
-        hdrs = dict(HttpUrlOperations._headers)
+        hdrs = dict(self._headers)
         if headers is not None:
             hdrs.update(headers)
         return hdrs


Test patch:
diff --git a/datalad_next/url_operations/tests/test_http.py b/datalad_next/url_operations/tests/test_http.py
index 62b1bc1..817d756 100644
--- a/datalad_next/url_operations/tests/test_http.py
+++ b/datalad_next/url_operations/tests/test_http.py
@@ -1,6 +1,7 @@
 from pathlib import Path
 import pytest
 
+from ..any import AnyUrlOperations
 from ..http import (
     HttpUrlOperations,
     UrlOperationsRemoteError,
@@ -42,3 +43,17 @@ def test_http_url_operations(credman, httpbin, tmp_path):
         ops.stat(f'{hbsurl}/status/404')
     with pytest.raises(UrlOperationsResourceUnknown):
         ops.download(f'{hbsurl}/status/404', tmp_path / 'dontmatter')
+
+
+def test_custom_http_headers_via_config(datalad_cfg):
+    for k, v in (
+            ('datalad.url-handler.http.*.class',
+             'datalad_next.url_operations.http.HttpUrlOperations'),
+            ('datalad.url-handler.http.*.kwargs',
+             '{"headers": {"X-Funky": "Stuff"}}'),
+    ):
+        datalad_cfg.set(k, v, scope='global', reload=False)
+    datalad_cfg.reload()
+    auo = AnyUrlOperations()
+    huo = auo._get_handler(f'http://example.com')
+    assert huo._headers['X-Funky'] == 'Stuff'