Repository: OCR-D/core (python)

Issue:
ocrd process: add METS Server CLI option
We have `-U path` in the processor CLIs now, but not for `ocrd process` yet.

Pull request:
ocrd process via METS server option
fixes #1241

Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 46a9b83d..049f804b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,13 +5,6 @@ Versioned according to [Semantic Versioning](http://semver.org/).
 
 ## Unreleased
 
-## [2.66.1] - 2024-06-26
-
-
-Fixed:
-
-  * GHA Docker: build docker.io first, then tag ghcr.io
-
 ## [2.66.0] - 2024-06-07
 
 Fixed:
@@ -2099,9 +2092,8 @@ Fixed
 ## [0.0.1] - 2018-04-17
 
 Initial Release
-
+]
 <!-- link-labels -->
-[2.66.1]: ../../compare/v2.66.1..v2.66.0
 [2.66.0]: ../../compare/v2.66.0..v2.65.0
 [2.65.0]: ../../compare/v2.65.0..v2.64.1
 [2.64.1]: ../../compare/v2.64.1..v2.64.0
diff --git a/VERSION b/VERSION
index 64fb9e5e..3d6ac35b 100644
--- a/VERSION
+++ b/VERSION
@@ -1,1 +1,1 @@
-2.66.1
+2.66.0
diff --git a/src/ocrd/cli/process.py b/src/ocrd/cli/process.py
index b71b74d0..9dcd5626 100644
--- a/src/ocrd/cli/process.py
+++ b/src/ocrd/cli/process.py
@@ -19,14 +19,15 @@ from ..decorators import ocrd_loglevel
 @click.command('process')
 @ocrd_loglevel
 @click.option('-m', '--mets', help="METS to process", default=DEFAULT_METS_BASENAME)
+@click.option('-U', '--mets-server-url', help="TCP host URI or UDS path of METS server")
 @click.option('-g', '--page-id', help="ID(s) of the pages to process")
 @click.option('--overwrite', is_flag=True, default=False, help="Remove output pages/images if they already exist")
 @click.argument('tasks', nargs=-1, required=True)
-def process_cli(log_level, mets, page_id, tasks, overwrite):
+def process_cli(log_level, mets, mets_server_url, page_id, tasks, overwrite):
     """
     Process a series of tasks
     """
     initLogging()
     log = getLogger('ocrd.cli.process')
-    run_tasks(mets, log_level, page_id, tasks, overwrite)
+    run_tasks(mets, log_level, page_id, tasks, overwrite=overwrite, mets_server_url=mets_server_url)
     log.info("Finished")
diff --git a/src/ocrd/cli/workspace.py b/src/ocrd/cli/workspace.py
index 52f48b7c..4c262ec4 100644
--- a/src/ocrd/cli/workspace.py
+++ b/src/ocrd/cli/workspace.py
@@ -48,7 +48,7 @@ pass_workspace = click.make_pass_decorator(WorkspaceCtx)
 @click.option('-d', '--directory', envvar='WORKSPACE_DIR', type=click.Path(file_okay=False), metavar='WORKSPACE_DIR', help='Changes the workspace folder location [default: METS_URL directory or .]"')
 @click.option('-M', '--mets-basename', default=None, help='METS file basename. Deprecated, use --mets/--directory')
 @click.option('-m', '--mets', default=None, help='The path/URL of the METS file [default: WORKSPACE_DIR/mets.xml]', metavar="METS_URL")
-@click.option('-U', '--mets-server-url', 'mets_server_url', help="TCP host of METS server")
+@click.option('-U', '--mets-server-url', 'mets_server_url', help="TCP host URI or UDS path of METS server")
 @click.option('--backup', default=False, help="Backup mets.xml whenever it is saved.", is_flag=True)
 @click.pass_context
 def workspace_cli(ctx, directory, mets, mets_basename, mets_server_url, backup):
diff --git a/src/ocrd/resolver.py b/src/ocrd/resolver.py
index 4b8fe6b2..fa98c82d 100644
--- a/src/ocrd/resolver.py
+++ b/src/ocrd/resolver.py
@@ -174,6 +174,9 @@ class Resolver():
                 By default existing ``mets.xml`` will raise an exception.
             download (boolean, False): Whether to also download all the files referenced by the METS
             src_baseurl (string, None): Base URL for resolving relative file locations
+            mets_server_url (string, None): URI of TCP or local path of UDS for METS server handling
+                the `OcrdMets` of the workspace. By default the METS will be read from and written to
+                the filesystem directly.
             **kwargs (): Passed on to ``OcrdMets.find_files`` if download == True
 
         Download (clone) :py:attr:`mets_url` to ``mets.xml`` in :py:attr:`dst_dir`, unless 
diff --git a/src/ocrd/task_sequence.py b/src/ocrd/task_sequence.py
index da691fbc..85e30b17 100644
--- a/src/ocrd/task_sequence.py
+++ b/src/ocrd/task_sequence.py
@@ -115,9 +115,11 @@ def validate_tasks(tasks, workspace, page_id=None, overwrite=False):
     return report
 
 
-def run_tasks(mets, log_level, page_id, task_strs, overwrite=False):
+def run_tasks(mets, log_level, page_id, task_strs, overwrite=False, mets_server_url=None):
     resolver = Resolver()
-    workspace = resolver.workspace_from_url(mets)
+    workdir, mets, basename, _ = resolver.resolve_mets_arguments(None, mets, None)
+    workspace = resolver.workspace_from_url(mets, workdir, mets_basename=basename,
+                                            mets_server_url=mets_server_url)
     log = getLogger('ocrd.task_sequence.run_tasks')
     tasks = [ProcessorTask.parse(task_str) for task_str in task_strs]
 
@@ -139,7 +141,8 @@ def run_tasks(mets, log_level, page_id, task_strs, overwrite=False):
             overwrite=overwrite,
             input_file_grp=','.join(task.input_file_grps),
             output_file_grp=','.join(task.output_file_grps),
-            parameter=json.dumps(task.parameters)
+            parameter=json.dumps(task.parameters),
+            mets_server_url=mets_server_url
         )
 
         # check return code
@@ -149,7 +152,8 @@ def run_tasks(mets, log_level, page_id, task_strs, overwrite=False):
         log.info("Finished processing task '%s'", task)
 
         # reload mets
-        workspace.reload_mets()
+        if mets_server_url is None:
+            workspace.reload_mets()
 
         # check output file groups are in mets
         for output_file_grp in task.output_file_grps:
diff --git a/src/ocrd/workspace.py b/src/ocrd/workspace.py
index 41ea8e90..2fb3dd28 100644
--- a/src/ocrd/workspace.py
+++ b/src/ocrd/workspace.py
@@ -64,13 +64,17 @@ class Workspace():
     :py:class:`ocrd.resolver.Resolver`.
 
     Args:
-
-        directory (string) : Filesystem folder to work in
+        resolver (:py:class:`ocrd.Resolver`) : `Resolver` instance
+        directory (string) : Filesystem path to work in
         mets (:py:class:`ocrd_models.ocrd_mets.OcrdMets`) : `OcrdMets` representing this workspace.
-            Loaded from `'mets.xml'` if `None`.
-        mets_basename (string) : Basename of the METS XML file. Default: Last URL segment of the mets_url.
-        overwrite_mode (boolean) : Whether to force add operations on this workspace globally
-        baseurl (string) : Base URL to prefix to relative URL.
+            If `None`, then loaded from ``directory``/``mets_basename``
+            or delegated to ``mets_server_url``.
+        mets_basename (string, mets.xml) : Basename of the METS XML file in the workspace directory.
+        mets_server_url (string, None) : URI of TCP or local path of UDS for METS server handling the
+            `OcrdMets` of this workspace. If `None`, then the METS will be read from and written to
+            the filesystem directly.
+        baseurl (string, None) : Base URL to prefix to relative URL.
+        overwrite_mode (boolean, False) : Whether to force add operations on this workspace globally
     """
 
     def __init__(
diff --git a/src/ocrd_utils/config.py b/src/ocrd_utils/config.py
index 5c99800c..b3a3e953 100644
--- a/src/ocrd_utils/config.py
+++ b/src/ocrd_utils/config.py
@@ -178,12 +178,12 @@ config.add("HOME",
     default=(True, lambda: Path.home()))
 
 config.add("XDG_DATA_HOME",
-    description="Directory to look for `./ocrd/resources.yml` (i.e. `ocrd resmgr` user database)",
+    description="Directory to look for `./ocrd-resources/*` (i.e. `ocrd resmgr` data location)",
     parser=lambda val: Path(val),
     default=(True, lambda: Path(config.HOME, '.local/share')))
 
 config.add("XDG_CONFIG_HOME",
-    description="Directory to look for `./ocrd-resources/*` (i.e. `ocrd resmgr` data location)",
+    description="Directory to look for `./ocrd/resources.yml` (i.e. `ocrd resmgr` user database)",
     parser=lambda val: Path(val),
     default=(True, lambda: Path(config.HOME, '.config')))
 


Test patch:
diff --git a/tests/cli/test_process.py b/tests/cli/test_process.py
index d0e5dc21..32fc02fe 100644
--- a/tests/cli/test_process.py
+++ b/tests/cli/test_process.py
@@ -1,16 +1,59 @@
+from os.path import exists
+from os import remove, getcwd
+from time import sleep
+from contextlib import ExitStack
+from multiprocessing import Process, set_start_method
+# necessary for macos
+set_start_method("fork")
+
+from ocrd import Resolver, Workspace, OcrdMetsServer
 from ocrd.cli import process_cli
-from ocrd_utils import pushd_popd, disableLogging
+from ocrd_utils import pushd_popd
 
 from tests.base import CapturingTestCase as TestCase, main, assets, copy_of_directory
 
-class TestLogCli(TestCase):
+class TestCli(TestCase):
+
+    def setUp(self):
+        super().setUp()
+        # make sure we get an isolated temporary copy of the testdata each time
+        # as long as we are not using pytest but unittest, we need to manage contexts
+        # (enterContext is only supported starting with py311)
+        with ExitStack() as stack:
+            self.workdir = stack.enter_context(copy_of_directory(assets.path_to('kant_aufklaerung_1784/data')))
+            stack.enter_context(pushd_popd(self.workdir))
+            self.addCleanup(stack.pop_all().close)
 
     def test_cli_process_smoke(self):
-        disableLogging()
-        with copy_of_directory(assets.path_to('kant_aufklaerung_1784/data')) as wsdir:
-            with pushd_popd(wsdir):
-                with self.assertRaisesRegex(Exception, "Executable not found in PATH: ocrd-foo"):
-                    self.invoke_cli(process_cli, ['foo'])
+        with self.assertRaisesRegex(Exception, "Executable not found in PATH: ocrd-foo"):
+            self.invoke_cli(process_cli, ['foo'])
+
+    def test_cli_process_dummy(self):
+        code, out, err = self.invoke_cli(process_cli, ['dummy -I OCR-D-GT-PAGE -O OCR-D-DUMMY'])
+        print(code, out, err)
+        self.assertFalse(code)
+        self.assertTrue(exists('OCR-D-DUMMY'))
+
+    def test_cli_process_mets_server(self):
+        # stolen from test_mets_server.fixture_start_mets_server ...
+        def _start_mets_server(*args, **kwargs):
+            mets_server = OcrdMetsServer(*args, **kwargs)
+            mets_server.startup()
+        if exists('mets.sock'):
+            remove('mets.sock')
+        ws = Workspace(Resolver(), getcwd())
+        p = Process(target=_start_mets_server, kwargs={'workspace': ws, 'url': 'mets.sock'})
+        p.daemon = True
+        p.start()
+        sleep(1)  # sleep to start up server
+        self.assertTrue(exists('mets.sock'))
+        code, out, err = self.invoke_cli(process_cli, ['-U', 'mets.sock', 'dummy -I OCR-D-GT-PAGE -O OCR-D-DUMMY'])
+        print(code, out, err)
+        self.assertFalse(code)
+        self.assertTrue(exists('OCR-D-DUMMY'))
+        p.terminate()
+        ws.reload_mets()
+        self.assertIn('OCR-D-DUMMY', ws.mets.file_groups)
 
 if __name__ == '__main__':
     main(__file__)
diff --git a/tests/test_mets_server.py b/tests/test_mets_server.py
index da0b9589..58ff6e2a 100644
--- a/tests/test_mets_server.py
+++ b/tests/test_mets_server.py
@@ -6,8 +6,11 @@ from tests.base import assets
 
 from itertools import repeat
 from multiprocessing import Process, Pool, Pipe, set_start_method
-# necessary for macos
-set_start_method("fork")
+try:
+    # necessary for macos
+    set_start_method("fork")
+except RuntimeError:
+    pass
 from shutil import rmtree, copytree
 from os import remove, stat as os_stat
 from os.path import exists
diff --git a/tests/test_resource_manager.py b/tests/test_resource_manager.py
index 221b0a3a..e76073e3 100644
--- a/tests/test_resource_manager.py
+++ b/tests/test_resource_manager.py
@@ -2,8 +2,11 @@ import os
 from pathlib import Path
 # import pdb
 
-from ocrd.resource_manager import OcrdResourceManager
-from ocrd_utils import config
+# avoid importing early (before mocking environ)
+# because ocrd_utils.config module-level init is crucial:
+#from ocrd.resource_manager import OcrdResourceManager
+#from ocrd_utils import config
+
 from ocrd_utils.os import get_ocrd_tool_json
 
 from pytest import raises, fixture
@@ -23,14 +26,18 @@ def test_resources_manager_config_default(monkeypatch, tmp_path):
     monkeypatch.setenv('HOME', str(tmp_path))
     if 'XDG_CONFIG_HOME' in os.environ:
         monkeypatch.delenv('XDG_CONFIG_HOME', raising=False)
+    if 'XDG_DATA_HOME' in os.environ:
+        monkeypatch.delenv('XDG_DATA_HOME', raising=False)
 
     # act
+    from ocrd.resource_manager import OcrdResourceManager
     mgr = OcrdResourceManager()
 
     # assert
     default_config_dir = os.path.join(os.environ['HOME'], '.config', 'ocrd')
     f = Path(default_config_dir) / CONST_RESOURCE_YML
     assert os.environ['HOME'] == str(tmp_path)
+    from ocrd_utils import config
     assert config.HOME == tmp_path
     assert Path.home() == tmp_path
     assert f == mgr.user_list
@@ -54,6 +61,7 @@ def test_resources_manager_from_environment(tmp_path, monkeypatch):
     monkeypatch.setenv('HOME', str(tmp_path))
 
     # act
+    from ocrd.resource_manager import OcrdResourceManager
     mgr = OcrdResourceManager()
 
     # assert
@@ -72,6 +80,7 @@ def test_resources_manager_from_environment(tmp_path, monkeypatch):
 def test_resources_manager_config_explicite(tmp_path):
 
     # act
+    from ocrd.resource_manager import OcrdResourceManager
     mgr = OcrdResourceManager(xdg_config_home=str(tmp_path / 'config'), xdg_data_home=str(tmp_path / 'data'))
 
     # assert
@@ -88,6 +97,7 @@ def test_resources_manager_config_explicite(tmp_path):
 def test_resources_manager_config_explicit_invalid(tmp_path):
 
     # act
+    from ocrd.resource_manager import OcrdResourceManager
     (tmp_path / 'ocrd').mkdir()
     (tmp_path / 'ocrd' / CONST_RESOURCE_YML).write_text('::INVALID::')
 
@@ -98,6 +108,7 @@ def test_resources_manager_config_explicit_invalid(tmp_path):
 def test_find_resources(tmp_path):
 
     # act
+    from ocrd.resource_manager import OcrdResourceManager
     f = tmp_path / 'ocrd-foo' / 'foo.bar'
     f.parent.mkdir()
     f.write_text('foobar')
@@ -110,6 +121,7 @@ def test_find_resources(tmp_path):
     assert 'ocrd-foo' in [x for x, _ in mgr.list_available(url='http://foo/bar')]
 
 def test_parameter_usage(tmp_path):
+    from ocrd.resource_manager import OcrdResourceManager
     mgr = OcrdResourceManager(xdg_config_home=tmp_path)
     assert mgr.parameter_usage('foo.bar') == 'foo.bar'
     assert mgr.parameter_usage('foo.bar', 'without-extension') == 'foo'
@@ -117,21 +129,25 @@ def test_parameter_usage(tmp_path):
         mgr.parameter_usage('foo.bar', 'baz')
 
 def test_default_resource_dir(tmp_path):
+    from ocrd.resource_manager import OcrdResourceManager
     mgr = OcrdResourceManager(xdg_data_home=tmp_path)
     assert mgr.xdg_config_home != mgr.xdg_data_home
     assert mgr.default_resource_dir == str(mgr.xdg_data_home / 'ocrd-resources')
 
 def test_list_available0(tmp_path):
+    from ocrd.resource_manager import OcrdResourceManager
     mgr = OcrdResourceManager(xdg_data_home=tmp_path)
     res = mgr.list_available()
     assert len(res) > 0
 
 def test_list_available_with_unknown_executable(tmp_path):
+    from ocrd.resource_manager import OcrdResourceManager
     mgr = OcrdResourceManager(xdg_data_home=tmp_path)
     res = mgr.list_available(executable="ocrd-non-existing-processor")
     assert len(res[0][1]) == 0
 
 def test_date_as_string(tmp_path):
+    from ocrd.resource_manager import OcrdResourceManager
     mgr = OcrdResourceManager(xdg_data_home=tmp_path)
     test_list = tmp_path / 'test-list.yml'
     with open(test_list, 'w', encoding='utf-8') as fout:
@@ -148,6 +164,7 @@ def test_date_as_string(tmp_path):
     mgr.list_available(executable='ocrd-eynollah-segment')
 
 def test_download_archive(tmp_path):
+    from ocrd.resource_manager import OcrdResourceManager
     mgr = OcrdResourceManager(xdg_data_home=tmp_path)
     for archive_type in ('.zip', '.tar.gz', '.tar.xz'):
         mgr.download(
diff --git a/tests/utils/test_os.py b/tests/utils/test_os.py
index a651ef60..2bdf33d2 100644
--- a/tests/utils/test_os.py
+++ b/tests/utils/test_os.py
@@ -11,6 +11,7 @@ from ocrd_utils.os import (
     redirect_stderr_and_stdout_to_file,
     guess_media_type,
 )
+from ocrd_utils import config
 
 class TestOsUtils(TestCase):
 
@@ -34,7 +35,7 @@ class TestOsUtils(TestCase):
         self.assertEqual(cands, [join(x, fname) for x in [
             dehomify(join(getcwd())),
             dehomify(self.tempdir_path),
-            '$HOME/.local/share/ocrd-resources/ocrd-dummy',
+            dehomify(join(config.XDG_DATA_HOME, 'ocrd-resources', 'ocrd-dummy')),
             '/usr/local/share/ocrd-resources/ocrd-dummy',
         ]])