Repository: radiocosmology/alpenhorn

Issue:
Resuscitate the docker test thing?
There's a fairly robust test suite these days using `pyfakefs` and in-memory sqlite databases to provide a test arena, so docker isn't as necessary as it once was, but the docker image could still be handy to provide a space for interactive testing/development.

Discussion:

Patch:
diff --git a/alpenhorn/cli/node/init.py b/alpenhorn/cli/node/init.py
index 0c4c2bf..5a31151 100644
--- a/alpenhorn/cli/node/init.py
+++ b/alpenhorn/cli/node/init.py
@@ -30,4 +30,4 @@ def init(name):
         # Add request
         ArchiveFileImportRequest.create(node=node, path="ALPENHORN_NODE")
 
-    echo('Requested initialisation of Node "{name}".')
+    echo(f'Requested initialisation of Node "{name}".')
diff --git a/alpenhorn/cli/node/show.py b/alpenhorn/cli/node/show.py
index 3654f73..4c6e8e1 100644
--- a/alpenhorn/cli/node/show.py
+++ b/alpenhorn/cli/node/show.py
@@ -10,6 +10,7 @@ from ...common.util import pretty_bytes
 from ...db import (
     ArchiveFile,
     ArchiveFileCopyRequest,
+    ArchiveFileImportRequest,
     StorageGroup,
     StorageTransferAction,
 )
@@ -26,11 +27,14 @@ from .stats import get_stats
     help="Show post-transfer auto-actions affecting this group.",
 )
 @cli_option("all_", help="Show all additional data.")
+@click.option(
+    "--imports", is_flag=True, help="Show pending import requests for the node."
+)
 @click.option("--stats", is_flag=True, help="Show usage stats of the node.")
 @click.option(
     "--transfers", is_flag=True, help="Show pending transfers out from the node."
 )
-def show(name, actions, all_, stats, transfers):
+def show(name, actions, all_, imports, stats, transfers):
     """Show details of a Storage Node.
 
     Shows details of the Storage Node named NODE.
@@ -38,6 +42,7 @@ def show(name, actions, all_, stats, transfers):
 
     if all_:
         actions = True
+        imports = True
         stats = True
         transfers = True
 
@@ -111,6 +116,42 @@ def show(name, actions, all_, stats, transfers):
         echo("     Total Size: " + stats["size"])
         echo("          Usage: " + stats["percent"].lstrip() + "%")
 
+    # List imports, if requested
+    if imports:
+        echo("\nPending import requests:\n")
+
+        query = (
+            ArchiveFileImportRequest.select()
+            .where(
+                ArchiveFileImportRequest.node == node,
+                ArchiveFileImportRequest.completed == 0,
+            )
+            .order_by(ArchiveFileImportRequest.timestamp)
+        )
+
+        inits = []
+        reqs = []
+        for req in query.execute():
+            if req.path == "ALPENHORN_NODE":
+                # Handle node init requests separately
+                inits.append(("[Node Init]", "-", "-", req.timestamp))
+            else:
+                reqs.append(
+                    (
+                        req.path,
+                        "Yes" if req.recurse else "No",
+                        "Yes" if req.register else "No",
+                        req.timestamp,
+                    )
+                )
+
+        # The node init requests are always put at the top of the table
+        echo(
+            tabulate(
+                inits + reqs, headers=["Path", "Scan", "Register New", "Request Time"]
+            )
+        )
+
     # List transfers, if requested
     if transfers:
         echo("\nPending outbound transfers:\n")
diff --git a/alpenhorn/daemon/auto_import.py b/alpenhorn/daemon/auto_import.py
index caf62fc..84f9700 100644
--- a/alpenhorn/daemon/auto_import.py
+++ b/alpenhorn/daemon/auto_import.py
@@ -14,7 +14,6 @@ from typing import TYPE_CHECKING
 import peewee as pw
 from watchdog.events import FileSystemEventHandler
 
-from .. import db
 from ..common import config, extensions, metrics
 from ..common.util import invalid_import_path
 from ..db import (
@@ -96,6 +95,8 @@ def import_file(
         True if we should register new files (files without ArchiveFile records),
         or False to only import already registered files.
     req : ArchiveFileImportRequest or None
+        This will be None when the import request was triggered by the auto-import
+        watchdog.  Otherwise, this is the import request which is being handled.
         If not None, req will be marked as complete if the import isn't skipped.
     """
 
@@ -171,7 +172,7 @@ def _import_file(
     # Skip non-files
     fullpath = pathlib.Path(node.db.root).joinpath(path)
     if fullpath.is_symlink() or not fullpath.is_file():
-        log.debug(f'Not importing "{path}": not a file.')
+        log.info(f'Not importing "{path}": not a file.')
         import_request_done(req, "invalid")
         return
 
@@ -179,14 +180,14 @@ def _import_file(
 
     # Wait for file to become ready
     while not node.io.ready_path(path):
-        log.debug(
-            f'Path "{path}" not ready for I/O during import.  Waiting 60 seconds.'
+        log.info(
+            f'Path "{path}" not ready for I/O during import.  Waiting 600 seconds.'
         )
         yield 600  # wait 600 seconds and re-queue
 
     # Skip the file if there is still a lock on it.
     if node.io.locked(path):
-        log.debug(f'Skipping "{path}": locked.')
+        log.info(f'Skipping "{path}": locked.')
         # In this case we don't complete the import request.
         return
 
@@ -217,67 +218,84 @@ def _import_file(
         import_request_done(req, "duplicate")
         return
 
-    # Begin a transaction
-    with db.database_proxy.atomic():
-        # Add the acqusition, if necessary
-        try:
-            acq = ArchiveAcq.get(ArchiveAcq.name == acq_name)
-            new_acq = None
-            log.debug(f'Acquisition "{acq_name}" already in DB.')
-        except pw.DoesNotExist:
-            if register:
+    # Add the acqusition, if necessary
+    try:
+        acq = ArchiveAcq.get(ArchiveAcq.name == acq_name)
+        new_acq = None
+        log.debug(f'Acquisition "{acq_name}" already in DB.')
+    except pw.DoesNotExist:
+        if register:
+            try:
                 acq = ArchiveAcq.create(name=acq_name)
                 new_acq = acq
                 log.info(f'Acquisition "{acq_name}" added to DB.')
-            else:
-                log.info(f'Not importing unregistered acquistion: "{acq_name}".')
-                import_request_done(req, "unregistered")
-                return
-
-        # Add the file, if necessary.
-        try:
-            file_ = ArchiveFile.get(
-                ArchiveFile.name == file_name, ArchiveFile.acq == acq
-            )
-            new_file = None
-            log.debug(f'File "{path}" already in DB.')
-        except pw.DoesNotExist:
-            if register:
-                log.debug(f'Computing md5sum of "{path}".')
-                md5sum = node.io.md5(acq_name, file_name)
-                size_b = node.io.filesize(path)
+            except pw.IntegrityError:
+                # i.e. record was created by someone else between the first get and
+                #      the subsequent create
+                acq = ArchiveAcq.get(ArchiveAcq.name == acq_name)
+                new_acq = None
+                log.debug(f'Acquisition "{acq_name}" already in DB.')
+        else:
+            log.info(f'Not importing unregistered acquistion: "{acq_name}".')
+            import_request_done(req, "unregistered")
+            return
 
+    # Add the file, if necessary.
+    try:
+        file_ = ArchiveFile.get(ArchiveFile.name == file_name, ArchiveFile.acq == acq)
+        new_file = None
+        log.debug(f'File "{path}" already in DB.')
+    except pw.DoesNotExist:
+        if register:
+            log.debug(f'Computing md5sum of "{path}".')
+            md5sum = node.io.md5(acq_name, file_name)
+            size_b = node.io.filesize(path)
+
+            try:
                 file_ = ArchiveFile.create(
                     acq=acq, name=file_name, size_b=size_b, md5sum=md5sum
                 )
                 new_file = file_
                 log.info(f'File "{path}" added to DB.')
-            else:
-                log.info(f'Not importing unregistered file: "{path}".')
-                import_request_done(req, "unregistered")
-                return
+            except pw.IntegrityError:
+                # i.e. record was created by someone else between the first get and
+                #      the subsequent create
+                #
+                # We _cannot_ assume that this is due to another worker in _this_
+                # daemon: there may be an import for the same file happening at the
+                # same time on another host, as unlikely as that seems.
+                file_ = ArchiveFile.get(
+                    ArchiveFile.name == file_name, ArchiveFile.acq == acq
+                )
+                new_file = None
+                log.debug(f'File "{path}" already in DB.')
+        else:
+            log.info(f'Not importing unregistered file: "{path}".')
+            import_request_done(req, "unregistered")
+            return
 
+    try:
+        copy = ArchiveFileCopy.get(file=file_, node=node.db)
+        # If we're importing a file that's missing (has_file == N but
+        # wants_file == Y), set has_file='M' to trigger a integrity check.
+        # If it's recorded as having been properly removed, though, just
+        # set it to 'Y' and assume it's good now.
+        if copy.wants_file == "Y":
+            copy.has_file = "M"
+            log.warning(
+                f'Imported missing file "{path}" on node {node.name}.'
+                "  Marking suspect."
+            )
+        else:
+            copy.has_file = "Y"
+            copy.wants_file = "Y"
+            log.info(f'Imported file copy "{path}" on node "{node.name}".')
+        copy.ready = True
+        copy.last_update = utcnow()
+        copy.save()
+    except pw.DoesNotExist:
+        # No existing file copy; create a new one.
         try:
-            copy = ArchiveFileCopy.get(file=file_, node=node.db)
-            # If we're importing a file that's missing (has_file == N but
-            # wants_file == Y), set has_file='M' to trigger a integrity check.
-            # If it's recorded as having been properly removed, though, just
-            # set it to 'Y' and assume it's good now.
-            if copy.wants_file == "Y":
-                copy.has_file = "M"
-                log.warning(
-                    f'Imported missing file "{path}" on node {node.name}.'
-                    "  Marking suspect."
-                )
-            else:
-                copy.has_file = "Y"
-                copy.wants_file = "Y"
-                log.info(f'Imported file copy "{path}" on node "{node.name}".')
-            copy.ready = True
-            copy.last_update = utcnow()
-            copy.save()
-        except pw.DoesNotExist:
-            # No existing file copy; create a new one.
             copy = ArchiveFileCopy.create(
                 file=file_,
                 node=node.db,
@@ -288,6 +306,18 @@ def _import_file(
                 last_update=utcnow(),
             )
             log.info(f'Imported file copy "{path}" on node "{node.name}".')
+        except pw.IntegrityError:
+            log.debug("ArchiveFileCopy created by another worker!")
+            # The ArchiveFileCopy record has been created by someone else
+            # between our initial .get() and the subsequent .create().
+            #
+            # In this case, we assume another worker from _this_ daemon
+            # has just imported the file, likely due to multiple idential
+            # import requests, so just mark the request we're working on as
+            # completed and let the other worker deal with fixing up the
+            # copy and doing all the post-import stuff
+            import_request_done(req, "duplicate")
+            return
 
     import_request_done(req, "success")
 
@@ -316,32 +346,39 @@ class RegisterFile(FileSystemEventHandler):
         The task queue.  Import tasks will be submitted to this queue.
     """
 
+    def _is_lock_file(self, path):
+        """Returns True if event does not refer to a lock file."""
+        basename = pathlib.PurePath(path).name
+        return basename[0] == "." and basename[-5:] == ".lock"
+
     def __init__(self, node: UpdateableNode, queue: FairMultiFIFOQueue) -> None:
         self.node = node
         self.queue = queue
         super().__init__()
 
     def on_created(self, event):
-        import_file(self.node, self.queue, pathlib.PurePath(event.src_path), True, None)
-        return
-
-    def on_modified(self, event):
-        import_file(self.node, self.queue, pathlib.PurePath(event.src_path), True, None)
+        if not event.is_directory and not self._is_lock_file(event.src_path):
+            import_file(
+                self.node, self.queue, pathlib.PurePath(event.src_path), True, None
+            )
         return
 
     def on_moved(self, event):
-        import_file(self.node, self.queue, pathlib.PurePath(event.src_path), True, None)
+        if not event.is_directory and not self._is_lock_file(event.dest_path):
+            import_file(
+                self.node, self.queue, pathlib.PurePath(event.dest_path), True, None
+            )
         return
 
     def on_deleted(self, event):
         # For lockfiles: ensure that the file that was locked is added: it is
         # possible that the watchdog notices that a file has been closed before the
         # lockfile is deleted.
-        path = pathlib.Path(event.src_path)
-        basename = path.name
-        if basename[0] == "." and basename[-5:] == ".lock":
-            basename = basename[1:-5]
-            import_file(self.node, self.queue, path.with_name(basename), True, None)
+        if not event.is_directory and self._is_lock_file(event.src_path):
+            path = pathlib.Path(event.src_path)
+            import_file(
+                self.node, self.queue, path.with_name(path.name[1:-5]), True, None
+            )
 
 
 # Routines to control the filesystem watchdogs.
diff --git a/demo/Dockerfile.alpenhorn b/demo/Dockerfile.alpenhorn
index 8719213..05d10a1 100644
--- a/demo/Dockerfile.alpenhorn
+++ b/demo/Dockerfile.alpenhorn
@@ -2,7 +2,7 @@
 ## transfers)
 
 # Use an official Python runtime as a base image
-FROM python:3.11
+FROM python:latest
 
 LABEL maintainer="D. V. Wiebe <dvw@phas.ubc.ca>"
 
diff --git a/demo/alpenhorn.conf b/demo/alpenhorn.conf
index bf609a1..dc68923 100644
--- a/demo/alpenhorn.conf
+++ b/demo/alpenhorn.conf
@@ -1,9 +1,10 @@
 database:
-    url: mysql://root@db/alpenhorn_db
+    url: mysql://root@alpdb/alpenhorn_db
 
-service:
-    update_interval: 2
-    auto_import_interval: 2
+daemon:
+    update_interval: 10
+    auto_import_interval: 10
+    num_workers: 2
 
 extensions:
     - pattern_importer
diff --git a/demo/data/host_1/2017/03/21/acq_xy1_45678901T000000Z_inst_zab/acq_data/x_123_1_data/raw/acq_123_1.zxc b/demo/data/host_1/2017/03/21/acq_xy1_45678901T000000Z_inst_zab/acq_data/x_123_1_data/raw/acq_123_1.zxc
deleted file mode 100644
index b9a5c8d..0000000
--- a/demo/data/host_1/2017/03/21/acq_xy1_45678901T000000Z_inst_zab/acq_data/x_123_1_data/raw/acq_123_1.zxc
+++ /dev/null
@@ -1,1 +0,0 @@
-I'm a ZXC FRB.
diff --git a/demo/data/host_1/2017/03/21/acq_xy1_45678901T000000Z_inst_zab/summary.txt b/demo/data/host_1/2017/03/21/acq_xy1_45678901T000000Z_inst_zab/summary.txt
deleted file mode 100644
index e69de29..0000000
diff --git a/demo/data/host_1/ALPENHORN_NODE b/demo/data/host_1/ALPENHORN_NODE
deleted file mode 100644
index 0edb0d1..0000000
--- a/demo/data/host_1/ALPENHORN_NODE
+++ /dev/null
@@ -1,1 +0,0 @@
-node_1
\ No newline at end of file
diff --git a/demo/data/host_2/ALPENHORN_NODE b/demo/data/host_2/ALPENHORN_NODE
deleted file mode 100644
index 6477c07..0000000
--- a/demo/data/host_2/ALPENHORN_NODE
+++ /dev/null
@@ -1,1 +0,0 @@
-node_2
\ No newline at end of file
diff --git a/demo/data/host_3/ALPENHORN_NODE b/demo/data/host_3/ALPENHORN_NODE
deleted file mode 100644
index 2d0f52c..0000000
--- a/demo/data/host_3/ALPENHORN_NODE
+++ /dev/null
@@ -1,1 +0,0 @@
-node_3
\ No newline at end of file
diff --git a/demo/data/transport/t1/ALPENHORN_NODE b/demo/data/transport/t1/ALPENHORN_NODE
deleted file mode 100644
index 3e0bb63..0000000
--- a/demo/data/transport/t1/ALPENHORN_NODE
+++ /dev/null
@@ -1,1 +0,0 @@
-t1
\ No newline at end of file
diff --git a/demo/demo-script.md b/demo/demo-script.md
index 8df228d..372c60b 100644
--- a/demo/demo-script.md
+++ b/demo/demo-script.md
@@ -1,231 +1,1547 @@
-Alpenhorn
-=========
+# Alpenhorn Demo
 
-Preliminaries
-------------
+## Introduction
 
+This is a short demonstration of using alpenhorn intended to show off most of
+the major features of the system.
 
-1. Start the DB and enter the first container
-    ```
-    docker compose up --remove-orphans --detach db
-    docker compose run host_1 bash -l
-    ```
+## Demo Set-up
 
-2. Initialize Alpenhorn database
+Because alpenhorn is designed to run as a distributed system, with both data and
+software present at multiple dispersed, independent places, this demo uses
+[docker](https://docs.docker.com/) to run several virtual images simulating
+independent hosts.  Additionally, [Docker Compose](https://docs.docker.com/compose/)
+is used to manage the multi-container set-up for this demo.
 
-    ```
-    python -c 'import pattern_importer; pattern_importer.demo_init()'
-    alpenhorn db init
-    alpenhorn group create group_1
-    alpenhorn node create node_1 --auto-import --root=/data --group=group_1 --host=host_1
-    alpenhorn node activate node_1 --username root --address host_1
-    ```
+Installing docker itself is beyond the scope of this demo.  The
+[Docker install documentation](https://docs.docker.com/get-started/get-docker/) may
+help.  You may also be able to get help from your friendly neighbourhood sysadmin.
+Once docker is properly installed you can test the installation by running the `hello-world`
+container:
+```(console)
+$ docker run hello-world
+Unable to find image 'hello-world:latest' locally
+latest: Pulling from library/hello-world
+e6590344b1a5: Pull complete
+Digest: sha256:d715f14f9eca81473d9112df50457893aa4d099adeb4729f679006bf5ea12407
+Status: Downloaded newer image for hello-world:latest
 
+Hello from Docker!
+This message shows that your installation appears to be working correctly.
 
-Auto-Importing files
---------------------
+To generate this message, Docker took the following steps:
+ 1. The Docker client contacted the Docker daemon.
+ 2. The Docker daemon pulled the "hello-world" image from the Docker Hub.
+    (amd64)
+ 3. The Docker daemon created a new container from that image which runs the
+    executable that produces the output you are currently reading.
+ 4. The Docker daemon streamed that output to the Docker client, which sent it
+    to your terminal.
 
-3. Start the service
-    ```
-    docker compose up --remove-orphans --detach host_1
-    docker compose logs host_1
-    ```
-   
-   Note files being imported. 
-   (Could also do `docker compose logs host_1 | head -30`.)
-   
-4. Client
-    ```
-    docker compose exec host_1 bash -l
-    alpenhorn node stats
-    ```
-   
-   Two files on node_1:
-
-   ```
-   root@host_1:/# alpenhorn node stats
-   Name      File Count    Total Size    % Full
-   ------  ------------  ------------  --------
-   node_1             2          15 B         -
-   ```
-
-   ```
-   root@host_1:/# alpenhorn file list --node=node_1 --details
-File                                                                                  Size    MD5 Hash                          Registration Time             State    Size on Node
-------------------------------------------------------------------------------------  ------  --------------------------------  ----------------------------  -------  --------------
-2017/03/21/acq_xy1_45678901T000000Z_inst_zab/acq_data/x_123_1_data/raw/acq_123_1.zxc  15 B    9ef14b877f34ed94c8dc3e700db49dad  Wed Jan 22 22:14:03 2025 UTC  Healthy  4.000 kiB
-2017/03/21/acq_xy1_45678901T000000Z_inst_zab/summary.txt                              0 B     d41d8cd98f00b204e9800998ecf8427e  Wed Jan 22 22:14:03 2025 UTC  Healthy  0 B
-   ```
-
-
-Start more nodes
----------------
-
-5. Create storage
-
-    ```
-    alpenhorn group create group_2
-    alpenhorn node create node_2 --auto-import --root=/data --group=group_2 --host=host_2
-    alpenhorn node activate node_2 --username root --address host_2
-
-    alpenhorn group create group_3
-    alpenhorn node create node_3 --auto-import --root=/data --group=group_3 --host=host_3
-    alpenhorn node activate node_3 --username root --address host_3
-    ```
-
-6. Start the service
-    ```
-    docker compose up -d host_1
-    docker compose up host_2
-    docker compose up host_3
-    ```
-   
-   Note no files are being imported.
-   
+To try something more ambitious, you can run an Ubuntu container with:
+ $ docker run -it ubuntu bash
 
-Syncing files
----------------
+Share images, automate workflows, and more with a free Docker ID:
+ https://hub.docker.com/
 
-7. Client
-    ```
-    docker compose exec host_2 bash -l
-    find /data
-    ```
-   
-    No files on host_2
-
-8. Start a sync
-    ```
-    alpenhorn node sync node_1 group_2 --show_files
-    ```
-    
-    Note the server copying
-
-    ```
-    find /data
-    ```
-    The files are here!
-
-9. Sync an explicit acquisition
-    ```
-    docker compose exec host_3 bash -l
-    find /data
-    alpenhorn sync node_1 group_3 --acq=12345678T000000Z_inst_zab --show_files
-    ```
-    
-    Note the server copying
-
-
-10. Client
-    ```
-    alpenhorn status
-    alpenhorn verify node_2
-    alpenhorn verify node_3
-    ```
-   
-   Two files on host_2, one file on host_3. 
+For more examples and ideas, visit:
+ https://docs.docker.com/get-started/
+```
+
+Once docker itself is running, you'll also need to
+[install the docker compose plugin](https://docs.docker.com/compose/install/linux/).
+Afer installing docker compose, you should be able to run
+```
+  docker compose version
+```
+and see the version of the plugin you just installed:
+```(console)
+$ docker compose version
+Docker Compose version v2.31.0
+```
+
+If that's all working, you should be able to proceed with the alpenhorn demo itself!
+
+## Starting the demo database
+
+There are four docker containers that comprise this demo:
+* a database container, which runs the MySQL server containing the alpenhorn Data Index
+* three containers implementing the separate alpenhorn hosts, each containing a StorageNodes
+
+These containers will be automatically built when we first start the demo.
+
+The demo must be run from the `/demo/` subdirectory.  (The directory containing this file.)
+
+Let's start off by starting the database container in the background.  Because alpenhorn
+is a distributed system, it is not expected that the database itself runs on an alpenhorn
+node.  We simulate this in the demo by running the database out of a standard mysql container.
+
+To start the database container, run the following from the `/demo` subdirectory:
+```
+    docker compose up --detach alpdb
+```
+
+If you get a `no configuration file provided: not found` error, you're not in the right
+directory.  (The directory also should have the `Dockerfile.alpenhorn` file and the
+`docker-compose.yaml` file, both of which came with this demo.)
+
+Doing this the first time will probably cause docker to download the latest MySQL image:
+```(console)
+$ docker compose up --detach alpdb
+[+] Running 11/11
+ ✔ alpdb Pulled                                                             15.9s
+ ✔ 1d19e87a21f5 Pull complete                                                3.0s
+ ✔ 16ec22ff04f9 Pull complete                                                3.1s
+ ✔ 9f789b8d2675 Pull complete                                                3.1s
+ ✔ 96f4da41c548 Pull complete                                                3.5s
+ ✔ fb087646189b Pull complete                                                3.5s
+ ✔ 023374826adc Pull complete                                                3.5s
+ ✔ 8293a632aa25 Pull complete                                                4.6s
+ ✔ c3947540e0c6 Pull complete                                                4.7s
+ ✔ c38bed95fb4b Pull complete                                               14.5s
+ ✔ 712eb897f1e5 Pull complete                                               14.5s
+[+] Running 1/1
+ ✔ Container demo-alpdb-1  Started 
+```
+
+You can use `docker status` or `docker container ls` to verify that the alpdb container is running:
+```(console)
+$ docker container ls
+CONTAINER ID   IMAGE          COMMAND         CREATED         STATUS         PORTS                 NAMES
+7e19895eb701   mysql:latest   "docker-ent…"   5 minutes ago   Up 5 minutes   3306/tcp, 33060/tcp   demo-alpdb-1
+```
+
+## Stopping and resetting the demo
+
+Before we continue, a few words about stopping and resetting this demo.
+
+You can stop the docker containers running this demo at any time by executing:
+```
+docker compose stop
+```
+This will stop all running containers.  To restart the demo, run the appropriate `docker compose up`
+commands.  Stopping the demo does not delete the containers or volumes containing the database and
+the storage node data.
+
+If you want to also remove the demo containers:
+```
+docker compose down --remove-orphans
+```
+
+To remove the containers _and_ the volumes containing the database and the storage node data:
+```
+docker compose down --remove-orphans --volumes
+```
+Removing the volumes will require rebuilding the database as described below.
+
+Finally, to remove the alpenhorn container image, which gets built the first time the containers is
+run:
+```
+docker rmi alpenhorn:latest
+```
+You should do this if you want to update the version of alpenhorn used by the demo, or if you've
+made changes to the demo's `Dockerfile.alpenhorn` or `docker-compose.yaml` files.  (You can
+also remove the `mysql:latest` image if you want to run a newer version of the database
+container.)
+
+
+## Initialising the database
+
+Now we need to use some `alpenhorn` commands to create the Data Index (the alpenhorn database)
+and the define the start of our storage infrastructure in it.  The data index must exist before we
+can start the first alpenhorn daemon.
+
+To create the data index we'll need access to the database.  We'll do that by creating a
+temporary container running the image from the first alpenhorn node (`alpen1`):
+
+To build the container and start a bash session in it, run:
+```
+docker compose run --rm alpen1 bash -l
+```
+
+Running this the first time will cause docker compose to build the `alpenhorn` docker image.
+This may take some time.  Eventually you should be presented with a bash prompt as root inside
+the `alpen1` container:
+```(console)
+$ docker compose run --rm alpen1 bash -l
+[+] Creating 3/0
+ ✔ Volume "demo_node1_vol"      Created                                                                                                                                                                                 0.0s 
+ ✔ Volume "demo_transport_vol"  Created                                                                                                                                                                                 0.0s 
+ ✔ Container demo-alpdb-1       Running                                                                                                                                                                                 0.0s 
+[+] Running 1/1
+ ! alpen1 Warning pull access denied for alpenhorn, repository does not exist or may require 'docker login': denied: requested access to the resource is denied                                                          0.9s 
+[+] Building 78.4s (17/17) FINISHED                                                                                                                                                                           docker:default
+ => [alpen1 internal] load build definition from Dockerfile.alpenhorn                                                                                                                                                   0.0s
+ => => transferring dockerfile: 1.20kB                                                                                                                                                                                  0.0s
+ => [alpen1 internal] load metadata for docker.io/library/python:3.11                                                                                                                                                   1.2s
+ => [alpen1 internal] load .dockerignore                                                                                                                                                                                0.0s
+ => => transferring context: 2B                                                                                                                                                                                         0.0s
+ => [alpen1 internal] load build context                                                                                                                                                                                0.2s
+ => => transferring context: 2.89MB                                                                                                                                                                                     0.2s
+ => [alpen1  1/11] FROM docker.io/library/python:3.11@sha256:14b4620f59a90f163dfa6bd252b68743f9a41d494a9fde935f9d7669d98094bb                                                                                          18.8s
+ => => resolve docker.io/library/python:3.11@sha256:14b4620f59a90f163dfa6bd252b68743f9a41d494a9fde935f9d7669d98094bb                                                                                                    0.0s
+ => => sha256:14b4620f59a90f163dfa6bd252b68743f9a41d494a9fde935f9d7669d98094bb 9.08kB / 9.08kB                                                                                                                          0.0s
+ => => sha256:fa951df28e3fef5b5736bf5d0c285f91e7c8d1c814bfc3784c1a4b3d216b39ee 2.33kB / 2.33kB                                                                                                                          0.0s
+ => => sha256:35af2a7690f2b43e7237d1fae8e3f2350dfb25f3249e9cf65121866f9c56c772 64.39MB / 64.39MB                                                                                                                        1.7s
+ => => sha256:78a74fb73bfb12a8641cc50cbc82f57c610aaafa73b628896cb71a475497922c 6.18kB / 6.18kB                                                                                                                          0.0s
+ => => sha256:a492eee5e55976c7d3feecce4c564aaf6f14fb07fdc5019d06f4154eddc93fde 48.48MB / 48.48MB                                                                                                                        1.8s
+ => => sha256:32b550be6cb62359a0f3a96bc0dc289f8b45d097eaad275887f163c6780b4108 24.06MB / 24.06MB                                                                                                                        1.3s
+ => => sha256:7576b00d9bb10cc967bb5bdeeb3d5fa078ac8800e112aa03ed15ec199662d4f7 211.33MB / 211.33MB                                                                                                                      8.0s
+ => => sha256:3fd67c6ea72187077ad551000d527ae7c24d461e7c9944dc74312e3afac50fb4 6.16MB / 6.16MB                                                                                                                          2.1s
+ => => sha256:dcaa1b9153e7d7edb7678e2ef9933b57b19a97ec9bce49dc8630911aa18664d1 24.31MB / 24.31MB                                                                                                                        2.8s
+ => => extracting sha256:a492eee5e55976c7d3feecce4c564aaf6f14fb07fdc5019d06f4154eddc93fde                                                                                                                               2.7s
+ => => sha256:8630e3071c887d36db54d1924e21eee3419c5afc9874a068e03fb7978b2fb7d8 249B / 249B                                                                                                                              2.3s
+ => => extracting sha256:32b550be6cb62359a0f3a96bc0dc289f8b45d097eaad275887f163c6780b4108                                                                                                                               0.7s
+ => => extracting sha256:35af2a7690f2b43e7237d1fae8e3f2350dfb25f3249e9cf65121866f9c56c772                                                                                                                               3.2s
+ => => extracting sha256:7576b00d9bb10cc967bb5bdeeb3d5fa078ac8800e112aa03ed15ec199662d4f7                                                                                                                               7.8s
+ => => extracting sha256:3fd67c6ea72187077ad551000d527ae7c24d461e7c9944dc74312e3afac50fb4                                                                                                                               0.4s
+ => => extracting sha256:dcaa1b9153e7d7edb7678e2ef9933b57b19a97ec9bce49dc8630911aa18664d1                                                                                                                               1.1s
+ => => extracting sha256:8630e3071c887d36db54d1924e21eee3419c5afc9874a068e03fb7978b2fb7d8                                                                                                                               0.0s
+ => [alpen1  2/11] RUN apt-get update && apt-get install --no-install-recommends -y     vim     ssh     rsync     netcat-openbsd     default-mysql-client                                                              13.2s
+ => [alpen1  3/11] RUN pip install --no-cache-dir mysqlclient                                                                                                                                                           8.3s 
+ => [alpen1  4/11] RUN ssh-keygen -t rsa -N '' -f /root/.ssh/id_rsa                                                                                                                                                     2.2s 
+ => [alpen1  5/11] RUN cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys                                                                                                                                              0.6s 
+ => [alpen1  6/11] RUN echo 'Host *\n    StrictHostKeyChecking no\n' > /root/.ssh/config                                                                                                                                0.6s 
+ => [alpen1  7/11] COPY demo/alpenhorn.conf /etc/alpenhorn/alpenhorn.conf                                                                                                                                               0.1s 
+ => [alpen1  8/11] RUN mkdir /var/log/alpenhorn                                                                                                                                                                         0.5s 
+ => [alpen1  9/11] COPY examples/pattern_importer.py /root/python/pattern_importer.py                                                                                                                                   0.1s 
+ => [alpen1 10/11] ADD . /build                                                                                                                                                                                         0.2s
+ => [alpen1 11/11] RUN cd /build && pip install .                                                                                                                                                                      31.3s
+ => [alpen1] exporting to image                                                                                                                                                                                         1.1s 
+ => => exporting layers                                                                                                                                                                                                 1.0s 
+ => => writing image sha256:cbbb72cb54b858da771eae6590640747fc884282239056c127a5881f008532a3                                                                                                                            0.0s 
+ => => naming to docker.io/library/alpenhorn                                                                                                                                                                            0.0s 
+ => [alpen1] resolving provenance for metadata file                                                                                                                                                                     0.0s 
+root@alpen1:/#
+```
+
+Once at the root prompt, we can build the data index and start populating it.
+
+### Setting up the import extension
+
+Because alpenhorn is data agnostic, it doesn't have any facilities out-of-the-box to import files.
+To be able to import files, alpenhorn needs one or more "import-detect extensions" to be loaded.
+For the purposes of this demo, we'll use the simple `pattern_importer` example extension provided in
+the `/examples` directory.  This extension has already been incorporated into the alpenhorn
+container image that we're running, and alpenhorn has been set up to use it.
+
+As explained in the documentation for the `pattern_importer` example, the extension adds fields to
+two alpenhorn tables: `ArchiveAcq` and `ArchiveFile`, as well as creating some of its own tables in
+the Data Index.  Because of this, we need to run the extension initialisation before we can create
+the rest of the data index proper.
+
+To initialise the database for the extension, run the `demo_init` function provided by the
+extension:
+```
+  python -c 'import pattern_importer; pattern_importer.demo_init()'
+```
+If you get a `ModuleNotFoundError: No module named 'pattern_importer'` error, you're probably not
+executing this command in the root-shell in the `alpen1` container.
+
+You should see a success message:
+```(console)
+root@alpen1:/# python -c 'import pattern_importer; pattern_importer.demo_init()'
+Plugin init complete complete.
+```
+
+### Setting up the data index
+
+Once the pattern importer has been set up, set up the rest of the data index using the alpenhorn
+CLI:
+```
+  alpenhorn db init
+```
+Remember that all these alpenhorn commands need to be run inside the `alpen1` container that we
+started in the last section.  The `db init` command outputs nothing when successful:
+```(console)
+root@alpen1:/# alpenhorn db init
+root@alpen1:/#
+```
+
+### Create the first StorageNode
+
+We need to start with a place to put some files.  We'll create the first `StorageNode`, which will
+be hosted on `alpen1`.  Before we can do that, though we first need to create a `StorageGroup` to
+house the node.  All `StorageNode`s need to be contained in a `StorageGroup`.  Typically each group
+contains only a single node, but certain group classes support or require multiple nodes (such as
+the transport group that we'll create later).
+
+To create the group, which we'll call `demo_storage1`, run:
+```
+  alpenhorn group create demo_storage1
+```
+This should create the group:
+```
+root@alpen1:/# alpenhorn group create demo_storage1
+Created storage group "demo_storage1".
+```
+If instead you get an error: `Error: Group "demo_storage1" already exists.` then likely you're
+trying to run this demo using an old instance of the database.  In this case, you can stop the demo
+and delete the old database volume as explained above, if you want to start with a clean demo.
+
+Now that the group is created, we can create a node within it.  We'll also call the node
+`demo_storage1`.  (By convention, when a StorageGroup contains only one StorageNode, the node and
+group have the same name, though that's not required.)
+```
+  alpenhorn node create demo_storage1 --group=demo_storage1 --auto-import --root=/data --host=alpen1
+```
+This command will create a new StorageNode called `demo_storage1` and put it in the
+identically-named group.  Auto-import will be turned on; the mount point in the filesystem will be
+set to `/data` and we declare it to be available on host `alpen1`:
+```(console)
+root@alpen1:/# alpenhorn node create demo_storage1 --group=demo_storage1 --auto-import 
+              --root=/data --host=alpen1
+Created storage node "demo_storage1".
+```
+
+That's enough to get us started.  Exit the temporary `alpen1` container:
+```
+exit
+```
+
+Docker should remove the temporary container once you've exited.
+
+## Start the first daemon
+
+Now it's time to start the first daemon.  The alpenhorn container is set-up to run the alpenhorn
+daemon automatically.  Start it by running the `docker compose up` command:
+```
+docker compose up --detach alpen1
+```
+
+Note: if you're following along with this demo, the database container should already be running:
+```(console)
+$ docker compose up --detach alpen1
+[+] Running 2/2
+ ✔ Container demo-alpdb-1   Running                                                           0.0s
+ ✔ Container demo-alpen1-1  Started                                                           0.4s
+```
+(If the database container is not running, docker compose will start it first).
+
+You should now check the logs for the daemon:
+```
+docker compose logs alpen1
+```
+(You can add `--follow` if you wish to have the logs continuously update.)  You'll see the
+alpenhorn daemon start up:
+```
+alpen1-1  | Feb 21 00:38:32 INFO >> [MainThread] Alpenhorn start.
+alpen1-1  | Feb 21 00:38:32 INFO >> [MainThread] Loading config file /etc/alpenhorn/alpenhorn.conf
+alpen1-1  | Feb 21 00:38:32 INFO >> [MainThread] Loading extension pattern_importer
+alpen1-1  | Feb 21 00:38:32 INFO >> [Worker#1] Started.
+alpen1-1  | Feb 21 00:38:32 INFO >> [Worker#2] Started.
+```
+Two worker threads are started because that's what's specified in the `demo/alpenhornd.conf` file.
+It has also loaded the `pattern_exporter` extension, since that's also specified in the config
+file.
+
+Almost immediately, the daemon will notice that there are no _active_ ndoes on `alpen1`.  It
+will perform this check roughly every ten seconds, which is the update interval time set in
+the `demo/alpenhornd.conf` file.
+```
+alpen1-1  | Feb 21 00:38:32 WARNING >> [MainThread] No active nodes on host (alpen1)!
+alpen1-1  | Feb 21 00:38:32 INFO >> [MainThread] Main loop execution was 0.0s.
+alpen1-1  | Feb 21 00:38:32 INFO >> [MainThread] Tasks: 0 queued, 0 deferred, 0 in-progress on 2 workers
+alpen1-1  | Feb 21 00:38:42 WARNING >> [MainThread] No active nodes on host (alpen1)!
+alpen1-1  | Feb 21 00:38:42 INFO >> [MainThread] Main loop execution was 0.0s.
+alpen1-1  | Feb 21 00:38:42 INFO >> [MainThread] Tasks: 0 queued, 0 deferred, 0 in-progress on 2 workers
+```
+
+We can fix this by activating the node we created.
+
+Start a bash session in the runing `alpen1` container:
+```
+docker compose exec alpen1 bash -l
+```
+(Note the use of `exec` here instead of `run` which we used to start the bash session earlier.  The
+difference between `exec` and `run` is: `exec` will execute the command in the running `alpen1`
+container.  Using `run` would have created a separate instance of the `alpen1` container to run the
+command.
+
+You will be issuing a lot of `alpenhorn` commands over the course of this demo.  We suggest leaving
+this root shell open to make it more convenient to issue them.  If you ever need open a new root
+session on `alpen1`, just run the above `docker compose exec` command again.
+
+In the `alpen1` container, at the root prompt, we can now activate the node:
+```
+alpenhorn node activate demo_storage1
+```
+
+```(console)
+root@alpen1:/# alpenhorn node activate demo_storage1
+Storage node "demo_storage1" activated.
+```
+
+Now the daemon will find the active node, but there's still a problem:
+```
+alpen1-1  | Feb 21 00:40:22 INFO >> [MainThread] Node "demo_storage1" now available.
+alpen1-1  | Feb 21 00:40:22 WARNING >> [MainThread] Node file "/data/ALPENHORN_NODE" could not be read.
+alpen1-1  | Feb 21 00:40:22 WARNING >> [MainThread] Ignoring node "demo_storage1": not initialised.
+alpen1-1  | Feb 21 00:40:22 INFO >> [MainThread] Main loop execution was 0.0s.
+alpen1-1  | Feb 21 00:40:22 INFO >> [MainThread] Tasks: 0 queued, 0 deferred, 0 in-progress on 2 workers
+```
+
+We need to initialise the node so `alpenhorn` can use it.  In this case, we could do this by
+manually creating the `/data/ALPENHORN_NODE` file that it can't find.  But, generally, it's easier
+to tell the daemon to initialise the node for us:
+```
+alpenhorn node init demo_storage1
+```
+
+```(console)
+root@alpen1:/# alpenhorn node init demo_storage1
+Requested initialisation of Node "demo_storage1".
+```
+A node only ever needs to be initialised once, when it is first created, but it's always safe to run
+this command: a request to initialise an already-initialised node is simply ignored.
+
+You should see the node being initialised by one of the daemon workers:
+```
+alpen1-1  | Feb 21 00:40:52 INFO >> [MainThread] Node "demo_storage1" now available.
+alpen1-1  | Feb 21 00:40:52 WARNING >> [MainThread] Node file "/data/ALPENHORN_NODE" could not be read.
+alpen1-1  | Feb 21 00:40:52 INFO >> [MainThread] Requesting init of node "demo_storage1".
+alpen1-1  | Feb 21 00:40:52 INFO >> [MainThread] Main loop execution was 0.0s.
+alpen1-1  | Feb 21 00:40:52 INFO >> [Worker#1] Beginning task Init Node "demo_storage1"
+alpen1-1  | Feb 21 00:40:52 INFO >> [MainThread] Tasks: 0 queued, 0 deferred, 1 in-progress on 2 workers
+alpen1-1  | Feb 21 00:40:52 WARNING >> [Worker#1] Node file "/data/ALPENHORN_NODE" could not be read.
+alpen1-1  | Feb 21 00:40:52 WARNING >> [Worker#1] Node file "/data/ALPENHORN_NODE" could not be read.
+alpen1-1  | Feb 21 00:40:52 INFO >> [Worker#1] Node "demo_storage1" initialised.
+alpen1-1  | Feb 21 00:40:52 INFO >> [Worker#1] Finished task: Init Node "demo_storage1"
+```
+
+After initialisation is complete, the daemon will finally be happy with the Storage Node and start
+the auto-import monitor.  The start of auto-import triggers a "catch-up" job which searches for
+unknown, pre-existing files that need import.  As this is an empty node, though, it won't find
+anything:
+```
+alpen1-1  | Feb 21 00:41:02 INFO >> [MainThread] Node "demo_storage1" now available.
+alpen1-1  | Feb 21 00:41:02 INFO >> [MainThread] Group "demo_storage1" now available.
+alpen1-1  | Feb 21 00:41:02 INFO >> [MainThread] Watching node "demo_storage1" root "/data" for auto import.
+alpen1-1  | Feb 21 00:41:02 INFO >> [Worker#1] Beginning task Catch-up on demo_storage1
+alpen1-1  | Feb 21 00:41:02 INFO >> [Worker#1] Scanning "." on "demo_storage1" for new files.
+alpen1-1  | Feb 21 00:41:02 INFO >> [Worker#1] Scanning ".".
+alpen1-1  | Feb 21 00:41:02 INFO >> [Worker#1] Finished task: Catch-up on demo_storage1
+alpen1-1  | Feb 21 00:41:02 INFO >> [MainThread] Node demo_storage1: 46.77 GiB available.
+alpen1-1  | Feb 21 00:41:02 INFO >> [MainThread] Updating node "demo_storage1".
+alpen1-1  | Feb 21 00:41:02 INFO >> [MainThread] Updating group "demo_storage1".
+alpen1-1  | Feb 21 00:41:02 INFO >> [MainThread] Main loop execution was 0.0s.
+alpen1-1  | Feb 21 00:41:02 INFO >> [MainThread] Tasks: 1 queued, 0 deferred, 0 in-progress on 2 workers
+```
+
+It will also run a job to see if there's anything needing clean-up on the node. This "tidy up" job
+helps the alpenhorn daemon recover from unexpected crashes.  The job is generally run when a node
+first becomes available to the daemon, and then periodically after that.  Again, because this
+is a brand-new node, there isn't anything needing tidying:
+```
+alpen1-1  | Feb 21 00:41:02 INFO >> [Worker#2] Beginning task Tidy up demo_storage1
+alpen1-1  | Feb 21 00:41:02 INFO >> [Worker#2] Finished task: Tidy up demo_storage1
+alpen1-1  | Feb 21 00:41:12 INFO >> [MainThread] Node demo_storage1: 46.77 GiB available.
+alpen1-1  | Feb 21 00:41:12 INFO >> [MainThread] Updating node "demo_storage1".
+alpen1-1  | Feb 21 00:41:12 INFO >> [MainThread] Updating group "demo_storage1".
+alpen1-1  | Feb 21 00:41:12 INFO >> [MainThread] Main loop execution was 0.0s.
+alpen1-1  | Feb 21 00:41:12 INFO >> [MainThread] Tasks: 0 queued, 0 deferred, 0 in-progress on 2 workers
+```
+
+
+## Importing files
+
+Let's experiment now with importing files into alpenhorn, using both the auto-import system and
+manually importing them.
+
+### What kind of files can be imported?
+
+As mentioned before, alpenhorn itself is agnostic to data file contents.  All decisions on which
+files are imported into the data index are made by the import detect extensions, which can be
+tailored to the specific data being managed.  For this demo, the only import detect function we're
+using is the example `pattern_importer` extension.  This extension uses a regular expressions to
+match against the pathnames of candidate files to determine whether they should be imported or not.
+
+The `demo_init` function that we called earlier to initialise the database for this demo, added one
+allowed ArchiveAcq name pattern consisting of a nested directory tree with the date: `YYYY/MM/DD`
+and two allowed ArchiveFile name patterns.  The first of these is a file called "meta.txt" in the
+top acquisition directory (i.e. `YYYY/MM/DD/meta.txt`), which provides metadata for our notional
+acquisition, and then data files with the time of day, sorted further into hourly directories (i.e.
+`YYYY/MM/DD/hh/mmss.dat`).
+
+It bears repeating: the _contents_ of these files are not interesting to alpenhorn per se, but an
+import detect extension may be implemented which inspects the data of the files being imported, if
+desired.
+
+We'll continue this demo by creating files with the above-mentioned naming conventions, without much
+concern about the file contents.
+
+### Auto-importing files and lock files
+
+Let's start with auto-importing files.  When auto-import is turned on for a node, like it has been
+for our `demo_storage1` node, then files will automatically be discovered by alpenhorn as they are
+added to the node filesystem.
+
+Care must be taken when writing files to a node filesystem when auto-import is turned on to prevent
+alpenhorn from trying to import a file before it is fully written.  To prevent this from happening,
+before creating a file on the node filesystem, we can create a _lock file_.
+
+For a file at the path `AAA/BBB/name.ext`, the corresponding lock file will be called
+`AAA/BBB/.name.ext.lock` (i.e. the name of a lock file is the name of the file it's locking plus a
+leading `.` and a `.lock` suffix.
+
+Let's create the first file we want to import into alpenhorn, first creating it's lockfile.  This
+should be done in the `alpen1` root shell we started earlier:
+```
+mkdir -p /data/2025/02/21
+touch /data/2025/02/21/.meta.txt.lock
+echo "This is the first acquistion in the alpenhorn demo" > /data/2025/02/21/meta.txt
+```
+
+When creating the file in this last step, you'll see alpenhorn notice the file, but skip it because
+it's locked:
+```
+alpen1-1  | Feb 21 23:04:21 INFO >> [Worker#1] Beginning task Import 2025/02/21/meta.txt on demo_storage1
+alpen1-1  | Feb 21 23:04:21 INFO >> [Worker#1] Skipping "2025/02/21/meta.txt": locked.
+alpen1-1  | Feb 21 23:04:21 INFO >> [Worker#1] Finished task: Import 2025/02/21/meta.txt on demo_storage1
+```
+Note: in some cases file creation can cause multiple import requests to be scheduled.  This is
+harmless: alpenhorn is prepared to handle multiple simultaneous attempts to import the same file and
+will only ever import a file once.
+
+Once the file has been created, the lock file can be deleted, to trigger import of the file:
+```
+rm -f /data/2025/02/21/.meta.txt.lock
+```
+
+This will trigger alpenhorn to finally actually import the file:
+```
+alpen1-1  | Feb 21 23:07:07 INFO >> [Worker#1] Beginning task Import 2025/02/21/meta.txt on demo_storage1
+alpen1-1  | Feb 21 23:07:07 INFO >> [Worker#1] Acquisition "2025/02/21" added to DB.
+alpen1-1  | Feb 21 23:07:07 INFO >> [Worker#1] File "2025/02/21/meta.txt" added to DB.
+alpen1-1  | Feb 21 23:07:07 INFO >> [Worker#1] Imported file copy "2025/02/21/meta.txt" on node "demo_storage1".
+alpen1-1  | Feb 21 23:07:07 INFO >> [Worker#1] Finished task: Import 2025/02/21/meta.txt on demo_storage1
+```
+
+Note here that the the three lines in the middle of the daemon output above indicate that the daemon
+has created three new records in the database:
+* an `ArchiveAcq` record for the new acquisition, with name `2025/02/21`
+* an `ArchiveFile` record for the new file, with name `21/meta.txt` in the new acqusition
+* an `ArchiveFileCopy` record recording that a copy of the newly-created `ArchiveFile` exists on
+    `demo_storage1`
+
+
+You can use the alpenhorn CLI to see that this file is now present on the `demo_storage1` node:
+```
+root@alpen1:/# alpenhorn node stats
+Name             File Count    Total Size    % Full
+-------------  ------------  ------------  --------
+demo_storage1             1          51 B         -
+root@alpen1:/# alpenhorn file list --node=demo_storage1 --details
+File                 Size    MD5 Hash                          Registration Time             State    Size on Node
+-------------------  ------  --------------------------------  ----------------------------  -------  --------------
+2025/02/21/meta.txt  51 B    c2607e3dbaf6a1e2467b82c6a79f6b46  Fri Feb 21 23:07:08 2025 UTC  Healthy  4.000 kiB
+```
+
+### Auto-importing files and temporary names
+
+Another option for writing files to a node filesystem when auto-import is turned on, may be to
+write them to a path which the import detect extensions won't accept, although whether this
+is possible will depend on the particular import detect extensions being used.  For this demo,
+we can use any name which doesn't match the patterns which the `pattern_importer` will accept.
+
+As an example, let's create a `.dat` file with a temporary name by appending, say, `.temp` to
+the name of the file we want to create:
+```
+mkdir /data/2025/02/21/23
+echo "0 1 2 3 4 5" > /data/2025/02/21/23/1324.dat.temp
+```
+
+This file creation will be noticed by alpenhorn, but no import will occur, because the
+`pattern_exporter` won't accept the name as valid:
+```
+alpen1-1  | Feb 21 23:51:59 INFO >> [Worker#1] Beginning task Import 2025/02/21/23/1324.dat.temp on demo_storage1
+alpen1-1  | Feb 21 23:51:59 INFO >> [Worker#1] Not importing non-acquisition path: 2025/02/21/23/1324.dat.temp
+alpen1-1  | Feb 21 23:51:59 INFO >> [Worker#1] Finished task: Import 2025/02/21/23/1324.dat.temp on demo_storage1
+```
+
+After file is fully written, it can be moved to the correct name.  On most filesystems, this is an
+atomic operation:
+```
+mv /data/2025/02/21/23/1324.dat.temp /data/2025/02/21/23/1324.dat
+```
+
+This will trigger import of the file:
+```
+alpen1-1  | Feb 21 23:52:20 INFO >> [Worker#2] Beginning task Import 2025/02/21/23/1324.dat on demo_storage1
+alpen1-1  | Feb 21 23:52:20 INFO >> [Worker#2] File "2025/02/21/23/1324.dat" added to DB.
+alpen1-1  | Feb 21 23:52:20 INFO >> [Worker#2] Imported file copy "2025/02/21/23/1324.dat" on node "demo_storage1".
+alpen1-1  | Feb 21 23:52:20 INFO >> [Worker#2] Finished task: Import 2025/02/21/23/1324.dat on demo_storage1
+```
+
+Unlike when we imported the first file, now only two new records are created in the database because
+the acquistion record already existsed:
+* an `ArchiveFile` for the new file
+* an `ArchiveFileCopy` for the copy of the new file on `demo_storage1`
+
+Now there are two files on the node:
+```
+root@alpen1:/# alpenhorn node stats
+Name             File Count    Total Size    % Full
+-------------  ------------  ------------  --------
+demo_storage1             2          63 B         -
+root@alpen1:/# alpenhorn file list --node=demo_storage1 --details
+File                    Size    MD5 Hash                          Registration Time             State    Size on Node
+----------------------  ------  --------------------------------  ----------------------------  -------  --------------
+2025/02/21/23/1324.dat  12 B    4c79018e00ddef11af0b9cfc14dd3261  Fri Feb 21 23:52:21 2025 UTC  Healthy  4.000 kiB
+2025/02/21/meta.txt     51 B    c2607e3dbaf6a1e2467b82c6a79f6b46  Fri Feb 21 23:07:08 2025 UTC  Healthy  4.000 kiB
+```
+
+### Manually importing files
+
+Let's now turn to the case where we _don't_ have auto-import turned on for a node.  In this case
+there's no difficulty writing to the node, since filesystem events won't trigger automatic attempts
+to import files.
+
+First, turn off auto-import on the node by modifying its properties:
+```
+alpenhorn node modify demo_storage1 --no-auto-import
+```
+
+If you want, you can verify that auto-import has been turned off for the node by checking its
+metadata:
+```
+root@alpen1:/# alpenhorn node modify demo_storage1 --no-auto-import
+Node updated.
+root@alpen1:/# alpenhorn node show demo_storage1
+   Storage Node: demo_storage1
+  Storage Group: demo_storage1
+         Active: Yes
+           Type: -
+          Notes: 
+      I/O Class: Default
+
+    Daemon Host: alpen1
+ Log-in Address: 
+Log-in Username: 
+
+    Auto-Import: Off
+    Auto-Verify: Off
+      Max Total: -
+      Available: 46.47 GiB
+  Min Available: -
+   Last Checked: Sat Feb 22 00:03:36 2025 UTC
+
+I/O Config:
+
+  none
+```
+
+With that done, let's create some more data files:
+```
+echo "0 1 2 3 4 5" > /data/2025/02/21/23/1330.dat
+echo "3 4 5 6 7 8" > /data/2025/02/21/23/1342.dat
+echo "9 10 11 12 13" > /data/2025/02/21/23/1349.dat
+```
+
+None of these files have been added to the database.  We can use the alpenhorn CLI to see this: as
+far as alpenhorn is concerned, there are still only two files on the node.
+```
+root@alpen1:/# alpenhorn node stats
+Name             File Count    Total Size    % Full
+-------------  ------------  ------------  --------
+demo_storage1             2          63 B         -
+```
+
+But, now that we've finished writing these files, we can tell alpenhorn to import them.  This can
+be done for an individual file:
+```
+alpenhorn file import --register-new 2025/02/21/23/1330.dat demo_storage1
+```
+
+The `--register-new` flag tells alpenhorn that it is allowed to create a new `ArchiveFile` (and,
+were it necessary, an `ArchiveAcq` record, too) for newly discovered files.  Without this flag,
+alpenhorn will only import files which are already represented by an existing `ArchvieFile`.  This
+second mode is more appropriate in cases where a node should not be receiving new files.
+
+The CLI will create an import request for this file:
+```
+root@alpen1:/# alpenhorn file import --register-new 2025/02/21/23/1330.dat demo_storage1
+Added new import request.
+```
+
+The import request should be shortly handled by the daemon:
+```
+alpen1-1  | Feb 22 00:09:36 INFO >> [Worker#1] Beginning task Import 2025/02/21/23/1330.dat on demo_storage1
+alpen1-1  | Feb 22 00:09:36 INFO >> [Worker#1] File "2025/02/21/23/1330.dat" added to DB.
+alpen1-1  | Feb 22 00:09:36 INFO >> [Worker#1] Imported file copy "2025/02/21/23/1330.dat" on node "demo_storage1".
+alpen1-1  | Feb 22 00:09:36 INFO >> [Worker#1] Completed import request #2.
+alpen1-1  | Feb 22 00:09:36 INFO >> [Worker#1] Finished task: Import 2025/02/21/23/1330.dat on demo_storage1
+```
+
+It's also possible to tell alpenhorn to scan an entire directory for new files.
+```
+alpenhorn node scan demo_storage1 --register-new 2025/02/21
+```
+
+Which will add another import request:
+```
+root@alpen1:/# alpenhorn node scan demo_storage1 --register-new 2025/02/21
+Added request for scan of "2025/02/21" on Node "demo_storage1".
+```
+
+Now alpenhorn will scan the requested path and find the other two files we just created:
+```
+alpen1-1  | Feb 22 00:12:56 INFO >> [Worker#2] Beginning task Scan "2025/02/21" on demo_storage1
+alpen1-1  | Feb 22 00:12:56 INFO >> [Worker#2] Scanning "2025/02/21" on "demo_storage1" for new files.
+alpen1-1  | Feb 22 00:12:56 INFO >> [Worker#2] Scanning "2025/02/21".
+alpen1-1  | Feb 22 00:12:56 INFO >> [Worker#2] Scanning "2025/02/21/23".
+alpen1-1  | Feb 22 00:12:56 INFO >> [Worker#1] Beginning task Import 2025/02/21/23/1349.dat on demo_storage1
+alpen1-1  | Feb 22 00:12:56 INFO >> [Worker#2] Completed import request #4.
+alpen1-1  | Feb 22 00:12:56 INFO >> [Worker#2] Finished task: Scan "2025/02/21" on demo_storage1
+alpen1-1  | Feb 22 00:12:56 INFO >> [Worker#2] Beginning task Import 2025/02/21/23/1342.dat on demo_storage1
+alpen1-1  | Feb 22 00:12:56 INFO >> [Worker#1] File "2025/02/21/23/1349.dat" added to DB.
+alpen1-1  | Feb 22 00:12:56 INFO >> [Worker#1] Imported file copy "2025/02/21/23/1349.dat" on node "demo_storage1".
+alpen1-1  | Feb 22 00:12:56 INFO >> [Worker#2] File "2025/02/21/23/1342.dat" added to DB.
+alpen1-1  | Feb 22 00:12:56 INFO >> [Worker#2] Imported file copy "2025/02/21/23/1342.dat" on node "demo_storage1".
+alpen1-1  | Feb 22 00:12:56 INFO >> [Worker#1] Finished task: Import 2025/02/21/23/1349.dat on demo_storage1
+alpen1-1  | Feb 22 00:12:56 INFO >> [Worker#2] Finished task: Import 2025/02/21/23/1342.dat on demo_storage1
+```
+
+Now there are five files on the storage node:
+```
+root@alpen1:/# alpenhorn node stats
+Name             File Count    Total Size    % Full
+-------------  ------------  ------------  --------
+demo_storage1             5         101 B         -
+```
+
+## Syncing files between nodes
+
+Let's now move on to syncing, or transferring, files between different hosts.
+
+### Starting up the second and third nodes
+
+Before being able to transfer files, we need to create somewhere to transfer them to.
+We'll start by creating the second storage node on the second host:
+
+```
+alpenhorn node create demo_storage2 --create-group --root=/data --host=alpen2
+```
+
+Note here that the `--create-group` option to `node create` tells alpenhorn to also create a
+`StorageGroup` for the new node with the same name (i.e. the same thing we did manually
+for `demo_storage1` above):
+```
+root@alpen1:/# alpenhorn node create demo_storage2 --create-group --root=/data --host=alpen2
+Created storage group "demo_storage2".
+Created storage node "demo_storage2".
+```
+
+Let's also make sure this node gets initialised, though this won't happen immediately,
+since we haven't activated the Storage Node, nor are we running the second daemon yet.
+```
+alpenhorn node init demo_storage2
+```
+Requests created by the alpenhorn CLI, be they initialisation requests, import requests,
+or transfer requests, do not require the target node to be active, nor do they require an
+alpenhorn daemon to be managing them.  Requests made on inactive nodes will remain pending
+in the database until they can be handled by an alpenhorn daemon instance.
+
+You can see pending requests, including this init request, using the alpenhorn CLI:
+```
+root@alpen1:/# alpenhorn node show demo_storage2 --all
+   Storage Node: demo_storage2
+  Storage Group: demo_storage2
+         Active: No
+           Type: -
+          Notes:
+      I/O Class: Default
+
+    Daemon Host: alpen2
+ Log-in Address:
+Log-in Username:
+
+    Auto-Import: Off
+    Auto-Verify: Off
+      Max Total: -
+      Available: -
+  Min Available: -
+   Last Checked: -
+
+I/O Config:
+
+  none
+
+Stats:
+
+    Total Files: 0
+     Total Size: -
+          Usage: -%
+
+Pending import requests:
+
+Path         Scan    Register New    Request Time
+-----------  ------  --------------  -------------------
+[Node Init]  -       -               2025-02-26 22:54:14
+
+Pending outbound transfers:
+
+Dest. Group    Request Count    Total Size
+-------------  ---------------  ------------
+
+Auto-actions:
+
+  none
+```
+(Node init requests are handled, under the hood, as a special kind of import request, which is
+why the Node Init request appears in the import request table.)
+
+This node is initially empty:
+```
+root@alpen1:/# alpenhorn node stats
+Name             File Count    Total Size    % Full
+-------------  ------------  ------------  --------
+demo_storage1             5         101 B         -
+demo_storage2             0             -         -
+```
+
+Before starting transfers we have to record log-in details for the hosts containing the nodes.
+alpenhorn uses SSH to log in to remote nodes when performing transfers, meaning we need to
+specify a username and login-in address for the node.  For `demo_storage1`, which is already
+active we can do this by modifying the node record:
+```
+alpenhorn node modify demo_storage1 --username root --address alpen1
+```
+
+For the second node, we can do it when we activate it.  We could have also specified
+these values when we created the node:
+```
+alpenhorn node activate demo_storage2 --username root --address alpen2
+```
+
+Note: it's very important to distinguish the name used for a node's _host_ (where the
+daemon managing the node is running) and the node's _address_ (the name or IP address
+used by remote daemons to access the node via SSH).  Often these two fields have the same
+value, but there's no requirement that they do.
+
+Let's start up the second alpenhorn container to get the second node running:
+```
+docker compose up --detach alpen2
+```
+
+You can monitor this nodes in the same way you did with alpen1:
+```
+docker compose logs alpen2
+```
+but it's also possible to monitor all nodes at once:
+```
+docker compose logs --follow
+```
+
+For now, the new node should initialise itself, and then idle: there are no pending requests:
+```
+alpen2-1  | Feb 26 23:05:02 INFO >> [MainThread] Node "demo_storage2" now available.
+alpen2-1  | Feb 26 23:05:02 WARNING >> [MainThread] Node file "/data/ALPENHORN_NODE" could not be read.
+alpen2-1  | Feb 26 23:05:02 INFO >> [MainThread] Requesting init of node "demo_storage2".
+alpen2-1  | Feb 26 23:05:02 INFO >> [MainThread] Main loop execution was 0.0s.
+alpen2-1  | Feb 26 23:05:02 INFO >> [MainThread] Tasks: 1 queued, 0 deferred, 0 in-progress on 2 workers
+alpen2-1  | Feb 26 23:05:02 INFO >> [Worker#1] Beginning task Init Node "demo_storage2"
+alpen2-1  | Feb 26 23:05:02 WARNING >> [Worker#1] Node file "/data/ALPENHORN_NODE" could not be read.
+alpen2-1  | Feb 26 23:05:02 WARNING >> [Worker#1] Node file "/data/ALPENHORN_NODE" could not be read.
+alpen2-1  | Feb 26 23:05:02 INFO >> [Worker#1] Node "demo_storage2" initialised.
+alpen2-1  | Feb 26 23:05:02 INFO >> [Worker#1] Finished task: Init Node "demo_storage2"
+alpen2-1  | Feb 26 23:05:12 INFO >> [MainThread] Node "demo_storage2" now available.
+alpen2-1  | Feb 26 23:05:12 INFO >> [MainThread] Group "demo_storage2" now available.
+alpen2-1  | Feb 26 23:05:12 INFO >> [MainThread] Node demo_storage2: 45.51 GiB available.
+alpen2-1  | Feb 26 23:05:12 INFO >> [MainThread] Updating node "demo_storage2".
+alpen2-1  | Feb 26 23:05:12 INFO >> [MainThread] Updating group "demo_storage2".
+alpen2-1  | Feb 26 23:05:12 INFO >> [MainThread] Main loop execution was 0.0s.
+alpen2-1  | Feb 26 23:05:12 INFO >> [MainThread] Tasks: 1 queued, 0 deferred, 0 in-progress on 2 workers
+alpen2-1  | Feb 26 23:05:12 INFO >> [Worker#1] Beginning task Tidy up demo_storage2
+alpen2-1  | Feb 26 23:05:12 INFO >> [Worker#1] Finished task: Tidy up demo_storage2
+```
    
+### Transferring a file
+
+The alpenhorn daemon has the ability to transfer files between Storage Nodes.  To trigger
+file movement, we need to issue sync or transfer requests.  Transfer requests _always_
+request movement of a file from a Storage Node into a Storage Group.  Because all the
+groups we have for now have a single node in them, this distinction isn't terribly important,
+but we'll revisit this later, when we experiment with multi-node groups.
+
+We can transfer any existing file explicitly by issuing a transfer request for it:
+```
+alpenhorn file sync --from demo_storage1 --to demo_storage2 2025/02/21/meta.txt
+```
+
+This will submit a new transfer request:
+```(console)
+root@alpen1:/# alpenhorn file sync --from demo_storage1 --to demo_storage2 2025/02/21/meta.txt
+Request submitted.
+```
+Transfers are always handled on the receiving side.  After, perhaps, a short while, the
+daemon on `alpen2` will notice this request.  First, it will look at the local filesystem to
+see if the requested file already exists.  If it did, there would be no need for a transfer:
+```
+alpen2-1  | Feb 26 23:18:52 INFO >> [Worker#2] Beginning task Pre-pull search for 2025/02/21/meta.txt in demo_storage2
+alpen2-1  | Feb 26 23:18:52 INFO >> [Worker#2] Finished task: Pre-pull search for 2025/02/21/meta.txt in demo_storage2
+```
+
+But, in this case, the search will fail to find an existing copy of the file, so then a file
+transfer will be started:
+```
+alpen2-1  | Feb 26 23:18:52 INFO >> [Worker#1] Beginning task AFCR#1: demo_storage1 -> demo_storage2
+alpen2-1  | Feb 26 23:18:52 INFO >> [Worker#1] Creating directory "/data/2025/02/21".
+alpen2-1  | Feb 26 23:18:52 INFO >> [Worker#1] Pulling remote file 2025/02/21/meta.txt using rsync
+alpen2-1  | Feb 26 23:18:52 INFO >> [Worker#1] Pull of 2025/02/21/meta.txt complete. Transferred 51 B in 0.4s [139 B/s]
+alpen2-1  | Feb 26 23:18:52 INFO >> [Worker#1] Finished task: AFCR#1: demo_storage1 -> demo_storage2
+```
+
+The default tool for remote transfers is `rsync`, but alpenhorn will also try to use
+[bbcp](https://www.slac.stanford.edu/~abh/bbcp/), a GridFTP implementation, which may allow for
+higher-rate transfers, if it is available on to the daemon.
+
+Now there is one file on `demo_storage2`:
+```
+root@alpen1:/# alpenhorn node stats
+Name             File Count    Total Size    % Full
+-------------  ------------  ------------  --------
+demo_storage1             5         101 B         -
+demo_storage2             1          51 B         -
+```
+
+You can check the filesystem on `alpen2` to see that this file now exists on that node:
+```(console)
+$ docker compose exec alpen2 find /data
+/data
+/data/ALPENHORN_NODE
+/data/2025
+/data/2025/02
+/data/2025/02/21
+/data/2025/02/21/meta.txt
+```
+
+### Bulk transfers
+
+Rather than the tedious operation of requesting individual files for transfer, it is more typical
+to request _all_ files present on a source node and absent from a destination group be transferred:
+```
+alpenhorn node sync demo_storage1 demo_storage2 --show-files
+```
+This will cause the alpenhorn CLI to create transfer requests for all files not present on
+`demo_storage2` which are present on `demo_storage1`.
+
+This command will require confirmation:
+```
+root@alpen1:/# alpenhorn node sync demo_storage1 demo_storage2 --show-files
+Would sync 4 files (50 B) from Node "demo_storage1" to Group "demo_storage2":
+
+2025/02/21/23/1324.dat
+2025/02/21/23/1330.dat
+2025/02/21/23/1342.dat
+2025/02/21/23/1349.dat
+
+Continue? [y/N]: y
+
+Syncing 4 files (50 B) from Node "demo_storage1" to Group "demo_storage2".
+
+Added 4 new copy requests.
+```
+Although there are five files on the node, only four of them will be transferred, because
+the first file we transferred is already on `demo_storage2`.
+
+The daemon on alpen2 will churn through these requests:
+```
+alpen2-1  | Feb 26 23:34:32 INFO >> [Worker#2] Beginning task Pre-pull search for 2025/02/21/23/1330.dat in demo_storage2
+alpen2-1  | Feb 26 23:34:32 INFO >> [Worker#2] Finished task: Pre-pull search for 2025/02/21/23/1330.dat in demo_storage2
+alpen2-1  | Feb 26 23:34:32 INFO >> [Worker#1] Beginning task AFCR#2: demo_storage1 -> demo_storage2
+alpen2-1  | Feb 26 23:34:32 INFO >> [Worker#2] Beginning task Pre-pull search for 2025/02/21/23/1324.dat in demo_storage2
+alpen2-1  | Feb 26 23:34:32 INFO >> [Worker#1] Creating directory "/data/2025/02/21/23".
+alpen2-1  | Feb 26 23:34:32 INFO >> [Worker#2] Finished task: Pre-pull search for 2025/02/21/23/1324.dat in demo_storage2
+alpen2-1  | Feb 26 23:34:32 INFO >> [Worker#2] Beginning task AFCR#3: demo_storage1 -> demo_storage2
+alpen2-1  | Feb 26 23:34:32 INFO >> [Worker#2] Pulling remote file 2025/02/21/23/1324.dat using rsync
+alpen2-1  | Feb 26 23:34:32 INFO >> [Worker#1] Pulling remote file 2025/02/21/23/1330.dat using rsync
+alpen2-1  | Feb 26 23:34:32 INFO >> [MainThread] Main loop execution was 0.1s.
+alpen2-1  | Feb 26 23:34:32 INFO >> [MainThread] Tasks: 2 queued, 0 deferred, 2 in-progress on 2 workers
+alpen2-1  | Feb 26 23:34:32 INFO >> [Worker#2] Pull of 2025/02/21/23/1324.dat complete. Transferred 12 B in 0.3s [36 B/s]
+alpen2-1  | Feb 26 23:34:32 INFO >> [Worker#1] Pull of 2025/02/21/23/1330.dat complete. Transferred 12 B in 0.3s [36 B/s]
+alpen2-1  | Feb 26 23:34:32 INFO >> [Worker#2] Finished task: AFCR#3: demo_storage1 -> demo_storage2
+alpen2-1  | Feb 26 23:34:32 INFO >> [Worker#1] Finished task: AFCR#2: demo_storage1 -> demo_storage2
+alpen2-1  | Feb 26 23:34:32 INFO >> [Worker#2] Beginning task Pre-pull search for 2025/02/21/23/1349.dat in demo_storage2
+alpen2-1  | Feb 26 23:34:32 INFO >> [Worker#1] Beginning task Pre-pull search for 2025/02/21/23/1342.dat in demo_storage2
+alpen2-1  | Feb 26 23:34:32 INFO >> [Worker#2] Finished task: Pre-pull search for 2025/02/21/23/1349.dat in demo_storage2
+alpen2-1  | Feb 26 23:34:32 INFO >> [Worker#2] Beginning task AFCR#4: demo_storage1 -> demo_storage2
+alpen2-1  | Feb 26 23:34:32 INFO >> [Worker#1] Finished task: Pre-pull search for 2025/02/21/23/1342.dat in demo_storage2
+alpen2-1  | Feb 26 23:34:32 INFO >> [Worker#1] Beginning task AFCR#5: demo_storage1 -> demo_storage2
+alpen2-1  | Feb 26 23:34:32 INFO >> [Worker#2] Pulling remote file 2025/02/21/23/1349.dat using rsync
+alpen2-1  | Feb 26 23:34:32 INFO >> [Worker#1] Pulling remote file 2025/02/21/23/1342.dat using rsync
+alpen2-1  | Feb 26 23:34:32 INFO >> [Worker#1] Pull of 2025/02/21/23/1342.dat complete. Transferred 12 B in 0.4s [32 B/s]
+alpen2-1  | Feb 26 23:34:32 INFO >> [Worker#2] Pull of 2025/02/21/23/1349.dat complete. Transferred 14 B in 0.4s [37 B/s]
+alpen2-1  | Feb 26 23:34:32 INFO >> [Worker#1] Finished task: AFCR#5: demo_storage1 -> demo_storage2
+alpen2-1  | Feb 26 23:34:32 INFO >> [Worker#2] Finished task: AFCR#4: demo_storage1 -> demo_storage2
+```
+
+And eventually all files will be transfered to `alpen2`:
+```
+root@alpen1:/# alpenhorn node stats
+Name             File Count    Total Size    % Full
+-------------  ------------  ------------  --------
+demo_storage1             5         101 B         -
+demo_storage2             5         101 B         -
+```
+
+Note: If you were to try the identical sync request a second time, after alpen2 has finished all the
+transfers, alpenhorn will decide that nothing needs transferring:
+```(console)
+root@alpen1:/# alpenhorn node sync demo_storage1 demo_storage2 --show-files
+No files to sync.
+```
+
+One last note on the `node sync` command: if you prefer thinking about the destination side
+of transfers, you can use `group sync` to peform the same task.
+
+The command
+```
+alpenhorn node sync demo_storage1 demo_storage2 --show-files
+```
+is equivalent to
+```
+alpenhorn group sync demo_storage2 demo_storage1 --show-files
+```
+though note that with `node sync` the arguments are source node and then destination group
+but with `group sync` these are reversed: the first argument is the destination group and the
+second argument the source node.
+
+## Dealing with corruption
+
+More than just helping you copy files around, alpenhorn can monitor your files for corruption.
+
+### MD5 Digest Hashes
+
+Although, as mentioned earlier, alpenhorn doesn't really know what's in the files its managing,
+whenever it registers a new file, it computes the MD5 digest hash for the file.  This means that,
+if a file is changed after registration, alpenhorn can detect this change by re-computing the
+MD5 hash and comparing it to the hash value it recorded when first registering the file.
+
+You can see the stored hash value for a file using the alpenhorn CLI:
+```(console)
+root@alpen1:/# alpenhorn file show 2025/02/21/23/1324.dat                  
+       Name: 23/1324.dat
+Acquisition: 2025/02/21
+       Path: 2025/02/21/23/1324.dat
+
+       Size: 12 B
+   MD5 Hash: 4c79018e00ddef11af0b9cfc14dd3261
+ Registered: Thu Mar  6 22:54:37 2025 UTC
+```
+
+If we were to manually compute the MD5 digest for this file we would get the same result:
+```(console)
+root@alpen1:/# md5sum /data/2025/02/21/23/1324.dat
+4c79018e00ddef11af0b9cfc14dd3261  /data/2025/02/21/23/1324.dat
+```
+
+Let's corrupt a file by changing its contents:
+
+```
+echo "bad data" > /data/2025/02/21/23/1324.dat
+```
+
+Now if we manually compute the MD5 hash, we can see that's it's different than
+what alpenhorn has recorded:
+
+```(console)
+root@alpen1:/# md5sum /data/2025/02/21/23/1324.dat
+3412f7b66a30b90ae3d3085c96615f00  /data/2025/02/21/23/1324.dat
+```
+
+However, alpenhorn hasn't noticed this:
+
+```(console)
+root@alpen1:/# alpenhorn node stats --extra-stats
+Name             File Count    Total Size    % Full    Corrupt Files    Suspect Files    Missing Files
+-------------  ------------  ------------  --------  ---------------  ---------------  ---------------
+demo_storage1             5         101 B         -                -                -                -
+demo_storage2             5         101 B         -                -                -                -
+```
+It still lists no corrupt files on `demo_storage1`.  This is because alpenhorn
+doesn't normally automatically detect corruption to files it is managing.  You
+can turn on "auto-verify" on a node, but that won't result in instantaneous detection
+of corruption either, and can be I/O expensive, (and, so, should be used with caution).
+
+In some cases, file corruption will be detected by alenhorn when copying an unexpectedly corrupt
+file from one node to another.  For now, we can manually request a verification of the file.
+We'll do this by requesting verifciation for the entire acqusition, even though we've only
+corrupted one of the files.
+
+To request verifcation of all files in the acqusition on the node `demo_storage1`, run:
+```
+alpenhorn node verify --all --acq=2025/02/21 demo_storage1
+```
+
+You will have to confirm this request:
+```(console)
+root@alpen1:/# alpenhorn node verify --all --acq=2025/02/21 demo_storage1
+Would request verification of 5 files (101 B).
+
+Continue? [y/N]: y
+
+Requesting verification of 5 files (101 B).
+Updated 5 files.
+```
+
+The daemon on `alpen1` will respond to this command by re-verifying all files in that acqusition:
+```
+alpen1-1  | Mar 07 01:48:25 INFO >> [MainThread] Checking copy "2025/02/21/meta.txt" on node demo_storage1.
+alpen1-1  | Mar 07 01:48:25 INFO >> [MainThread] Checking copy "2025/02/21/23/1324.dat" on node demo_storage1.
+alpen1-1  | Mar 07 01:48:25 INFO >> [MainThread] Checking copy "2025/02/21/23/1330.dat" on node demo_storage1.
+alpen1-1  | Mar 07 01:48:25 INFO >> [MainThread] Checking copy "2025/02/21/23/1349.dat" on node demo_storage1.
+alpen1-1  | Mar 07 01:48:25 INFO >> [MainThread] Checking copy "2025/02/21/23/1342.dat" on node demo_storage1.
+alpen1-1  | Mar 07 01:48:25 ERROR >> [Worker#2] File 2025/02/21/23/1324.dat on node demo_storage1 is corrupt! Size: 9; expected: 12
+alpen1-1  | Mar 07 01:48:25 INFO >> [Worker#2] Updating file copy #2 for file 2025/02/21/23/1324.dat on node demo_storage1.
+alpen1-1  | Mar 07 01:48:25 INFO >> [MainThread] Updating group "demo_storage1".
+alpen1-1  | Mar 07 01:48:25 INFO >> [Worker#2] Finished task: Check file 2025/02/21/23/1324.dat on demo_storage1
+alpen1-1  | Mar 07 01:48:25 INFO >> [Worker#2] Beginning task Check file 2025/02/21/23/1330.dat on demo_storage1
+alpen1-1  | Mar 07 01:48:25 INFO >> [MainThread] Main loop execution was 0.0s.
+alpen1-1  | Mar 07 01:48:25 INFO >> [Worker#1] File 2025/02/21/meta.txt on node demo_storage1 is A-OK!
+alpen1-1  | Mar 07 01:48:25 INFO >> [MainThread] Tasks: 2 queued, 0 deferred, 2 in-progress on 2 workers
+alpen1-1  | Mar 07 01:48:25 INFO >> [Worker#1] Updating file copy #1 for file 2025/02/21/meta.txt on node demo_storage1.
+alpen1-1  | Mar 07 01:48:25 INFO >> [Worker#1] Finished task: Check file 2025/02/21/meta.txt on demo_storage1
+alpen1-1  | Mar 07 01:48:25 INFO >> [Worker#1] Beginning task Check file 2025/02/21/23/1349.dat on demo_storage1
+alpen1-1  | Mar 07 01:48:25 INFO >> [Worker#2] File 2025/02/21/23/1330.dat on node demo_storage1 is A-OK!
+alpen1-1  | Mar 07 01:48:25 INFO >> [Worker#2] Updating file copy #3 for file 2025/02/21/23/1330.dat on node demo_storage1.
+alpen1-1  | Mar 07 01:48:25 INFO >> [Worker#2] Finished task: Check file 2025/02/21/23/1330.dat on demo_storage1
+alpen1-1  | Mar 07 01:48:25 INFO >> [Worker#2] Beginning task Check file 2025/02/21/23/1342.dat on demo_storage1
+alpen1-1  | Mar 07 01:48:25 INFO >> [Worker#1] File 2025/02/21/23/1349.dat on node demo_storage1 is A-OK!
+alpen1-1  | Mar 07 01:48:25 INFO >> [Worker#1] Updating file copy #4 for file 2025/02/21/23/1349.dat on node demo_storage1.
+alpen1-1  | Mar 07 01:48:25 INFO >> [Worker#2] File 2025/02/21/23/1342.dat on node demo_storage1 is A-OK!
+alpen1-1  | Mar 07 01:48:25 INFO >> [Worker#2] Updating file copy #5 for file 2025/02/21/23/1342.dat on node demo_storage1.
+alpen1-1  | Mar 07 01:48:25 INFO >> [Worker#1] Finished task: Check file 2025/02/21/23/1349.dat on demo_storage1
+alpen1-1  | Mar 07 01:48:25 INFO >> [Worker#2] Finished task: Check file 2025/02/21/23/1342.dat on demo_storage1
+```
+As you can see, it has discovered our corruption of `2025/02/21/23/1324.dat`.
+
+Now if we check the node stats, we can see one corrupt file on this node.
+```
+root@alpen1:/# alpenhorn node stats --extra-stats
+Name             File Count    Total Size    % Full    Corrupt Files    Suspect Files    Missing Files
+-------------  ------------  ------------  --------  ---------------  ---------------  ---------------
+demo_storage1             4          89 B         -                1                -                -
+demo_storage2             5         101 B         -                -                -                -
+root@alpen1:/# alpenhorn file state 2025/02/21/23/1324.dat demo_storage1
+Corrupt Ready
+```
+Also note that the file count for `demo_storage1` is down to four: a known corrupt file is not
+considered "present" on a node, since it can't be used as a good copy of the file.
+
+## Recovering corrupt files
+
+The standard way to recover a corrupt file copy is to re-transfer a known-good copy of the
+file over top of the corrupt version.  We can do this by syncing the file back from `alpen2`:
+
+```
+alpenhorn node sync demo_storage2 demo_storage1
+```
+
+It will tell you there is only one file to transfer (the corrupt file) and ask for confirmation:
+```(console)
+root@alpen1:/# alpenhorn node sync demo_storage2 demo_storage1
+Would sync 1 file (12 B) from Node "demo_storage2" to Group "demo_storage1".
+
+Continue? [y/N]: y
+
+Syncing 1 file (12 B) from Node "demo_storage2" to Group "demo_storage1".
+
+Added 1 new copy request.
+```
+
+Wait for the daemon on `alpen1` to pull the file from `alpen2`:
+```
+alpen1-1  | Mar 07 01:52:15 INFO >> [Worker#1] Beginning task AFCR#6: demo_storage2 -> demo_storage1
+alpen1-1  | Mar 07 01:52:15 INFO >> [MainThread] Tasks: 0 queued, 0 deferred, 1 in-progress on 2 workers
+alpen1-1  | Mar 07 01:52:15 INFO >> [Worker#1] Pulling remote file 2025/02/21/23/1324.dat using rsync
+alpen1-1  | Mar 07 01:52:15 INFO >> [Worker#1] Pull of 2025/02/21/23/1324.dat complete. Transferred 12 B in 0.4s [32 B/s]
+alpen1-1  | Mar 07 01:52:15 INFO >> [Worker#1] Finished task: AFCR#6: demo_storage2 -> demo_storage1
+```
+
+After transferring the file back, now alpenhorn now considers the file healthy again:
+```(console)
+root@alpen1:/# alpenhorn node stats --extra-stats
+Name             File Count    Total Size    % Full    Corrupt Files    Suspect Files    Missing Files
+-------------  ------------  ------------  --------  ---------------  ---------------  ---------------
+demo_storage1             5         101 B         -                -                -                -
+demo_storage2             5         101 B         -                -                -                -
+```
+
+## Deleting files
+
+Typically you'll want to delete files off your acqusition nodes once they've been transferred
+off-site.  File deletion can be accomplished with the `clean` command.
+
+Since we've copied some files from `alpen1` to `alpen2`, let's try deleting one of the files from
+`alpen1`:
+```
+alpenhorn file clean --now --node=demo_storage1 2025/02/21/meta.txt
+```
+
+The CLI should release the file immediately:
+```(console)
+root@alpen1:/# alpenhorn file clean --now --node=demo_storage1 2025/02/21/meta.txt
+Released "2025/02/21/meta.txt" for immediate removal on Node "demo_storage1".
+```
+The `--now` flag tells alpenhorn to delete the file as soon as possible.  Without that flag,
+instead of being released for removal, the file is marked for "discretionary cleaning", which
+tells alpenhorn that it can decide to delete the file if it wants to clear space on the node, but
+in this demo alpenhorn would never decide to do that, so we'll opt for immediate removal.
+
+Despite our request, if you look at the daemon log on `alpen1`, you'll see that it's refused to
+delete the file:
+```
+alpen1-1  | Mar 07 02:21:25 INFO >> [MainThread] Tasks: 0 queued, 0 deferred, 1 in-progress on 2 workers
+alpen1-1  | Mar 07 02:21:25 WARNING >> [Worker#1] Too few archive copies (0) to delete 2025/02/21/meta.txt on demo_storage1.
+alpen1-1  | Mar 07 02:21:25 INFO >> [Worker#1] Finished task: Delete copies [1] from demo_storage1
+```
+
+To prevent data loss, alpenhorn will only delete file copies from a node if at least two other
+copies of the file exist on other archive nodes.  Currently we have no archive nodes, so we can't
+delete files.
+
+Let's fix that.  While we do, the `alpen1` daemon will keep checking whether it can delete that
+file.
+
+### Archive nodes
+
+An archive node is any storage node with the "archive" storage type.  Let's change `demo_storage2`
+into an archive node.  We do that by modifying it's metadata:
+```
+alpenhorn node modify --archive demo_storage2
+```
+
+After running this command, you can look at the node metadata to see that it now has the "archive"
+storage type:
+```(console)
+root@alpen1:/# alpenhorn node modify --archive demo_storage2
+Node updated.
+root@alpen1:/# alpenhorn node show demo_storage2
+   Storage Node: demo_storage2
+  Storage Group: demo_storage2
+         Active: Yes
+           Type: Archive
+          Notes: 
+      I/O Class: Default
+
+    Daemon Host: alpen2
+ Log-in Address: alpen2
+Log-in Username: root
+
+    Auto-Import: Off
+    Auto-Verify: Off
+      Max Total: -
+      Available: 45.38 GiB
+  Min Available: -
+   Last Checked: Fri Mar  7 02:27:47 2025 UTC
+
+I/O Config:
+
+  none
+```
+
+Now if we look at the `alpen1` daemon log, the file it's trying to delete is now found on
+one archive node (out of the two needed):
+```
+alpen1-1  | Mar 07 02:28:55 INFO >> [MainThread] Tasks: 0 queued, 0 deferred, 1 in-progress on 2 workers
+alpen1-1  | Mar 07 02:28:55 WARNING >> [Worker#1] Too few archive copies (1) to delete 2025/02/21/meta.txt on demo_storage1.
+alpen1-1  | Mar 07 02:28:55 INFO >> [Worker#1] Finished task: Delete copies [1] from demo_storage1
+```
+
+We'll need another archive node with this file on it if we want the deletion to happen.  So, let's
+set up the final storage host, `alpen3`.
+
+First let's create the storage node in the database.  We'll make this one an archive node when we
+create it:
+```
+alpenhorn node create demo_storage3 --create-group --archive --root=/data --host=alpen3 \
+                                    --username root --address alpen3 --init --activate
+```
+The `--init` and `--activate` flags save us from having to run those commands on the new node later.
+
+Now let's start the third docker container and take a look at its logs:
+```
+docker compose up --detach alpen3
+docker compose logs --follow alpen3
+```
+
+Sync everything on `demo_storage2` to `demo_storage3`:
+```
+alpenhorn node sync --force demo_storage2 demo_storage3
+```
+Using `--force` here skips the confirmation step.  You can use `--force` with any alpenhorn
+command that would ask for confirmation, but you should be careful when using it.
+
+As soon as the file is transferred to `demo_storage3`, the daemon on `alpen1` will finally
+delete the file:
+
+```
+alpen1-1  | Mar 07 02:38:45 INFO >> [Worker#1] Beginning task Delete copies [1] from demo_storage1
+alpen1-1  | Mar 07 02:38:45 INFO >> [Worker#1] Removed file copy 2025/02/21/meta.txt on demo_storage1
+alpen1-1  | Mar 07 02:38:45 INFO >> [Worker#1] Finished task: Delete copies [1] from demo_storage1
+alpen1-1  | Mar 07 02:38:45 INFO >> [MainThread] Main loop execution was 0.1s.
+alpen1-1  | Mar 07 02:38:45 INFO >> [MainThread] Tasks: 0 queued, 0 deferred, 0 in-progress on 2 workers
+```
+
+Now there are only four files on `demo_storage1`:
+```(console)
+root@alpen1:/# alpenhorn node stats --extra-stats
+Name             File Count    Total Size    % Full    Corrupt Files    Suspect Files    Missing Files
+-------------  ------------  ------------  --------  ---------------  ---------------  ---------------
+demo_storage1             4          50 B         -                -                -                -
+demo_storage2             5         101 B         -                -                -                -
+demo_storage3             5         101 B         -                -                -                -
+```
+
+Rather than cleaning individual files, we can do bulk operations.  To tell alpenhorn to delete
+everything from `demo_storage1` that already exists on `demo_storage3`:
+```
+alpenhorn node clean demo_storage1 --now --target demo_storage3
+```
+
+It will find four files to clean, which you'll have to confirm:
+```
+root@alpen1:/# alpenhorn node clean demo_storage1 --now --target demo_storage3
+Would release 4 files (50 B).
+
+Continue? [y/N]: y
+
+Releasing 4 files (50 B).
+Updated 4 files.
+```
+
+The files will be removed from `demo_storage1` by the daemon:
+```
+alpen1-1  | Mar 07 02:43:05 INFO >> [Worker#1] Beginning task Delete copies [2, 3, 4, 5] from demo_storage1
+alpen1-1  | Mar 07 02:43:05 INFO >> [Worker#1] Removed file copy 2025/02/21/23/1324.dat on demo_storage1
+alpen1-1  | Mar 07 02:43:05 INFO >> [Worker#1] Removed file copy 2025/02/21/23/1330.dat on demo_storage1
+alpen1-1  | Mar 07 02:43:05 INFO >> [Worker#1] Removed file copy 2025/02/21/23/1349.dat on demo_storage1
+alpen1-1  | Mar 07 02:43:05 INFO >> [Worker#1] Removed file copy 2025/02/21/23/1342.dat on demo_storage1
+alpen1-1  | Mar 07 02:43:05 INFO >> [Worker#1] Removed directory /data/2025/02/21/23 on demo_storage1
+alpen1-1  | Mar 07 02:43:05 INFO >> [Worker#1] Removed directory /data/2025/02/21 on demo_storage1
+alpen1-1  | Mar 07 02:43:05 INFO >> [Worker#1] Removed directory /data/2025/02 on demo_storage1
+alpen1-1  | Mar 07 02:43:05 INFO >> [Worker#1] Removed directory /data/2025 on demo_storage1
+alpen1-1  | Mar 07 02:43:05 INFO >> [Worker#1] Finished task: Delete copies [2, 3, 4, 5] from demo_storage1
+```
+Note that the daemon will also delete directories on the node which end up empty after file deletion
+to keep the storage node directory tree tidy.
+
+Now `demo_storage1` is empty:
+```(console)
+root@alpen1:/# alpenhorn node stats --extra-stats
+Name             File Count    Total Size    % Full    Corrupt Files    Suspect Files    Missing Files
+-------------  ------------  ------------  --------  ---------------  ---------------  ---------------
+demo_storage1             0             -         -                -                -                -
+demo_storage2             5         101 B         -                -                -                -
+demo_storage3             5         101 B         -                -                -                -
+root@alpen1:/# find /data
+/data
+/data/ALPENHORN_NODE
+```
+
+## Transport disks and the Sneakernet
+
+Alpenhorn has been designed to work with instruments in remote locations where network transport
+of data may be difficult or impossible to accomplish.  To help with this situation, alpenhorn
+can be used to manage transfer of data via physically moving storage media from site to site.
+(This is known as the Sneakernet).
+
+Alpenhorn can be configured to copy data onto a set of physical media at one location where
+data are produced and then, later, copy data off those media once they have been transported
+to a data ingest site.
+
+To demonstrate this, we'll use a transport device to simulate transferring data back from
+`demo_storage3` to `demo_storage1`, as if these two nodes were unable to communicate directly
+over the network.
+
+### The Transport Group and Transport Nodes
+
+In alpenhorn, each individual physical device holding data to transfer is represent by its own
+StorageNode which has the "transport" storage type.  All the transport nodes are collected into
+a StorageGroup which has the "Transport" I/O class.
+
+Our first job, then, is to create a transport group:
+```
+alpenhorn group create --class=Transport transport_group
+```
+This has I/O class "Transport" (the capital "T" is important).  Typically you only ever need one
+transport group, and you put all your transport nodes in the single group.  Normal logistics of
+the Sneakernet mean that typically different member nodes of this group will be located at different
+sites and/or be in-transit at any given time, and the locations of the nodes will change over time.
+Alpenhorn never requires, nor expects, multiple nodes in the transport group to be accessible to a
+single daemon.
+
+Now that we have the transport group, we can create storage nodes to put in it.  As mentioned
+above, each node is a single physical device (disk, tape, etc.) which is transferred through the
+Snearkernet.  Multiple nodes in the group can be available at a particular site, but we'll just
+create a single node for the purpose of this demo.
+
+When we create the new node, we'll tell alpenhorn that it's initally available on `alpen3`:
+```
+alpenhorn node create transport1 --transport --group transport_group --host=alpen3 \
+                                 --root=/mnt/transport --init --activate
+```
+Note the use of the `--transport` flag to set the node's storage type to "transport".  The
+"Transport" Group I/O class allows StorageNodes of any class to be added to the group, but
+requires all such nodes to have the "transport" storage type.
+
+The filesystem has already been made available in the `alpen3` container, so wait for the
+daemon on `alpen3` to initialise the node:
+```
+alpen3-1  | Mar 07 09:21:03 INFO >> [MainThread] Node "transport1" now available.
+alpen3-1  | Mar 07 09:21:03 WARNING >> [MainThread] Node file "/mnt/transport/ALPENHORN_NODE" could not be read.
+alpen3-1  | Mar 07 09:21:03 INFO >> [MainThread] Requesting init of node "transport1".
+alpen3-1  | Mar 07 09:21:03 INFO >> [Worker#1] Beginning task Init Node "transport1"
+alpen3-1  | Mar 07 09:21:03 WARNING >> [Worker#1] Node file "/mnt/transport/ALPENHORN_NODE" could not be read.
+alpen3-1  | Mar 07 09:21:03 WARNING >> [Worker#1] Node file "/mnt/transport/ALPENHORN_NODE" could not be read.
+alpen3-1  | Mar 07 09:21:03 INFO >> [Worker#1] Node "transport1" initialised.
+```
+
+### Copying Data to the Transport Group
+
+Remember that, when copy files, data always flows from a node to a group.  To get
+data onto the transport node, we need to transfer data into the transport group.  Logic
+defined by the Transport I/O class then determines which of the available transport nodes the
+transferred files will be written to.
+
+Briefly, the Transport logic works like this:
+* only local transfers are allowed into the group (i.e. transferring into the transport
+  group will only ever copy data onto transport nodes at the same location as the source
+  node).
+* the transport group will try to fill up one transport node before copying data to another
+* all other things being equal, all transport nodes have the same priority for accepting data
+
+In our case we only have a single transport node, so it's easy to figure out which node the
+data will end up on.
+
+Let's transfer data out of `demo_storage3` into the transport group with the intent of
+transferring data, ultimately, to `demo_storage1`:
+
+```
+alpenhorn node sync demo_storage3 transport_group --target=demo_storage1
+```
+The `--target` option indicates to alpenhorn the ultimate destination for the data
+we're syncing to transport.  This prevents alpenhorn from trying to transfer data already
+present on `demo_storage1` (though in our case, that's nothing).
+
+This should sync all five files we have:
+```
+root@alpen1:/# alpenhorn node sync demo_storage3 transport_group --target=demo_storage1
+Would sync 5 files (101 B) from Node "demo_storage3" to Group "transport_group".
+
+Continue? [y/N]: y
+
+Syncing 5 files (101 B) from Node "demo_storage3" to Group "transport_group".
+
+Added 5 new copy requests.
+```
+It may also be good to point out here that even though both `demo_storage3` and the
+transport node are on `alpen3`, and all the resulting transfers are local, we can
+run this command on `alpen1`.  Running commands with the CLI never need to occur
+where the storage nodes referenced are.  Anywhere that can access the alpenhorn
+databse can be used to run any alpenhorn command.
+
+After waiting for the daemon to process these requests, a look at the trasnport group
+should show us that they've all ended up on the transport node:
+```
+root@alpen1:/# alpenhorn group show --node-stats transport_group
+Storage Group: transport_group
+        Notes: 
+    I/O Class: Transport
+
+I/O Config:
+
+  none
+
+Nodes:
+
+Name          File Count  Total Size    % Full
+----------  ------------  ------------  --------
+transport1             5  101 B         -
+```
+
+### Transporting the transport node
+
+Now let's simulate what would happen if we wanted to move this transport node
+from `alpen3` to `alpen1` over the Snearkernet.  (Normally, to increase throughput
+of the Snearkernet, we would wait for the node to fill up, but we're not going to
+wait for that in this demo.)
+
+The first step is to deactivate the alpenhorn node to tell alpenhorn to stop
+managing it:
+```
+alpenhorn node deactivate transport1
+```
+
+The daemon on `alpen3` will notice this, and stop updating the transport node:
+```
+alpen3-1  | Mar 09 04:10:07 INFO >> [MainThread] Node "transport1" no longer available.
+alpen3-1  | Mar 09 04:10:07 INFO >> [MainThread] Group "transport_group" no longer available.
+```
+
+If this were a real transport device, the next steps would be to:
+* unmount the filesystem
+* eject the media
+* remove the physical storage device from the machine
+
+Next, the trasnport device would need to travel (via Sneakernet) from the site containing
+`alpen3` to the site containing `alpen1` where we would do the reverse procedure, installing
+the device in the `alpen1` machine and mounting the device's filesystem.
+
+For the purposes of this demo, however, we don't have to do any of that: the docker volume we're
+using to simulate the transport device has already been made available in the `alpen1` container,
+so let's proceed with the last step of the transport process, which is to update the alpenhorn
+data index to record the movement of the transport device.
+
+In addition to activating the node to tell alpenhorn to start managing it again, there are,
+generally, four fields we may need to upate for a transport device after it's been moved:
+* its `host`, to let alpenhorn know which daemon should now be able to access the disk
+* its `username` and `address` to set the log-in details for remote access to the device.  If
+  remote access to the transport node isn't needed, this may not be necessary to do.
+* its `root` to tell alpenhorn where we have mounted the transport device's filesystem.
+
+We can do this all using the `node activate` command, which has been designed with this use
+case in mind:
+```
+alpenhorn node activate transport1 --host=alpen1 --username=root --address=alpen1 \
+                                   --root=/mnt/transport
+```
+
+The node will now appear to the daemon on `alpen1`:
+```
+alpen1-1  | Mar 09 04:21:44 INFO >> [MainThread] Node "transport1" now available.
+alpen1-1  | Mar 09 04:21:44 INFO >> [MainThread] Group "transport_group" now available.
+```
+
+Now let's now copy all the data off the transport media onto the `demo_storage1` node to
+complete our long-distance transfer:
+```
+alpenhorn node sync transport1 demo_storage1
+```
+
+Once the transfers are complete, we've now got data back on `demo_storage1` courtesy
+our transport media:
+```(console)
+root@alpen1:/# alpenhorn node stats
+Name             File Count    Total Size    % Full
+-------------  ------------  ------------  --------
+demo_storage1             5         101 B         -
+demo_storage2             5         101 B         -
+demo_storage3             5         101 B         -
+transport1                5         101 B         -
+```
+
+Once we're happy with the transfer off of the transport device, we'll want to clear it
+out so we can ship it back to `alpen3` to be used to transfer more data in the future:
+
+```
+alpenhorn node clean --now --force transport1 --target=demo_storage1
+```
+The `--target` option ensures we only delete files from `transport1` which are present
+on `demo_storage1`.
+
+## Next steps
 
-Add a new file
---------------
-
-11. create a new file
-    ```
-    docker compose exec host_2 bash -l
-    echo foo bar > /data/12345678T000000Z_inst_zab/jim.out
-    ```
-
-12. see the new file get synced
-    ```
-    alpenhorn sync --acq 12345678T000000Z_inst_zab node_2 group_3 --show_files
-    alpenhorn status
-    docker compose up host_3
-    ```
-    
-    Highlight "transferring file ... jim.out"
-    
-13. See how many files are present
-    ```
-    alpenhorn status
-    ```
-
-Dealing with corruption
------------------------
-
-14. Modify a copy of a file:
-    ```
-    alpenhorn status
-    echo bla >> /data/12345678T000000Z_inst_zab/jim.out
-    alpenhorn status
-    ```
-
-    Notice in the output of the service:
-
-    ```
-    > INFO >> Checking file "/data/12345678T000000Z_inst_zab/jim.out" on node "node_2".
-    > ERROR >> File is corrupted!
-    > INFO >> Updating file copy status [id=6].
-    ```
-
-15. Repair with a sync from a known good copy:
-
-    ```
-    alpenhorn sync --acq 12345678T000000Z_inst_zab node_1 group_2 --show_files
-    alpenhorn status
-    ```
-
-Cleaning
---------
-
-16. Remove unneeded files with `clean`:
-    ```
-    alpenhorn clean --now node_2
-    ```
-
-    Observe only two are removed: "Too few backups to delete
-    2017/03/21/acq_xy1_45678901T000000Z_inst_zab/acq_data/x_123_1_data/raw/acq_123_1.zxc".
-    This is because this acquisition was never synced to node_3
-
-
-Transport disks
----------------
-
-17. Create storage
-
-    ```
-    alpenhorn create_group transport
-    alpenhorn create_node --storage_type=T t1 /mnt/t1 host_1 transport
-    ```
-
-18. Make it available
-
-    ```
-    alpenhorn mount --address=host_1 t1
-    ```
-
-19. Copy to transport for a target destination:
-
-    ```
-    alpenhorn sync node_1 transport --target group_3 --show_files
-    ```
-    
-    Note files being synced
-    ```
-    alpenhorn status
-    find /mnt/t1
-    alpenhorn unmount t1
-    ```
-    
-20. On the other side (host_3)...
-    ```
-    alpenhorn mount --address=host_3 t1
-    alpenhorn sync t1 group_3 --show_files
-    ```
-
-    Note files being synced
-    ```
-    alpenhorn status
-    alpenhorn clean --now t1
-    alpenhorn status
-    find /mnt/t1
-    ```
+This is the end of the curated part of the alpenhorn demo, but you can use this demo system to
+experiment with running alpenhorn.  Remember: you can always reset this demo to its inital state.
diff --git a/demo/docker-compose.yaml b/demo/docker-compose.yaml
index 02a222e..749de52 100644
--- a/demo/docker-compose.yaml
+++ b/demo/docker-compose.yaml
@@ -1,39 +1,43 @@
 volumes:
   db_vol:
+  node1_vol:
+  node2_vol:
+  node3_vol:
+  transport_vol:
 services:
-  db:
+  alpdb:
     image: mysql:latest
     environment:
       MYSQL_ALLOW_EMPTY_PASSWORD: 1
       MYSQL_DATABASE: alpenhorn_db
     volumes:
       - db_vol:/var/lib/mysql
-  host_1:
+  alpen1:
     build:
       context: ..
       dockerfile: demo/Dockerfile.alpenhorn
     image: alpenhorn
     command: sh -c "service ssh start && alpenhornd"
     depends_on:
-      - db
-    hostname: host_1
+      - alpdb
+    hostname: alpen1
     volumes:
-      - ./data/host_1:/data
-      - ./data/transport/t1:/mnt/t1
-  host_2:
+      - node1_vol:/data
+      - transport_vol:/mnt/transport
+  alpen2:
     image: alpenhorn
     command: sh -c "service ssh start && alpenhornd"
     depends_on:
-      - db
-    hostname: host_2
+      - alpdb
+    hostname: alpen2
     volumes:
-      - ./data/host_2:/data
-  host_3:
+      - node2_vol:/data
+  alpen3:
     image: alpenhorn
     command: sh -c "service ssh start && alpenhornd"
     depends_on:
-      - db
-    hostname: host_3
+      - alpdb
+    hostname: alpen3
     volumes:
-      - ./data/host_3:/data
-      - ./data/transport/t1:/mnt/t1
+      - node3_vol:/data
+      - transport_vol:/mnt/transport
diff --git a/examples/pattern_importer.py b/examples/pattern_importer.py
index 217831c..16117d2 100644
--- a/examples/pattern_importer.py
+++ b/examples/pattern_importer.py
@@ -103,7 +103,7 @@ class TypeBase(base_model):
 
         # Loop over patterns and check for matches
         for pattern in self._pattern_list:
-            result = re.match(pattern, name)
+            result = re.fullmatch(pattern, name)
             if result:
                 return result[0]
 
@@ -283,8 +283,8 @@ def detect(
 def demo_init() -> None:
     """Extension init for alpenhorn demo
 
-    This function initialised this extension for the alpenhorn demo
-    (see alpenhorn/dmeo/demo-script.md).
+    This function initialises the alpenhorn data index to support this extension
+    when used in the the alpenhorn demo (see alpenhorn/dmeo/demo-script.md).
 
     It creates the AcqType, FileType and extended ArchiveAcq and ArchiveFile
     tables and then populates the AcqType and FileType tables to allow the
@@ -309,24 +309,20 @@ def demo_init() -> None:
     # Populate AcqType.  There is only acq type in the demo
     AcqType.create(
         name="demo_acq",
-        patterns=json.dumps(
-            [
-                r"20[0-9][0-9]/(0[1-9]|1[012])/(0[1-9]|[12][0-9]|3[01])/acq_.*_inst_[a-z0-9]+"
-            ]
-        ),
+        patterns=json.dumps([r"20[0-9][0-9]/(0[1-9]|1[012])/(0[1-9]|[12][0-9]|3[01])"]),
         notes="AcqType for alpenhorn demo",
     )
 
     # Populate FileType.  There are two of these
     FileType.create(
-        name="acq_summary",
-        patterns=json.dumps([r"summary\.txt"]),
-        notes="Summary files for alpenhorn demo",
+        name="demo_data",
+        patterns=json.dumps([r"(0[0-9]|2[0-3])/[0-5][0-9][0-5][0-9].dat"]),
+        notes="Data files for alpenhorn demo",
     )
     FileType.create(
-        name="zcx",
-        patterns=json.dumps([r"acq_data/x_([0-9]*_[0-9])_data/raw/acq_\1\.zxc"]),
-        notes="ZXC file for alpenhorn demo",
+        name="demo_meta",
+        patterns=json.dumps([r"meta.txt$"]),
+        notes="Metadata file for alpenhorn demo",
     )
 
     # Indicate success


Test patch:
diff --git a/tests/cli/node/test_show.py b/tests/cli/node/test_show.py
index 03f71ca..b95dc4b 100644
--- a/tests/cli/node/test_show.py
+++ b/tests/cli/node/test_show.py
@@ -1,10 +1,13 @@
 """Test CLI: alpenhorn node show"""
 
+from datetime import timedelta
+
 from alpenhorn.db import (
     ArchiveAcq,
     ArchiveFile,
     ArchiveFileCopy,
     ArchiveFileCopyRequest,
+    ArchiveFileImportRequest,
     StorageGroup,
     StorageNode,
     StorageTransferAction,
@@ -226,6 +229,96 @@ def test_show_transfers(clidb, cli, assert_row_present):
     assert_row_present(result.output, "Group2", "1", "1.205 kiB")
 
 
+def test_show_imports(clidb, cli, assert_row_present):
+    """Test show --imports."""
+
+    group = StorageGroup.create(name="Group1")
+    node = StorageNode.create(name="Node", group=group)
+
+    time4 = utcnow().replace(microsecond=0)
+    time3 = time4 - timedelta(seconds=2)
+    time2 = time3 - timedelta(seconds=5)
+    time1 = time2 - timedelta(seconds=8)
+
+    ArchiveFileImportRequest.create(
+        node=node,
+        path="path/1",
+        recurse=False,
+        register=False,
+        completed=False,
+        timestamp=time1,
+    )
+    ArchiveFileImportRequest.create(
+        node=node,
+        path="path/2",
+        recurse=False,
+        register=False,
+        completed=True,
+        timestamp=time3,
+    )
+    ArchiveFileImportRequest.create(
+        node=node,
+        path="path/3",
+        recurse=False,
+        register=True,
+        completed=False,
+        timestamp=time4,
+    )
+    ArchiveFileImportRequest.create(
+        node=node,
+        path="path/4",
+        recurse=True,
+        register=False,
+        completed=False,
+        timestamp=time2,
+    )
+
+    # These are init requests
+    ArchiveFileImportRequest.create(
+        node=node,
+        path="ALPENHORN_NODE",
+        recurse=True,
+        register=False,
+        completed=False,
+        timestamp=time1,
+    )
+    ArchiveFileImportRequest.create(
+        node=node,
+        path="ALPENHORN_NODE",
+        recurse=True,
+        register=False,
+        completed=True,
+        timestamp=time2,
+    )
+    ArchiveFileImportRequest.create(
+        node=node,
+        path="ALPENHORN_NODE",
+        recurse=False,
+        register=False,
+        completed=True,
+        timestamp=time3,
+    )
+    ArchiveFileImportRequest.create(
+        node=node,
+        path="ALPENHORN_NODE",
+        recurse=False,
+        register=True,
+        completed=False,
+        timestamp=time4,
+    )
+
+    result = cli(0, ["node", "show", "Node", "--imports"])
+
+    assert_row_present(result.output, "[Node Init]", "-", "-", str(time1))
+    assert_row_present(result.output, "[Node Init]", "-", "-", str(time4))
+    assert result.output.count("[Node Init]") == 2
+
+    assert_row_present(result.output, "path/1", "No", "No", str(time1))
+    assert_row_present(result.output, "path/4", "Yes", "No", str(time2))
+    assert "path/2" not in result.output
+    assert_row_present(result.output, "path/3", "No", "Yes", str(time4))
+
+
 def test_show_all(clidb, cli, assert_row_present):
     """Test show --all."""
 
@@ -272,6 +365,25 @@ def test_show_all(clidb, cli, assert_row_present):
         node_from=node, group_to=group2, file=file, completed=1, cancelled=0
     )
 
+    # Imports (and inits)
+    now = utcnow().replace(microsecond=0)
+    ArchiveFileImportRequest.create(
+        node=node,
+        path="import/path",
+        recurse=False,
+        register=True,
+        completed=False,
+        timestamp=now,
+    )
+    ArchiveFileImportRequest.create(
+        node=node,
+        path="ALPENHORN_NODE",
+        recurse=False,
+        register=False,
+        completed=False,
+        timestamp=now,
+    )
+
     result = cli(0, ["node", "show", "Node", "--all"])
 
     assert "Total Files: 2" in result.output
@@ -282,6 +394,9 @@ def test_show_all(clidb, cli, assert_row_present):
     # 4.58 out of 8 == 57.25 percent
     assert "57.25%" in result.output
 
+    assert_row_present(result.output, "[Node Init]", "-", "-", str(now))
+    assert_row_present(result.output, "import/path", "No", "Yes", str(now))
+
     assert_row_present(result.output, "Group", "3", "10.12 kiB")
     assert_row_present(result.output, "Group2", "1", "3.375 kiB")