Repository: borgbackup/borg

Issue:
create borg(1) man page, polish man pages a bit
cont'd #208


Discussion:

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -214,6 +214,8 @@ def finalize_options(self):
 
     def run(self):
         print('generating usage docs')
+        import borg
+        borg.doc_mode = 'build_man'
         if not os.path.exists('docs/usage'):
             os.mkdir('docs/usage')
         # allows us to build docs without the C modules fully loaded during help generation
@@ -351,6 +353,7 @@ class build_man(Command):
     .. role:: ref(title)
 
     .. |project_name| replace:: Borg
+
     """)
 
     def initialize_options(self):
@@ -361,6 +364,8 @@ def finalize_options(self):
 
     def run(self):
         print('building man pages (in docs/man)', file=sys.stderr)
+        import borg
+        borg.doc_mode = 'build_man'
         os.makedirs('docs/man', exist_ok=True)
         # allows us to build docs without the C modules fully loaded during help generation
         from borg.archiver import Archiver
@@ -368,6 +373,7 @@ def run(self):
 
         self.generate_level('', parser, Archiver)
         self.build_topic_pages(Archiver)
+        self.build_intro_page()
 
     def generate_level(self, prefix, parser, Archiver):
         is_subcommand = False
@@ -387,26 +393,39 @@ def generate_level(self, prefix, parser, Archiver):
             man_title = 'borg-' + command.replace(' ', '-')
             print('building man page', man_title + '(1)', file=sys.stderr)
 
-            if self.generate_level(command + ' ', parser, Archiver):
-                continue
+            is_intermediary = self.generate_level(command + ' ', parser, Archiver)
 
             doc, write = self.new_doc()
             self.write_man_header(write, man_title, parser.description)
 
             self.write_heading(write, 'SYNOPSIS')
-            write('borg', command, end='')
-            self.write_usage(write, parser)
+            if is_intermediary:
+                subparsers = [action for action in parser._actions if 'SubParsersAction' in str(action.__class__)][0]
+                for subcommand in subparsers.choices:
+                    write('| borg', command, subcommand, '...')
+                    self.see_also.setdefault(command, []).append('%s-%s' % (command, subcommand))
+            else:
+                write('borg', command, end='')
+                self.write_usage(write, parser)
             write('\n')
 
-            self.write_heading(write, 'DESCRIPTION')
-            write(parser.epilog)
+            description, _, notes = parser.epilog.partition('\n.. man NOTES')
 
-            self.write_heading(write, 'OPTIONS')
-            write('See `borg-common(1)` for common options of Borg commands.')
-            write()
-            self.write_options(write, parser)
+            if description:
+                self.write_heading(write, 'DESCRIPTION')
+                write(description)
 
-            self.write_examples(write, command)
+            if not is_intermediary:
+                self.write_heading(write, 'OPTIONS')
+                write('See `borg-common(1)` for common options of Borg commands.')
+                write()
+                self.write_options(write, parser)
+
+                self.write_examples(write, command)
+
+            if notes:
+                self.write_heading(write, 'NOTES')
+                write(notes)
 
             self.write_see_also(write, man_title)
 
@@ -438,6 +457,12 @@ def build_topic_pages(self, Archiver):
             write(text)
             self.gen_man_page(man_title, doc.getvalue())
 
+    def build_intro_page(self):
+        print('building man page borg(1)', file=sys.stderr)
+        with open('docs/man_intro.rst') as fd:
+            man_intro = fd.read()
+        self.gen_man_page('borg', self.rst_prelude + man_intro)
+
     def new_doc(self):
         doc = io.StringIO(self.rst_prelude)
         doc.read()
@@ -493,7 +518,9 @@ def write_see_also(self, write, man_title):
     def gen_man_page(self, name, rst):
         from docutils.writers import manpage
         from docutils.core import publish_string
-        man_page = publish_string(source=rst, writer=manpage.Writer())
+        # We give the source_path so that docutils can find relative includes
+        # as-if the document where located in the docs/ directory.
+        man_page = publish_string(source=rst, source_path='docs/virtmanpage.rst', writer=manpage.Writer())
         with open('docs/man/%s.1' % name, 'wb') as fd:
             fd.write(man_page)
 
diff --git a/src/borg/archiver.py b/src/borg/archiver.py
--- a/src/borg/archiver.py
+++ b/src/borg/archiver.py
@@ -25,6 +25,7 @@
 
 import msgpack
 
+import borg
 from . import __version__
 from . import helpers
 from .archive import Archive, ArchiveChecker, ArchiveRecreater, Statistics, is_special
@@ -1657,6 +1658,16 @@ def preprocess_args(self, args):
         return args
 
     def build_parser(self, prog=None):
+        def process_epilog(epilog):
+            epilog = textwrap.dedent(epilog).splitlines()
+            try:
+                mode = borg.doc_mode
+            except AttributeError:
+                mode = 'command-line'
+            if mode in ('command-line', 'build_usage'):
+                epilog = [line for line in epilog if not line.startswith('.. man')]
+            return '\n'.join(epilog)
+
         common_parser = argparse.ArgumentParser(add_help=False, prog=prog)
 
         common_group = common_parser.add_argument_group('Common options')
@@ -1703,7 +1714,7 @@ def build_parser(self, prog=None):
                             help='show version number and exit')
         subparsers = parser.add_subparsers(title='required arguments', metavar='<command>')
 
-        serve_epilog = textwrap.dedent("""
+        serve_epilog = process_epilog("""
         This command starts a repository server process. This command is usually not used manually.
         """)
         subparser = subparsers.add_parser('serve', parents=[common_parser], add_help=False,
@@ -1718,7 +1729,7 @@ def build_parser(self, prog=None):
         subparser.add_argument('--append-only', dest='append_only', action='store_true',
                                help='only allow appending to repository segment files')
 
-        init_epilog = textwrap.dedent("""
+        init_epilog = process_epilog("""
         This command initializes an empty repository. A repository is a filesystem
         directory containing the deduplicated data from zero or more archives.
 
@@ -1765,32 +1776,33 @@ def build_parser(self, prog=None):
         Encryption modes
         ++++++++++++++++
 
-        repokey and keyfile use AES-CTR-256 for encryption and HMAC-SHA256 for
+        `repokey` and `keyfile` use AES-CTR-256 for encryption and HMAC-SHA256 for
         authentication in an encrypt-then-MAC (EtM) construction. The chunk ID hash
         is HMAC-SHA256 as well (with a separate key).
         These modes are compatible with borg 1.0.x.
 
-        repokey-blake2 and keyfile-blake2 are also authenticated encryption modes,
+        `repokey-blake2` and `keyfile-blake2` are also authenticated encryption modes,
         but use BLAKE2b-256 instead of HMAC-SHA256 for authentication. The chunk ID
         hash is a keyed BLAKE2b-256 hash.
-        These modes are new and not compatible with borg 1.0.x.
+        These modes are new and *not* compatible with borg 1.0.x.
 
-        "authenticated" mode uses no encryption, but authenticates repository contents
+        `authenticated` mode uses no encryption, but authenticates repository contents
         through the same keyed BLAKE2b-256 hash as the other blake2 modes (it uses it
         as chunk ID hash). The key is stored like repokey.
         This mode is new and not compatible with borg 1.0.x.
 
-        "none" mode uses no encryption and no authentication. It uses sha256 as chunk
+        `none` mode uses no encryption and no authentication. It uses sha256 as chunk
         ID hash. Not recommended, rather consider using an authenticated or
         authenticated/encrypted mode.
         This mode is compatible with borg 1.0.x.
 
         Hardware acceleration will be used automatically.
 
-        On modern Intel/AMD CPUs (except very cheap ones), AES is usually hw
-        accelerated. BLAKE2b is faster than sha256 on Intel/AMD 64bit CPUs.
+        On modern Intel/AMD CPUs (except very cheap ones), AES is usually
+        hardware-accelerated. BLAKE2b is faster than SHA256 on Intel/AMD 64bit CPUs,
+        which makes `authenticated` faster than `none`.
 
-        On modern ARM CPUs, NEON provides hw acceleration for sha256 making it faster
+        On modern ARM CPUs, NEON provides hardware acceleration for SHA256 making it faster
         than BLAKE2b-256 there.
         """)
         subparser = subparsers.add_parser('init', parents=[common_parser], add_help=False,
@@ -1804,11 +1816,11 @@ def build_parser(self, prog=None):
         subparser.add_argument('-e', '--encryption', dest='encryption',
                                choices=('none', 'keyfile', 'repokey', 'keyfile-blake2', 'repokey-blake2', 'authenticated'),
                                default=None,
-                               help='select encryption key mode (default: "%(default)s")')
+                               help='select encryption key mode')
         subparser.add_argument('-a', '--append-only', dest='append_only', action='store_true',
                                help='create an append-only mode repository')
 
-        check_epilog = textwrap.dedent("""
+        check_epilog = process_epilog("""
         The check command verifies the consistency of a repository and the corresponding archives.
 
         First, the underlying repository data files are checked:
@@ -1894,7 +1906,7 @@ def build_parser(self, prog=None):
         key_parsers = subparser.add_subparsers(title='required arguments', metavar='<command>')
         subparser.set_defaults(fallback_func=functools.partial(self.do_subcommand_help, subparser))
 
-        key_export_epilog = textwrap.dedent("""
+        key_export_epilog = process_epilog("""
         If repository encryption is used, the repository is inaccessible
         without the key. This command allows to backup this essential key.
 
@@ -1927,7 +1939,7 @@ def build_parser(self, prog=None):
                                default=False,
                                help='Create an export suitable for printing and later type-in')
 
-        key_import_epilog = textwrap.dedent("""
+        key_import_epilog = process_epilog("""
         This command allows to restore a key previously backed up with the
         export command.
 
@@ -1949,7 +1961,7 @@ def build_parser(self, prog=None):
                                default=False,
                                help='interactively import from a backup done with --paper')
 
-        change_passphrase_epilog = textwrap.dedent("""
+        change_passphrase_epilog = process_epilog("""
         The key files used for repository encryption are optionally passphrase
         protected. This command can be used to change this passphrase.
         """)
@@ -1972,7 +1984,7 @@ def build_parser(self, prog=None):
         subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='',
                                type=location_validator(archive=False))
 
-        migrate_to_repokey_epilog = textwrap.dedent("""
+        migrate_to_repokey_epilog = process_epilog("""
         This command migrates a repository from passphrase mode (removed in Borg 1.0)
         to repokey mode.
 
@@ -1999,7 +2011,7 @@ def build_parser(self, prog=None):
         subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='',
                                type=location_validator(archive=False))
 
-        create_epilog = textwrap.dedent("""
+        create_epilog = process_epilog("""
         This command creates a backup archive containing all files found while recursively
         traversing all paths specified. When giving '-' as path, borg will read data
         from standard input and create a file 'stdin' in the created archive from that
@@ -2022,6 +2034,52 @@ def build_parser(self, prog=None):
 
         See the output of the "borg help patterns" command for more help on exclude patterns.
         See the output of the "borg help placeholders" command for more help on placeholders.
+
+        .. man NOTES
+
+        The --exclude patterns are not like tar. In tar --exclude .bundler/gems will
+        exclude foo/.bundler/gems. In borg it will not, you need to use --exclude
+        '\*/.bundler/gems' to get the same effect. See ``borg help patterns`` for
+        more information.
+
+        Item flags
+        ++++++++++
+
+        ``--list`` outputs a list of all files, directories and other
+        file system items it considered (no matter whether they had content changes
+        or not). For each item, it prefixes a single-letter flag that indicates type
+        and/or status of the item.
+
+        If you are interested only in a subset of that output, you can give e.g.
+        ``--filter=AME`` and it will only show regular files with A, M or E status (see
+        below).
+
+        A uppercase character represents the status of a regular file relative to the
+        "files" cache (not relative to the repo -- this is an issue if the files cache
+        is not used). Metadata is stored in any case and for 'A' and 'M' also new data
+        chunks are stored. For 'U' all data chunks refer to already existing chunks.
+
+        - 'A' = regular file, added (see also :ref:`a_status_oddity` in the FAQ)
+        - 'M' = regular file, modified
+        - 'U' = regular file, unchanged
+        - 'E' = regular file, an error happened while accessing/reading *this* file
+
+        A lowercase character means a file type other than a regular file,
+        borg usually just stores their metadata:
+
+        - 'd' = directory
+        - 'b' = block device
+        - 'c' = char device
+        - 'h' = regular file, hardlink (to already seen inodes)
+        - 's' = symlink
+        - 'f' = fifo
+
+        Other flags used include:
+
+        - 'i' = backup data was read from standard input (stdin)
+        - '-' = dry run, item was *not* backed up
+        - 'x' = excluded, item was *not* backed up
+        - '?' = missing status code (if you see this, please file a bug report!)
         """)
 
         subparser = subparsers.add_parser('create', parents=[common_parser], add_help=False,
@@ -2062,12 +2120,12 @@ def build_parser(self, prog=None):
                                         'http://www.brynosaurus.com/cachedir/spec.html)')
         exclude_group.add_argument('--exclude-if-present', dest='exclude_if_present',
                                    metavar='NAME', action='append', type=str,
-                                   help='exclude directories that are tagged by containing a filesystem object with \
-                                         the given NAME')
+                                   help='exclude directories that are tagged by containing a filesystem object with '
+                                        'the given NAME')
         exclude_group.add_argument('--keep-exclude-tags', '--keep-tag-files', dest='keep_exclude_tags',
                                    action='store_true', default=False,
-                                   help='keep tag objects (i.e.: arguments to --exclude-if-present) in otherwise \
-                                         excluded caches/directories')
+                                   help='keep tag objects (i.e.: arguments to --exclude-if-present) in otherwise '
+                                        'excluded caches/directories')
 
         fs_group = subparser.add_argument_group('Filesystem options')
         fs_group.add_argument('-x', '--one-file-system', dest='one_file_system',
@@ -2122,7 +2180,7 @@ def build_parser(self, prog=None):
         subparser.add_argument('paths', metavar='PATH', nargs='+', type=str,
                                help='paths to archive')
 
-        extract_epilog = textwrap.dedent("""
+        extract_epilog = process_epilog("""
         This command extracts the contents of an archive. By default the entire
         archive is extracted but a subset of files and directories can be selected
         by passing a list of ``PATHs`` as arguments. The file selection can further
@@ -2173,7 +2231,7 @@ def build_parser(self, prog=None):
         subparser.add_argument('paths', metavar='PATH', nargs='*', type=str,
                                help='paths to extract; patterns are supported')
 
-        diff_epilog = textwrap.dedent("""
+        diff_epilog = process_epilog("""
             This command finds differences (file contents, user/group/mode) between archives.
 
             A repository location and an archive name must be specified for REPO_ARCHIVE1.
@@ -2221,7 +2279,7 @@ def build_parser(self, prog=None):
         subparser.add_argument('paths', metavar='PATH', nargs='*', type=str,
                                help='paths of items inside the archives to compare; patterns are supported')
 
-        rename_epilog = textwrap.dedent("""
+        rename_epilog = process_epilog("""
         This command renames an archive in the repository.
 
         This results in a different archive ID.
@@ -2239,7 +2297,7 @@ def build_parser(self, prog=None):
                                type=archivename_validator(),
                                help='the new archive name to use')
 
-        delete_epilog = textwrap.dedent("""
+        delete_epilog = process_epilog("""
         This command deletes an archive from the repository or the complete repository.
         Disk space is reclaimed accordingly. If you delete the complete repository, the
         local cache for it (if any) is also deleted.
@@ -2270,13 +2328,16 @@ def build_parser(self, prog=None):
                                help='archive or repository to delete')
         self.add_archives_filters_args(subparser)
 
-        list_epilog = textwrap.dedent("""
+        list_epilog = process_epilog("""
         This command lists the contents of a repository or an archive.
 
         See the "borg help patterns" command for more help on exclude patterns.
 
+        .. man NOTES
+
         The following keys are available for --format:
 
+
         """) + BaseFormatter.keys_help() + textwrap.dedent("""
 
         Keys for listing repository archives:
@@ -2311,7 +2372,7 @@ def build_parser(self, prog=None):
                                help='paths to list; patterns are supported')
         self.add_archives_filters_args(subparser)
 
-        mount_epilog = textwrap.dedent("""
+        mount_epilog = process_epilog("""
         This command mounts an archive as a FUSE filesystem. This can be useful for
         browsing an archive or restoring individual files. Unless the ``--foreground``
         option is given the command will run in the background until the filesystem
@@ -2362,7 +2423,7 @@ def build_parser(self, prog=None):
                                help='Extra mount options')
         self.add_archives_filters_args(subparser)
 
-        umount_epilog = textwrap.dedent("""
+        umount_epilog = process_epilog("""
         This command un-mounts a FUSE filesystem that was mounted with ``borg mount``.
 
         This is a convenience wrapper that just calls the platform-specific shell
@@ -2377,7 +2438,7 @@ def build_parser(self, prog=None):
         subparser.add_argument('mountpoint', metavar='MOUNTPOINT', type=str,
                                help='mountpoint of the filesystem to umount')
 
-        info_epilog = textwrap.dedent("""
+        info_epilog = process_epilog("""
         This command displays detailed information about the specified archive or repository.
 
         Please note that the deduplicated sizes of the individual archives do not add
@@ -2400,7 +2461,7 @@ def build_parser(self, prog=None):
                                help='archive or repository to display information about')
         self.add_archives_filters_args(subparser)
 
-        break_lock_epilog = textwrap.dedent("""
+        break_lock_epilog = process_epilog("""
         This command breaks the repository and cache locks.
         Please use carefully and only while no borg process (on any machine) is
         trying to access the Cache or the Repository.
@@ -2415,7 +2476,7 @@ def build_parser(self, prog=None):
                                type=location_validator(archive=False),
                                help='repository for which to break the locks')
 
-        prune_epilog = textwrap.dedent("""
+        prune_epilog = process_epilog("""
         The prune command prunes a repository by deleting all archives not matching
         any of the specified retention options. This command is normally used by
         automated backup scripts wanting to keep a certain number of historic backups.
@@ -2501,7 +2562,7 @@ def build_parser(self, prog=None):
                                type=location_validator(archive=False),
                                help='repository to prune')
 
-        upgrade_epilog = textwrap.dedent("""
+        upgrade_epilog = process_epilog("""
         Upgrade an existing Borg repository.
 
         Borg 1.x.y upgrades
@@ -2591,7 +2652,7 @@ def build_parser(self, prog=None):
                                type=location_validator(archive=False),
                                help='path to the repository to be upgraded')
 
-        recreate_epilog = textwrap.dedent("""
+        recreate_epilog = process_epilog("""
         Recreate the contents of existing archives.
 
         This is an *experimental* feature. Do *not* use this on your only backup.
@@ -2710,7 +2771,7 @@ def build_parser(self, prog=None):
         subparser.add_argument('paths', metavar='PATH', nargs='*', type=str,
                                help='paths to recreate; patterns are supported')
 
-        with_lock_epilog = textwrap.dedent("""
+        with_lock_epilog = process_epilog("""
         This command runs a user-specified command while the repository lock is held.
 
         It will first try to acquire the lock (make sure that no other operation is
@@ -2746,7 +2807,7 @@ def build_parser(self, prog=None):
         subparser.add_argument('topic', metavar='TOPIC', type=str, nargs='?',
                                help='additional help on TOPIC')
 
-        debug_epilog = textwrap.dedent("""
+        debug_epilog = process_epilog("""
         These commands are not intended for normal use and potentially very
         dangerous if used incorrectly.
 
@@ -2763,7 +2824,7 @@ def build_parser(self, prog=None):
         debug_parsers = subparser.add_subparsers(title='required arguments', metavar='<command>')
         subparser.set_defaults(fallback_func=functools.partial(self.do_subcommand_help, subparser))
 
-        debug_info_epilog = textwrap.dedent("""
+        debug_info_epilog = process_epilog("""
         This command displays some system information that might be useful for bug
         reports and debugging problems. If a traceback happens, this information is
         already appended at the end of the traceback.
@@ -2775,7 +2836,7 @@ def build_parser(self, prog=None):
                                           help='show system infos for debugging / bug reports (debug)')
         subparser.set_defaults(func=self.do_debug_info)
 
-        debug_dump_archive_items_epilog = textwrap.dedent("""
+        debug_dump_archive_items_epilog = process_epilog("""
         This command dumps raw (but decrypted and decompressed) archive items (only metadata) to files.
         """)
         subparser = debug_parsers.add_parser('dump-archive-items', parents=[common_parser], add_help=False,
@@ -2788,7 +2849,7 @@ def build_parser(self, prog=None):
                                type=location_validator(archive=True),
                                help='archive to dump')
 
-        debug_dump_archive_epilog = textwrap.dedent("""
+        debug_dump_archive_epilog = process_epilog("""
         This command dumps all metadata of an archive in a decoded form to a file.
         """)
         subparser = debug_parsers.add_parser('dump-archive', parents=[common_parser], add_help=False,
@@ -2803,7 +2864,7 @@ def build_parser(self, prog=None):
         subparser.add_argument('path', metavar='PATH', type=str,
                                help='file to dump data into')
 
-        debug_dump_manifest_epilog = textwrap.dedent("""
+        debug_dump_manifest_epilog = process_epilog("""
         This command dumps manifest metadata of a repository in a decoded form to a file.
         """)
         subparser = debug_parsers.add_parser('dump-manifest', parents=[common_parser], add_help=False,
@@ -2818,7 +2879,7 @@ def build_parser(self, prog=None):
         subparser.add_argument('path', metavar='PATH', type=str,
                                help='file to dump data into')
 
-        debug_dump_repo_objs_epilog = textwrap.dedent("""
+        debug_dump_repo_objs_epilog = process_epilog("""
         This command dumps raw (but decrypted and decompressed) repo objects to files.
         """)
         subparser = debug_parsers.add_parser('dump-repo-objs', parents=[common_parser], add_help=False,
@@ -2831,7 +2892,7 @@ def build_parser(self, prog=None):
                                type=location_validator(archive=False),
                                help='repo to dump')
 
-        debug_get_obj_epilog = textwrap.dedent("""
+        debug_get_obj_epilog = process_epilog("""
         This command gets an object from the repository.
         """)
         subparser = debug_parsers.add_parser('get-obj', parents=[common_parser], add_help=False,
@@ -2848,7 +2909,7 @@ def build_parser(self, prog=None):
         subparser.add_argument('path', metavar='PATH', type=str,
                                help='file to write object data into')
 
-        debug_put_obj_epilog = textwrap.dedent("""
+        debug_put_obj_epilog = process_epilog("""
         This command puts objects into the repository.
         """)
         subparser = debug_parsers.add_parser('put-obj', parents=[common_parser], add_help=False,
@@ -2863,7 +2924,7 @@ def build_parser(self, prog=None):
         subparser.add_argument('paths', metavar='PATH', nargs='+', type=str,
                                help='file(s) to read and create object(s) from')
 
-        debug_delete_obj_epilog = textwrap.dedent("""
+        debug_delete_obj_epilog = process_epilog("""
         This command deletes objects from the repository.
         """)
         subparser = debug_parsers.add_parser('delete-obj', parents=[common_parser], add_help=False,
@@ -2878,7 +2939,7 @@ def build_parser(self, prog=None):
         subparser.add_argument('ids', metavar='IDs', nargs='+', type=str,
                                help='hex object ID(s) to delete from the repo')
 
-        debug_refcount_obj_epilog = textwrap.dedent("""
+        debug_refcount_obj_epilog = process_epilog("""
         This command displays the reference count for objects from the repository.
         """)
         subparser = debug_parsers.add_parser('refcount-obj', parents=[common_parser], add_help=False,
diff --git a/src/borg/repository.py b/src/borg/repository.py
--- a/src/borg/repository.py
+++ b/src/borg/repository.py
@@ -788,6 +788,7 @@ def scan(self, limit=None, marker=None):
         fetching data in this order does linear reads and reuses stuff from disk cache.
 
         We rely on repository.check() has run already (either now or some time before) and that:
+
         - if we are called from a borg check command, self.index is a valid, fresh, in-sync repo index.
         - if we are called from elsewhere, either self.index or the on-disk index is valid and in-sync.
         - the repository segments are valid (no CRC errors).


Test patch: