Repository: borgbackup/borg

Issue:
query quota set / use from repository
Is there any way to query the quota introduced in https://github.com/borgbackup/borg/pull/2565? So that I, as a borg client/borg user, check how much size my remote repo already uses and how much space I still have left, so that I can potentially tweak my prune settings, manually delete large archives, etc.

---

:moneybag: [there is a bounty for this](https://www.bountysource.com/issues/47527011-query-quota-from-remote-repository)


Discussion:
should tell:
- storage quota use (as we track it in hints)
- storage quota set (from repo config or overridden via borg serve option)

implementation:
- for local repos should be rather easy (but of limited application as quotas might be most frequently used by backup space providers)
- for remote (ssh:) repos needs a new rpc call

From `repository.py`:
```
    def log_storage_quota(self):
        if self.storage_quota:
            logger.info('Storage quota: %s out of %s used.',
                        format_file_size(self.storage_quota_use), format_file_size(self.storage_quota))
```

@rugk looks like quota available and actual use is already logged - if you are logging at INFO level.

This code is called by `commit()` and `prepare_txn()`.

So, just logging at INFO level and looking in your logs seems to solve this, without introducing a new RPC api.

Okay, thanks. However, at least `borg info` or so should show it, should not it?

Looking deep into some (info) logs cannot be a solution…
As it is currently implemented, the used-quota stored in the `hints` file is only loaded when a transaction is started.

And why is looking into a log not a solution? :)
Asked otherwise. Why should not it be displayed in `borg info`? It is a very useful information for the user when backing up., especially as it is critical if the space runs low. Just a simple "Usage: XX bytes out of XY bytes"?

…or, for that reason, after a backup was created with `borg create`.
I'm thinking about implementing this. As I understand @rugk wants to add quota info to the output of `info` and `create`.
Where in the output should I put it? At the top so it looks like this?
```
Used quota: 123MB out of 5GB
Archive name: 1
Archive fingerprint: f2718537357293f6458b9c622b15c08577c036b50e650c129c12ffdbc2206fbe
...

Archive name: 2
...
```
And in `borg create` should I add it to the --stats option? `create` doesn't give any output by default.
@ThomasWaldmann Did you read my comment?
The obvious place to output the quota information is `borg info` I'ld say.

Not a big fan from putting more and more into borg create output.

Resolving patch:
diff --git a/src/borg/archiver/rinfo_cmd.py b/src/borg/archiver/rinfo_cmd.py
--- a/src/borg/archiver/rinfo_cmd.py
+++ b/src/borg/archiver/rinfo_cmd.py
@@ -3,7 +3,7 @@
 
 from ._common import with_repository
 from ..constants import *  # NOQA
-from ..helpers import bin_to_hex, json_print, basic_json_data
+from ..helpers import bin_to_hex, json_print, basic_json_data, format_file_size
 from ..manifest import Manifest
 
 from ..logger import create_logger
@@ -30,7 +30,7 @@ def do_rinfo(self, args, repository, manifest, cache):
                 encryption += "\nKey file: %s" % key.find_key()
             info["encryption"] = encryption
 
-            print(
+            output = (
                 textwrap.dedent(
                     """
             Repository ID: {id}
@@ -38,8 +38,6 @@ def do_rinfo(self, args, repository, manifest, cache):
             Repository version: {version}
             Append only: {append_only}
             {encryption}
-            Cache: {cache.path}
-            Security dir: {security_dir}
             """
                 )
                 .strip()
@@ -48,9 +46,31 @@ def do_rinfo(self, args, repository, manifest, cache):
                     location=repository._location.canonical_path(),
                     version=repository.version,
                     append_only=repository.append_only,
-                    **info,
+                    encryption=info["encryption"],
                 )
             )
+
+            response = repository.info()
+            storage_quota = response["storage_quota"]
+            used = format_file_size(response["storage_quota_use"])
+
+            output += f"Storage quota: {used} used"
+            if storage_quota:
+                output += f" out of {format_file_size(storage_quota)}"
+            output += "\n"
+
+            output += (
+                textwrap.dedent(
+                    """
+                    Cache: {cache.path}
+                    Security dir: {security_dir}
+                    """
+                )
+                .strip()
+                .format(**info)
+            )
+
+            print(output)
             print(str(cache))
         return self.exit_code
 
diff --git a/src/borg/repository.py b/src/borg/repository.py
--- a/src/borg/repository.py
+++ b/src/borg/repository.py
@@ -493,9 +493,22 @@ def open(self, path, exclusive, lock_wait=None, lock=True):
         self.id = unhexlify(self.config.get("repository", "id").strip())
         self.io = LoggedIO(self.path, self.max_segment_size, self.segments_per_dir)
 
+    def _load_hints(self):
+        if (transaction_id := self.get_transaction_id()) is None:
+            # self is a fresh repo, so transaction_id is None and there is no hints file
+            return
+        hints = self._unpack_hints(transaction_id)
+        self.version = hints["version"]
+        self.storage_quota_use = hints["storage_quota_use"]
+        self.shadow_index = hints["shadow_index"]
+
     def info(self):
         """return some infos about the repo (must be opened first)"""
-        return dict(id=self.id, version=self.version, append_only=self.append_only)
+        info = dict(id=self.id, version=self.version, append_only=self.append_only)
+        self._load_hints()
+        info["storage_quota"] = self.storage_quota
+        info["storage_quota_use"] = self.storage_quota_use
+        return info
 
     def close(self):
         if self.lock:
@@ -513,7 +526,6 @@ def commit(self, save_space=False, compact=True, threshold=0.1):
             self.rollback()
             raise exception
         self.check_free_space()
-        self.log_storage_quota()
         segment = self.io.write_commit()
         self.segments.setdefault(segment, 0)
         self.compact[segment] += LoggedIO.header_fmt.size
@@ -557,6 +569,12 @@ def open_index(self, transaction_id, auto_recover=True):
             self.commit(compact=False)
             return self.open_index(self.get_transaction_id())
 
+    def _unpack_hints(self, transaction_id):
+        hints_path = os.path.join(self.path, "hints.%d" % transaction_id)
+        integrity_data = self._read_integrity(transaction_id, "hints")
+        with IntegrityCheckedFile(hints_path, write=False, integrity_data=integrity_data) as fd:
+            return msgpack.unpack(fd)
+
     def prepare_txn(self, transaction_id, do_cleanup=True):
         self._active_txn = True
         if self.do_lock and not self.lock.got_exclusive_lock():
@@ -593,10 +611,8 @@ def prepare_txn(self, transaction_id, do_cleanup=True):
                 self.io.cleanup(transaction_id)
             hints_path = os.path.join(self.path, "hints.%d" % transaction_id)
             index_path = os.path.join(self.path, "index.%d" % transaction_id)
-            integrity_data = self._read_integrity(transaction_id, "hints")
             try:
-                with IntegrityCheckedFile(hints_path, write=False, integrity_data=integrity_data) as fd:
-                    hints = msgpack.unpack(fd)
+                hints = self._unpack_hints(transaction_id)
             except (msgpack.UnpackException, FileNotFoundError, FileIntegrityError) as e:
                 logger.warning("Repository hints file missing or corrupted, trying to recover: %s", e)
                 if not isinstance(e, FileNotFoundError):
@@ -623,7 +639,6 @@ def prepare_txn(self, transaction_id, do_cleanup=True):
                 self.compact = FreeSpace(hints["compact"])
                 self.storage_quota_use = hints.get("storage_quota_use", 0)
                 self.shadow_index = hints.get("shadow_index", {})
-            self.log_storage_quota()
             # Drop uncommitted segments in the shadow index
             for key, shadowed_segments in self.shadow_index.items():
                 for segment in list(shadowed_segments):
@@ -763,14 +778,6 @@ def check_free_space(self):
             formatted_free = format_file_size(free_space)
             raise self.InsufficientFreeSpaceError(formatted_required, formatted_free)
 
-    def log_storage_quota(self):
-        if self.storage_quota:
-            logger.info(
-                "Storage quota: %s out of %s used.",
-                format_file_size(self.storage_quota_use),
-                format_file_size(self.storage_quota),
-            )
-
     def compact_segments(self, threshold):
         """Compact sparse segments by copying data into new segments"""
         if not self.compact:
diff --git a/src/borg/testsuite/archiver/rinfo_cmd.py b/src/borg/testsuite/archiver/rinfo_cmd.py
--- a/src/borg/testsuite/archiver/rinfo_cmd.py
+++ b/src/borg/testsuite/archiver/rinfo_cmd.py
@@ -1,4 +1,5 @@
 import json
+from random import randbytes
 import unittest
 
 from ...constants import *  # NOQA
@@ -36,6 +37,20 @@ def test_info_json(self):
         assert all(isinstance(o, int) for o in stats.values())
         assert all(key in stats for key in ("total_chunks", "total_size", "total_unique_chunks", "unique_size"))
 
+    def test_info_on_repository_with_storage_quota(self):
+        self.create_regular_file("file1", contents=randbytes(1000 * 1000))
+        self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION, "--storage-quota=1G")
+        self.cmd(f"--repo={self.repository_location}", "create", "test", "input")
+        info_repo = self.cmd(f"--repo={self.repository_location}", "rinfo")
+        assert "Storage quota: 1.00 MB used out of 1.00 GB" in info_repo
+
+    def test_info_on_repository_without_storage_quota(self):
+        self.create_regular_file("file1", contents=randbytes(1000 * 1000))
+        self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION)
+        self.cmd(f"--repo={self.repository_location}", "create", "test", "input")
+        info_repo = self.cmd(f"--repo={self.repository_location}", "rinfo")
+        assert "Storage quota: 1.00 MB used" in info_repo
+
 
 class RemoteArchiverTestCase(RemoteArchiverTestCaseBase, ArchiverTestCase):
     """run the same tests, but with a remote repository"""


Test patch: