Repository: restic/restic (go)

Issue:
Support `-h` argument to `ls -l`
Output of `restic version`
--------------------------

restic 0.14.0 compiled with go1.19 on linux/amd64

What should restic do differently? Which functionality do you think we should add?
----------------------------------------------------------------------------------

Support a `-h` flag on `restic -l <snapshot>` to print human-readable byte counts like `ls` does. From `ls` manpage:

```
-h, --human-readable       with -l and -s, print sizes like 1K 234M 2G etc.
```

`-h` in restic is already occupied by help, but combining `-h` and `-l` does not make sense so if that combination is encountered, take `-h` argument as `--human-readable`. Alternatively, remove the `-h` argument for help only via `--help` is more common.

What are you trying to do? What problem would this solve?
---------------------------------------------------------

Large file sizes are hard to read with `restic ls -l`:

```
$ restic ls -l b64a60a1
repository 9cbe0bce opened (repository version 2) successfully, password is correct
snapshot b64a60a1 of [/hugefile] filtered by [] at 2023-01-17 04:55:21.228268714 +0100 CET):
-rw-r--r--     0     0 345945001259 2023-01-17 04:55:21 /hugefile
```

Ideally, it should print `345G` in this case.


Pull request description:
Add --human-readable to ls and find output
Closes #4159 

<!--
Thank you very much for contributing code or documentation to restic! Please
fill out the following questions to make it easier for us to review your
changes.
-->

What does this PR change? What problem does it solve?
-----------------------------------------------------

Modifies format module to add options for human readable storage size formatting, using size parsing already in ui/format. Cmd flag --human-readable added to ls and find commands. Additional option added to formatNode to support printing size in regular or new human readable format

<!--
Describe the changes and their purpose here, as detailed as needed.
-->

Was the change previously discussed in an issue or on the forum?
----------------------------------------------------------------
Discussion in #4159 

Expanded scope of the original issue a bit after finding formatNode function was common between `ls` and `find`; figured both could benefit from the additional formatting option. Also added a test for format


<!--
Link issues and relevant forum posts here.

If this PR resolves an issue on GitHub, use "Closes #1234" so that the issue
is closed automatically when this PR is merged.
-->

Checklist
---------

<!--
You do not need to check all the boxes below all at once. Feel free to take
your time and add more commits. If you're done and ready for review, please
check the last box. Enable a checkbox by replacing [ ] with [x].
-->

- [x] I have read the [contribution guidelines](https://github.com/restic/restic/blob/master/CONTRIBUTING.md#providing-patches).
- [x] I have [enabled maintainer edits](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/allowing-changes-to-a-pull-request-branch-created-from-a-fork).
- [x] I have added tests for all code changes.
- [x] I have added documentation for relevant changes (in the manual).
- [x] There's a new file in `changelog/unreleased/` that describes the changes for our users (see [template](https://github.com/restic/restic/blob/master/changelog/TEMPLATE)).
- [x] I have run `gofmt` on the code in all commits.
- [x] All commit messages are formatted in the same style as [the other commits in the repo](https://github.com/restic/restic/blob/master/CONTRIBUTING.md#git-commits).
- [x] I'm done! This pull request is ready for review.


Patch:
diff --git a/changelog/unreleased/issue-4159 b/changelog/unreleased/issue-4159
new file mode 100644
index 000000000..4f91ce240
--- /dev/null
+++ b/changelog/unreleased/issue-4159
@@ -0,0 +1,13 @@
+Enhancement: Add `--human-readable` flag to `ls` and `find` commands
+
+Previously, when using the -l option with the ls and find commands,
+the displayed size was always in bytes, without an option for a more
+human readable format such as MiB or GiB.
+
+The new `--human-readable` option will convert longer size values into
+more human friendly values with an appropriate suffix depending on the
+output size. For example, a size of `14680064` will be shown as
+`14.000 MiB`.
+
+https://github.com/restic/restic/issues/4159
+https://github.com/restic/restic/pull/4351
diff --git a/cmd/restic/cmd_find.go b/cmd/restic/cmd_find.go
index 6b5e32df7..e1161797b 100644
--- a/cmd/restic/cmd_find.go
+++ b/cmd/restic/cmd_find.go
@@ -51,6 +51,7 @@ type FindOptions struct {
 	PackID, ShowPackID bool
 	CaseInsensitive    bool
 	ListLong           bool
+	HumanReadable      bool
 	restic.SnapshotFilter
 }
 
@@ -69,6 +70,7 @@ func init() {
 	f.BoolVar(&findOptions.ShowPackID, "show-pack-id", false, "display the pack-ID the blobs belong to (with --blob or --tree)")
 	f.BoolVarP(&findOptions.CaseInsensitive, "ignore-case", "i", false, "ignore case for pattern")
 	f.BoolVarP(&findOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode")
+	f.BoolVar(&findOptions.HumanReadable, "human-readable", false, "print sizes in human readable format")
 
 	initMultiSnapshotFilter(f, &findOptions.SnapshotFilter, true)
 }
@@ -104,12 +106,13 @@ func parseTime(str string) (time.Time, error) {
 }
 
 type statefulOutput struct {
-	ListLong bool
-	JSON     bool
-	inuse    bool
-	newsn    *restic.Snapshot
-	oldsn    *restic.Snapshot
-	hits     int
+	ListLong      bool
+	HumanReadable bool
+	JSON          bool
+	inuse         bool
+	newsn         *restic.Snapshot
+	oldsn         *restic.Snapshot
+	hits          int
 }
 
 func (s *statefulOutput) PrintPatternJSON(path string, node *restic.Node) {
@@ -164,7 +167,7 @@ func (s *statefulOutput) PrintPatternNormal(path string, node *restic.Node) {
 		s.oldsn = s.newsn
 		Verbosef("Found matching entries in snapshot %s from %s\n", s.oldsn.ID().Str(), s.oldsn.Time.Local().Format(TimeFormat))
 	}
-	Println(formatNode(path, node, s.ListLong))
+	Println(formatNode(path, node, s.ListLong, s.HumanReadable))
 }
 
 func (s *statefulOutput) PrintPattern(path string, node *restic.Node) {
@@ -594,7 +597,7 @@ func runFind(ctx context.Context, opts FindOptions, gopts GlobalOptions, args []
 	f := &Finder{
 		repo:        repo,
 		pat:         pat,
-		out:         statefulOutput{ListLong: opts.ListLong, JSON: gopts.JSON},
+		out:         statefulOutput{ListLong: opts.ListLong, HumanReadable: opts.HumanReadable, JSON: gopts.JSON},
 		ignoreTrees: restic.NewIDSet(),
 	}
 
diff --git a/cmd/restic/cmd_ls.go b/cmd/restic/cmd_ls.go
index e8c27381c..1cd549e7c 100644
--- a/cmd/restic/cmd_ls.go
+++ b/cmd/restic/cmd_ls.go
@@ -50,7 +50,8 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 type LsOptions struct {
 	ListLong bool
 	restic.SnapshotFilter
-	Recursive bool
+	Recursive     bool
+	HumanReadable bool
 }
 
 var lsOptions LsOptions
@@ -62,6 +63,7 @@ func init() {
 	initSingleSnapshotFilter(flags, &lsOptions.SnapshotFilter)
 	flags.BoolVarP(&lsOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode")
 	flags.BoolVar(&lsOptions.Recursive, "recursive", false, "include files in subfolders of the listed directories")
+	flags.BoolVar(&lsOptions.HumanReadable, "human-readable", false, "print sizes in human readable format")
 }
 
 type lsSnapshot struct {
@@ -206,7 +208,7 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
 			Verbosef("snapshot %s of %v filtered by %v at %s):\n", sn.ID().Str(), sn.Paths, dirs, sn.Time)
 		}
 		printNode = func(path string, node *restic.Node) {
-			Printf("%s\n", formatNode(path, node, lsOptions.ListLong))
+			Printf("%s\n", formatNode(path, node, lsOptions.ListLong, lsOptions.HumanReadable))
 		}
 	}
 
diff --git a/cmd/restic/format.go b/cmd/restic/format.go
index 2f14a4575..063cd4e71 100644
--- a/cmd/restic/format.go
+++ b/cmd/restic/format.go
@@ -5,9 +5,10 @@ import (
 	"os"
 
 	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/ui"
 )
 
-func formatNode(path string, n *restic.Node, long bool) string {
+func formatNode(path string, n *restic.Node, long bool, human bool) string {
 	if !long {
 		return path
 	}
@@ -15,6 +16,13 @@ func formatNode(path string, n *restic.Node, long bool) string {
 	var mode os.FileMode
 	var target string
 
+	var size string
+	if human {
+		size = ui.FormatBytes(n.Size)
+	} else {
+		size = fmt.Sprintf("%6d", n.Size)
+	}
+
 	switch n.Type {
 	case "file":
 		mode = 0
@@ -33,8 +41,8 @@ func formatNode(path string, n *restic.Node, long bool) string {
 		mode = os.ModeSocket
 	}
 
-	return fmt.Sprintf("%s %5d %5d %6d %s %s%s",
-		mode|n.Mode, n.UID, n.GID, n.Size,
+	return fmt.Sprintf("%s %5d %5d %s %s %s%s",
+		mode|n.Mode, n.UID, n.GID, size,
 		n.ModTime.Local().Format(TimeFormat), path,
 		target)
 }


Test patch:
diff --git a/cmd/restic/format_test.go b/cmd/restic/format_test.go
new file mode 100644
index 000000000..689bd27a5
--- /dev/null
+++ b/cmd/restic/format_test.go
@@ -0,0 +1,61 @@
+package main
+
+import (
+	"testing"
+	"time"
+
+	"github.com/restic/restic/internal/restic"
+	rtest "github.com/restic/restic/internal/test"
+)
+
+func TestFormatNode(t *testing.T) {
+	// overwrite time zone to ensure the data is formatted reproducibly
+	tz := time.Local
+	time.Local = time.UTC
+	defer func() {
+		time.Local = tz
+	}()
+
+	testPath := "/test/path"
+	node := restic.Node{
+		Name:    "baz",
+		Type:    "file",
+		Size:    14680064,
+		UID:     1000,
+		GID:     2000,
+		ModTime: time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC),
+	}
+
+	for _, c := range []struct {
+		path string
+		restic.Node
+		long   bool
+		human  bool
+		expect string
+	}{
+		{
+			path:   testPath,
+			Node:   node,
+			long:   false,
+			human:  false,
+			expect: testPath,
+		},
+		{
+			path:   testPath,
+			Node:   node,
+			long:   true,
+			human:  false,
+			expect: "----------  1000  2000 14680064 2020-01-02 03:04:05 " + testPath,
+		},
+		{
+			path:   testPath,
+			Node:   node,
+			long:   true,
+			human:  true,
+			expect: "----------  1000  2000 14.000 MiB 2020-01-02 03:04:05 " + testPath,
+		},
+	} {
+		r := formatNode(c.path, &c.Node, c.long, c.human)
+		rtest.Equals(t, c.expect, r)
+	}
+}