Repository: juicedata/juicefs (go)

Issue:
Find out broken files in `fsck`
When doing `juicefs fsck`, we can scan out lost objects, but users probably want to know the broken files (paths) containing these objects.

Pull request description:
fsck: list broken files
close #949 

Patch:
diff --git a/cmd/fsck.go b/cmd/fsck.go
index ad553fb7..61e57a93 100644
--- a/cmd/fsck.go
+++ b/cmd/fsck.go
@@ -17,6 +17,7 @@ package main
 
 import (
 	"fmt"
+	"sort"
 	"strings"
 	"time"
 
@@ -100,16 +101,20 @@ func fsck(ctx *cli.Context) error {
 	logger.Infof("Listing all slices ...")
 	progress, bar := utils.NewProgressCounter("listed slices counter: ")
 	var c = meta.NewContext(0, 0, []uint32{0})
-	var slices []meta.Slice
-	r := m.ListSlices(c, &slices, false, bar.Increment)
+	slices := make(map[meta.Ino][]meta.Slice)
+	r := m.ListSlices(c, slices, false, bar.Increment)
 	if r != 0 {
 		logger.Fatalf("list all slices: %s", r)
 	}
 	bar.SetTotal(0, true)
 	progress.Wait()
 
+	var totalSlices int
+	for _, ss := range slices {
+		totalSlices += len(ss)
+	}
 	progress, bar = utils.NewDynProgressBar("scanning slices: ", false)
-	bar.SetTotal(int64(len(slices)), false)
+	bar.SetTotal(int64(totalSlices), false)
 	lost := progress.Add(0,
 		utils.NewSpinner(),
 		mpb.PrependDecorators(
@@ -127,24 +132,33 @@ func fsck(ctx *cli.Context) error {
 		mpb.BarFillerClearOnComplete(),
 	)
 
-	keys := make(map[uint64]uint32)
 	var totalBytes uint64
-	for _, s := range slices {
-		bar.Increment()
-		keys[s.Chunkid] = s.Size
-		totalBytes += uint64(s.Size)
-		n := (s.Size - 1) / uint32(chunkConf.BlockSize)
-		for i := uint32(0); i <= n; i++ {
-			sz := chunkConf.BlockSize
-			if i == n {
-				sz = int(s.Size) - int(i)*chunkConf.BlockSize
-			}
-			key := fmt.Sprintf("%d_%d_%d", s.Chunkid, i, sz)
-			if _, ok := blocks[key]; !ok {
-				if _, err := blob.Head(key); err != nil {
-					logger.Errorf("can't find block %s: %s", key, err)
-					lost.Increment()
-					lostBytes.IncrBy(sz)
+	brokens := make(map[meta.Ino]string)
+	for inode, ss := range slices {
+		for _, s := range ss {
+			bar.Increment()
+			totalBytes += uint64(s.Size)
+			n := (s.Size - 1) / uint32(chunkConf.BlockSize)
+			for i := uint32(0); i <= n; i++ {
+				sz := chunkConf.BlockSize
+				if i == n {
+					sz = int(s.Size) - int(i)*chunkConf.BlockSize
+				}
+				key := fmt.Sprintf("%d_%d_%d", s.Chunkid, i, sz)
+				if _, ok := blocks[key]; !ok {
+					if _, err := blob.Head(key); err != nil {
+						if _, ok := brokens[inode]; !ok {
+							if p, st := meta.GetPath(m, meta.Background, inode); st == 0 {
+								brokens[inode] = p
+							} else {
+								logger.Warnf("getpath of inode %d: %s", inode, st)
+								brokens[inode] = st.Error()
+							}
+						}
+						logger.Errorf("can't find block %s for file %s: %s", key, brokens[inode], err)
+						lost.Increment()
+						lostBytes.IncrBy(sz)
+					}
 				}
 			}
 		}
@@ -153,9 +167,17 @@ func fsck(ctx *cli.Context) error {
 	lost.SetTotal(0, true)
 	lostBytes.SetTotal(0, true)
 	progress.Wait()
-	logger.Infof("Used by %d slices (%d bytes)", len(keys), totalBytes)
+	logger.Infof("Used by %d slices (%d bytes)", totalSlices, totalBytes)
 	if lost.Current() > 0 {
-		logger.Fatalf("%d object is lost (%d bytes)", lost.Current(), lostBytes.Current())
+		msg := fmt.Sprintf("%d objects are lost (%d bytes), %d broken files:\n", lost.Current(), lostBytes.Current(), len(brokens))
+		msg += fmt.Sprintf("%13s: PATH\n", "INODE")
+		var fileList []string
+		for i, p := range brokens {
+			fileList = append(fileList, fmt.Sprintf("%13d: %s", i, p))
+		}
+		sort.Strings(fileList)
+		msg += strings.Join(fileList, "\n")
+		logger.Fatal(msg)
 	}
 
 	return nil
diff --git a/cmd/gc.go b/cmd/gc.go
index 25a4c37e..5a873d8a 100644
--- a/cmd/gc.go
+++ b/cmd/gc.go
@@ -150,8 +150,8 @@ func gc(ctx *cli.Context) error {
 
 	progress, bar := utils.NewProgressCounter("listed slices counter: ")
 	var c = meta.NewContext(0, 0, []uint32{0})
-	var slices []meta.Slice
-	r := m.ListSlices(c, &slices, ctx.Bool("delete"), bar.Increment)
+	slices := make(map[meta.Ino][]meta.Slice)
+	r := m.ListSlices(c, slices, ctx.Bool("delete"), bar.Increment)
 	if r != 0 {
 		logger.Fatalf("list all slices: %s", r)
 	}
@@ -161,10 +161,12 @@ func gc(ctx *cli.Context) error {
 	keys := make(map[uint64]uint32)
 	var total int64
 	var totalBytes uint64
-	for _, s := range slices {
-		keys[s.Chunkid] = s.Size
-		total += int64(int(s.Size-1)/chunkConf.BlockSize) + 1 // s.Size should be > 0
-		totalBytes += uint64(s.Size)
+	for _, ss := range slices {
+		for _, s := range ss {
+			keys[s.Chunkid] = s.Size
+			total += int64(int(s.Size-1)/chunkConf.BlockSize) + 1 // s.Size should be > 0
+			totalBytes += uint64(s.Size)
+		}
 	}
 	logger.Infof("using %d slices (%d bytes)", len(keys), totalBytes)
 
diff --git a/pkg/meta/interface.go b/pkg/meta/interface.go
index d155cd6e..c6aa79e4 100644
--- a/pkg/meta/interface.go
+++ b/pkg/meta/interface.go
@@ -307,7 +307,7 @@ type Meta interface {
 	// Compact all the chunks by merge small slices together
 	CompactAll(ctx Context) syscall.Errno
 	// ListSlices returns all slices used by all files.
-	ListSlices(ctx Context, slices *[]Slice, delete bool, showProgress func()) syscall.Errno
+	ListSlices(ctx Context, slices map[Ino][]Slice, delete bool, showProgress func()) syscall.Errno
 
 	// OnMsg add a callback for the given message type.
 	OnMsg(mtype uint32, cb MsgCallback)
@@ -377,3 +377,37 @@ func newSessionInfo() *SessionInfo {
 func timeit(start time.Time) {
 	opDist.Observe(time.Since(start).Seconds())
 }
+
+// Get full path of an inode; a random one is picked if it has multiple hard links
+func GetPath(m Meta, ctx Context, inode Ino) (string, syscall.Errno) {
+	var names []string
+	var attr Attr
+	for inode != 1 {
+		if st := m.GetAttr(ctx, inode, &attr); st != 0 {
+			logger.Debugf("getattr inode %d: %s", inode, st)
+			return "", st
+		}
+
+		var entries []*Entry
+		if st := m.Readdir(ctx, attr.Parent, 0, &entries); st != 0 {
+			return "", st
+		}
+		var name string
+		for _, e := range entries {
+			if e.Inode == inode {
+				name = string(e.Name)
+				break
+			}
+		}
+		if name == "" {
+			return "", syscall.ENOENT
+		}
+		names = append(names, name)
+		inode = attr.Parent
+	}
+
+	for i, j := 0, len(names)-1; i < j; i, j = i+1, j-1 { // reverse
+		names[i], names[j] = names[j], names[i]
+	}
+	return "/" + strings.Join(names, "/"), 0
+}
diff --git a/pkg/meta/redis.go b/pkg/meta/redis.go
index 32d5e471..c30f2c04 100644
--- a/pkg/meta/redis.go
+++ b/pkg/meta/redis.go
@@ -2849,11 +2849,11 @@ func (r *redisMeta) cleanupLeakedInodes(delete bool) {
 	}
 }
 
-func (r *redisMeta) ListSlices(ctx Context, slices *[]Slice, delete bool, showProgress func()) syscall.Errno {
+func (r *redisMeta) ListSlices(ctx Context, slices map[Ino][]Slice, delete bool, showProgress func()) syscall.Errno {
 	r.cleanupLeakedInodes(delete)
 	r.cleanupLeakedChunks()
 	r.cleanupOldSliceRefs()
-	*slices = nil
+
 	var cursor uint64
 	p := r.rdb.Pipeline()
 	for {
@@ -2871,11 +2871,13 @@ func (r *redisMeta) ListSlices(ctx Context, slices *[]Slice, delete bool, showPr
 			return errno(err)
 		}
 		for _, cmd := range cmds {
+			key := cmd.(*redis.StringSliceCmd).Args()[1].(string)
+			inode, _ := strconv.Atoi(strings.Split(key[1:], "_")[0])
 			vals := cmd.(*redis.StringSliceCmd).Val()
 			ss := readSlices(vals)
 			for _, s := range ss {
 				if s.chunkid > 0 {
-					*slices = append(*slices, Slice{Chunkid: s.chunkid, Size: s.size})
+					slices[Ino(inode)] = append(slices[Ino(inode)], Slice{Chunkid: s.chunkid, Size: s.size})
 					if showProgress != nil {
 						showProgress()
 					}
diff --git a/pkg/meta/sql.go b/pkg/meta/sql.go
index ba6f18bb..3c7d4af5 100644
--- a/pkg/meta/sql.go
+++ b/pkg/meta/sql.go
@@ -2475,7 +2475,7 @@ func (m *dbMeta) CompactAll(ctx Context) syscall.Errno {
 	return 0
 }
 
-func (m *dbMeta) ListSlices(ctx Context, slices *[]Slice, delete bool, showProgress func()) syscall.Errno {
+func (m *dbMeta) ListSlices(ctx Context, slices map[Ino][]Slice, delete bool, showProgress func()) syscall.Errno {
 	var c chunk
 	rows, err := m.engine.Rows(&c)
 	if err != nil {
@@ -2483,7 +2483,6 @@ func (m *dbMeta) ListSlices(ctx Context, slices *[]Slice, delete bool, showProgr
 	}
 	defer rows.Close()
 
-	*slices = nil
 	for rows.Next() {
 		err = rows.Scan(&c)
 		if err != nil {
@@ -2492,7 +2491,7 @@ func (m *dbMeta) ListSlices(ctx Context, slices *[]Slice, delete bool, showProgr
 		ss := readSliceBuf(c.Slices)
 		for _, s := range ss {
 			if s.chunkid > 0 {
-				*slices = append(*slices, Slice{Chunkid: s.chunkid, Size: s.size})
+				slices[c.Inode] = append(slices[c.Inode], Slice{Chunkid: s.chunkid, Size: s.size})
 				if showProgress != nil {
 					showProgress()
 				}
diff --git a/pkg/meta/tkv.go b/pkg/meta/tkv.go
index 78e90037..22ea44ce 100644
--- a/pkg/meta/tkv.go
+++ b/pkg/meta/tkv.go
@@ -2332,22 +2332,22 @@ func (r *kvMeta) CompactAll(ctx Context) syscall.Errno {
 	return 0
 }
 
-func (r *kvMeta) ListSlices(ctx Context, slices *[]Slice, delete bool, showProgress func()) syscall.Errno {
-	*slices = nil
+func (m *kvMeta) ListSlices(ctx Context, slices map[Ino][]Slice, delete bool, showProgress func()) syscall.Errno {
 	// AiiiiiiiiCnnnn     file chunks
 	klen := 1 + 8 + 1 + 4
-	result, err := r.scanValues(r.fmtKey("A"), func(k, v []byte) bool {
+	result, err := m.scanValues(m.fmtKey("A"), func(k, v []byte) bool {
 		return len(k) == klen && k[1+8] == 'C'
 	})
 	if err != nil {
 		logger.Warnf("scan chunks: %s", err)
 		return errno(err)
 	}
-	for _, value := range result {
+	for key, value := range result {
+		inode := m.decodeInode([]byte(key)[1:9])
 		ss := readSliceBuf(value)
 		for _, s := range ss {
 			if s.chunkid > 0 {
-				*slices = append(*slices, Slice{Chunkid: s.chunkid, Size: s.size})
+				slices[inode] = append(slices[inode], Slice{Chunkid: s.chunkid, Size: s.size})
 				if showProgress != nil {
 					showProgress()
 				}


Test patch:
diff --git a/pkg/meta/redis_test.go b/pkg/meta/redis_test.go
index 51facd29..ffc4bf6b 100644
--- a/pkg/meta/redis_test.go
+++ b/pkg/meta/redis_test.go
@@ -779,8 +779,8 @@ func testCompaction(t *testing.T, m Meta) {
 	if st := m.CompactAll(ctx); st != 0 {
 		logger.Fatalf("compactall: %s", st)
 	}
-	var slices []Slice
-	if st := m.ListSlices(ctx, &slices, false, nil); st != 0 {
+	slices := make(map[Ino][]Slice)
+	if st := m.ListSlices(ctx, slices, false, nil); st != 0 {
 		logger.Fatalf("list all slices: %s", st)
 	}
 
@@ -896,13 +896,17 @@ func testTruncateAndDelete(t *testing.T, m Meta) {
 		),
 		mpb.BarFillerClearOnComplete(),
 	)
-	var ss []Slice
-	m.ListSlices(ctx, &ss, false, func() {
+	slices := make(map[Ino][]Slice)
+	m.ListSlices(ctx, slices, false, func() {
 		bar.SetTotal(total+2048, false)
 		bar.Increment()
 	})
-	if len(ss) != 1 {
-		t.Fatalf("number of chunks: %d != 1, %+v", len(ss), ss)
+	var totalSlices int
+	for _, ss := range slices {
+		totalSlices += len(ss)
+	}
+	if totalSlices != 1 {
+		t.Fatalf("number of chunks: %d != 1, %+v", totalSlices, slices)
 	}
 	m.Close(ctx, inode)
 	if st := m.Unlink(ctx, 1, "f"); st != 0 {
@@ -910,10 +914,15 @@ func testTruncateAndDelete(t *testing.T, m Meta) {
 	}
 
 	time.Sleep(time.Millisecond * 100)
-	m.ListSlices(ctx, &ss, false, nil)
+	slices = make(map[Ino][]Slice)
+	m.ListSlices(ctx, slices, false, nil)
+	totalSlices = 0
+	for _, ss := range slices {
+		totalSlices += len(ss)
+	}
 	// the last chunk could be found and deleted
-	if len(ss) > 1 {
-		t.Fatalf("number of chunks: %d > 1, %+v", len(ss), ss)
+	if totalSlices > 1 {
+		t.Fatalf("number of chunks: %d > 1, %+v", totalSlices, slices)
 	}
 }