Repository: juicedata/juicefs
Pull request #5939 (2025-04-08 11:21:54)

Hadoop: hdfs command does not show the Quota for JuiceFS
This PR is for https://github.com/juicedata/juicefs/issues/5937


Patch:
diff --git a/cmd/object.go b/cmd/object.go
index 83811176..14271831 100644
--- a/cmd/object.go
+++ b/cmd/object.go
@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"math/rand"
@@ -187,8 +188,9 @@ func (j *juiceFS) Delete(key string, getters ...object.AttrGetter) error {
 }
 
 type jObj struct {
-	key string
-	fi  *fs.FileStat
+	key       string
+	fi        *fs.FileStat
+	isSymlink bool
 }
 
 func (o *jObj) Key() string { return o.key }
@@ -200,21 +202,32 @@ func (o *jObj) Size() int64 {
 }
 func (o *jObj) Mtime() time.Time     { return o.fi.ModTime() }
 func (o *jObj) IsDir() bool          { return o.fi.IsDir() }
-func (o *jObj) IsSymlink() bool      { return o.fi.IsSymlink() }
+func (o *jObj) IsSymlink() bool      { return o.isSymlink }
 func (o *jObj) Owner() string        { return utils.UserName(o.fi.Uid()) }
 func (o *jObj) Group() string        { return utils.GroupName(o.fi.Gid()) }
 func (o *jObj) Mode() os.FileMode    { return o.fi.Mode() }
 func (o *jObj) StorageClass() string { return "" }
 
 func (j *juiceFS) Head(key string) (object.Object, error) {
-	fi, eno := j.jfs.Stat(ctx, j.path(key))
-	if eno == syscall.ENOENT {
-		return nil, os.ErrNotExist
+	errConv := func(eno syscall.Errno) error {
+		if errors.Is(eno, syscall.ENOENT) {
+			return os.ErrNotExist
+		} else {
+			return eno
+		}
 	}
+	fi, eno := j.jfs.Lstat(ctx, j.path(key))
 	if eno != 0 {
-		return nil, eno
+		return nil, errConv(eno)
+	}
+	isSymlink := fi.IsSymlink()
+	if isSymlink {
+		fi, eno = j.jfs.Stat(ctx, j.path(key))
+		if eno != 0 {
+			return nil, errConv(eno)
+		}
 	}
-	return &jObj{key, fi}, nil
+	return &jObj{key, fi, isSymlink}, nil
 }
 
 func (j *juiceFS) List(prefix, marker, token, delimiter string, limit int64, followLink bool) ([]object.Object, bool, string, error) {
@@ -250,7 +263,7 @@ func (j *juiceFS) List(prefix, marker, token, delimiter string, limit int64, fol
 		if !strings.HasPrefix(key, prefix) || (marker != "" && key <= marker) {
 			continue
 		}
-		f := &jObj{key, e.fi}
+		f := &jObj{key, e.fi, e.fi.IsSymlink()}
 		objs = append(objs, f)
 		if len(objs) == int(limit) {
 			break
diff --git a/cmd/sync.go b/cmd/sync.go
index a51ebed8..bfe551b6 100644
--- a/cmd/sync.go
+++ b/cmd/sync.go
@@ -165,7 +165,7 @@ func selectionFlags() []cli.Flag {
 		},
 		&cli.StringFlag{
 			Name:  "files-from",
-			Usage: "generate the list of files or dirs to sync from FILE",
+			Usage: "read list of files or dirs to sync from FILE",
 		},
 	})
 }
diff --git a/pkg/fs/fs.go b/pkg/fs/fs.go
index 3e135d7d..624b7355 100644
--- a/pkg/fs/fs.go
+++ b/pkg/fs/fs.go
@@ -1401,3 +1401,29 @@ func (f *File) GetTreeSummary(ctx meta.Context, depth, entries uint8, strict boo
 	s.Path = path.Base(f.path)
 	return
 }
+
+func (f *File) GetQuota(ctx meta.Context) (quota *meta.Quota, err error) {
+	defer trace.StartRegion(context.TODO(), "fs.getQuota").End()
+	l := vfs.NewLogContext(ctx)
+	defer func() {
+		f.fs.log(l, "getQuota (%s): %s", f.path, errstr(err))
+	}()
+	err = nil
+	qs := make(map[string]*meta.Quota)
+	// get filesystem quota if root
+	if f.inode == meta.RootInode {
+		format := f.fs.m.GetFormat()
+		quota = &meta.Quota{
+			MaxSpace:  int64(format.Capacity),
+			MaxInodes: int64(format.Inodes),
+		}
+		return quota, err
+	}
+	// get directory quota
+	err = f.fs.m.HandleQuota(ctx, meta.QuotaGet, f.path, qs, false, false, false)
+	if err != nil {
+		return nil, err
+	}
+	quota = qs[f.path]
+	return quota, err
+}
diff --git a/pkg/object/file.go b/pkg/object/file.go
index c26c97a1..c6104fe7 100644
--- a/pkg/object/file.go
+++ b/pkg/object/file.go
@@ -76,11 +76,18 @@ func (d *filestore) path(key string) string {
 
 func (d *filestore) Head(key string) (Object, error) {
 	p := d.path(key)
-	fi, err := os.Stat(p)
+	fi, err := os.Lstat(p)
 	if err != nil {
 		return nil, err
 	}
-	return toFile(key, fi, false, getOwnerGroup), nil
+	isSymlink := fi.Mode()&os.ModeSymlink != 0
+	if isSymlink {
+		fi, err = os.Stat(p)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return toFile(key, fi, isSymlink, getOwnerGroup), nil
 }
 
 func toFile(key string, fi fs.FileInfo, isSymlink bool, ownerGetter func(fs.FileInfo) (string, string)) *file {
@@ -247,9 +254,10 @@ func readDirSorted(dir string, followLink bool) ([]*mEntry, error) {
 
 	mEntries := make([]*mEntry, len(entries))
 	for i, e := range entries {
+		isSymlink := e.Mode()&os.ModeSymlink != 0
 		if e.IsDir() {
 			mEntries[i] = &mEntry{e, e.Name() + dirSuffix, nil, false}
-		} else if !e.Mode().IsRegular() && followLink {
+		} else if isSymlink && followLink {
 			fi, err := os.Stat(filepath.Join(dir, e.Name()))
 			if err != nil {
 				mEntries[i] = &mEntry{e, e.Name(), nil, true}
@@ -261,7 +269,7 @@ func readDirSorted(dir string, followLink bool) ([]*mEntry, error) {
 			}
 			mEntries[i] = &mEntry{e, name, fi, false}
 		} else {
-			mEntries[i] = &mEntry{e, e.Name(), nil, !e.Mode().IsRegular()}
+			mEntries[i] = &mEntry{e, e.Name(), nil, isSymlink}
 		}
 	}
 	sort.Slice(mEntries, func(i, j int) bool { return mEntries[i].Name() < mEntries[j].Name() })
diff --git a/pkg/object/gluster.go b/pkg/object/gluster.go
index 44726fc5..aa423e99 100644
--- a/pkg/object/gluster.go
+++ b/pkg/object/gluster.go
@@ -178,9 +178,10 @@ func (g *gluster) readDirSorted(dirname string, followLink bool) ([]*mEntry, err
 		if name == "." || name == ".." {
 			continue
 		}
+		isSymlink := e.Mode()&os.ModeSymlink != 0
 		if e.IsDir() {
 			mEntries = append(mEntries, &mEntry{nil, name + dirSuffix, e, false})
-		} else if !e.Mode().IsRegular() && followLink {
+		} else if isSymlink && followLink {
 			fi, err := v.Stat(filepath.Join(dirname, name))
 			if err != nil {
 				mEntries = append(mEntries, &mEntry{nil, name, e, true})
@@ -191,7 +192,7 @@ func (g *gluster) readDirSorted(dirname string, followLink bool) ([]*mEntry, err
 			}
 			mEntries = append(mEntries, &mEntry{nil, name, fi, false})
 		} else {
-			mEntries = append(mEntries, &mEntry{nil, name, e, !e.Mode().IsRegular()})
+			mEntries = append(mEntries, &mEntry{nil, name, e, isSymlink})
 		}
 	}
 	sort.Slice(mEntries, func(i, j int) bool { return mEntries[i].Name() < mEntries[j].Name() })
diff --git a/pkg/object/nfs.go b/pkg/object/nfs.go
index ecdcd776..4f604b77 100644
--- a/pkg/object/nfs.go
+++ b/pkg/object/nfs.go
@@ -106,7 +106,14 @@ func (n *nfsStore) Head(key string) (Object, error) {
 			return nil, err
 		}
 		dir, _ := path.Split(p)
-		return n.Head(path.Join(dir, src))
+		ff, err := n.Head(path.Join(dir, src))
+		if err != nil {
+			return nil, err
+		}
+		if f2, ok := ff.(*file); ok {
+			f2.isSymlink = true
+		}
+		return ff, nil
 	}
 	return n.fileInfo(key, fi), nil
 }
@@ -236,7 +243,7 @@ func (n *nfsStore) Delete(key string, getters ...AttrGetter) error {
 
 func (n *nfsStore) fileInfo(key string, fi os.FileInfo) Object {
 	owner, group := n.getOwnerGroup(fi)
-	isSymlink := !fi.Mode().IsDir() && !fi.Mode().IsRegular()
+	isSymlink := fi.Mode()&os.ModeSymlink != 0
 	ff := &file{
 		obj{key, fi.Size(), fi.ModTime(), fi.IsDir(), ""},
 		owner,
diff --git a/pkg/object/sftp.go b/pkg/object/sftp.go
index 69f865a3..3cf9a8ee 100644
--- a/pkg/object/sftp.go
+++ b/pkg/object/sftp.go
@@ -168,11 +168,19 @@ func (f *sftpStore) Head(key string) (Object, error) {
 	}
 	defer f.putSftpConnection(&c, err)
 
-	info, err := c.sftpClient.Stat(f.path(key))
+	info, err := c.sftpClient.Lstat(f.path(key))
 	if err != nil {
 		return nil, err
 	}
-	return f.fileInfo(nil, key, info, true), nil
+	var isSymlink bool
+	if info.Mode()&os.ModeSymlink != 0 {
+		isSymlink = true
+		info, err = c.sftpClient.Stat(f.path(key))
+		if err != nil {
+			return nil, err
+		}
+	}
+	return f.fileInfo(key, info, isSymlink), nil
 }
 
 func (f *sftpStore) Get(key string, off, limit int64, getters ...AttrGetter) (io.ReadCloser, error) {
@@ -322,28 +330,35 @@ func (f *sftpStore) Delete(key string, getters ...AttrGetter) error {
 	return err
 }
 
-func (f *sftpStore) sortByName(c *sftp.Client, path string, fis []os.FileInfo, followLink bool) []Object {
-	var obs = make([]Object, 0, len(fis))
-	for _, fi := range fis {
-		p := path + fi.Name()
-		if strings.HasPrefix(p, f.root) {
-			key := p[len(f.root):]
-			obs = append(obs, f.fileInfo(c, key, fi, followLink))
+func (f *sftpStore) sortByName(c *sftp.Client, path string, fis []os.FileInfo, followLink bool) []*mEntry {
+	mEntries := make([]*mEntry, len(fis))
+	for i, e := range fis {
+		isSymlink := e.Mode()&os.ModeSymlink != 0
+		if e.IsDir() {
+			mEntries[i] = &mEntry{e, e.Name() + dirSuffix, nil, false}
+		} else if isSymlink && followLink {
+			var fi os.FileInfo
+			p := path + e.Name()
+			fi, err := c.Stat(p)
+			if err != nil {
+				mEntries[i] = &mEntry{e, e.Name(), nil, true}
+				continue
+			}
+			name := e.Name()
+			if fi.IsDir() {
+				name = e.Name() + dirSuffix
+			}
+			mEntries[i] = &mEntry{e, name, fi, false}
+		} else {
+			mEntries[i] = &mEntry{e, e.Name(), nil, isSymlink}
 		}
 	}
-	sort.Slice(obs, func(i, j int) bool { return obs[i].Key() < obs[j].Key() })
-	return obs
+	sort.Slice(mEntries, func(i, j int) bool { return mEntries[i].Name() < mEntries[j].Name() })
+	return mEntries
 }
 
-func (f *sftpStore) fileInfo(c *sftp.Client, key string, fi os.FileInfo, followLink bool) Object {
+func (f *sftpStore) fileInfo(key string, fi os.FileInfo, isSymlink bool) Object {
 	owner, group := getOwnerGroup(fi)
-	isSymlink := !fi.Mode().IsDir() && !fi.Mode().IsRegular()
-	if isSymlink && c != nil && followLink {
-		if fi2, err := c.Stat(f.root + key); err == nil {
-			fi = fi2
-			isSymlink = false
-		}
-	}
 	ff := &file{
 		obj{key, fi.Size(), fi.ModTime(), fi.IsDir(), ""},
 		owner,
@@ -401,12 +416,21 @@ func (f *sftpStore) List(prefix, marker, token, delimiter string, limit int64, f
 	}
 
 	entries := f.sortByName(c.sftpClient, dir, infos, followLink)
-	for _, o := range entries {
-		key := o.Key()
+	for _, e := range entries {
+		p := path.Join(dir, e.Name())
+		if e.IsDir() {
+			p = p + "/"
+		}
+		if !strings.HasPrefix(p, f.root) {
+			continue
+		}
+		key := p[len(f.root):]
 		if !strings.HasPrefix(key, prefix) || (marker != "" && key <= marker) {
 			continue
 		}
-		objs = append(objs, o)
+		info := e.Info()
+		f := toFile(key, info, e.isSymlink, getOwnerGroup)
+		objs = append(objs, f)
 		if len(objs) == int(limit) {
 			break
 		}
diff --git a/pkg/sync/sync.go b/pkg/sync/sync.go
index 4fb61ce2..9a923865 100644
--- a/pkg/sync/sync.go
+++ b/pkg/sync/sync.go
@@ -1396,6 +1396,13 @@ func produceFromList(tasks chan<- object.Object, src, dst object.ObjectStorage,
 		go func() {
 			defer wg.Done()
 			for key := range prefixs {
+				if !strings.HasSuffix(key, "/") {
+					if err := produceSingleObject(tasks, src, dst, key, config); err == nil {
+						continue
+					} else if errors.Is(err, ignoreDir) {
+						key += "/"
+					}
+				}
 				logger.Debugf("start listing prefix %s", key)
 				err = startProducer(tasks, src, dst, key, config.ListDepth, config)
 				if err != nil {
@@ -1426,31 +1433,43 @@ func produceFromList(tasks chan<- object.Object, src, dst object.ObjectStorage,
 	return nil
 }
 
+var ignoreDir = errors.New("ignore dir")
+
+func produceSingleObject(tasks chan<- object.Object, src, dst object.ObjectStorage, key string, config *Config) error {
+	obj, err := src.Head(key)
+	if err == nil && (!obj.IsDir() || obj.IsSymlink() && config.Links || obj.IsDir() && config.Dirs && strings.HasSuffix(key, "/")) {
+		var srckeys = make(chan object.Object, 1)
+		srckeys <- obj
+		close(srckeys)
+		if dobj, e := dst.Head(key); e == nil || os.IsNotExist(e) {
+			var dstkeys = make(chan object.Object, 1)
+			if dobj != nil {
+				dstkeys <- dobj
+			}
+			close(dstkeys)
+			logger.Debugf("produce single key %s", key)
+			_ = produce(tasks, srckeys, dstkeys, config)
+			return nil
+		} else {
+			logger.Warnf("head %s from %s: %s", key, dst, e)
+			err = e
+		}
+	} else if err != nil {
+		logger.Warnf("head %s from %s: %s", key, src, err)
+	} else {
+		err = ignoreDir
+	}
+	return err
+}
+
 func startProducer(tasks chan<- object.Object, src, dst object.ObjectStorage, prefix string, listDepth int, config *Config) error {
 	config.concurrentList <- 1
 	defer func() {
 		<-config.concurrentList
 	}()
 	if config.Limit == 1 && len(config.rules) == 0 {
-		// fast path for single key
-		obj, err := src.Head(config.Start)
-		if err == nil && (!obj.IsDir() || config.Dirs) {
-			var srckeys = make(chan object.Object, 1)
-			srckeys <- obj
-			close(srckeys)
-			var dstkeys = make(chan object.Object, 1)
-			if dobj, err := dst.Head(config.Start); err == nil || os.IsNotExist(err) {
-				if dobj != nil {
-					dstkeys <- dobj
-				}
-				close(dstkeys)
-				logger.Debugf("produce single key %s", config.Start)
-				return produce(tasks, srckeys, dstkeys, config)
-			} else {
-				logger.Warnf("head %s from %s: %s", config.Start, dst, err)
-			}
-		} else if err != nil && !os.IsNotExist(err) {
-			logger.Warnf("head %s from %s: %s", config.Start, src, err)
+		if produceSingleObject(tasks, src, dst, prefix, config) == nil {
+			return nil
 		}
 	}
 	if config.ListThreads <= 1 || listDepth <= 0 {
diff --git a/sdk/java/libjfs/main.go b/sdk/java/libjfs/main.go
index 17cc2182..eca9c90b 100644
--- a/sdk/java/libjfs/main.go
+++ b/sdk/java/libjfs/main.go
@@ -1265,11 +1265,21 @@ func jfs_summary(pid int64, h int64, cpath *C.char, buf uintptr) int32 {
 	if err != 0 {
 		return errno(err)
 	}
-	wb := utils.NewNativeBuffer(toBuf(buf, 24))
+	wb := utils.NewNativeBuffer(toBuf(buf, 40))
 	wb.Put64(summary.Length)
 	wb.Put64(summary.Files)
 	wb.Put64(summary.Dirs)
-	return 24
+
+	// quota
+	quota, _ := f.GetQuota(ctx)
+	if quota != nil {
+		wb.Put64(uint64(quota.MaxInodes))
+		wb.Put64(uint64(quota.MaxSpace))
+	} else {
+		wb.Put64(0)
+		wb.Put64(0)
+	}
+	return 40
 }
 
 //export jfs_info
diff --git a/sdk/java/src/main/java/io/juicefs/JuiceFileSystemImpl.java b/sdk/java/src/main/java/io/juicefs/JuiceFileSystemImpl.java
index 36618ecb..34f0a840 100644
--- a/sdk/java/src/main/java/io/juicefs/JuiceFileSystemImpl.java
+++ b/sdk/java/src/main/java/io/juicefs/JuiceFileSystemImpl.java
@@ -1639,7 +1639,7 @@ public class JuiceFileSystemImpl extends FileSystem {
     }
     statistics.incrementReadOps(1);
     String path = normalizePath(f);
-    Pointer buf = Memory.allocate(Runtime.getRuntime(lib), 24);
+    Pointer buf = Memory.allocate(Runtime.getRuntime(lib), 40);
     int r = lib.jfs_summary(Thread.currentThread().getId(), handle, path, buf);
     if (r < 0) {
       throw error(r, f);
@@ -1647,7 +1647,11 @@ public class JuiceFileSystemImpl extends FileSystem {
     long size = buf.getLongLong(0);
     long files = buf.getLongLong(8);
     long dirs = buf.getLongLong(16);
-    return new ContentSummary(size, files, dirs);
+    long quota = buf.getLongLong(24);
+    long spaceQuota = buf.getLongLong(32);
+    quota = quota == 0 ? -1L : quota;
+    spaceQuota = spaceQuota == 0 ? -1L : spaceQuota;
+    return new ContentSummary(size, files, dirs, quota, size, spaceQuota);
   }
 
   private FileStatus newFileStatus(Path p, Pointer buf, int size, boolean readlink) throws IOException {


Test patch:
diff --git a/cmd/object_test.go b/cmd/object_test.go
index 8ac46c7a..a914ebd7 100644
--- a/cmd/object_test.go
+++ b/cmd/object_test.go
@@ -17,7 +17,9 @@ package cmd
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
+	"os"
 	"strings"
 	"testing"
 	"time"
@@ -135,6 +137,16 @@ func testFileSystem(t *testing.T, s object.ObjectStorage) {
 		if target, err := ss.Readlink("b"); err != nil || target != "/xyz/notExist/" {
 			t.Fatalf("readlink b %s %s", target, err)
 		}
+		head, err := s.Head("a")
+		if err != nil || !head.IsSymlink() {
+			t.Fatalf("head a %s %s", head, err)
+		}
+		ss.Symlink("notExit", "brokenLink")
+		_, err = s.Head("brokenLink")
+		if !errors.Is(err, os.ErrNotExist) {
+			t.Fatalf("head b %s %s", head, err)
+		}
+		s.Delete("brokenLink")
 		objs, err = listAll(s, "", "", 100)
 		if err != nil {
 			t.Fatalf("listall failed: %s", err)
diff --git a/pkg/fs/fs_test.go b/pkg/fs/fs_test.go
index e5cb037a..beb15c0f 100644
--- a/pkg/fs/fs_test.go
+++ b/pkg/fs/fs_test.go
@@ -221,6 +221,9 @@ func TestFileSystem(t *testing.T) {
 	if s, e := d.Summary(ctx, true, true); e != 0 || s.Dirs != 2 || s.Files != 2 || s.Length != 7 || s.Size != 16<<10 {
 		t.Fatalf("summary: %s %+v", e, s)
 	}
+	if q, e := d.GetQuota(ctx); e != nil || q.MaxInodes != 0 || q.MaxSpace != (1<<30) {
+		t.Fatalf("quota: %s %+v", e, q)
+	}
 	if e := fs.Delete(ctx, "/d"); e == 0 || !IsNotEmpty(e) {
 		t.Fatalf("rmdir: %s", e)
 	}
diff --git a/pkg/object/filesystem_test.go b/pkg/object/filesystem_test.go
index c24b4345..32597a62 100644
--- a/pkg/object/filesystem_test.go
+++ b/pkg/object/filesystem_test.go
@@ -18,6 +18,7 @@ package object
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
 	"os"
 	"strings"
@@ -195,9 +196,17 @@ func testFileSystem(t *testing.T, s ObjectStorage) {
 		} else if target != "../b0" {
 			t.Fatalf("target should be ../b0, but got %s", target)
 		}
-		if fi, err := s.Head("bb/b1"); err != nil || fi.Size() != 10 {
-			t.Fatalf("size of symlink: %s, %d != %d", err, fi.Size(), 10)
+		if fi, err := s.Head("bb/b1"); err != nil || !fi.IsSymlink() || fi.Size() != 10 {
+			t.Fatalf("haed of symlink: err=%s, size=%d isSymlink=%v", err, fi.Size(), fi.IsSymlink())
 		}
+		err = ss.Symlink("../notExist", "bb/brokenLink")
+		if err != nil {
+			t.Fatalf("symlink: %s", err)
+		}
+		if _, err := s.Head("bb/brokenLink"); !errors.Is(err, os.ErrNotExist) {
+			t.Fatalf("head broken symlink: err=%s, should be os.ErrNotExist", err)
+		}
+		_ = s.Delete("bb/brokenLink")
 		if err = ss.Symlink("xyz/ol1/", "a"); err != nil {
 			t.Fatalf("symlink: a: %s", err)
 		}
diff --git a/sdk/java/src/test/java/io/juicefs/JuiceFileSystemTest.java b/sdk/java/src/test/java/io/juicefs/JuiceFileSystemTest.java
index cfc7173a..09615050 100644
--- a/sdk/java/src/test/java/io/juicefs/JuiceFileSystemTest.java
+++ b/sdk/java/src/test/java/io/juicefs/JuiceFileSystemTest.java
@@ -92,6 +92,8 @@ public class JuiceFileSystemTest extends TestCase {
     assertEquals(6, summary.getLength());
     assertEquals(1, summary.getFileCount());
     assertEquals(0, summary.getDirectoryCount());
+    assertEquals(-1L, summary.getQuota());
+    assertEquals(-1L, summary.getSpaceQuota());
   }
 
   public void testLongName() throws IOException {