Repository: tetratelabs/wazero
Pull request #1152 (2023-02-22 08:54:59)

wasi: manually adds dot and dot-dot directory entries
This manually adds dot and dot-dot directory entries discarded by compilers like Go. Doing so complies with the latest interpretation of wasi preview1 without forcing us to do the same in preview2 or in GOOS=js. There's a cost penalty of one stat per directory list, which will be only measurable when using real file I/O.

This should result in us being able to remove the exclusion here https://github.com/WebAssembly/wasi-testsuite/pull/55/files#diff-8a3ffd323d75a12f8deb01b053f062876d83dda0b89c1fa24b293cee4195bcfd

See https://github.com/WebAssembly/wasi-testsuite/issues/52


Patch:
diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml
index a72a1685..cfc4dfae 100644
--- a/.github/workflows/integration.yaml
+++ b/.github/workflows/integration.yaml
@@ -297,5 +297,4 @@ jobs:
             -t ./tests/assemblyscript/testsuite/ \
             ./tests/c/testsuite/ \
             ./tests/rust/testsuite/ \
-            -f ./adapters/wazero_skip.json \
             -r ./adapters/wazero.py
diff --git a/imports/wasi_snapshot_preview1/fs.go b/imports/wasi_snapshot_preview1/fs.go
index 0e045768..867ccc0e 100644
--- a/imports/wasi_snapshot_preview1/fs.go
+++ b/imports/wasi_snapshot_preview1/fs.go
@@ -836,6 +836,18 @@ func fdReaddirFn(_ context.Context, mod api.Module, params []uint64) Errno {
 		return errno
 	}
 
+	// Add entries for dot and dot-dot as wasi-testsuite requires them.
+	if cookie == 0 && entries == nil {
+		var err error
+		if f, ok := fsc.LookupFile(fd); !ok {
+			return ErrnoBadf
+		} else if entries, err = sys.DotEntries(f.File); err != nil {
+			return ToErrno(err)
+		}
+		dir.Entries = entries
+		dir.CountRead = 2 // . and ..
+	}
+
 	// Check if we have maxDirEntries, and read more from the FS as needed.
 	if entryCount := len(entries); entryCount < maxDirEntries {
 		l, err := rd.ReadDir(maxDirEntries - entryCount)
diff --git a/internal/sys/fs.go b/internal/sys/fs.go
index 2a51efc8..a5a7095a 100644
--- a/internal/sys/fs.go
+++ b/internal/sys/fs.go
@@ -203,6 +203,11 @@ type ReadDir struct {
 	// Entries is the contents of the last fs.ReadDirFile call. Notably,
 	// directory listing are not rewindable, so we keep entries around in case
 	// the caller mis-estimated their buffer and needs a few still cached.
+	//
+	// Note: This is wasi-specific and needs to be refactored.
+	// In wasi preview1, dot and dot-dot entries are required to exist, but the
+	// reverse is true for preview2. More importantly, preview2 holds separate
+	// stateful dir-entry-streams per file.
 	Entries []fs.DirEntry
 }
 
@@ -449,3 +454,46 @@ func WriterForFile(fsc *FSContext, fd uint32) (writer io.Writer) {
 	}
 	return
 }
+
+// DotEntries returns "." and "..", where "." has a real stat because
+// wasi-testsuite does inode validation.
+func DotEntries(f fs.File) ([]fs.DirEntry, error) {
+	var st platform.Stat_t
+	if err := platform.StatFile(f, &st); err != nil {
+		return nil, err
+	}
+	return []fs.DirEntry{&dotEntry{stat: &st}, dotDotEntry{}}, nil
+}
+
+// dotEntry is a fs.DirEntry representing the directory being listed.
+type dotEntry struct {
+	// stat is the stat of the opened directory
+	stat *platform.Stat_t
+}
+
+func (i *dotEntry) Name() string               { return "." }
+func (i *dotEntry) Type() fs.FileMode          { return i.stat.Mode.Type() }
+func (i *dotEntry) Info() (fs.FileInfo, error) { return i, nil }
+func (i *dotEntry) Size() int64                { return i.stat.Size }
+func (i *dotEntry) Mode() fs.FileMode          { return i.stat.Mode }
+func (i *dotEntry) ModTime() time.Time {
+	return time.Unix(i.stat.Mtim/1e9, i.stat.Mtim%1e9)
+}
+func (i *dotEntry) IsDir() bool      { return true }
+func (i *dotEntry) Sys() interface{} { return nil }
+
+// dotDotEntry is a fake entry for dot-dot (".."), added to satisfy WASI tests.
+//
+// Note: This is intentionally invalid as WASI decided that it must be present
+// on any list, including the root directory: No values are tested apart from
+// the name.
+type dotDotEntry struct{}
+
+func (dotDotEntry) Name() string               { return ".." }
+func (dotDotEntry) Type() fs.FileMode          { return fs.ModeDir }
+func (dotDotEntry) Info() (fs.FileInfo, error) { return dotDotEntry{}, nil }
+func (dotDotEntry) Size() int64                { return 0 }
+func (dotDotEntry) Mode() fs.FileMode          { return fs.ModeDir }
+func (dotDotEntry) ModTime() time.Time         { return time.Unix(0, 0) }
+func (dotDotEntry) IsDir() bool                { return true }
+func (dotDotEntry) Sys() interface{}           { return nil }


Test patch:
diff --git a/imports/wasi_snapshot_preview1/fs_test.go b/imports/wasi_snapshot_preview1/fs_test.go
index f0a1e98a..60e8756f 100644
--- a/imports/wasi_snapshot_preview1/fs_test.go
+++ b/imports/wasi_snapshot_preview1/fs_test.go
@@ -1806,30 +1806,55 @@ var (
 		if err != nil {
 			panic(err)
 		}
-		return entries
+		d, err := fstest.FS.Open("dir")
+		if err != nil {
+			panic(err)
+		}
+		defer d.Close()
+		dots, err := sys.DotEntries(d)
+		if err != nil {
+			panic(err)
+		}
+		return append(dots, entries...)
 	}()
 
-	dirent1 = []byte{
+	direntDot = []byte{
 		1, 0, 0, 0, 0, 0, 0, 0, // d_next = 1
 		0, 0, 0, 0, 0, 0, 0, 0, // d_ino = 0
 		1, 0, 0, 0, // d_namlen = 1 character
+		3, 0, 0, 0, // d_type =  directory
+		'.', // name
+	}
+	direntDotDot = []byte{
+		2, 0, 0, 0, 0, 0, 0, 0, // d_next = 2
+		0, 0, 0, 0, 0, 0, 0, 0, // d_ino = 0
+		2, 0, 0, 0, // d_namlen = 2 characters
+		3, 0, 0, 0, // d_type =  directory
+		'.', '.', // name
+	}
+	dirent1 = []byte{
+		3, 0, 0, 0, 0, 0, 0, 0, // d_next = 3
+		0, 0, 0, 0, 0, 0, 0, 0, // d_ino = 0
+		1, 0, 0, 0, // d_namlen = 1 character
 		4, 0, 0, 0, // d_type = regular_file
 		'-', // name
 	}
 	dirent2 = []byte{
-		2, 0, 0, 0, 0, 0, 0, 0, // d_next = 2
+		4, 0, 0, 0, 0, 0, 0, 0, // d_next = 4
 		0, 0, 0, 0, 0, 0, 0, 0, // d_ino = 0
 		2, 0, 0, 0, // d_namlen = 1 character
 		3, 0, 0, 0, // d_type =  directory
 		'a', '-', // name
 	}
 	dirent3 = []byte{
-		3, 0, 0, 0, 0, 0, 0, 0, // d_next = 3
+		5, 0, 0, 0, 0, 0, 0, 0, // d_next = 5
 		0, 0, 0, 0, 0, 0, 0, 0, // d_ino = 0
 		3, 0, 0, 0, // d_namlen = 3 characters
 		4, 0, 0, 0, // d_type = regular_file
 		'a', 'b', '-', // name
 	}
+
+	dirents = append(append(append(append(direntDot, direntDotDot...), dirent1...), dirent2...), dirent3...)
 )
 
 func Test_fdReaddir(t *testing.T) {
@@ -1860,11 +1885,14 @@ func Test_fdReaddir(t *testing.T) {
 
 				return &sys.FileEntry{File: dir}
 			},
-			bufLen:          DirentSize,
+			bufLen:          DirentSize + 1, // size of one entry
 			cookie:          0,
-			expectedBufused: 0,
-			expectedMem:     []byte{},
-			expectedReadDir: &sys.ReadDir{},
+			expectedBufused: DirentSize + 1, // one dot entry
+			expectedMem:     direntDot,
+			expectedReadDir: &sys.ReadDir{
+				CountRead: 2,
+				Entries:   testDirEntries[0:2], // dot and dot-dot
+			},
 		},
 		{
 			name: "full read",
@@ -1876,10 +1904,10 @@ func Test_fdReaddir(t *testing.T) {
 			},
 			bufLen:          4096,
 			cookie:          0,
-			expectedBufused: 78, // length of all entries
-			expectedMem:     append(append(dirent1, dirent2...), dirent3...),
+			expectedBufused: 129, // length of all entries
+			expectedMem:     dirents,
 			expectedReadDir: &sys.ReadDir{
-				CountRead: 3,
+				CountRead: 5,
 				Entries:   testDirEntries,
 			},
 		},
@@ -1891,13 +1919,13 @@ func Test_fdReaddir(t *testing.T) {
 
 				return &sys.FileEntry{File: dir}
 			},
-			bufLen:          24, // length is long enough for first, but not the name.
+			bufLen:          DirentSize, // length is long enough for first, but not the name.
 			cookie:          0,
-			expectedBufused: 24,           // == bufLen which is the size of the dirent
-			expectedMem:     dirent1[:24], // header without name
+			expectedBufused: DirentSize,             // == bufLen which is the size of the dirent
+			expectedMem:     direntDot[:DirentSize], // header without name
 			expectedReadDir: &sys.ReadDir{
 				CountRead: 3,
-				Entries:   testDirEntries,
+				Entries:   testDirEntries[0:3],
 			},
 		},
 		{
@@ -1911,10 +1939,10 @@ func Test_fdReaddir(t *testing.T) {
 			bufLen:          25, // length is long enough for first + the name, but not more.
 			cookie:          0,
 			expectedBufused: 25, // length to read exactly first.
-			expectedMem:     dirent1,
+			expectedMem:     direntDot,
 			expectedReadDir: &sys.ReadDir{
 				CountRead: 3,
-				Entries:   testDirEntries,
+				Entries:   testDirEntries[0:3],
 			},
 		},
 		{
@@ -1928,18 +1956,18 @@ func Test_fdReaddir(t *testing.T) {
 				return &sys.FileEntry{
 					File: dir,
 					ReadDir: &sys.ReadDir{
-						CountRead: 1,
-						Entries:   entry,
+						CountRead: 3,
+						Entries:   append(testDirEntries[0:2], entry...),
 					},
 				}
 			},
-			bufLen:          26, // length is long enough for exactly second.
+			bufLen:          27, // length is long enough for exactly second.
 			cookie:          1,  // d_next of first
-			expectedBufused: 26, // length to read exactly second.
-			expectedMem:     dirent2,
+			expectedBufused: 27, // length to read exactly second.
+			expectedMem:     direntDotDot,
 			expectedReadDir: &sys.ReadDir{
-				CountRead: 3,
-				Entries:   testDirEntries[1:],
+				CountRead: 4,
+				Entries:   testDirEntries[1:4],
 			},
 		},
 		{
@@ -1953,19 +1981,19 @@ func Test_fdReaddir(t *testing.T) {
 				return &sys.FileEntry{
 					File: dir,
 					ReadDir: &sys.ReadDir{
-						CountRead: 1,
-						Entries:   entry,
+						CountRead: 3,
+						Entries:   append(testDirEntries[0:2], entry...),
 					},
 				}
 			},
 			bufLen:          30, // length is longer than the second entry, but not long enough for a header.
 			cookie:          1,  // d_next of first
 			expectedBufused: 30, // length to read some more, but not enough for a header, so buf was exhausted.
-			expectedMem:     dirent2,
-			expectedMemSize: len(dirent2), // we do not want to compare the full buffer since we don't know what the leftover 4 bytes will contain.
+			expectedMem:     direntDotDot,
+			expectedMemSize: len(direntDotDot), // we do not want to compare the full buffer since we don't know what the leftover 4 bytes will contain.
 			expectedReadDir: &sys.ReadDir{
-				CountRead: 3,
-				Entries:   testDirEntries[1:],
+				CountRead: 4,
+				Entries:   testDirEntries[1:4],
 			},
 		},
 		{
@@ -1979,18 +2007,18 @@ func Test_fdReaddir(t *testing.T) {
 				return &sys.FileEntry{
 					File: dir,
 					ReadDir: &sys.ReadDir{
-						CountRead: 1,
-						Entries:   entry,
+						CountRead: 3,
+						Entries:   append(testDirEntries[0:2], entry...),
 					},
 				}
 			},
 			bufLen:          50, // length is longer than the second entry + enough for the header of third.
 			cookie:          1,  // d_next of first
 			expectedBufused: 50, // length to read exactly second and the header of third.
-			expectedMem:     append(dirent2, dirent3[0:24]...),
+			expectedMem:     append(direntDotDot, dirent1[0:24]...),
 			expectedReadDir: &sys.ReadDir{
-				CountRead: 3,
-				Entries:   testDirEntries[1:],
+				CountRead: 5,
+				Entries:   testDirEntries[1:5],
 			},
 		},
 		{
@@ -2004,18 +2032,18 @@ func Test_fdReaddir(t *testing.T) {
 				return &sys.FileEntry{
 					File: dir,
 					ReadDir: &sys.ReadDir{
-						CountRead: 1,
-						Entries:   entry,
+						CountRead: 3,
+						Entries:   append(testDirEntries[0:2], entry...),
 					},
 				}
 			},
 			bufLen:          53, // length is long enough for second and third.
 			cookie:          1,  // d_next of first
 			expectedBufused: 53, // length to read exactly one second and third.
-			expectedMem:     append(dirent2, dirent3...),
+			expectedMem:     append(direntDotDot, dirent1...),
 			expectedReadDir: &sys.ReadDir{
-				CountRead: 3,
-				Entries:   testDirEntries[1:],
+				CountRead: 5,
+				Entries:   testDirEntries[1:5],
 			},
 		},
 		{
@@ -2029,17 +2057,17 @@ func Test_fdReaddir(t *testing.T) {
 				return &sys.FileEntry{
 					File: dir,
 					ReadDir: &sys.ReadDir{
-						CountRead: 2,
-						Entries:   two[1:],
+						CountRead: 4,
+						Entries:   append(testDirEntries[0:2], two[0:]...),
 					},
 				}
 			},
 			bufLen:          27, // length is long enough for exactly third.
 			cookie:          2,  // d_next of second.
 			expectedBufused: 27, // length to read exactly third.
-			expectedMem:     dirent3,
+			expectedMem:     dirent1,
 			expectedReadDir: &sys.ReadDir{
-				CountRead: 3,
+				CountRead: 5,
 				Entries:   testDirEntries[2:],
 			},
 		},
@@ -2054,17 +2082,17 @@ func Test_fdReaddir(t *testing.T) {
 				return &sys.FileEntry{
 					File: dir,
 					ReadDir: &sys.ReadDir{
-						CountRead: 2,
-						Entries:   two[1:],
+						CountRead: 4,
+						Entries:   append(testDirEntries[0:2], two[0:]...),
 					},
 				}
 			},
-			bufLen:          100, // length is long enough for third and more, but there is nothing more.
+			bufLen:          300, // length is long enough for third and more
 			cookie:          2,   // d_next of second.
-			expectedBufused: 27,  // length to read exactly third.
-			expectedMem:     dirent3,
+			expectedBufused: 78,  // length to read the rest
+			expectedMem:     append(dirent1, dirent2...),
 			expectedReadDir: &sys.ReadDir{
-				CountRead: 3,
+				CountRead: 5,
 				Entries:   testDirEntries[2:],
 			},
 		},
@@ -2121,7 +2149,7 @@ func Test_fdReaddir_Rewind(t *testing.T) {
 	require.NoError(t, err)
 
 	mem := mod.Memory()
-	const resultBufUsed, buf, bufSize = 0, 8, 100
+	const resultBufUsed, buf, bufSize = 0, 8, 200
 	read := func(cookie, bufSize uint64) (bufUsed uint32) {
 		requireErrnoResult(t, ErrnoSuccess, mod, FdReaddirName,
 			uint64(fd), buf, bufSize, cookie, uint64(resultBufUsed))
@@ -2135,10 +2163,10 @@ func Test_fdReaddir_Rewind(t *testing.T) {
 	// Initial read.
 	initialBufUsed := read(cookie, bufSize)
 	// Ensure that all is read.
-	require.Equal(t, len(dirent1)+len(dirent2)+len(dirent3), int(initialBufUsed))
+	require.Equal(t, len(dirents), int(initialBufUsed))
 	resultBuf, ok := mem.Read(buf, initialBufUsed)
 	require.True(t, ok)
-	require.Equal(t, append(append(dirent1, dirent2...), dirent3...), resultBuf)
+	require.Equal(t, dirents, resultBuf)
 
 	// Mask the result.
 	for i := range resultBuf {
@@ -2146,9 +2174,9 @@ func Test_fdReaddir_Rewind(t *testing.T) {
 	}
 
 	// Advance the cookie beyond the existing entries.
-	cookie += 3
+	cookie += 5
 	// Nothing to read from, so bufUsed must be zero.
-	require.Equal(t, 0, int(read(cookie, bufSize)))
+	require.Zero(t, int(read(cookie, bufSize)))
 
 	// Ensure buffer is intact.
 	for i := range resultBuf {
@@ -2159,10 +2187,10 @@ func Test_fdReaddir_Rewind(t *testing.T) {
 	cookie = 0
 	usedAfterRewind := read(cookie, bufSize)
 	// Ensure that all is read.
-	require.Equal(t, len(dirent1)+len(dirent2)+len(dirent3), int(usedAfterRewind))
+	require.Equal(t, len(dirents), int(usedAfterRewind))
 	resultBuf, ok = mem.Read(buf, usedAfterRewind)
 	require.True(t, ok)
-	require.Equal(t, append(append(dirent1, dirent2...), dirent3...), resultBuf)
+	require.Equal(t, dirents, resultBuf)
 }
 
 func Test_fdReaddir_Errors(t *testing.T) {
@@ -4425,11 +4453,11 @@ func Test_fdReaddir_opened_file_written(t *testing.T) {
 	used, _ := mem.ReadUint32Le(resultBufused)
 
 	results, _ := mem.Read(buf, used)
-	require.Equal(t, []byte{
-		1, 0, 0, 0, 0, 0, 0, 0, // d_next = 1
+	require.Equal(t, append(append(direntDot, direntDotDot...),
+		3, 0, 0, 0, 0, 0, 0, 0, // d_next = 3
 		0, 0, 0, 0, 0, 0, 0, 0, // d_ino = 0
 		5, 0, 0, 0, // d_namlen = 4 character
 		4, 0, 0, 0, // d_type = regular_file
 		'a', 'f', 'i', 'l', 'e', // name
-	}, results)
+	), results)
 }
diff --git a/imports/wasi_snapshot_preview1/wasi_stdlib_test.go b/imports/wasi_snapshot_preview1/wasi_stdlib_test.go
index e424f337..047fb9bd 100644
--- a/imports/wasi_snapshot_preview1/wasi_stdlib_test.go
+++ b/imports/wasi_snapshot_preview1/wasi_stdlib_test.go
@@ -42,12 +42,13 @@ func Test_fdReaddir_ls(t *testing.T) {
 		toolchain := toolchain
 		bin := bin
 		t.Run(toolchain, func(t *testing.T) {
-			testFdReaddirLs(t, bin)
+			expectDots := toolchain == "zig-cc"
+			testFdReaddirLs(t, bin, expectDots)
 		})
 	}
 }
 
-func testFdReaddirLs(t *testing.T, bin []byte) {
+func testFdReaddirLs(t *testing.T, bin []byte, expectDots bool) {
 	// TODO: make a subfs
 	moduleConfig := wazero.NewModuleConfig().
 		WithFS(fstest.MapFS{
@@ -59,7 +60,7 @@ func testFdReaddirLs(t *testing.T, bin []byte) {
 	t.Run("empty directory", func(t *testing.T) {
 		console := compileAndRun(t, moduleConfig.WithArgs("wasi", "ls", "./a-"), bin)
 
-		require.Zero(t, console)
+		requireLsOut(t, "\n", expectDots, console)
 	})
 
 	t.Run("not a directory", func(t *testing.T) {
@@ -72,17 +73,30 @@ ENOTDIR
 
 	t.Run("directory with entries", func(t *testing.T) {
 		console := compileAndRun(t, moduleConfig.WithArgs("wasi", "ls", "."), bin)
-
-		require.Equal(t, `
+		requireLsOut(t, `
 ./-
 ./a-
 ./ab-
-`, "\n"+console)
+`, expectDots, console)
 	})
 
 	t.Run("directory with entries - read twice", func(t *testing.T) {
 		console := compileAndRun(t, moduleConfig.WithArgs("wasi", "ls", ".", "repeat"), bin)
-		require.Equal(t, `
+		if expectDots {
+			require.Equal(t, `
+./.
+./..
+./-
+./a-
+./ab-
+./.
+./..
+./-
+./a-
+./ab-
+`, "\n"+console)
+		} else {
+			require.Equal(t, `
 ./-
 ./a-
 ./ab-
@@ -90,6 +104,7 @@ ENOTDIR
 ./a-
 ./ab-
 `, "\n"+console)
+		}
 	})
 
 	t.Run("directory with tons of entries", func(t *testing.T) {
@@ -102,10 +117,25 @@ ENOTDIR
 		console := compileAndRun(t, config, bin)
 
 		lines := strings.Split(console, "\n")
-		require.Equal(t, count+1 /* trailing newline */, len(lines))
+		expected := count + 1 /* trailing newline */
+		if expectDots {
+			expected += 2
+		}
+		require.Equal(t, expected, len(lines))
 	})
 }
 
+func requireLsOut(t *testing.T, expected string, expectDots bool, console string) {
+	dots := `
+./.
+./..
+`
+	if expectDots {
+		expected = dots + expected[1:]
+	}
+	require.Equal(t, expected, "\n"+console)
+}
+
 func Test_fdReaddir_stat(t *testing.T) {
 	for toolchain, bin := range map[string][]byte{
 		"cargo-wasi": wasmCargoWasi,