Repository: spf13/afero
Pull request #266 (2020-09-13 17:03:08)

Add tarfs implementation (reopen #265)
Hi,

Heavily inspired by the already existing zipfs implementation, I have implemented the "tar" version. It is a read-only, in-memory representation of the contents of a tarball. All feedback is welcome :)

This was originally opened as #265, but after rebase+force-push, it seems you can't reopen a PR in GH.

Thanks!


Patch:
diff --git a/.travis.yml b/.travis.yml
index fdaa999..1459644 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -19,4 +19,4 @@ script:
   - go build -v ./...
   - go test -count=1 -cover -race -v ./...
   - go vet ./...
-  - FILES=$(gofmt -s -l . zipfs sftpfs mem); if [[ -n "${FILES}" ]]; then  echo "You have go format errors; gofmt your changes"; exit 1; fi
+  - FILES=$(gofmt -s -l . zipfs sftpfs mem tarfs); if [[ -n "${FILES}" ]]; then  echo "You have go format errors; gofmt your changes"; exit 1; fi
diff --git a/README.md b/README.md
index acd4930..c3e807a 100644
--- a/README.md
+++ b/README.md
@@ -380,7 +380,6 @@ The following is a short list of possible backends we hope someone will
 implement:
 
 * SSH
-* TAR
 * S3
 
 # About the project
diff --git a/memmap.go b/memmap.go
index bbcc238..0fa9592 100644
--- a/memmap.go
+++ b/memmap.go
@@ -317,6 +317,11 @@ func (m *MemMapFs) Rename(oldname, newname string) error {
 	return nil
 }
 
+func (m *MemMapFs) LstatIfPossible(name string) (os.FileInfo, bool, error) {
+	fileInfo, err := m.Stat(name)
+	return fileInfo, false, err
+}
+
 func (m *MemMapFs) Stat(name string) (os.FileInfo, error) {
 	f, err := m.Open(name)
 	if err != nil {
diff --git a/tarfs/file.go b/tarfs/file.go
new file mode 100644
index 0000000..e1d63ed
--- /dev/null
+++ b/tarfs/file.go
@@ -0,0 +1,144 @@
+package tarfs
+
+import (
+	"archive/tar"
+	"bytes"
+	"os"
+	"path/filepath"
+	"sort"
+	"syscall"
+
+	"github.com/spf13/afero"
+)
+
+type File struct {
+	h      *tar.Header
+	data   *bytes.Reader
+	closed bool
+	fs     *Fs
+}
+
+func (f *File) Close() error {
+	if f.closed {
+		return afero.ErrFileClosed
+	}
+
+	f.closed = true
+	f.h = nil
+	f.data = nil
+	f.fs = nil
+
+	return nil
+}
+
+func (f *File) Read(p []byte) (n int, err error) {
+	if f.closed {
+		return 0, afero.ErrFileClosed
+	}
+
+	if f.h.Typeflag == tar.TypeDir {
+		return 0, syscall.EISDIR
+	}
+
+	return f.data.Read(p)
+}
+
+func (f *File) ReadAt(p []byte, off int64) (n int, err error) {
+	if f.closed {
+		return 0, afero.ErrFileClosed
+	}
+
+	if f.h.Typeflag == tar.TypeDir {
+		return 0, syscall.EISDIR
+	}
+
+	return f.data.ReadAt(p, off)
+}
+
+func (f *File) Seek(offset int64, whence int) (int64, error) {
+	if f.closed {
+		return 0, afero.ErrFileClosed
+	}
+
+	if f.h.Typeflag == tar.TypeDir {
+		return 0, syscall.EISDIR
+	}
+
+	return f.data.Seek(offset, whence)
+}
+
+func (f *File) Write(p []byte) (n int, err error) { return 0, syscall.EROFS }
+
+func (f *File) WriteAt(p []byte, off int64) (n int, err error) { return 0, syscall.EROFS }
+
+func (f *File) Name() string {
+	return filepath.Join(splitpath(f.h.Name))
+}
+
+func (f *File) getDirectoryNames() ([]string, error) {
+	d, ok := f.fs.files[f.Name()]
+	if !ok {
+		return nil, &os.PathError{Op: "readdir", Path: f.Name(), Err: syscall.ENOENT}
+	}
+
+	var names []string
+	for n := range d {
+		names = append(names, n)
+	}
+	sort.Strings(names)
+
+	return names, nil
+}
+
+func (f *File) Readdir(count int) ([]os.FileInfo, error) {
+	if f.closed {
+		return nil, afero.ErrFileClosed
+	}
+
+	if !f.h.FileInfo().IsDir() {
+		return nil, syscall.ENOTDIR
+	}
+
+	names, err := f.getDirectoryNames()
+	if err != nil {
+		return nil, err
+	}
+
+	d := f.fs.files[f.Name()]
+	var fi []os.FileInfo
+	for _, n := range names {
+		if n == "" {
+			continue
+		}
+
+		f := d[n]
+		fi = append(fi, f.h.FileInfo())
+		if count > 0 && len(fi) >= count {
+			break
+		}
+	}
+
+	return fi, nil
+}
+
+func (f *File) Readdirnames(n int) ([]string, error) {
+	fi, err := f.Readdir(n)
+	if err != nil {
+		return nil, err
+	}
+
+	var names []string
+	for _, f := range fi {
+		names = append(names, f.Name())
+	}
+
+	return names, nil
+}
+
+func (f *File) Stat() (os.FileInfo, error) { return f.h.FileInfo(), nil }
+
+func (f *File) Sync() error { return nil }
+
+func (f *File) Truncate(size int64) error { return syscall.EROFS }
+
+func (f *File) WriteString(s string) (ret int, err error) { return 0, syscall.EROFS }
diff --git a/tarfs/fs.go b/tarfs/fs.go
new file mode 100644
index 0000000..ba3212b
--- /dev/null
+++ b/tarfs/fs.go
@@ -0,0 +1,134 @@
+// package tarfs implements a read-only in-memory representation of a tar archive
+package tarfs
+
+import (
+	"archive/tar"
+	"bytes"
+	"io"
+	"os"
+	"path/filepath"
+	"syscall"
+	"time"
+
+	"github.com/spf13/afero"
+)
+
+type Fs struct {
+	files map[string]map[string]*File
+}
+
+func splitpath(name string) (dir, file string) {
+	name = filepath.ToSlash(name)
+	if len(name) == 0 || name[0] != '/' {
+		name = "/" + name
+	}
+	name = filepath.Clean(name)
+	dir, file = filepath.Split(name)
+	dir = filepath.Clean(dir)
+	return
+}
+
+func New(t *tar.Reader) *Fs {
+	fs := &Fs{files: make(map[string]map[string]*File)}
+	for {
+		hdr, err := t.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return nil
+		}
+
+		d, f := splitpath(hdr.Name)
+		if _, ok := fs.files[d]; !ok {
+			fs.files[d] = make(map[string]*File)
+		}
+
+		var buf bytes.Buffer
+		size, err := buf.ReadFrom(t)
+		if err != nil {
+			panic("tarfs: reading from tar:" + err.Error())
+		}
+
+		if size != hdr.Size {
+			panic("tarfs: size mismatch")
+		}
+
+		file := &File{
+			h:    hdr,
+			data: bytes.NewReader(buf.Bytes()),
+			fs:   fs,
+		}
+		fs.files[d][f] = file
+
+	}
+
+	// Add a pseudoroot
+	fs.files[afero.FilePathSeparator][""] = &File{
+		h: &tar.Header{
+			Name:     afero.FilePathSeparator,
+			Typeflag: tar.TypeDir,
+			Size:     0,
+		},
+		data: bytes.NewReader(nil),
+		fs:   fs,
+	}
+
+	return fs
+}
+
+func (fs *Fs) Open(name string) (afero.File, error) {
+	d, f := splitpath(name)
+	if _, ok := fs.files[d]; !ok {
+		return nil, &os.PathError{Op: "open", Path: name, Err: syscall.ENOENT}
+	}
+
+	file, ok := fs.files[d][f]
+	if !ok {
+		return nil, &os.PathError{Op: "open", Path: name, Err: syscall.ENOENT}
+	}
+
+	nf := *file
+
+	return &nf, nil
+}
+
+func (fs *Fs) Name() string { return "tarfs" }
+
+func (fs *Fs) Create(name string) (afero.File, error) { return nil, syscall.EROFS }
+
+func (fs *Fs) Mkdir(name string, perm os.FileMode) error { return syscall.EROFS }
+
+func (fs *Fs) MkdirAll(path string, perm os.FileMode) error { return syscall.EROFS }
+
+func (fs *Fs) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) {
+	if flag != os.O_RDONLY {
+		return nil, &os.PathError{Op: "open", Path: name, Err: syscall.EPERM}
+	}
+
+	return fs.Open(name)
+}
+
+func (fs *Fs) Remove(name string) error { return syscall.EROFS }
+
+func (fs *Fs) RemoveAll(path string) error { return syscall.EROFS }
+
+func (fs *Fs) Rename(oldname string, newname string) error { return syscall.EROFS }
+
+func (fs *Fs) Stat(name string) (os.FileInfo, error) {
+	d, f := splitpath(name)
+	if _, ok := fs.files[d]; !ok {
+		return nil, &os.PathError{Op: "stat", Path: name, Err: syscall.ENOENT}
+	}
+
+	file, ok := fs.files[d][f]
+	if !ok {
+		return nil, &os.PathError{Op: "stat", Path: name, Err: syscall.ENOENT}
+	}
+
+	return file.h.FileInfo(), nil
+}
+
+func (fs *Fs) Chmod(name string, mode os.FileMode) error { return syscall.EROFS }
+
+func (fs *Fs) Chtimes(name string, atime time.Time, mtime time.Time) error { return syscall.EROFS }


Test patch:
diff --git a/memmap_test.go b/memmap_test.go
index 6ce742c..476909d 100644
--- a/memmap_test.go
+++ b/memmap_test.go
@@ -650,3 +650,31 @@ func TestMemFsOpenFileModeIllegal(t *testing.T) {
 		t.Fatalf("should not be able to use OpenFile to set illegal mode: %s", info.Mode().String())
 	}
 }
+
+// LstatIfPossible should always return false, since MemMapFs does not
+// support symlinks.
+func TestMemFsLstatIfPossible(t *testing.T) {
+	t.Parallel()
+
+	fs := NewMemMapFs()
+
+	// We assert that fs implements Lstater
+	fsAsserted, ok := fs.(Lstater)
+	if !ok {
+		t.Fatalf("The filesytem does not implement Lstater")
+	}
+
+	file, err := fs.OpenFile("/a.txt", os.O_CREATE, 0o644)
+	if err != nil {
+		t.Fatalf("Error when opening file: %v", err)
+	}
+	defer file.Close()
+
+	_, lstatCalled, err := fsAsserted.LstatIfPossible("/a.txt")
+	if err != nil {
+		t.Fatalf("Function returned err: %v", err)
+	}
+	if lstatCalled {
+		t.Fatalf("Function indicated lstat was called. This should never be true.")
+	}
+}
diff --git a/tarfs/tarfs_test.go b/tarfs/tarfs_test.go
new file mode 100644
index 0000000..8d5eaab
--- /dev/null
+++ b/tarfs/tarfs_test.go
@@ -0,0 +1,402 @@
+// Most of the tests are stolen from the zipfs implementation
+package tarfs
+
+import (
+	"archive/tar"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"reflect"
+	"syscall"
+	"testing"
+
+	"github.com/spf13/afero"
+)
+
+var files = []struct {
+	name        string
+	exists      bool
+	isdir       bool
+	size        int64
+	content     string
+	contentAt4k string
+}{
+	{"/", true, true, 0, "", ""},
+	{"/sub", true, true, 0, "", ""},
+	{"/sub/testDir2", true, true, 0, "", ""},
+	{"/sub/testDir2/testFile", true, false, 8192, "cccccccc", "ccccdddd"},
+	{"/testFile", true, false, 8192, "aaaaaaaa", "aaaabbbb"},
+	{"/testDir1/testFile", true, false, 8192, "bbbbbbbb", "bbbbcccc"},
+
+	{"/nonExisting", false, false, 0, "", ""},
+}
+
+var dirs = []struct {
+	name     string
+	children []string
+}{
+	{"/", []string{"sub", "testDir1", "testFile"}},
+	{"/sub", []string{"testDir2"}},
+	{"/sub/testDir2", []string{"testFile"}},
+	{"/testDir1", []string{"testFile"}},
+}
+
+var afs *afero.Afero
+
+func TestMain(m *testing.M) {
+	tf, err := os.Open("testdata/t.tar")
+	if err != nil {
+		fmt.Print(err)
+		os.Exit(1)
+	}
+
+	tfs := New(tar.NewReader(tf))
+	afs = &afero.Afero{Fs: tfs}
+	os.Exit(m.Run())
+}
+
+func TestFsOpen(t *testing.T) {
+	for _, f := range files {
+		file, err := afs.Open(f.name)
+		if (err == nil) != f.exists {
+			t.Errorf("%v exists = %v, but got err = %v", f.name, f.exists, err)
+		}
+
+		if !f.exists {
+			continue
+		}
+		if err != nil {
+			t.Fatalf("%v: %v", f.name, err)
+		}
+
+		if file.Name() != filepath.FromSlash(f.name) {
+			t.Errorf("Name(), got %v, expected %v", file.Name(), filepath.FromSlash(f.name))
+		}
+
+		s, err := file.Stat()
+		if err != nil {
+			t.Fatalf("stat %v: got error '%v'", file.Name(), err)
+		}
+
+		if isdir := s.IsDir(); isdir != f.isdir {
+			t.Errorf("%v directory, got: %v, expected: %v", file.Name(), isdir, f.isdir)
+		}
+
+		if size := s.Size(); size != f.size {
+			t.Errorf("%v size, got: %v, expected: %v", file.Name(), size, f.size)
+		}
+	}
+}
+
+func TestRead(t *testing.T) {
+	for _, f := range files {
+		if !f.exists {
+			continue
+		}
+
+		file, err := afs.Open(f.name)
+		if err != nil {
+			t.Fatalf("opening %v: %v", f.name, err)
+		}
+
+		buf := make([]byte, 8)
+		n, err := file.Read(buf)
+		if err != nil {
+			if f.isdir && (err != syscall.EISDIR) {
+				t.Errorf("%v got error %v, expected EISDIR", f.name, err)
+			} else if !f.isdir {
+				t.Errorf("%v: %v", f.name, err)
+			}
+		} else if n != 8 {
+			t.Errorf("%v: got %d read bytes, expected 8", f.name, n)
+		} else if string(buf) != f.content {
+			t.Errorf("%v: got <%s>, expected <%s>", f.name, f.content, string(buf))
+		}
+
+	}
+}
+
+func TestReadAt(t *testing.T) {
+	for _, f := range files {
+		if !f.exists {
+			continue
+		}
+
+		file, err := afs.Open(f.name)
+		if err != nil {
+			t.Fatalf("opening %v: %v", f.name, err)
+		}
+
+		buf := make([]byte, 8)
+		n, err := file.ReadAt(buf, 4092)
+		if err != nil {
+			if f.isdir && (err != syscall.EISDIR) {
+				t.Errorf("%v got error %v, expected EISDIR", f.name, err)
+			} else if !f.isdir {
+				t.Errorf("%v: %v", f.name, err)
+			}
+		} else if n != 8 {
+			t.Errorf("%v: got %d read bytes, expected 8", f.name, n)
+		} else if string(buf) != f.contentAt4k {
+			t.Errorf("%v: got <%s>, expected <%s>", f.name, f.contentAt4k, string(buf))
+		}
+
+	}
+}
+
+func TestSeek(t *testing.T) {
+	for _, f := range files {
+		if !f.exists {
+			continue
+		}
+
+		file, err := afs.Open(f.name)
+		if err != nil {
+			t.Fatalf("opening %v: %v", f.name, err)
+		}
+
+		var tests = []struct {
+			offin  int64
+			whence int
+			offout int64
+		}{
+			{0, io.SeekStart, 0},
+			{10, io.SeekStart, 10},
+			{1, io.SeekCurrent, 11},
+			{10, io.SeekCurrent, 21},
+			{0, io.SeekEnd, f.size},
+			{-1, io.SeekEnd, f.size - 1},
+		}
+
+		for _, s := range tests {
+			n, err := file.Seek(s.offin, s.whence)
+			if err != nil {
+				if f.isdir && err == syscall.EISDIR {
+					continue
+				}
+
+				t.Errorf("%v: %v", f.name, err)
+			}
+
+			if n != s.offout {
+				t.Errorf("%v: (off: %v, whence: %v): got %v, expected %v", f.name, s.offin, s.whence, n, s.offout)
+			}
+		}
+
+	}
+}
+
+func TestName(t *testing.T) {
+	for _, f := range files {
+		if !f.exists {
+			continue
+		}
+
+		file, err := afs.Open(f.name)
+		if err != nil {
+			t.Fatalf("opening %v: %v", f.name, err)
+		}
+
+		n := file.Name()
+		if n != filepath.FromSlash(f.name) {
+			t.Errorf("got: %v, expected: %v", n, filepath.FromSlash(f.name))
+		}
+
+	}
+}
+
+func TestClose(t *testing.T) {
+	for _, f := range files {
+		if !f.exists {
+			continue
+		}
+
+		file, err := afs.Open(f.name)
+		if err != nil {
+			t.Fatalf("opening %v: %v", f.name, err)
+		}
+
+		err = file.Close()
+		if err != nil {
+			t.Errorf("%v: %v", f.name, err)
+		}
+
+		err = file.Close()
+		if err == nil {
+			t.Errorf("%v: closing twice should return an error", f.name)
+		}
+
+		buf := make([]byte, 8)
+		n, err := file.Read(buf)
+		if n != 0 || err == nil {
+			t.Errorf("%v: could read from a closed file", f.name)
+		}
+
+		n, err = file.ReadAt(buf, 256)
+		if n != 0 || err == nil {
+			t.Errorf("%v: could readAt from a closed file", f.name)
+		}
+
+		off, err := file.Seek(0, io.SeekStart)
+		if off != 0 || err == nil {
+			t.Errorf("%v: could seek from a closed file", f.name)
+		}
+	}
+}
+
+func TestOpenFile(t *testing.T) {
+	for _, f := range files {
+		file, err := afs.OpenFile(f.name, os.O_RDONLY, 0400)
+		if !f.exists {
+			if !errors.Is(err, syscall.ENOENT) {
+				t.Errorf("%v: got %v, expected%v", f.name, err, syscall.ENOENT)
+			}
+
+			continue
+		}
+
+		if err != nil {
+			t.Fatalf("%v: %v", f.name, err)
+		}
+		file.Close()
+
+		file, err = afs.OpenFile(f.name, os.O_CREATE, 0600)
+		if !errors.Is(err, syscall.EPERM) {
+			t.Errorf("%v: open for write: got %v, expected %v", f.name, err, syscall.EPERM)
+		}
+
+	}
+}
+
+func TestFsStat(t *testing.T) {
+	for _, f := range files {
+		fi, err := afs.Stat(f.name)
+		if !f.exists {
+			if !errors.Is(err, syscall.ENOENT) {
+				t.Errorf("%v: got %v, expected%v", f.name, err, syscall.ENOENT)
+			}
+
+			continue
+		}
+
+		if err != nil {
+			t.Fatalf("stat %v: got error '%v'", f.name, err)
+		}
+
+		if isdir := fi.IsDir(); isdir != f.isdir {
+			t.Errorf("%v directory, got: %v, expected: %v", f.name, isdir, f.isdir)
+		}
+
+		if size := fi.Size(); size != f.size {
+			t.Errorf("%v size, got: %v, expected: %v", f.name, size, f.size)
+		}
+	}
+}
+
+func TestReaddir(t *testing.T) {
+	for _, d := range dirs {
+		dir, err := afs.Open(d.name)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		fi, err := dir.Readdir(0)
+		if err != nil {
+			t.Fatal(err)
+		}
+		var names []string
+		for _, f := range fi {
+			names = append(names, f.Name())
+		}
+
+		if !reflect.DeepEqual(names, d.children) {
+			t.Errorf("%v: children, got '%v', expected '%v'", d.name, names, d.children)
+		}
+
+		fi, err = dir.Readdir(1)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		names = []string{}
+		for _, f := range fi {
+			names = append(names, f.Name())
+		}
+
+		if !reflect.DeepEqual(names, d.children[0:1]) {
+			t.Errorf("%v: children, got '%v', expected '%v'", d.name, names, d.children[0:1])
+		}
+	}
+
+	dir, err := afs.Open("/testFile")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = dir.Readdir(-1)
+	if err != syscall.ENOTDIR {
+		t.Fatal("Expected error")
+	}
+}
+
+func TestReaddirnames(t *testing.T) {
+	for _, d := range dirs {
+		dir, err := afs.Open(d.name)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		names, err := dir.Readdirnames(0)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if !reflect.DeepEqual(names, d.children) {
+			t.Errorf("%v: children, got '%v', expected '%v'", d.name, names, d.children)
+		}
+
+		names, err = dir.Readdirnames(1)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if !reflect.DeepEqual(names, d.children[0:1]) {
+			t.Errorf("%v: children, got '%v', expected '%v'", d.name, names, d.children[0:1])
+		}
+	}
+
+	dir, err := afs.Open("/testFile")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = dir.Readdir(-1)
+	if err != syscall.ENOTDIR {
+		t.Fatal("Expected error")
+	}
+}
+
+func TestGlob(t *testing.T) {
+	for _, s := range []struct {
+		glob    string
+		entries []string
+	}{
+		{filepath.FromSlash("/*"), []string{filepath.FromSlash("/sub"), filepath.FromSlash("/testDir1"), filepath.FromSlash("/testFile")}},
+		{filepath.FromSlash("*"), []string{filepath.FromSlash("sub"), filepath.FromSlash("testDir1"), filepath.FromSlash("testFile")}},
+		{filepath.FromSlash("sub/*"), []string{filepath.FromSlash("sub/testDir2")}},
+		{filepath.FromSlash("sub/testDir2/*"), []string{filepath.FromSlash("sub/testDir2/testFile")}},
+		{filepath.FromSlash("testDir1/*"), []string{filepath.FromSlash("testDir1/testFile")}},
+	} {
+		entries, err := afero.Glob(afs.Fs, s.glob)
+		if err != nil {
+			t.Error(err)
+		}
+		if reflect.DeepEqual(entries, s.entries) {
+			t.Logf("glob: %s: glob ok", s.glob)
+		} else {
+			t.Errorf("glob: %s: got %#v, expected %#v", s.glob, entries, s.entries)
+		}
+	}
+}