Repository: kopia/kopia
Pull request #47 (2019-06-08 22:06:45)

Deprecate hashcache and replace it with simply walking previous snapshot directory tree
This cleans up the code a lot and removes many ugly hacks.
The performance is pretty reasonable and with separate metadata cache it's likely to stay that way.

coveralls: 
[![Coverage Status](https://coveralls.io/builds/23879490/badge)](https://coveralls.io/builds/23879490)

Coverage increased (+0.7%) to 65.59% when pulling **85817a8cb0fb873293aa65aba24734b6bd5a43c1 on jkowalski:deprecate-hashcache** into **bf311400f481861a4059d7d787abbafaab7a7dbc on kopia:master**.


Patch:
diff --git a/cli/command_snapshot_create.go b/cli/command_snapshot_create.go
index 6a0cb2e8..e7411900 100644
--- a/cli/command_snapshot_create.go
+++ b/cli/command_snapshot_create.go
@@ -29,7 +29,6 @@
 	snapshotCreateCheckpointUploadLimitMB = snapshotCreateCommand.Flag("upload-limit-mb", "Stop the backup process after the specified amount of data (in MB) has been uploaded.").PlaceHolder("MB").Default("0").Int64()
 	snapshotCreateDescription             = snapshotCreateCommand.Flag("description", "Free-form snapshot description.").String()
 	snapshotCreateForceHash               = snapshotCreateCommand.Flag("force-hash", "Force hashing of source files for a given percentage of files [0..100]").Default("0").Int()
-	snapshotCreateHashCacheMinAge         = snapshotCreateCommand.Flag("hash-cache-min-age", "Do not hash-cache files below certain age").Default("10m").Duration()
 	snapshotCreateParallelUploads         = snapshotCreateCommand.Flag("parallel", "Upload N files in parallel").PlaceHolder("N").Default("0").Int()
 )
 
@@ -50,7 +49,6 @@ func runBackupCommand(ctx context.Context, rep *repo.Repository) error {
 	u := snapshotfs.NewUploader(rep)
 	u.MaxUploadBytes = *snapshotCreateCheckpointUploadLimitMB * 1024 * 1024
 	u.ForceHashPercentage = *snapshotCreateForceHash
-	u.HashCacheMinAge = *snapshotCreateHashCacheMinAge
 	u.ParallelUploads = *snapshotCreateParallelUploads
 	onCtrlC(u.Cancel)
 
@@ -89,7 +87,7 @@ func snapshotSingleSource(ctx context.Context, rep *repo.Repository, u *snapshot
 
 	localEntry := mustGetLocalFSEntry(sourceInfo.Path)
 
-	previousManifest, err := findPreviousSnapshotManifest(ctx, rep, sourceInfo, nil)
+	previous, err := findPreviousSnapshotManifest(ctx, rep, sourceInfo, nil)
 	if err != nil {
 		return err
 	}
@@ -99,8 +97,8 @@ func snapshotSingleSource(ctx context.Context, rep *repo.Repository, u *snapshot
 		return err
 	}
 
-	log.Debugf("uploading %v using previous manifest %v", sourceInfo, previousManifest)
-	manifest, err := u.Upload(ctx, localEntry, sourceInfo, previousManifest)
+	log.Infof("uploading %v using %v previous manifests", sourceInfo, len(previous))
+	manifest, err := u.Upload(ctx, localEntry, sourceInfo, previous...)
 	if err != nil {
 		return err
 	}
@@ -113,35 +111,49 @@ func snapshotSingleSource(ctx context.Context, rep *repo.Repository, u *snapshot
 	}
 
 	printStderr("uploaded snapshot %v (root %v) in %v\n", snapID, manifest.RootObjectID(), time.Since(t0))
-	log.Debugf("Hash Cache: %v", manifest.HashCacheID.String())
 
 	_, err = policy.ApplyRetentionPolicy(ctx, rep, sourceInfo, true)
 	return err
 }
 
-func findPreviousSnapshotManifest(ctx context.Context, rep *repo.Repository, sourceInfo snapshot.SourceInfo, noLaterThan *time.Time) (*snapshot.Manifest, error) {
-	previous, err := snapshot.ListSnapshots(ctx, rep, sourceInfo)
+// findPreviousSnapshotManifest returns the list of previous snapshots for a given source, including
+// last complete snapshot and possibly some number of incomplete snapshots following it.
+func findPreviousSnapshotManifest(ctx context.Context, rep *repo.Repository, sourceInfo snapshot.SourceInfo, noLaterThan *time.Time) ([]*snapshot.Manifest, error) {
+	man, err := snapshot.ListSnapshots(ctx, rep, sourceInfo)
 	if err != nil {
-		return nil, errors.Wrap(err, "error listing previous backups")
+		return nil, errors.Wrap(err, "error listing previous snapshots")
 	}
 
-	var previousManifest *snapshot.Manifest
-	for _, p := range previous {
+	// phase 1 - find latest complete snapshot.
+	var previousComplete *snapshot.Manifest
+	var previousCompleteStartTime time.Time
+	var result []*snapshot.Manifest
+
+	for _, p := range man {
 		if noLaterThan != nil && p.StartTime.After(*noLaterThan) {
 			continue
 		}
-		if previousManifest == nil || p.StartTime.After(previousManifest.StartTime) {
-			previousManifest = p
+		if p.IncompleteReason == "" && (previousComplete == nil || p.StartTime.After(previousComplete.StartTime)) {
+			previousComplete = p
+			previousCompleteStartTime = p.StartTime
 		}
 	}
 
-	if previousManifest != nil {
-		log.Debugf("found previous manifest for %v with start time %v", sourceInfo, previousManifest.StartTime)
-	} else {
-		log.Debugf("no previous manifest for %v", sourceInfo)
+	if previousComplete != nil {
+		result = append(result, previousComplete)
+	}
+
+	// add all incomplete snapshots after that
+	for _, p := range man {
+		if noLaterThan != nil && p.StartTime.After(*noLaterThan) {
+			continue
+		}
+		if p.IncompleteReason != "" && p.StartTime.After(previousCompleteStartTime) {
+			result = append(result, p)
+		}
 	}
 
-	return previousManifest, nil
+	return result, nil
 }
 
 func getLocalBackupPaths(ctx context.Context, rep *repo.Repository) ([]string, error) {
diff --git a/cli/command_snapshot_list.go b/cli/command_snapshot_list.go
index 6b05352a..af331bd0 100644
--- a/cli/command_snapshot_list.go
+++ b/cli/command_snapshot_list.go
@@ -26,7 +26,6 @@
 	snapshotListShowHumanReadable    = snapshotListCommand.Flag("human-readable", "Show human-readable units").Default("true").Bool()
 	snapshotListShowDelta            = snapshotListCommand.Flag("delta", "Include deltas.").Short('d').Bool()
 	snapshotListShowItemID           = snapshotListCommand.Flag("manifest-id", "Include manifest item ID.").Short('m').Bool()
-	snapshotListShowHashCache        = snapshotListCommand.Flag("hashcache", "Include hashcache object ID.").Bool()
 	snapshotListShowRetentionReasons = snapshotListCommand.Flag("retention", "Include retention reasons.").Default("true").Bool()
 	snapshotListShowModTime          = snapshotListCommand.Flag("mtime", "Include file mod time").Bool()
 	shapshotListShowOwner            = snapshotListCommand.Flag("owner", "Include owner").Bool()
@@ -205,9 +204,6 @@ func outputManifestFromSingleSource(ctx context.Context, rep *repo.Repository, m
 		if *snapshotListShowItemID {
 			bits = append(bits, "manifest:"+string(m.ID))
 		}
-		if *snapshotListShowHashCache {
-			bits = append(bits, "hashcache:"+m.HashCacheID.String())
-		}
 
 		if *snapshotListShowDelta {
 			bits = append(bits, deltaBytes(ent.Size()-lastTotalFileSize))
diff --git a/cli/command_snapshot_migrate.go b/cli/command_snapshot_migrate.go
index 094a10a8..58224d73 100644
--- a/cli/command_snapshot_migrate.go
+++ b/cli/command_snapshot_migrate.go
@@ -152,12 +152,12 @@ func migrateSingleSourceSnapshot(ctx context.Context, uploader *snapshotfs.Uploa
 	}
 
 	log.Infof("migrating snapshot of %v at %v", s, formatTimestamp(m.StartTime))
-	previousManifest, err := findPreviousSnapshotManifest(ctx, destRepo, m.Source, &m.StartTime)
+	previous, err := findPreviousSnapshotManifest(ctx, destRepo, m.Source, &m.StartTime)
 	if err != nil {
 		return err
 	}
 
-	newm, err := uploader.Upload(ctx, sourceEntry, m.Source, previousManifest)
+	newm, err := uploader.Upload(ctx, sourceEntry, m.Source, previous...)
 	if err != nil {
 		return errors.Wrapf(err, "error migrating shapshot %v @ %v", m.Source, m.StartTime)
 	}
diff --git a/fs/localfs/local_fs.go b/fs/localfs/local_fs.go
index acd17106..14e7d423 100644
--- a/fs/localfs/local_fs.go
+++ b/fs/localfs/local_fs.go
@@ -100,7 +100,7 @@ func (f *fileWithMetadata) Entry() (fs.Entry, error) {
 	if err != nil {
 		return nil, err
 	}
-	return filesystemFile{newEntry(fi, f.Name())}, nil
+	return &filesystemFile{newEntry(fi, f.Name())}, nil
 }
 
 func (fsf *filesystemFile) Open(ctx context.Context) (fs.Reader, error) {
diff --git a/internal/hashcache/hashcache.go b/internal/hashcache/hashcache.go
deleted file mode 100644
index a7cc4db4..00000000
--- a/internal/hashcache/hashcache.go
+++ /dev/null
@@ -1,13 +0,0 @@
-// Package hashcache implements streaming cache of file hashes.
-package hashcache
-
-import "github.com/kopia/kopia/repo/object"
-
-var hashCacheStreamType = "kopia:hashcache"
-
-// Entry represents an entry in hash cache database about single file or directory.
-type Entry struct {
-	Name     string    `json:"name,omitempty"`
-	Hash     uint64    `json:"hash,omitempty"`
-	ObjectID object.ID `json:"oid,omitempty"`
-}
diff --git a/internal/hashcache/hashcache_reader.go b/internal/hashcache/hashcache_reader.go
deleted file mode 100644
index eca44e49..00000000
--- a/internal/hashcache/hashcache_reader.go
+++ /dev/null
@@ -1,116 +0,0 @@
-package hashcache
-
-import (
-	"bufio"
-	"io"
-
-	"github.com/kopia/kopia/internal/jsonstream"
-	"github.com/kopia/kopia/internal/kopialogging"
-)
-
-var log = kopialogging.Logger("kopia/hashcache")
-
-// Reader supports reading a stream of hash cache entries.
-type Reader interface {
-	FindEntry(relativeName string) *Entry
-	CopyTo(w Writer) error
-}
-
-type reader struct {
-	reader    *jsonstream.Reader
-	nextEntry *Entry
-	entry0    Entry
-	entry1    Entry
-	odd       bool
-	first     bool
-}
-
-// FindEntry looks for an entry with a given name in hash cache stream and returns it or nil if not found.
-func (hcr *reader) FindEntry(relativeName string) *Entry {
-	for hcr.nextEntry != nil && isLess(hcr.nextEntry.Name, relativeName) {
-		log.Debugf("skipping %v while looking for %v", hcr.nextEntry.Name, relativeName)
-		hcr.readahead()
-	}
-
-	if hcr.nextEntry != nil && relativeName == hcr.nextEntry.Name {
-		e := hcr.nextEntry
-		hcr.nextEntry = nil
-		hcr.readahead()
-		return e
-	}
-
-	if hcr.nextEntry == nil {
-		log.Debugf("end of cache while looking for %v", relativeName)
-		return nil
-	}
-
-	log.Debugf("skipping %v while looking for %v", hcr.nextEntry.Name, relativeName)
-	return nil
-}
-
-func (hcr *reader) CopyTo(w Writer) error {
-	for hcr.nextEntry != nil {
-		if err := w.WriteEntry(*hcr.nextEntry); err != nil {
-			return err
-		}
-		hcr.readahead()
-	}
-
-	return nil
-}
-
-func (hcr *reader) readahead() {
-	if hcr.reader != nil {
-		hcr.nextEntry = nil
-		e := hcr.nextManifestEntry()
-		*e = Entry{}
-		if err := hcr.reader.Read(e); err == nil {
-			hcr.nextEntry = e
-		} else if err != io.EOF {
-			log.Debugf("unable to read next hash cache entry: %v", err)
-		}
-	}
-
-	if hcr.nextEntry == nil {
-		hcr.reader = nil
-	}
-}
-
-func (hcr *reader) nextManifestEntry() *Entry {
-	hcr.odd = !hcr.odd
-	if hcr.odd {
-		return &hcr.entry1
-	}
-
-	return &hcr.entry0
-}
-
-type nullReader struct {
-}
-
-func (*nullReader) FindEntry(relativeName string) *Entry {
-	return nil
-}
-
-func (*nullReader) CopyTo(w Writer) error {
-	return nil
-}
-
-// Open starts reading hash cache content.
-func Open(r io.Reader) Reader {
-	if r == nil {
-		return &nullReader{}
-	}
-
-	jsr, err := jsonstream.NewReader(bufio.NewReader(r), hashCacheStreamType, nil)
-	if err != nil {
-		return &nullReader{}
-	}
-	var hcr reader
-	hcr.reader = jsr
-	hcr.nextEntry = nil
-	hcr.first = true
-	hcr.readahead()
-	log.Debugf("nextEntry: %v", hcr.nextEntry)
-	return &hcr
-}
diff --git a/internal/hashcache/hashcache_writer.go b/internal/hashcache/hashcache_writer.go
deleted file mode 100644
index aeb1e770..00000000
--- a/internal/hashcache/hashcache_writer.go
+++ /dev/null
@@ -1,49 +0,0 @@
-package hashcache
-
-import (
-	"io"
-
-	"github.com/pkg/errors"
-
-	"github.com/kopia/kopia/internal/jsonstream"
-)
-
-// Writer emits hash cache entries.
-type Writer interface {
-	WriteEntry(e Entry) error
-	Finalize() error
-}
-
-type writer struct {
-	writer          *jsonstream.Writer
-	lastNameWritten string
-}
-
-// NewWriter creates new hash cache Writer.
-func NewWriter(w io.Writer) Writer {
-	hcw := &writer{
-		writer: jsonstream.NewWriter(w, hashCacheStreamType),
-	}
-	return hcw
-}
-
-// WriteEntry emits the specified hash cache entry.
-func (hcw *writer) WriteEntry(e Entry) error {
-	if err := e.ObjectID.Validate(); err != nil {
-		panic("invalid object ID: " + err.Error())
-	}
-
-	if hcw.lastNameWritten != "" {
-		if isLessOrEqual(e.Name, hcw.lastNameWritten) {
-			return errors.Errorf("out-of-order directory entry, previous '%v' current '%v'", hcw.lastNameWritten, e.Name)
-		}
-		hcw.lastNameWritten = e.Name
-	}
-
-	return hcw.writer.Write(&e)
-}
-
-// Finalize closes hashcache stream and must be invoked after emitting all entries.
-func (hcw *writer) Finalize() error {
-	return hcw.writer.Finalize()
-}
diff --git a/internal/hashcache/utils.go b/internal/hashcache/utils.go
deleted file mode 100644
index c81cafb0..00000000
--- a/internal/hashcache/utils.go
+++ /dev/null
@@ -1,41 +0,0 @@
-package hashcache
-
-import "strings"
-
-func isLess(name1, name2 string) bool {
-	if name1 == name2 {
-		return false
-	}
-
-	return isLessOrEqual(name1, name2)
-}
-
-func isLessOrEqual(name1, name2 string) bool {
-	parts1 := strings.Split(name1, "/")
-	parts2 := strings.Split(name2, "/")
-
-	i := 0
-	for i < len(parts1) && i < len(parts2) {
-		// an element is a directory if it has another element following it
-		dir1 := i+1 < len(parts1)
-		dir2 := i+1 < len(parts2)
-		if dir1 != dir2 {
-			// directories trump non-directories
-			return dir1
-		}
-
-		if parts1[i] == parts2[i] {
-			i++
-			continue
-		}
-		if parts1[i] == "" {
-			return false
-		}
-		if parts2[i] == "" {
-			return true
-		}
-		return parts1[i] < parts2[i]
-	}
-
-	return len(parts1) <= len(parts2)
-}
diff --git a/internal/server/source_manager.go b/internal/server/source_manager.go
index fd3a8b8b..3b20b946 100644
--- a/internal/server/source_manager.go
+++ b/internal/server/source_manager.go
@@ -173,7 +173,7 @@ func (s *sourceManager) snapshot(ctx context.Context) {
 	u.Progress = s
 
 	log.Infof("starting upload of %v", s.src)
-	manifest, err := u.Upload(ctx, localEntry, s.src, s.lastSnapshot)
+	manifest, err := u.Upload(ctx, localEntry, s.src, s.lastCompleteSnapshot, s.lastSnapshot)
 	if err != nil {
 		log.Errorf("upload error: %v", err)
 		return
diff --git a/repo/open.go b/repo/open.go
index 1e1a49b7..8c69bf76 100644
--- a/repo/open.go
+++ b/repo/open.go
@@ -28,11 +28,8 @@ type Options struct {
 
 // Open opens a Repository specified in the configuration file.
 func Open(ctx context.Context, configFile, password string, options *Options) (rep *Repository, err error) {
-	log.Debugf("opening repository from %v", configFile)
 	defer func() {
-		if err == nil {
-			log.Debugf("opened repository")
-		} else {
+		if err != nil {
 			log.Errorf("failed to open repository: %v", err)
 		}
 	}()
@@ -46,14 +43,11 @@ func Open(ctx context.Context, configFile, password string, options *Options) (r
 		return nil, err
 	}
 
-	log.Debugf("loading config from file: %v", configFile)
 	lc, err := loadConfigFromFile(configFile)
 	if err != nil {
 		return nil, err
 	}
 
-	log.Debugf("opening storage: %v", lc.Storage.Type)
-
 	st, err := blob.NewStorage(ctx, lc.Storage)
 	if err != nil {
 		return nil, errors.Wrap(err, "cannot open storage")
@@ -76,7 +70,6 @@ func Open(ctx context.Context, configFile, password string, options *Options) (r
 
 // OpenWithConfig opens the repository with a given configuration, avoiding the need for a config file.
 func OpenWithConfig(ctx context.Context, st blob.Storage, lc *LocalConfig, password string, options *Options, caching content.CachingOptions) (*Repository, error) {
-	log.Debugf("reading encrypted format blob")
 	// Read format blob, potentially from cache.
 	fb, err := readAndCacheFormatBlobBytes(ctx, st, caching.CacheDirectory)
 	if err != nil {
@@ -110,19 +103,16 @@ func OpenWithConfig(ctx context.Context, st blob.Storage, lc *LocalConfig, passw
 		fo.MaxPackSize = 20 << 20 // 20 MB
 	}
 
-	log.Debugf("initializing content-addressable storage manager")
 	cm, err := content.NewManager(ctx, st, fo, caching, fb)
 	if err != nil {
 		return nil, errors.Wrap(err, "unable to open content manager")
 	}
 
-	log.Debugf("initializing object manager")
 	om, err := object.NewObjectManager(ctx, cm, repoConfig.Format, options.ObjectManagerOptions)
 	if err != nil {
 		return nil, errors.Wrap(err, "unable to open object manager")
 	}
 
-	log.Debugf("initializing manifest manager")
 	manifests, err := manifest.NewManager(ctx, cm)
 	if err != nil {
 		return nil, errors.Wrap(err, "unable to open manifests")
diff --git a/snapshot/manifest.go b/snapshot/manifest.go
index ff866a85..ac9b59e1 100644
--- a/snapshot/manifest.go
+++ b/snapshot/manifest.go
@@ -20,10 +20,8 @@ type Manifest struct {
 	StartTime   time.Time `json:"startTime"`
 	EndTime     time.Time `json:"endTime"`
 
-	HashCacheID         object.ID `json:"hashCache"`
-	HashCacheCutoffTime time.Time `json:"hashCacheCutoff"`
-	Stats               Stats     `json:"stats"`
-	IncompleteReason    string    `json:"incomplete,omitempty"`
+	Stats            Stats  `json:"stats"`
+	IncompleteReason string `json:"incomplete,omitempty"`
 
 	RootEntry *DirEntry `json:"rootEntry"`
 
diff --git a/snapshot/policy/retention_policy.go b/snapshot/policy/retention_policy.go
index f130e168..99517276 100644
--- a/snapshot/policy/retention_policy.go
+++ b/snapshot/policy/retention_policy.go
@@ -46,8 +46,12 @@ func (r *RetentionPolicy) ComputeRetentionReasons(manifests []*snapshot.Manifest
 	for i, s := range sorted {
 		s.RetentionReasons = r.getRetentionReasons(i, s, cutoff, ids, idCounters)
 	}
-	if len(sorted) > 0 && sorted[0].IncompleteReason != "" {
-		sorted[0].RetentionReasons = append(sorted[0].RetentionReasons, "last-incomplete")
+	for _, s := range sorted {
+		if s.IncompleteReason != "" {
+			s.RetentionReasons = append(s.RetentionReasons, "incomplete")
+		} else {
+			break
+		}
 	}
 }
 
diff --git a/snapshot/snapshotfs/repofs.go b/snapshot/snapshotfs/repofs.go
index a3118d1f..de0af201 100644
--- a/snapshot/snapshotfs/repofs.go
+++ b/snapshot/snapshotfs/repofs.go
@@ -3,7 +3,6 @@
 
 import (
 	"context"
-	"fmt"
 	"io/ioutil"
 	"os"
 	"time"
@@ -94,7 +93,10 @@ func (rd *repositoryDirectory) Readdir(ctx context.Context) (fs.Entries, error)
 
 	entries := make(fs.Entries, len(metadata))
 	for i, m := range metadata {
-		entries[i] = newRepoEntry(rd.repo, m)
+		entries[i], err = newRepoEntry(rd.repo, m)
+		if err != nil {
+			return nil, errors.Wrapf(err, "error parsing entry %v", m)
+		}
 	}
 
 	entries.Sort()
@@ -126,7 +128,7 @@ func (rsl *repositorySymlink) Readlink(ctx context.Context) (string, error) {
 	return string(b), nil
 }
 
-func newRepoEntry(r *repo.Repository, md *snapshot.DirEntry) fs.Entry {
+func newRepoEntry(r *repo.Repository, md *snapshot.DirEntry) (fs.Entry, error) {
 	re := repositoryEntry{
 		metadata: md,
 		repo:     r,
@@ -139,16 +141,16 @@ func newRepoEntry(r *repo.Repository, md *snapshot.DirEntry) fs.Entry {
 			md.ModTime = md.DirSummary.MaxModTime
 		}
 
-		return fs.Directory(&repositoryDirectory{re, md.DirSummary})
+		return fs.Directory(&repositoryDirectory{re, md.DirSummary}), nil
 
 	case snapshot.EntryTypeSymlink:
-		return fs.Symlink(&repositorySymlink{re})
+		return fs.Symlink(&repositorySymlink{re}), nil
 
 	case snapshot.EntryTypeFile:
-		return fs.File(&repositoryFile{re})
+		return fs.File(&repositoryFile{re}), nil
 
 	default:
-		panic(fmt.Sprintf("not supported entry metadata type: %v", md.Type))
+		return nil, errors.Errorf("not supported entry metadata type: %q", md.Type)
 	}
 }
 
@@ -168,7 +170,7 @@ func withFileInfo(r object.Reader, e fs.Entry) fs.Reader {
 // DirectoryEntry returns fs.Directory based on repository object with the specified ID.
 // The existence or validity of the directory object is not validated until its contents are read.
 func DirectoryEntry(rep *repo.Repository, objectID object.ID, dirSummary *fs.DirectorySummary) fs.Directory {
-	d := newRepoEntry(rep, &snapshot.DirEntry{
+	d, _ := newRepoEntry(rep, &snapshot.DirEntry{
 		Name:        "/",
 		Permissions: 0555,
 		Type:        snapshot.EntryTypeDirectory,
@@ -186,7 +188,7 @@ func SnapshotRoot(rep *repo.Repository, man *snapshot.Manifest) (fs.Entry, error
 		return nil, errors.New("manifest root object ID")
 	}
 
-	return newRepoEntry(rep, man.RootEntry), nil
+	return newRepoEntry(rep, man.RootEntry)
 }
 
 var _ fs.Directory = &repositoryDirectory{}
diff --git a/snapshot/snapshotfs/source_snapshots.go b/snapshot/snapshotfs/source_snapshots.go
index bfaeaba1..c137cd94 100644
--- a/snapshot/snapshotfs/source_snapshots.go
+++ b/snapshot/snapshotfs/source_snapshots.go
@@ -7,6 +7,8 @@
 	"strings"
 	"time"
 
+	"github.com/pkg/errors"
+
 	"github.com/kopia/kopia/fs"
 	"github.com/kopia/kopia/repo"
 	"github.com/kopia/kopia/snapshot"
@@ -80,7 +82,10 @@ func (s *sourceSnapshots) Readdir(ctx context.Context) (fs.Entries, error) {
 			de.DirSummary = m.RootEntry.DirSummary
 		}
 
-		e := newRepoEntry(s.rep, de)
+		e, err := newRepoEntry(s.rep, de)
+		if err != nil {
+			return nil, errors.Wrap(err, "unable to create entry")
+		}
 
 		result = append(result, e)
 	}
diff --git a/snapshot/snapshotfs/upload.go b/snapshot/snapshotfs/upload.go
index 863e1f00..85c6026e 100644
--- a/snapshot/snapshotfs/upload.go
+++ b/snapshot/snapshotfs/upload.go
@@ -3,10 +3,8 @@
 import (
 	"bytes"
 	"context"
-	"encoding/binary"
 	"hash/fnv"
 	"io"
-	"math/rand"
 	"os"
 	"runtime"
 	"sync"
@@ -17,7 +15,6 @@
 
 	"github.com/kopia/kopia/fs"
 	"github.com/kopia/kopia/fs/ignorefs"
-	"github.com/kopia/kopia/internal/hashcache"
 	"github.com/kopia/kopia/internal/kopialogging"
 	"github.com/kopia/kopia/repo"
 	"github.com/kopia/kopia/repo/object"
@@ -26,21 +23,6 @@
 
 var log = kopialogging.Logger("kopia/upload")
 
-func hashEntryMetadata(w io.Writer, e fs.Entry, fileSize int64) {
-	io.WriteString(w, e.Name())                                  //nolint:errcheck
-	binary.Write(w, binary.LittleEndian, e.ModTime().UnixNano()) //nolint:errcheck
-	binary.Write(w, binary.LittleEndian, e.Mode())               //nolint:errcheck
-	binary.Write(w, binary.LittleEndian, fileSize)               //nolint:errcheck
-	binary.Write(w, binary.LittleEndian, e.Owner().UserID)       //nolint:errcheck
-	binary.Write(w, binary.LittleEndian, e.Owner().GroupID)      //nolint:errcheck
-}
-
-func metadataHash(e fs.Entry, fileSize int64) uint64 {
-	h := fnv.New64a()
-	hashEntryMetadata(h, e, fileSize)
-	return h.Sum64()
-}
-
 var errCancelled = errors.New("canceled")
 
 // Uploader supports efficient uploading files and directories to repository.
@@ -55,25 +37,18 @@ type Uploader struct {
 	// ignore file read errors
 	IgnoreFileErrors bool
 
-	// probability with hich hashcache entries will be ignored, must be [0..100]
-	// 0=always use hash cache if possible
-	// 100=never use hash cache
+	// probability with cached entries will be ignored, must be [0..100]
+	// 0=always use cached object entries if possible
+	// 100=never use cached entries
 	ForceHashPercentage int
 
-	// Do not hash-cache files younger than this age.
-	// Protects from accidentally caching incorrect hashes of files that are being modified.
-	HashCacheMinAge time.Duration
-
 	// Number of files to hash and upload in parallel.
 	ParallelUploads int
 
-	repo        *repo.Repository
-	cacheWriter hashcache.Writer
-	cacheReader hashcache.Reader
+	repo *repo.Repository
 
-	hashCacheCutoff time.Time
-	stats           snapshot.Stats
-	canceled        int32
+	stats    snapshot.Stats
+	canceled int32
 
 	progressMutex          sync.Mutex
 	nextProgressReportTime time.Time
@@ -127,10 +102,13 @@ func (u *Uploader) uploadFileInternal(ctx context.Context, f fs.File) entryResul
 		return entryResult{err: err}
 	}
 
-	de := newDirEntry(fi2, r)
+	de, err := newDirEntry(fi2, r)
+	if err != nil {
+		return entryResult{err: errors.Wrap(err, "unable to create dir entry")}
+	}
 	de.FileSize = written
 
-	return entryResult{de: de, hash: metadataHash(fi2, written)}
+	return entryResult{de: de}
 }
 
 func (u *Uploader) uploadSymlinkInternal(ctx context.Context, f fs.Symlink) entryResult {
@@ -154,9 +132,12 @@ func (u *Uploader) uploadSymlinkInternal(ctx context.Context, f fs.Symlink) entr
 		return entryResult{err: err}
 	}
 
-	de := newDirEntry(f, r)
+	de, err := newDirEntry(f, r)
+	if err != nil {
+		return entryResult{err: errors.Wrap(err, "unable to create dir entry")}
+	}
 	de.FileSize = written
-	return entryResult{de: de, hash: metadataHash(f, written)}
+	return entryResult{de: de}
 }
 
 func (u *Uploader) addDirProgress(length int64) {
@@ -219,18 +200,18 @@ func (u *Uploader) copyWithProgress(dst io.Writer, src io.Reader, completed, len
 	return written, nil
 }
 
-func newDirEntry(md fs.Entry, oid object.ID) *snapshot.DirEntry {
+func newDirEntry(md fs.Entry, oid object.ID) (*snapshot.DirEntry, error) {
 	var entryType snapshot.EntryType
 
-	switch md.Mode() & os.ModeType {
-	case os.ModeDir:
+	switch md := md.(type) {
+	case fs.Directory:
 		entryType = snapshot.EntryTypeDirectory
-	case os.ModeSymlink:
+	case fs.Symlink:
 		entryType = snapshot.EntryTypeSymlink
-	case 0:
+	case fs.File:
 		entryType = snapshot.EntryTypeFile
 	default:
-		entryType = snapshot.EntryTypeUnknown
+		return nil, errors.Errorf("invalid entry type %T", md)
 	}
 
 	return &snapshot.DirEntry{
@@ -242,7 +223,7 @@ func newDirEntry(md fs.Entry, oid object.ID) *snapshot.DirEntry {
 		UserID:      md.Owner().UserID,
 		GroupID:     md.Owner().GroupID,
 		ObjectID:    oid,
-	}
+	}, nil
 }
 
 // uploadFile uploads the specified File to the repository.
@@ -252,7 +233,10 @@ func (u *Uploader) uploadFile(ctx context.Context, file fs.File) (*snapshot.DirE
 		return nil, res.err
 	}
 
-	de := newDirEntry(file, res.de.ObjectID)
+	de, err := newDirEntry(file, res.de.ObjectID)
+	if err != nil {
+		return nil, errors.Wrap(err, "unable to create dir entry")
+	}
 	de.DirSummary = &fs.DirectorySummary{
 		TotalFileCount: 1,
 		TotalFileSize:  res.de.FileSize,
@@ -265,30 +249,19 @@ func (u *Uploader) uploadFile(ctx context.Context, file fs.File) (*snapshot.DirE
 // uploadDir uploads the specified Directory to the repository.
 // An optional ID of a hash-cache object may be provided, in which case the Uploader will use its
 // contents to avoid hashing
-func (u *Uploader) uploadDir(ctx context.Context, rootDir fs.Directory) (*snapshot.DirEntry, object.ID, error) {
-	mw := u.repo.Objects.NewWriter(ctx, object.WriterOptions{
-		Description: "HASHCACHE:" + rootDir.Name(),
-		Prefix:      "h",
-	})
-	defer mw.Close() //nolint:errcheck
-	u.cacheWriter = hashcache.NewWriter(mw)
-	oid, summ, err := uploadDirInternal(ctx, u, rootDir, ".")
-	if u.IsCancelled() {
-		if err2 := u.cacheReader.CopyTo(u.cacheWriter); err != nil {
-			return nil, "", err2
-		}
+func (u *Uploader) uploadDir(ctx context.Context, rootDir fs.Directory, previousDirs []fs.Directory) (*snapshot.DirEntry, error) {
+	oid, summ, err := uploadDirInternal(ctx, u, rootDir, previousDirs, ".")
+	if err != nil {
+		return nil, err
 	}
-	defer u.cacheWriter.Finalize() //nolint:errcheck
-	u.cacheWriter = nil
 
+	de, err := newDirEntry(rootDir, oid)
 	if err != nil {
-		return nil, "", err
+		return nil, errors.Wrap(err, "unable to create dir entry")
 	}
 
-	hcid, err := mw.Result()
-	de := newDirEntry(rootDir, oid)
 	de.DirSummary = &summ
-	return de, hcid, err
+	return de, err
 }
 
 func (u *Uploader) foreachEntryUnlessCancelled(relativePath string, entries fs.Entries, cb func(entry fs.Entry, entryRelativePath string) error) error {
@@ -308,12 +281,11 @@ func (u *Uploader) foreachEntryUnlessCancelled(relativePath string, entries fs.E
 }
 
 type entryResult struct {
-	err  error
-	de   *snapshot.DirEntry
-	hash uint64
+	err error
+	de  *snapshot.DirEntry
 }
 
-func (u *Uploader) processSubdirectories(ctx context.Context, relativePath string, entries fs.Entries, dw *dirWriter, summ *fs.DirectorySummary) error {
+func (u *Uploader) processSubdirectories(ctx context.Context, relativePath string, entries fs.Entries, previousEntries []fs.Entries, dw *dirWriter, summ *fs.DirectorySummary) error {
 	return u.foreachEntryUnlessCancelled(relativePath, entries, func(entry fs.Entry, entryRelativePath string) error {
 		dir, ok := entry.(fs.Directory)
 		if !ok {
@@ -321,7 +293,16 @@ func (u *Uploader) processSubdirectories(ctx context.Context, relativePath strin
 			return nil
 		}
 
-		oid, subdirsumm, err := uploadDirInternal(ctx, u, dir, entryRelativePath)
+		var previousDirs []fs.Directory
+		for _, e := range previousEntries {
+			if d, _ := e.FindByName(entry.Name()).(fs.Directory); d != nil {
+				previousDirs = append(previousDirs, d)
+			}
+		}
+
+		previousDirs = uniqueDirectories(previousDirs)
+
+		oid, subdirsumm, err := uploadDirInternal(ctx, u, dir, previousDirs, entryRelativePath)
 		if err == errCancelled {
 			return err
 		}
@@ -337,7 +318,11 @@ func (u *Uploader) processSubdirectories(ctx context.Context, relativePath strin
 			return errors.Errorf("unable to process directory %q: %s", entry.Name(), err)
 		}
 
-		de := newDirEntry(dir, oid)
+		de, err := newDirEntry(dir, oid)
+		if err != nil {
+			return errors.Wrap(err, "unable to create dir entry")
+		}
+
 		de.DirSummary = &subdirsumm
 		if err := dw.WriteEntry(de); err != nil {
 			return errors.Wrap(err, "unable to write dir entry")
@@ -373,7 +358,52 @@ type uploadWorkItem struct {
 	resultChan        chan entryResult
 }
 
-func (u *Uploader) prepareWorkItems(ctx context.Context, dirRelativePath string, entries fs.Entries, summ *fs.DirectorySummary) ([]*uploadWorkItem, error) {
+func metadataEquals(e1, e2 fs.Entry) bool {
+	if l, r := e1.ModTime(), e2.ModTime(); l != r {
+		return false
+	}
+	if l, r := e1.Mode(), e2.Mode(); l != r {
+		return false
+	}
+	if l, r := e1.Size(), e2.Size(); l != r {
+		return false
+	}
+	if l, r := e1.Owner(), e2.Owner(); l != r {
+		return false
+	}
+	return true
+}
+
+func findCachedEntry(entry fs.Entry, prevEntries []fs.Entries) fs.Entry {
+	for _, e := range prevEntries {
+		if ent := e.FindByName(entry.Name()); ent != nil && metadataEquals(entry, ent) {
+			return ent
+		}
+	}
+
+	return nil
+}
+
+// objectIDPercent arbitrarily maps given object ID onto a number 0.99
+func objectIDPercent(obj object.ID) int {
+	h := fnv.New32a()
+	io.WriteString(h, obj.String()) //nolint:errcheck
+	return int(h.Sum32() % 100)
+}
+
+func (u *Uploader) maybeIgnoreCachedEntry(ent fs.Entry) fs.Entry {
+	if h, ok := ent.(object.HasObjectID); ok {
+		if objectIDPercent(h.ObjectID()) < u.ForceHashPercentage {
+			log.Debugf("ignoring valid cached object: %v", h.ObjectID())
+			return nil
+		}
+		return ent
+	}
+
+	return nil
+}
+
+func (u *Uploader) prepareWorkItems(ctx context.Context, dirRelativePath string, entries fs.Entries, prevEntries []fs.Entries, summ *fs.DirectorySummary) ([]*uploadWorkItem, error) {
 	var result []*uploadWorkItem
 
 	resultErr := u.foreachEntryUnlessCancelled(dirRelativePath, entries, func(entry fs.Entry, entryRelativePath string) error {
@@ -383,16 +413,6 @@ func (u *Uploader) prepareWorkItems(ctx context.Context, dirRelativePath string,
 		}
 
 		// regular file
-		// See if we had this name during previous pass.
-		cachedEntry := u.maybeIgnoreHashCacheEntry(u.cacheReader.FindEntry(entryRelativePath))
-		var cachedHash uint64
-		if cachedEntry != nil {
-			cachedHash = cachedEntry.Hash
-		}
-
-		// ... and whether file metadata is identical to the previous one.
-		computedHash := metadataHash(entry, entry.Size())
-
 		if entry, ok := entry.(fs.File); ok {
 			u.stats.TotalFileCount++
 			u.stats.TotalFileSize += entry.Size()
@@ -403,14 +423,15 @@ func (u *Uploader) prepareWorkItems(ctx context.Context, dirRelativePath string,
 			}
 		}
 
-		if cachedHash == computedHash {
+		// See if we had this name during either of previous passes.
+		if cachedEntry := u.maybeIgnoreCachedEntry(findCachedEntry(entry, prevEntries)); cachedEntry != nil {
 			u.stats.CachedFiles++
 			u.addDirProgress(entry.Size())
 
 			// compute entryResult now, cachedEntry is short-lived
-			cachedResult := entryResult{
-				de:   newDirEntry(entry, cachedEntry.ObjectID),
-				hash: cachedEntry.Hash,
+			cachedDirEntry, err := newDirEntry(entry, cachedEntry.(object.HasObjectID).ObjectID())
+			if err != nil {
+				return errors.Wrap(err, "unable to create dir entry")
 			}
 
 			// Avoid hashing by reusing previous object ID.
@@ -418,12 +439,10 @@ func (u *Uploader) prepareWorkItems(ctx context.Context, dirRelativePath string,
 				entry:             entry,
 				entryRelativePath: entryRelativePath,
 				uploadFunc: func() entryResult {
-					return cachedResult
+					return entryResult{de: cachedDirEntry}
 				},
 			})
 		} else {
-			log.Debugf("hash cache miss for %v (cached %v computed %v)", entryRelativePath, cachedHash, computedHash)
-
 			switch entry := entry.(type) {
 			case fs.Symlink:
 				result = append(result, &uploadWorkItem{
@@ -515,16 +534,6 @@ func (u *Uploader) processUploadWorkItems(workItems []*uploadWorkItem, dw *dirWr
 		if err := dw.WriteEntry(result.de); err != nil {
 			return errors.Wrap(err, "unable to write directory entry")
 		}
-
-		if result.hash != 0 && it.entry.ModTime().Before(u.hashCacheCutoff) {
-			if err := u.cacheWriter.WriteEntry(hashcache.Entry{
-				Name:     it.entryRelativePath,
-				Hash:     result.hash,
-				ObjectID: result.de.ObjectID,
-			}); err != nil {
-				return errors.Wrap(err, "unable to write hash cache entry")
-			}
-		}
 	}
 
 	// wait for workers, this is technically not needed, but let's make sure we don't leak goroutines
@@ -533,10 +542,46 @@ func (u *Uploader) processUploadWorkItems(workItems []*uploadWorkItem, dw *dirWr
 	return nil
 }
 
+func maybeReadDirectoryEntries(ctx context.Context, dir fs.Directory) fs.Entries {
+	if dir == nil {
+		return nil
+	}
+
+	ent, err := dir.Readdir(ctx)
+	if err != nil {
+		log.Warningf("unable to read previous directory entries: %v", err)
+		return nil
+	}
+
+	return ent
+}
+
+func uniqueDirectories(dirs []fs.Directory) []fs.Directory {
+	if len(dirs) <= 1 {
+		return dirs
+	}
+
+	var unique = map[object.ID]fs.Directory{}
+	for _, dir := range dirs {
+		unique[dir.(object.HasObjectID).ObjectID()] = dir
+	}
+
+	if len(unique) == len(dirs) {
+		return dirs
+	}
+
+	var result []fs.Directory
+	for _, d := range unique {
+		result = append(result, d)
+	}
+	return result
+}
+
 func uploadDirInternal(
 	ctx context.Context,
 	u *Uploader,
 	directory fs.Directory,
+	previousDirs []fs.Directory,
 	dirRelativePath string,
 ) (object.ID, fs.DirectorySummary, error) {
 	u.stats.TotalDirectoryCount++
@@ -552,6 +597,14 @@ func uploadDirInternal(
 	if direrr != nil {
 		return "", fs.DirectorySummary{}, direrr
 	}
+
+	var prevEntries []fs.Entries
+	for _, d := range uniqueDirectories(previousDirs) {
+		if ent := maybeReadDirectoryEntries(ctx, d); ent != nil {
+			prevEntries = append(prevEntries, ent)
+		}
+
+	}
 	if len(entries) == 0 {
 		summ.MaxModTime = directory.ModTime()
 	}
@@ -564,12 +617,12 @@ func uploadDirInternal(
 	dw := newDirWriter(writer)
 	defer writer.Close() //nolint:errcheck
 
-	if err := u.processSubdirectories(ctx, dirRelativePath, entries, dw, &summ); err != nil && err != errCancelled {
+	if err := u.processSubdirectories(ctx, dirRelativePath, entries, prevEntries, dw, &summ); err != nil && err != errCancelled {
 		return "", fs.DirectorySummary{}, err
 	}
 	u.prepareProgress(dirRelativePath, entries)
 
-	workItems, workItemErr := u.prepareWorkItems(ctx, dirRelativePath, entries, &summ)
+	workItems, workItemErr := u.prepareWorkItems(ctx, dirRelativePath, entries, prevEntries, &summ)
 	if workItemErr != nil && workItemErr != errCancelled {
 		return "", fs.DirectorySummary{}, workItemErr
 	}
@@ -584,20 +637,11 @@ func uploadDirInternal(
 	return oid, summ, err
 }
 
-func (u *Uploader) maybeIgnoreHashCacheEntry(e *hashcache.Entry) *hashcache.Entry {
-	if rand.Intn(100) < u.ForceHashPercentage {
-		return nil
-	}
-
-	return e
-}
-
 // NewUploader creates new Uploader object for a given repository.
 func NewUploader(r *repo.Repository) *Uploader {
 	return &Uploader{
 		repo:             r,
 		Progress:         &nullUploadProgress{},
-		HashCacheMinAge:  1 * time.Hour,
 		IgnoreFileErrors: true,
 		ParallelUploads:  1,
 	}
@@ -608,45 +652,60 @@ func (u *Uploader) Cancel() {
 	atomic.StoreInt32(&u.canceled, 1)
 }
 
+func (u *Uploader) maybeOpenDirectoryFromManifest(man *snapshot.Manifest) fs.Directory {
+	if man == nil {
+		return nil
+	}
+
+	ent, err := newRepoEntry(u.repo, man.RootEntry)
+	if err != nil {
+		log.Warningf("invalid previous manifest root entry %v: %v", man.RootEntry, err)
+		return nil
+	}
+
+	dir, ok := ent.(fs.Directory)
+	if !ok {
+		log.Debugf("previous manifest root is not a directory (was %T %+v)", ent, man.RootEntry)
+		return nil
+	}
+
+	return dir
+}
+
 // Upload uploads contents of the specified filesystem entry (file or directory) to the repository and returns snapshot.Manifest with statistics.
 // Old snapshot manifest, when provided can be used to speed up uploads by utilizing hash cache.
 func (u *Uploader) Upload(
 	ctx context.Context,
 	source fs.Entry,
 	sourceInfo snapshot.SourceInfo,
-	old *snapshot.Manifest,
+	previousManifests ...*snapshot.Manifest,
 ) (*snapshot.Manifest, error) {
+	log.Debugf("Uploading %v", sourceInfo)
 	s := &snapshot.Manifest{
 		Source: sourceInfo,
 	}
 
 	defer u.Progress.UploadFinished()
 
-	u.cacheReader = hashcache.Open(nil)
 	u.stats = snapshot.Stats{}
-	if old != nil {
-		log.Debugf("opening hash cache: %v", old.HashCacheID)
-		if r, err := u.repo.Objects.Open(ctx, old.HashCacheID); err == nil {
-			u.cacheReader = hashcache.Open(r)
-			log.Debugf("opened hash cache: %v", old.HashCacheID)
-		} else {
-			log.Warningf("unable to open hash cache %v: %v", old.HashCacheID, err)
-
-		}
-	}
 
 	var err error
 
 	s.StartTime = time.Now()
-	u.hashCacheCutoff = time.Now().Add(-u.HashCacheMinAge)
-	s.HashCacheCutoffTime = u.hashCacheCutoff
 
 	switch entry := source.(type) {
 	case fs.Directory:
+		var previousDirs []fs.Directory
+		for _, m := range previousManifests {
+			if d := u.maybeOpenDirectoryFromManifest(m); d != nil {
+				previousDirs = append(previousDirs, d)
+			}
+		}
+
 		entry = ignorefs.New(entry, u.FilesPolicy, ignorefs.ReportIgnoredFiles(func(_ string, md fs.Entry) {
 			u.stats.AddExcluded(md)
 		}))
-		s.RootEntry, s.HashCacheID, err = u.uploadDir(ctx, entry)
+		s.RootEntry, err = u.uploadDir(ctx, entry, previousDirs)
 
 	case fs.File:
 		s.RootEntry, err = u.uploadFile(ctx, entry)


Test patch:
diff --git a/internal/hashcache/utils_test.go b/internal/hashcache/utils_test.go
deleted file mode 100644
index 7e5d0a90..00000000
--- a/internal/hashcache/utils_test.go
+++ /dev/null
@@ -1,35 +0,0 @@
-package hashcache
-
-import "testing"
-
-func TestDirectoryNameOrder(t *testing.T) {
-	sortedNames := []string{
-		"a/a/a",
-		"a/b/c",
-		"a/a",
-		"a/b",
-		"a/b1",
-		"a/b2",
-		"bar/a/d/a",
-		"bar/a/a",
-		"bar/a/b",
-		"bar/a1/a",
-		"bar/a.b",
-		"bar/a2",
-		"bar/a3",
-		"foo/a/a",
-		"foo/c/a",
-		"foo/b",
-		"goo/a/a",
-	}
-
-	for i, n1 := range sortedNames {
-		for j, n2 := range sortedNames {
-			expected := i <= j
-			actual := isLessOrEqual(n1, n2)
-			if actual != expected {
-				t.Errorf("unexpected value for isLessOrEqual('%v','%v'), expected: %v, got: %v", n1, n2, expected, actual)
-			}
-		}
-	}
-}
diff --git a/snapshot/snapshotfs/upload_test.go b/snapshot/snapshotfs/upload_test.go
index b295f7cc..40b008f2 100644
--- a/snapshot/snapshotfs/upload_test.go
+++ b/snapshot/snapshotfs/upload_test.go
@@ -101,11 +101,14 @@ func TestUpload(t *testing.T) {
 	defer th.cleanup()
 
 	u := NewUploader(th.repo)
-	s1, err := u.Upload(ctx, th.sourceDir, snapshot.SourceInfo{}, nil)
+	log.Infof("Uploading s1")
+	s1, err := u.Upload(ctx, th.sourceDir, snapshot.SourceInfo{})
 	if err != nil {
 		t.Errorf("Upload error: %v", err)
 	}
+	log.Infof("s1: %v", s1.RootEntry)
 
+	log.Infof("Uploading s2")
 	s2, err := u.Upload(ctx, th.sourceDir, snapshot.SourceInfo{}, s1)
 	if err != nil {
 		t.Errorf("Upload error: %v", err)
@@ -115,17 +118,17 @@ func TestUpload(t *testing.T) {
 		t.Errorf("expected s1.RootObjectID==s2.RootObjectID, got %v and %v", s1.RootObjectID().String(), s2.RootObjectID().String())
 	}
 
-	if !objectIDsEqual(s2.HashCacheID, s1.HashCacheID) {
-		t.Errorf("expected s2.HashCacheID==s1.HashCacheID, got %v and %v", s2.HashCacheID.String(), s1.HashCacheID.String())
+	if got, want := s1.Stats.CachedFiles, 0; got != want {
+		t.Errorf("unexpected s1 cached files: %v, want %v", got, want)
 	}
 
-	if s1.Stats.CachedFiles != 0 {
-		t.Errorf("unexpected s1 stats: %+v", s1.Stats)
+	// All non-cached files from s1 are now cached and there are no non-cached files since nothing changed.
+	if got, want := s2.Stats.CachedFiles, s1.Stats.NonCachedFiles; got != want {
+		t.Errorf("unexpected s2 cached files: %v, want %v", got, want)
 	}
 
-	// All non-cached files from s1 are now cached and there are no non-cached files since nothing changed.
-	if s2.Stats.CachedFiles != s1.Stats.NonCachedFiles || s2.Stats.NonCachedFiles != 0 {
-		t.Errorf("unexpected s2 stats: %+v, vs s1: %+v", s2.Stats, s1.Stats)
+	if got, want := s2.Stats.NonCachedFiles, 0; got != want {
+		t.Errorf("unexpected non-cached files: %v", got)
 	}
 
 	// Add one more file, the s1.RootObjectID should change.
@@ -139,10 +142,6 @@ func TestUpload(t *testing.T) {
 		t.Errorf("expected s3.RootObjectID!=s2.RootObjectID, got %v", s3.RootObjectID().String())
 	}
 
-	if objectIDsEqual(s2.HashCacheID, s3.HashCacheID) {
-		t.Errorf("expected s3.HashCacheID!=s2.HashCacheID, got %v", s3.HashCacheID.String())
-	}
-
 	if s3.Stats.NonCachedFiles != 1 {
 		// one file is not cached, which causes "./d2/d1/", "./d2/" and "./" to be changed.
 		t.Errorf("unexpected s3 stats: %+v", s3.Stats)
@@ -159,9 +158,6 @@ func TestUpload(t *testing.T) {
 	if !objectIDsEqual(s4.RootObjectID(), s1.RootObjectID()) {
 		t.Errorf("expected s4.RootObjectID==s1.RootObjectID, got %v and %v", s4.RootObjectID(), s1.RootObjectID())
 	}
-	if !objectIDsEqual(s4.HashCacheID, s1.HashCacheID) {
-		t.Errorf("expected s4.HashCacheID==s1.HashCacheID, got %v and %v", s4.HashCacheID, s1.HashCacheID)
-	}
 
 	// Everything is still cached.
 	if s4.Stats.CachedFiles != s1.Stats.NonCachedFiles || s4.Stats.NonCachedFiles != 0 {
@@ -176,9 +172,6 @@ func TestUpload(t *testing.T) {
 	if !objectIDsEqual(s4.RootObjectID(), s5.RootObjectID()) {
 		t.Errorf("expected s4.RootObjectID==s5.RootObjectID, got %v and %v", s4.RootObjectID(), s5.RootObjectID())
 	}
-	if !objectIDsEqual(s4.HashCacheID, s5.HashCacheID) {
-		t.Errorf("expected s4.HashCacheID==s5.HashCacheID, got %v and %v", s4.HashCacheID, s5.HashCacheID)
-	}
 
 	if s5.Stats.NonCachedFiles != 0 {
 		// no files are changed, but one file disappeared which caused "./d2/d1/", "./d2/" and "./" to be changed.
@@ -197,7 +190,7 @@ func TestUpload_TopLevelDirectoryReadFailure(t *testing.T) {
 	th.sourceDir.FailReaddir(errTest)
 
 	u := NewUploader(th.repo)
-	s, err := u.Upload(ctx, th.sourceDir, snapshot.SourceInfo{}, nil)
+	s, err := u.Upload(ctx, th.sourceDir, snapshot.SourceInfo{})
 	if err != errTest {
 		t.Errorf("expected error: %v", err)
 	}
@@ -216,7 +209,7 @@ func TestUpload_SubDirectoryReadFailure(t *testing.T) {
 
 	u := NewUploader(th.repo)
 	u.IgnoreFileErrors = false
-	_, err := u.Upload(ctx, th.sourceDir, snapshot.SourceInfo{}, nil)
+	_, err := u.Upload(ctx, th.sourceDir, snapshot.SourceInfo{})
 	if err == nil {
 		t.Errorf("expected error")
 	}