Repository: kopia/kopia
Pull request #2628 (2022-12-05 17:20:15)

fix(general): prevent storage retries on ESTALE errors
Running Kopia over certain (like NFS) backends can lead to ESTALE errors when working with a stale mount or file-descriptors. Currently an ESTALE from FS calls like o.Stat() end up getting retried unnecessarily. This patch introduces a fix to prevent retries on UNIX platforms.


Patch:
diff --git a/cli/command_repository_set_parameters.go b/cli/command_repository_set_parameters.go
index 95a0189f..30cc0df0 100644
--- a/cli/command_repository_set_parameters.go
+++ b/cli/command_repository_set_parameters.go
@@ -122,9 +122,58 @@ func (c *commandRepositorySetParameters) setRetentionModeParameter(ctx context.C
 	log(ctx).Infof(" - setting %v to %s.\n", desc, v)
 }
 
-func (c *commandRepositorySetParameters) run(ctx context.Context, rep repo.DirectRepositoryWriter) error {
-	var anyChange bool
+func updateRepositoryParameters(
+	ctx context.Context,
+	upgradeToEpochManager bool,
+	mp format.MutableParameters,
+	rep repo.DirectRepositoryWriter,
+	blobcfg format.BlobStorageConfiguration,
+	requiredFeatures []feature.Required,
+) error {
+	if upgradeToEpochManager {
+		log(ctx).Infof("migrating current indexes to epoch format")
+
+		if err := rep.ContentManager().PrepareUpgradeToIndexBlobManagerV1(ctx, mp.EpochParameters); err != nil {
+			return errors.Wrap(err, "error upgrading indexes")
+		}
+	}
+
+	if err := rep.FormatManager().SetParameters(ctx, mp, blobcfg, requiredFeatures); err != nil {
+		return errors.Wrap(err, "error setting parameters")
+	}
+
+	if upgradeToEpochManager {
+		if err := content.WriteLegacyIndexPoisonBlob(ctx, rep.BlobStorage()); err != nil {
+			log(ctx).Errorf("unable to write legacy index poison blob: %v", err)
+		}
+	}
+
+	return nil
+}
+
+func updateEpochParameters(mp *format.MutableParameters, anyChange, upgradeToEpochManager *bool) {
+	*anyChange = true
+
+	if !mp.EpochParameters.Enabled {
+		mp.EpochParameters = epoch.DefaultParameters()
+		mp.IndexVersion = 2
+		*upgradeToEpochManager = true
+	}
+
+	if mp.Version < format.FormatVersion2 {
+		mp.Version = format.FormatVersion2
+	}
+}
 
+func (c *commandRepositorySetParameters) disableBlobRetention(ctx context.Context, blobcfg *format.BlobStorageConfiguration, anyChange *bool) {
+	log(ctx).Infof("disabling blob retention")
+
+	blobcfg.RetentionMode = ""
+	blobcfg.RetentionPeriod = 0
+	*anyChange = true
+}
+
+func (c *commandRepositorySetParameters) run(ctx context.Context, rep repo.DirectRepositoryWriter) error {
 	mp, err := rep.FormatManager().GetMutableParameters()
 	if err != nil {
 		return errors.Wrap(err, "mutable parameters")
@@ -140,32 +189,28 @@ func (c *commandRepositorySetParameters) run(ctx context.Context, rep repo.Direc
 		return errors.Wrap(err, "unable to get required features")
 	}
 
+	anyChange := false
 	upgradeToEpochManager := false
 
 	if c.upgradeRepositoryFormat {
-		anyChange = true
+		updateEpochParameters(&mp, &anyChange, &upgradeToEpochManager)
+	}
 
-		if !mp.EpochParameters.Enabled {
-			mp.EpochParameters = epoch.DefaultParameters()
-			upgradeToEpochManager = true
-			mp.IndexVersion = 2
-		}
+	c.setSizeMBParameter(ctx, c.maxPackSizeMB, "maximum pack size", &mp.MaxPackSize, &anyChange)
 
-		if mp.Version < format.FormatVersion2 {
-			mp.Version = format.FormatVersion2
+	// prevent downgrade of index format
+	if c.indexFormatVersion != 0 && c.indexFormatVersion != mp.IndexVersion {
+		if c.indexFormatVersion > mp.IndexVersion {
+			c.setIntParameter(ctx, c.indexFormatVersion, "index format version", &mp.IndexVersion, &anyChange)
+		} else {
+			return errors.Errorf("index format version can only be upgraded")
 		}
 	}
 
-	c.setSizeMBParameter(ctx, c.maxPackSizeMB, "maximum pack size", &mp.MaxPackSize, &anyChange)
-	c.setIntParameter(ctx, c.indexFormatVersion, "index format version", &mp.IndexVersion, &anyChange)
-
 	if c.retentionMode == "none" {
 		if blobcfg.IsRetentionEnabled() {
-			log(ctx).Infof("disabling blob retention")
-
-			blobcfg.RetentionMode = ""
-			blobcfg.RetentionPeriod = 0
-			anyChange = true
+			// disable blob retention if already enabled
+			c.disableBlobRetention(ctx, &blobcfg, &anyChange)
 		}
 	} else {
 		c.setRetentionModeParameter(ctx, blob.RetentionMode(c.retentionMode), "storage backend blob retention mode", &blobcfg.RetentionMode, &anyChange)
@@ -186,22 +231,8 @@ func (c *commandRepositorySetParameters) run(ctx context.Context, rep repo.Direc
 		return errors.Errorf("no changes")
 	}
 
-	if upgradeToEpochManager {
-		log(ctx).Infof("migrating current indexes to epoch format")
-
-		if err := rep.ContentManager().PrepareUpgradeToIndexBlobManagerV1(ctx, mp.EpochParameters); err != nil {
-			return errors.Wrap(err, "error upgrading indexes")
-		}
-	}
-
-	if err := rep.FormatManager().SetParameters(ctx, mp, blobcfg, requiredFeatures); err != nil {
-		return errors.Wrap(err, "error setting parameters")
-	}
-
-	if upgradeToEpochManager {
-		if err := content.WriteLegacyIndexPoisonBlob(ctx, rep.BlobStorage()); err != nil {
-			log(ctx).Errorf("unable to write legacy index poison blob: %v", err)
-		}
+	if err := updateRepositoryParameters(ctx, upgradeToEpochManager, mp, rep, blobcfg, requiredFeatures); err != nil {
+		return errors.Wrap(err, "error updating repository parameters")
 	}
 
 	log(ctx).Infof("NOTE: Repository parameters updated, you must disconnect and re-connect all other Kopia clients.")
diff --git a/fs/entry.go b/fs/entry.go
index 20f10134..2fb5a7ff 100644
--- a/fs/entry.go
+++ b/fs/entry.go
@@ -9,6 +9,9 @@
 	"github.com/pkg/errors"
 )
 
+// ModBits is a bitmask representing the mode flags supported.
+const ModBits = os.ModePerm | os.ModeSetgid | os.ModeSetuid | os.ModeSticky
+
 // ErrUnknown is returned by ErrorEntry.ErrorInfo() to indicate that type of an entry is unknown.
 var ErrUnknown = errors.Errorf("unknown or unsupported entry type")
 
diff --git a/go.mod b/go.mod
index cf79b2d2..8f3413f0 100644
--- a/go.mod
+++ b/go.mod
@@ -35,7 +35,7 @@ require (
 	github.com/natefinch/atomic v1.0.1
 	github.com/pierrec/lz4 v2.6.1+incompatible
 	github.com/pkg/errors v0.9.1
-	github.com/pkg/profile v1.6.0
+	github.com/pkg/profile v1.7.0
 	github.com/pkg/sftp v1.13.5
 	github.com/pquerna/ffjson v0.0.0-20190930134022-aa0246cd15f7 // indirect
 	github.com/prometheus/client_golang v1.14.0
@@ -48,13 +48,13 @@ require (
 	github.com/zalando/go-keyring v0.2.1
 	github.com/zeebo/blake3 v0.2.3
 	go.opencensus.io v0.24.0 // indirect
-	go.uber.org/zap v1.23.0
+	go.uber.org/zap v1.24.0
 	golang.org/x/crypto v0.3.0
 	golang.org/x/mod v0.7.0
 	golang.org/x/net v0.2.0
 	golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783
 	golang.org/x/sync v0.1.0
-	golang.org/x/sys v0.2.0
+	golang.org/x/sys v0.3.0
 	golang.org/x/term v0.2.0
 	golang.org/x/text v0.4.0
 	google.golang.org/api v0.103.0
@@ -121,11 +121,13 @@ require (
 	cloud.google.com/go/iam v0.7.0 // indirect
 	github.com/alessio/shellescape v1.4.1 // indirect
 	github.com/chromedp/sysutil v1.0.0 // indirect
+	github.com/felixge/fgprof v0.9.3 // indirect
 	github.com/go-logr/logr v1.2.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/gobwas/httphead v0.1.0 // indirect
 	github.com/gobwas/pool v0.2.1 // indirect
 	github.com/gobwas/ws v1.1.0 // indirect
+	github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
diff --git a/go.sum b/go.sum
index 10ac76e4..811d8173 100644
--- a/go.sum
+++ b/go.sum
@@ -128,6 +128,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
 github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
+github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g=
+github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw=
 github.com/foomo/htpasswd v0.0.0-20200116085101-e3a90e78da9c h1:DBGU7zCwrrPPDsD6+gqKG8UfMxenWg9BOJE/Nmfph+4=
 github.com/foomo/htpasswd v0.0.0-20200116085101-e3a90e78da9c/go.mod h1:SHawtolbB0ZOFoRWgDwakX5WpwuIWAK88bUXVZqK0Ss=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
@@ -225,6 +227,8 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf
 github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/pprof v0.0.0-20211214055906-6f57359322fd h1:1FjCyPC+syAzJ5/2S8fqdZK1R22vvA0J7JZKcuOIQ7Y=
+github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg=
 github.com/google/readahead v0.0.0-20161222183148-eaceba169032 h1:6Be3nkuJFyRfCgr6qTIzmRp8y9QwDIbqy/nYr9WDPos=
 github.com/google/readahead v0.0.0-20161222183148-eaceba169032/go.mod h1:qYysrqQXuV4tzsizt4oOQ6mrBZQ0xnQXP3ylXX8Jk5Y=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
@@ -248,6 +252,7 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ
 github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
 github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
 github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
@@ -333,8 +338,8 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/profile v1.6.0 h1:hUDfIISABYI59DyeB3OTay/HxSRwTQ8rB/H83k6r5dM=
-github.com/pkg/profile v1.6.0/go.mod h1:qBsxPvzyUincmltOk6iyRVxHYg4adc0OFOv72ZdLa18=
+github.com/pkg/profile v1.7.0 h1:hnbDkaNWPCLMO9wGLdBFTIZvzDrDfBM2072E1S9gJkA=
+github.com/pkg/profile v1.7.0/go.mod h1:8Uer0jas47ZQMJ7VD+OHknK4YDY07LPUC6dEvqDjvNo=
 github.com/pkg/sftp v1.13.5 h1:a3RLUqkyjYRtBTZJZ1VRrKbN3zhuPLlUc3sphVz81go=
 github.com/pkg/sftp v1.13.5/go.mod h1:wHDZ0IZX6JcBYRK1TH9bcVq8G7TLpVHYIGJRFnmPfxg=
 github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -436,8 +441,8 @@ go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
 go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8=
 go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
-go.uber.org/zap v1.23.0 h1:OjGQ5KQDEUawVHxNwQgPpiypGHOxo2mNZsOqTak4fFY=
-go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY=
+go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60=
+go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190228161510-8dd112bcdc25/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
@@ -592,6 +597,7 @@ golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210819135213-f52c844e1c1c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -600,8 +606,8 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A=
-golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ=
+golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
diff --git a/internal/fusemount/fusefs.go b/internal/fusemount/fusefs.go
index feed3adb..85e437da 100644
--- a/internal/fusemount/fusefs.go
+++ b/internal/fusemount/fusefs.go
@@ -30,8 +30,26 @@ type fuseNode struct {
 	entry fs.Entry
 }
 
+func goModeToUnixMode(mode os.FileMode) uint32 {
+	unixmode := uint32(mode.Perm())
+
+	if mode&os.ModeSetuid != 0 {
+		unixmode |= 0o4000
+	}
+
+	if mode&os.ModeSetgid != 0 {
+		unixmode |= 0o2000
+	}
+
+	if mode&os.ModeSticky != 0 {
+		unixmode |= 0o1000
+	}
+
+	return unixmode
+}
+
 func populateAttributes(a *fuse.Attr, e fs.Entry) {
-	a.Mode = uint32(e.Mode()) & uint32(os.ModePerm)
+	a.Mode = goModeToUnixMode(e.Mode())
 	a.Size = uint64(e.Size())
 	a.Mtime = uint64(e.ModTime().Unix())
 	a.Ctime = a.Mtime
diff --git a/repo/blob/filesystem/filesystem_storage.go b/repo/blob/filesystem/filesystem_storage.go
index 9340a313..626bf4c5 100644
--- a/repo/blob/filesystem/filesystem_storage.go
+++ b/repo/blob/filesystem/filesystem_storage.go
@@ -51,6 +51,12 @@ func (fs *fsImpl) isRetriable(err error) bool {
 
 	err = errors.Cause(err)
 
+	if fs.osi.IsESTALE(err) {
+		// errors indicative of stale resource handle or invalid
+		// descriptors should not be retried
+		return false
+	}
+
 	if fs.osi.IsNotExist(err) {
 		return false
 	}
diff --git a/repo/blob/filesystem/osinterface.go b/repo/blob/filesystem/osinterface.go
index 92723e09..06179a1d 100644
--- a/repo/blob/filesystem/osinterface.go
+++ b/repo/blob/filesystem/osinterface.go
@@ -27,6 +27,9 @@ type osInterface interface {
 	Chtimes(fname string, atime, mtime time.Time) error
 	Geteuid() int
 	Chown(fname string, uid, gid int) error
+
+	// Errno
+	IsESTALE(err error) bool
 }
 
 type osReadFile interface {
diff --git a/repo/blob/filesystem/osinterface_realos_other.go b/repo/blob/filesystem/osinterface_realos_other.go
new file mode 100644
index 00000000..665c22b8
--- /dev/null
+++ b/repo/blob/filesystem/osinterface_realos_other.go
@@ -0,0 +1,8 @@
+//go:build !linux && !freebsd && !darwin
+// +build !linux,!freebsd,!darwin
+
+package filesystem
+
+func (realOS) IsESTALE(err error) bool {
+	return false
+}
diff --git a/repo/blob/filesystem/osinterface_realos_unix.go b/repo/blob/filesystem/osinterface_realos_unix.go
new file mode 100644
index 00000000..15c71f0e
--- /dev/null
+++ b/repo/blob/filesystem/osinterface_realos_unix.go
@@ -0,0 +1,16 @@
+//go:build linux || freebsd || darwin
+// +build linux freebsd darwin
+
+package filesystem
+
+import (
+	"syscall"
+
+	"github.com/pkg/errors"
+)
+
+func (realOS) IsESTALE(err error) bool {
+	var errno syscall.Errno
+
+	return errors.As(err, &errno) && errno == syscall.ESTALE
+}
diff --git a/site/content/docs/Installation/_index.md b/site/content/docs/Installation/_index.md
index ef7df666..ce4bf09e 100644
--- a/site/content/docs/Installation/_index.md
+++ b/site/content/docs/Installation/_index.md
@@ -98,6 +98,22 @@ Alternatively, to install the latest unreleased version of Kopia use the followi
 $ brew install kopia/test-builds/kopia
 ```
 
+#### macOS GUI using Homebrew
+
+On macOS, you can use [Homebrew](https://brew.sh) to install and keep Kopia up-to-date.
+
+To install:
+
+```shell
+$ brew install kopiaui
+```
+
+To upgrade Kopia:
+
+```shell
+$ brew upgrade kopiaui
+```
+
 #### macOS GUI installer
 
 MacOS package with `KopiaUI` is available in DMG and ZIP formats on the [releases page](https://github.com/kopia/kopia/releases/latest).
diff --git a/snapshot/restore/local_fs_output.go b/snapshot/restore/local_fs_output.go
index a37a5e13..37f93706 100644
--- a/snapshot/restore/local_fs_output.go
+++ b/snapshot/restore/local_fs_output.go
@@ -21,7 +21,6 @@
 )
 
 const (
-	modBits                           = os.ModePerm | os.ModeSetgid | os.ModeSetuid | os.ModeSticky
 	outputDirMode                     = 0o700 // default mode to create directories in before setting their ACLs
 	maxTimeDeltaToConsiderFileTheSame = 2 * time.Second
 )
@@ -273,7 +272,7 @@ func (o *FilesystemOutput) setAttributes(targetPath string, e fs.Entry, modclear
 
 	// Set file permissions from e
 	if o.shouldUpdatePermissions(le, e, modclear) {
-		if err = o.maybeIgnorePermissionError(osChmod(targetPath, (e.Mode()&modBits)&^modclear)); err != nil {
+		if err = o.maybeIgnorePermissionError(osChmod(targetPath, (e.Mode()&fs.ModBits)&^modclear)); err != nil {
 			return errors.Wrap(err, "could not change permissions on "+targetPath)
 		}
 	}
@@ -317,7 +316,7 @@ func (o *FilesystemOutput) shouldUpdatePermissions(local, remote fs.Entry, modcl
 		return false
 	}
 
-	return ((local.Mode() & modBits) &^ modclear) != (remote.Mode() & modBits)
+	return ((local.Mode() & fs.ModBits) &^ modclear) != (remote.Mode() & fs.ModBits)
 }
 
 func (o *FilesystemOutput) shouldUpdateTimes(local, remote fs.Entry) bool {
diff --git a/snapshot/snapshotfs/upload.go b/snapshot/snapshotfs/upload.go
index efbb7eac..e08bcb8e 100644
--- a/snapshot/snapshotfs/upload.go
+++ b/snapshot/snapshotfs/upload.go
@@ -454,7 +454,7 @@ func newDirEntry(md fs.Entry, fname string, oid object.ID) (*snapshot.DirEntry,
 	return &snapshot.DirEntry{
 		Name:        fname,
 		Type:        entryType,
-		Permissions: snapshot.Permissions(md.Mode() & os.ModePerm),
+		Permissions: snapshot.Permissions(md.Mode() & fs.ModBits),
 		FileSize:    md.Size(),
 		ModTime:     fs.UTCTimestampFromTime(md.ModTime()),
 		UserID:      md.Owner().UserID,


Test patch:
diff --git a/cli/command_repository_set_parameters_test.go b/cli/command_repository_set_parameters_test.go
index 136785ac..87caecaf 100644
--- a/cli/command_repository_set_parameters_test.go
+++ b/cli/command_repository_set_parameters_test.go
@@ -182,6 +182,49 @@ func (s *formatSpecificTestSuite) TestRepositorySetParametersUpgrade(t *testing.
 	env.RunAndExpectSuccess(t, "index", "epoch", "list")
 }
 
+// TestRepositorySetParametersDowngrade test that a repository cannot be downgraded by using `set-parameters`.
+func (s *formatSpecificTestSuite) TestRepositorySetParametersDowngrade(t *testing.T) {
+	env := s.setupInMemoryRepo(t)
+
+	// checkStatusForVersion is a function with stanzas to check that the repository has the expected version.
+	// 	its saved into a variable to prevent repetition and enforce that nothing has changed between invocations
+	//  if `set-parameters`
+	checkStatusForVersion := func() {
+		out := env.RunAndExpectSuccess(t, "repository", "status")
+
+		// default values
+		require.Contains(t, out, "Max pack length:     21 MB")
+
+		switch s.formatVersion {
+		case format.FormatVersion1:
+			require.Contains(t, out, "Format version:      1")
+			require.Contains(t, out, "Epoch Manager:       disabled")
+			env.RunAndExpectFailure(t, "index", "epoch", "list")
+		case format.FormatVersion2:
+			require.Contains(t, out, "Format version:      2")
+			require.Contains(t, out, "Epoch Manager:       enabled")
+			env.RunAndExpectSuccess(t, "index", "epoch", "list")
+		default:
+			require.Contains(t, out, "Format version:      3")
+			require.Contains(t, out, "Epoch Manager:       enabled")
+			env.RunAndExpectSuccess(t, "index", "epoch", "list")
+		}
+	}
+
+	checkStatusForVersion()
+
+	env.RunAndExpectFailure(t, "repository", "set-parameters", "--index-version=1")
+
+	checkStatusForVersion()
+
+	// run basic check to ensure that an upgrade can still be performed as expected
+	env.RunAndExpectSuccess(t, "repository", "set-parameters", "--upgrade")
+
+	out := env.RunAndExpectSuccess(t, "repository", "status")
+	require.Contains(t, out, "Epoch Manager:       enabled")
+	require.Contains(t, out, "Index Format:        v2")
+}
+
 func (s *formatSpecificTestSuite) TestRepositorySetParametersRequiredFeatures(t *testing.T) {
 	env := s.setupInMemoryRepo(t)
 
diff --git a/repo/blob/filesystem/filesystem_storage_unix_test.go b/repo/blob/filesystem/filesystem_storage_unix_test.go
new file mode 100644
index 00000000..e6d2a21f
--- /dev/null
+++ b/repo/blob/filesystem/filesystem_storage_unix_test.go
@@ -0,0 +1,48 @@
+//go:build linux || freebsd || darwin
+// +build linux freebsd darwin
+
+package filesystem
+
+import (
+	"syscall"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/kopia/kopia/internal/gather"
+	"github.com/kopia/kopia/internal/testlogging"
+	"github.com/kopia/kopia/internal/testutil"
+	"github.com/kopia/kopia/repo/blob"
+	"github.com/kopia/kopia/repo/blob/sharded"
+)
+
+func TestFileStorage_ESTALE_ErrorHandling(t *testing.T) {
+	t.Parallel()
+
+	ctx := testlogging.Context(t)
+
+	dataDir := testutil.TempDirectory(t)
+
+	osi := newMockOS()
+
+	st, err := New(ctx, &Options{
+		Path: dataDir,
+		Options: sharded.Options{
+			DirectoryShards: []int{5, 2},
+		},
+	}, true)
+	require.NoError(t, err)
+
+	st.(*fsStorage).Impl.(*fsImpl).osi = osi
+
+	require.False(t, st.(*fsStorage).Impl.(*fsImpl).isRetriable(syscall.ESTALE), "ESTALE should not be retryable")
+
+	defer st.Close(ctx)
+
+	require.NoError(t, st.PutBlob(ctx, "someblob1234567812345678", gather.FromSlice([]byte{1, 2, 3}), blob.PutOptions{}))
+
+	osi.eStaleRemainingErrors.Store(1)
+
+	_, err = st.GetMetadata(ctx, "someblob1234567812345678")
+	require.ErrorIs(t, err, syscall.ESTALE)
+}
diff --git a/repo/blob/filesystem/osinterface_mock_other_test.go b/repo/blob/filesystem/osinterface_mock_other_test.go
new file mode 100644
index 00000000..e521fbc4
--- /dev/null
+++ b/repo/blob/filesystem/osinterface_mock_other_test.go
@@ -0,0 +1,19 @@
+//go:build !linux && !freebsd && !darwin
+// +build !linux,!freebsd,!darwin
+
+package filesystem
+
+import (
+	"io/fs"
+	"os"
+
+	"github.com/pkg/errors"
+)
+
+func (osi *mockOS) Stat(fname string) (fs.FileInfo, error) {
+	if osi.statRemainingErrors.Add(-1) >= 0 {
+		return nil, &os.PathError{Op: "stat", Err: errors.Errorf("underlying problem")}
+	}
+
+	return osi.osInterface.Stat(fname)
+}
diff --git a/repo/blob/filesystem/osinterface_mock_test.go b/repo/blob/filesystem/osinterface_mock_test.go
index c8972e9c..e2a816d1 100644
--- a/repo/blob/filesystem/osinterface_mock_test.go
+++ b/repo/blob/filesystem/osinterface_mock_test.go
@@ -31,6 +31,10 @@ type mockOS struct {
 
 	effectiveUID int
 
+	// remaining syscall errnos
+	//nolint:unused // Used with platform specific code
+	eStaleRemainingErrors atomic.Int32
+
 	osInterface
 }
 
@@ -94,14 +98,6 @@ func (osi *mockOS) Remove(fname string) error {
 	return osi.osInterface.Remove(fname)
 }
 
-func (osi *mockOS) Stat(fname string) (fs.FileInfo, error) {
-	if osi.statRemainingErrors.Add(-1) >= 0 {
-		return nil, &os.PathError{Op: "stat", Err: errors.Errorf("underlying problem")}
-	}
-
-	return osi.osInterface.Stat(fname)
-}
-
 func (osi *mockOS) Chtimes(fname string, atime, mtime time.Time) error {
 	if osi.chtimesRemainingErrors.Add(-1) >= 0 {
 		return &os.PathError{Op: "chtimes", Err: errors.Errorf("underlying problem")}
diff --git a/repo/blob/filesystem/osinterface_mock_unix_test.go b/repo/blob/filesystem/osinterface_mock_unix_test.go
new file mode 100644
index 00000000..424408e1
--- /dev/null
+++ b/repo/blob/filesystem/osinterface_mock_unix_test.go
@@ -0,0 +1,24 @@
+//go:build linux || freebsd || darwin
+// +build linux freebsd darwin
+
+package filesystem
+
+import (
+	"io/fs"
+	"os"
+	"syscall"
+
+	"github.com/pkg/errors"
+)
+
+func (osi *mockOS) Stat(fname string) (fs.FileInfo, error) {
+	if osi.statRemainingErrors.Add(-1) >= 0 {
+		return nil, &os.PathError{Op: "stat", Err: errors.Errorf("underlying problem")}
+	}
+
+	if osi.eStaleRemainingErrors.Add(-1) >= 0 {
+		return nil, &os.PathError{Op: "stat", Err: syscall.ESTALE}
+	}
+
+	return osi.osInterface.Stat(fname)
+}
diff --git a/tests/end_to_end_test/restore_test.go b/tests/end_to_end_test/restore_test.go
index 2e5f5781..c1b456ca 100644
--- a/tests/end_to_end_test/restore_test.go
+++ b/tests/end_to_end_test/restore_test.go
@@ -259,7 +259,7 @@ func TestSnapshotRestore(t *testing.T) {
 
 	// restoring single file
 	e.RunAndExpectSuccess(t, "snapshot", "restore", rootID+"/single-file", filepath.Join(restoreByOIDFile, "output-file"))
-	verifyFileMode(t, filepath.Join(restoreByOIDFile, "output-file"), os.FileMode(0o651))
+	verifyFileMode(t, filepath.Join(restoreByOIDFile, "output-file"), overriddenFilePermissions)
 
 	// Restore last snapshot using the snapshot ID
 	e.RunAndExpectSuccess(t, "snapshot", "restore", snapID, restoreDir)
@@ -461,21 +461,22 @@ func TestRestoreSnapshotOfSingleFile(t *testing.T) {
 	verifyFileMode(t, filepath.Join(restoreDir, "restored-2"), os.FileMode(0o653))
 
 	if runtime.GOOS != windowsOSName {
+		overriddenFilePermissions := 0o654 | os.ModeSetuid
+
 		// change source file permissions and create one more snapshot
 		// at this poing we will have multiple snapshot manifests with one root but different attributes.
-		os.Chmod(sourceFile, 0o654)
+		os.Chmod(sourceFile, overriddenFilePermissions)
 		e.RunAndExpectSuccess(t, "snapshot", "create", sourceFile)
 
 		// when restoring by root Kopia needs to pick which manifest to apply since they are conflicting
 		// We're passing --consistent-attributes which causes it to fail, since otherwise we'd restore arbitrary
 		// top-level object permissions.
 		e.RunAndExpectFailure(t, "snapshot", "restore", rootID, "--consistent-attributes", filepath.Join(restoreDir, "restored-3"))
-	}
 
-	// Without the flag kopia picks the attributes from the latest snapshot.
-	e.RunAndExpectSuccess(t, "snapshot", "restore", rootID, filepath.Join(restoreDir, "restored-3"))
-
-	verifyFileMode(t, filepath.Join(restoreDir, "restored-3"), os.FileMode(0o654))
+		// Without the flag kopia picks the attributes from the latest snapshot.
+		e.RunAndExpectSuccess(t, "snapshot", "restore", rootID, filepath.Join(restoreDir, "restored-3"))
+		verifyFileMode(t, filepath.Join(restoreDir, "restored-3"), overriddenFilePermissions)
+	}
 
 	// restoring using snapshot ID is unambiguous and always produces file with 0o653
 	e.RunAndExpectSuccess(t, "snapshot", "restore", snapID, filepath.Join(restoreDir, "restored-4"))