Repository: gohugoio/hugo (language: go)

Issue:
"Error while watching" when removing lots of content dirs
```
ERROR 2022/12/14 11:32:12 Error while watching: fsnotify.sendDirectoryChangeEvents: open /Users/.... no such file or directory
```

I have checked, and we do ignore `os.IsNotExist` errors, but the above is wrapped or something. This also smells like a bug in the upstream library.


Discussion / hints:

Fix patch:
diff --git a/cache/filecache/filecache_pruner.go b/cache/filecache/filecache_pruner.go
index e5e5719725d..5734af1995d 100644
--- a/cache/filecache/filecache_pruner.go
+++ b/cache/filecache/filecache_pruner.go
@@ -18,6 +18,7 @@ import (
 	"io"
 	"os"
 
+	"github.com/gohugoio/hugo/common/herrors"
 	"github.com/gohugoio/hugo/hugofs"
 
 	"github.com/spf13/afero"
@@ -36,7 +37,7 @@ func (c Caches) Prune() (int, error) {
 		counter += count
 
 		if err != nil {
-			if os.IsNotExist(err) {
+			if herrors.IsNotExist(err) {
 				continue
 			}
 			return counter, fmt.Errorf("failed to prune cache %q: %w", k, err)
@@ -76,7 +77,7 @@ func (c *Cache) Prune(force bool) (int, error) {
 				err = c.Fs.Remove(name)
 			}
 
-			if err != nil && !os.IsNotExist(err) {
+			if err != nil && !herrors.IsNotExist(err) {
 				return err
 			}
 
@@ -97,7 +98,7 @@ func (c *Cache) Prune(force bool) (int, error) {
 				counter++
 			}
 
-			if err != nil && !os.IsNotExist(err) {
+			if err != nil && !herrors.IsNotExist(err) {
 				return err
 			}
 
@@ -112,7 +113,7 @@ func (c *Cache) Prune(force bool) (int, error) {
 func (c *Cache) pruneRootDir(force bool) (int, error) {
 	info, err := c.Fs.Stat(c.pruneAllRootDir)
 	if err != nil {
-		if os.IsNotExist(err) {
+		if herrors.IsNotExist(err) {
 			return 0, nil
 		}
 		return 0, err
diff --git a/commands/hugo.go b/commands/hugo.go
index e26f052d451..e247fca275a 100644
--- a/commands/hugo.go
+++ b/commands/hugo.go
@@ -579,7 +579,7 @@ func (c *commandeer) serverBuild() error {
 
 func (c *commandeer) copyStatic() (map[string]uint64, error) {
 	m, err := c.doWithPublishDirs(c.copyStaticTo)
-	if err == nil || os.IsNotExist(err) {
+	if err == nil || herrors.IsNotExist(err) {
 		return m, nil
 	}
 	return m, err
@@ -899,7 +899,7 @@ func (c *commandeer) newWatcher(pollIntervalStr string, dirList ...string) (*wat
 				}
 				unlock()
 			case err := <-watcher.Errors():
-				if err != nil && !os.IsNotExist(err) {
+				if err != nil && !herrors.IsNotExist(err) {
 					c.logger.Errorln("Error while watching:", err)
 				}
 			}
diff --git a/commands/static_syncer.go b/commands/static_syncer.go
index b97c4df7ade..c248ca152c3 100644
--- a/commands/static_syncer.go
+++ b/commands/static_syncer.go
@@ -14,9 +14,9 @@
 package commands
 
 import (
-	"os"
 	"path/filepath"
 
+	"github.com/gohugoio/hugo/common/herrors"
 	"github.com/gohugoio/hugo/hugolib/filesystems"
 
 	"github.com/fsnotify/fsnotify"
@@ -95,7 +95,7 @@ func (s *staticSyncer) syncsStaticEvents(staticEvents []fsnotify.Event) error {
 			// the source of that static file. In this case Hugo will incorrectly remove that file
 			// from the published directory.
 			if ev.Op&fsnotify.Rename == fsnotify.Rename || ev.Op&fsnotify.Remove == fsnotify.Remove {
-				if _, err := sourceFs.Fs.Stat(relPath); os.IsNotExist(err) {
+				if _, err := sourceFs.Fs.Stat(relPath); herrors.IsNotExist(err) {
 					// If file doesn't exist in any static dir, remove it
 					logger.Println("File no longer exists in static dir, removing", relPath)
 					_ = c.Fs.PublishDirStatic.RemoveAll(relPath)
diff --git a/common/herrors/errors.go b/common/herrors/errors.go
index 6ce90885353..822271ef202 100644
--- a/common/herrors/errors.go
+++ b/common/herrors/errors.go
@@ -1,4 +1,4 @@
-// Copyright 2018 The Hugo Authors. All rights reserved.
+// Copyright 2022 The Hugo Authors. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -19,6 +19,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"os"
 	"runtime"
 	"runtime/debug"
 	"strconv"
@@ -38,7 +39,8 @@ type ErrorSender interface {
 
 // Recover is a helper function that can be used to capture panics.
 // Put this at the top of a method/function that crashes in a template:
-//     defer herrors.Recover()
+//
+//	defer herrors.Recover()
 func Recover(args ...any) {
 	if r := recover(); r != nil {
 		fmt.Println("ERR:", r)
@@ -69,3 +71,18 @@ func Must(err error) {
 		panic(err)
 	}
 }
+
+// IsNotExist returns true if the error is a file not found error.
+// Unlike os.IsNotExist, this also considers wrapped errors.
+func IsNotExist(err error) bool {
+	if os.IsNotExist(err) {
+		return true
+	}
+
+	// os.IsNotExist does not consider wrapped errors.
+	if os.IsNotExist(errors.Unwrap(err)) {
+		return true
+	}
+
+	return false
+}
diff --git a/helpers/path.go b/helpers/path.go
index 0fb365f43bb..ad2ff765803 100644
--- a/helpers/path.go
+++ b/helpers/path.go
@@ -24,6 +24,7 @@ import (
 	"strings"
 	"unicode"
 
+	"github.com/gohugoio/hugo/common/herrors"
 	"github.com/gohugoio/hugo/common/text"
 
 	"github.com/gohugoio/hugo/config"
@@ -378,7 +379,7 @@ func OpenFileForWriting(fs afero.Fs, filename string) (afero.File, error) {
 	// os.Create will create any new files with mode 0666 (before umask).
 	f, err := fs.Create(filename)
 	if err != nil {
-		if !os.IsNotExist(err) {
+		if !herrors.IsNotExist(err) {
 			return nil, err
 		}
 		if err = fs.MkdirAll(filepath.Dir(filename), 0777); err != nil { //  before umask
diff --git a/hugofs/decorators.go b/hugofs/decorators.go
index 3762d753b0e..47b4266df4c 100644
--- a/hugofs/decorators.go
+++ b/hugofs/decorators.go
@@ -19,6 +19,7 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/gohugoio/hugo/common/herrors"
 	"github.com/spf13/afero"
 )
 
@@ -224,7 +225,7 @@ func (l *baseFileDecoratorFile) Readdir(c int) (ofi []os.FileInfo, err error) {
 		// We need to resolve any symlink info.
 		fi, _, err := lstatIfPossible(l.fs.Fs, filename)
 		if err != nil {
-			if os.IsNotExist(err) {
+			if herrors.IsNotExist(err) {
 				continue
 			}
 			return nil, err
diff --git a/hugofs/rootmapping_fs.go b/hugofs/rootmapping_fs.go
index 90df48f8cae..a37e21a8b3b 100644
--- a/hugofs/rootmapping_fs.go
+++ b/hugofs/rootmapping_fs.go
@@ -19,6 +19,7 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/gohugoio/hugo/common/herrors"
 	"github.com/gohugoio/hugo/hugofs/files"
 
 	radix "github.com/armon/go-radix"
@@ -45,7 +46,7 @@ func NewRootMappingFs(fs afero.Fs, rms ...RootMapping) (*RootMappingFs, error) {
 
 		fi, err := fs.Stat(rm.To)
 		if err != nil {
-			if os.IsNotExist(err) {
+			if herrors.IsNotExist(err) {
 				continue
 			}
 			return nil, err
diff --git a/hugofs/slice_fs.go b/hugofs/slice_fs.go
index 7edaf75137d..574a5cb5fd8 100644
--- a/hugofs/slice_fs.go
+++ b/hugofs/slice_fs.go
@@ -21,6 +21,7 @@ import (
 
 	"errors"
 
+	"github.com/gohugoio/hugo/common/herrors"
 	"github.com/spf13/afero"
 )
 
@@ -161,7 +162,7 @@ func (fs *SliceFs) pickFirst(name string) (os.FileInfo, int, error) {
 			return fi, i, nil
 		}
 
-		if !os.IsNotExist(err) {
+		if !herrors.IsNotExist(err) {
 			// Real error
 			return nil, -1, err
 		}
@@ -175,7 +176,7 @@ func (fs *SliceFs) readDirs(name string, startIdx, count int) ([]os.FileInfo, er
 	collect := func(lfs *FileMeta) ([]os.FileInfo, error) {
 		d, err := lfs.Fs.Open(name)
 		if err != nil {
-			if !os.IsNotExist(err) {
+			if !herrors.IsNotExist(err) {
 				return nil, err
 			}
 			return nil, nil
diff --git a/hugofs/walk.go b/hugofs/walk.go
index 22a99402f32..e847174c672 100644
--- a/hugofs/walk.go
+++ b/hugofs/walk.go
@@ -20,6 +20,7 @@ import (
 	"sort"
 	"strings"
 
+	"github.com/gohugoio/hugo/common/herrors"
 	"github.com/gohugoio/hugo/common/loggers"
 
 	"errors"
@@ -118,7 +119,7 @@ func (w *Walkway) Walk() error {
 	} else {
 		info, _, err := lstatIfPossible(w.fs, w.root)
 		if err != nil {
-			if os.IsNotExist(err) {
+			if herrors.IsNotExist(err) {
 				return nil
 			}
 
@@ -154,7 +155,7 @@ func (w *Walkway) checkErr(filename string, err error) bool {
 		return true
 	}
 
-	if os.IsNotExist(err) {
+	if herrors.IsNotExist(err) {
 		// The file may be removed in process.
 		// This may be a ERROR situation, but it is not possible
 		// to determine as a general case.
diff --git a/hugolib/codeowners.go b/hugolib/codeowners.go
index 17e956981e7..162ee16ae90 100644
--- a/hugolib/codeowners.go
+++ b/hugolib/codeowners.go
@@ -15,9 +15,9 @@ package hugolib
 
 import (
 	"io"
-	"os"
 	"path"
 
+	"github.com/gohugoio/hugo/common/herrors"
 	"github.com/gohugoio/hugo/config"
 	"github.com/gohugoio/hugo/resources/page"
 	"github.com/hairyhenderson/go-codeowners"
@@ -32,7 +32,7 @@ func findCodeOwnersFile(dir string) (io.Reader, error) {
 
 		_, err := afs.Stat(f)
 		if err != nil {
-			if os.IsNotExist(err) {
+			if herrors.IsNotExist(err) {
 				continue
 			}
 			return nil, err
diff --git a/hugolib/filesystems/basefs.go b/hugolib/filesystems/basefs.go
index e0fed6f3e6b..5a98be47ee2 100644
--- a/hugolib/filesystems/basefs.go
+++ b/hugolib/filesystems/basefs.go
@@ -28,6 +28,7 @@ import (
 	"github.com/gohugoio/hugo/htesting"
 	"github.com/gohugoio/hugo/hugofs/glob"
 
+	"github.com/gohugoio/hugo/common/herrors"
 	"github.com/gohugoio/hugo/common/types"
 
 	"github.com/gohugoio/hugo/common/loggers"
@@ -295,15 +296,15 @@ func (s SourceFilesystems) StaticFs(lang string) afero.Fs {
 
 // StatResource looks for a resource in these filesystems in order: static, assets and finally content.
 // If found in any of them, it returns FileInfo and the relevant filesystem.
-// Any non os.IsNotExist error will be returned.
-// An os.IsNotExist error wil be returned only if all filesystems return such an error.
+// Any non herrors.IsNotExist error will be returned.
+// An herrors.IsNotExist error wil be returned only if all filesystems return such an error.
 // Note that if we only wanted to find the file, we could create a composite Afero fs,
 // but we also need to know which filesystem root it lives in.
 func (s SourceFilesystems) StatResource(lang, filename string) (fi os.FileInfo, fs afero.Fs, err error) {
 	for _, fsToCheck := range []afero.Fs{s.StaticFs(lang), s.Assets.Fs, s.Content.Fs} {
 		fs = fsToCheck
 		fi, err = fs.Stat(filename)
-		if err == nil || !os.IsNotExist(err) {
+		if err == nil || !herrors.IsNotExist(err) {
 			return
 		}
 	}
diff --git a/hugolib/page.go b/hugolib/page.go
index ec7b8227771..5acfbc67720 100644
--- a/hugolib/page.go
+++ b/hugolib/page.go
@@ -16,7 +16,6 @@ package hugolib
 import (
 	"bytes"
 	"fmt"
-	"os"
 	"path"
 	"path/filepath"
 	"sort"
@@ -489,7 +488,7 @@ func (p *pageState) renderResources() (err error) {
 			}
 
 			if err := src.Publish(); err != nil {
-				if os.IsNotExist(err) {
+				if herrors.IsNotExist(err) {
 					// The resource has been deleted from the file system.
 					// This should be extremely rare, but can happen on live reload in server
 					// mode when the same resource is member of different page bundles.
diff --git a/hugolib/pages_capture.go b/hugolib/pages_capture.go
index da7515fc22b..b72ae7e8542 100644
--- a/hugolib/pages_capture.go
+++ b/hugolib/pages_capture.go
@@ -16,11 +16,11 @@ package hugolib
 import (
 	"context"
 	"fmt"
-	"os"
 	pth "path"
 	"path/filepath"
 	"reflect"
 
+	"github.com/gohugoio/hugo/common/herrors"
 	"github.com/gohugoio/hugo/common/maps"
 
 	"github.com/gohugoio/hugo/parser/pageparser"
@@ -318,7 +318,7 @@ func (c *pagesCollector) cloneFileInfo(fi hugofs.FileMetaInfo) hugofs.FileMetaIn
 func (c *pagesCollector) collectDir(dirname string, partial bool, inFilter func(fim hugofs.FileMetaInfo) bool) error {
 	fi, err := c.fs.Stat(dirname)
 	if err != nil {
-		if os.IsNotExist(err) {
+		if herrors.IsNotExist(err) {
 			// May have been deleted.
 			return nil
 		}
diff --git a/hugolib/site.go b/hugolib/site.go
index cbfc4d836e0..8fb39a1eac8 100644
--- a/hugolib/site.go
+++ b/hugolib/site.go
@@ -20,7 +20,6 @@ import (
 	"log"
 	"mime"
 	"net/url"
-	"os"
 	"path"
 	"path/filepath"
 	"regexp"
@@ -30,6 +29,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/gohugoio/hugo/common/herrors"
 	"github.com/gohugoio/hugo/common/htime"
 	"github.com/gohugoio/hugo/common/hugio"
 	"github.com/gohugoio/hugo/common/types"
@@ -90,16 +90,16 @@ import (
 //
 // 1. A list of Files is parsed and then converted into Pages.
 //
-// 2. Pages contain sections (based on the file they were generated from),
-//    aliases and slugs (included in a pages frontmatter) which are the
-//    various targets that will get generated.  There will be canonical
-//    listing.  The canonical path can be overruled based on a pattern.
+//  2. Pages contain sections (based on the file they were generated from),
+//     aliases and slugs (included in a pages frontmatter) which are the
+//     various targets that will get generated.  There will be canonical
+//     listing.  The canonical path can be overruled based on a pattern.
 //
-// 3. Taxonomies are created via configuration and will present some aspect of
-//    the final page and typically a perm url.
+//  3. Taxonomies are created via configuration and will present some aspect of
+//     the final page and typically a perm url.
 //
-// 4. All Pages are passed through a template based on their desired
-//    layout based on numerous different elements.
+//  4. All Pages are passed through a template based on their desired
+//     layout based on numerous different elements.
 //
 // 5. The entire collection of files is written to disk.
 type Site struct {
@@ -954,7 +954,7 @@ func (s *Site) filterFileEvents(events []fsnotify.Event) []fsnotify.Event {
 
 		// Throw away any directories
 		isRegular, err := s.SourceSpec.IsRegularSourceFile(ev.Name)
-		if err != nil && os.IsNotExist(err) && (ev.Op&fsnotify.Remove == fsnotify.Remove || ev.Op&fsnotify.Rename == fsnotify.Rename) {
+		if err != nil && herrors.IsNotExist(err) && (ev.Op&fsnotify.Remove == fsnotify.Remove || ev.Op&fsnotify.Rename == fsnotify.Rename) {
 			// Force keep of event
 			isRegular = true
 		}
diff --git a/magefile.go b/magefile.go
index b2dc5477723..c8f2a048e9c 100644
--- a/magefile.go
+++ b/magefile.go
@@ -268,7 +268,7 @@ func Lint() error {
 	return nil
 }
 
-//  Run go vet linter
+// Run go vet linter
 func Vet() error {
 	if err := sh.Run(goexe, "vet", "./..."); err != nil {
 		return fmt.Errorf("error running go vet: %v", err)
diff --git a/modules/client.go b/modules/client.go
index 78ba9f5aeec..953391e5934 100644
--- a/modules/client.go
+++ b/modules/client.go
@@ -29,6 +29,7 @@ import (
 	"time"
 
 	"github.com/gohugoio/hugo/common/collections"
+	"github.com/gohugoio/hugo/common/herrors"
 	"github.com/gohugoio/hugo/common/hexec"
 
 	hglob "github.com/gohugoio/hugo/hugofs/glob"
@@ -193,7 +194,8 @@ func (c *Client) Tidy() error {
 //
 // We, by default, use the /_vendor folder first, if found. To disable,
 // run with
-//    hugo --ignoreVendorPaths=".*"
+//
+//	hugo --ignoreVendorPaths=".*"
 //
 // Given a module tree, Hugo will pick the first module for a given path,
 // meaning that if the top-level module is vendored, that will be the full
@@ -297,7 +299,7 @@ func (c *Client) Vendor() error {
 		configFiles = append(configFiles, filepath.Join(dir, "theme.toml"))
 		for _, configFile := range configFiles {
 			if err := hugio.CopyFile(c.fs, configFile, filepath.Join(vendorDir, t.Path(), filepath.Base(configFile))); err != nil {
-				if !os.IsNotExist(err) {
+				if !herrors.IsNotExist(err) {
 					return err
 				}
 			}
@@ -560,7 +562,7 @@ func (c *Client) rewriteGoModRewrite(name string, isGoMod map[string]bool) ([]by
 	b := &bytes.Buffer{}
 	f, err := c.fs.Open(filepath.Join(c.ccfg.WorkingDir, name))
 	if err != nil {
-		if os.IsNotExist(err) {
+		if herrors.IsNotExist(err) {
 			// It's been deleted.
 			return nil, nil
 		}
diff --git a/modules/collect.go b/modules/collect.go
index ff83f9ecca8..7d92e30456e 100644
--- a/modules/collect.go
+++ b/modules/collect.go
@@ -23,6 +23,7 @@ import (
 	"time"
 
 	"github.com/bep/debounce"
+	"github.com/gohugoio/hugo/common/herrors"
 	"github.com/gohugoio/hugo/common/loggers"
 
 	"github.com/spf13/cast"
@@ -539,7 +540,7 @@ func (c *collector) collectModulesTXT(owner Module) error {
 
 	f, err := c.fs.Open(filename)
 	if err != nil {
-		if os.IsNotExist(err) {
+		if herrors.IsNotExist(err) {
 			return nil
 		}
 
diff --git a/resources/resource_spec.go b/resources/resource_spec.go
index fd9653012ad..13920be7e66 100644
--- a/resources/resource_spec.go
+++ b/resources/resource_spec.go
@@ -258,7 +258,7 @@ func (r *Spec) newResource(sourceFs afero.Fs, fd ResourceSourceDescriptor) (reso
 		var err error
 		fi, err = sourceFs.Stat(fd.SourceFilename)
 		if err != nil {
-			if os.IsNotExist(err) {
+			if herrors.IsNotExist(err) {
 				return nil, nil
 			}
 			return nil, err
diff --git a/tpl/tplimpl/template.go b/tpl/tplimpl/template.go
index 9a9e82a804c..c01863ebb4b 100644
--- a/tpl/tplimpl/template.go
+++ b/tpl/tplimpl/template.go
@@ -20,7 +20,6 @@ import (
 	"fmt"
 	"io"
 	"io/fs"
-	"os"
 	"path/filepath"
 	"reflect"
 	"regexp"
@@ -824,7 +823,7 @@ func (t *templateHandler) loadTemplates() error {
 	}
 
 	if err := helpers.SymbolicWalk(t.Layouts.Fs, "", walker); err != nil {
-		if !os.IsNotExist(err) {
+		if !herrors.IsNotExist(err) {
 			return err
 		}
 		return nil
diff --git a/watcher/filenotify/poller.go b/watcher/filenotify/poller.go
index 71d806209d4..7479dcbdde6 100644
--- a/watcher/filenotify/poller.go
+++ b/watcher/filenotify/poller.go
@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/fsnotify/fsnotify"
+	"github.com/gohugoio/hugo/common/herrors"
 )
 
 var (
@@ -191,7 +192,7 @@ func (r *recording) record(filename string) error {
 	r.clear()
 
 	fi, err := os.Stat(filename)
-	if err != nil && !os.IsNotExist(err) {
+	if err != nil && !herrors.IsNotExist(err) {
 		return err
 	}
 
@@ -206,7 +207,7 @@ func (r *recording) record(filename string) error {
 	if fi.IsDir() {
 		f, err := os.Open(filename)
 		if err != nil {
-			if os.IsNotExist(err) {
+			if herrors.IsNotExist(err) {
 				return nil
 			}
 			return err
@@ -215,7 +216,7 @@ func (r *recording) record(filename string) error {
 
 		fis, err := f.Readdir(-1)
 		if err != nil {
-			if os.IsNotExist(err) {
+			if herrors.IsNotExist(err) {
 				return nil
 			}
 			return err
@@ -260,7 +261,7 @@ func (item *itemToWatch) checkForChanges() ([]fsnotify.Event, error) {
 	}
 
 	err := item.right.record(item.filename)
-	if err != nil && !os.IsNotExist(err) {
+	if err != nil && !herrors.IsNotExist(err) {
 		return nil, err
 	}
 


Test patch:
diff --git a/common/herrors/errors_test.go b/common/herrors/errors_test.go
new file mode 100644
index 00000000000..1e07300284e
--- /dev/null
+++ b/common/herrors/errors_test.go
@@ -0,0 +1,36 @@
+// Copyright 2022 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package herrors
+
+import (
+	"fmt"
+	"testing"
+
+	qt "github.com/frankban/quicktest"
+	"github.com/spf13/afero"
+)
+
+func TestIsNotExist(t *testing.T) {
+	c := qt.New(t)
+
+	c.Assert(IsNotExist(afero.ErrFileNotFound), qt.Equals, true)
+	c.Assert(IsNotExist(afero.ErrFileExists), qt.Equals, false)
+	c.Assert(IsNotExist(afero.ErrDestinationExists), qt.Equals, false)
+	c.Assert(IsNotExist(nil), qt.Equals, false)
+
+	c.Assert(IsNotExist(fmt.Errorf("foo")), qt.Equals, false)
+
+	// os.IsNotExist returns false for wrapped errors.
+	c.Assert(IsNotExist(fmt.Errorf("foo: %w", afero.ErrFileNotFound)), qt.Equals, true)
+}