Repository: gohugoio/hugo (language: go)

Issue:
syncing static files seems 'slow' since 103.0 macOS
Hey there, since 0.103.0 I noticed a significant 'slowdown' in syncing static files on macOS (hugo_extended_0.103.0_darwin-universal). The slowdown appears when public is empty so *only the initial* `hugo` call is affected. After that it's fast again (makes sense) but the *problem* persists for every `hugo server` call which went up from seconds to minutes since 0.102.3. I was wondering if this is intended (or expected) behaviour and I have to change my project or if there is something wrong?

Here is a quick overview of my os, project and the timings I get with 0.103.0 vs. 0.102.3 (104 is ~same as 103):

I run a macbook Pro M1 with 12.6 Monterey. The project looks as follows and the resulting public folder is ~388MB (161MB videos + 147MB images). Since 103, the syncing of static files is very slow in comparison (checked with --debug, it's them):

```
                   | EN
-------------------+------
  Pages            | 181
  Paginator pages  |   0
  Non-page files   |   0
  Static files     | 935
  Processed images |   0
  Aliases          |   0
  Sitemaps         |   1
  Cleaned          |   0
```

The timings have changed as follows:

```
# hugo_extended_0.103.0_darwin-universal:
hugo (initial call)    -> Total in 347228 ms
hugo ('cached' public) -> Total in   1281 ms
hugo server            -> Built in 347726 ms

# hugo_extended_0.102.3_macOS-universal:
hugo (initial call)    -> Total in  673 ms
hugo ('cached' public) -> Total in  659 ms
hugo server            -> Built in 1825 ms
```

Thanks in advance and let me know if I can or should provide more info on this - Johannes


Discussion / hints:

Fix patch:
diff --git a/commands/hugo.go b/commands/hugo.go
index d127d37212f..e26f052d451 100644
--- a/commands/hugo.go
+++ b/commands/hugo.go
@@ -653,10 +653,7 @@ func (c *commandeer) copyStaticTo(sourceFs *filesystems.SourceFilesystem) (uint6
 	syncer.NoChmod = c.Cfg.GetBool("noChmod")
 	syncer.ChmodFilter = chmodFilter
 	syncer.SrcFs = fs
-	syncer.DestFs = c.Fs.PublishDir
-	if c.renderStaticToDisk {
-		syncer.DestFs = c.Fs.PublishDirStatic
-	}
+	syncer.DestFs = c.Fs.PublishDirStatic
 	// Now that we are using a unionFs for the static directories
 	// We can effectively clean the publishDir on initial sync
 	syncer.Delete = c.Cfg.GetBool("cleanDestinationDir")
diff --git a/deps/deps.go b/deps/deps.go
index e1cbfce069e..02730e825a0 100644
--- a/deps/deps.go
+++ b/deps/deps.go
@@ -2,6 +2,8 @@ package deps
 
 import (
 	"fmt"
+	"path/filepath"
+	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -246,16 +248,30 @@ func New(cfg DepsCfg) (*Deps, error) {
 	execHelper := hexec.New(securityConfig)
 
 	var filenameHasPostProcessPrefixMu sync.Mutex
-	cb := func(name string, match bool) {
+	hashBytesReceiverFunc := func(name string, match bool) {
 		if !match {
 			return
 		}
 		filenameHasPostProcessPrefixMu.Lock()
 		d.FilenameHasPostProcessPrefix = append(d.FilenameHasPostProcessPrefix, name)
 		filenameHasPostProcessPrefixMu.Unlock()
+	}
 
+	// Skip binary files.
+	hashBytesSHouldCheck := func(name string) bool {
+		ext := strings.TrimPrefix(filepath.Ext(name), ".")
+		mime, _, found := cfg.MediaTypes.GetBySuffix(ext)
+		if !found {
+			return false
+		}
+		switch mime.MainType {
+		case "text", "application":
+			return true
+		default:
+			return false
+		}
 	}
-	fs.PublishDir = hugofs.NewHasBytesReceiver(fs.PublishDir, cb, []byte(postpub.PostProcessPrefix))
+	fs.PublishDir = hugofs.NewHasBytesReceiver(fs.PublishDir, hashBytesSHouldCheck, hashBytesReceiverFunc, []byte(postpub.PostProcessPrefix))
 
 	ps, err := helpers.NewPathSpec(fs, cfg.Language, logger)
 	if err != nil {
diff --git a/hugofs/fs.go b/hugofs/fs.go
index 63c25a4c0d7..51bbe061925 100644
--- a/hugofs/fs.go
+++ b/hugofs/fs.go
@@ -40,8 +40,7 @@ type Fs struct {
 	// It's mounted inside publishDir (default /public).
 	PublishDir afero.Fs
 
-	// PublishDirStatic is the file system used for static files  when --renderStaticToDisk is set.
-	// When this is set, PublishDir is set to write to memory.
+	// PublishDirStatic is the file system used for static files.
 	PublishDirStatic afero.Fs
 
 	// PublishDirServer is the file system used for serving the public directory with Hugo's development server.
@@ -142,7 +141,6 @@ func isWrite(flag int) bool {
 // MakeReadableAndRemoveAllModulePkgDir makes any subdir in dir readable and then
 // removes the root.
 // TODO(bep) move this to a more suitable place.
-//
 func MakeReadableAndRemoveAllModulePkgDir(fs afero.Fs, dir string) (int, error) {
 	// Safe guard
 	if !strings.Contains(dir, "pkg") {
diff --git a/hugofs/hasbytes_fs.go b/hugofs/hasbytes_fs.go
index b5f82877e8c..3d32a828f1c 100644
--- a/hugofs/hasbytes_fs.go
+++ b/hugofs/hasbytes_fs.go
@@ -27,12 +27,13 @@ var (
 
 type hasBytesFs struct {
 	afero.Fs
+	shouldCheck      func(name string) bool
 	hasBytesCallback func(name string, match bool)
 	pattern          []byte
 }
 
-func NewHasBytesReceiver(delegate afero.Fs, hasBytesCallback func(name string, match bool), pattern []byte) afero.Fs {
-	return &hasBytesFs{Fs: delegate, hasBytesCallback: hasBytesCallback, pattern: pattern}
+func NewHasBytesReceiver(delegate afero.Fs, shouldCheck func(name string) bool, hasBytesCallback func(name string, match bool), pattern []byte) afero.Fs {
+	return &hasBytesFs{Fs: delegate, shouldCheck: shouldCheck, hasBytesCallback: hasBytesCallback, pattern: pattern}
 }
 
 func (fs *hasBytesFs) UnwrapFilesystem() afero.Fs {
@@ -56,6 +57,9 @@ func (fs *hasBytesFs) OpenFile(name string, flag int, perm os.FileMode) (afero.F
 }
 
 func (fs *hasBytesFs) wrapFile(f afero.File) afero.File {
+	if !fs.shouldCheck(f.Name()) {
+		return f
+	}
 	return &hasBytesFile{
 		File: f,
 		hbw: &hugio.HasBytesWriter{
diff --git a/hugolib/filesystems/basefs.go b/hugolib/filesystems/basefs.go
index a380857cddd..e0fed6f3e6b 100644
--- a/hugolib/filesystems/basefs.go
+++ b/hugolib/filesystems/basefs.go
@@ -67,7 +67,7 @@ type BaseFs struct {
 	// This usually maps to /my-project/public.
 	PublishFs afero.Fs
 
-	// The filesystem used for renderStaticToDisk.
+	// The filesystem used for static files.
 	PublishFsStatic afero.Fs
 
 	// A read-only filesystem starting from the project workDir.
diff --git a/hugolib/pages_process.go b/hugolib/pages_process.go
index 04ac0218ad4..196a566f095 100644
--- a/hugolib/pages_process.go
+++ b/hugolib/pages_process.go
@@ -32,10 +32,9 @@ func newPagesProcessor(h *HugoSites, sp *source.SourceSpec) *pagesProcessor {
 	procs := make(map[string]pagesCollectorProcessorProvider)
 	for _, s := range h.Sites {
 		procs[s.Lang()] = &sitePagesProcessor{
-			m:                  s.pageMap,
-			errorSender:        s.h,
-			itemChan:           make(chan interface{}, config.GetNumWorkerMultiplier()*2),
-			renderStaticToDisk: h.Cfg.GetBool("renderStaticToDisk"),
+			m:           s.pageMap,
+			errorSender: s.h,
+			itemChan:    make(chan interface{}, config.GetNumWorkerMultiplier()*2),
 		}
 	}
 	return &pagesProcessor{
@@ -118,8 +117,6 @@ type sitePagesProcessor struct {
 	ctx       context.Context
 	itemChan  chan any
 	itemGroup *errgroup.Group
-
-	renderStaticToDisk bool
 }
 
 func (p *sitePagesProcessor) Process(item any) error {
@@ -164,10 +161,7 @@ func (p *sitePagesProcessor) copyFile(fim hugofs.FileMetaInfo) error {
 
 	defer f.Close()
 
-	fs := s.PublishFs
-	if p.renderStaticToDisk {
-		fs = s.PublishFsStatic
-	}
+	fs := s.PublishFsStatic
 
 	return s.publish(&s.PathSpec.ProcessingStats.Files, target, f, fs)
 }


Test patch: