Repository: gohugoio/hugo (language: go)

Issue:
Fix /static and add some better end-to-end tests
I was a little puzzled that my "git revert" in `0.76.2` did not work, as I just assumed it should.
[v0.97.0] hugo server PostProcess wrong/unprocessed permalink
```html
<!doctype html>
<html lang="{{ .Lang }}">
  <head>
    {{- $options := dict "targetPath" "assets/bundle.min.css" "outputStyle" "compressed" "includePaths" (slice "node_modules") -}}
    {{- $style := resources.Get "scss/index.scss" | toCSS $options | resources.PostCSS | fingerprint | resources.PostProcess -}}
    <link rel="stylesheet" href="{{ $style.Permalink }}" integrity="{{ $style.Data.Integrity }}" crossorigin="anonymous">
  </head>
  <body>
    <h1>Hello World</h1>
  </body>
</html>
```

1. `hugo server` not working.

```text
hugo v0.97.0-c07f3626e7c8160943591f4d209977efa02c3dca+extended linux/amd64 BuildDate=2022-04-14T08:45:07Z VendorInfo=gohugoio

<link rel="stylesheet" href="__h_pp_l1_2_Permalink__e=" integrity="__h_pp_l1_2_Data.Integrity__e=" crossorigin="anonymous">
```

2. `hugo server --renderToDisk` works as expected.

```text
hugo v0.97.0-c07f3626e7c8160943591f4d209977efa02c3dca+extended linux/amd64 BuildDate=2022-04-14T08:45:07Z VendorInfo=gohugoio

<link rel="stylesheet" href="http://localhost:37979/assets/bundle.min.de5eadfd48d2043210f12189a24cb4ada1fb1f5a3d085b292a389d40dc75099f.css" integrity="sha256-3l6t/UjSBDIQ8SGJoky0raH7H1o9CFspKjidQNx1CZ8=" crossorigin="anonymous">
```

3. `hugo-v0.96.0 server` works as expected.


Discussion / hints:

This is a known limitation (I think there is an open issue) with the server. What you want to do is:

```
 {{- $style := resources.Get "scss/index.scss" | toCSS $options | resources.PostCSS -}}
{{ if hugo.IsProduction }}
  {{ $style = $style | minify | fingerprint | resources.PostProcess }}
{{ end }}
```

Or something like that.
Thanks for your reply.

I was confused by the inconsistent default behavior of `hugo server` between `0.96.0` and `0.97.0`. The former version works great, the CSS was purged, so that the final site is predictable. 

> If you’re only using this for the production build, you should consider placing it below [config/production](https://gohugo.io/getting-started/configuration/#configuration-directory). 
> ------ [PostProcess](https://gohugo.io/hugo-pipes/postprocess/)

I think it's reasonable to apply it to local environment as well.

>I was confused by the inconsistent default behavior of hugo server between 0.96.0 and 0.97.0.

OK, are you saying that this worked by default in `hugo server´ (render to memory) in earlier releases?

>I think it's reasonable to apply it to local environment as well.

Yea, well, if you're fingerprinting your CSS running the local dev server, you effectively have to regenerate every HTML page on every CSS change, which is not something I would want.
Yea, OK, I think I understand where I got this wrong, I will have a look at it. Sorry for the noise.

Fix patch:
diff --git a/commands/commandeer.go b/commands/commandeer.go
index 1162a4b7054..c42de5d11a5 100644
--- a/commands/commandeer.go
+++ b/commands/commandeer.go
@@ -395,23 +395,23 @@ func (c *commandeer) loadConfig() error {
 	}
 
 	c.fsCreate.Do(func() {
-		fs := hugofs.NewFrom(sourceFs, config)
+		// Assume both source and destination are using same filesystem.
+		fs := hugofs.NewFromSourceAndDestination(sourceFs, sourceFs, config)
 
 		if c.publishDirFs != nil {
 			// Need to reuse the destination on server rebuilds.
 			fs.PublishDir = c.publishDirFs
 			fs.PublishDirServer = c.publishDirServerFs
 		} else {
-			publishDir := config.GetString("publishDir")
-			publishDirStatic := config.GetString("publishDirStatic")
-			workingDir := config.GetString("workingDir")
-			absPublishDir := paths.AbsPathify(workingDir, publishDir)
-			absPublishDirStatic := paths.AbsPathify(workingDir, publishDirStatic)
-
 			if c.renderStaticToDisk {
-				// Writes the dynamic output oton memory,
+				publishDirStatic := config.GetString("publishDirStatic")
+				workingDir := config.GetString("workingDir")
+				absPublishDirStatic := paths.AbsPathify(workingDir, publishDirStatic)
+
+				fs = hugofs.NewFromSourceAndDestination(sourceFs, afero.NewMemMapFs(), config)
+				// Writes the dynamic output to memory,
 				// while serve others directly from /public on disk.
-				dynamicFs := afero.NewMemMapFs()
+				dynamicFs := fs.PublishDir
 				staticFs := afero.NewBasePathFs(afero.NewOsFs(), absPublishDirStatic)
 
 				// Serve from both the static and dynamic fs,
@@ -427,18 +427,10 @@ func (c *commandeer) loadConfig() error {
 						},
 					},
 				)
-				fs.PublishDir = dynamicFs
 				fs.PublishDirStatic = staticFs
 			} else if createMemFs {
 				// Hugo writes the output to memory instead of the disk.
-				fs.PublishDir = new(afero.MemMapFs)
-				fs.PublishDirServer = fs.PublishDir
-				fs.PublishDirStatic = fs.PublishDir
-			} else {
-				// Write everything to disk.
-				fs.PublishDir = afero.NewBasePathFs(afero.NewOsFs(), absPublishDir)
-				fs.PublishDirServer = fs.PublishDir
-				fs.PublishDirStatic = fs.PublishDir
+				fs = hugofs.NewFromSourceAndDestination(sourceFs, afero.NewMemMapFs(), config)
 			}
 		}
 
diff --git a/go.mod b/go.mod
index 397be1fadd2..2ec35fc1d8c 100644
--- a/go.mod
+++ b/go.mod
@@ -12,7 +12,7 @@ require (
 	github.com/bep/godartsass v0.14.0
 	github.com/bep/golibsass v1.0.0
 	github.com/bep/gowebp v0.1.0
-	github.com/bep/overlayfs v0.5.0
+	github.com/bep/overlayfs v0.6.0
 	github.com/bep/tmc v0.5.1
 	github.com/clbanning/mxj/v2 v2.5.5
 	github.com/cli/safeexec v1.0.0
diff --git a/go.sum b/go.sum
index 71eec8a9a36..d2bca3174d0 100644
--- a/go.sum
+++ b/go.sum
@@ -236,6 +236,8 @@ github.com/bep/overlayfs v0.4.0 h1:J/G5YltfU2BxO2KV/VcFzJo94jpRMjtthRNEZ+7V7uA=
 github.com/bep/overlayfs v0.4.0/go.mod h1:NFjSmn3kCqG7KX2Lmz8qT8VhPPCwZap3UNogXawoQHM=
 github.com/bep/overlayfs v0.5.0 h1:5F7xbjVhPSMv/PX5KYeCeEMR/17Fm20G2A0Phe+dUmc=
 github.com/bep/overlayfs v0.5.0/go.mod h1:NFjSmn3kCqG7KX2Lmz8qT8VhPPCwZap3UNogXawoQHM=
+github.com/bep/overlayfs v0.6.0 h1:sgLcq/qtIzbaQNl2TldGXOkHvqeZB025sPvHOQL+DYo=
+github.com/bep/overlayfs v0.6.0/go.mod h1:NFjSmn3kCqG7KX2Lmz8qT8VhPPCwZap3UNogXawoQHM=
 github.com/bep/tmc v0.5.1 h1:CsQnSC6MsomH64gw0cT5f+EwQDcvZz4AazKunFwTpuI=
 github.com/bep/tmc v0.5.1/go.mod h1:tGYHN8fS85aJPhDLgXETVKp+PR382OvFi2+q2GkGsq0=
 github.com/bep/workers v1.0.0 h1:U+H8YmEaBCEaFZBst7GcRVEoqeRC9dzH2dWOwGmOchg=
diff --git a/hugofs/fs.go b/hugofs/fs.go
index 436387f13f5..63c25a4c0d7 100644
--- a/hugofs/fs.go
+++ b/hugofs/fs.go
@@ -65,7 +65,7 @@ type Fs struct {
 // as source and destination file systems.
 func NewDefault(cfg config.Provider) *Fs {
 	fs := Os
-	return newFs(fs, cfg)
+	return newFs(fs, fs, cfg)
 }
 
 // NewMem creates a new Fs with the MemMapFs
@@ -73,17 +73,23 @@ func NewDefault(cfg config.Provider) *Fs {
 // Useful for testing.
 func NewMem(cfg config.Provider) *Fs {
 	fs := &afero.MemMapFs{}
-	return newFs(fs, cfg)
+	return newFs(fs, fs, cfg)
 }
 
 // NewFrom creates a new Fs based on the provided Afero Fs
 // as source and destination file systems.
 // Useful for testing.
 func NewFrom(fs afero.Fs, cfg config.Provider) *Fs {
-	return newFs(fs, cfg)
+	return newFs(fs, fs, cfg)
 }
 
-func newFs(base afero.Fs, cfg config.Provider) *Fs {
+// NewFrom creates a new Fs based on the provided Afero Fss
+// as the source and destination file systems.
+func NewFromSourceAndDestination(source, destination afero.Fs, cfg config.Provider) *Fs {
+	return newFs(source, destination, cfg)
+}
+
+func newFs(source, destination afero.Fs, cfg config.Provider) *Fs {
 	workingDir := cfg.GetString("workingDir")
 	publishDir := cfg.GetString("publishDir")
 	if publishDir == "" {
@@ -91,27 +97,27 @@ func newFs(base afero.Fs, cfg config.Provider) *Fs {
 	}
 
 	// Sanity check
-	if IsOsFs(base) && len(workingDir) < 2 {
+	if IsOsFs(source) && len(workingDir) < 2 {
 		panic("workingDir is too short")
 	}
 
 	absPublishDir := paths.AbsPathify(workingDir, publishDir)
 
 	// Make sure we always have the /public folder ready to use.
-	if err := base.MkdirAll(absPublishDir, 0777); err != nil && !os.IsExist(err) {
+	if err := source.MkdirAll(absPublishDir, 0777); err != nil && !os.IsExist(err) {
 		panic(err)
 	}
 
-	pubFs := afero.NewBasePathFs(base, absPublishDir)
+	pubFs := afero.NewBasePathFs(destination, absPublishDir)
 
 	return &Fs{
-		Source:             base,
+		Source:             source,
 		PublishDir:         pubFs,
 		PublishDirServer:   pubFs,
 		PublishDirStatic:   pubFs,
 		Os:                 &afero.OsFs{},
-		WorkingDirReadOnly: getWorkingDirFsReadOnly(base, workingDir),
-		WorkingDirWritable: getWorkingDirFsWritable(base, workingDir),
+		WorkingDirReadOnly: getWorkingDirFsReadOnly(source, workingDir),
+		WorkingDirWritable: getWorkingDirFsWritable(source, workingDir),
 	}
 }
 


Test patch:
diff --git a/commands/commands_test.go b/commands/commands_test.go
index e3ec7bd99a6..97d81ec6ee9 100644
--- a/commands/commands_test.go
+++ b/commands/commands_test.go
@@ -375,6 +375,10 @@ Single: {{ .Title }}
 List: {{ .Title }}
 Environment: {{ hugo.Environment }}
 
+For issue 9788:
+{{ $foo :="abc" | resources.FromString "foo.css" | minify | resources.PostProcess }}
+PostProcess: {{ $foo.RelPermalink }}
+
 `)
 
 	return dir
diff --git a/commands/hugo_test.go b/commands/hugo_test.go
index aca3de2fd2b..bdad25cfcd2 100644
--- a/commands/hugo_test.go
+++ b/commands/hugo_test.go
@@ -14,32 +14,138 @@
 package commands
 
 import (
+	"bytes"
+	"fmt"
+	"math/rand"
+	"path/filepath"
+	"strings"
 	"testing"
 
 	qt "github.com/frankban/quicktest"
+	"github.com/gohugoio/hugo/hugofs"
+	"github.com/spf13/afero"
+	"golang.org/x/tools/txtar"
 )
 
 // Issue #5662
 func TestHugoWithContentDirOverride(t *testing.T) {
+	t.Parallel()
 	c := qt.New(t)
 
-	hugoCmd := newCommandsBuilder().addAll().build()
-	cmd := hugoCmd.getCommand()
+	files := `
+-- config.toml --
+baseURL = "https://example.org"
+title = "Hugo Commands"
+-- mycontent/p1.md --
+---
+title: "P1"
+---
+-- layouts/_default/single.html --
+Page: {{ .Title }}|
 
-	contentDir := "contentOverride"
+`
+	s := newTestHugoCmdBuilder(c, files, []string{"-c", "mycontent"}).Build()
+	s.AssertFileContent("public/p1/index.html", `Page: P1|`)
 
-	cfgStr := `
+}
 
-baseURL = "https://example.org"
-title = "Hugo Commands"
+// Issue #9794
+func TestHugoStaticFilesMultipleStaticAndManyFolders(t *testing.T) {
+	t.Parallel()
+	c := qt.New(t)
 
-contentDir = "thisdoesnotexist"
+	files := `
+-- config.toml --
+baseURL = "https://example.org"
+theme = "mytheme"
+-- layouts/index.html --
+Home.
 
 `
-	dir := createSimpleTestSite(t, testSiteConfig{configTOML: cfgStr, contentDir: contentDir})
+	const (
+		numDirs     = 33
+		numFilesMax = 12
+	)
+
+	r := rand.New(rand.NewSource(32))
+
+	for i := 0; i < numDirs; i++ {
+		for j := 0; j < r.Intn(numFilesMax); j++ {
+			if j%3 == 0 {
+				files += fmt.Sprintf("-- themes/mytheme/static/d%d/f%d.txt --\nHellot%d-%d\n", i, j, i, j)
+				files += fmt.Sprintf("-- themes/mytheme/static/d%d/ft%d.txt --\nHellot%d-%d\n", i, j, i, j)
+			}
+			files += fmt.Sprintf("-- static/d%d/f%d.txt --\nHello%d-%d\n", i, j, i, j)
+		}
+	}
 
-	cmd.SetArgs([]string{"-s=" + dir, "-c=" + contentDir})
+	r = rand.New(rand.NewSource(32))
+
+	s := newTestHugoCmdBuilder(c, files, []string{"-c", "mycontent"}).Build()
+	for i := 0; i < numDirs; i++ {
+		for j := 0; j < r.Intn(numFilesMax); j++ {
+			if j%3 == 0 {
+				if j%3 == 0 {
+					s.AssertFileContent(fmt.Sprintf("public/d%d/ft%d.txt", i, j), fmt.Sprintf("Hellot%d-%d", i, j))
+				}
+				s.AssertFileContent(fmt.Sprintf("public/d%d/f%d.txt", i, j), fmt.Sprintf("Hello%d-%d", i, j))
+			}
+		}
+	}
+
+}
+
+type testHugoCmdBuilder struct {
+	*qt.C
+
+	fs    afero.Fs
+	dir   string
+	files string
+	args  []string
+}
+
+func newTestHugoCmdBuilder(c *qt.C, files string, args []string) *testHugoCmdBuilder {
+	s := &testHugoCmdBuilder{C: c, files: files, args: args}
+	s.dir = s.TempDir()
+	s.fs = afero.NewBasePathFs(hugofs.Os, s.dir)
+
+	return s
+}
+
+func (s *testHugoCmdBuilder) Build() *testHugoCmdBuilder {
+	data := txtar.Parse([]byte(s.files))
+
+	for _, f := range data.Files {
+		filename := filepath.Clean(f.Name)
+		data := bytes.TrimSuffix(f.Data, []byte("\n"))
+		s.Assert(s.fs.MkdirAll(filepath.Dir(filename), 0777), qt.IsNil)
+		s.Assert(afero.WriteFile(s.fs, filename, data, 0666), qt.IsNil)
+	}
+
+	hugoCmd := newCommandsBuilder().addAll().build()
+	cmd := hugoCmd.getCommand()
+	args := append(s.args, "-s="+s.dir, "--quiet")
+	cmd.SetArgs(args)
 
 	_, err := cmd.ExecuteC()
-	c.Assert(err, qt.IsNil)
+	s.Assert(err, qt.IsNil)
+
+	return s
+}
+
+func (s *testHugoCmdBuilder) AssertFileContent(filename string, matches ...string) {
+	s.Helper()
+	data, err := afero.ReadFile(s.fs, filename)
+	s.Assert(err, qt.IsNil)
+	content := strings.TrimSpace(string(data))
+	for _, m := range matches {
+		lines := strings.Split(m, "\n")
+		for _, match := range lines {
+			match = strings.TrimSpace(match)
+			if match == "" || strings.HasPrefix(match, "#") {
+				continue
+			}
+			s.Assert(content, qt.Contains, match, qt.Commentf(m))
+		}
+	}
 }
diff --git a/commands/server_test.go b/commands/server_test.go
index ea50afd94d2..c2aa0dfd5d7 100644
--- a/commands/server_test.go
+++ b/commands/server_test.go
@@ -31,16 +31,6 @@ import (
 	qt "github.com/frankban/quicktest"
 )
 
-func TestServer(t *testing.T) {
-	c := qt.New(t)
-
-	r := runServerTest(c, true, "")
-
-	c.Assert(r.err, qt.IsNil)
-	c.Assert(r.homeContent, qt.Contains, "List: Hugo Commands")
-	c.Assert(r.homeContent, qt.Contains, "Environment: development")
-}
-
 // Issue 9518
 func TestServerPanicOnConfigError(t *testing.T) {
 	c := qt.New(t)
@@ -101,6 +91,42 @@ baseURL="https://example.org"
 
 }
 
+func TestServerBugs(t *testing.T) {
+	c := qt.New(t)
+
+	for _, test := range []struct {
+		name   string
+		flag   string
+		assert func(c *qt.C, r serverTestResult)
+	}{
+		// Issue 9788
+		{"PostProcess, memory", "", func(c *qt.C, r serverTestResult) {
+			c.Assert(r.err, qt.IsNil)
+			c.Assert(r.homeContent, qt.Contains, "PostProcess: /foo.min.css")
+		}},
+		{"PostProcess, disk", "--renderToDisk", func(c *qt.C, r serverTestResult) {
+			c.Assert(r.err, qt.IsNil)
+			c.Assert(r.homeContent, qt.Contains, "PostProcess: /foo.min.css")
+		}},
+	} {
+		c.Run(test.name, func(c *qt.C) {
+			config := `
+baseURL="https://example.org"
+`
+
+			var args []string
+			if test.flag != "" {
+				args = strings.Split(test.flag, "=")
+			}
+			r := runServerTest(c, true, config, args...)
+			test.assert(c, r)
+
+		})
+
+	}
+
+}
+
 type serverTestResult struct {
 	err            error
 	homeContent    string