Repository: gohugoio/hugo (language: go)

Issue:
resources.GetRemote HEAD method errors with unexpected EOF
Reference: <https://discourse.gohugo.io/t/42272/3>

When I do this:
```
{{ $url := "https://gohugo.io/img/hugo.png" }}
{{ $opts := dict "method" "head" }}
{{ with resources.GetRemote $url $opts }}
  {{ with .Err }}
    {{ errorf "Unable to get remote resource: %s" . }}
  {{ else }}
    {{ .Content }}
  {{ end }}
{{ else }}
  {{ errorf "Unable to get remote resource: %s" $url }}
{{ end }}
```

I get:

> Unable to get remote resource: error calling resources.GetRemote: failed to read remote resource “https://gohugo.io/img/hugo.png”: unexpected EOF

But the response _is_ captured and cached in:
`/tmp/hugo_cache/my-project/filecache/getresource/6247740232174893946`


Discussion / hints:
cc: @vanbroup 
A quick code scan, we should check if the response body is `nil` bore reading it, as a HEAD response doesn't contain a body.

https://github.com/gohugoio/hugo/blob/master/resources/resource_factories/create/remote.go#L138

should check if `res.Body` is `nil`.

I can create a PR to fix and test this next week.


Fix patch:
diff --git a/docs/content/en/hugo-pipes/introduction.md b/docs/content/en/hugo-pipes/introduction.md
index c16572e4884..116523fd501 100755
--- a/docs/content/en/hugo-pipes/introduction.md
+++ b/docs/content/en/hugo-pipes/introduction.md
@@ -50,6 +50,20 @@ With `resources.GetRemote`, the first argument is a remote URL:
 
 `resources.Get` and `resources.GetRemote` return `nil` if the resource is not found.
 
+{{< new-in "0.110.0" >}} You can get information about the HTTP Response using `.Data` in the returned `Resource`. This is especially useful for HEAD request without any body. The Data object contains:
+
+StatusCode
+: The HTTP status code, e.g. 200
+Status
+: The HTTP status text, e.g. "200 OK"
+TransferEncoding
+: The transfer encoding, e.g. "chunked"
+ContentLength
+: The content length, e.g. 1234
+ContentType
+: The content type, e.g. "text/html"
+
+
 ## Copy a Resource
 
 {{< new-in "0.100.0" >}}
diff --git a/media/mediaType.go b/media/mediaType.go
index cdfb1c6542b..5732b9030c9 100644
--- a/media/mediaType.go
+++ b/media/mediaType.go
@@ -117,7 +117,7 @@ func FromContent(types Types, extensionHints []string, content []byte) Type {
 
 // FromStringAndExt creates a Type from a MIME string and a given extension.
 func FromStringAndExt(t, ext string) (Type, error) {
-	tp, err := fromString(t)
+	tp, err := FromString(t)
 	if err != nil {
 		return tp, err
 	}
@@ -129,7 +129,7 @@ func FromStringAndExt(t, ext string) (Type, error) {
 
 // FromString creates a new Type given a type string on the form MainType/SubType and
 // an optional suffix, e.g. "text/html" or "text/html+html".
-func fromString(t string) (Type, error) {
+func FromString(t string) (Type, error) {
 	t = strings.ToLower(t)
 	parts := strings.Split(t, "/")
 	if len(parts) != 2 {
@@ -470,7 +470,7 @@ func DecodeTypes(mms ...map[string]any) (Types, error) {
 			mediaType, found := mmm[k]
 			if !found {
 				var err error
-				mediaType, err = fromString(k)
+				mediaType, err = FromString(k)
 				if err != nil {
 					return m, err
 				}
diff --git a/resources/resource.go b/resources/resource.go
index fd60fd4f6e1..0d7d1d85a57 100644
--- a/resources/resource.go
+++ b/resources/resource.go
@@ -70,6 +70,8 @@ type ResourceSourceDescriptor struct {
 
 	Fs afero.Fs
 
+	Data map[string]any
+
 	// Set when its known up front, else it's resolved from the target filename.
 	MediaType media.Type
 
diff --git a/resources/resource_factories/create/remote.go b/resources/resource_factories/create/remote.go
index 51199dc935b..1ae5e095b00 100644
--- a/resources/resource_factories/create/remote.go
+++ b/resources/resource_factories/create/remote.go
@@ -45,7 +45,29 @@ type HTTPError struct {
 	Body       string
 }
 
-func toHTTPError(err error, res *http.Response) *HTTPError {
+func responseToData(res *http.Response, readBody bool) map[string]any {
+	var body []byte
+	if readBody {
+		body, _ = ioutil.ReadAll(res.Body)
+	}
+
+	m := map[string]any{
+		"StatusCode":       res.StatusCode,
+		"Status":           res.Status,
+		"TransferEncoding": res.TransferEncoding,
+		"ContentLength":    res.ContentLength,
+		"ContentType":      res.Header.Get("Content-Type"),
+	}
+
+	if readBody {
+		m["Body"] = string(body)
+	}
+
+	return m
+
+}
+
+func toHTTPError(err error, res *http.Response, readBody bool) *HTTPError {
 	if err == nil {
 		panic("err is nil")
 	}
@@ -56,19 +78,9 @@ func toHTTPError(err error, res *http.Response) *HTTPError {
 		}
 	}
 
-	var body []byte
-	body, _ = ioutil.ReadAll(res.Body)
-
 	return &HTTPError{
 		error: err,
-		Data: map[string]any{
-			"StatusCode":       res.StatusCode,
-			"Status":           res.Status,
-			"Body":             string(body),
-			"TransferEncoding": res.TransferEncoding,
-			"ContentLength":    res.ContentLength,
-			"ContentType":      res.Header.Get("Content-Type"),
-		},
+		Data:  responseToData(res, readBody),
 	}
 }
 
@@ -80,6 +92,12 @@ func (c *Client) FromRemote(uri string, optionsm map[string]any) (resource.Resou
 		return nil, fmt.Errorf("failed to parse URL for resource %s: %w", uri, err)
 	}
 
+	method := "GET"
+	if s, ok := maps.LookupEqualFold(optionsm, "method"); ok {
+		method = strings.ToUpper(s.(string))
+	}
+	isHeadMethod := method == "HEAD"
+
 	resourceID := calculateResourceID(uri, optionsm)
 
 	_, httpResponse, err := c.cacheGetResource.GetOrCreate(resourceID, func() (io.ReadCloser, error) {
@@ -91,29 +109,25 @@ func (c *Client) FromRemote(uri string, optionsm map[string]any) (resource.Resou
 			return nil, err
 		}
 
-		req, err := http.NewRequest(options.Method, uri, options.BodyReader())
+		req, err := options.NewRequest(uri)
 		if err != nil {
 			return nil, fmt.Errorf("failed to create request for resource %s: %w", uri, err)
 		}
-		addDefaultHeaders(req)
-
-		if options.Headers != nil {
-			addUserProvidedHeaders(options.Headers, req)
-		}
 
 		res, err := c.httpClient.Do(req)
 		if err != nil {
 			return nil, err
 		}
+		defer res.Body.Close()
 
 		httpResponse, err := httputil.DumpResponse(res, true)
 		if err != nil {
-			return nil, toHTTPError(err, res)
+			return nil, toHTTPError(err, res, !isHeadMethod)
 		}
 
 		if res.StatusCode != http.StatusNotFound {
 			if res.StatusCode < 200 || res.StatusCode > 299 {
-				return nil, toHTTPError(fmt.Errorf("failed to fetch remote resource: %s", http.StatusText(res.StatusCode)), res)
+				return nil, toHTTPError(fmt.Errorf("failed to fetch remote resource: %s", http.StatusText(res.StatusCode)), res, !isHeadMethod)
 
 			}
 		}
@@ -129,15 +143,24 @@ func (c *Client) FromRemote(uri string, optionsm map[string]any) (resource.Resou
 	if err != nil {
 		return nil, err
 	}
+	defer res.Body.Close()
 
 	if res.StatusCode == http.StatusNotFound {
 		// Not found. This matches how looksup for local resources work.
 		return nil, nil
 	}
 
-	body, err := ioutil.ReadAll(res.Body)
-	if err != nil {
-		return nil, fmt.Errorf("failed to read remote resource %q: %w", uri, err)
+	var (
+		body      []byte
+		mediaType media.Type
+	)
+	// A response to a HEAD method should not have a body. If it has one anyway, that body must be ignored.
+	// See https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/HEAD
+	if !isHeadMethod && res.Body != nil {
+		body, err = ioutil.ReadAll(res.Body)
+		if err != nil {
+			return nil, fmt.Errorf("failed to read remote resource %q: %w", uri, err)
+		}
 	}
 
 	filename := path.Base(rURL.Path)
@@ -147,39 +170,49 @@ func (c *Client) FromRemote(uri string, optionsm map[string]any) (resource.Resou
 		}
 	}
 
-	var extensionHints []string
-
 	contentType := res.Header.Get("Content-Type")
 
-	// mime.ExtensionsByType gives a long list of extensions for text/plain,
-	// just use ".txt".
-	if strings.HasPrefix(contentType, "text/plain") {
-		extensionHints = []string{".txt"}
+	if isHeadMethod {
+		// We have no body to work with, so we need to use the Content-Type header.
+		mediaType, _ = media.FromString(contentType)
 	} else {
-		exts, _ := mime.ExtensionsByType(contentType)
-		if exts != nil {
-			extensionHints = exts
+
+		var extensionHints []string
+
+		// mime.ExtensionsByType gives a long list of extensions for text/plain,
+		// just use ".txt".
+		if strings.HasPrefix(contentType, "text/plain") {
+			extensionHints = []string{".txt"}
+		} else {
+			exts, _ := mime.ExtensionsByType(contentType)
+			if exts != nil {
+				extensionHints = exts
+			}
 		}
-	}
 
-	// Look for a file extension. If it's .txt, look for a more specific.
-	if extensionHints == nil || extensionHints[0] == ".txt" {
-		if ext := path.Ext(filename); ext != "" {
-			extensionHints = []string{ext}
+		// Look for a file extension. If it's .txt, look for a more specific.
+		if extensionHints == nil || extensionHints[0] == ".txt" {
+			if ext := path.Ext(filename); ext != "" {
+				extensionHints = []string{ext}
+			}
 		}
+
+		// Now resolve the media type primarily using the content.
+		mediaType = media.FromContent(c.rs.MediaTypes, extensionHints, body)
+
 	}
 
-	// Now resolve the media type primarily using the content.
-	mediaType := media.FromContent(c.rs.MediaTypes, extensionHints, body)
 	if mediaType.IsZero() {
 		return nil, fmt.Errorf("failed to resolve media type for remote resource %q", uri)
 	}
 
 	resourceID = filename[:len(filename)-len(path.Ext(filename))] + "_" + resourceID + mediaType.FirstSuffix.FullSuffix
+	data := responseToData(res, false)
 
 	return c.rs.New(
 		resources.ResourceSourceDescriptor{
 			MediaType:   mediaType,
+			Data:        data,
 			LazyPublish: true,
 			OpenReadSeekCloser: func() (hugio.ReadSeekCloser, error) {
 				return hugio.NewReadSeekerNoOpCloser(bytes.NewReader(body)), nil
@@ -207,12 +240,7 @@ func calculateResourceID(uri string, optionsm map[string]any) string {
 	return helpers.HashString(uri, optionsm)
 }
 
-func addDefaultHeaders(req *http.Request, accepts ...string) {
-	for _, accept := range accepts {
-		if !hasHeaderValue(req.Header, "Accept", accept) {
-			req.Header.Add("Accept", accept)
-		}
-	}
+func addDefaultHeaders(req *http.Request) {
 	if !hasHeaderKey(req.Header, "User-Agent") {
 		req.Header.Add("User-Agent", "Hugo Static Site Generator")
 	}
@@ -264,6 +292,23 @@ func (o fromRemoteOptions) BodyReader() io.Reader {
 	return bytes.NewBuffer(o.Body)
 }
 
+func (o fromRemoteOptions) NewRequest(url string) (*http.Request, error) {
+	req, err := http.NewRequest(o.Method, url, o.BodyReader())
+	if err != nil {
+		return nil, err
+	}
+
+	// First add any user provided headers.
+	if o.Headers != nil {
+		addUserProvidedHeaders(o.Headers, req)
+	}
+
+	// Then add default headers not provided by the user.
+	addDefaultHeaders(req)
+
+	return req, nil
+}
+
 func decodeRemoteOptions(optionsm map[string]any) (fromRemoteOptions, error) {
 	options := fromRemoteOptions{
 		Method: "GET",
diff --git a/resources/resource_spec.go b/resources/resource_spec.go
index 13920be7e66..8ef69318362 100644
--- a/resources/resource_spec.go
+++ b/resources/resource_spec.go
@@ -192,6 +192,7 @@ func (r *Spec) newGenericResource(sourceFs afero.Fs,
 		sourceFilename,
 		baseFilename,
 		mediaType,
+		nil,
 	)
 }
 
@@ -203,7 +204,9 @@ func (r *Spec) newGenericResourceWithBase(
 	osFileInfo os.FileInfo,
 	sourceFilename,
 	baseFilename string,
-	mediaType media.Type) *genericResource {
+	mediaType media.Type,
+	data map[string]any,
+) *genericResource {
 	if osFileInfo != nil && osFileInfo.IsDir() {
 		panic(fmt.Sprintf("dirs not supported resource types: %v", osFileInfo))
 	}
@@ -244,6 +247,7 @@ func (r *Spec) newGenericResourceWithBase(
 		name:                   baseFilename,
 		title:                  baseFilename,
 		resourceContent:        &resourceContent{},
+		data:                   data,
 	}
 
 	return g
@@ -305,7 +309,8 @@ func (r *Spec) newResource(sourceFs afero.Fs, fd ResourceSourceDescriptor) (reso
 		fi,
 		sourceFilename,
 		fd.RelTargetFilename,
-		mimeType)
+		mimeType,
+		fd.Data)
 
 	if mimeType.MainType == "image" {
 		imgFormat, ok := images.ImageFormatFromMediaSubType(mimeType.SubType)


Test patch:
diff --git a/media/mediaType_test.go b/media/mediaType_test.go
index 2a1b4884999..3d12c31bb69 100644
--- a/media/mediaType_test.go
+++ b/media/mediaType_test.go
@@ -132,22 +132,22 @@ func TestGetFirstBySuffix(t *testing.T) {
 
 func TestFromTypeString(t *testing.T) {
 	c := qt.New(t)
-	f, err := fromString("text/html")
+	f, err := FromString("text/html")
 	c.Assert(err, qt.IsNil)
 	c.Assert(f.Type(), qt.Equals, HTMLType.Type())
 
-	f, err = fromString("application/custom")
+	f, err = FromString("application/custom")
 	c.Assert(err, qt.IsNil)
 	c.Assert(f, qt.Equals, Type{MainType: "application", SubType: "custom", mimeSuffix: ""})
 
-	f, err = fromString("application/custom+sfx")
+	f, err = FromString("application/custom+sfx")
 	c.Assert(err, qt.IsNil)
 	c.Assert(f, qt.Equals, Type{MainType: "application", SubType: "custom", mimeSuffix: "sfx"})
 
-	_, err = fromString("noslash")
+	_, err = FromString("noslash")
 	c.Assert(err, qt.Not(qt.IsNil))
 
-	f, err = fromString("text/xml; charset=utf-8")
+	f, err = FromString("text/xml; charset=utf-8")
 	c.Assert(err, qt.IsNil)
 
 	c.Assert(f, qt.Equals, Type{MainType: "text", SubType: "xml", mimeSuffix: ""})
diff --git a/resources/resource_factories/create/integration_test.go b/resources/resource_factories/create/integration_test.go
new file mode 100644
index 00000000000..2d9a700d39d
--- /dev/null
+++ b/resources/resource_factories/create/integration_test.go
@@ -0,0 +1,59 @@
+// Copyright 2023 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package create_test
+
+import (
+	"testing"
+
+	"github.com/gohugoio/hugo/hugolib"
+)
+
+func TestGetResourceHead(t *testing.T) {
+
+	files := `
+-- config.toml --
+[security]
+  [security.http]
+    methods = ['(?i)GET|POST|HEAD']
+    urls = ['.*gohugo\.io.*']
+
+-- layouts/index.html --
+{{ $url := "https://gohugo.io/img/hugo.png" }}
+{{ $opts := dict "method" "head" }}
+{{ with resources.GetRemote $url $opts }}
+  {{ with .Err }}
+    {{ errorf "Unable to get remote resource: %s" . }}
+  {{ else }}
+    Head Content: {{ .Content }}. Head Data: {{ .Data }}
+  {{ end }}
+{{ else }}
+  {{ errorf "Unable to get remote resource: %s" $url }}
+{{ end }}
+`
+
+	b := hugolib.NewIntegrationTestBuilder(
+		hugolib.IntegrationTestConfig{
+			T:           t,
+			TxtarString: files,
+		},
+	)
+
+	b.Build()
+
+	b.AssertFileContent("public/index.html",
+		"Head Content: .",
+		"Head Data: map[ContentLength:18210 ContentType:image/png Status:200 OK StatusCode:200 TransferEncoding:[]]",
+	)
+
+}
diff --git a/resources/resource_factories/create/remote_test.go b/resources/resource_factories/create/remote_test.go
index c2a3b7b3275..a443a063394 100644
--- a/resources/resource_factories/create/remote_test.go
+++ b/resources/resource_factories/create/remote_test.go
@@ -20,6 +20,8 @@ import (
 )
 
 func TestDecodeRemoteOptions(t *testing.T) {
+	t.Parallel()
+
 	c := qt.New(t)
 
 	for _, test := range []struct {
@@ -81,10 +83,43 @@ func TestDecodeRemoteOptions(t *testing.T) {
 		})
 
 	}
+}
+
+func TestOptionsNewRequest(t *testing.T) {
+	t.Parallel()
+
+	c := qt.New(t)
+
+	opts := fromRemoteOptions{
+		Method: "GET",
+		Body:   []byte("foo"),
+	}
+
+	req, err := opts.NewRequest("https://example.com/api")
+
+	c.Assert(err, qt.IsNil)
+	c.Assert(req.Method, qt.Equals, "GET")
+	c.Assert(req.Header["User-Agent"], qt.DeepEquals, []string{"Hugo Static Site Generator"})
+
+	opts = fromRemoteOptions{
+		Method: "GET",
+		Body:   []byte("foo"),
+		Headers: map[string]any{
+			"User-Agent": "foo",
+		},
+	}
+
+	req, err = opts.NewRequest("https://example.com/api")
+
+	c.Assert(err, qt.IsNil)
+	c.Assert(req.Method, qt.Equals, "GET")
+	c.Assert(req.Header["User-Agent"], qt.DeepEquals, []string{"foo"})
 
 }
 
 func TestCalculateResourceID(t *testing.T) {
+	t.Parallel()
+
 	c := qt.New(t)
 
 	c.Assert(calculateResourceID("foo", nil), qt.Equals, "5917621528921068675")