Repository: raviqqe/muffet (go)

Issue:
Feature: Support metadata URLs used for open graph/twitter cards
Websites can provide metadata like thumbnails/preview images using the `<meta>` tag. [Open Graph](https://ogp.me/) is used by most platforms and Twitter has its own format but they all use `<meta>` tags where the `content=` attribute can contain a URL.

My use case is that I crawl my website locally to make sure that I don't have any broken links in the meta URLs. So I would find this feature helpful.

I am happy to send a PR but wanted to check if there is interest in this feature first.

Pull request description:
Support scraping opengraph/twitter meta tags
`<meta>` tags with specific `og:` and `twitter:` properties that map to URLs are now fetched.

Specific properties that are supported are:

- `og:image`
- `og:audio`
- `og:video`
- `og:image:url`
- `og:image:secure_url`
- `twitter:image`

Fixes #268 

Patch:
diff --git a/link_finder.go b/link_finder.go
index fcee4f8..edd7f68 100644
--- a/link_finder.go
+++ b/link_finder.go
@@ -25,6 +25,7 @@ var atomToAttributes = map[atom.Atom][]string{
 	atom.Script: {"src"},
 	atom.Source: {"src", "srcset"},
 	atom.Track:  {"src"},
+	atom.Meta:   {"content"},
 }
 
 var imageDescriptorPattern = regexp.MustCompile(" [^ ]*$")
@@ -81,6 +82,12 @@ func (linkFinder) parseLinks(n *html.Node, a string) []string {
 		for _, s := range strings.Split(s, ",") {
 			ss = append(ss, imageDescriptorPattern.ReplaceAllString(strings.TrimSpace(s), ""))
 		}
+	} else if a == "content" {
+		prop := scrape.Attr(n, "property")
+		switch prop {
+		case "og:image", "og:audio", "og:video", "og:image:url", "og:image:secure_url", "twitter:image":
+			ss = append(ss, s)
+		}
 	} else {
 		ss = append(ss, s)
 	}


Test patch:
diff --git a/link_finder_test.go b/link_finder_test.go
index 2054bc2..4da52e8 100644
--- a/link_finder_test.go
+++ b/link_finder_test.go
@@ -253,6 +253,40 @@ func TestLinkFinderFindMultipleLinksInSrcSetWithDescriptors(t *testing.T) {
 	assert.Nil(t, err)
 }
 
+func TestLinkFinderFindMetaTags(t *testing.T) {
+	b, err := url.Parse("http://foo.com")
+	assert.Nil(t, err)
+
+	n, err := html.Parse(strings.NewReader(
+		htmlWithHead(`<meta property="og:image" content="foo.png" />`)),
+	)
+	assert.Nil(t, err)
+
+	ls := newLinkFinder(nil, nil).Find(n, b)
+
+	err, ok := ls["http://foo.com/foo.png"]
+	assert.True(t, ok)
+	assert.Nil(t, err)
+}
+
+func TestLinkFinderIgnoreMetaTags(t *testing.T) {
+	b, err := url.Parse("http://foo.com")
+	assert.Nil(t, err)
+
+	n, err := html.Parse(strings.NewReader(
+		htmlWithHead(`<meta property="og:title" content="title" />`)),
+	)
+	assert.Nil(t, err)
+
+	ls := newLinkFinder(nil, nil).Find(n, b)
+
+	assert.Len(t, ls, 0)
+}
+
 func htmlWithBody(b string) string {
 	return fmt.Sprintf(`<html><body>%v</body></html>`, b)
 }
+
+func htmlWithHead(b string) string {
+	return fmt.Sprintf(`<html><head>%v</head><body><p>hi</p></body></html>`, b)
+}