Repository: raviqqe/muffet (go)

Issue:
Spaces removed from links (all URLs)
Related to #44, however, now I've come across a site (being published on Read the Docs) which has many spaces in the filenames. Though the IDs are fine, the spaces get removed from the path, which breaks the links and results in many false 404's.

Perhaps instead of removing spaces, using the `net/url` Parse function only? Or removing spaces only from the `URL.Fragment`?
https://github.com/raviqqe/muffet/blob/4998c9b377f664bcad596990afeac7679cb306c8/scraper.go#L48-L54
https://github.com/raviqqe/muffet/blob/4998c9b377f664bcad596990afeac7679cb306c8/scraper.go#L82-L90
https://play.golang.org/p/42kUw1Rg23m

Pull request:
Fix space handling in URLs
Close #72 

Patch:
diff --git a/configuration.go b/configuration.go
index d6779a9..8661b2b 100644
--- a/configuration.go
+++ b/configuration.go
@@ -3,7 +3,7 @@ package main
 import "time"
 
 const (
-	version                = "1.5.5"
+	version                = "1.5.6"
 	defaultBufferSize      = 4096
 	defaultConcurrency     = 512
 	defaultMaxRedirections = 64
diff --git a/link_finder.go b/link_finder.go
index 06cd028..74d9acc 100644
--- a/link_finder.go
+++ b/link_finder.go
@@ -4,7 +4,6 @@ import (
 	"net/url"
 	"regexp"
 	"strings"
-	"unicode"
 
 	"github.com/yhat/scrape"
 	"golang.org/x/net/html"
@@ -44,7 +43,7 @@ func (f linkFinder) Find(n *html.Node, base *url.URL) map[string]error {
 		return ok
 	}) {
 		for _, a := range atomToAttributes[n.DataAtom] {
-			s := normalizeURL(scrape.Attr(n, a))
+			s := strings.TrimSpace(scrape.Attr(n, a))
 
 			if s == "" || f.isLinkExcluded(s) {
 				continue
@@ -72,13 +71,3 @@ func (f linkFinder) isLinkExcluded(u string) bool {
 
 	return false
 }
-
-func normalizeURL(s string) string {
-	return strings.Map(func(r rune) rune {
-		if unicode.IsSpace(r) {
-			return -1
-		}
-
-		return r
-	}, s)
-}


Test patch:
diff --git a/link_finder_test.go b/link_finder_test.go
index aecd17b..2a6ec5a 100644
--- a/link_finder_test.go
+++ b/link_finder_test.go
@@ -20,8 +20,7 @@ func TestLinkFinderFindLinks(t *testing.T) {
 	}{
 		{``, 0},
 		{`<a href="/" />`, 1},
-		{`<a href="/f
-		o 	o" />`, 1},
+		{`<a href="/foo" />`, 1},
 		// TODO: Test <frame> tag.
 		{`<iframe src="/iframe"></iframe>`, 1},
 		{`<img src="/foo.jpg" />`, 1},
@@ -52,7 +51,55 @@ func TestLinkFinderFindLinks(t *testing.T) {
 	}
 }
 
-func TestLinkFinderScrapePageError(t *testing.T) {
+func TestLinkFinderFindLinkWithoutEncodedSpaces(t *testing.T) {
+	b, err := url.Parse("http://foo.com")
+	assert.Nil(t, err)
+
+	n, err := html.Parse(strings.NewReader(
+		htmlWithBody(`<a href="http://foo.com/a%20b" />`)),
+	)
+	assert.Nil(t, err)
+
+	ls := newLinkFinder(nil).Find(n, b)
+
+	err, ok := ls["http://foo.com/a%20b"]
+	assert.True(t, ok)
+	assert.Nil(t, err)
+}
+
+func TestLinkFinderFindLinkWithoutSpacesNotEncoded(t *testing.T) {
+	b, err := url.Parse("http://foo.com")
+	assert.Nil(t, err)
+
+	n, err := html.Parse(strings.NewReader(
+		htmlWithBody(`<a href="http://foo.com/a b" />`)),
+	)
+	assert.Nil(t, err)
+
+	ls := newLinkFinder(nil).Find(n, b)
+
+	err, ok := ls["http://foo.com/a%20b"]
+	assert.True(t, ok)
+	assert.Nil(t, err)
+}
+
+func TestLinkFinderFindLinkWithLeadingAndTrailingSpaces(t *testing.T) {
+	b, err := url.Parse("http://foo.com")
+	assert.Nil(t, err)
+
+	n, err := html.Parse(strings.NewReader(
+		htmlWithBody(`<a href=" http://foo.com " />`)),
+	)
+	assert.Nil(t, err)
+
+	ls := newLinkFinder(nil).Find(n, b)
+
+	err, ok := ls["http://foo.com"]
+	assert.True(t, ok)
+	assert.Nil(t, err)
+}
+
+func TestLinkFinderFailWithInvalidURL(t *testing.T) {
 	b, err := url.Parse("http://foo.com")
 	assert.Nil(t, err)