Repository: raviqqe/muffet
Pull request #546 (2025-07-02 23:41:07)

Revert network error ignorance
- **Revert "Refactor network error ignorance (#542)"**
- **Revert "Added --ignore-network-errors to optionally ignore network errors (#535)"**



Patch:
diff --git a/arguments.go b/arguments.go
index 487a513..71195fb 100644
--- a/arguments.go
+++ b/arguments.go
@@ -22,10 +22,9 @@ type arguments struct {
 	FollowSitemapXML       bool     `long:"follow-sitemap-xml" description:"Scrape only pages listed in sitemap.xml (deprecated)"`
 	RawHeaders             []string `long:"header" value-name:"<header>..." description:"Custom headers"`
 	// TODO Remove a short option.
-	IgnoreFragments     bool   `short:"f" long:"ignore-fragments" description:"Ignore URL fragments"`
-	IgnoreNetworkErrors string `long:"ignore-network-errors" description:"Ignore network errors and timeouts" choice:"none" choice:"all" choice:"external" default:"none"`
-	DnsResolver         string `long:"dns-resolver" value-name:"<address>" description:"Custom DNS resolver"`
-	Format              string `long:"format" description:"Output format" default:"text" choice:"text" choice:"json" choice:"junit"`
+	IgnoreFragments bool   `short:"f" long:"ignore-fragments" description:"Ignore URL fragments"`
+	DnsResolver     string `long:"dns-resolver" value-name:"<address>" description:"Custom DNS resolver"`
+	Format          string `long:"format" description:"Output format" default:"text" choice:"text" choice:"json" choice:"junit"`
 	// TODO Remove this option.
 	JSONOutput bool `long:"json" description:"Output results in JSON (deprecated)"`
 	// TODO Remove this option.
diff --git a/command.go b/command.go
index 05b90b5..bea32cb 100644
--- a/command.go
+++ b/command.go
@@ -108,23 +108,10 @@ func (c *command) runWithError(ss []string) (bool, error) {
 		}
 	}
 
-	g := networkErrorGroupNone
-
-	switch args.IgnoreNetworkErrors {
-	case "none":
-	case "all":
-		g = networkErrorGroupAll
-	case "external":
-		g = networkErrorGroupExternal
-	default:
-		return false, fmt.Errorf("invalid network error: %v", args.IgnoreNetworkErrors)
-	}
-
 	checker := newPageChecker(
 		f,
 		newLinkValidator(p.URL().Hostname(), rd, sm),
 		args.OnePageOnly,
-		g,
 	)
 
 	go checker.Check(p)
diff --git a/network_error.go b/network_error.go
deleted file mode 100644
index b5792b8..0000000
--- a/network_error.go
+++ /dev/null
@@ -1,9 +0,0 @@
-package main
-
-type networkErrorGroup int
-
-const (
-	networkErrorGroupNone networkErrorGroup = iota
-	networkErrorGroupAll
-	networkErrorGroupExternal
-)
diff --git a/page_checker.go b/page_checker.go
index 0143208..7205df0 100644
--- a/page_checker.go
+++ b/page_checker.go
@@ -1,24 +1,17 @@
 package main
 
-import (
-	"context"
-	"errors"
-	"net"
-	"net/url"
-	"sync"
-)
+import "sync"
 
 type pageChecker struct {
-	fetcher             *linkFetcher
-	linkValidator       *linkValidator
-	daemonManager       *daemonManager
-	results             chan *pageResult
-	donePages           concurrentStringSet
-	onePageOnly         bool
-	ignoreNetworkErrors networkErrorGroup
+	fetcher       *linkFetcher
+	linkValidator *linkValidator
+	daemonManager *daemonManager
+	results       chan *pageResult
+	donePages     concurrentStringSet
+	onePageOnly   bool
 }
 
-func newPageChecker(f *linkFetcher, v *linkValidator, onePageOnly bool, ignore networkErrorGroup) *pageChecker {
+func newPageChecker(f *linkFetcher, v *linkValidator, onePageOnly bool) *pageChecker {
 	return &pageChecker{
 		f,
 		v,
@@ -26,7 +19,6 @@ func newPageChecker(f *linkFetcher, v *linkValidator, onePageOnly bool, ignore n
 		make(chan *pageResult, concurrency),
 		newConcurrentStringSet(),
 		onePageOnly,
-		ignore,
 	}
 }
 
@@ -63,7 +55,7 @@ func (c *pageChecker) checkPage(p page) {
 
 			if err == nil {
 				sc <- &successLinkResult{u, status}
-			} else if !c.shouldIgnoreNetworkError(err, u) {
+			} else {
 				ec <- &errorLinkResult{u, err}
 			}
 
@@ -98,19 +90,3 @@ func (c *pageChecker) addPage(p page) {
 		c.daemonManager.Add(func() { c.checkPage(p) })
 	}
 }
-
-func (c *pageChecker) shouldIgnoreNetworkError(err error, rawURL string) bool {
-	if c.ignoreNetworkErrors == networkErrorGroupNone || !isNetworkError(err) {
-		return false
-	}
-
-	u, err := url.Parse(rawURL)
-	return err == nil &&
-		(c.ignoreNetworkErrors == networkErrorGroupAll ||
-			u.Hostname() != c.linkValidator.hostname)
-}
-
-func isNetworkError(err error) bool {
-	var netErr net.Error
-	return errors.As(err, &netErr) || errors.Is(err, context.DeadlineExceeded)
-}


Test patch:
diff --git a/.snapshots/TestHelp b/.snapshots/TestHelp
index 255f166..fbddea1 100644
--- a/.snapshots/TestHelp
+++ b/.snapshots/TestHelp
@@ -2,53 +2,45 @@ Usage:
   muffet.test [options] <url>
 
 Application Options:
-      --accepted-status-codes=<codes>                Accepted HTTP response
-                                                     status codes (e.g.
-                                                     '200..300,403') (default:
-                                                     200..300)
-  -b, --buffer-size=<size>                           HTTP response buffer size
-                                                     in bytes (default: 4096)
-  -c, --max-connections=<count>                      Maximum number of HTTP
-                                                     connections (default: 512)
-      --max-connections-per-host=<count>             Maximum number of HTTP
-                                                     connections per host
-                                                     (default: 512)
-      --max-response-body-size=<size>                Maximum response body size
-                                                     to read (default: 10000000)
-  -e, --exclude=<pattern>...                         Exclude URLs matched with
-                                                     given regular expressions
-  -i, --include=<pattern>...                         Include URLs matched with
-                                                     given regular expressions
-      --follow-robots-txt                            Follow robots.txt when
-                                                     scraping pages
-      --follow-sitemap-xml                           Scrape only pages listed
-                                                     in sitemap.xml (deprecated)
-      --header=<header>...                           Custom headers
-  -f, --ignore-fragments                             Ignore URL fragments
-      --ignore-network-errors=[none|all|external]    Ignore network errors and
-                                                     timeouts (default: none)
-      --dns-resolver=<address>                       Custom DNS resolver
-      --format=[text|json|junit]                     Output format (default:
-                                                     text)
-      --json                                         Output results in JSON
-                                                     (deprecated)
-      --experimental-verbose-json                    Include successful results
-                                                     in JSON (deprecated)
-      --junit                                        Output results as JUnit
-                                                     XML file (deprecated)
-  -r, --max-redirections=<count>                     Maximum number of
-                                                     redirections (default: 64)
-      --rate-limit=<rate>                            Max requests per second
-  -t, --timeout=<seconds>                            Timeout for HTTP requests
-                                                     in seconds (default: 10)
-  -v, --verbose                                      Show successful results too
-      --proxy=<host>                                 HTTP proxy host
-      --skip-tls-verification                        Skip TLS certificate
-                                                     verification
-      --one-page-only                                Only check links found in
-                                                     the given URL
-      --color=[auto|always|never]                    Color output (default:
-                                                     auto)
-  -h, --help                                         Show this help
-      --version                                      Show version
+      --accepted-status-codes=<codes>       Accepted HTTP response status codes
+                                            (e.g. '200..300,403') (default:
+                                            200..300)
+  -b, --buffer-size=<size>                  HTTP response buffer size in bytes
+                                            (default: 4096)
+  -c, --max-connections=<count>             Maximum number of HTTP connections
+                                            (default: 512)
+      --max-connections-per-host=<count>    Maximum number of HTTP connections
+                                            per host (default: 512)
+      --max-response-body-size=<size>       Maximum response body size to read
+                                            (default: 10000000)
+  -e, --exclude=<pattern>...                Exclude URLs matched with given
+                                            regular expressions
+  -i, --include=<pattern>...                Include URLs matched with given
+                                            regular expressions
+      --follow-robots-txt                   Follow robots.txt when scraping
+                                            pages
+      --follow-sitemap-xml                  Scrape only pages listed in
+                                            sitemap.xml (deprecated)
+      --header=<header>...                  Custom headers
+  -f, --ignore-fragments                    Ignore URL fragments
+      --dns-resolver=<address>              Custom DNS resolver
+      --format=[text|json|junit]            Output format (default: text)
+      --json                                Output results in JSON (deprecated)
+      --experimental-verbose-json           Include successful results in JSON
+                                            (deprecated)
+      --junit                               Output results as JUnit XML file
+                                            (deprecated)
+  -r, --max-redirections=<count>            Maximum number of redirections
+                                            (default: 64)
+      --rate-limit=<rate>                   Max requests per second
+  -t, --timeout=<seconds>                   Timeout for HTTP requests in
+                                            seconds (default: 10)
+  -v, --verbose                             Show successful results too
+      --proxy=<host>                        HTTP proxy host
+      --skip-tls-verification               Skip TLS certificate verification
+      --one-page-only                       Only check links found in the given
+                                            URL
+      --color=[auto|always|never]           Color output (default: auto)
+  -h, --help                                Show this help
+      --version                             Show version
 
diff --git a/arguments_test.go b/arguments_test.go
index d57ebda..983ad69 100644
--- a/arguments_test.go
+++ b/arguments_test.go
@@ -35,8 +35,6 @@ func TestGetArguments(t *testing.T) {
 		{"--verbose", "https://foo.com"},
 		{"-v", "-f", "https://foo.com"},
 		{"-v", "--ignore-fragments", "https://foo.com"},
-		{"--ignore-network-errors", "all", "https://foo.com"},
-		{"--ignore-network-errors", "external", "https://foo.com"},
 		{"--one-page-only", "https://foo.com"},
 		{"--json", "https://foo.com"},
 		{"-h"},
@@ -63,7 +61,6 @@ func TestGetArgumentsError(t *testing.T) {
 		{"--max-redirections", "foo", "https://foo.com"},
 		{"-t", "foo", "https://foo.com"},
 		{"--timeout", "foo", "https://foo.com"},
-		{"--ignore-network-errors", "foo", "https://foo.com"},
 	} {
 		_, err := getArguments(ss)
 		assert.NotNil(t, err)
diff --git a/page_checker_test.go b/page_checker_test.go
index c4e0c07..82a2a86 100644
--- a/page_checker_test.go
+++ b/page_checker_test.go
@@ -9,10 +9,6 @@ import (
 )
 
 func newTestPageChecker(c *fakeHttpClient) *pageChecker {
-	return newTestPageCheckerWithIgnore(c, networkErrorGroupNone)
-}
-
-func newTestPageCheckerWithIgnore(c *fakeHttpClient, ignore networkErrorGroup) *pageChecker {
 	return newPageChecker(
 		newLinkFetcher(
 			c,
@@ -21,16 +17,9 @@ func newTestPageCheckerWithIgnore(c *fakeHttpClient, ignore networkErrorGroup) *
 		),
 		newLinkValidator("foo.com", nil, nil),
 		false,
-		ignore,
 	)
 }
 
-type fakeNetError struct{}
-
-func (fakeNetError) Error() string   { return "network error" }
-func (fakeNetError) Timeout() bool   { return true }
-func (fakeNetError) Temporary() bool { return true }
-
 func newTestPage(t *testing.T, fragments map[string]struct{}, links map[string]error) page {
 	u, err := url.Parse("http://foo.com")
 	assert.Nil(t, err)
@@ -123,57 +112,3 @@ func TestPageCheckerDoNotCheckSamePageTwice(t *testing.T) {
 
 	assert.Equal(t, 1, i)
 }
-
-func TestPageCheckerIgnoreAllNetworkErrors(t *testing.T) {
-	c := newTestPageCheckerWithIgnore(
-		newFakeHttpClient(
-			func(u *url.URL) (*fakeHttpResponse, error) {
-				if u.String() == "http://foo.com" {
-					return newFakeHtmlResponse(
-						"http://foo.com",
-						`<html><body><a href="http://bar.com" /></body></html>`,
-					), nil
-				}
-
-				return nil, fakeNetError{}
-			},
-		),
-		networkErrorGroupAll,
-	)
-
-	go c.Check(newTestPage(t, nil, map[string]error{"http://bar.com": nil}))
-
-	r := <-c.Results()
-	assert.True(t, r.OK())
-	assert.Equal(t, 0, len(r.ErrorLinkResults))
-}
-
-func TestPageCheckerIgnoreExternalNetworkError(t *testing.T) {
-	c := newTestPageCheckerWithIgnore(
-		newFakeHttpClient(func(u *url.URL) (*fakeHttpResponse, error) {
-			return nil, fakeNetError{}
-		}),
-		networkErrorGroupExternal,
-	)
-
-	go c.Check(newTestPage(t, nil, map[string]error{"http://bar.com": nil}))
-
-	r := <-c.Results()
-	assert.True(t, r.OK())
-	assert.Equal(t, 0, len(r.ErrorLinkResults))
-}
-
-func TestPageCheckerDoNotIgnoreInternalNetworkError(t *testing.T) {
-	c := newTestPageCheckerWithIgnore(
-		newFakeHttpClient(func(u *url.URL) (*fakeHttpResponse, error) {
-			return nil, fakeNetError{}
-		}),
-		networkErrorGroupExternal,
-	)
-
-	go c.Check(newTestPage(t, nil, map[string]error{"http://foo.com/foo": nil}))
-
-	r := <-c.Results()
-	assert.False(t, r.OK())
-	assert.Equal(t, 1, len(r.ErrorLinkResults))
-}