Repository: gocolly/colly

Issue:
Stale Cache system
Hey I'm using colly for a personal project of mine, and I'd like to not spam the web page I'm scraping.
I saw there was a `CacheDir` function to colly, but what happens if the webpage changes? Maybe it has some new data.

How can I set a timeout that a cached page is stale after a day or so, and colly would automatically fetch the current one?

Discussion:
@nebojsaj1726: Hey, I would like to contribute to the project. I’ve submitted a PR that resolves this issue: https://github.com/gocolly/colly/pull/852
It adds support for cache expiration. Let me know if any adjustments are needed — happy to help.

Solution patch:
diff --git a/_examples/coursera_courses/coursera_courses.go b/_examples/coursera_courses/coursera_courses.go
index 8526b9a..7017930 100644
--- a/_examples/coursera_courses/coursera_courses.go
+++ b/_examples/coursera_courses/coursera_courses.go
@@ -5,6 +5,7 @@ import (
 	"log"
 	"os"
 	"strings"
+	"time"
 
 	"github.com/gocolly/colly/v2"
 )
@@ -38,6 +39,8 @@ func main() {
 		// Cache responses to prevent multiple download of pages
 		// even if the collector is restarted
 		colly.CacheDir("./coursera_cache"),
+		// Cached responses older than the specified duration will be refreshed
+		colly.CacheExpiration(24*time.Hour),
 	)
 
 	// Create another collector to scrape course details
diff --git a/colly.go b/colly.go
index d701057..833ca4f 100644
--- a/colly.go
+++ b/colly.go
@@ -135,6 +135,9 @@ type Collector struct {
 	backend                  *httpBackend
 	wg                       *sync.WaitGroup
 	lock                     *sync.RWMutex
+	// CacheExpiration sets the maximum age for cache files.
+	// If a cached file is older than this duration, it will be ignored and refreshed.
+	CacheExpiration time.Duration
 }
 
 // RequestCallback is a type alias for OnRequest callback functions
@@ -467,6 +470,14 @@ func CheckHead() CollectorOption {
 	}
 }
 
+// CacheExpiration sets the maximum age for cache files.
+// If a cached file is older than this duration, it will be ignored and refreshed.
+func CacheExpiration(d time.Duration) CollectorOption {
+	return func(c *Collector) {
+		c.CacheExpiration = d
+	}
+}
+
 // Init initializes the Collector's private variables and sets default
 // configuration for the Collector
 func (c *Collector) Init() {
@@ -711,7 +722,7 @@ func (c *Collector) fetch(u, method string, depth int, requestData io.Reader, ct
 		c.handleOnResponseHeaders(&Response{Ctx: ctx, Request: request, StatusCode: statusCode, Headers: &headers})
 		return !request.abort
 	}
-	response, err := c.backend.Cache(req, c.MaxBodySize, checkHeadersFunc, c.CacheDir)
+	response, err := c.backend.Cache(req, c.MaxBodySize, checkHeadersFunc, c.CacheDir, c.CacheExpiration)
 	if proxyURL, ok := req.Context().Value(ProxyURLKey).(string); ok {
 		request.ProxyURL = proxyURL
 	}
diff --git a/http_backend.go b/http_backend.go
index e580f7a..3d178f5 100644
--- a/http_backend.go
+++ b/http_backend.go
@@ -128,7 +128,7 @@ func (h *httpBackend) GetMatchingRule(domain string) *LimitRule {
 	return nil
 }
 
-func (h *httpBackend) Cache(request *http.Request, bodySize int, checkHeadersFunc checkHeadersFunc, cacheDir string) (*Response, error) {
+func (h *httpBackend) Cache(request *http.Request, bodySize int, checkHeadersFunc checkHeadersFunc, cacheDir string, cacheExpiration time.Duration) (*Response, error) {
 	if cacheDir == "" || request.Method != "GET" || request.Header.Get("Cache-Control") == "no-cache" {
 		return h.Do(request, bodySize, checkHeadersFunc)
 	}
@@ -136,6 +136,13 @@ func (h *httpBackend) Cache(request *http.Request, bodySize int, checkHeadersFun
 	hash := hex.EncodeToString(sum[:])
 	dir := path.Join(cacheDir, hash[:2])
 	filename := path.Join(dir, hash)
+
+	if fileInfo, err := os.Stat(filename); err == nil && cacheExpiration > 0 {
+		if time.Since(fileInfo.ModTime()) > cacheExpiration {
+			_ = os.Remove(filename)
+		}
+	}
+
 	if file, err := os.Open(filename); err == nil {
 		resp := new(Response)
 		err := gob.NewDecoder(file).Decode(resp)



Test patch:
diff --git a/colly_test.go b/colly_test.go
index 59ad396..d73d951 100644
--- a/colly_test.go
+++ b/colly_test.go
@@ -401,6 +401,19 @@ var newCollectorTests = map[string]func(*testing.T){
 			}
 		}
 	},
+	"CacheExpiration": func(t *testing.T) {
+		for _, d := range []time.Duration{
+			5 * time.Second,
+			10 * time.Minute,
+			0,
+		} {
+			c := NewCollector(CacheExpiration(d))
+
+			if got, want := c.CacheExpiration, d; got != want {
+				t.Fatalf("c.CacheExpiration = %v, want %v", got, want)
+			}
+		}
+	},
 	"IgnoreRobotsTxt": func(t *testing.T) {
 		c := NewCollector(IgnoreRobotsTxt())