Repository: internetofwater/nabu
Pull request #76: Core gleaner functionality working via cli
Author: login: C-Loftus
Created: 2025-04-29T20:51:03Z | Merged: 2025-04-30T13:42:33Z

Gleaner functionality for:

- limiting concurrency
- log level
- harvesting to disk or minio
- specifying cli args

is all working

To test you can run: 

```sh
go run cmd/gleaner/root.go --to-disk --log-level DEBUG --sitemap-index https://geoconnex.us/sitemap.xml --concurrent-sitemaps 1
```


Diff:
diff --git a/cmd/gleaner/root.go b/cmd/gleaner/root.go
new file mode 100644
index 00000000..3fba9c39
--- /dev/null
+++ b/cmd/gleaner/root.go
@@ -0,0 +1,107 @@
+package main
+
+import (
+	"fmt"
+	"nabu/internal/config"
+	"nabu/internal/interfaces"
+	"nabu/internal/synchronizer/s3"
+	"os"
+
+	crawl "nabu/internal/crawl"
+
+	arg "github.com/alexflint/go-arg"
+	log "github.com/sirupsen/logrus"
+)
+
+type GleanerArgs struct {
+	Address      string `arg:"--address" default:"127.0.0.1"`
+	Port         int    `arg:"--port" default:"7200"`
+	Bucket       string `arg:"--bucket" default:"iow"` // minio bucket to put data
+	SitemapIndex string `arg:"--sitemap-index"`
+	Source       string `arg:"--source"` // source to crawl from the config
+	Config       string `arg:"--cfg"`    // full path to config
+	Mode         string `arg:"--mode"`
+	SecretKey    string `arg:"--secret-key" default:"minioadmin"` // secret key for minio
+	AccessKey    string `arg:"--access-key" default:"minioadmin"` // access key for minio
+	SSL          bool   `arg:"--ssl"`                             // use SSL for HTTP requests
+	SetupBuckets bool   `arg:"--setup-buckets"`                   // setup buckets before crawling
+	Rude         bool   `arg:"--rude"`                            // ignore robots.txt
+	ToDisk       bool   `arg:"--to-disk" default:"false"`         // save to disk instead of minio
+	LogLevel     string `arg:"--log-level" default:"INFO"`
+
+	ConcurrentSitemaps int `arg:"--concurrent-sitemaps" default:"10"`
+	SitemapWorkers     int `arg:"--sitemap-workers" default:"10"`
+}
+
+type GleanerRunner struct {
+	args GleanerArgs
+}
+
+func NewGleanerRunner(cliArgs []string) GleanerRunner {
+	args := GleanerArgs{}
+	allArgsButBinary := cliArgs[1:]
+	const dummyBinaryName = "gleaner" // we need to add some arbitrary binary name
+	os.Args = append([]string{dummyBinaryName}, allArgsButBinary...)
+	arg.MustParse(&args)
+	return GleanerRunner{
+		args: args,
+	}
+}
+
+func (g GleanerRunner) Run() error {
+	level, err := log.ParseLevel(g.args.LogLevel)
+	if err != nil {
+		return fmt.Errorf("invalid log level %s: %w", g.args.LogLevel, err)
+	}
+	log.SetLevel(level)
+	log.SetOutput(os.Stdout)
+	log.Info("Starting Gleaner")
+
+	if g.args.SitemapIndex != "" {
+		index, err := crawl.NewSitemapIndexHarvester(g.args.SitemapIndex)
+		if err != nil {
+			return err
+		}
+		var configuredSitemap crawl.Index
+		if g.args.ToDisk {
+			log.Info("Saving to fetched files to disk")
+			tmpFSStorage, err := interfaces.NewLocalTempFSCrawlStorage()
+			if err != nil {
+				return err
+			}
+			configuredSitemap = index.WithStorageDestination(tmpFSStorage)
+		} else {
+			log.Infof("Saving to fetched files to s3 bucket at %s:%d", g.args.Address, g.args.Port)
+			minioS3, err := s3.NewMinioClientWrapper(config.MinioConfig{
+				Address:   g.args.Address,
+				Port:      g.args.Port,
+				Ssl:       g.args.SSL,
+				Accesskey: g.args.AccessKey,
+				Secretkey: g.args.SecretKey,
+				Bucket:    g.args.Bucket,
+			})
+			if err != nil {
+				return err
+			}
+			configuredSitemap = index.WithStorageDestination(minioS3)
+		}
+
+		configuredSitemap = configuredSitemap.WithConcurrencyConfig(g.args.ConcurrentSitemaps, g.args.SitemapWorkers)
+
+		if g.args.Source != "" {
+			return configuredSitemap.HarvestSitemap(g.args.Source)
+		} else {
+			return configuredSitemap.HarvestAll()
+		}
+	} else if g.args.Source != "" {
+		panic("not implemented")
+	}
+
+	return nil
+}
+
+func main() {
+	if err := NewGleanerRunner(os.Args).Run(); err != nil {
+		log.Fatalf("Failed to run: %v", err)
+	}
+}
diff --git a/cmd/gleaner/root_test.go b/cmd/gleaner/root_test.go
new file mode 100644
index 00000000..cb565792
--- /dev/null
+++ b/cmd/gleaner/root_test.go
@@ -0,0 +1,67 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"testing"
+
+	"nabu/internal/synchronizer/s3"
+
+	"github.com/h2non/gock"
+	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/suite"
+)
+
+func startMocks() {
+	gock.New("https://geoconnex.us/sitemap.xml").
+		Reply(200).
+		File("testdata/sitemap_index.xml")
+
+	gock.New("https://geoconnex.us/sitemap/iow/wqp/stations__5.xml").
+		Reply(200).
+		File("testdata/sitemap.xml")
+}
+
+func (s *GleanerRootSuite) TestHarvestToS3() {
+	startMocks()
+	args := fmt.Sprintf("gleaner --log-level DEBUG --sitemap-index https://geoconnex.us/sitemap.xml --address %s --port %d --bucket %s", s.minioContainer.Hostname, s.minioContainer.APIPort, s.minioContainer.ClientWrapper.DefaultBucket)
+	err := NewGleanerRunner(strings.Split(args, " ")).Run()
+	require.NoError(s.T(), err)
+	objs, err := s.minioContainer.ClientWrapper.ObjectList("summoned/")
+	require.NoError(s.T(), err)
+	require.Len(s.T(), objs, 3)
+}
+
+func (s *GleanerRootSuite) TestHarvestToDisk() {
+	startMocks()
+	args := "gleaner --log-level DEBUG --to-disk --sitemap-index testdata/sitemap_index.xml"
+	err := NewGleanerRunner(strings.Split(args, " ")).Run()
+	require.NoError(s.T(), err)
+}
+
+// Wrapper struct to store a handle to the container for all
+type GleanerRootSuite struct {
+	suite.Suite
+	minioContainer s3.MinioContainer
+}
+
+// Setup common dependencies before starting the test suite
+func (suite *GleanerRootSuite) SetupSuite() {
+	container, err := s3.NewDefaultMinioContainer()
+	require.NoError(suite.T(), err)
+	suite.minioContainer = container
+
+}
+
+func (s *GleanerRootSuite) TearDownSuite() {
+	defer gock.Off()
+	c := *s.minioContainer.Container
+	err := c.Terminate(context.Background())
+	require.NoError(s.T(), err)
+}
+
+// Run the entire test suite
+func TestS3ClientSuite(t *testing.T) {
+	suite.Run(t, new(GleanerRootSuite))
+}
diff --git a/go.mod b/go.mod
index caab474d..34ed1ad1 100644
--- a/go.mod
+++ b/go.mod
@@ -5,6 +5,7 @@ go 1.23
 toolchain go1.23.3
 
 require (
+	github.com/alexflint/go-arg v1.5.1
 	github.com/h2non/gock v1.2.0
 	github.com/knakk/rdf v0.0.0-20190304171630-8521bf4c5042
 	github.com/minio/minio-go/v7 v7.0.84
@@ -29,7 +30,10 @@ require (
 	google.golang.org/protobuf v1.36.5 // indirect
 )
 
-require github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect
+require (
+	github.com/alexflint/go-scalar v1.2.0 // indirect
+	github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect
+)
 
 require (
 	dario.cat/mergo v1.0.0 // indirect
diff --git a/go.sum b/go.sum
index 26c57854..19d3ad81 100644
--- a/go.sum
+++ b/go.sum
@@ -56,6 +56,10 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym
 github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
 github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
+github.com/alexflint/go-arg v1.5.1 h1:nBuWUCpuRy0snAG+uIJ6N0UvYxpxA0/ghA/AaHxlT8Y=
+github.com/alexflint/go-arg v1.5.1/go.mod h1:A7vTJzvjoaSTypg4biM5uYNTkJ27SkNTArtYXnlqVO8=
+github.com/alexflint/go-scalar v1.2.0 h1:WR7JPKkeNpnYIOfHRa7ivM21aWAdHD0gEWHCx+WQBRw=
+github.com/alexflint/go-scalar v1.2.0/go.mod h1:LoFvNMqS1CPrMVltza4LvnGKhaSpc3oyLEBUZVhhS2o=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
diff --git a/cmd/gleaner/err_group.go b/internal/crawl/err_group.go
similarity index 98%
rename from cmd/gleaner/err_group.go
rename to internal/crawl/err_group.go
index e65e5030..fc349df0 100644
--- a/cmd/gleaner/err_group.go
+++ b/internal/crawl/err_group.go
@@ -1,4 +1,4 @@
-package gleaner
+package crawl
 
 import (
 	"sync"
diff --git a/cmd/gleaner/helpers.go b/internal/crawl/helpers.go
similarity index 86%
rename from cmd/gleaner/helpers.go
rename to internal/crawl/helpers.go
index e4ed0442..a1160a33 100644
--- a/cmd/gleaner/helpers.go
+++ b/internal/crawl/helpers.go
@@ -1,4 +1,4 @@
-package gleaner
+package crawl
 
 import (
 	"bytes"
@@ -46,7 +46,7 @@ func newRobots(urlToCheck string) (*robotstxt.Group, error) {
 	return robots.FindGroup(gleanerAgent), nil
 }
 
-func copyReaderAndReturnHash(reader io.Reader) (io.Reader, string, error) {
+func copyReaderAndGenerateHashFilename(reader io.Reader) (io.Reader, string, error) {
 	var buf bytes.Buffer
 	tee := io.TeeReader(reader, &buf)
 
@@ -56,5 +56,5 @@ func copyReaderAndReturnHash(reader io.Reader) (io.Reader, string, error) {
 	}
 
 	// Create a new reader from the buffered copy
-	return bytes.NewReader(buf.Bytes()), fmt.Sprintf("%x", hasher.Sum(nil)), nil
+	return bytes.NewReader(buf.Bytes()), fmt.Sprintf("%x.jsonld", hasher.Sum(nil)), nil
 }
diff --git a/cmd/gleaner/helpers_test.go b/internal/crawl/helpers_test.go
similarity index 93%
rename from cmd/gleaner/helpers_test.go
rename to internal/crawl/helpers_test.go
index e66a2c80..175d6174 100644
--- a/cmd/gleaner/helpers_test.go
+++ b/internal/crawl/helpers_test.go
@@ -1,4 +1,4 @@
-package gleaner
+package crawl
 
 import (
 	"bytes"
@@ -89,7 +89,7 @@ func FuzzCopyReaderAndReturnHash(f *testing.F) {
 		// Run function with fuzz input
 		reader := bytes.NewReader(input)
 
-		readerCopy, hash, err := copyReaderAndReturnHash(reader)
+		readerCopy, hash, err := copyReaderAndGenerateHashFilename(reader)
 		require.NoError(t, err)
 
 		// Read copied data
@@ -100,7 +100,7 @@ func FuzzCopyReaderAndReturnHash(f *testing.F) {
 		require.Equal(t, string(copiedData), string(input), "copied data should match input")
 
 		// rehash to verify correctness
-		expectedHash := fmt.Sprintf("%x", md5.Sum(input))
+		expectedHash := fmt.Sprintf("%x.jsonld", md5.Sum(input))
 		require.Equal(t, hash, expectedHash, "hash should match expected MD5")
 	})
 }
diff --git a/cmd/gleaner/sitemap.go b/internal/crawl/sitemap.go
similarity index 61%
rename from cmd/gleaner/sitemap.go
rename to internal/crawl/sitemap.go
index 72bc809e..bddf7d38 100644
--- a/cmd/gleaner/sitemap.go
+++ b/internal/crawl/sitemap.go
@@ -1,29 +1,18 @@
-package gleaner
+package crawl
 
 import (
 	"encoding/xml"
 	"fmt"
 	"nabu/internal/common"
+	"nabu/internal/interfaces"
 	"strings"
 	"time"
 
-	crawl "nabu/internal/crawl"
-
 	sitemap "github.com/oxffaa/gopher-parse-sitemap"
+	log "github.com/sirupsen/logrus"
+	"golang.org/x/sync/errgroup"
 )
 
-// Index is a structure of <sitemapindex>
-type Index struct {
-	XMLName xml.Name `xml:"sitemapindex"`
-	Sitemap []parts  `xml:"sitemap"`
-}
-
-// parts is a structure of <sitemap> in <sitemapindex>
-type parts struct {
-	Loc     string `xml:"loc"`
-	LastMod string `xml:"lastmod"`
-}
-
 // Represents an XML sitemap
 type Sitemap struct {
 	XMLName xml.Name `xml:":urlset"`
@@ -33,35 +22,37 @@ type Sitemap struct {
 	// - explicitly ignores xml marshaling
 	// since this is not an xml field but rather
 	// associated data with the sitemap struct
-	storageDestination crawl.CrawlStorage `xml:"-"`
+	storageDestination interfaces.CrawlStorage `xml:"-"`
 }
 
 // Set the storage strategy for the struct
-func (s Sitemap) SetStorageDestination(storageDestination crawl.CrawlStorage) Sitemap {
+func (s Sitemap) SetStorageDestination(storageDestination interfaces.CrawlStorage) Sitemap {
 	s.storageDestination = storageDestination
 	return s
 }
 
 // Harvest all the URLs in the sitemap
-func (s Sitemap) Harvest(workers int) []error {
-	var group MultiErrGroup
+func (s Sitemap) Harvest(workers int) error {
+	var group errgroup.Group
 	group.SetLimit(workers)
 
 	// For the time being, we assume that the first URL in the sitemap has the
 	// same robots.txt as the rest of the items
 	if len(s.URL) == 0 {
-		return []error{fmt.Errorf("no URLs found in sitemap")}
+		return fmt.Errorf("no URLs found in sitemap")
 	} else if s.storageDestination == nil {
-		return []error{fmt.Errorf("no storage destination set")}
+		return fmt.Errorf("no storage destination set")
+	} else if workers < 1 {
+		return fmt.Errorf("no workers set")
 	}
 
 	firstUrl := s.URL[0]
 	robotstxt, err := newRobots(firstUrl.Loc)
 	if err != nil {
-		return []error{err}
+		return err
 	}
 	if !robotstxt.Test(gleanerAgent) {
-		return []error{fmt.Errorf("robots.txt does not allow us to crawl %s", firstUrl.Loc)}
+		return fmt.Errorf("robots.txt does not allow us to crawl %s", firstUrl.Loc)
 	}
 
 	client := common.NewRetryableHTTPClient()
@@ -79,21 +70,26 @@ func (s Sitemap) Harvest(workers int) []error {
 			}
 
 			// To generate a hash we need to copy the response body
-			respBodyCopy, itemHash, err := copyReaderAndReturnHash(resp.Body)
+			respBodyCopy, itemHash, err := copyReaderAndGenerateHashFilename(resp.Body)
 			if err != nil {
 				return err
 			}
 
-			exists, err := s.storageDestination.Exists(itemHash)
+			summonedPath := fmt.Sprintf("summoned/%s", itemHash)
+
+			exists, err := s.storageDestination.Exists(summonedPath)
 			if err != nil {
 				return err
 			}
 
 			if !exists {
 				// Store from the buffered copy
-				if err = s.storageDestination.Store(itemHash, respBodyCopy); err != nil {
+				if err = s.storageDestination.Store(summonedPath, respBodyCopy); err != nil {
 					return err
 				}
+				log.Debugf("stored %s as %s", url.Loc, itemHash)
+			} else {
+				log.Debugf("%s already exists so skipping", url.Loc)
 			}
 
 			if robotstxt.CrawlDelay > 0 {
@@ -132,15 +128,3 @@ func NewSitemap(sitemapURL string) (Sitemap, error) {
 	serializedSitemap.URL = urls
 	return serializedSitemap, nil
 }
-
-// // This function takes a top level sitemap index like geoconnex.us/sitemap.xml and returns a list of sitemap urls
-// A sitemap index is a file that lists the URLs for multiple sitemaps
-func GetSitemapListFromIndex(sitemapURL string) ([]string, error) {
-	result := []string{}
-	err := sitemap.ParseIndexFromSite(sitemapURL, func(e sitemap.IndexEntry) error {
-		result = append(result, strings.TrimSpace(e.GetLocation()))
-		return nil
-	})
-
-	return result, err
-}
diff --git a/internal/crawl/sitemap_index.go b/internal/crawl/sitemap_index.go
new file mode 100644
index 00000000..a7de9e84
--- /dev/null
+++ b/internal/crawl/sitemap_index.go
@@ -0,0 +1,122 @@
+package crawl
+
+import (
+	"encoding/xml"
+	"fmt"
+	"nabu/internal/interfaces"
+	"net/url"
+	"strings"
+	"time"
+
+	sitemap "github.com/oxffaa/gopher-parse-sitemap"
+	"golang.org/x/sync/errgroup"
+)
+
+// Index is a structure of <sitemapindex>
+type Index struct {
+	XMLName  xml.Name `xml:"sitemapindex"`
+	Sitemaps []parts  `xml:"sitemap"`
+
+	storageDestination interfaces.CrawlStorage `xml:"-"`
+	concurrentSitemaps int                     `xml:"-"`
+	sitemapWorkers     int                     `xml:"-"`
+}
+
+// parts is a structure of <sitemap> in <sitemapindex>
+type parts struct {
+	Loc     string `xml:"loc"`
+	LastMod string `xml:"lastmod"`
+}
+
+func isUrl(str string) bool {
+	u, err := url.Parse(str)
+	return err == nil && u.Scheme != "" && u.Host != ""
+}
+
+func NewSitemapIndexHarvester(sitemapRef string) (Index, error) {
+
+	serializedSitemapIndex := Index{}
+
+	if isUrl(sitemapRef) {
+		return serializedSitemapIndex, sitemap.ParseIndexFromSite(sitemapRef, func(ie sitemap.IndexEntry) error {
+			part := parts{}
+			part.Loc = strings.TrimSpace(ie.GetLocation())
+			part.LastMod = ie.GetLastModified().Format(time.RFC3339)
+			serializedSitemapIndex.Sitemaps = append(serializedSitemapIndex.Sitemaps, part)
+			return nil
+		})
+	} else {
+		return serializedSitemapIndex, sitemap.ParseIndexFromFile(sitemapRef, func(ie sitemap.IndexEntry) error {
+			part := parts{}
+			part.Loc = strings.TrimSpace(ie.GetLocation())
+			part.LastMod = ie.GetLastModified().Format(time.RFC3339)
+			serializedSitemapIndex.Sitemaps = append(serializedSitemapIndex.Sitemaps, part)
+			return nil
+		})
+	}
+
+}
+
+func (i Index) GetUrlList() []string {
+	result := []string{}
+	for _, part := range i.Sitemaps {
+		result = append(result, part.Loc)
+	}
+	return result
+}
+
+// Harvest all the URLs in the sitemap
+func (i Index) HarvestAll() error {
+
+	var group errgroup.Group
+	group.SetLimit(i.concurrentSitemaps)
+
+	for _, part := range i.Sitemaps {
+		part := part
+		group.Go(func() error {
+			sitemap, err := NewSitemap(part.Loc)
+			if err != nil {
+				return err
+			}
+			return sitemap.SetStorageDestination(i.storageDestination).Harvest(i.sitemapWorkers)
+		})
+	}
+
+	return group.Wait()
+}
+
+// Harvest one particular sitemap
+func (i Index) HarvestSitemap(sitemap string) error {
+
+	for _, part := range i.Sitemaps {
+		if part.Loc != sitemap {
+			continue
+		}
+		sitemap, err := NewSitemap(part.Loc)
+		if err != nil {
+			return err
+		}
+		return sitemap.SetStorageDestination(i.storageDestination).Harvest(i.sitemapWorkers)
+	}
+	return fmt.Errorf("sitemap %s not found in sitemap", sitemap)
+}
+
+func (i Index) WithStorageDestination(storageDestination interfaces.CrawlStorage) Index {
+	i.storageDestination = storageDestination
+	return i
+}
+
+func (i Index) WithConcurrencyConfig(concurrentSitemaps int, sitemapWorkers int) Index {
+	// Make sure concurrency is at least 1
+	// otherwise go will block indefinitely
+	if concurrentSitemaps < 1 {
+		concurrentSitemaps = 1
+	}
+	if sitemapWorkers < 1 {
+		sitemapWorkers = 1
+	}
+
+	i.concurrentSitemaps = concurrentSitemaps
+	i.sitemapWorkers = sitemapWorkers
+	return i
+}
diff --git a/cmd/gleaner/sitemap_test.go b/internal/crawl/sitemap_test.go
similarity index 80%
rename from cmd/gleaner/sitemap_test.go
rename to internal/crawl/sitemap_test.go
index 5c4da1cd..303613c3 100644
--- a/cmd/gleaner/sitemap_test.go
+++ b/internal/crawl/sitemap_test.go
@@ -1,9 +1,9 @@
-package gleaner
+package crawl
 
 import (
 	"testing"
 
-	crawl "nabu/internal/crawl"
+	"nabu/internal/interfaces"
 	"nabu/internal/synchronizer/s3"
 
 	"github.com/h2non/gock"
@@ -24,12 +24,12 @@ func TestParseSitemapIndex(t *testing.T) {
 		Reply(200).
 		File("testdata/sitemap.xml")
 
-	sitemapUrls, err := GetSitemapListFromIndex("https://geoconnex.us/sitemap.xml")
-	require.NotEmpty(t, sitemapUrls)
+	indexHarvester, err := NewSitemapIndexHarvester("https://geoconnex.us/sitemap.xml")
+	require.NotEmpty(t, indexHarvester)
 	assert.NoError(t, err)
 	var emptyMaps []string
 
-	for _, url := range sitemapUrls {
+	for _, url := range indexHarvester.GetUrlList() {
 		assert.NotEmpty(t, url)
 		sitemap, err := NewSitemap(url)
 		require.NoError(t, err)
@@ -53,16 +53,16 @@ func TestHarvestSitemap(t *testing.T) {
 		File("testdata/sitemap.xml")
 
 	// initialize storage types
-	tmpStore, err := crawl.NewLocalTempFSCrawlStorage()
+	tmpStore, err := interfaces.NewLocalTempFSCrawlStorage()
 	require.NoError(t, err)
 	container, err := s3.NewDefaultMinioContainer()
 	require.NoError(t, err)
 
 	// get the sitemap index
-	sitemapUrls, err := GetSitemapListFromIndex("https://geoconnex.us/sitemap.xml")
+	sitemapUrls, err := NewSitemapIndexHarvester("https://geoconnex.us/sitemap.xml")
 	require.NoError(t, err)
 	// just test the first for the sake of brevity
-	sitemap, err := NewSitemap(sitemapUrls[0])
+	sitemap, err := NewSitemap(sitemapUrls.GetUrlList()[0])
 	require.NoError(t, err)
 
 	errs := sitemap.SetStorageDestination(tmpStore).Harvest(10)
diff --git a/internal/crawl/testdata/sitemap.xml b/internal/crawl/testdata/sitemap.xml
new file mode 100644
index 00000000..105ffe5d
--- /dev/null
+++ b/internal/crawl/testdata/sitemap.xml
@@ -0,0 +1,13 @@
+<?xml version='1.0' encoding='utf-8'?>
+<ns0:urlset xmlns:ns0="http://www.sitemaps.org/schemas/sitemap/0.9">
+<url>
+    <loc>https://geoconnex.us/iow/wqp/BPMWQX-1084-WR-CC01C</loc>
+    <lastmod>2025-02-05T16:39:34Z</lastmod>
+</url><url>
+    <loc>https://geoconnex.us/iow/wqp/BPMWQX-1085-WR-CC01C2</loc>
+    <lastmod>2025-02-05T16:39:34Z</lastmod>
+</url><url>
+    <loc>https://geoconnex.us/iow/wqp/BPMWQX-1086-WR-CC02A</loc>
+    <lastmod>2025-02-05T16:39:34Z</lastmod>
+</url>
+</ns0:urlset>
\ No newline at end of file
diff --git a/internal/crawl/testdata/sitemap_index.xml b/internal/crawl/testdata/sitemap_index.xml
new file mode 100644
index 00000000..b1db27e4
--- /dev/null
+++ b/internal/crawl/testdata/sitemap_index.xml
@@ -0,0 +1,6 @@
+<?xml version='1.0' encoding='utf-8'?>
+<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+<sitemap>
+    <loc>https://geoconnex.us/sitemap/iow/wqp/stations__5.xml</loc>
+    <lastmod>2020-06-12T16:33:18Z</lastmod>
+</sitemap></sitemapindex>
\ No newline at end of file
diff --git a/internal/crawl/storage.go b/internal/interfaces/storage.go
similarity index 89%
rename from internal/crawl/storage.go
rename to internal/interfaces/storage.go
index beba4353..b170ca82 100644
--- a/internal/crawl/storage.go
+++ b/internal/interfaces/storage.go
@@ -1,10 +1,12 @@
-package gleaner
+package interfaces
 
 import (
 	"fmt"
 	"io"
 	"os"
 	"path/filepath"
+
+	log "github.com/sirupsen/logrus"
 )
 
 type BatchFileObject struct {
@@ -41,12 +43,15 @@ func NewLocalTempFSCrawlStorage() (*LocalTempFSCrawlStorage, error) {
 }
 
 // Store saves the contents from the reader into a file named after `object`
-func (l *LocalTempFSCrawlStorage) Store(object string, reader io.Reader) error {
+func (l *LocalTempFSCrawlStorage) Store(name string, reader io.Reader) error {
+
 	if l.baseDir == "" {
 		return fmt.Errorf("baseDir is empty")
 	}
 
-	destPath := filepath.Join(l.baseDir, object)
+	destPath := filepath.Join(l.baseDir, name)
+
+	log.Debugf("saving data to %s", destPath)
 
 	// Make sure directory exists
 	if err := os.MkdirAll(filepath.Dir(destPath), 0755); err != nil {
diff --git a/internal/crawl/storage_test.go b/internal/interfaces/storage_test.go
similarity index 97%
rename from internal/crawl/storage_test.go
rename to internal/interfaces/storage_test.go
index d9c375db..177a93cf 100644
--- a/internal/crawl/storage_test.go
+++ b/internal/interfaces/storage_test.go
@@ -1,4 +1,4 @@
-package gleaner
+package interfaces
 
 import (
 	"bytes"
diff --git a/internal/synchronizer/s3/client.go b/internal/synchronizer/s3/client.go
index 652cfc85..5bce0c35 100644
--- a/internal/synchronizer/s3/client.go
+++ b/internal/synchronizer/s3/client.go
@@ -6,11 +6,12 @@ import (
 	"io"
 	"nabu/internal/common"
 	"nabu/internal/config"
-	crawl "nabu/internal/crawl"
 	"os"
 	"strings"
 	"sync"
 
+	interfaces "nabu/internal/interfaces"
+
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
 	"github.com/piprate/json-gold/ld"
@@ -250,7 +251,7 @@ func (m MinioClientWrapper) Exists(path string) (bool, error) {
 	return false, err
 }
 
-func (m MinioClientWrapper) BatchStore(batch chan crawl.BatchFileObject) error {
+func (m MinioClientWrapper) BatchStore(batch chan interfaces.BatchFileObject) error {
 	snowBallChan := make(chan minio.SnowballObject)
 
 	go func() {
@@ -265,4 +266,4 @@ func (m MinioClientWrapper) BatchStore(batch chan crawl.BatchFileObject) error {
 	return m.Client.PutObjectsSnowball(context.Background(), m.DefaultBucket, minio.SnowballOptions{}, snowBallChan)
 }
 
-var _ crawl.BatchCrawlStorage = MinioClientWrapper{}
+var _ interfaces.BatchCrawlStorage = MinioClientWrapper{}


Reviews:
user: login: webb-ben
body: +1
state: APPROVED
author_association: MEMBER
submitted_at: 2025-04-30T12:17:58Z
user: login: C-Loftus
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-30T12:27:46Z

Review comments:
diff_hunk: @@ -0,0 +1,57 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"testing"
+
+	"nabu/internal/synchronizer/s3"
+
+	"github.com/h2non/gock"
+	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/suite"
+)
+
+func (s *GleanerRootSuite) TestRunOnSitemapIndex() {
+	defer gock.Off()
+	gock.New("https://geoconnex.us/sitemap.xml").
path: cmd/gleaner/root_test.go
user: login: webb-ben
body: Can you test against https://pids.geoconnex.dev/sitemap.xml? Technically anything in https://geoconnex.us is not stable
created_at: 2025-04-30T12:15:14Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,57 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"testing"
+
+	"nabu/internal/synchronizer/s3"
+
+	"github.com/h2non/gock"
+	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/suite"
+)
+
+func (s *GleanerRootSuite) TestRunOnSitemapIndex() {
+	defer gock.Off()
+	gock.New("https://geoconnex.us/sitemap.xml").
path: cmd/gleaner/root_test.go
user: login: C-Loftus
body: Yup I will definitely for larger tests, this is all mocked so it doesn't end up hitting the real sitemap anyways
created_at: 2025-04-30T12:27:46Z
author_association: MEMBER

PR comments:
user: login: C-Loftus
created_at: 2025-04-29T21:13:27Z
body: stacked PR of https://github.com/internetofwater/nabu/pull/77 into this branch
author_association: MEMBER
user: login: codecov[bot]
created_at: 2025-04-30T12:41:16Z
body: ## [Codecov](https://app.codecov.io/gh/internetofwater/nabu/pull/76?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=internetofwater) Report
Attention: Patch coverage is `47.71242%` with `80 lines` in your changes missing coverage. Please review.
> Project coverage is 49.87%. Comparing base [(`f834097`)](https://app.codecov.io/gh/internetofwater/nabu/commit/f83409787e040798de1185543d911a3e78985f33?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=internetofwater) to head [(`2288bb7`)](https://app.codecov.io/gh/internetofwater/nabu/commit/2288bb7236fc0d9426f1f9f34ed4341eede87239?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=internetofwater).
> Report is 1 commits behind head on master.

| [Files with missing lines](https://app.codecov.io/gh/internetofwater/nabu/pull/76?dropdown=coverage&src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=internetofwater) | Patch % | Lines |
|---|---|---|
| [internal/crawl/sitemap\_index.go](https://app.codecov.io/gh/internetofwater/nabu/pull/76?src=pr&el=tree&filepath=internal%2Fcrawl%2Fsitemap_index.go&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=internetofwater#diff-aW50ZXJuYWwvY3Jhd2wvc2l0ZW1hcF9pbmRleC5nbw==) | 30.43% | [48 Missing :warning: ](https://app.codecov.io/gh/internetofwater/nabu/pull/76?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=internetofwater) |
| [cmd/gleaner/root.go](https://app.codecov.io/gh/internetofwater/nabu/pull/76?src=pr&el=tree&filepath=cmd%2Fgleaner%2Froot.go&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=internetofwater#diff-Y21kL2dsZWFuZXIvcm9vdC5nbw==) | 62.71% | [16 Missing and 6 partials :warning: ](https://app.codecov.io/gh/internetofwater/nabu/pull/76?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=internetofwater) |
| [internal/crawl/sitemap.go](https://app.codecov.io/gh/internetofwater/nabu/pull/76?src=pr&el=tree&filepath=internal%2Fcrawl%2Fsitemap.go&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=internetofwater#diff-aW50ZXJuYWwvY3Jhd2wvc2l0ZW1hcC5nbw==) | 47.05% | [7 Missing and 2 partials :warning: ](https://app.codecov.io/gh/internetofwater/nabu/pull/76?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=internetofwater) |
| [internal/synchronizer/s3/client.go](https://app.codecov.io/gh/internetofwater/nabu/pull/76?src=pr&el=tree&filepath=internal%2Fsynchronizer%2Fs3%2Fclient.go&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=internetofwater#diff-aW50ZXJuYWwvc3luY2hyb25pemVyL3MzL2NsaWVudC5nbw==) | 0.00% | [1 Missing :warning: ](https://app.codecov.io/gh/internetofwater/nabu/pull/76?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=internetofwater) |

<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##           master      #76      +/-   ##
==========================================
- Coverage   51.24%   49.87%   -1.38%     
==========================================
  Files          29       31       +2     
  Lines        1883     2015     +132     
==========================================
+ Hits          965     1005      +40     
- Misses        754      840      +86     
- Partials      164      170       +6     
```

</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/internetofwater/nabu/pull/76?dropdown=coverage&src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=internetofwater).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=internetofwater).

<details><summary> :rocket: New features to boost your workflow: </summary>

- :snowflake: [Test Analytics](https://docs.codecov.com/docs/test-analytics): Detect flaky tests, report on failures, and find test suite problems.
</details>
author_association: NONE