Repository: sa7mon/S3Scanner (language: python)

Issue:
Create fetchRegions cmd
For each of the providers that require a list of regions, go check their documentation for newly-added regions to notify us to add support

* DigitalOcean
  * [/regions API](https://docs.digitalocean.com/reference/api/api-reference/#tag/Regions) (requires bearer token)
  * [Feature Matrix](https://docs.digitalocean.com/products/platform/availability-matrix/#other-product-availability) - parse HTML table
* Linode
  * Anonymous [API](https://api.linode.com/v4/regions) - check if `capabilities` contains `Object Storage`
* Dreamhost
  * Nothing in their docs. Will have to rely on crt.sh or similar to look for DNS records like `objects-us-east-1.dream.io`


Discussion / hints:

Fix patch:
diff --git a/.github/workflows/check-regions.yml b/.github/workflows/check-regions.yml
new file mode 100644
index 0000000..e47c822
--- /dev/null
+++ b/.github/workflows/check-regions.yml
@@ -0,0 +1,17 @@
+name: Check provider regions
+
+on:
+  schedule:
+    - cron: '5 4 * * 3'
+
+jobs:
+  region-check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: 1.19
+      - name: region-check
+        run: go run cmd/regioncheck/regioncheck.go
\ No newline at end of file
diff --git a/Makefile b/Makefile
index e92f305..48cbaa4 100644
--- a/Makefile
+++ b/Makefile
@@ -10,6 +10,9 @@ dev:
 docker-image:
 	docker build -t $(DOCKER_IMAGE):$(VERSION) -f packaging/docker/Dockerfile .
 
+lint:
+	docker run -t --rm -v $(pwd):/app -w /app golangci/golangci-lint:v1.53.3 golangci-lint run -v 
+
 test:
 	go test ./...
 
diff --git a/cmd/regioncheck/regioncheck.go b/cmd/regioncheck/regioncheck.go
new file mode 100644
index 0000000..bc6310b
--- /dev/null
+++ b/cmd/regioncheck/regioncheck.go
@@ -0,0 +1,194 @@
+package main
+
+import (
+	"fmt"
+	"github.com/PuerkitoBio/goquery"
+	"github.com/sa7mon/s3scanner/collection"
+	"github.com/sa7mon/s3scanner/provider"
+	"log"
+	"net"
+	"net/http"
+	"os"
+	"regexp"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+)
+
+// eq compares sorted string slices. Once we move to Golang 1.21, use slices.Equal instead.
+func eq(f []string, s []string) bool {
+	if len(f) != len(s) {
+		return false
+	}
+	for i := range f {
+		if f[i] != s[i] {
+			return false
+		}
+	}
+	return true
+}
+
+// GetRegionsDO fetches regions from the DigitalOcean docs HTML page.
+func GetRegionsDO() ([]string, error) {
+	requestURL := "https://docs.digitalocean.com/products/platform/availability-matrix/#other-product-availability"
+	res, err := http.Get(requestURL)
+	if err != nil {
+		return nil, err
+	}
+	defer res.Body.Close()
+	if res.StatusCode != 200 {
+		return nil, fmt.Errorf("status code error: %d %s", res.StatusCode, res.Status)
+	}
+
+	doc, err := goquery.NewDocumentFromReader(res.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	regions := []string{}
+	doc.Find("h2#other-product-availability + table thead tr th").Each(func(i int, t *goquery.Selection) {
+		regions = append(regions, t.Text())
+	})
+
+	spaces_supported := []bool{}
+	doc.Find("h2#other-product-availability + table tbody tr").Each(func(i int, t *goquery.Selection) {
+		// For each row, check the first cell for a value of "Spaces"
+		rowHeader := t.Find("td").First().Text()
+		if rowHeader == "Spaces" {
+			// For each cell in the "Spaces" row, check if the contents are not empty - meaning Spaces is supported
+			t.Find("td").Each(func(j int, v *goquery.Selection) {
+				supported := v.Text() != ""
+				spaces_supported = append(spaces_supported, supported)
+			})
+		}
+	})
+
+	supported_regions := []string{}
+	for i := 0; i < len(regions); i++ {
+		if regions[i] == "Product" {
+			continue
+		}
+		if spaces_supported[i] {
+			supported_regions = append(supported_regions, strings.ToLower(regions[i]))
+		}
+	}
+
+	// Return slice of region names
+	return supported_regions, nil
+}
+
+// GetRegionsLinode fetches region names from Linode docs HTML page. Linode also provides this info via
+// unauthenticated API (https://api.linode.com/v4/regions) but the region names do not include the trailing digit "-1".
+func GetRegionsLinode() ([]string, error) {
+	requestURL := "https://www.linode.com/docs/products/storage/object-storage/"
+	res, err := http.Get(requestURL)
+	if err != nil {
+		return nil, err
+	}
+	defer res.Body.Close()
+	if res.StatusCode != 200 {
+		return nil, fmt.Errorf("status code error: %d %s", res.StatusCode, res.Status)
+	}
+
+	doc, err := goquery.NewDocumentFromReader(res.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	regions := []string{}
+	doc.Find("h2#availability + p + table tbody tr td:nth-of-type(2)").Each(func(i int, t *goquery.Selection) {
+		regions = append(regions, t.Text())
+	})
+
+	return regions, nil
+}
+
+// GetRegionsDreamhost fetches subdomains of dream.io like 'objects-us-east-1.dream.io' via crt.sh since Dreamhost
+// doesn't have a documentation page listing the regions.
+func GetRegionsDreamhost() ([]string, error) {
+	var domainRe = regexp.MustCompile(`objects-([^\.]+)\.dream\.io`)
+	requestURL := "https://crt.sh/?q=.dream.io"
+	// Request the HTML page.
+	res, err := http.Get(requestURL)
+	if err != nil {
+		return nil, err
+	}
+	defer res.Body.Close()
+	if res.StatusCode != 200 {
+		return nil, fmt.Errorf("status code error: %d %s", res.StatusCode, res.Status)
+	}
+
+	// Load the HTML document
+	doc, err := goquery.NewDocumentFromReader(res.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	certCNs := collection.StringSet{}
+	// For each cell in the Common Name column
+	doc.Find("body > table table tbody tr > td:nth-of-type(5)").Each(func(i int, t *goquery.Selection) {
+		matches := domainRe.FindAllStringSubmatch(t.Text(), -1)
+		for _, match := range matches {
+			if !strings.HasPrefix(match[1], "website-") { // regions like 'objects-website-us-east-1' are not for Object Storage
+				certCNs.Add(match[1])
+			}
+		}
+	})
+
+	// crt.sh may return old or invalid SSL certs
+	// verify regions found resolve and respond on port 443
+	resolvableRegions := []string{}
+	for _, s := range certCNs.Slice() {
+		timeout := 1 * time.Second
+		_, cerr := net.DialTimeout("tcp", fmt.Sprintf("objects-%s.dream.io:443", s), timeout)
+		if cerr == nil {
+			resolvableRegions = append(resolvableRegions, s)
+		}
+
+	}
+
+	return resolvableRegions, nil
+}
+
+func main() {
+	wg := sync.WaitGroup{}
+	wg.Add(3)
+
+	results := map[string][]string{}
+	errors := map[string]error{}
+
+	go func(w *sync.WaitGroup) {
+		results["digitalocean"], errors["digitalocean"] = GetRegionsDO()
+		wg.Done()
+	}(&wg)
+	go func(w *sync.WaitGroup) {
+		results["dreamhost"], errors["dreamhost"] = GetRegionsDreamhost()
+		wg.Done()
+	}(&wg)
+	go func(w *sync.WaitGroup) {
+		results["linode"], errors["linode"] = GetRegionsLinode()
+		wg.Done()
+	}(&wg)
+	wg.Wait()
+
+	exit := 0
+
+	for p, knownRegions := range provider.ProviderRegions {
+		if errors[p] != nil {
+			log.Printf("[%s]: %v\n", p, errors[p])
+			continue
+		}
+		foundRegions := results[p]
+		sort.Strings(foundRegions)
+		sort.Strings(knownRegions)
+
+		if !eq(foundRegions, knownRegions) {
+			log.Printf("[%s] regions differ! Existing: %v, found: %v", p, knownRegions, foundRegions)
+			exit = 1
+		} else {
+			log.Printf("[%s} OK", p)
+		}
+	}
+	os.Exit(exit)
+}
diff --git a/collection/collection.go b/collection/collection.go
new file mode 100644
index 0000000..1ec43bd
--- /dev/null
+++ b/collection/collection.go
@@ -0,0 +1,28 @@
+package collection
+
+// StringSet is a simple implementeation of the Set data structure.
+// An empty struct allegedly takes zero memory
+type StringSet map[string]struct{}
+
+func (ss StringSet) Add(s string) {
+	ss[s] = struct{}{}
+}
+
+func (ss StringSet) Remove(s string) {
+	delete(ss, s)
+}
+
+func (ss StringSet) Has(s string) bool {
+	_, ok := ss[s]
+	return ok
+}
+
+func (ss StringSet) Slice() []string {
+	slice := make([]string, len(ss))
+	i := 0
+	for s := range ss {
+		slice[i] = s
+		i++
+	}
+	return slice
+}
diff --git a/go.mod b/go.mod
index 6f8abb0..90c86cc 100644
--- a/go.mod
+++ b/go.mod
@@ -3,6 +3,7 @@ module github.com/sa7mon/s3scanner
 go 1.19
 
 require (
+	github.com/PuerkitoBio/goquery v1.8.1
 	github.com/aws/aws-sdk-go-v2 v1.21.0
 	github.com/aws/aws-sdk-go-v2/config v1.18.39
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.83
@@ -18,6 +19,7 @@ require (
 )
 
 require (
+	github.com/andybalholm/cascadia v1.3.1 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 // indirect
 	github.com/aws/aws-sdk-go-v2/credentials v1.13.37 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11 // indirect
@@ -52,6 +54,7 @@ require (
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
 	golang.org/x/crypto v0.12.0 // indirect
+	golang.org/x/net v0.10.0 // indirect
 	golang.org/x/sys v0.11.0 // indirect
 	golang.org/x/text v0.12.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
diff --git a/go.sum b/go.sum
index 042da93..8ef3dfc 100644
--- a/go.sum
+++ b/go.sum
@@ -38,6 +38,10 @@ cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3f
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM=
+github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ=
+github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
+github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
 github.com/aws/aws-sdk-go-v2 v1.21.0 h1:gMT0IW+03wtYJhRqTVYn0wLzwdnK9sRMcxmtfGzRdJc=
 github.com/aws/aws-sdk-go-v2 v1.21.0/go.mod h1:/RfNgGmRxI+iFOB1OeJUyxiU+9s88k3pfHvDagGEp0M=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 h1:OPLEkmhXf6xFPiz0bLeDArZIDx1NNS4oJyG4nv3Gct0=
@@ -231,6 +235,7 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
@@ -243,6 +248,7 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk=
 golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw=
@@ -279,6 +285,7 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -310,7 +317,12 @@ golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwY
 golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
+golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -330,6 +342,7 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -367,11 +380,16 @@ golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
 golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -380,6 +398,7 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc=
 golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -432,6 +451,7 @@ golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4f
 golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
diff --git a/provider/digitalocean.go b/provider/digitalocean.go
index 82a199e..7260102 100644
--- a/provider/digitalocean.go
+++ b/provider/digitalocean.go
@@ -10,7 +10,6 @@ import (
 )
 
 type providerDO struct {
-	regions []string
 	clients *clientmap.ClientMap
 }
 
@@ -60,18 +59,10 @@ func (pdo providerDO) Enumerate(b *bucket.Bucket) error {
 	return nil
 }
 
-func (pdo *providerDO) Regions() []string {
-	urls := make([]string, len(pdo.regions))
-	for i, r := range pdo.regions {
-		urls[i] = fmt.Sprintf("https://%s.digitaloceanspaces.com", r)
-	}
-	return urls
-}
-
 func (pdo *providerDO) newClients() (*clientmap.ClientMap, error) {
-	clients := clientmap.WithCapacity(len(pdo.regions))
-	for _, r := range pdo.Regions() {
-		client, err := newNonAWSClient(pdo, r)
+	clients := clientmap.WithCapacity(len(ProviderRegions[pdo.Name()]))
+	for _, r := range ProviderRegions[pdo.Name()] {
+		client, err := newNonAWSClient(pdo, fmt.Sprintf("https://%s.digitaloceanspaces.com", r))
 		if err != nil {
 			return nil, err
 		}
@@ -87,7 +78,6 @@ func (pdo *providerDO) getRegionClient(region string) *s3.Client {
 
 func NewProviderDO() (*providerDO, error) {
 	pdo := new(providerDO)
-	pdo.regions = []string{"nyc3", "sfo2", "sfo3", "ams3", "sgp1", "fra1", "syd1"}
 
 	clients, err := pdo.newClients()
 	if err != nil {
diff --git a/provider/dreamhost.go b/provider/dreamhost.go
index 8803001..3528583 100644
--- a/provider/dreamhost.go
+++ b/provider/dreamhost.go
@@ -3,14 +3,12 @@ package provider
 import (
 	"errors"
 	"fmt"
-
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/sa7mon/s3scanner/bucket"
 	"github.com/sa7mon/s3scanner/provider/clientmap"
 )
 
 type ProviderDreamhost struct {
-	regions []string
 	clients *clientmap.ClientMap
 }
 
@@ -64,18 +62,10 @@ func (p ProviderDreamhost) Enumerate(b *bucket.Bucket) error {
 	return nil
 }
 
-func (p ProviderDreamhost) Regions() []string {
-	urls := make([]string, len(p.regions))
-	for i, r := range p.regions {
-		urls[i] = fmt.Sprintf("https://objects-%s.dream.io", r)
-	}
-	return urls
-}
-
 func (p *ProviderDreamhost) newClients() (*clientmap.ClientMap, error) {
-	clients := clientmap.WithCapacity(len(p.regions))
-	for _, r := range p.Regions() {
-		client, err := newNonAWSClient(p, r)
+	clients := clientmap.WithCapacity(len(ProviderRegions[p.Name()]))
+	for _, r := range ProviderRegions[p.Name()] {
+		client, err := newNonAWSClient(p, fmt.Sprintf("https://objects-%s.dream.io", r))
 		if err != nil {
 			return nil, err
 		}
@@ -87,7 +77,6 @@ func (p *ProviderDreamhost) newClients() (*clientmap.ClientMap, error) {
 
 func NewProviderDreamhost() (*ProviderDreamhost, error) {
 	pd := new(ProviderDreamhost)
-	pd.regions = []string{"us-east-1"}
 
 	clients, err := pd.newClients()
 	if err != nil {
diff --git a/provider/linode.go b/provider/linode.go
index 067a443..7cb4ad8 100644
--- a/provider/linode.go
+++ b/provider/linode.go
@@ -10,13 +10,11 @@ import (
 )
 
 type providerLinode struct {
-	regions []string
 	clients *clientmap.ClientMap
 }
 
 func NewProviderLinode() (*providerLinode, error) {
 	pl := new(providerLinode)
-	pl.regions = []string{"us-east-1", "us-southeast-1", "eu-central-1", "ap-south-1"}
 
 	clients, err := pl.newClients()
 	if err != nil {
@@ -60,9 +58,9 @@ func (pl *providerLinode) Enumerate(b *bucket.Bucket) error {
 }
 
 func (pl *providerLinode) newClients() (*clientmap.ClientMap, error) {
-	clients := clientmap.WithCapacity(len(pl.regions))
-	for _, r := range pl.Regions() {
-		client, err := newNonAWSClient(pl, r)
+	clients := clientmap.WithCapacity(len(ProviderRegions[pl.Name()]))
+	for _, r := range ProviderRegions[pl.Name()] {
+		client, err := newNonAWSClient(pl, fmt.Sprintf("https://%s.linodeobjects.com", r))
 		if err != nil {
 			return nil, err
 		}
@@ -85,14 +83,6 @@ func (*providerLinode) Name() string {
 	return "linode"
 }
 
-func (pl *providerLinode) Regions() []string {
-	urls := make([]string, len(pl.regions))
-	for i, r := range pl.regions {
-		urls[i] = fmt.Sprintf("https://%s.linodeobjects.com", r)
-	}
-	return urls
-}
-
 func (*providerLinode) AddressStyle() int {
 	return VirtualHostStyle
 }
diff --git a/provider/providers.go b/provider/providers.go
index 800927e..65c8782 100644
--- a/provider/providers.go
+++ b/provider/providers.go
@@ -43,6 +43,12 @@ var AllProviders = []string{
 	"aws", "custom", "digitalocean", "dreamhost", "gcp", "linode",
 }
 
+var ProviderRegions = map[string][]string{
+	"digitalocean": {"ams3", "blr1", "fra1", "nyc3", "sfo2", "sfo3", "sgp1", "syd1"},
+	"dreamhost":    {"us-east-1"},
+	"linode":       {"ap-south-1", "eu-central-1", "fr-par-1", "se-sto-1", "us-east-1", "us-iad-1", "us-ord-1", "us-southeast-1"},
+}
+
 func NewProvider(name string) (StorageProvider, error) {
 	var (
 		provider StorageProvider


Test patch:
diff --git a/cmd/regioncheck/regioncheck_test.go b/cmd/regioncheck/regioncheck_test.go
new file mode 100644
index 0000000..3567320
--- /dev/null
+++ b/cmd/regioncheck/regioncheck_test.go
@@ -0,0 +1,27 @@
+package main
+
+import (
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func TestGetRegionsDO(t *testing.T) {
+	r, err := GetRegionsDO()
+	assert.Nil(t, err)
+	assert.GreaterOrEqual(t, len(r), 1)
+	assert.Contains(t, r, "nyc3")
+}
+
+func TestGetRegionsLinode(t *testing.T) {
+	r, err := GetRegionsLinode()
+	assert.Nil(t, err)
+	assert.GreaterOrEqual(t, len(r), 1)
+	assert.Contains(t, r, "us-east-1")
+}
+
+func TestGetRegionsDreamhost(t *testing.T) {
+	dor, err := GetRegionsDreamhost()
+	assert.Nil(t, err)
+	assert.GreaterOrEqual(t, len(dor), 1)
+	assert.Contains(t, dor, "us-east-1")
+}