Repository: sa7mon/S3Scanner
Pull request #376 (2024-12-28 05:52:29)

chore: update region checks
Update linode, dreamhost, and digitalocean region checks

github-actions[bot]: ![Code Coverage](https://img.shields.io/badge/Code%20Coverage-58%25-yellow?style=flat)

Package | Line Rate | Complexity | Health
-------- | --------- | ---------- | ------
github.com/sa7mon/s3scanner | 0% | 0 | ❌
github.com/sa7mon/s3scanner/bucket | 91% | 0 | ✔
github.com/sa7mon/s3scanner/cmd/mqingest | 0% | 0 | ❌
github.com/sa7mon/s3scanner/cmd/regioncheck | 58% | 0 | ➖
github.com/sa7mon/s3scanner/cmd/s3scanner | 19% | 0 | ❌
github.com/sa7mon/s3scanner/collection | 0% | 0 | ❌
github.com/sa7mon/s3scanner/db | 73% | 0 | ➖
github.com/sa7mon/s3scanner/log | 0% | 0 | ❌
github.com/sa7mon/s3scanner/mq | 71% | 0 | ➖
github.com/sa7mon/s3scanner/permission | 28% | 0 | ❌
github.com/sa7mon/s3scanner/provider | 79% | 0 | ✔
github.com/sa7mon/s3scanner/provider/clientmap | 100% | 0 | ✔
github.com/sa7mon/s3scanner/worker | 49% | 0 | ❌
**Summary** | **58%** (632 / 1090) | **0** | ➖

<!-- Sticky Pull Request Comment -->

Patch:
diff --git a/cmd/regioncheck/regioncheck.go b/cmd/regioncheck/regioncheck.go
index bcd2238..ac10a34 100644
--- a/cmd/regioncheck/regioncheck.go
+++ b/cmd/regioncheck/regioncheck.go
@@ -1,6 +1,7 @@
 package main
 
 import (
+	"compress/gzip"
 	"fmt"
 	"github.com/PuerkitoBio/goquery"
 	"github.com/sa7mon/s3scanner/provider"
@@ -45,12 +46,12 @@ func GetRegionsDO() ([]string, error) {
 	}
 
 	regions := []string{}
-	doc.Find("h3#other-products + table thead tr th").Each(func(_ int, t *goquery.Selection) {
+	doc.Find("h2#other-digitalocean-products + table thead tr th").Each(func(_ int, t *goquery.Selection) {
 		regions = append(regions, t.Text())
 	})
 
 	spacesSupported := []bool{}
-	doc.Find("h3#other-products + table tbody tr").Each(func(_ int, t *goquery.Selection) {
+	doc.Find("h2#other-digitalocean-products + table tbody tr").Each(func(_ int, t *goquery.Selection) {
 		// For each row, check the first cell for a value of "Spaces"
 		rowHeader := t.Find("td").First().Text()
 		if rowHeader == "Spaces" {
@@ -76,30 +77,54 @@ func GetRegionsDO() ([]string, error) {
 	return supportedRegions, nil
 }
 
+func GetRegionsDreamhost() ([]string, error) {
+	return []string{"us-east-1"}, nil
+}
+
 // GetRegionsLinode fetches region names from Linode docs HTML page. Linode also provides this info via
 // unauthenticated API (https://api.linode.com/v4/regions) but the region names do not include the trailing digit "-1".
 func GetRegionsLinode() ([]string, error) {
-	requestURL := "https://techdocs.akamai.com/cloud-computing/docs/object-storage"
-
-	req, err := http.NewRequest("GET", requestURL, nil)
+	// Akamai docs return a strange HTTP2 internal error if you don't request HTTP/2 with compression
+	req, err := http.NewRequest(http.MethodGet, "https://techdocs.akamai.com/cloud-computing/docs/object-storage", nil)
 	if err != nil {
 		return nil, err
 	}
-	// server drops requests if user agent is set to the default go-http-client:
-	// `HTTP/2 stream 1 was not closed cleanly: INTERNAL_ERROR (err 2)`
-	req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36")
+	req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:128.0) Gecko/20100101 Firefox/128.0")
+	req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/png,image/svg+xml,*/*;q=0.8")
+	req.Header.Set("Accept-Language", "en-US,en;q=0.5")
+	req.Header.Set("Accept-Encoding", "gzip, deflate, br, zstd")
+	req.Header.Set("Connection", "keep-alive")
 
-	client := &http.Client{}
-	res, err := client.Do(req)
+	resp, err := http.DefaultClient.Do(req)
 	if err != nil {
 		return nil, err
 	}
-	defer res.Body.Close()
-	if res.StatusCode != 200 {
-		return nil, fmt.Errorf("status code error: %d %s", res.StatusCode, res.Status)
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		return nil, fmt.Errorf("status code error: %d %s", resp.StatusCode, resp.Status)
 	}
 
-	doc, err := goquery.NewDocumentFromReader(res.Body)
+	// Check that the server actually sent compressed data
+	var reader io.ReadCloser
+	switch resp.Header.Get("Content-Encoding") {
+	case "gzip":
+		reader, err = gzip.NewReader(resp.Body)
+		if err != nil {
+			return nil, err
+		}
+		defer reader.Close()
+	default:
+		reader = resp.Body
+	}
+
+	buf := new(strings.Builder)
+	_, err = io.Copy(buf, reader) //nolint:gosec
+	if err != nil {
+		return nil, err
+	}
+
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(buf.String()))
 	if err != nil {
 		return nil, err
 	}
@@ -142,6 +167,7 @@ func main() {
 
 	p := map[string]func() ([]string, error){
 		"digitalocean": GetRegionsDO,
+		"dreamhost":    GetRegionsDreamhost,
 		"linode":       GetRegionsLinode,
 		"scaleway":     GetRegionsScaleway,
 	}
diff --git a/provider/providers.go b/provider/providers.go
index 4d5c14f..efb2452 100644
--- a/provider/providers.go
+++ b/provider/providers.go
@@ -44,7 +44,7 @@ var AllProviders = []string{
 }
 
 var ProviderRegions = map[string][]string{
-	"digitalocean": {"ams3", "blr1", "fra1", "lon1", "nyc3", "sfo2", "sfo3", "sgp1", "syd1"},
+	"digitalocean": {"ams3", "blr1", "fra1", "lon1", "nyc3", "sfo2", "sfo3", "sgp1", "syd1", "tor1"},
 	"dreamhost":    {"us-east-1"},
 	"linode": {"ap-south-1", "br-gru-1", "es-mad-1", "eu-central-1", "fr-par-1", "id-cgk-1", "in-maa-1",
 		"it-mil-1", "jp-osa-1", "nl-ams-1", "se-sto-1", "us-east-1", "us-iad-1", "us-lax-1", "us-mia-1",


Test patch:
diff --git a/provider/providers_test.go b/provider/providers_test.go
index 5e05243..27efbf1 100644
--- a/provider/providers_test.go
+++ b/provider/providers_test.go
@@ -175,7 +175,7 @@ func Test_StorageProvider_Enum(t *testing.T) {
 		{name: "DO", provider: providers["digitalocean"], goodBucket: bucket.NewBucket("action"), numObjects: 2},
 		{name: "Dreamhost", provider: providers["dreamhost"], goodBucket: bucket.NewBucket("acc"), numObjects: 310},
 		{name: "GCP", provider: providers["gcp"], goodBucket: bucket.NewBucket("assets"), numObjects: 3},
-		{name: "Linode", provider: providers["linode"], goodBucket: bucket.NewBucket("vantage"), numObjects: 49},
+		{name: "Linode", provider: providers["linode"], goodBucket: bucket.NewBucket("vantage"), numObjects: 50},
 		{name: "Scaleway", provider: providers["scaleway"], goodBucket: bucket.NewBucket("3d-builder"), numObjects: 1},
 	}