Repository: gomods/athens
Pull request #1630 (2020-06-22 02:36:38)

implement /index endpoint
This is the first of possibly a couple of more PRs to try and integrate pkg.go.dev (pkgsite) with Athens. 

Pkgsite uses https://index.golang.org to discover new module versions so that they are searchable and are parsed ahead of time. More on pkgsite's design [here](https://github.com/golang/pkgsite/blob/master/doc/design.md). 

This PR adds a `/index` endpoint that tries to adhere to the same logic and format in `index.golang.org/index`

To support a /index endpoint we create a new Indexer interface: 

```
type Indexer interface {
	Index(ctx context.Context, mod, ver string) error
	Lines(ctx context.Context, since time.Time, limit int) ([]*Line, error)
}
```

This is different (and separate) from a storage.Backend for one important reason: 

- A storage backend might not store modules in a chronological/append-only way. For example, blob storage stores modules based on path names, and not in an append only fashion. 

Furthermore, a storage backend stores a module's go.mod file and the entire zip while an index backend only cares about indexing a module's metadata. This allows us to add index backends that make sense for indexing and storing metadata but would not make sense for storing large zips, such as SQL stores. 

This PR adds 3 implementations of an index backend: memory, MySQL, and Postgres, defaulting to none to keep backwards compatibility (though ideally, I'd love to have /index be turned on and using in-memory by default so people can play with the endpoint immediately). 

More index backends can be added as seen fit but ideally we'll also add an external index backend similar to pkg/storage/external so users can bring in their own solution without the maintenance overhead. 

Lastly, this PR also adds compliance tests to ensure all implementations comply to the interface expectations but I will probably need to add more test cases, handler tests, and integration tests as follow up PRs.

Updates https://github.com/gomods/athens/issues/1186
Updates https://github.com/gomods/athens/issues/1546

marwan-at-work: @twexler thank you for the review 🙏 -- All comments should be addressed, let me know if it looks good to you :v: 

Patch:
diff --git a/appveyor.yml b/appveyor.yml
index 2da8d497..58917515 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -2,6 +2,8 @@ build: off
 
 clone_folder: c:\gopath\src\github.com\gomods\athens
 
+image: Previous Visual Studio 2019
+
 environment:
   GOPATH: c:\gopath
   GO111MODULE: on
@@ -11,5 +13,6 @@ environment:
 stack: go 1.14
 
 test_script:
+  - go version
   - go test ./...
 
diff --git a/cmd/proxy/actions/app_proxy.go b/cmd/proxy/actions/app_proxy.go
index 7a2e2685..3d1e3b7f 100644
--- a/cmd/proxy/actions/app_proxy.go
+++ b/cmd/proxy/actions/app_proxy.go
@@ -11,6 +11,11 @@ import (
 	"github.com/gomods/athens/pkg/download"
 	"github.com/gomods/athens/pkg/download/addons"
 	"github.com/gomods/athens/pkg/download/mode"
+	"github.com/gomods/athens/pkg/index"
+	"github.com/gomods/athens/pkg/index/mem"
+	"github.com/gomods/athens/pkg/index/mysql"
+	"github.com/gomods/athens/pkg/index/nop"
+	"github.com/gomods/athens/pkg/index/postgres"
 	"github.com/gomods/athens/pkg/log"
 	"github.com/gomods/athens/pkg/module"
 	"github.com/gomods/athens/pkg/stash"
@@ -32,6 +37,12 @@ func addProxyRoutes(
 	r.HandleFunc("/catalog", catalogHandler(s))
 	r.HandleFunc("/robots.txt", robotsHandler(c))
 
+	indexer, err := getIndex(c)
+	if err != nil {
+		return err
+	}
+	r.HandleFunc("/index", indexHandler(indexer))
+
 	for _, sumdb := range c.SumDBs {
 		sumdbURL, err := url.Parse(sumdb)
 		if err != nil {
@@ -96,7 +107,7 @@ func addProxyRoutes(
 	if err != nil {
 		return err
 	}
-	st := stash.New(mf, s, stash.WithPool(c.GoGetWorkers), withSingleFlight)
+	st := stash.New(mf, s, indexer, stash.WithPool(c.GoGetWorkers), withSingleFlight)
 
 	df, err := mode.NewFile(c.DownloadMode, c.DownloadURL)
 	if err != nil {
@@ -157,3 +168,17 @@ func getSingleFlight(c *config.Config, checker storage.Checker) (stash.Wrapper,
 		return nil, fmt.Errorf("unrecognized single flight type: %v", c.SingleFlightType)
 	}
 }
+
+func getIndex(c *config.Config) (index.Indexer, error) {
+	switch c.IndexType {
+	case "", "none":
+		return nop.New(), nil
+	case "memory":
+		return mem.New(), nil
+	case "mysql":
+		return mysql.New(c.Index.MySQL)
+	case "postgres":
+		return postgres.New(c.Index.Postgres)
+	}
+	return nil, fmt.Errorf("unknown index type: %q", c.IndexType)
+}
diff --git a/cmd/proxy/actions/index.go b/cmd/proxy/actions/index.go
new file mode 100644
index 00000000..3be942c0
--- /dev/null
+++ b/cmd/proxy/actions/index.go
@@ -0,0 +1,52 @@
+package actions
+
+import (
+	"encoding/json"
+	"net/http"
+	"strconv"
+	"time"
+
+	"github.com/gomods/athens/pkg/errors"
+	"github.com/gomods/athens/pkg/index"
+)
+
+// indexHandler implements GET baseURL/index
+func indexHandler(index index.Indexer) http.HandlerFunc {
+	const op errors.Op = "actions.IndexHandler"
+	return func(w http.ResponseWriter, r *http.Request) {
+		var (
+			err   error
+			limit int
+			since time.Time
+		)
+		if limitStr := r.FormValue("limit"); limitStr != "" {
+			limit, err = strconv.Atoi(limitStr)
+			if err != nil {
+				http.Error(w, err.Error(), http.StatusBadRequest)
+				return
+			}
+		}
+		if sinceStr := r.FormValue("since"); sinceStr != "" {
+			since, err = time.Parse(time.RFC3339, sinceStr)
+			if err != nil {
+				http.Error(w, err.Error(), http.StatusBadRequest)
+				return
+			}
+		}
+		if limit <= 0 {
+			limit = 2000
+		}
+		list, err := index.Lines(r.Context(), since, limit)
+		if err != nil {
+			http.Error(w, err.Error(), 500)
+			return
+		}
+		enc := json.NewEncoder(w)
+		for _, meta := range list {
+			if err = enc.Encode(meta); err != nil {
+				http.Error(w, err.Error(), 500)
+				return
+			}
+		}
+	}
+}
diff --git a/cmd/proxy/actions/storage.go b/cmd/proxy/actions/storage.go
index 6dceb1a0..b1e51b5a 100644
--- a/cmd/proxy/actions/storage.go
+++ b/cmd/proxy/actions/storage.go
@@ -21,7 +21,7 @@ import (
 )
 
 // GetStorage returns storage backend based on env configuration
-func GetStorage(storageType string, storageConfig *config.StorageConfig, timeout time.Duration, client *http.Client) (storage.Backend, error) {
+func GetStorage(storageType string, storageConfig *config.Storage, timeout time.Duration, client *http.Client) (storage.Backend, error) {
 	const op errors.Op = "actions.GetStorage"
 	switch storageType {
 	case "memory":
diff --git a/config.dev.toml b/config.dev.toml
index b27e1291..cf579ecb 100755
--- a/config.dev.toml
+++ b/config.dev.toml
@@ -282,6 +282,12 @@ DownloadURL = ""
 # Env override: ATHENS_SINGLE_FLIGHT_TYPE
 SingleFlightType = "memory"
 
+# IndexType sets the type of an index backend Athens will use.
+# Possible values are none, memory, mysql, postgres
+# Defaults to none
+# Env override: ATHENS_INDEX_TYPE
+IndexType = "none"
+
 [SingleFlight]
     [SingleFlight.Etcd]
         # Endpoints are comma separated URLs that determine all distributed etcd servers.
@@ -475,3 +481,65 @@ SingleFlightType = "memory"
         # details.
         # Env override: ATHENS_EXTERNAL_STORAGE_URL
         URL = ""
+
+[Index]
+    [Index.MySQL]
+        # MySQL protocol
+        # Env override: ATHENS_INDEX_MYSQL_PROTOCOL
+        Protocol = "tcp"
+
+        # MySQL user name
+        # Env override: ATHENS_INDEX_MYSQL_HOST
+        Host = "localhost"
+
+        # MySQL user name
+        # Env override: ATHENS_INDEX_MYSQL_PORT
+        Port = 3306
+
+        # MySQL user name
+        # Env override: ATHENS_INDEX_MYSQL_USER
+        User = "root"
+
+        # MySQL user name
+        # Env override: ATHENS_INDEX_MYSQL_PASSWORD
+        Password = ""
+
+        # MySQL database
+        # Env override: ATHENS_INDEX_MYSQL_DATABASE
+        Database = "athens"
+
+        # MySQL query parameters
+        # Environment overrides must be in the following format:
+        # ATHENS_INDEX_MYSQL_PARAMS="parseTime:true,timeout=90s"
+        # Env override: ATHENS_INDEX_MYSQL_PARAMS
+        [Index.MySQL.Params]
+            parseTime = "true"
+            timeout = "30s"
+    [Index.Postgres]
+        # Postgres user name
+        # Env override: ATHENS_INDEX_POSTGRES_HOST
+        Host = "localhost"
+
+        # Postgres user name
+        # Env override: ATHENS_INDEX_POSTGRES_PORT
+        Port = 5432
+
+        # Postgres user name
+        # Env override: ATHENS_INDEX_POSTGRES_USER
+        User = "postgres"
+
+        # Postgres user name
+        # Env override: ATHENS_INDEX_POSTGRES_PASSWORD
+        Password = ""
+
+        # Postgres database
+        # Env override: ATHENS_INDEX_POSTGRES_DATABASE
+        Database = "athens"
+
+        # Postgres query parameters
+        # Environment overrides must be in the following format:
+        # ATHENS_INDEX_POSTGRES_PARAMS="connect_timeout:30s,sslmode=disable"
+        # Env override: ATHENS_INDEX_POSTGRES_PARAMS
+        [Index.Postgres.Params]
+            connect_timeout = "30s"
+            sslmode = "disable"
diff --git a/go.mod b/go.mod
index 35017d0c..3cf48d1b 100644
--- a/go.mod
+++ b/go.mod
@@ -17,10 +17,10 @@ require (
 	github.com/bsm/redislock v0.4.2
 	github.com/codegangsta/negroni v1.0.0 // indirect
 	github.com/fatih/color v1.7.0
-	github.com/go-ini/ini v1.25.4 // indirect
 	github.com/go-playground/locales v0.12.1 // indirect
 	github.com/go-playground/universal-translator v0.16.0 // indirect
 	github.com/go-redis/redis/v7 v7.2.0
+	github.com/go-sql-driver/mysql v1.5.0
 	github.com/gobuffalo/envy v1.6.7
 	github.com/gobuffalo/httptest v1.0.4
 	github.com/golang/protobuf v1.3.3 // indirect
@@ -32,6 +32,7 @@ require (
 	github.com/hashicorp/hcl2 v0.0.0-20190503213020-640445e16309
 	github.com/kelseyhightower/envconfig v1.3.0
 	github.com/kylelemons/godebug v1.1.0 // indirect
+	github.com/lib/pq v1.7.0
 	github.com/minio/minio-go/v6 v6.0.43
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/philhofer/fwd v1.0.0 // indirect
diff --git a/go.sum b/go.sum
index 06ab9913..d8cc4cac 100644
--- a/go.sum
+++ b/go.sum
@@ -68,8 +68,6 @@ github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb
 github.com/apparentlymart/go-dump v0.0.0-20180507223929-23540a00eaa3/go.mod h1:oL81AME2rN47vu18xqj1S1jPIPuN7afo62yKTNn3XMM=
 github.com/apparentlymart/go-textseg v1.0.0 h1:rRmlIsPEEhUTIKQb7T++Nz/A5Q6C9IuX2wFoYVvnCs0=
 github.com/apparentlymart/go-textseg v1.0.0/go.mod h1:z96Txxhf3xSFMPmb5X/1W05FF/Nj9VFpLOpjS5yuumk=
-github.com/aws/aws-sdk-go v1.15.24 h1:xLAdTA/ore6xdPAljzZRed7IGqQgC+nY+ERS5vaj4Ro=
-github.com/aws/aws-sdk-go v1.15.24/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0=
 github.com/aws/aws-sdk-go v1.32.7 h1:H4VgdCSF1cHw0VD8zGc98T1bGdACoLkh/vK2L6wgOUU=
 github.com/aws/aws-sdk-go v1.32.7/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
@@ -113,8 +111,6 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-ini/ini v1.25.4 h1:Mujh4R/dH6YL8bxuISne3xX2+qcQ9p0IxKAP6ExWoUo=
-github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-playground/locales v0.12.1 h1:2FITxuFt/xuCNP1Acdhv62OzaCiviiE4kotfhkmOqEc=
@@ -124,6 +120,7 @@ github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEK
 github.com/go-redis/redis/v7 v7.0.0-beta.4/go.mod h1:xhhSbUMTsleRPur+Vgx9sUHtyN33bdjxY+9/0n9Ig8s=
 github.com/go-redis/redis/v7 v7.2.0 h1:CrCexy/jYWZjW0AyVoHlcJUeZN19VWlbepTh1Vq6dJs=
 github.com/go-redis/redis/v7 v7.2.0/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg=
+github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
@@ -216,8 +213,6 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
-github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8 h1:12VvqtR6Aowv3l/EQUlocDHW2Cp4G9WJVH7uyH8QFJE=
-github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc=
 github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
 github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc=
@@ -246,6 +241,8 @@ github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3v
 github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+github.com/lib/pq v1.7.0 h1:h93mCPfUSkaul3Ka/VG8uZdmW1uMHDGxzu0NWHuJmHY=
+github.com/lib/pq v1.7.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/markbates/hmax v1.0.0 h1:yo2N0gBoCnUMKhV/VRLHomT6Y9wUm+oQQENuWJqCdlM=
 github.com/markbates/hmax v1.0.0/go.mod h1:cOkR9dktiESxIMu+65oc/r/bdY4bE8zZw3OLhLx0X2c=
 github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4=
@@ -286,8 +283,7 @@ github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG
 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
 github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
-github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -338,6 +334,7 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/technosophos/moniker v0.0.0-20180509230615-a5dbd03a2245 h1:DNVk+NIkGS0RbLkjQOLCJb/759yfCysThkMbl7EXxyY=
 github.com/technosophos/moniker v0.0.0-20180509230615-a5dbd03a2245/go.mod h1:O1c8HleITsZqzNZDjSNzirUGsMT0oGu9LhHKoJrqO+A=
diff --git a/pkg/config/config.go b/pkg/config/config.go
index 2975e942..108c0a6a 100644
--- a/pkg/config/config.go
+++ b/pkg/config/config.go
@@ -54,8 +54,10 @@ type Config struct {
 	DownloadURL      string    `envconfig:"ATHENS_DOWNLOAD_URL"`
 	SingleFlightType string    `envconfig:"ATHENS_SINGLE_FLIGHT_TYPE"`
 	RobotsFile       string    `envconfig:"ATHENS_ROBOTS_FILE"`
+	IndexType        string    `envconfig:"ATHENS_INDEX_TYPE"`
 	SingleFlight     *SingleFlight
-	Storage          *StorageConfig
+	Storage          *Storage
+	Index            *Index
 }
 
 // EnvList is a list of key-value environment
@@ -161,6 +163,7 @@ func defaultConfig() *Config {
 		DownloadMode:     "sync",
 		DownloadURL:      "",
 		RobotsFile:       "robots.txt",
+		IndexType:        "none",
 		SingleFlight: &SingleFlight{
 			Etcd:  &Etcd{"localhost:2379,localhost:22379,localhost:32379"},
 			Redis: &Redis{"127.0.0.1:6379", ""},
@@ -170,6 +173,31 @@ func defaultConfig() *Config {
 				SentinelPassword: "sekret",
 			},
 		},
+		Index: &Index{
+			MySQL: &MySQL{
+				Protocol: "tcp",
+				Host:     "localhost",
+				Port:     3306,
+				User:     "root",
+				Password: "",
+				Database: "athens",
+				Params: map[string]string{
+					"parseTime": "true",
+					"timeout":   "30s",
+				},
+			},
+			Postgres: &Postgres{
+				Host:     "localhost",
+				Port:     5432,
+				User:     "postgres",
+				Password: "",
+				Database: "athens",
+				Params: map[string]string{
+					"connect_timeout": "30",
+					"sslmode":         "disable",
+				},
+			},
+		},
 	}
 }
 
@@ -267,29 +295,54 @@ func ensurePortFormat(s string) string {
 
 func validateConfig(config Config) error {
 	validate := validator.New()
-	err := validate.StructExcept(config, "Storage")
+	err := validate.StructExcept(config, "Storage", "Index")
+	if err != nil {
+		return err
+	}
+	err = validateStorage(validate, config.StorageType, config.Storage)
+	if err != nil {
+		return err
+	}
+	err = validateIndex(validate, config.IndexType, config.Index)
 	if err != nil {
 		return err
 	}
-	switch config.StorageType {
+	return nil
+}
+
+func validateStorage(validate *validator.Validate, storageType string, config *Storage) error {
+	switch storageType {
 	case "memory":
 		return nil
 	case "mongo":
-		return validate.Struct(config.Storage.Mongo)
+		return validate.Struct(config.Mongo)
 	case "disk":
-		return validate.Struct(config.Storage.Disk)
+		return validate.Struct(config.Disk)
 	case "minio":
-		return validate.Struct(config.Storage.Minio)
+		return validate.Struct(config.Minio)
 	case "gcp":
-		return validate.Struct(config.Storage.GCP)
+		return validate.Struct(config.GCP)
 	case "s3":
-		return validate.Struct(config.Storage.S3)
+		return validate.Struct(config.S3)
 	case "azureblob":
-		return validate.Struct(config.Storage.AzureBlob)
+		return validate.Struct(config.AzureBlob)
 	case "external":
-		return validate.Struct(config.Storage.External)
+		return validate.Struct(config.External)
+	default:
+		return fmt.Errorf("storage type %q is unknown", storageType)
+	}
+}
+
+func validateIndex(validate *validator.Validate, indexType string, config *Index) error {
+	switch indexType {
+	case "", "none", "memory":
+		return nil
+	case "mysql":
+		return validate.Struct(config.MySQL)
+	case "postgres":
+		return validate.Struct(config.Postgres)
 	default:
-		return fmt.Errorf("storage type %s is unknown", config.StorageType)
+		return fmt.Errorf("index type %q is unknown", indexType)
 	}
 }
 
diff --git a/pkg/config/index.go b/pkg/config/index.go
new file mode 100644
index 00000000..ed050ea9
--- /dev/null
+++ b/pkg/config/index.go
@@ -0,0 +1,7 @@
+package config
+
+// Index is the config for various index storage backends
+type Index struct {
+	MySQL    *MySQL
+	Postgres *Postgres
+}
diff --git a/pkg/config/mysql.go b/pkg/config/mysql.go
new file mode 100644
index 00000000..3234bc10
--- /dev/null
+++ b/pkg/config/mysql.go
@@ -0,0 +1,12 @@
+package config
+
+// MySQL config
+type MySQL struct {
+	Protocol string            `validate:"required" envconfig:"ATHENS_INDEX_MYSQL_PROTOCOL"`
+	Host     string            `validate:"required" envconfig:"ATHENS_INDEX_MYSQL_HOST"`
+	Port     int               `validate:"" envconfig:"ATHENS_INDEX_MYSQL_PORT"`
+	User     string            `validate:"required" envconfig:"ATHENS_INDEX_MYSQL_USER"`
+	Password string            `validate:"" envconfig:"ATHENS_INDEX_MYSQL_PASSWORD"`
+	Database string            `validate:"required" envconfig:"ATHENS_INDEX_MYSQL_DATABASE"`
+	Params   map[string]string `validate:"required" envconfig:"ATHENS_INDEX_MYSQL_PARAMS"`
+}
diff --git a/pkg/config/postgres.go b/pkg/config/postgres.go
new file mode 100644
index 00000000..4a4c07fc
--- /dev/null
+++ b/pkg/config/postgres.go
@@ -0,0 +1,11 @@
+package config
+
+// Postgres config
+type Postgres struct {
+	Host     string            `validate:"required" envconfig:"ATHENS_INDEX_POSTGRES_HOST"`
+	Port     int               `validate:"required" envconfig:"ATHENS_INDEX_POSTGRES_PORT"`
+	User     string            `validate:"required" envconfig:"ATHENS_INDEX_POSTGRES_USER"`
+	Password string            `validate:"" envconfig:"ATHENS_INDEX_POSTGRES_PASSWORD"`
+	Database string            `validate:"required" envconfig:"ATHENS_INDEX_POSTGRES_DATABASE"`
+	Params   map[string]string `validate:"required" envconfig:"ATHENS_INDEX_POSTGRES_PARAMS"`
+}
diff --git a/pkg/config/storage.go b/pkg/config/storage.go
index 749977bd..9dd8ffa9 100644
--- a/pkg/config/storage.go
+++ b/pkg/config/storage.go
@@ -1,7 +1,7 @@
 package config
 
-// StorageConfig provides configs for various storage backends
-type StorageConfig struct {
+// Storage provides configs for various storage backends
+type Storage struct {
 	Disk      *DiskConfig
 	GCP       *GCPConfig
 	Minio     *MinioConfig
diff --git a/pkg/index/compliance/compliance.go b/pkg/index/compliance/compliance.go
new file mode 100644
index 00000000..6c1fdf1b
--- /dev/null
+++ b/pkg/index/compliance/compliance.go
@@ -0,0 +1,118 @@
+package compliance
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/gomods/athens/pkg/index"
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/technosophos/moniker"
+)
+
+func RunTests(t *testing.T, indexer index.Indexer, clearIndex func() error) {
+	if err := clearIndex(); err != nil {
+		t.Fatal(err)
+	}
+
+	var tests = []struct {
+		name    string
+		desc    string
+		limit   int
+		preTest func(t *testing.T) ([]*index.Line, time.Time)
+	}{
+		{
+			name:    "empty",
+			desc:    "an empty index should return an empty slice",
+			preTest: func(t *testing.T) ([]*index.Line, time.Time) { return []*index.Line{}, time.Time{} },
+			limit:   2000,
+		},
+		{
+			name: "happy path",
+			desc: "given 10 modules, return all of them in correct order",
+			preTest: func(t *testing.T) ([]*index.Line, time.Time) {
+				return seed(t, indexer, 10), time.Time{}
+			},
+			limit: 2000,
+		},
+		{
+			name: "respect the limit",
+			desc: "givn 10 modules and a 'limit' of 5, only return the first five lines",
+			preTest: func(t *testing.T) ([]*index.Line, time.Time) {
+				lines := seed(t, indexer, 10)
+				return lines[0:5], time.Time{}
+			},
+			limit: 5,
+		},
+		{
+			name: "respect the time",
+			desc: "given 10 modules, 'since' should filter out the ones that came before it",
+			preTest: func(t *testing.T) ([]*index.Line, time.Time) {
+				err := indexer.Index(context.Background(), "tobeignored", "v1.2.3")
+				if err != nil {
+					t.Fatal(err)
+				}
+				time.Sleep(50 * time.Millisecond)
+				now := time.Now()
+				lines := seed(t, indexer, 5)
+				return lines, now
+			},
+			limit: 2000,
+		},
+		{
+			name: "ignore the past",
+			desc: "no line should be returned if 'since' is after all of the indexed modules",
+			preTest: func(t *testing.T) ([]*index.Line, time.Time) {
+				seed(t, indexer, 5)
+				time.Sleep(50 * time.Millisecond)
+				return []*index.Line{}, time.Now()
+			},
+			limit: 2000,
+		},
+		{
+			name: "no limit no line",
+			desc: "if limit is set to zero, then nothing should be returned",
+			preTest: func(t *testing.T) ([]*index.Line, time.Time) {
+				seed(t, indexer, 5)
+				return []*index.Line{}, time.Time{}
+			},
+			limit: 0,
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Log(tc.desc)
+			t.Cleanup(func() {
+				if err := clearIndex(); err != nil {
+					t.Fatal(err)
+				}
+			})
+			expected, since := tc.preTest(t)
+			given, err := indexer.Lines(context.Background(), since, tc.limit)
+			if err != nil {
+				t.Fatal(err)
+			}
+			opts := cmpopts.IgnoreFields(index.Line{}, "Timestamp")
+			if !cmp.Equal(given, expected, opts) {
+				t.Fatal(cmp.Diff(expected, given, opts))
+			}
+		})
+	}
+}
+
+func seed(t *testing.T, indexer index.Indexer, num int) []*index.Line {
+	lines := []*index.Line{}
+	t.Helper()
+	for i := 0; i < num; i++ {
+		mod := moniker.New().NameSep("_")
+		ver := fmt.Sprintf("%d.0.0", i)
+		err := indexer.Index(context.Background(), mod, ver)
+		if err != nil {
+			t.Fatal(err)
+		}
+		lines = append(lines, &index.Line{Path: mod, Version: ver})
+	}
+	return lines
+}
diff --git a/pkg/index/indexer.go b/pkg/index/indexer.go
new file mode 100644
index 00000000..395aa0c5
--- /dev/null
+++ b/pkg/index/indexer.go
@@ -0,0 +1,26 @@
+package index
+
+import (
+	"context"
+	"time"
+)
+
+// Line represents a module@version line
+// with its metadata such as creation time.
+type Line struct {
+	Path, Version string
+	Timestamp     time.Time
+}
+
+// Indexer is an interface that can process new module@versions
+// and also retrieve 'limit' module@versions that were indexed after 'since'
+type Indexer interface {
+	// Index stores the module@version into the index backend.
+	// Implementer must create the Timestamp at the time and set it
+	// to the time this method is call.
+	Index(ctx context.Context, mod, ver string) error
+
+	// Lines returns the module@version lines given the time and limit
+	// constraints
+	Lines(ctx context.Context, since time.Time, limit int) ([]*Line, error)
+}
diff --git a/pkg/index/mem/mem.go b/pkg/index/mem/mem.go
new file mode 100644
index 00000000..4ffb5ffc
--- /dev/null
+++ b/pkg/index/mem/mem.go
@@ -0,0 +1,51 @@
+package mem
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"github.com/gomods/athens/pkg/errors"
+	"github.com/gomods/athens/pkg/index"
+)
+
+// New returns a new in-memory indexer
+func New() index.Indexer {
+	return &indexer{}
+}
+
+type indexer struct {
+	mu    sync.RWMutex
+	lines []*index.Line
+}
+
+func (i *indexer) Index(ctx context.Context, mod, ver string) error {
+	const op errors.Op = "mem.Index"
+	i.mu.Lock()
+	i.lines = append(i.lines, &index.Line{
+		Path:      mod,
+		Version:   ver,
+		Timestamp: time.Now(),
+	})
+	i.mu.Unlock()
+	return nil
+}
+
+func (i *indexer) Lines(ctx context.Context, since time.Time, limit int) ([]*index.Line, error) {
+	const op errors.Op = "mem.Lines"
+	lines := []*index.Line{}
+	var count int
+	i.mu.RLock()
+	defer i.mu.RUnlock()
+	for _, line := range i.lines {
+		if count >= limit {
+			break
+		}
+		if since.After(line.Timestamp) {
+			continue
+		}
+		lines = append(lines, line)
+		count++
+	}
+	return lines, nil
+}
diff --git a/pkg/index/mysql/mysql.go b/pkg/index/mysql/mysql.go
new file mode 100644
index 00000000..1ca85956
--- /dev/null
+++ b/pkg/index/mysql/mysql.go
@@ -0,0 +1,105 @@
+package mysql
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"time"
+
+	"github.com/go-sql-driver/mysql"
+	"github.com/gomods/athens/pkg/config"
+	"github.com/gomods/athens/pkg/errors"
+	"github.com/gomods/athens/pkg/index"
+)
+
+func New(cfg *config.MySQL) (index.Indexer, error) {
+	dataSource := getMySQLSource(cfg)
+	db, err := sql.Open("mysql", dataSource)
+	if err != nil {
+		return nil, err
+	}
+	if err = db.Ping(); err != nil {
+		return nil, err
+	}
+	_, err = db.Exec(schema)
+	if err != nil {
+		return nil, err
+	}
+	return &indexer{db}, nil
+}
+
+const schema = `
+	CREATE TABLE IF NOT EXISTS indexes(
+	id INT
+		AUTO_INCREMENT
+		PRIMARY KEY
+		COMMENT 'Unique identifier for a module line',
+
+	path VARCHAR(255)
+		NOT NULL
+		COMMENT 'Import path of the module',
+
+	version VARCHAR(255)
+		NOT NULL
+		COMMENT 'Module version',
+
+	timestamp TIMESTAMP(6)
+		COMMENT 'Date and time when the module was first created',
+
+	INDEX (timestamp),
+	UNIQUE INDEX idx_module_version (path, version)
+	) CHARACTER SET utf8;
+`
+
+type indexer struct {
+	db *sql.DB
+}
+
+func (i *indexer) Index(ctx context.Context, mod, ver string) error {
+	const op errors.Op = "mysql.Index"
+	_, err := i.db.ExecContext(
+		ctx,
+		`INSERT INTO indexes (path, version, timestamp) VALUES (?, ?, ?)`,
+		mod,
+		ver,
+		time.Now().Format(time.RFC3339Nano),
+	)
+	if err != nil {
+		return errors.E(op, err)
+	}
+	return nil
+}
+
+func (i *indexer) Lines(ctx context.Context, since time.Time, limit int) ([]*index.Line, error) {
+	const op errors.Op = "mysql.Lines"
+	if since.IsZero() {
+		since = time.Unix(0, 0)
+	}
+	sinceStr := since.Format(time.RFC3339Nano)
+	rows, err := i.db.QueryContext(ctx, `SELECT path, version, timestamp FROM indexes WHERE timestamp >= ? LIMIT ?`, sinceStr, limit)
+	if err != nil {
+		return nil, errors.E(op, err)
+	}
+	defer rows.Close()
+	lines := []*index.Line{}
+	for rows.Next() {
+		var line index.Line
+		err = rows.Scan(&line.Path, &line.Version, &line.Timestamp)
+		if err != nil {
+			return nil, errors.E(op, err)
+		}
+		lines = append(lines, &line)
+	}
+	return lines, nil
+}
+
+func getMySQLSource(cfg *config.MySQL) string {
+	c := mysql.NewConfig()
+	c.Net = cfg.Protocol
+	c.Addr = fmt.Sprintf("%s:%d", cfg.Host, cfg.Port)
+	c.User = cfg.User
+	c.Passwd = cfg.Password
+	c.DBName = cfg.Database
+	c.Params = cfg.Params
+	return c.FormatDSN()
+}
diff --git a/pkg/index/nop/nop.go b/pkg/index/nop/nop.go
new file mode 100644
index 00000000..5140d249
--- /dev/null
+++ b/pkg/index/nop/nop.go
@@ -0,0 +1,22 @@
+package nop
+
+import (
+	"context"
+	"time"
+
+	"github.com/gomods/athens/pkg/index"
+)
+
+// New returns a no-op Indexer
+func New() index.Indexer {
+	return indexer{}
+}
+
+type indexer struct{}
+
+func (indexer) Index(ctx context.Context, mod, ver string) error {
+	return nil
+}
+func (indexer) Lines(ctx context.Context, since time.Time, limit int) ([]*index.Line, error) {
+	return []*index.Line{}, nil
+}
diff --git a/pkg/index/postgres/postgres.go b/pkg/index/postgres/postgres.go
new file mode 100644
index 00000000..2f3c0b5a
--- /dev/null
+++ b/pkg/index/postgres/postgres.go
@@ -0,0 +1,106 @@
+package postgres
+
+import (
+	"context"
+	"database/sql"
+	"strconv"
+	"strings"
+	"time"
+
+	// register the driver with database/sql
+	_ "github.com/lib/pq"
+
+	"github.com/gomods/athens/pkg/config"
+	"github.com/gomods/athens/pkg/errors"
+	"github.com/gomods/athens/pkg/index"
+)
+
+func New(cfg *config.Postgres) (index.Indexer, error) {
+	dataSource := getPostgresSource(cfg)
+	db, err := sql.Open("postgres", dataSource)
+	if err != nil {
+		return nil, err
+	}
+	if err = db.Ping(); err != nil {
+		return nil, err
+	}
+	for _, statement := range schema {
+		_, err = db.Exec(statement)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return &indexer{db}, nil
+}
+
+var schema = [...]string{
+	`
+		CREATE TABLE IF NOT EXISTS indexes(
+			id SERIAL PRIMARY KEY,
+			path VARCHAR(255) NOT NULL,
+			version VARCHAR(255) NOT NULL,
+			timestamp timestamp NOT NULL
+		)
+	`,
+	`
+		CREATE INDEX IF NOT EXISTS idx_timestamp ON indexes (timestamp)
+	`,
+	`
+		CREATE UNIQUE INDEX IF NOT EXISTS idx_module_version ON indexes (path, version)
+	`,
+}
+
+type indexer struct {
+	db *sql.DB
+}
+
+func (i *indexer) Index(ctx context.Context, mod, ver string) error {
+	const op errors.Op = "postgres.Index"
+	_, err := i.db.ExecContext(
+		ctx,
+		`INSERT INTO indexes (path, version, timestamp) VALUES ($1, $2, $3)`,
+		mod,
+		ver,
+		time.Now().Format(time.RFC3339Nano),
+	)
+	if err != nil {
+		return errors.E(op, err)
+	}
+	return nil
+}
+
+func (i *indexer) Lines(ctx context.Context, since time.Time, limit int) ([]*index.Line, error) {
+	const op errors.Op = "postgres.Lines"
+	if since.IsZero() {
+		since = time.Unix(0, 0)
+	}
+	sinceStr := since.Format(time.RFC3339Nano)
+	rows, err := i.db.QueryContext(ctx, `SELECT path, version, timestamp FROM indexes WHERE timestamp >= $1 LIMIT $2`, sinceStr, limit)
+	if err != nil {
+		return nil, errors.E(op, err)
+	}
+	defer rows.Close()
+	lines := []*index.Line{}
+	for rows.Next() {
+		var line index.Line
+		err = rows.Scan(&line.Path, &line.Version, &line.Timestamp)
+		if err != nil {
+			return nil, errors.E(op, err)
+		}
+		lines = append(lines, &line)
+	}
+	return lines, nil
+}
+
+func getPostgresSource(cfg *config.Postgres) string {
+	args := []string{}
+	args = append(args, "host="+cfg.Host)
+	args = append(args, "port=", strconv.Itoa(cfg.Port))
+	args = append(args, "user=", cfg.User)
+	args = append(args, "dbname=", cfg.Database)
+	args = append(args, "password="+cfg.Password)
+	for k, v := range cfg.Params {
+		args = append(args, k+"="+v)
+	}
+	return strings.Join(args, " ")
+}
diff --git a/pkg/stash/stasher.go b/pkg/stash/stasher.go
index 4a35fc85..8d567cb5 100644
--- a/pkg/stash/stasher.go
+++ b/pkg/stash/stasher.go
@@ -5,6 +5,7 @@ import (
 	"time"
 
 	"github.com/gomods/athens/pkg/errors"
+	"github.com/gomods/athens/pkg/index"
 	"github.com/gomods/athens/pkg/log"
 	"github.com/gomods/athens/pkg/module"
 	"github.com/gomods/athens/pkg/observ"
@@ -13,7 +14,7 @@ import (
 )
 
 // Stasher has the job of taking a module
-// from an upstream entity and stashing it to a Storage Backend.
+// from an upstream entity and stashing it to a Storage Backend and Index.
 // It also returns a string that represents a semver version of
 // what was requested, this is helpful if what was requested
 // was a descriptive version such as a branch name or a full commit sha.
@@ -27,8 +28,8 @@ type Wrapper func(Stasher) Stasher
 // New returns a plain stasher that takes
 // a module from a download.Protocol and
 // stashes it into a backend.Storage.
-func New(f module.Fetcher, s storage.Backend, wrappers ...Wrapper) Stasher {
-	var st Stasher = &stasher{f, s, storage.WithChecker(s)}
+func New(f module.Fetcher, s storage.Backend, indexer index.Indexer, wrappers ...Wrapper) Stasher {
+	var st Stasher = &stasher{f, s, storage.WithChecker(s), indexer}
 	for _, w := range wrappers {
 		st = w(st)
 	}
@@ -40,6 +41,7 @@ type stasher struct {
 	fetcher module.Fetcher
 	storage storage.Backend
 	checker storage.Checker
+	indexer index.Indexer
 }
 
 func (s *stasher) Stash(ctx context.Context, mod, ver string) (string, error) {
@@ -71,6 +73,10 @@ func (s *stasher) Stash(ctx context.Context, mod, ver string) (string, error) {
 	if err != nil {
 		return "", errors.E(op, err)
 	}
+	err = s.indexer.Index(ctx, mod, v.Semver)
+	if err != nil {
+		return "", errors.E(op, err)
+	}
 	return v.Semver, nil
 }
 
@@ -80,6 +86,5 @@ func (s *stasher) fetchModule(ctx context.Context, mod, ver string) (*storage.Ve
 	if err != nil {
 		return nil, errors.E(op, err)
 	}
-
 	return v, nil
 }


Test patch:
diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go
index 3c06a9da..3592a2cb 100644
--- a/pkg/config/config_test.go
+++ b/pkg/config/config_test.go
@@ -24,14 +24,14 @@ func testConfigFile(t *testing.T) (testConfigFile string) {
 }
 
 func compareConfigs(parsedConf *Config, expConf *Config, t *testing.T) {
-	opts := cmpopts.IgnoreTypes(StorageConfig{}, SingleFlight{})
+	opts := cmpopts.IgnoreTypes(Storage{}, SingleFlight{}, Index{})
 	eq := cmp.Equal(parsedConf, expConf, opts)
 	if !eq {
 		t.Errorf("Parsed Example configuration did not match expected values. Expected: %+v. Actual: %+v", expConf, parsedConf)
 	}
 }
 
-func compareStorageConfigs(parsedStorage *StorageConfig, expStorage *StorageConfig, t *testing.T) {
+func compareStorageConfigs(parsedStorage *Storage, expStorage *Storage, t *testing.T) {
 	eq := cmp.Equal(parsedStorage.Mongo, expStorage.Mongo)
 	if !eq {
 		t.Errorf("Parsed Example Storage configuration did not match expected values. Expected: %+v. Actual: %+v", expStorage.Mongo, parsedStorage.Mongo)
@@ -91,10 +91,11 @@ func TestEnvOverrides(t *testing.T) {
 		PathPrefix:      "prefix",
 		NETRCPath:       "/test/path/.netrc",
 		HGRCPath:        "/test/path/.hgrc",
-		Storage:         &StorageConfig{},
+		Storage:         &Storage{},
 		GoBinaryEnvVars: []string{"GOPROXY=direct"},
 		SingleFlight:    &SingleFlight{},
 		RobotsFile:      "robots.txt",
+		Index:           &Index{},
 	}
 
 	envVars := getEnvMap(expConf)
@@ -157,7 +158,7 @@ func TestEnsurePortFormat(t *testing.T) {
 }
 
 func TestStorageEnvOverrides(t *testing.T) {
-	expStorage := &StorageConfig{
+	expStorage := &Storage{
 		Disk: &DiskConfig{
 			RootPath: "/my/root/path",
 		},
@@ -209,7 +210,7 @@ func TestParseExampleConfig(t *testing.T) {
 
 	// initialize all struct pointers so we get all applicable env variables
 	emptyConf := &Config{
-		Storage: &StorageConfig{
+		Storage: &Storage{
 			Disk: &DiskConfig{},
 			GCP:  &GCPConfig{},
 			Minio: &MinioConfig{
@@ -219,6 +220,7 @@ func TestParseExampleConfig(t *testing.T) {
 			S3:    &S3Config{},
 		},
 		SingleFlight: &SingleFlight{},
+		Index:        &Index{},
 	}
 	// unset all environment variables
 	envVars := getEnvMap(emptyConf)
@@ -229,7 +231,7 @@ func TestParseExampleConfig(t *testing.T) {
 		os.Unsetenv(k)
 	}
 
-	expStorage := &StorageConfig{
+	expStorage := &Storage{
 		Disk: &DiskConfig{
 			RootPath: "/path/on/disk",
 		},
@@ -289,6 +291,8 @@ func TestParseExampleConfig(t *testing.T) {
 		NoSumPatterns:    []string{},
 		DownloadMode:     "sync",
 		RobotsFile:       "robots.txt",
+		IndexType:        "none",
+		Index:            &Index{},
 	}
 
 	absPath, err := filepath.Abs(testConfigFile(t))
@@ -477,7 +481,7 @@ func TestDefaultConfigMatchesConfigFile(t *testing.T) {
 
 	defConf := defaultConfig()
 
-	ignoreStorageOpts := cmpopts.IgnoreTypes(&StorageConfig{})
+	ignoreStorageOpts := cmpopts.IgnoreTypes(&Storage{}, &Index{})
 	ignoreGoEnvOpts := cmpopts.IgnoreFields(Config{}, "GoEnv")
 	eq := cmp.Equal(defConf, parsedConf, ignoreStorageOpts, ignoreGoEnvOpts)
 	if !eq {
diff --git a/pkg/download/protocol_test.go b/pkg/download/protocol_test.go
index 068e2887..3c307190 100644
--- a/pkg/download/protocol_test.go
+++ b/pkg/download/protocol_test.go
@@ -15,6 +15,7 @@ import (
 	"github.com/gomods/athens/pkg/config"
 	"github.com/gomods/athens/pkg/download/mode"
 	"github.com/gomods/athens/pkg/errors"
+	"github.com/gomods/athens/pkg/index/nop"
 	"github.com/gomods/athens/pkg/module"
 	"github.com/gomods/athens/pkg/stash"
 	"github.com/gomods/athens/pkg/storage"
@@ -44,7 +45,7 @@ func getDP(t *testing.T) Protocol {
 	if err != nil {
 		t.Fatal(err)
 	}
-	st := stash.New(mf, s)
+	st := stash.New(mf, s, nop.New())
 	return New(&Opts{s, st, module.NewVCSLister(goBin, conf.GoBinaryEnvVars, fs), nil})
 }
 
@@ -280,7 +281,7 @@ func TestDownloadProtocol(t *testing.T) {
 		t.Fatal(err)
 	}
 	mp := &mockFetcher{}
-	st := stash.New(mp, s)
+	st := stash.New(mp, s, nop.New())
 	dp := New(&Opts{s, st, nil, nil})
 	ctx := context.Background()
 
@@ -332,7 +333,7 @@ func TestDownloadProtocolWhenFetchFails(t *testing.T) {
 		t.Fatal(err)
 	}
 	mp := &notFoundFetcher{}
-	st := stash.New(mp, s)
+	st := stash.New(mp, s, nop.New())
 	dp := New(&Opts{s, st, nil, nil})
 	ctx := context.Background()
 	_, err = dp.GoMod(ctx, fakeMod.mod, fakeMod.ver)
diff --git a/pkg/index/mem/mem_test.go b/pkg/index/mem/mem_test.go
new file mode 100644
index 00000000..657effdf
--- /dev/null
+++ b/pkg/index/mem/mem_test.go
@@ -0,0 +1,20 @@
+package mem
+
+import (
+	"testing"
+
+	"github.com/gomods/athens/pkg/index"
+	"github.com/gomods/athens/pkg/index/compliance"
+)
+
+func TestMem(t *testing.T) {
+	indexer := &indexer{}
+	compliance.RunTests(t, indexer, indexer.clear)
+}
+
+func (i *indexer) clear() error {
+	i.mu.Lock()
+	i.lines = []*index.Line{}
+	i.mu.Unlock()
+	return nil
+}
diff --git a/pkg/index/mysql/mysql_test.go b/pkg/index/mysql/mysql_test.go
new file mode 100644
index 00000000..93fcd9cf
--- /dev/null
+++ b/pkg/index/mysql/mysql_test.go
@@ -0,0 +1,35 @@
+package mysql
+
+import (
+	"os"
+	"testing"
+
+	"github.com/gomods/athens/pkg/config"
+	"github.com/gomods/athens/pkg/index/compliance"
+)
+
+func TestMySQL(t *testing.T) {
+	if os.Getenv("TEST_INDEX_MYSQL") != "true" {
+		t.SkipNow()
+	}
+	cfg := getTestConfig(t)
+	i, err := New(cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	compliance.RunTests(t, i, i.(*indexer).clear)
+}
+
+func (i *indexer) clear() error {
+	_, err := i.db.Exec(`DELETE FROM indexes`)
+	return err
+}
+
+func getTestConfig(t *testing.T) *config.MySQL {
+	t.Helper()
+	cfg, err := config.Load("")
+	if err != nil {
+		t.Fatal(err)
+	}
+	return cfg.Index.MySQL
+}
diff --git a/pkg/index/postgres/postgres_test.go b/pkg/index/postgres/postgres_test.go
new file mode 100644
index 00000000..287877d6
--- /dev/null
+++ b/pkg/index/postgres/postgres_test.go
@@ -0,0 +1,35 @@
+package postgres
+
+import (
+	"os"
+	"testing"
+
+	"github.com/gomods/athens/pkg/config"
+	"github.com/gomods/athens/pkg/index/compliance"
+)
+
+func TestPostgres(t *testing.T) {
+	if os.Getenv("TEST_INDEX_POSTGRES") != "true" {
+		t.SkipNow()
+	}
+	cfg := getTestConfig(t)
+	i, err := New(cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	compliance.RunTests(t, i, i.(*indexer).clear)
+}
+
+func (i *indexer) clear() error {
+	_, err := i.db.Exec(`DELETE FROM indexes`)
+	return err
+}
+
+func getTestConfig(t *testing.T) *config.Postgres {
+	t.Helper()
+	cfg, err := config.Load("")
+	if err != nil {
+		t.Fatal(err)
+	}
+	return cfg.Index.Postgres
+}
diff --git a/pkg/stash/stasher_test.go b/pkg/stash/stasher_test.go
index ae406e83..0f53b788 100644
--- a/pkg/stash/stasher_test.go
+++ b/pkg/stash/stasher_test.go
@@ -7,6 +7,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/gomods/athens/pkg/index/nop"
 	"github.com/gomods/athens/pkg/storage"
 )
 
@@ -54,7 +55,7 @@ func TestStash(t *testing.T) {
 			var mf mockFetcher
 			mf.ver = testCase.modVer
 
-			s := New(&mf, &ms)
+			s := New(&mf, &ms, nop.New())
 			newVersion, err := s.Stash(context.Background(), "module", testCase.ver)
 			if err != nil {
 				t.Fatal(err)