implement /index endpoint (#1630)

* implement /index endpoint

* rename to Module to Path

appveyor.yml

@@ -2,6 +2,8 @@ build: off
 
 clone_folder: c:\gopath\src\github.com\gomods\athens
 
+image: Previous Visual Studio 2019
+
 environment:
   GOPATH: c:\gopath
   GO111MODULE: on
@@ -11,5 +13,6 @@ environment:
 stack: go 1.14
 
 test_script:
+  - go version
   - go test ./...
 

cmd/proxy/actions/app_proxy.go

@@ -11,6 +11,11 @@ import (
 	"github.com/gomods/athens/pkg/download"
 	"github.com/gomods/athens/pkg/download/addons"
 	"github.com/gomods/athens/pkg/download/mode"
+	"github.com/gomods/athens/pkg/index"
+	"github.com/gomods/athens/pkg/index/mem"
+	"github.com/gomods/athens/pkg/index/mysql"
+	"github.com/gomods/athens/pkg/index/nop"
+	"github.com/gomods/athens/pkg/index/postgres"
 	"github.com/gomods/athens/pkg/log"
 	"github.com/gomods/athens/pkg/module"
 	"github.com/gomods/athens/pkg/stash"
@@ -32,6 +37,12 @@ func addProxyRoutes(
 	r.HandleFunc("/catalog", catalogHandler(s))
 	r.HandleFunc("/robots.txt", robotsHandler(c))
 
+	indexer, err := getIndex(c)
+	if err != nil {
+		return err
+	}
+	r.HandleFunc("/index", indexHandler(indexer))
+
 	for _, sumdb := range c.SumDBs {
 		sumdbURL, err := url.Parse(sumdb)
 		if err != nil {
@@ -96,7 +107,7 @@ func addProxyRoutes(
 	if err != nil {
 		return err
 	}
-	st := stash.New(mf, s, stash.WithPool(c.GoGetWorkers), withSingleFlight)
+	st := stash.New(mf, s, indexer, stash.WithPool(c.GoGetWorkers), withSingleFlight)
 
 	df, err := mode.NewFile(c.DownloadMode, c.DownloadURL)
 	if err != nil {
@@ -157,3 +168,17 @@ func getSingleFlight(c *config.Config, checker storage.Checker) (stash.Wrapper,
 		return nil, fmt.Errorf("unrecognized single flight type: %v", c.SingleFlightType)
 	}
 }
+
+func getIndex(c *config.Config) (index.Indexer, error) {
+	switch c.IndexType {
+	case "", "none":
+		return nop.New(), nil
+	case "memory":
+		return mem.New(), nil
+	case "mysql":
+		return mysql.New(c.Index.MySQL)
+	case "postgres":
+		return postgres.New(c.Index.Postgres)
+	}
+	return nil, fmt.Errorf("unknown index type: %q", c.IndexType)
+}

cmd/proxy/actions/index.go

@@ -0,0 +1,52 @@
+package actions
+
+import (
+	"encoding/json"
+	"net/http"
+	"strconv"
+	"time"
+
+	"github.com/gomods/athens/pkg/errors"
+	"github.com/gomods/athens/pkg/index"
+)
+
+// indexHandler implements GET baseURL/index
+func indexHandler(index index.Indexer) http.HandlerFunc {
+	const op errors.Op = "actions.IndexHandler"
+	return func(w http.ResponseWriter, r *http.Request) {
+		var (
+			err   error
+			limit int
+			since time.Time
+		)
+		if limitStr := r.FormValue("limit"); limitStr != "" {
+			limit, err = strconv.Atoi(limitStr)
+			if err != nil {
+				http.Error(w, err.Error(), http.StatusBadRequest)
+				return
+			}
+		}
+		if sinceStr := r.FormValue("since"); sinceStr != "" {
+			since, err = time.Parse(time.RFC3339, sinceStr)
+			if err != nil {
+				http.Error(w, err.Error(), http.StatusBadRequest)
+				return
+			}
+		}
+		if limit <= 0 {
+			limit = 2000
+		}
+		list, err := index.Lines(r.Context(), since, limit)
+		if err != nil {
+			http.Error(w, err.Error(), 500)
+			return
+		}
+		enc := json.NewEncoder(w)
+		for _, meta := range list {
+			if err = enc.Encode(meta); err != nil {
+				http.Error(w, err.Error(), 500)
+				return
+			}
+		}
+	}
+}

cmd/proxy/actions/storage.go

@@ -21,7 +21,7 @@ import (
 )
 
 // GetStorage returns storage backend based on env configuration
-func GetStorage(storageType string, storageConfig *config.StorageConfig, timeout time.Duration, client *http.Client) (storage.Backend, error) {
+func GetStorage(storageType string, storageConfig *config.Storage, timeout time.Duration, client *http.Client) (storage.Backend, error) {
 	const op errors.Op = "actions.GetStorage"
 	switch storageType {
 	case "memory":

config.dev.toml

@@ -282,6 +282,12 @@ DownloadURL = ""
 # Env override: ATHENS_SINGLE_FLIGHT_TYPE
 SingleFlightType = "memory"
 
+# IndexType sets the type of an index backend Athens will use.
+# Possible values are none, memory, mysql, postgres
+# Defaults to none
+# Env override: ATHENS_INDEX_TYPE
+IndexType = "none"
+
 [SingleFlight]
     [SingleFlight.Etcd]
         # Endpoints are comma separated URLs that determine all distributed etcd servers.
@@ -475,3 +481,65 @@ SingleFlightType = "memory"
         # details.
         # Env override: ATHENS_EXTERNAL_STORAGE_URL
         URL = ""
+
+[Index]
+    [Index.MySQL]
+        # MySQL protocol
+        # Env override: ATHENS_INDEX_MYSQL_PROTOCOL
+        Protocol = "tcp"
+
+        # MySQL user name
+        # Env override: ATHENS_INDEX_MYSQL_HOST
+        Host = "localhost"
+
+        # MySQL user name
+        # Env override: ATHENS_INDEX_MYSQL_PORT
+        Port = 3306
+
+        # MySQL user name
+        # Env override: ATHENS_INDEX_MYSQL_USER
+        User = "root"
+
+        # MySQL user name
+        # Env override: ATHENS_INDEX_MYSQL_PASSWORD
+        Password = ""
+
+        # MySQL database
+        # Env override: ATHENS_INDEX_MYSQL_DATABASE
+        Database = "athens"
+
+        # MySQL query parameters
+        # Environment overrides must be in the following format:
+        # ATHENS_INDEX_MYSQL_PARAMS="parseTime:true,timeout=90s"
+        # Env override: ATHENS_INDEX_MYSQL_PARAMS
+        [Index.MySQL.Params]
+            parseTime = "true"
+            timeout = "30s"
+    [Index.Postgres]
+        # Postgres user name
+        # Env override: ATHENS_INDEX_POSTGRES_HOST
+        Host = "localhost"
+
+        # Postgres user name
+        # Env override: ATHENS_INDEX_POSTGRES_PORT
+        Port = 5432
+
+        # Postgres user name
+        # Env override: ATHENS_INDEX_POSTGRES_USER
+        User = "postgres"
+
+        # Postgres user name
+        # Env override: ATHENS_INDEX_POSTGRES_PASSWORD
+        Password = ""
+
+        # Postgres database
+        # Env override: ATHENS_INDEX_POSTGRES_DATABASE
+        Database = "athens"
+
+        # Postgres query parameters
+        # Environment overrides must be in the following format:
+        # ATHENS_INDEX_POSTGRES_PARAMS="connect_timeout:30s,sslmode=disable"
+        # Env override: ATHENS_INDEX_POSTGRES_PARAMS
+        [Index.Postgres.Params]
+            connect_timeout = "30s"
+            sslmode = "disable"

go.mod

@@ -17,10 +17,10 @@ require (
 	github.com/bsm/redislock v0.4.2
 	github.com/codegangsta/negroni v1.0.0 // indirect
 	github.com/fatih/color v1.7.0
-	github.com/go-ini/ini v1.25.4 // indirect
 	github.com/go-playground/locales v0.12.1 // indirect
 	github.com/go-playground/universal-translator v0.16.0 // indirect
 	github.com/go-redis/redis/v7 v7.2.0
+	github.com/go-sql-driver/mysql v1.5.0
 	github.com/gobuffalo/envy v1.6.7
 	github.com/gobuffalo/httptest v1.0.4
 	github.com/golang/protobuf v1.3.3 // indirect
@@ -32,6 +32,7 @@ require (
 	github.com/hashicorp/hcl2 v0.0.0-20190503213020-640445e16309
 	github.com/kelseyhightower/envconfig v1.3.0
 	github.com/kylelemons/godebug v1.1.0 // indirect
+	github.com/lib/pq v1.7.0
 	github.com/minio/minio-go/v6 v6.0.43
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/philhofer/fwd v1.0.0 // indirect

go.sum

@@ -68,8 +68,6 @@ github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb
 github.com/apparentlymart/go-dump v0.0.0-20180507223929-23540a00eaa3/go.mod h1:oL81AME2rN47vu18xqj1S1jPIPuN7afo62yKTNn3XMM=
 github.com/apparentlymart/go-textseg v1.0.0 h1:rRmlIsPEEhUTIKQb7T++Nz/A5Q6C9IuX2wFoYVvnCs0=
 github.com/apparentlymart/go-textseg v1.0.0/go.mod h1:z96Txxhf3xSFMPmb5X/1W05FF/Nj9VFpLOpjS5yuumk=
-github.com/aws/aws-sdk-go v1.15.24 h1:xLAdTA/ore6xdPAljzZRed7IGqQgC+nY+ERS5vaj4Ro=
-github.com/aws/aws-sdk-go v1.15.24/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0=
 github.com/aws/aws-sdk-go v1.32.7 h1:H4VgdCSF1cHw0VD8zGc98T1bGdACoLkh/vK2L6wgOUU=
 github.com/aws/aws-sdk-go v1.32.7/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
@@ -113,8 +111,6 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
-github.com/go-ini/ini v1.25.4 h1:Mujh4R/dH6YL8bxuISne3xX2+qcQ9p0IxKAP6ExWoUo=
-github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-playground/locales v0.12.1 h1:2FITxuFt/xuCNP1Acdhv62OzaCiviiE4kotfhkmOqEc=
@@ -124,6 +120,7 @@ github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEK
 github.com/go-redis/redis/v7 v7.0.0-beta.4/go.mod h1:xhhSbUMTsleRPur+Vgx9sUHtyN33bdjxY+9/0n9Ig8s=
 github.com/go-redis/redis/v7 v7.2.0 h1:CrCexy/jYWZjW0AyVoHlcJUeZN19VWlbepTh1Vq6dJs=
 github.com/go-redis/redis/v7 v7.2.0/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg=
+github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
@@ -216,8 +213,6 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
-github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8 h1:12VvqtR6Aowv3l/EQUlocDHW2Cp4G9WJVH7uyH8QFJE=
-github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc=
 github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
 github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc=
@@ -246,6 +241,8 @@ github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3v
 github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+github.com/lib/pq v1.7.0 h1:h93mCPfUSkaul3Ka/VG8uZdmW1uMHDGxzu0NWHuJmHY=
+github.com/lib/pq v1.7.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/markbates/hmax v1.0.0 h1:yo2N0gBoCnUMKhV/VRLHomT6Y9wUm+oQQENuWJqCdlM=
 github.com/markbates/hmax v1.0.0/go.mod h1:cOkR9dktiESxIMu+65oc/r/bdY4bE8zZw3OLhLx0X2c=
 github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4=
@@ -286,8 +283,7 @@ github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG
 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
 github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
-github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -338,6 +334,7 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/technosophos/moniker v0.0.0-20180509230615-a5dbd03a2245 h1:DNVk+NIkGS0RbLkjQOLCJb/759yfCysThkMbl7EXxyY=
 github.com/technosophos/moniker v0.0.0-20180509230615-a5dbd03a2245/go.mod h1:O1c8HleITsZqzNZDjSNzirUGsMT0oGu9LhHKoJrqO+A=

pkg/config/config.go

@@ -54,8 +54,10 @@ type Config struct {
 	DownloadURL      string    `envconfig:"ATHENS_DOWNLOAD_URL"`
 	SingleFlightType string    `envconfig:"ATHENS_SINGLE_FLIGHT_TYPE"`
 	RobotsFile       string    `envconfig:"ATHENS_ROBOTS_FILE"`
+	IndexType        string    `envconfig:"ATHENS_INDEX_TYPE"`
 	SingleFlight     *SingleFlight
-	Storage          *StorageConfig
+	Storage          *Storage
+	Index            *Index
 }
 
 // EnvList is a list of key-value environment
@@ -161,6 +163,7 @@ func defaultConfig() *Config {
 		DownloadMode:     "sync",
 		DownloadURL:      "",
 		RobotsFile:       "robots.txt",
+		IndexType:        "none",
 		SingleFlight: &SingleFlight{
 			Etcd:  &Etcd{"localhost:2379,localhost:22379,localhost:32379"},
 			Redis: &Redis{"127.0.0.1:6379", ""},
@@ -170,6 +173,31 @@ func defaultConfig() *Config {
 				SentinelPassword: "sekret",
 			},
 		},
+		Index: &Index{
+			MySQL: &MySQL{
+				Protocol: "tcp",
+				Host:     "localhost",
+				Port:     3306,
+				User:     "root",
+				Password: "",
+				Database: "athens",
+				Params: map[string]string{
+					"parseTime": "true",
+					"timeout":   "30s",
+				},
+			},
+			Postgres: &Postgres{
+				Host:     "localhost",
+				Port:     5432,
+				User:     "postgres",
+				Password: "",
+				Database: "athens",
+				Params: map[string]string{
+					"connect_timeout": "30",
+					"sslmode":         "disable",
+				},
+			},
+		},
 	}
 }
 
@@ -267,29 +295,54 @@ func ensurePortFormat(s string) string {
 
 func validateConfig(config Config) error {
 	validate := validator.New()
-	err := validate.StructExcept(config, "Storage")
+	err := validate.StructExcept(config, "Storage", "Index")
 	if err != nil {
 		return err
 	}
-	switch config.StorageType {
+	err = validateStorage(validate, config.StorageType, config.Storage)
+	if err != nil {
+		return err
+	}
+	err = validateIndex(validate, config.IndexType, config.Index)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func validateStorage(validate *validator.Validate, storageType string, config *Storage) error {
+	switch storageType {
 	case "memory":
 		return nil
 	case "mongo":
-		return validate.Struct(config.Storage.Mongo)
+		return validate.Struct(config.Mongo)
 	case "disk":
-		return validate.Struct(config.Storage.Disk)
+		return validate.Struct(config.Disk)
 	case "minio":
-		return validate.Struct(config.Storage.Minio)
+		return validate.Struct(config.Minio)
 	case "gcp":
-		return validate.Struct(config.Storage.GCP)
+		return validate.Struct(config.GCP)
 	case "s3":
-		return validate.Struct(config.Storage.S3)
+		return validate.Struct(config.S3)
 	case "azureblob":
-		return validate.Struct(config.Storage.AzureBlob)
+		return validate.Struct(config.AzureBlob)
 	case "external":
-		return validate.Struct(config.Storage.External)
+		return validate.Struct(config.External)
 	default:
-		return fmt.Errorf("storage type %s is unknown", config.StorageType)
+		return fmt.Errorf("storage type %q is unknown", storageType)
+	}
+}
+
+func validateIndex(validate *validator.Validate, indexType string, config *Index) error {
+	switch indexType {
+	case "", "none", "memory":
+		return nil
+	case "mysql":
+		return validate.Struct(config.MySQL)
+	case "postgres":
+		return validate.Struct(config.Postgres)
+	default:
+		return fmt.Errorf("index type %q is unknown", indexType)
 	}
 }
 

pkg/config/config_test.go

@@ -24,14 +24,14 @@ func testConfigFile(t *testing.T) (testConfigFile string) {
 }
 
 func compareConfigs(parsedConf *Config, expConf *Config, t *testing.T) {
-	opts := cmpopts.IgnoreTypes(StorageConfig{}, SingleFlight{})
+	opts := cmpopts.IgnoreTypes(Storage{}, SingleFlight{}, Index{})
 	eq := cmp.Equal(parsedConf, expConf, opts)
 	if !eq {
 		t.Errorf("Parsed Example configuration did not match expected values. Expected: %+v. Actual: %+v", expConf, parsedConf)
 	}
 }
 
-func compareStorageConfigs(parsedStorage *StorageConfig, expStorage *StorageConfig, t *testing.T) {
+func compareStorageConfigs(parsedStorage *Storage, expStorage *Storage, t *testing.T) {
 	eq := cmp.Equal(parsedStorage.Mongo, expStorage.Mongo)
 	if !eq {
 		t.Errorf("Parsed Example Storage configuration did not match expected values. Expected: %+v. Actual: %+v", expStorage.Mongo, parsedStorage.Mongo)
@@ -91,10 +91,11 @@ func TestEnvOverrides(t *testing.T) {
 		PathPrefix:      "prefix",
 		NETRCPath:       "/test/path/.netrc",
 		HGRCPath:        "/test/path/.hgrc",
-		Storage:         &StorageConfig{},
+		Storage:         &Storage{},
 		GoBinaryEnvVars: []string{"GOPROXY=direct"},
 		SingleFlight:    &SingleFlight{},
 		RobotsFile:      "robots.txt",
+		Index:           &Index{},
 	}
 
 	envVars := getEnvMap(expConf)
@@ -157,7 +158,7 @@ func TestEnsurePortFormat(t *testing.T) {
 }
 
 func TestStorageEnvOverrides(t *testing.T) {
-	expStorage := &StorageConfig{
+	expStorage := &Storage{
 		Disk: &DiskConfig{
 			RootPath: "/my/root/path",
 		},
@@ -209,7 +210,7 @@ func TestParseExampleConfig(t *testing.T) {
 
 	// initialize all struct pointers so we get all applicable env variables
 	emptyConf := &Config{
-		Storage: &StorageConfig{
+		Storage: &Storage{
 			Disk: &DiskConfig{},
 			GCP:  &GCPConfig{},
 			Minio: &MinioConfig{
@@ -219,6 +220,7 @@ func TestParseExampleConfig(t *testing.T) {
 			S3:    &S3Config{},
 		},
 		SingleFlight: &SingleFlight{},
+		Index:        &Index{},
 	}
 	// unset all environment variables
 	envVars := getEnvMap(emptyConf)
@@ -229,7 +231,7 @@ func TestParseExampleConfig(t *testing.T) {
 		os.Unsetenv(k)
 	}
 
-	expStorage := &StorageConfig{
+	expStorage := &Storage{
 		Disk: &DiskConfig{
 			RootPath: "/path/on/disk",
 		},
@@ -289,6 +291,8 @@ func TestParseExampleConfig(t *testing.T) {
 		NoSumPatterns:    []string{},
 		DownloadMode:     "sync",
 		RobotsFile:       "robots.txt",
+		IndexType:        "none",
+		Index:            &Index{},
 	}
 
 	absPath, err := filepath.Abs(testConfigFile(t))
@@ -477,7 +481,7 @@ func TestDefaultConfigMatchesConfigFile(t *testing.T) {
 
 	defConf := defaultConfig()
 
-	ignoreStorageOpts := cmpopts.IgnoreTypes(&StorageConfig{})
+	ignoreStorageOpts := cmpopts.IgnoreTypes(&Storage{}, &Index{})
 	ignoreGoEnvOpts := cmpopts.IgnoreFields(Config{}, "GoEnv")
 	eq := cmp.Equal(defConf, parsedConf, ignoreStorageOpts, ignoreGoEnvOpts)
 	if !eq {

pkg/config/index.go

@@ -0,0 +1,7 @@
+package config
+
+// Index is the config for various index storage backends
+type Index struct {
+	MySQL    *MySQL
+	Postgres *Postgres
+}

pkg/config/mysql.go

@@ -0,0 +1,12 @@
+package config
+
+// MySQL config
+type MySQL struct {
+	Protocol string            `validate:"required" envconfig:"ATHENS_INDEX_MYSQL_PROTOCOL"`
+	Host     string            `validate:"required" envconfig:"ATHENS_INDEX_MYSQL_HOST"`
+	Port     int               `validate:"" envconfig:"ATHENS_INDEX_MYSQL_PORT"`
+	User     string            `validate:"required" envconfig:"ATHENS_INDEX_MYSQL_USER"`
+	Password string            `validate:"" envconfig:"ATHENS_INDEX_MYSQL_PASSWORD"`
+	Database string            `validate:"required" envconfig:"ATHENS_INDEX_MYSQL_DATABASE"`
+	Params   map[string]string `validate:"required" envconfig:"ATHENS_INDEX_MYSQL_PARAMS"`
+}

pkg/config/postgres.go

@@ -0,0 +1,11 @@
+package config
+
+// Postgres config
+type Postgres struct {
+	Host     string            `validate:"required" envconfig:"ATHENS_INDEX_POSTGRES_HOST"`
+	Port     int               `validate:"required" envconfig:"ATHENS_INDEX_POSTGRES_PORT"`
+	User     string            `validate:"required" envconfig:"ATHENS_INDEX_POSTGRES_USER"`
+	Password string            `validate:"" envconfig:"ATHENS_INDEX_POSTGRES_PASSWORD"`
+	Database string            `validate:"required" envconfig:"ATHENS_INDEX_POSTGRES_DATABASE"`
+	Params   map[string]string `validate:"required" envconfig:"ATHENS_INDEX_POSTGRES_PARAMS"`
+}

pkg/config/storage.go

@@ -1,7 +1,7 @@
 package config
 
-// StorageConfig provides configs for various storage backends
-type StorageConfig struct {
+// Storage provides configs for various storage backends
+type Storage struct {
 	Disk      *DiskConfig
 	GCP       *GCPConfig
 	Minio     *MinioConfig

pkg/download/protocol_test.go

@@ -15,6 +15,7 @@ import (
 	"github.com/gomods/athens/pkg/config"
 	"github.com/gomods/athens/pkg/download/mode"
 	"github.com/gomods/athens/pkg/errors"
+	"github.com/gomods/athens/pkg/index/nop"
 	"github.com/gomods/athens/pkg/module"
 	"github.com/gomods/athens/pkg/stash"
 	"github.com/gomods/athens/pkg/storage"
@@ -44,7 +45,7 @@ func getDP(t *testing.T) Protocol {
 	if err != nil {
 		t.Fatal(err)
 	}
-	st := stash.New(mf, s)
+	st := stash.New(mf, s, nop.New())
 	return New(&Opts{s, st, module.NewVCSLister(goBin, conf.GoBinaryEnvVars, fs), nil})
 }
 
@@ -280,7 +281,7 @@ func TestDownloadProtocol(t *testing.T) {
 		t.Fatal(err)
 	}
 	mp := &mockFetcher{}
-	st := stash.New(mp, s)
+	st := stash.New(mp, s, nop.New())
 	dp := New(&Opts{s, st, nil, nil})
 	ctx := context.Background()
 
@@ -332,7 +333,7 @@ func TestDownloadProtocolWhenFetchFails(t *testing.T) {
 		t.Fatal(err)
 	}
 	mp := &notFoundFetcher{}
-	st := stash.New(mp, s)
+	st := stash.New(mp, s, nop.New())
 	dp := New(&Opts{s, st, nil, nil})
 	ctx := context.Background()
 	_, err = dp.GoMod(ctx, fakeMod.mod, fakeMod.ver)

pkg/index/compliance/compliance.go

@@ -0,0 +1,118 @@
+package compliance
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/gomods/athens/pkg/index"
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/technosophos/moniker"
+)
+
+func RunTests(t *testing.T, indexer index.Indexer, clearIndex func() error) {
+	if err := clearIndex(); err != nil {
+		t.Fatal(err)
+	}
+
+	var tests = []struct {
+		name    string
+		desc    string
+		limit   int
+		preTest func(t *testing.T) ([]*index.Line, time.Time)
+	}{
+		{
+			name:    "empty",
+			desc:    "an empty index should return an empty slice",
+			preTest: func(t *testing.T) ([]*index.Line, time.Time) { return []*index.Line{}, time.Time{} },
+			limit:   2000,
+		},
+		{
+			name: "happy path",
+			desc: "given 10 modules, return all of them in correct order",
+			preTest: func(t *testing.T) ([]*index.Line, time.Time) {
+				return seed(t, indexer, 10), time.Time{}
+			},
+			limit: 2000,
+		},
+		{
+			name: "respect the limit",
+			desc: "givn 10 modules and a 'limit' of 5, only return the first five lines",
+			preTest: func(t *testing.T) ([]*index.Line, time.Time) {
+				lines := seed(t, indexer, 10)
+				return lines[0:5], time.Time{}
+			},
+			limit: 5,
+		},
+		{
+			name: "respect the time",
+			desc: "given 10 modules, 'since' should filter out the ones that came before it",
+			preTest: func(t *testing.T) ([]*index.Line, time.Time) {
+				err := indexer.Index(context.Background(), "tobeignored", "v1.2.3")
+				if err != nil {
+					t.Fatal(err)
+				}
+				time.Sleep(50 * time.Millisecond)
+				now := time.Now()
+				lines := seed(t, indexer, 5)
+				return lines, now
+			},
+			limit: 2000,
+		},
+		{
+			name: "ignore the past",
+			desc: "no line should be returned if 'since' is after all of the indexed modules",
+			preTest: func(t *testing.T) ([]*index.Line, time.Time) {
+				seed(t, indexer, 5)
+				time.Sleep(50 * time.Millisecond)
+				return []*index.Line{}, time.Now()
+			},
+			limit: 2000,
+		},
+		{
+			name: "no limit no line",
+			desc: "if limit is set to zero, then nothing should be returned",
+			preTest: func(t *testing.T) ([]*index.Line, time.Time) {
+				seed(t, indexer, 5)
+				return []*index.Line{}, time.Time{}
+			},
+			limit: 0,
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Log(tc.desc)
+			t.Cleanup(func() {
+				if err := clearIndex(); err != nil {
+					t.Fatal(err)
+				}
+			})
+			expected, since := tc.preTest(t)
+			given, err := indexer.Lines(context.Background(), since, tc.limit)
+			if err != nil {
+				t.Fatal(err)
+			}
+			opts := cmpopts.IgnoreFields(index.Line{}, "Timestamp")
+			if !cmp.Equal(given, expected, opts) {
+				t.Fatal(cmp.Diff(expected, given, opts))
+			}
+		})
+	}
+}
+
+func seed(t *testing.T, indexer index.Indexer, num int) []*index.Line {
+	lines := []*index.Line{}
+	t.Helper()
+	for i := 0; i < num; i++ {
+		mod := moniker.New().NameSep("_")
+		ver := fmt.Sprintf("%d.0.0", i)
+		err := indexer.Index(context.Background(), mod, ver)
+		if err != nil {
+			t.Fatal(err)
+		}
+		lines = append(lines, &index.Line{Path: mod, Version: ver})
+	}
+	return lines
+}

pkg/index/indexer.go

@@ -0,0 +1,26 @@
+package index
+
+import (
+	"context"
+	"time"
+)
+
+// Line represents a module@version line
+// with its metadata such as creation time.
+type Line struct {
+	Path, Version string
+	Timestamp     time.Time
+}
+
+// Indexer is an interface that can process new module@versions
+// and also retrieve 'limit' module@versions that were indexed after 'since'
+type Indexer interface {
+	// Index stores the module@version into the index backend.
+	// Implementer must create the Timestamp at the time and set it
+	// to the time this method is call.
+	Index(ctx context.Context, mod, ver string) error
+
+	// Lines returns the module@version lines given the time and limit
+	// constraints
+	Lines(ctx context.Context, since time.Time, limit int) ([]*Line, error)
+}

pkg/index/mem/mem.go

@@ -0,0 +1,51 @@
+package mem
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"github.com/gomods/athens/pkg/errors"
+	"github.com/gomods/athens/pkg/index"
+)
+
+// New returns a new in-memory indexer
+func New() index.Indexer {
+	return &indexer{}
+}
+
+type indexer struct {
+	mu    sync.RWMutex
+	lines []*index.Line
+}
+
+func (i *indexer) Index(ctx context.Context, mod, ver string) error {
+	const op errors.Op = "mem.Index"
+	i.mu.Lock()
+	i.lines = append(i.lines, &index.Line{
+		Path:      mod,
+		Version:   ver,
+		Timestamp: time.Now(),
+	})
+	i.mu.Unlock()
+	return nil
+}
+
+func (i *indexer) Lines(ctx context.Context, since time.Time, limit int) ([]*index.Line, error) {
+	const op errors.Op = "mem.Lines"
+	lines := []*index.Line{}
+	var count int
+	i.mu.RLock()
+	defer i.mu.RUnlock()
+	for _, line := range i.lines {
+		if count >= limit {
+			break
+		}
+		if since.After(line.Timestamp) {
+			continue
+		}
+		lines = append(lines, line)
+		count++
+	}
+	return lines, nil
+}

pkg/index/mem/mem_test.go

@@ -0,0 +1,20 @@
+package mem
+
+import (
+	"testing"
+
+	"github.com/gomods/athens/pkg/index"
+	"github.com/gomods/athens/pkg/index/compliance"
+)
+
+func TestMem(t *testing.T) {
+	indexer := &indexer{}
+	compliance.RunTests(t, indexer, indexer.clear)
+}
+
+func (i *indexer) clear() error {
+	i.mu.Lock()
+	i.lines = []*index.Line{}
+	i.mu.Unlock()
+	return nil
+}

pkg/index/mysql/mysql.go

@@ -0,0 +1,105 @@
+package mysql
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"time"
+
+	"github.com/go-sql-driver/mysql"
+	"github.com/gomods/athens/pkg/config"
+	"github.com/gomods/athens/pkg/errors"
+	"github.com/gomods/athens/pkg/index"
+)
+
+func New(cfg *config.MySQL) (index.Indexer, error) {
+	dataSource := getMySQLSource(cfg)
+	db, err := sql.Open("mysql", dataSource)
+	if err != nil {
+		return nil, err
+	}
+	if err = db.Ping(); err != nil {
+		return nil, err
+	}
+	_, err = db.Exec(schema)
+	if err != nil {
+		return nil, err
+	}
+	return &indexer{db}, nil
+}
+
+const schema = `
+	CREATE TABLE IF NOT EXISTS indexes(
+	id INT
+		AUTO_INCREMENT
+		PRIMARY KEY
+		COMMENT 'Unique identifier for a module line',
+
+	path VARCHAR(255)
+		NOT NULL
+		COMMENT 'Import path of the module',
+
+	version VARCHAR(255)
+		NOT NULL
+		COMMENT 'Module version',
+
+	timestamp TIMESTAMP(6)
+		COMMENT 'Date and time when the module was first created',
+
+	INDEX (timestamp),
+	UNIQUE INDEX idx_module_version (path, version)
+	) CHARACTER SET utf8;
+`
+
+type indexer struct {
+	db *sql.DB
+}
+
+func (i *indexer) Index(ctx context.Context, mod, ver string) error {
+	const op errors.Op = "mysql.Index"
+	_, err := i.db.ExecContext(
+		ctx,
+		`INSERT INTO indexes (path, version, timestamp) VALUES (?, ?, ?)`,
+		mod,
+		ver,
+		time.Now().Format(time.RFC3339Nano),
+	)
+	if err != nil {
+		return errors.E(op, err)
+	}
+	return nil
+}
+
+func (i *indexer) Lines(ctx context.Context, since time.Time, limit int) ([]*index.Line, error) {
+	const op errors.Op = "mysql.Lines"
+	if since.IsZero() {
+		since = time.Unix(0, 0)
+	}
+	sinceStr := since.Format(time.RFC3339Nano)
+	rows, err := i.db.QueryContext(ctx, `SELECT path, version, timestamp FROM indexes WHERE timestamp >= ? LIMIT ?`, sinceStr, limit)
+	if err != nil {
+		return nil, errors.E(op, err)
+	}
+	defer rows.Close()
+	lines := []*index.Line{}
+	for rows.Next() {
+		var line index.Line
+		err = rows.Scan(&line.Path, &line.Version, &line.Timestamp)
+		if err != nil {
+			return nil, errors.E(op, err)
+		}
+		lines = append(lines, &line)
+	}
+	return lines, nil
+}
+
+func getMySQLSource(cfg *config.MySQL) string {
+	c := mysql.NewConfig()
+	c.Net = cfg.Protocol
+	c.Addr = fmt.Sprintf("%s:%d", cfg.Host, cfg.Port)
+	c.User = cfg.User
+	c.Passwd = cfg.Password
+	c.DBName = cfg.Database
+	c.Params = cfg.Params
+	return c.FormatDSN()
+}

pkg/index/mysql/mysql_test.go

@@ -0,0 +1,35 @@
+package mysql
+
+import (
+	"os"
+	"testing"
+
+	"github.com/gomods/athens/pkg/config"
+	"github.com/gomods/athens/pkg/index/compliance"
+)
+
+func TestMySQL(t *testing.T) {
+	if os.Getenv("TEST_INDEX_MYSQL") != "true" {
+		t.SkipNow()
+	}
+	cfg := getTestConfig(t)
+	i, err := New(cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	compliance.RunTests(t, i, i.(*indexer).clear)
+}
+
+func (i *indexer) clear() error {
+	_, err := i.db.Exec(`DELETE FROM indexes`)
+	return err
+}
+
+func getTestConfig(t *testing.T) *config.MySQL {
+	t.Helper()
+	cfg, err := config.Load("")
+	if err != nil {
+		t.Fatal(err)
+	}
+	return cfg.Index.MySQL
+}

pkg/index/nop/nop.go

@@ -0,0 +1,22 @@
+package nop
+
+import (
+	"context"
+	"time"
+
+	"github.com/gomods/athens/pkg/index"
+)
+
+// New returns a no-op Indexer
+func New() index.Indexer {
+	return indexer{}
+}
+
+type indexer struct{}
+
+func (indexer) Index(ctx context.Context, mod, ver string) error {
+	return nil
+}
+func (indexer) Lines(ctx context.Context, since time.Time, limit int) ([]*index.Line, error) {
+	return []*index.Line{}, nil
+}

pkg/index/postgres/postgres.go

@@ -0,0 +1,106 @@
+package postgres
+
+import (
+	"context"
+	"database/sql"
+	"strconv"
+	"strings"
+	"time"
+
+	// register the driver with database/sql
+	_ "github.com/lib/pq"
+
+	"github.com/gomods/athens/pkg/config"
+	"github.com/gomods/athens/pkg/errors"
+	"github.com/gomods/athens/pkg/index"
+)
+
+func New(cfg *config.Postgres) (index.Indexer, error) {
+	dataSource := getPostgresSource(cfg)
+	db, err := sql.Open("postgres", dataSource)
+	if err != nil {
+		return nil, err
+	}
+	if err = db.Ping(); err != nil {
+		return nil, err
+	}
+	for _, statement := range schema {
+		_, err = db.Exec(statement)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return &indexer{db}, nil
+}
+
+var schema = [...]string{
+	`
+		CREATE TABLE IF NOT EXISTS indexes(
+			id SERIAL PRIMARY KEY,
+			path VARCHAR(255) NOT NULL,
+			version VARCHAR(255) NOT NULL,
+			timestamp timestamp NOT NULL
+		)
+	`,
+	`
+		CREATE INDEX IF NOT EXISTS idx_timestamp ON indexes (timestamp)
+	`,
+	`
+		CREATE UNIQUE INDEX IF NOT EXISTS idx_module_version ON indexes (path, version)
+	`,
+}
+
+type indexer struct {
+	db *sql.DB
+}
+
+func (i *indexer) Index(ctx context.Context, mod, ver string) error {
+	const op errors.Op = "postgres.Index"
+	_, err := i.db.ExecContext(
+		ctx,
+		`INSERT INTO indexes (path, version, timestamp) VALUES ($1, $2, $3)`,
+		mod,
+		ver,
+		time.Now().Format(time.RFC3339Nano),
+	)
+	if err != nil {
+		return errors.E(op, err)
+	}
+	return nil
+}
+
+func (i *indexer) Lines(ctx context.Context, since time.Time, limit int) ([]*index.Line, error) {
+	const op errors.Op = "postgres.Lines"
+	if since.IsZero() {
+		since = time.Unix(0, 0)
+	}
+	sinceStr := since.Format(time.RFC3339Nano)
+	rows, err := i.db.QueryContext(ctx, `SELECT path, version, timestamp FROM indexes WHERE timestamp >= $1 LIMIT $2`, sinceStr, limit)
+	if err != nil {
+		return nil, errors.E(op, err)
+	}
+	defer rows.Close()
+	lines := []*index.Line{}
+	for rows.Next() {
+		var line index.Line
+		err = rows.Scan(&line.Path, &line.Version, &line.Timestamp)
+		if err != nil {
+			return nil, errors.E(op, err)
+		}
+		lines = append(lines, &line)
+	}
+	return lines, nil
+}
+
+func getPostgresSource(cfg *config.Postgres) string {
+	args := []string{}
+	args = append(args, "host="+cfg.Host)
+	args = append(args, "port=", strconv.Itoa(cfg.Port))
+	args = append(args, "user=", cfg.User)
+	args = append(args, "dbname=", cfg.Database)
+	args = append(args, "password="+cfg.Password)
+	for k, v := range cfg.Params {
+		args = append(args, k+"="+v)
+	}
+	return strings.Join(args, " ")
+}

pkg/index/postgres/postgres_test.go

@@ -0,0 +1,35 @@
+package postgres
+
+import (
+	"os"
+	"testing"
+
+	"github.com/gomods/athens/pkg/config"
+	"github.com/gomods/athens/pkg/index/compliance"
+)
+
+func TestPostgres(t *testing.T) {
+	if os.Getenv("TEST_INDEX_POSTGRES") != "true" {
+		t.SkipNow()
+	}
+	cfg := getTestConfig(t)
+	i, err := New(cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	compliance.RunTests(t, i, i.(*indexer).clear)
+}
+
+func (i *indexer) clear() error {
+	_, err := i.db.Exec(`DELETE FROM indexes`)
+	return err
+}
+
+func getTestConfig(t *testing.T) *config.Postgres {
+	t.Helper()
+	cfg, err := config.Load("")
+	if err != nil {
+		t.Fatal(err)
+	}
+	return cfg.Index.Postgres
+}

pkg/stash/stasher.go

@@ -5,6 +5,7 @@ import (
 	"time"
 
 	"github.com/gomods/athens/pkg/errors"
+	"github.com/gomods/athens/pkg/index"
 	"github.com/gomods/athens/pkg/log"
 	"github.com/gomods/athens/pkg/module"
 	"github.com/gomods/athens/pkg/observ"
@@ -13,7 +14,7 @@ import (
 )
 
 // Stasher has the job of taking a module
-// from an upstream entity and stashing it to a Storage Backend.
+// from an upstream entity and stashing it to a Storage Backend and Index.
 // It also returns a string that represents a semver version of
 // what was requested, this is helpful if what was requested
 // was a descriptive version such as a branch name or a full commit sha.
@@ -27,8 +28,8 @@ type Wrapper func(Stasher) Stasher
 // New returns a plain stasher that takes
 // a module from a download.Protocol and
 // stashes it into a backend.Storage.
-func New(f module.Fetcher, s storage.Backend, wrappers ...Wrapper) Stasher {
-	var st Stasher = &stasher{f, s, storage.WithChecker(s)}
+func New(f module.Fetcher, s storage.Backend, indexer index.Indexer, wrappers ...Wrapper) Stasher {
+	var st Stasher = &stasher{f, s, storage.WithChecker(s), indexer}
 	for _, w := range wrappers {
 		st = w(st)
 	}
@@ -40,6 +41,7 @@ type stasher struct {
 	fetcher module.Fetcher
 	storage storage.Backend
 	checker storage.Checker
+	indexer index.Indexer
 }
 
 func (s *stasher) Stash(ctx context.Context, mod, ver string) (string, error) {
@@ -71,6 +73,10 @@ func (s *stasher) Stash(ctx context.Context, mod, ver string) (string, error) {
 	if err != nil {
 		return "", errors.E(op, err)
 	}
+	err = s.indexer.Index(ctx, mod, v.Semver)
+	if err != nil {
+		return "", errors.E(op, err)
+	}
 	return v.Semver, nil
 }
 
@@ -80,6 +86,5 @@ func (s *stasher) fetchModule(ctx context.Context, mod, ver string) (*storage.Ve
 	if err != nil {
 		return nil, errors.E(op, err)
 	}
-
 	return v, nil
 }

pkg/stash/stasher_test.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/gomods/athens/pkg/index/nop"
 	"github.com/gomods/athens/pkg/storage"
 )
 
@@ -54,7 +55,7 @@ func TestStash(t *testing.T) {
 			var mf mockFetcher
 			mf.ver = testCase.modVer
 
-			s := New(&mf, &ms)
+			s := New(&mf, &ms, nop.New())
 			newVersion, err := s.Stash(context.Background(), "module", testCase.ver)
 			if err != nil {
 				t.Fatal(err)