Catalogendpoint (#955)

* Added new cataloger interface

* Implementing catalog protocol

* Propagated to protocol and over

* First round of fixes

* S3 almost ready, need to be tested

* Going on with testing s3

* Better testing with s3

* Simplified catalog tests

* Preparing gcp tests to access a gcp instance

* Fixing initialization errors

* Removed some prints

* Gcp ready, to be tested

* Gcp working

* Aligned bucket mock to catalog method

* Switched res payload to json

* Added catalog method to all storage instances

* Added catalog method to unsupported storages

* Fixed with pool test

* Restored tests

* Fixed gcp constructor

* Implemented catalog for fs

* Removed trace

* E2e tests, fixed fs

* Fixed module name return value

* Added cataloger method to azure storage

* Added docs

* Changed pagesize parameter name

* Fixed gofmt error

* Added json tags to result. Fixed lint warning

* Removed extra line

* Changed not implemented error to http.KindNotImplemented

* Checking for inequality on results

* Lower-cased json keys

* Added cleaning of path separator

* Fixed review comments

* Fixed docs

docs/content/design/proxy.md

@@ -62,3 +62,26 @@ Private module filters are string globs that tell the Athens proxy what is a pri
 ### Exclude Lists for Public Modules
 
 Exclude lists for public modules are also globs that tell the Athens proxy what modules it should never download from any upstream proxy. For example, the string `github.com/arschles/**` tells the Athens proxy to always return `404 Not Found` to clients.
+
+## Catalog Endpoint
+
+The proxy provides a `/catalog` service endpoint to fetch all the modules and their versions contained in the local storage.
+The endpoint accepts a continuation token and a page size parameter in order to provide paginated results.
+
+A query is of the form
+
+`https://proxyurl/catalog?token=foo&limit=47`
+
+Where token is an optional continuation token and limit is the desired size of the returned page.
+The `token` parameter is not required for the first call and it's needed for handling paginated results.
+
+
+The result is a json with the following structure:
+
+```
+{"modules": [{"module":"github.com/athens-artifacts/no-tags","version":"v1.0.0"}],
+ "next":""}'
+```
+
+An empty `next` token means that no more pages are available.
+The default page size is 1000.

pkg/config/module.go

@@ -1,9 +1,13 @@
 package config
 
-import "fmt"
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+)
 
 // PackageVersionedName return package full name used in storage.
-// E.g athens/@v/v1.0/go.mod
+// E.g athens/@v/v1.0.mod
 func PackageVersionedName(module, version, ext string) string {
 	return fmt.Sprintf("%s/@v/%s.%s", module, version, ext)
 }
@@ -13,3 +17,15 @@ func PackageVersionedName(module, version, ext string) string {
 func FmtModVer(mod, ver string) string {
 	return fmt.Sprintf("%s@%s", mod, ver)
 }
+
+// ModuleVersionFromPath returns module and version from a
+// storage path
+// E.g athens/@v/v1.0.info -> athens and v.1.0
+func ModuleVersionFromPath(path string) (string, string) {
+	segments := strings.Split(path, "/@v/")
+	if len(segments) != 2 {
+		return "", ""
+	}
+	version := strings.TrimSuffix(segments[1], filepath.Ext(segments[1]))
+	return segments[0], version
+}

pkg/download/addons/with_pool.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/gomods/athens/pkg/download"
 	"github.com/gomods/athens/pkg/errors"
+	"github.com/gomods/athens/pkg/paths"
 	"github.com/gomods/athens/pkg/storage"
 )
 
@@ -127,3 +128,21 @@ func (p *withpool) Zip(ctx context.Context, mod, ver string) (io.ReadCloser, err
 	}
 	return zip, nil
 }
+
+func (p *withpool) Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error) {
+	const op errors.Op = "pool.Catalog"
+	var modsVers []paths.AllPathParams
+	var nextToken string
+	var err error
+	done := make(chan struct{}, 1)
+	p.jobCh <- func() {
+		modsVers, nextToken, err = p.dp.Catalog(ctx, token, pageSize)
+		close(done)
+	}
+	<-done
+	if err != nil {
+		return nil, "", errors.E(op, err)
+	}
+
+	return modsVers, nextToken, nil
+}

pkg/download/addons/with_pool_test.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/gomods/athens/pkg/download"
+	"github.com/gomods/athens/pkg/paths"
 	"github.com/gomods/athens/pkg/storage"
 )
 
@@ -61,6 +62,9 @@ func TestPoolWrapper(t *testing.T) {
 	m.inputMod = mod
 	m.inputVer = ver
 	m.list = []string{"v0.0.0", "v0.1.0"}
+	m.catalog = []paths.AllPathParams{
+		{"pkg", "v0.1.0"},
+	}
 	givenList, err := dp.List(ctx, mod)
 	if err != m.err {
 		t.Fatalf("expected dp.List err to be %v but got %v", m.err, err)
@@ -96,6 +100,7 @@ type mockDP struct {
 	zip      io.ReadCloser
 	inputMod string
 	inputVer string
+	catalog  []paths.AllPathParams
 }
 
 // List implements GET /{module}/@v/list
@@ -147,6 +152,11 @@ func (m *mockDP) Zip(ctx context.Context, mod, ver string) (io.ReadCloser, error
 	return m.zip, m.err
 }
 
+// Catalog implements GET /catalog
+func (m *mockDP) Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error) {
+	return m.catalog, "", m.err
+}
+
 // Version is a helper method to get Info, GoMod, and Zip together.
 func (m *mockDP) Version(ctx context.Context, mod, ver string) (*storage.Version, error) {
 	panic("skipped")

pkg/download/catalog.go

@@ -0,0 +1,54 @@
+package download
+
+import (
+	"net/http"
+	"strconv"
+
+	"github.com/gobuffalo/buffalo"
+	"github.com/gobuffalo/buffalo/render"
+	"github.com/gomods/athens/pkg/errors"
+	"github.com/gomods/athens/pkg/log"
+	"github.com/gomods/athens/pkg/paths"
+)
+
+// PathCatalog URL.
+const PathCatalog = "/catalog"
+const defaultPageSize = 1000
+
+type catalogRes struct {
+	ModsAndVersions []paths.AllPathParams `json:"modules"`
+	NextPageToken   string                `json:"next"`
+}
+
+// CatalogHandler implements GET baseURL/catalog
+func CatalogHandler(dp Protocol, lggr log.Entry, eng *render.Engine) buffalo.Handler {
+	const op errors.Op = "download.CatalogHandler"
+
+	return func(c buffalo.Context) error {
+		token := c.Param("token")
+		pageSize, err := getLimitFromParam(c.Param("pagesize"))
+		if err != nil {
+			lggr.SystemErr(err)
+			return c.Render(http.StatusInternalServerError, nil)
+		}
+
+		modulesAndVersions, newToken, err := dp.Catalog(c, token, pageSize)
+
+		if err != nil {
+			if errors.Kind(err) != errors.KindNotImplemented {
+				lggr.SystemErr(errors.E(op, err))
+			}
+			return c.Render(errors.Kind(err), eng.JSON(errors.KindText(err)))
+		}
+
+		res := catalogRes{modulesAndVersions, newToken}
+		return c.Render(http.StatusOK, eng.JSON(res))
+	}
+}
+
+func getLimitFromParam(param string) (int, error) {
+	if param == "" {
+		return defaultPageSize, nil
+	}
+	return strconv.Atoi(param)
+}

pkg/download/handler.go

@@ -50,4 +50,5 @@ func RegisterHandlers(app *buffalo.App, opts *HandlerOpts) {
 	app.GET(PathVersionInfo, LogEntryHandler(VersionInfoHandler, opts))
 	app.GET(PathVersionModule, LogEntryHandler(VersionModuleHandler, opts))
 	app.GET(PathVersionZip, LogEntryHandler(VersionZipHandler, opts))
+	app.GET(PathCatalog, LogEntryHandler(CatalogHandler, opts))
 }

pkg/download/protocol.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/gomods/athens/pkg/errors"
 	"github.com/gomods/athens/pkg/observ"
+	"github.com/gomods/athens/pkg/paths"
 	"github.com/gomods/athens/pkg/stash"
 	"github.com/gomods/athens/pkg/storage"
 )
@@ -28,6 +29,9 @@ type Protocol interface {
 
 	// Zip implements GET /{module}/@v/{version}.zip
 	Zip(ctx context.Context, mod, ver string) (io.ReadCloser, error)
+
+	// Catalog implements GET /catalog
+	Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error)
 }
 
 // Wrapper helps extend the main protocol's functionality with addons.
@@ -173,6 +177,19 @@ func (p *protocol) Zip(ctx context.Context, mod, ver string) (io.ReadCloser, err
 	return zip, nil
 }
 
+func (p *protocol) Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error) {
+	const op errors.Op = "protocol.Catalog"
+	ctx, span := observ.StartSpan(ctx, op.String())
+	defer span.End()
+	modulesAndVersions, newToken, err := p.storage.Catalog(ctx, token, pageSize)
+
+	if err != nil {
+		return nil, "", errors.E(op, err)
+	}
+
+	return modulesAndVersions, newToken, err
+}
+
 // union concatenates two version lists and removes duplicates
 func union(list1, list2 []string) []string {
 	if list1 == nil {

pkg/errors/errors.go

@@ -11,11 +11,12 @@ import (
 
 // Kind enums
 const (
-	KindNotFound      = http.StatusNotFound
+	KindNotFound       = http.StatusNotFound
-	KindBadRequest    = http.StatusBadRequest
+	KindBadRequest     = http.StatusBadRequest
-	KindUnexpected    = http.StatusInternalServerError
+	KindUnexpected     = http.StatusInternalServerError
-	KindAlreadyExists = http.StatusConflict
+	KindAlreadyExists  = http.StatusConflict
-	KindRateLimit     = http.StatusTooManyRequests
+	KindRateLimit      = http.StatusTooManyRequests
+	KindNotImplemented = http.StatusNotImplemented
 )
 
 // Error is an Athens system error.

pkg/paths/path.go

@@ -31,8 +31,8 @@ func GetVersion(c buffalo.Context) (string, error) {
 // AllPathParams holds the module and version in the path of a ?go-get=1
 // request
 type AllPathParams struct {
-	Module  string
+	Module  string `json:"module"`
-	Version string
+	Version string `json:"version"`
 }
 
 // GetAllParams fetches the path patams from c and returns them

pkg/storage/azureblob/cataloger.go

@@ -0,0 +1,15 @@
+package azureblob
+
+import (
+	"context"
+
+	"github.com/gomods/athens/pkg/errors"
+	"github.com/gomods/athens/pkg/paths"
+)
+
+// Catalog implements the (./pkg/storage).Cataloger interface
+// It returns a list of modules and versions contained in the storage
+func (s *Storage) Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error) {
+	const op errors.Op = "azure.Catalog"
+	return nil, "", errors.E(op, errors.KindNotImplemented)
+}

pkg/storage/backend.go

@@ -7,4 +7,5 @@ type Backend interface {
 	Checker
 	Saver
 	Deleter
+	Cataloger
 }

pkg/storage/cataloger.go

@@ -0,0 +1,13 @@
+package storage
+
+import (
+	"context"
+
+	"github.com/gomods/athens/pkg/paths"
+)
+
+// Cataloger is the interface that lists all the modules and version contained in the storage
+type Cataloger interface {
+	// Catalog gets all the modules / versions.
+	Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error)
+}

pkg/storage/compliance/tests.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"math/rand"
+	"sort"
 	"testing"
 
 	"github.com/gomods/athens/pkg/errors"
@@ -22,6 +23,9 @@ func RunTests(t *testing.T, b storage.Backend, clearBackend func() error) {
 	testList(t, b)
 	testDelete(t, b)
 	testGet(t, b)
+	if isCatalogImplemented(b) {
+		testCatalog(t, b)
+	}
 }
 
 // testNotFound ensures that a storage Backend
@@ -71,6 +75,11 @@ func testList(t *testing.T, b storage.Backend) {
 		)
 		require.NoError(t, err, "Save for storage failed")
 	}
+	defer func() {
+		for _, ver := range versions {
+			b.Delete(ctx, modname, ver)
+		}
+	}()
 	retVersions, err := b.List(ctx, modname)
 	require.NoError(t, err)
 	require.Equal(t, versions, retVersions)
@@ -84,6 +93,7 @@ func testGet(t *testing.T, b storage.Backend) {
 	mock := getMockModule()
 	zipBts, _ := ioutil.ReadAll(mock.Zip)
 	b.Save(ctx, modname, ver, mock.Mod, bytes.NewReader(zipBts), mock.Info)
+	defer b.Delete(ctx, modname, ver)
 
 	info, err := b.Info(ctx, modname, ver)
 	require.NoError(t, err)
@@ -120,6 +130,49 @@ func testDelete(t *testing.T, b storage.Backend) {
 	require.Equal(t, false, exists)
 }
 
+func testCatalog(t *testing.T, b storage.Backend) {
+	ctx := context.Background()
+
+	mock := getMockModule()
+	zipBts, _ := ioutil.ReadAll(mock.Zip)
+	modname := "github.com/gomods/testCatalogModule"
+	for i := 0; i < 1005; i++ {
+		ver := fmt.Sprintf("v1.2.%04d", i)
+		b.Save(ctx, modname, ver, mock.Mod, bytes.NewReader(zipBts), mock.Info)
+	}
+	defer func() {
+		for i := 0; i < 1005; i++ {
+			ver := fmt.Sprintf("v1.2.%04d", i)
+			b.Delete(ctx, modname, ver)
+		}
+	}()
+
+	allres, next, err := b.Catalog(ctx, "", 1001)
+
+	require.NoError(t, err)
+	require.Equal(t, 1001, len(allres))
+
+	res, next, err := b.Catalog(ctx, next, 50)
+	allres = append(allres, res...)
+	require.NoError(t, err)
+	require.Equal(t, 4, len(res))
+	require.Equal(t, "", next)
+
+	sort.Slice(allres, func(i, j int) bool {
+		if allres[i].Module == allres[j].Module {
+			return allres[i].Version < allres[j].Version
+		}
+		return allres[i].Module < allres[j].Module
+	})
+	require.Equal(t, modname, allres[0].Module)
+	require.Equal(t, "v1.2.0000", allres[0].Version)
+	require.Equal(t, "v1.2.1004", allres[1004].Version)
+
+	for i := 1; i < len(allres); i++ {
+		require.NotEqual(t, allres[i].Version, allres[i-1].Version)
+	}
+}
+
 func getMockModule() *storage.Version {
 	return &storage.Version{
 		Info: []byte("123"),
@@ -127,3 +180,11 @@ func getMockModule() *storage.Version {
 		Zip:  ioutil.NopCloser(bytes.NewReader([]byte("789"))),
 	}
 }
+
+func isCatalogImplemented(b storage.Backend) bool {
+	ctx := context.Background()
+	if _, _, err := b.Catalog(ctx, "", 1); errors.Kind(err) == errors.KindNotImplemented {
+		return false
+	}
+	return true
+}

pkg/storage/fs/cataloger.go

@@ -0,0 +1,81 @@
+package fs
+
+import (
+	"context"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/gomods/athens/pkg/errors"
+	"github.com/gomods/athens/pkg/observ"
+	"github.com/gomods/athens/pkg/paths"
+	"github.com/spf13/afero"
+)
+
+const tokenSeparator = "|"
+
+// Catalog implements the (./pkg/storage).Cataloger interface
+// It returns a list of modules and versions contained in the storage
+func (s *storageImpl) Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error) {
+	const op errors.Op = "fs.Catalog"
+	ctx, span := observ.StartSpan(ctx, op.String())
+	defer span.End()
+
+	fromModule, fromVersion, err := modVerFromToken(token)
+	if err != nil {
+		return nil, "", errors.E(op, err, errors.KindBadRequest)
+	}
+
+	res := make([]paths.AllPathParams, 0)
+	resToken := ""
+	count := pageSize
+
+	err = afero.Walk(s.filesystem, s.rootDir, func(path string, info os.FileInfo, err error) error {
+		if strings.HasSuffix(info.Name(), ".info") {
+			verDir := filepath.Dir(path)
+			modVer, err := filepath.Rel(s.rootDir, verDir)
+			if err != nil {
+				return err
+			}
+
+			m, version := filepath.Split(modVer)
+			module := filepath.Clean(m)
+			module = strings.Replace(module, string(os.PathSeparator), "/", -1)
+
+			if fromModule != "" && module < fromModule { // it is ok to land on the same module
+				return nil
+			}
+
+			if fromVersion != "" && version <= fromVersion { // we must skip same version
+				return nil
+			}
+
+			res = append(res, paths.AllPathParams{module, version})
+			count--
+			if count == 0 {
+				resToken = tokenFromModVer(module, version)
+				return io.EOF
+			}
+		}
+		return nil
+	})
+
+	return res, resToken, nil
+}
+
+func tokenFromModVer(module, version string) string {
+	return module + tokenSeparator + version
+}
+
+func modVerFromToken(token string) (string, string, error) {
+	const op errors.Op = "fs.Catalog"
+	if token == "" {
+		return "", "", nil
+	}
+	values := strings.Split(token, tokenSeparator)
+	if len(values) < 2 {
+		return "", "", errors.E(op, "Invalid token")
+	}
+	return values[0], values[1], nil
+}

pkg/storage/gcp/all_test.go

@@ -2,10 +2,13 @@ package gcp
 
 import (
 	"context"
+	"flag"
 	"net/url"
 	"testing"
 	"time"
 
+	"github.com/gobuffalo/envy"
+	"github.com/gomods/athens/pkg/config"
 	"github.com/stretchr/testify/suite"
 )
 
@@ -25,15 +28,58 @@ type GcpTests struct {
 	bucket  *bucketMock
 }
 
+var realGcp = flag.Bool("realgcp", false, "tests against a real gcp instance")
+var project = flag.String("gcpprj", "", "the gcp project to test against")
+var bucket = flag.String("gcpbucket", "", "the gcp bucket to test against")
+
 func (g *GcpTests) SetupSuite() {
 	g.context = context.Background()
-	g.module = "gcp-test"
+	g.module = "github.com/foo/gcp-test" + time.Now().String()
 	g.version = "v1.2.3"
-	g.url, _ = url.Parse("https://storage.googleapis.com/testbucket")
+
-	g.bucket = newBucketMock()
+	if !*realGcp {
-	g.store = newWithBucket(g.bucket, g.url, time.Second)
+		setupMockStorage(g)
+	} else {
+		setupRealStorage(g)
+	}
 }
 
 func TestGcpStorage(t *testing.T) {
 	suite.Run(t, new(GcpTests))
 }
+
+func (g *GcpTests) BucketReadClosed() bool {
+	if g.bucket != nil {
+		return g.bucket.ReadClosed()
+	}
+	return true
+}
+
+func (g *GcpTests) BucketWriteClosed() bool {
+	if g.bucket != nil {
+		return g.bucket.WriteClosed()
+	}
+	return true
+}
+
+func setupMockStorage(g *GcpTests) {
+	g.url, _ = url.Parse("https://storage.googleapis.com/testbucket")
+	g.bucket = newBucketMock()
+	g.store = newWithBucket(g.bucket, g.url, time.Second)
+}
+
+func setupRealStorage(g *GcpTests) {
+	_, err := envy.MustGet("GOOGLE_APPLICATION_CREDENTIALS")
+	if err != nil {
+		g.T().Skip()
+	}
+	if *project == "" || *bucket == "" {
+		g.T().Skip()
+	}
+
+	g.store, err = New(context.Background(), &config.GCPConfig{
+		ProjectID: *project,
+		Bucket:    *bucket,
+	}, 300*time.Second)
+	g.Require().NoError(err)
+}

pkg/storage/gcp/bucket.go

@@ -18,4 +18,6 @@ type Bucket interface {
 	List(ctx context.Context, prefix string) ([]string, error)
 	// Exists returns true if the file exists
 	Exists(ctx context.Context, path string) (bool, error)
+	// Catalog returns a slice of paths starting from the page given by the token and max elementNum
+	Catalog(ctx context.Context, token string, pageSize int) ([]string, string, error)
 }

pkg/storage/gcp/bucket_cloud.go

@@ -70,3 +70,22 @@ func (b *gcpBucket) Exists(ctx context.Context, path string) (bool, error) {
 
 	return true, nil
 }
+
+func (b *gcpBucket) Catalog(ctx context.Context, token string, pageSize int) ([]string, string, error) {
+	const op errors.Op = "gcpBucket.Catalog"
+
+	it := b.Objects(ctx, nil)
+	p := iterator.NewPager(it, pageSize, token)
+
+	attrs := make([]*storage.ObjectAttrs, 0)
+	nextToken, err := p.NextPage(&attrs)
+	if err != nil {
+		return nil, "", errors.E(op, err)
+	}
+
+	res := []string{}
+	for _, attr := range attrs {
+		res = append(res, attr.Name)
+	}
+	return res, nextToken, nil
+}

pkg/storage/gcp/bucket_mock.go

@@ -5,6 +5,8 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"sort"
+	"strconv"
 	"strings"
 	"sync"
 )
@@ -95,6 +97,36 @@ func (m *bucketMock) Exists(ctx context.Context, path string) (bool, error) {
 	return found, nil
 }
 
+func (m *bucketMock) Catalog(ctx context.Context, token string, elementNum int) ([]string, string, error) {
+	keys := make([]string, 0)
+
+	m.lock.RLock()
+	defer m.lock.RUnlock()
+
+	for k := range m.db {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	from := 0
+	if token != "" {
+		var err error
+		from, err = strconv.Atoi(token)
+		if err != nil {
+			return nil, "", err
+		}
+	}
+	if from > len(keys)-1 {
+		return make([]string, 0), "", nil
+	}
+	to := from + elementNum
+	resToken := strconv.Itoa(to)
+	if to > len(keys)-1 {
+		to = len(keys) - 1
+		resToken = ""
+	}
+	return keys[from:to], resToken, nil
+}
+
 func (m *bucketMock) ReadClosed() bool {
 	return (m.readLockCount == 0)
 }

pkg/storage/gcp/cataloger.go

@@ -0,0 +1,67 @@
+package gcp
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/gomods/athens/pkg/config"
+	"github.com/gomods/athens/pkg/paths"
+
+	"github.com/gomods/athens/pkg/errors"
+	"github.com/gomods/athens/pkg/observ"
+)
+
+// Catalog implements the (./pkg/storage).Catalog interface
+// It returns a list of versions, if any, for a given module
+func (s *Storage) Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error) {
+	const op errors.Op = "gcp.Catalog"
+	ctx, span := observ.StartSpan(ctx, op.String())
+	defer span.End()
+	res := make([]paths.AllPathParams, 0)
+	var resToken string
+	count := pageSize
+
+	for count > 0 {
+		var catalog []string
+		var err error
+		catalog, resToken, err = s.bucket.Catalog(ctx, token, 3*count)
+		if err != nil {
+			return nil, "", errors.E(op, err)
+		}
+		pathsAndVers := fetchModsAndVersions(catalog)
+		res = append(res, pathsAndVers...)
+		count -= len(pathsAndVers)
+
+		if resToken == "" { // meaning we reached the end
+			break
+		}
+	}
+	return res, resToken, nil
+}
+
+func fetchModsAndVersions(catalog []string) []paths.AllPathParams {
+	res := make([]paths.AllPathParams, 0)
+	for _, p := range catalog {
+		if !strings.HasSuffix(p, ".info") {
+			continue
+		}
+		p, err := parseGcpKey(p)
+		if err != nil {
+			continue
+		}
+		res = append(res, p)
+	}
+	return res
+}
+
+func parseGcpKey(p string) (paths.AllPathParams, error) {
+	const op errors.Op = "gcp.parseS3Key"
+	// github.com/gomods/testCatalogModule/@v/v1.2.0976.info
+	m, v := config.ModuleVersionFromPath(p)
+
+	if m == "" || v == "" {
+		return paths.AllPathParams{}, errors.E(op, fmt.Errorf("invalid object key format %s", p))
+	}
+	return paths.AllPathParams{m, v}, nil
+}

pkg/storage/gcp/gcp_test.go

@@ -3,6 +3,7 @@ package gcp
 import (
 	"bytes"
 	"context"
+	"fmt"
 	"io/ioutil"
 	"testing"
 	"time"
@@ -50,9 +51,14 @@ func (g *GcpTests) TestSaveGetListExistsRoundTrip() {
 		r.Equal(true, exists)
 	})
 
+	g.T().Run("Delete storage", func(t *testing.T) {
+		err := g.store.Delete(g.context, g.module, g.version)
+		r.NoError(err)
+	})
+
 	g.T().Run("Resources closed", func(t *testing.T) {
-		r.Equal(true, g.bucket.ReadClosed())
+		r.Equal(true, g.BucketReadClosed())
-		r.Equal(true, g.bucket.WriteClosed())
+		r.Equal(true, g.BucketWriteClosed())
 	})
 }
 
@@ -95,3 +101,32 @@ func (g *GcpTests) TestNotFounds() {
 		r.Equal(0, len(list))
 	})
 }
+
+func (g *GcpTests) TestCatalog() {
+	r := g.Require()
+	for i := 0; i < 50; i++ {
+		ver := fmt.Sprintf("v1.2.%04d", i)
+		err := g.store.Save(g.context, g.module, ver, mod, bytes.NewReader(zip), info)
+		r.NoError(err)
+	}
+	defer func() {
+		for i := 0; i < 50; i++ {
+			ver := fmt.Sprintf("v1.2.%04d", i)
+			err := g.store.Delete(g.context, g.module, ver)
+			r.NoError(err)
+		}
+	}()
+
+	allres, nextToken, err := g.store.Catalog(g.context, "", 2)
+	r.NoError(err)
+	r.Equal(len(allres), 2)
+	r.NotEqual("", nextToken)
+	r.Equal(allres[0].Module, g.module)
+
+	res, nextToken, err := g.store.Catalog(g.context, nextToken, 50)
+	allres = append(allres, res...)
+	r.NoError(err)
+	r.Equal(len(allres), 50)
+	r.Equal(len(res), 48)
+	r.Equal("", nextToken)
+}

pkg/storage/minio/cataloger.go

@@ -0,0 +1,16 @@
+package minio
+
+import (
+	"context"
+
+	"github.com/gomods/athens/pkg/paths"
+
+	"github.com/gomods/athens/pkg/errors"
+)
+
+// Catalog implements the (./pkg/storage).Cataloger interface
+// It returns a list of modules and versions contained in the storage
+func (s *storageImpl) Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error) {
+	const op errors.Op = "minio.Catalog"
+	return nil, "", errors.E(op, errors.KindNotImplemented)
+}

pkg/storage/mongo/cataloger.go

@@ -0,0 +1,16 @@
+package mongo
+
+import (
+	"context"
+
+	"github.com/gomods/athens/pkg/paths"
+
+	"github.com/gomods/athens/pkg/errors"
+)
+
+// Catalog implements the (./pkg/storage).Cataloger interface
+// It returns a list of modules and versions contained in the storage
+func (s *ModuleStore) Catalog(ctx context.Context, token string, elements int) ([]paths.AllPathParams, string, error) {
+	const op errors.Op = "mongo.Catalog"
+	return nil, "", errors.E(op, errors.KindNotImplemented)
+}

pkg/storage/s3/cataloger.go

@@ -0,0 +1,85 @@
+package s3
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/gomods/athens/pkg/config"
+
+	"github.com/gomods/athens/pkg/paths"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/gomods/athens/pkg/errors"
+	"github.com/gomods/athens/pkg/observ"
+)
+
+// Catalog implements the (./pkg/storage).Cataloger interface
+// It returns a list of modules and versions contained in the storage
+func (s *Storage) Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error) {
+	const op errors.Op = "s3.Catalog"
+	ctx, span := observ.StartSpan(ctx, op.String())
+	defer span.End()
+	queryToken := token
+	res := make([]paths.AllPathParams, 0)
+	count := pageSize
+	for count > 0 {
+		lsParams := &s3.ListObjectsInput{
+			Bucket: aws.String(s.bucket),
+			Marker: &queryToken,
+		}
+
+		loo, err := s.s3API.ListObjectsWithContext(ctx, lsParams)
+		if err != nil {
+			return nil, "", errors.E(op, err)
+		}
+
+		m, lastKey := fetchModsAndVersions(loo.Contents, count)
+
+		res = append(res, m...)
+		count -= len(m)
+		queryToken = lastKey
+
+		if !*loo.IsTruncated { // not truncated, there is no point in asking more
+			if count > 0 { // it means we reached the end, no subsequent requests are necessary
+				queryToken = ""
+			}
+			break
+		}
+	}
+	return res, queryToken, nil
+}
+
+func fetchModsAndVersions(objects []*s3.Object, elementsNum int) ([]paths.AllPathParams, string) {
+	res := make([]paths.AllPathParams, 0)
+	lastKey := ""
+	for _, o := range objects {
+		if !strings.HasSuffix(*o.Key, ".info") {
+			continue
+		}
+		p, err := parseS3Key(o)
+		if err != nil {
+			continue
+		}
+
+		res = append(res, p)
+		lastKey = *o.Key
+
+		elementsNum--
+		if elementsNum == 0 {
+			break
+		}
+	}
+	return res, lastKey
+}
+
+func parseS3Key(o *s3.Object) (paths.AllPathParams, error) {
+	const op errors.Op = "s3.parseS3Key"
+	m, v := config.ModuleVersionFromPath(*o.Key)
+
+	if m == "" || v == "" {
+		return paths.AllPathParams{}, errors.E(op, fmt.Errorf("invalid object key format %s", *o.Key))
+	}
+	return paths.AllPathParams{m, v}, nil
+}

scripts/test_e2e.sh

@@ -59,3 +59,11 @@ clearGoModCache
 # Verify that the test works against the proxy
 export GOPROXY=http://localhost:3000
 $GO_BINARY_PATH run .
+
+CATALOG_RES=$(curl localhost:3000/catalog)
+CATALOG_EXPECTED='{"modules":[{"module":"github.com/athens-artifacts/no-tags","version":"v0.0.0-20180803171426-1a540c5d67ab"}],"next":""}'
+
+if [[ "$CATALOG_RES" != "$CATALOG_EXPECTED" ]]; then
+  echo ERROR: catalog endpoint failed
+  exit 1 # terminate and indicate error
+fi