Catalogendpoint (#955)

* Added new cataloger interface

* Implementing catalog protocol

* Propagated to protocol and over

* First round of fixes

* S3 almost ready, need to be tested

* Going on with testing s3

* Better testing with s3

* Simplified catalog tests

* Preparing gcp tests to access a gcp instance

* Fixing initialization errors

* Removed some prints

* Gcp ready, to be tested

* Gcp working

* Aligned bucket mock to catalog method

* Switched res payload to json

* Added catalog method to all storage instances

* Added catalog method to unsupported storages

* Fixed with pool test

* Restored tests

* Fixed gcp constructor

* Implemented catalog for fs

* Removed trace

* E2e tests, fixed fs

* Fixed module name return value

* Added cataloger method to azure storage

* Added docs

* Changed pagesize parameter name

* Fixed gofmt error

* Added json tags to result. Fixed lint warning

* Removed extra line

* Changed not implemented error to http.KindNotImplemented

* Checking for inequality on results

* Lower-cased json keys

* Added cleaning of path separator

* Fixed review comments

* Fixed docs

pkg/download/addons/with_pool.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/gomods/athens/pkg/download"
 	"github.com/gomods/athens/pkg/errors"
+	"github.com/gomods/athens/pkg/paths"
 	"github.com/gomods/athens/pkg/storage"
 )
 
@@ -127,3 +128,21 @@ func (p *withpool) Zip(ctx context.Context, mod, ver string) (io.ReadCloser, err
 	}
 	return zip, nil
 }
+
+func (p *withpool) Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error) {
+	const op errors.Op = "pool.Catalog"
+	var modsVers []paths.AllPathParams
+	var nextToken string
+	var err error
+	done := make(chan struct{}, 1)
+	p.jobCh <- func() {
+		modsVers, nextToken, err = p.dp.Catalog(ctx, token, pageSize)
+		close(done)
+	}
+	<-done
+	if err != nil {
+		return nil, "", errors.E(op, err)
+	}
+
+	return modsVers, nextToken, nil
+}

pkg/download/addons/with_pool_test.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/gomods/athens/pkg/download"
+	"github.com/gomods/athens/pkg/paths"
 	"github.com/gomods/athens/pkg/storage"
 )
 
@@ -61,6 +62,9 @@ func TestPoolWrapper(t *testing.T) {
 	m.inputMod = mod
 	m.inputVer = ver
 	m.list = []string{"v0.0.0", "v0.1.0"}
+	m.catalog = []paths.AllPathParams{
+		{"pkg", "v0.1.0"},
+	}
 	givenList, err := dp.List(ctx, mod)
 	if err != m.err {
 		t.Fatalf("expected dp.List err to be %v but got %v", m.err, err)
@@ -96,6 +100,7 @@ type mockDP struct {
 	zip      io.ReadCloser
 	inputMod string
 	inputVer string
+	catalog  []paths.AllPathParams
 }
 
 // List implements GET /{module}/@v/list
@@ -147,6 +152,11 @@ func (m *mockDP) Zip(ctx context.Context, mod, ver string) (io.ReadCloser, error
 	return m.zip, m.err
 }
 
+// Catalog implements GET /catalog
+func (m *mockDP) Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error) {
+	return m.catalog, "", m.err
+}
+
 // Version is a helper method to get Info, GoMod, and Zip together.
 func (m *mockDP) Version(ctx context.Context, mod, ver string) (*storage.Version, error) {
 	panic("skipped")

pkg/download/catalog.go

@@ -0,0 +1,54 @@
+package download
+
+import (
+	"net/http"
+	"strconv"
+
+	"github.com/gobuffalo/buffalo"
+	"github.com/gobuffalo/buffalo/render"
+	"github.com/gomods/athens/pkg/errors"
+	"github.com/gomods/athens/pkg/log"
+	"github.com/gomods/athens/pkg/paths"
+)
+
+// PathCatalog URL.
+const PathCatalog = "/catalog"
+const defaultPageSize = 1000
+
+type catalogRes struct {
+	ModsAndVersions []paths.AllPathParams `json:"modules"`
+	NextPageToken   string                `json:"next"`
+}
+
+// CatalogHandler implements GET baseURL/catalog
+func CatalogHandler(dp Protocol, lggr log.Entry, eng *render.Engine) buffalo.Handler {
+	const op errors.Op = "download.CatalogHandler"
+
+	return func(c buffalo.Context) error {
+		token := c.Param("token")
+		pageSize, err := getLimitFromParam(c.Param("pagesize"))
+		if err != nil {
+			lggr.SystemErr(err)
+			return c.Render(http.StatusInternalServerError, nil)
+		}
+
+		modulesAndVersions, newToken, err := dp.Catalog(c, token, pageSize)
+
+		if err != nil {
+			if errors.Kind(err) != errors.KindNotImplemented {
+				lggr.SystemErr(errors.E(op, err))
+			}
+			return c.Render(errors.Kind(err), eng.JSON(errors.KindText(err)))
+		}
+
+		res := catalogRes{modulesAndVersions, newToken}
+		return c.Render(http.StatusOK, eng.JSON(res))
+	}
+}
+
+func getLimitFromParam(param string) (int, error) {
+	if param == "" {
+		return defaultPageSize, nil
+	}
+	return strconv.Atoi(param)
+}

pkg/download/handler.go

@@ -50,4 +50,5 @@ func RegisterHandlers(app *buffalo.App, opts *HandlerOpts) {
 	app.GET(PathVersionInfo, LogEntryHandler(VersionInfoHandler, opts))
 	app.GET(PathVersionModule, LogEntryHandler(VersionModuleHandler, opts))
 	app.GET(PathVersionZip, LogEntryHandler(VersionZipHandler, opts))
+	app.GET(PathCatalog, LogEntryHandler(CatalogHandler, opts))
 }

pkg/download/protocol.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/gomods/athens/pkg/errors"
 	"github.com/gomods/athens/pkg/observ"
+	"github.com/gomods/athens/pkg/paths"
 	"github.com/gomods/athens/pkg/stash"
 	"github.com/gomods/athens/pkg/storage"
 )
@@ -28,6 +29,9 @@ type Protocol interface {
 
 	// Zip implements GET /{module}/@v/{version}.zip
 	Zip(ctx context.Context, mod, ver string) (io.ReadCloser, error)
+
+	// Catalog implements GET /catalog
+	Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error)
 }
 
 // Wrapper helps extend the main protocol's functionality with addons.
@@ -173,6 +177,19 @@ func (p *protocol) Zip(ctx context.Context, mod, ver string) (io.ReadCloser, err
 	return zip, nil
 }
 
+func (p *protocol) Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error) {
+	const op errors.Op = "protocol.Catalog"
+	ctx, span := observ.StartSpan(ctx, op.String())
+	defer span.End()
+	modulesAndVersions, newToken, err := p.storage.Catalog(ctx, token, pageSize)
+
+	if err != nil {
+		return nil, "", errors.E(op, err)
+	}
+
+	return modulesAndVersions, newToken, err
+}
+
 // union concatenates two version lists and removes duplicates
 func union(list1, list2 []string) []string {
 	if list1 == nil {