Catalogendpoint (#955)

* Added new cataloger interface

* Implementing catalog protocol

* Propagated to protocol and over

* First round of fixes

* S3 almost ready, need to be tested

* Going on with testing s3

* Better testing with s3

* Simplified catalog tests

* Preparing gcp tests to access a gcp instance

* Fixing initialization errors

* Removed some prints

* Gcp ready, to be tested

* Gcp working

* Aligned bucket mock to catalog method

* Switched res payload to json

* Added catalog method to all storage instances

* Added catalog method to unsupported storages

* Fixed with pool test

* Restored tests

* Fixed gcp constructor

* Implemented catalog for fs

* Removed trace

* E2e tests, fixed fs

* Fixed module name return value

* Added cataloger method to azure storage

* Added docs

* Changed pagesize parameter name

* Fixed gofmt error

* Added json tags to result. Fixed lint warning

* Removed extra line

* Changed not implemented error to http.KindNotImplemented

* Checking for inequality on results

* Lower-cased json keys

* Added cleaning of path separator

* Fixed review comments

* Fixed docs

pkg/storage/s3/cataloger.go

@@ -0,0 +1,85 @@
+package s3
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/gomods/athens/pkg/config"
+
+	"github.com/gomods/athens/pkg/paths"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/gomods/athens/pkg/errors"
+	"github.com/gomods/athens/pkg/observ"
+)
+
+// Catalog implements the (./pkg/storage).Cataloger interface
+// It returns a list of modules and versions contained in the storage
+func (s *Storage) Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error) {
+	const op errors.Op = "s3.Catalog"
+	ctx, span := observ.StartSpan(ctx, op.String())
+	defer span.End()
+	queryToken := token
+	res := make([]paths.AllPathParams, 0)
+	count := pageSize
+	for count > 0 {
+		lsParams := &s3.ListObjectsInput{
+			Bucket: aws.String(s.bucket),
+			Marker: &queryToken,
+		}
+
+		loo, err := s.s3API.ListObjectsWithContext(ctx, lsParams)
+		if err != nil {
+			return nil, "", errors.E(op, err)
+		}
+
+		m, lastKey := fetchModsAndVersions(loo.Contents, count)
+
+		res = append(res, m...)
+		count -= len(m)
+		queryToken = lastKey
+
+		if !*loo.IsTruncated { // not truncated, there is no point in asking more
+			if count > 0 { // it means we reached the end, no subsequent requests are necessary
+				queryToken = ""
+			}
+			break
+		}
+	}
+	return res, queryToken, nil
+}
+
+func fetchModsAndVersions(objects []*s3.Object, elementsNum int) ([]paths.AllPathParams, string) {
+	res := make([]paths.AllPathParams, 0)
+	lastKey := ""
+	for _, o := range objects {
+		if !strings.HasSuffix(*o.Key, ".info") {
+			continue
+		}
+		p, err := parseS3Key(o)
+		if err != nil {
+			continue
+		}
+
+		res = append(res, p)
+		lastKey = *o.Key
+
+		elementsNum--
+		if elementsNum == 0 {
+			break
+		}
+	}
+	return res, lastKey
+}
+
+func parseS3Key(o *s3.Object) (paths.AllPathParams, error) {
+	const op errors.Op = "s3.parseS3Key"
+	m, v := config.ModuleVersionFromPath(*o.Key)
+
+	if m == "" || v == "" {
+		return paths.AllPathParams{}, errors.E(op, fmt.Errorf("invalid object key format %s", *o.Key))
+	}
+	return paths.AllPathParams{m, v}, nil
+}