Catalogendpoint (#955)

* Added new cataloger interface

* Implementing catalog protocol

* Propagated to protocol and over

* First round of fixes

* S3 almost ready, need to be tested

* Going on with testing s3

* Better testing with s3

* Simplified catalog tests

* Preparing gcp tests to access a gcp instance

* Fixing initialization errors

* Removed some prints

* Gcp ready, to be tested

* Gcp working

* Aligned bucket mock to catalog method

* Switched res payload to json

* Added catalog method to all storage instances

* Added catalog method to unsupported storages

* Fixed with pool test

* Restored tests

* Fixed gcp constructor

* Implemented catalog for fs

* Removed trace

* E2e tests, fixed fs

* Fixed module name return value

* Added cataloger method to azure storage

* Added docs

* Changed pagesize parameter name

* Fixed gofmt error

* Added json tags to result. Fixed lint warning

* Removed extra line

* Changed not implemented error to http.KindNotImplemented

* Checking for inequality on results

* Lower-cased json keys

* Added cleaning of path separator

* Fixed review comments

* Fixed docs

pkg/storage/compliance/tests.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"math/rand"
+	"sort"
 	"testing"
 
 	"github.com/gomods/athens/pkg/errors"
@@ -22,6 +23,9 @@ func RunTests(t *testing.T, b storage.Backend, clearBackend func() error) {
 	testList(t, b)
 	testDelete(t, b)
 	testGet(t, b)
+	if isCatalogImplemented(b) {
+		testCatalog(t, b)
+	}
 }
 
 // testNotFound ensures that a storage Backend
@@ -71,6 +75,11 @@ func testList(t *testing.T, b storage.Backend) {
 		)
 		require.NoError(t, err, "Save for storage failed")
 	}
+	defer func() {
+		for _, ver := range versions {
+			b.Delete(ctx, modname, ver)
+		}
+	}()
 	retVersions, err := b.List(ctx, modname)
 	require.NoError(t, err)
 	require.Equal(t, versions, retVersions)
@@ -84,6 +93,7 @@ func testGet(t *testing.T, b storage.Backend) {
 	mock := getMockModule()
 	zipBts, _ := ioutil.ReadAll(mock.Zip)
 	b.Save(ctx, modname, ver, mock.Mod, bytes.NewReader(zipBts), mock.Info)
+	defer b.Delete(ctx, modname, ver)
 
 	info, err := b.Info(ctx, modname, ver)
 	require.NoError(t, err)
@@ -120,6 +130,49 @@ func testDelete(t *testing.T, b storage.Backend) {
 	require.Equal(t, false, exists)
 }
 
+func testCatalog(t *testing.T, b storage.Backend) {
+	ctx := context.Background()
+
+	mock := getMockModule()
+	zipBts, _ := ioutil.ReadAll(mock.Zip)
+	modname := "github.com/gomods/testCatalogModule"
+	for i := 0; i < 1005; i++ {
+		ver := fmt.Sprintf("v1.2.%04d", i)
+		b.Save(ctx, modname, ver, mock.Mod, bytes.NewReader(zipBts), mock.Info)
+	}
+	defer func() {
+		for i := 0; i < 1005; i++ {
+			ver := fmt.Sprintf("v1.2.%04d", i)
+			b.Delete(ctx, modname, ver)
+		}
+	}()
+
+	allres, next, err := b.Catalog(ctx, "", 1001)
+
+	require.NoError(t, err)
+	require.Equal(t, 1001, len(allres))
+
+	res, next, err := b.Catalog(ctx, next, 50)
+	allres = append(allres, res...)
+	require.NoError(t, err)
+	require.Equal(t, 4, len(res))
+	require.Equal(t, "", next)
+
+	sort.Slice(allres, func(i, j int) bool {
+		if allres[i].Module == allres[j].Module {
+			return allres[i].Version < allres[j].Version
+		}
+		return allres[i].Module < allres[j].Module
+	})
+	require.Equal(t, modname, allres[0].Module)
+	require.Equal(t, "v1.2.0000", allres[0].Version)
+	require.Equal(t, "v1.2.1004", allres[1004].Version)
+
+	for i := 1; i < len(allres); i++ {
+		require.NotEqual(t, allres[i].Version, allres[i-1].Version)
+	}
+}
+
 func getMockModule() *storage.Version {
 	return &storage.Version{
 		Info: []byte("123"),
@@ -127,3 +180,11 @@ func getMockModule() *storage.Version {
 		Zip:  ioutil.NopCloser(bytes.NewReader([]byte("789"))),
 	}
 }
+
+func isCatalogImplemented(b storage.Backend) bool {
+	ctx := context.Background()
+	if _, _, err := b.Catalog(ctx, "", 1); errors.Kind(err) == errors.KindNotImplemented {
+		return false
+	}
+	return true
+}