Catalog endpoint support for mongo (#985)

* Added new cataloger interface

* Implementing catalog protocol

* Propagated to protocol and over

* First round of fixes

* S3 almost ready, need to be tested

* Going on with testing s3

* Better testing with s3

* Simplified catalog tests

* Preparing gcp tests to access a gcp instance

* Fixing initialization errors

* Removed some prints

* Gcp ready, to be tested

* Gcp working

* Aligned bucket mock to catalog method

* Switched res payload to json

* Added catalog method to all storage instances

* Added catalog method to unsupported storages

* Fixed with pool test

* Restored tests

* Fixed gcp constructor

* Implemented catalog for fs

* Removed trace

* E2e tests, fixed fs

* Fixed module name return value

* Added cataloger method to azure storage

* Added docs

* Changed pagesize parameter name

* Fixed gofmt error

* Added json tags to result. Fixed lint warning

* Removed extra line

* Changed not implemented error to http.KindNotImplemented

* Checking for inequality on results

* Lower-cased json keys

* Added cleaning of path separator

* Fixed review comments

* Add catalog endpoint for mongo

* Add omitempty to ID

* Fix catalog tests

* update for next token

* fix e2e

* Make query readable

* Fix language and e2e script

* remove new line

config.dev.toml

@@ -198,7 +198,7 @@ StatsExporter = "prometheus"
         # Should be used for testing or development only
         # Env override: ATHENS_MONGO_INSECURE
         Insecure = false
-        
+
     [Storage.S3]
         # Region for S3 storage
         # Env override: AWS_REGION

docs/content/design/proxy.md

@@ -65,8 +65,7 @@ Exclude lists for public modules are also globs that tell the Athens proxy what
 
 ## Catalog Endpoint
 
-The proxy provides a `/catalog` service endpoint to fetch all the modules and their versions contained in the local storage.
-The endpoint accepts a continuation token and a page size parameter in order to provide paginated results.
+The proxy provides a `/catalog` service endpoint to fetch all the modules and their versions contained in the local storage. The endpoint accepts a continuation token and a page size parameter in order to provide paginated results.
 
 A query is of the form
 
@@ -83,5 +82,4 @@ The result is a json with the following structure:
  "next":""}'
 ```
 
-An empty `next` token means that no more pages are available.
-The default page size is 1000.
+If a `next` token is not returned, then it means that no more pages are available. The default page size is 1000.

pkg/storage/handler.go

@@ -15,7 +15,7 @@ const defaultPageSize = 1000
 
 type catalogRes struct {
 	ModsAndVersions []paths.AllPathParams `json:"modules"`
-	NextPageToken   string                `json:"next"`
+	NextPageToken   string                `json:"next,omitempty"`
 }
 
 // RegisterHandlers is a convenience method that registers

pkg/storage/module.go

@@ -1,10 +1,15 @@
 package storage
 
+import (
+	"github.com/globalsign/mgo/bson"
+)
+
 // Module represents a vgo module saved in a storage backend.
 type Module struct {
-	Module  string `bson:"module"`
-	Version string `bson:"version"`
-	Mod     []byte `bson:"mod"`
-	Zip     []byte `bson:"zip"`
-	Info    []byte `bson:"info"`
+	ID      bson.ObjectId `bson:"_id,omitempty"`
+	Module  string        `bson:"module"`
+	Version string        `bson:"version"`
+	Mod     []byte        `bson:"mod"`
+	Zip     []byte        `bson:"zip"`
+	Info    []byte        `bson:"info"`
 }

pkg/storage/mongo/cataloger.go

@@ -3,14 +3,49 @@ package mongo
 import (
 	"context"
 
-	"github.com/gomods/athens/pkg/paths"
-
+	"github.com/globalsign/mgo/bson"
 	"github.com/gomods/athens/pkg/errors"
+	"github.com/gomods/athens/pkg/paths"
+	"github.com/gomods/athens/pkg/storage"
 )
 
 // Catalog implements the (./pkg/storage).Cataloger interface
 // It returns a list of modules and versions contained in the storage
-func (s *ModuleStore) Catalog(ctx context.Context, token string, elements int) ([]paths.AllPathParams, string, error) {
+func (s *ModuleStore) Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error) {
 	const op errors.Op = "mongo.Catalog"
-	return nil, "", errors.E(op, errors.KindNotImplemented)
+	q := bson.M{}
+	if token != "" {
+		q = bson.M{"_id": bson.M{"$gt": bson.ObjectIdHex(token)}}
+	}
+
+	fields := bson.M{"module": 1, "version": 1}
+
+	c := s.s.DB(s.d).C(s.c)
+	modules := make([]storage.Module, 0)
+	err := c.Find(q).
+		Select(fields).
+		Sort("_id").
+		Limit(pageSize).
+		All(&modules)
+
+	if err != nil {
+		return nil, "", errors.E(op, err)
+	}
+
+	// If there are 0 results, return empty results without an error
+	if len(modules) == 0 {
+		return nil, "", nil
+	}
+
+	var versions = make([]paths.AllPathParams, len(modules))
+	for i := range modules {
+		versions[i].Module = modules[i].Module
+		versions[i].Version = modules[i].Version
+	}
+
+	var next = modules[len(modules)-1].ID.Hex()
+	if len(modules) < pageSize {
+		return versions, "", nil
+	}
+	return versions, next, nil
 }

pkg/storage/mongo/mongo_test.go

@@ -1,7 +1,7 @@
 package mongo
 
 import (
-	"os"
+	"path/filepath"
 	"testing"
 
 	"github.com/gomods/athens/pkg/config"
@@ -9,14 +9,17 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+var (
+	testConfigFile = filepath.Join("..", "..", "..", "config.dev.toml")
+)
+
 func TestBackend(t *testing.T) {
 	backend := getStorage(t)
 	compliance.RunTests(t, backend, backend.clear)
 }
 
 func (m *ModuleStore) clear() error {
-	m.s.DB(m.d).C(m.c).DropCollection()
-
+	m.s.DB(m.d).DropDatabase()
 	return m.initDatabase()
 }
 
@@ -26,12 +29,15 @@ func BenchmarkBackend(b *testing.B) {
 }
 
 func getStorage(tb testing.TB) *ModuleStore {
-	url := os.Getenv("ATHENS_MONGO_STORAGE_URL")
-	if url == "" {
+	conf, err := config.GetConf(testConfigFile)
+	if err != nil {
+		tb.Fatalf("Unable to parse config file: %s", err.Error())
+	}
+	if conf.Storage.Mongo.URL == "" {
 		tb.SkipNow()
 	}
 
-	backend, err := NewStorage(&config.MongoConfig{URL: url}, config.GetTimeoutDuration(300))
+	backend, err := NewStorage(&config.MongoConfig{URL: conf.Storage.Mongo.URL}, config.GetTimeoutDuration(300))
 	require.NoError(tb, err)
 
 	return backend

scripts/test_e2e.sh

@@ -61,7 +61,7 @@ export GOPROXY=http://localhost:3000
 $GO_BINARY_PATH run .
 
 CATALOG_RES=$(curl localhost:3000/catalog)
-CATALOG_EXPECTED='{"modules":[{"module":"github.com/athens-artifacts/no-tags","version":"v0.0.0-20180803171426-1a540c5d67ab"}],"next":""}'
+CATALOG_EXPECTED='{"modules":[{"module":"github.com/athens-artifacts/no-tags","version":"v0.0.0-20180803171426-1a540c5d67ab"}]}'
 
 if [[ "$CATALOG_RES" != "$CATALOG_EXPECTED" ]]; then
   echo ERROR: catalog endpoint failed