diff --git a/docs/content/design/proxy.md b/docs/content/design/proxy.md index 5ac159fb..eb08a7de 100644 --- a/docs/content/design/proxy.md +++ b/docs/content/design/proxy.md @@ -62,3 +62,26 @@ Private module filters are string globs that tell the Athens proxy what is a pri ### Exclude Lists for Public Modules Exclude lists for public modules are also globs that tell the Athens proxy what modules it should never download from any upstream proxy. For example, the string `github.com/arschles/**` tells the Athens proxy to always return `404 Not Found` to clients. + +## Catalog Endpoint + +The proxy provides a `/catalog` service endpoint to fetch all the modules and their versions contained in the local storage. +The endpoint accepts a continuation token and a page size parameter in order to provide paginated results. + +A query is of the form + +`https://proxyurl/catalog?token=foo&limit=47` + +Where token is an optional continuation token and limit is the desired size of the returned page. +The `token` parameter is not required for the first call and it's needed for handling paginated results. + + +The result is a json with the following structure: + +``` +{"modules": [{"module":"github.com/athens-artifacts/no-tags","version":"v1.0.0"}], + "next":""}' +``` + +An empty `next` token means that no more pages are available. +The default page size is 1000. diff --git a/pkg/config/module.go b/pkg/config/module.go index 39d4a3f8..48412f84 100644 --- a/pkg/config/module.go +++ b/pkg/config/module.go @@ -1,9 +1,13 @@ package config -import "fmt" +import ( + "fmt" + "path/filepath" + "strings" +) // PackageVersionedName return package full name used in storage. -// E.g athens/@v/v1.0/go.mod +// E.g athens/@v/v1.0.mod func PackageVersionedName(module, version, ext string) string { return fmt.Sprintf("%s/@v/%s.%s", module, version, ext) } @@ -13,3 +17,15 @@ func PackageVersionedName(module, version, ext string) string { func FmtModVer(mod, ver string) string { return fmt.Sprintf("%s@%s", mod, ver) } + +// ModuleVersionFromPath returns module and version from a +// storage path +// E.g athens/@v/v1.0.info -> athens and v.1.0 +func ModuleVersionFromPath(path string) (string, string) { + segments := strings.Split(path, "/@v/") + if len(segments) != 2 { + return "", "" + } + version := strings.TrimSuffix(segments[1], filepath.Ext(segments[1])) + return segments[0], version +} diff --git a/pkg/download/addons/with_pool.go b/pkg/download/addons/with_pool.go index 605dad80..6d36c786 100644 --- a/pkg/download/addons/with_pool.go +++ b/pkg/download/addons/with_pool.go @@ -6,6 +6,7 @@ import ( "github.com/gomods/athens/pkg/download" "github.com/gomods/athens/pkg/errors" + "github.com/gomods/athens/pkg/paths" "github.com/gomods/athens/pkg/storage" ) @@ -127,3 +128,21 @@ func (p *withpool) Zip(ctx context.Context, mod, ver string) (io.ReadCloser, err } return zip, nil } + +func (p *withpool) Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error) { + const op errors.Op = "pool.Catalog" + var modsVers []paths.AllPathParams + var nextToken string + var err error + done := make(chan struct{}, 1) + p.jobCh <- func() { + modsVers, nextToken, err = p.dp.Catalog(ctx, token, pageSize) + close(done) + } + <-done + if err != nil { + return nil, "", errors.E(op, err) + } + + return modsVers, nextToken, nil +} diff --git a/pkg/download/addons/with_pool_test.go b/pkg/download/addons/with_pool_test.go index 307e4588..d1fe8151 100644 --- a/pkg/download/addons/with_pool_test.go +++ b/pkg/download/addons/with_pool_test.go @@ -11,6 +11,7 @@ import ( "time" "github.com/gomods/athens/pkg/download" + "github.com/gomods/athens/pkg/paths" "github.com/gomods/athens/pkg/storage" ) @@ -61,6 +62,9 @@ func TestPoolWrapper(t *testing.T) { m.inputMod = mod m.inputVer = ver m.list = []string{"v0.0.0", "v0.1.0"} + m.catalog = []paths.AllPathParams{ + {"pkg", "v0.1.0"}, + } givenList, err := dp.List(ctx, mod) if err != m.err { t.Fatalf("expected dp.List err to be %v but got %v", m.err, err) @@ -96,6 +100,7 @@ type mockDP struct { zip io.ReadCloser inputMod string inputVer string + catalog []paths.AllPathParams } // List implements GET /{module}/@v/list @@ -147,6 +152,11 @@ func (m *mockDP) Zip(ctx context.Context, mod, ver string) (io.ReadCloser, error return m.zip, m.err } +// Catalog implements GET /catalog +func (m *mockDP) Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error) { + return m.catalog, "", m.err +} + // Version is a helper method to get Info, GoMod, and Zip together. func (m *mockDP) Version(ctx context.Context, mod, ver string) (*storage.Version, error) { panic("skipped") diff --git a/pkg/download/catalog.go b/pkg/download/catalog.go new file mode 100644 index 00000000..8c3cbc5c --- /dev/null +++ b/pkg/download/catalog.go @@ -0,0 +1,54 @@ +package download + +import ( + "net/http" + "strconv" + + "github.com/gobuffalo/buffalo" + "github.com/gobuffalo/buffalo/render" + "github.com/gomods/athens/pkg/errors" + "github.com/gomods/athens/pkg/log" + "github.com/gomods/athens/pkg/paths" +) + +// PathCatalog URL. +const PathCatalog = "/catalog" +const defaultPageSize = 1000 + +type catalogRes struct { + ModsAndVersions []paths.AllPathParams `json:"modules"` + NextPageToken string `json:"next"` +} + +// CatalogHandler implements GET baseURL/catalog +func CatalogHandler(dp Protocol, lggr log.Entry, eng *render.Engine) buffalo.Handler { + const op errors.Op = "download.CatalogHandler" + + return func(c buffalo.Context) error { + token := c.Param("token") + pageSize, err := getLimitFromParam(c.Param("pagesize")) + if err != nil { + lggr.SystemErr(err) + return c.Render(http.StatusInternalServerError, nil) + } + + modulesAndVersions, newToken, err := dp.Catalog(c, token, pageSize) + + if err != nil { + if errors.Kind(err) != errors.KindNotImplemented { + lggr.SystemErr(errors.E(op, err)) + } + return c.Render(errors.Kind(err), eng.JSON(errors.KindText(err))) + } + + res := catalogRes{modulesAndVersions, newToken} + return c.Render(http.StatusOK, eng.JSON(res)) + } +} + +func getLimitFromParam(param string) (int, error) { + if param == "" { + return defaultPageSize, nil + } + return strconv.Atoi(param) +} diff --git a/pkg/download/handler.go b/pkg/download/handler.go index 7cbb96a7..40b636f9 100644 --- a/pkg/download/handler.go +++ b/pkg/download/handler.go @@ -50,4 +50,5 @@ func RegisterHandlers(app *buffalo.App, opts *HandlerOpts) { app.GET(PathVersionInfo, LogEntryHandler(VersionInfoHandler, opts)) app.GET(PathVersionModule, LogEntryHandler(VersionModuleHandler, opts)) app.GET(PathVersionZip, LogEntryHandler(VersionZipHandler, opts)) + app.GET(PathCatalog, LogEntryHandler(CatalogHandler, opts)) } diff --git a/pkg/download/protocol.go b/pkg/download/protocol.go index f4fe77f7..b4ca49ff 100644 --- a/pkg/download/protocol.go +++ b/pkg/download/protocol.go @@ -7,6 +7,7 @@ import ( "github.com/gomods/athens/pkg/errors" "github.com/gomods/athens/pkg/observ" + "github.com/gomods/athens/pkg/paths" "github.com/gomods/athens/pkg/stash" "github.com/gomods/athens/pkg/storage" ) @@ -28,6 +29,9 @@ type Protocol interface { // Zip implements GET /{module}/@v/{version}.zip Zip(ctx context.Context, mod, ver string) (io.ReadCloser, error) + + // Catalog implements GET /catalog + Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error) } // Wrapper helps extend the main protocol's functionality with addons. @@ -173,6 +177,19 @@ func (p *protocol) Zip(ctx context.Context, mod, ver string) (io.ReadCloser, err return zip, nil } +func (p *protocol) Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error) { + const op errors.Op = "protocol.Catalog" + ctx, span := observ.StartSpan(ctx, op.String()) + defer span.End() + modulesAndVersions, newToken, err := p.storage.Catalog(ctx, token, pageSize) + + if err != nil { + return nil, "", errors.E(op, err) + } + + return modulesAndVersions, newToken, err +} + // union concatenates two version lists and removes duplicates func union(list1, list2 []string) []string { if list1 == nil { diff --git a/pkg/errors/errors.go b/pkg/errors/errors.go index ce93baa0..8feb7e8b 100644 --- a/pkg/errors/errors.go +++ b/pkg/errors/errors.go @@ -11,11 +11,12 @@ import ( // Kind enums const ( - KindNotFound = http.StatusNotFound - KindBadRequest = http.StatusBadRequest - KindUnexpected = http.StatusInternalServerError - KindAlreadyExists = http.StatusConflict - KindRateLimit = http.StatusTooManyRequests + KindNotFound = http.StatusNotFound + KindBadRequest = http.StatusBadRequest + KindUnexpected = http.StatusInternalServerError + KindAlreadyExists = http.StatusConflict + KindRateLimit = http.StatusTooManyRequests + KindNotImplemented = http.StatusNotImplemented ) // Error is an Athens system error. diff --git a/pkg/paths/path.go b/pkg/paths/path.go index ba4214f8..80d1df2c 100644 --- a/pkg/paths/path.go +++ b/pkg/paths/path.go @@ -31,8 +31,8 @@ func GetVersion(c buffalo.Context) (string, error) { // AllPathParams holds the module and version in the path of a ?go-get=1 // request type AllPathParams struct { - Module string - Version string + Module string `json:"module"` + Version string `json:"version"` } // GetAllParams fetches the path patams from c and returns them diff --git a/pkg/storage/azureblob/cataloger.go b/pkg/storage/azureblob/cataloger.go new file mode 100644 index 00000000..0f75bb6e --- /dev/null +++ b/pkg/storage/azureblob/cataloger.go @@ -0,0 +1,15 @@ +package azureblob + +import ( + "context" + + "github.com/gomods/athens/pkg/errors" + "github.com/gomods/athens/pkg/paths" +) + +// Catalog implements the (./pkg/storage).Cataloger interface +// It returns a list of modules and versions contained in the storage +func (s *Storage) Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error) { + const op errors.Op = "azure.Catalog" + return nil, "", errors.E(op, errors.KindNotImplemented) +} diff --git a/pkg/storage/backend.go b/pkg/storage/backend.go index 9c789afb..285e92f0 100644 --- a/pkg/storage/backend.go +++ b/pkg/storage/backend.go @@ -7,4 +7,5 @@ type Backend interface { Checker Saver Deleter + Cataloger } diff --git a/pkg/storage/cataloger.go b/pkg/storage/cataloger.go new file mode 100644 index 00000000..b1bbd163 --- /dev/null +++ b/pkg/storage/cataloger.go @@ -0,0 +1,13 @@ +package storage + +import ( + "context" + + "github.com/gomods/athens/pkg/paths" +) + +// Cataloger is the interface that lists all the modules and version contained in the storage +type Cataloger interface { + // Catalog gets all the modules / versions. + Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error) +} diff --git a/pkg/storage/compliance/tests.go b/pkg/storage/compliance/tests.go index 80b04cef..0407b6f9 100644 --- a/pkg/storage/compliance/tests.go +++ b/pkg/storage/compliance/tests.go @@ -6,6 +6,7 @@ import ( "fmt" "io/ioutil" "math/rand" + "sort" "testing" "github.com/gomods/athens/pkg/errors" @@ -22,6 +23,9 @@ func RunTests(t *testing.T, b storage.Backend, clearBackend func() error) { testList(t, b) testDelete(t, b) testGet(t, b) + if isCatalogImplemented(b) { + testCatalog(t, b) + } } // testNotFound ensures that a storage Backend @@ -71,6 +75,11 @@ func testList(t *testing.T, b storage.Backend) { ) require.NoError(t, err, "Save for storage failed") } + defer func() { + for _, ver := range versions { + b.Delete(ctx, modname, ver) + } + }() retVersions, err := b.List(ctx, modname) require.NoError(t, err) require.Equal(t, versions, retVersions) @@ -84,6 +93,7 @@ func testGet(t *testing.T, b storage.Backend) { mock := getMockModule() zipBts, _ := ioutil.ReadAll(mock.Zip) b.Save(ctx, modname, ver, mock.Mod, bytes.NewReader(zipBts), mock.Info) + defer b.Delete(ctx, modname, ver) info, err := b.Info(ctx, modname, ver) require.NoError(t, err) @@ -120,6 +130,49 @@ func testDelete(t *testing.T, b storage.Backend) { require.Equal(t, false, exists) } +func testCatalog(t *testing.T, b storage.Backend) { + ctx := context.Background() + + mock := getMockModule() + zipBts, _ := ioutil.ReadAll(mock.Zip) + modname := "github.com/gomods/testCatalogModule" + for i := 0; i < 1005; i++ { + ver := fmt.Sprintf("v1.2.%04d", i) + b.Save(ctx, modname, ver, mock.Mod, bytes.NewReader(zipBts), mock.Info) + } + defer func() { + for i := 0; i < 1005; i++ { + ver := fmt.Sprintf("v1.2.%04d", i) + b.Delete(ctx, modname, ver) + } + }() + + allres, next, err := b.Catalog(ctx, "", 1001) + + require.NoError(t, err) + require.Equal(t, 1001, len(allres)) + + res, next, err := b.Catalog(ctx, next, 50) + allres = append(allres, res...) + require.NoError(t, err) + require.Equal(t, 4, len(res)) + require.Equal(t, "", next) + + sort.Slice(allres, func(i, j int) bool { + if allres[i].Module == allres[j].Module { + return allres[i].Version < allres[j].Version + } + return allres[i].Module < allres[j].Module + }) + require.Equal(t, modname, allres[0].Module) + require.Equal(t, "v1.2.0000", allres[0].Version) + require.Equal(t, "v1.2.1004", allres[1004].Version) + + for i := 1; i < len(allres); i++ { + require.NotEqual(t, allres[i].Version, allres[i-1].Version) + } +} + func getMockModule() *storage.Version { return &storage.Version{ Info: []byte("123"), @@ -127,3 +180,11 @@ func getMockModule() *storage.Version { Zip: ioutil.NopCloser(bytes.NewReader([]byte("789"))), } } + +func isCatalogImplemented(b storage.Backend) bool { + ctx := context.Background() + if _, _, err := b.Catalog(ctx, "", 1); errors.Kind(err) == errors.KindNotImplemented { + return false + } + return true +} diff --git a/pkg/storage/fs/cataloger.go b/pkg/storage/fs/cataloger.go new file mode 100644 index 00000000..33ebf940 --- /dev/null +++ b/pkg/storage/fs/cataloger.go @@ -0,0 +1,81 @@ +package fs + +import ( + "context" + "io" + "os" + "path/filepath" + "strings" + + "github.com/gomods/athens/pkg/errors" + "github.com/gomods/athens/pkg/observ" + "github.com/gomods/athens/pkg/paths" + "github.com/spf13/afero" +) + +const tokenSeparator = "|" + +// Catalog implements the (./pkg/storage).Cataloger interface +// It returns a list of modules and versions contained in the storage +func (s *storageImpl) Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error) { + const op errors.Op = "fs.Catalog" + ctx, span := observ.StartSpan(ctx, op.String()) + defer span.End() + + fromModule, fromVersion, err := modVerFromToken(token) + if err != nil { + return nil, "", errors.E(op, err, errors.KindBadRequest) + } + + res := make([]paths.AllPathParams, 0) + resToken := "" + count := pageSize + + err = afero.Walk(s.filesystem, s.rootDir, func(path string, info os.FileInfo, err error) error { + if strings.HasSuffix(info.Name(), ".info") { + verDir := filepath.Dir(path) + modVer, err := filepath.Rel(s.rootDir, verDir) + if err != nil { + return err + } + + m, version := filepath.Split(modVer) + module := filepath.Clean(m) + module = strings.Replace(module, string(os.PathSeparator), "/", -1) + + if fromModule != "" && module < fromModule { // it is ok to land on the same module + return nil + } + + if fromVersion != "" && version <= fromVersion { // we must skip same version + return nil + } + + res = append(res, paths.AllPathParams{module, version}) + count-- + if count == 0 { + resToken = tokenFromModVer(module, version) + return io.EOF + } + } + return nil + }) + + return res, resToken, nil +} + +func tokenFromModVer(module, version string) string { + return module + tokenSeparator + version +} + +func modVerFromToken(token string) (string, string, error) { + const op errors.Op = "fs.Catalog" + if token == "" { + return "", "", nil + } + values := strings.Split(token, tokenSeparator) + if len(values) < 2 { + return "", "", errors.E(op, "Invalid token") + } + return values[0], values[1], nil +} diff --git a/pkg/storage/gcp/all_test.go b/pkg/storage/gcp/all_test.go index 19128c0c..63b17eba 100644 --- a/pkg/storage/gcp/all_test.go +++ b/pkg/storage/gcp/all_test.go @@ -2,10 +2,13 @@ package gcp import ( "context" + "flag" "net/url" "testing" "time" + "github.com/gobuffalo/envy" + "github.com/gomods/athens/pkg/config" "github.com/stretchr/testify/suite" ) @@ -25,15 +28,58 @@ type GcpTests struct { bucket *bucketMock } +var realGcp = flag.Bool("realgcp", false, "tests against a real gcp instance") +var project = flag.String("gcpprj", "", "the gcp project to test against") +var bucket = flag.String("gcpbucket", "", "the gcp bucket to test against") + func (g *GcpTests) SetupSuite() { g.context = context.Background() - g.module = "gcp-test" + g.module = "github.com/foo/gcp-test" + time.Now().String() g.version = "v1.2.3" - g.url, _ = url.Parse("https://storage.googleapis.com/testbucket") - g.bucket = newBucketMock() - g.store = newWithBucket(g.bucket, g.url, time.Second) + + if !*realGcp { + setupMockStorage(g) + } else { + setupRealStorage(g) + } } func TestGcpStorage(t *testing.T) { suite.Run(t, new(GcpTests)) } + +func (g *GcpTests) BucketReadClosed() bool { + if g.bucket != nil { + return g.bucket.ReadClosed() + } + return true +} + +func (g *GcpTests) BucketWriteClosed() bool { + if g.bucket != nil { + return g.bucket.WriteClosed() + } + return true +} + +func setupMockStorage(g *GcpTests) { + g.url, _ = url.Parse("https://storage.googleapis.com/testbucket") + g.bucket = newBucketMock() + g.store = newWithBucket(g.bucket, g.url, time.Second) +} + +func setupRealStorage(g *GcpTests) { + _, err := envy.MustGet("GOOGLE_APPLICATION_CREDENTIALS") + if err != nil { + g.T().Skip() + } + if *project == "" || *bucket == "" { + g.T().Skip() + } + + g.store, err = New(context.Background(), &config.GCPConfig{ + ProjectID: *project, + Bucket: *bucket, + }, 300*time.Second) + g.Require().NoError(err) +} diff --git a/pkg/storage/gcp/bucket.go b/pkg/storage/gcp/bucket.go index 73e4a1b4..a1ae6a01 100644 --- a/pkg/storage/gcp/bucket.go +++ b/pkg/storage/gcp/bucket.go @@ -18,4 +18,6 @@ type Bucket interface { List(ctx context.Context, prefix string) ([]string, error) // Exists returns true if the file exists Exists(ctx context.Context, path string) (bool, error) + // Catalog returns a slice of paths starting from the page given by the token and max elementNum + Catalog(ctx context.Context, token string, pageSize int) ([]string, string, error) } diff --git a/pkg/storage/gcp/bucket_cloud.go b/pkg/storage/gcp/bucket_cloud.go index 85d8bda9..8a568d05 100644 --- a/pkg/storage/gcp/bucket_cloud.go +++ b/pkg/storage/gcp/bucket_cloud.go @@ -70,3 +70,22 @@ func (b *gcpBucket) Exists(ctx context.Context, path string) (bool, error) { return true, nil } + +func (b *gcpBucket) Catalog(ctx context.Context, token string, pageSize int) ([]string, string, error) { + const op errors.Op = "gcpBucket.Catalog" + + it := b.Objects(ctx, nil) + p := iterator.NewPager(it, pageSize, token) + + attrs := make([]*storage.ObjectAttrs, 0) + nextToken, err := p.NextPage(&attrs) + if err != nil { + return nil, "", errors.E(op, err) + } + + res := []string{} + for _, attr := range attrs { + res = append(res, attr.Name) + } + return res, nextToken, nil +} diff --git a/pkg/storage/gcp/bucket_mock.go b/pkg/storage/gcp/bucket_mock.go index 7145b0aa..7ca9c29c 100644 --- a/pkg/storage/gcp/bucket_mock.go +++ b/pkg/storage/gcp/bucket_mock.go @@ -5,6 +5,8 @@ import ( "context" "fmt" "io" + "sort" + "strconv" "strings" "sync" ) @@ -95,6 +97,36 @@ func (m *bucketMock) Exists(ctx context.Context, path string) (bool, error) { return found, nil } +func (m *bucketMock) Catalog(ctx context.Context, token string, elementNum int) ([]string, string, error) { + keys := make([]string, 0) + + m.lock.RLock() + defer m.lock.RUnlock() + + for k := range m.db { + keys = append(keys, k) + } + sort.Strings(keys) + from := 0 + if token != "" { + var err error + from, err = strconv.Atoi(token) + if err != nil { + return nil, "", err + } + } + if from > len(keys)-1 { + return make([]string, 0), "", nil + } + to := from + elementNum + resToken := strconv.Itoa(to) + if to > len(keys)-1 { + to = len(keys) - 1 + resToken = "" + } + return keys[from:to], resToken, nil +} + func (m *bucketMock) ReadClosed() bool { return (m.readLockCount == 0) } diff --git a/pkg/storage/gcp/cataloger.go b/pkg/storage/gcp/cataloger.go new file mode 100644 index 00000000..9e34d970 --- /dev/null +++ b/pkg/storage/gcp/cataloger.go @@ -0,0 +1,67 @@ +package gcp + +import ( + "context" + "fmt" + "strings" + + "github.com/gomods/athens/pkg/config" + "github.com/gomods/athens/pkg/paths" + + "github.com/gomods/athens/pkg/errors" + "github.com/gomods/athens/pkg/observ" +) + +// Catalog implements the (./pkg/storage).Catalog interface +// It returns a list of versions, if any, for a given module +func (s *Storage) Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error) { + const op errors.Op = "gcp.Catalog" + ctx, span := observ.StartSpan(ctx, op.String()) + defer span.End() + res := make([]paths.AllPathParams, 0) + var resToken string + count := pageSize + + for count > 0 { + var catalog []string + var err error + catalog, resToken, err = s.bucket.Catalog(ctx, token, 3*count) + if err != nil { + return nil, "", errors.E(op, err) + } + pathsAndVers := fetchModsAndVersions(catalog) + res = append(res, pathsAndVers...) + count -= len(pathsAndVers) + + if resToken == "" { // meaning we reached the end + break + } + } + return res, resToken, nil +} + +func fetchModsAndVersions(catalog []string) []paths.AllPathParams { + res := make([]paths.AllPathParams, 0) + for _, p := range catalog { + if !strings.HasSuffix(p, ".info") { + continue + } + p, err := parseGcpKey(p) + if err != nil { + continue + } + res = append(res, p) + } + return res +} + +func parseGcpKey(p string) (paths.AllPathParams, error) { + const op errors.Op = "gcp.parseS3Key" + // github.com/gomods/testCatalogModule/@v/v1.2.0976.info + m, v := config.ModuleVersionFromPath(p) + + if m == "" || v == "" { + return paths.AllPathParams{}, errors.E(op, fmt.Errorf("invalid object key format %s", p)) + } + return paths.AllPathParams{m, v}, nil +} diff --git a/pkg/storage/gcp/gcp_test.go b/pkg/storage/gcp/gcp_test.go index c9baaa7c..e9bd0591 100644 --- a/pkg/storage/gcp/gcp_test.go +++ b/pkg/storage/gcp/gcp_test.go @@ -3,6 +3,7 @@ package gcp import ( "bytes" "context" + "fmt" "io/ioutil" "testing" "time" @@ -50,9 +51,14 @@ func (g *GcpTests) TestSaveGetListExistsRoundTrip() { r.Equal(true, exists) }) + g.T().Run("Delete storage", func(t *testing.T) { + err := g.store.Delete(g.context, g.module, g.version) + r.NoError(err) + }) + g.T().Run("Resources closed", func(t *testing.T) { - r.Equal(true, g.bucket.ReadClosed()) - r.Equal(true, g.bucket.WriteClosed()) + r.Equal(true, g.BucketReadClosed()) + r.Equal(true, g.BucketWriteClosed()) }) } @@ -95,3 +101,32 @@ func (g *GcpTests) TestNotFounds() { r.Equal(0, len(list)) }) } + +func (g *GcpTests) TestCatalog() { + r := g.Require() + for i := 0; i < 50; i++ { + ver := fmt.Sprintf("v1.2.%04d", i) + err := g.store.Save(g.context, g.module, ver, mod, bytes.NewReader(zip), info) + r.NoError(err) + } + defer func() { + for i := 0; i < 50; i++ { + ver := fmt.Sprintf("v1.2.%04d", i) + err := g.store.Delete(g.context, g.module, ver) + r.NoError(err) + } + }() + + allres, nextToken, err := g.store.Catalog(g.context, "", 2) + r.NoError(err) + r.Equal(len(allres), 2) + r.NotEqual("", nextToken) + r.Equal(allres[0].Module, g.module) + + res, nextToken, err := g.store.Catalog(g.context, nextToken, 50) + allres = append(allres, res...) + r.NoError(err) + r.Equal(len(allres), 50) + r.Equal(len(res), 48) + r.Equal("", nextToken) +} diff --git a/pkg/storage/minio/cataloger.go b/pkg/storage/minio/cataloger.go new file mode 100644 index 00000000..372ad012 --- /dev/null +++ b/pkg/storage/minio/cataloger.go @@ -0,0 +1,16 @@ +package minio + +import ( + "context" + + "github.com/gomods/athens/pkg/paths" + + "github.com/gomods/athens/pkg/errors" +) + +// Catalog implements the (./pkg/storage).Cataloger interface +// It returns a list of modules and versions contained in the storage +func (s *storageImpl) Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error) { + const op errors.Op = "minio.Catalog" + return nil, "", errors.E(op, errors.KindNotImplemented) +} diff --git a/pkg/storage/mongo/cataloger.go b/pkg/storage/mongo/cataloger.go new file mode 100644 index 00000000..6a158c87 --- /dev/null +++ b/pkg/storage/mongo/cataloger.go @@ -0,0 +1,16 @@ +package mongo + +import ( + "context" + + "github.com/gomods/athens/pkg/paths" + + "github.com/gomods/athens/pkg/errors" +) + +// Catalog implements the (./pkg/storage).Cataloger interface +// It returns a list of modules and versions contained in the storage +func (s *ModuleStore) Catalog(ctx context.Context, token string, elements int) ([]paths.AllPathParams, string, error) { + const op errors.Op = "mongo.Catalog" + return nil, "", errors.E(op, errors.KindNotImplemented) +} diff --git a/pkg/storage/s3/cataloger.go b/pkg/storage/s3/cataloger.go new file mode 100644 index 00000000..1c31d638 --- /dev/null +++ b/pkg/storage/s3/cataloger.go @@ -0,0 +1,85 @@ +package s3 + +import ( + "context" + "fmt" + "strings" + + "github.com/gomods/athens/pkg/config" + + "github.com/gomods/athens/pkg/paths" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/gomods/athens/pkg/errors" + "github.com/gomods/athens/pkg/observ" +) + +// Catalog implements the (./pkg/storage).Cataloger interface +// It returns a list of modules and versions contained in the storage +func (s *Storage) Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error) { + const op errors.Op = "s3.Catalog" + ctx, span := observ.StartSpan(ctx, op.String()) + defer span.End() + queryToken := token + res := make([]paths.AllPathParams, 0) + count := pageSize + for count > 0 { + lsParams := &s3.ListObjectsInput{ + Bucket: aws.String(s.bucket), + Marker: &queryToken, + } + + loo, err := s.s3API.ListObjectsWithContext(ctx, lsParams) + if err != nil { + return nil, "", errors.E(op, err) + } + + m, lastKey := fetchModsAndVersions(loo.Contents, count) + + res = append(res, m...) + count -= len(m) + queryToken = lastKey + + if !*loo.IsTruncated { // not truncated, there is no point in asking more + if count > 0 { // it means we reached the end, no subsequent requests are necessary + queryToken = "" + } + break + } + } + return res, queryToken, nil +} + +func fetchModsAndVersions(objects []*s3.Object, elementsNum int) ([]paths.AllPathParams, string) { + res := make([]paths.AllPathParams, 0) + lastKey := "" + for _, o := range objects { + if !strings.HasSuffix(*o.Key, ".info") { + continue + } + p, err := parseS3Key(o) + if err != nil { + continue + } + + res = append(res, p) + lastKey = *o.Key + + elementsNum-- + if elementsNum == 0 { + break + } + } + return res, lastKey +} + +func parseS3Key(o *s3.Object) (paths.AllPathParams, error) { + const op errors.Op = "s3.parseS3Key" + m, v := config.ModuleVersionFromPath(*o.Key) + + if m == "" || v == "" { + return paths.AllPathParams{}, errors.E(op, fmt.Errorf("invalid object key format %s", *o.Key)) + } + return paths.AllPathParams{m, v}, nil +} diff --git a/scripts/test_e2e.sh b/scripts/test_e2e.sh index f742e508..538b446c 100755 --- a/scripts/test_e2e.sh +++ b/scripts/test_e2e.sh @@ -59,3 +59,11 @@ clearGoModCache # Verify that the test works against the proxy export GOPROXY=http://localhost:3000 $GO_BINARY_PATH run . + +CATALOG_RES=$(curl localhost:3000/catalog) +CATALOG_EXPECTED='{"modules":[{"module":"github.com/athens-artifacts/no-tags","version":"v0.0.0-20180803171426-1a540c5d67ab"}],"next":""}' + +if [[ "$CATALOG_RES" != "$CATALOG_EXPECTED" ]]; then + echo ERROR: catalog endpoint failed + exit 1 # terminate and indicate error +fi