diff --git a/go.mod b/go.mod index e5a1105e..cfce1896 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.12 require ( cloud.google.com/go v0.26.0 contrib.go.opencensus.io/exporter/stackdriver v0.6.0 - github.com/Azure/azure-storage-blob-go v0.0.0-20181022225951-5152f14ace1c + github.com/Azure/azure-storage-blob-go v0.7.0 github.com/BurntSushi/toml v0.3.1 github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895 // indirect github.com/DataDog/opencensus-go-exporter-datadog v0.0.0-20180917103902-e6c7f767dc57 diff --git a/pkg/storage/azureblob/cataloger.go b/pkg/storage/azureblob/cataloger.go new file mode 100644 index 00000000..d7906936 --- /dev/null +++ b/pkg/storage/azureblob/cataloger.go @@ -0,0 +1,61 @@ +package azureblob + +import ( + "context" + "fmt" + "strings" + + "github.com/Azure/azure-storage-blob-go/azblob" + "github.com/gomods/athens/pkg/config" + "github.com/gomods/athens/pkg/errors" + "github.com/gomods/athens/pkg/observ" + "github.com/gomods/athens/pkg/paths" +) + +// Catalog implements the (./pkg/storage).Catalog interface +// It returns a list of versions, if any, for a given module +func (s *Storage) Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error) { + const op errors.Op = "azblob.Catalog" + ctx, span := observ.StartSpan(ctx, op.String()) + defer span.End() + + res := make([]paths.AllPathParams, 0) + + // one module@version consists of 3 pieces - info, mod, zip + objCount := 3 * pageSize + + marker := azblob.Marker{ + Val: &token, + } + blobs, err := s.client.containerURL.ListBlobsFlatSegment(ctx, marker, azblob.ListBlobsSegmentOptions{ + MaxResults: int32(objCount), + }) + if err != nil { + return nil, "", errors.E(op, err) + } + + nextToken := *blobs.NextMarker.Val + + for _, blob := range blobs.Segment.BlobItems { + if strings.HasSuffix(blob.Name, ".info") { + p, err := parsModVer(blob.Name) + if err != nil { + continue + } + res = append(res, p) + } + } + + return res, nextToken, nil +} + +func parsModVer(p string) (paths.AllPathParams, error) { + const op errors.Op = "azureblob.parseS3Key" + // github.com/gomods/testCatalogModule/@v/v1.2.0976.info + m, v := config.ModuleVersionFromPath(p) + + if m == "" || v == "" { + return paths.AllPathParams{}, errors.E(op, fmt.Errorf("invalid object key format %s", p)) + } + return paths.AllPathParams{Module: m, Version: v}, nil +}