athens/pkg/storage/s3/cataloger.go

package s3

import (
	"context"
	"fmt"
	"strings"

	"github.com/aws/aws-sdk-go-v2/aws"
	"github.com/aws/aws-sdk-go-v2/service/s3"
	"github.com/aws/aws-sdk-go-v2/service/s3/types"
	"github.com/gomods/athens/pkg/config"
	"github.com/gomods/athens/pkg/errors"
	"github.com/gomods/athens/pkg/observ"
	"github.com/gomods/athens/pkg/paths"
)

// Catalog implements the (./pkg/storage).Cataloger interface.
// It returns a list of modules and versions contained in the storage.
func (s *Storage) Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error) {
	const op errors.Op = "s3.Catalog"
	ctx, span := observ.StartSpan(ctx, op.String())
	defer span.End()
	queryToken := token
	res := make([]paths.AllPathParams, 0)
	count := pageSize
	for count > 0 {
		lsParams := &s3.ListObjectsInput{
			Bucket: aws.String(s.bucket),
			Marker: &queryToken,
		}

		loo, err := s.s3API.ListObjects(ctx, lsParams)
		if err != nil {
			return nil, "", errors.E(op, err)
		}

		m, lastKey := fetchModsAndVersions(loo.Contents, count)

		res = append(res, m...)
		count -= len(m)
		queryToken = lastKey

		if !*loo.IsTruncated { // not truncated, there is no point in asking more
			if count > 0 { // it means we reached the end, no subsequent requests are necessary
				queryToken = ""
			}
			break
		}
	}
	return res, queryToken, nil
}

func fetchModsAndVersions(objects []types.Object, elementsNum int) ([]paths.AllPathParams, string) {
	res := make([]paths.AllPathParams, 0)
	lastKey := ""
	for _, o := range objects {
		if !strings.HasSuffix(*o.Key, ".info") {
			continue
		}
		p, err := parseS3Key(o)
		if err != nil {
			continue
		}

		res = append(res, p)
		lastKey = *o.Key

		elementsNum--
		if elementsNum == 0 {
			break
		}
	}
	return res, lastKey
}

func parseS3Key(o types.Object) (paths.AllPathParams, error) {
	const op errors.Op = "s3.parseS3Key"
	m, v := config.ModuleVersionFromPath(*o.Key)

	if m == "" || v == "" {
		return paths.AllPathParams{}, errors.E(op, fmt.Errorf("invalid object key format %s", *o.Key))
	}
	return paths.AllPathParams{Module: m, Version: v}, nil
}