feat: add pagination to s3 lister (#2037)

By default `ListObjectsV2()` returns the first 1000 objects matching the
list parameters. Normally this is fine, as it supports up to 333
versions (1000 / 3 files in proxy-triplet). For modules with more
versions, this is insufficient and must be upgraded to paginate.

pkg/storage/s3/lister.go

@@ -2,6 +2,7 @@ package s3
 
 import (
 	"context"
+	"slices"
 	"strings"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
@@ -24,13 +25,17 @@ func (s *Storage) List(ctx context.Context, module string) ([]string, error) {
 		Bucket: aws.String(s.bucket),
 		Prefix: aws.String(modulePrefix),
 	}
+	paginator := s3.NewListObjectsV2Paginator(s.s3API, lsParams)
 
-	loo, err := s.s3API.ListObjectsV2(ctx, lsParams)
+	var versions []string
-	if err != nil {
+	for paginator.HasMorePages() {
-		return nil, errors.E(op, err, errors.M(module))
+		loo, err := paginator.NextPage(ctx)
+		if err != nil {
+			return nil, errors.E(op, err, errors.M(module))
+		}
+		versions = slices.Concat(versions, extractVersions(loo.Contents))
 	}
-
+	return versions, nil
-	return extractVersions(loo.Contents), nil
 }
 
 func extractVersions(objects []types.Object) []string {