From ebb5ac698b77333989656f7fc20616105c8d4c59 Mon Sep 17 00:00:00 2001
From: Connor McCarthy <64061225+connor15mcc@users.noreply.github.com>
Date: Tue, 1 Apr 2025 01:01:35 -0400
Subject: [PATCH] feat: add pagination to s3 lister (#2037)

By default `ListObjectsV2()` returns the first 1000 objects matching the
list parameters. Normally this is fine, as it supports up to 333
versions (1000 / 3 files in proxy-triplet). For modules with more
versions, this is insufficient and must be upgraded to paginate.
---
 pkg/storage/s3/lister.go | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/pkg/storage/s3/lister.go b/pkg/storage/s3/lister.go
index 09ca6093..ab5c71c8 100644
--- a/pkg/storage/s3/lister.go
+++ b/pkg/storage/s3/lister.go
@@ -2,6 +2,7 @@ package s3
 
 import (
 	"context"
+	"slices"
 	"strings"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
@@ -24,13 +25,17 @@ func (s *Storage) List(ctx context.Context, module string) ([]string, error) {
 		Bucket: aws.String(s.bucket),
 		Prefix: aws.String(modulePrefix),
 	}
+	paginator := s3.NewListObjectsV2Paginator(s.s3API, lsParams)
 
-	loo, err := s.s3API.ListObjectsV2(ctx, lsParams)
-	if err != nil {
-		return nil, errors.E(op, err, errors.M(module))
+	var versions []string
+	for paginator.HasMorePages() {
+		loo, err := paginator.NextPage(ctx)
+		if err != nil {
+			return nil, errors.E(op, err, errors.M(module))
+		}
+		versions = slices.Concat(versions, extractVersions(loo.Contents))
 	}
-
-	return extractVersions(loo.Contents), nil
+	return versions, nil
 }
 
 func extractVersions(objects []types.Object) []string {