primal/athens
1
0
Fork 0
mirror of https://github.com/gomods/athens synced 2026-02-03 12:10:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: add pagination to s3 lister (#2037)

Browse Source 
By default `ListObjectsV2()` returns the first 1000 objects matching the
list parameters. Normally this is fine, as it supports up to 333
versions (1000 / 3 files in proxy-triplet). For modules with more
versions, this is insufficient and must be upgraded to paginate.
This commit is contained in:
Connor McCarthy
2025-04-01 01:01:35 -04:00
committed by GitHub
parent 18041d7364
commit ebb5ac698b
1 changed files with 10 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/storage/s3/lister.go
+10 -5
View File
@@ -2,6 +2,7 @@ package s3
import (
"context"
"slices"
"strings"
"github.com/aws/aws-sdk-go-v2/aws"
@@ -24,13 +25,17 @@ func (s *Storage) List(ctx context.Context, module string) ([]string, error) {
Bucket: aws.String(s.bucket),
Prefix: aws.String(modulePrefix),
}
paginator := s3.NewListObjectsV2Paginator(s.s3API, lsParams)
loo, err := s.s3API.ListObjectsV2(ctx, lsParams)
if err != nil {
return nil, errors.E(op, err, errors.M(module))
var versions []string
for paginator.HasMorePages() {
loo, err := paginator.NextPage(ctx)
if err != nil {
return nil, errors.E(op, err, errors.M(module))
}
versions = slices.Concat(versions, extractVersions(loo.Contents))
}
return extractVersions(loo.Contents), nil
return versions, nil
}
func extractVersions(objects []types.Object) []string {