primal/athens
1
0
Fork 0
mirror of https://github.com/gomods/athens synced 2026-02-03 11:00:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

Handle duplicate keys returned by S3 ListObject (#1793)

Browse Source 
S3 API sometimes returns duplicate keys, causing the number of file for a specific module greater than 3. This PR makes Athens handle that situation gracefully and log a warning message instead of treating the module as not found.

logging string content

Co-authored-by: Manu Gupta <manugupt1@gmail.com>
Co-authored-by: Ashish Ranjan <ashishranjan2912@gmail.com>
This commit is contained in:
Zhongpeng Lin
2023-01-29 21:01:09 -08:00
committed by GitHub
parent b3bc773e29
commit 6aaf9e5b17
1 changed files with 12 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/storage/s3/checker.go
+12 -12
View File
@@ -8,6 +8,7 @@ import (
"github.com/aws/aws-sdk-go/service/s3"
"github.com/gomods/athens/pkg/config"
"github.com/gomods/athens/pkg/errors"
"github.com/gomods/athens/pkg/log"
"github.com/gomods/athens/pkg/observ"
)
@@ -22,25 +23,24 @@ func (s *Storage) Exists(ctx context.Context, module, version string) (bool, err
Bucket: aws.String(s.bucket),
Prefix: aws.String(fmt.Sprintf("%s/@v", module)),
}
var count int
found := make(map[string]struct{}, 3)
err := s.s3API.ListObjectsPagesWithContext(ctx, lsParams, func(loo *s3.ListObjectsOutput, lastPage bool) bool {
for _, o := range loo.Contents {
// sane assumption: no duplicate keys.
switch *o.Key {
case config.PackageVersionedName(module, version, "info"):
count++
case config.PackageVersionedName(module, version, "mod"):
count++
case config.PackageVersionedName(module, version, "zip"):
count++
if _, exists := found[*o.Key]; exists {
log.EntryFromContext(ctx).Warnf("duplicate key in prefix %q: %q", *lsParams.Prefix, *o.Key)
continue
}
if *o.Key == config.PackageVersionedName(module, version, "info") ||
*o.Key == config.PackageVersionedName(module, version, "mod") ||
*o.Key == config.PackageVersionedName(module, version, "zip") {
found[*o.Key] = struct{}{}
}
}
return count != 3
return len(found) < 3
})
if err != nil {
return false, errors.E(op, err, errors.M(module), errors.V(version))
}
return count == 3, nil
return len(found) == 3, nil
}