primal/athens
1
0
Fork 0
mirror of https://github.com/gomods/athens synced 2026-02-03 07:30:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: GCP checksum (#2052)

Browse Source
This commit is contained in:
south-mer
2025-08-06 15:24:54 +09:00
committed by GitHub
parent 59253bd64d
commit 11d674c8fb
21 changed files with 74 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/download/list_merge_test.go
+1 -1
View File
@@ -127,7 +127,7 @@ func TestListMerge(t *testing.T) {
t.Fatal(err)
}
for _, v := range tc.strVersions {
s.Save(ctx, testModName, v, bts, io.NopCloser(bytes.NewReader(bts)), bts)
s.Save(ctx, testModName, v, bts, io.NopCloser(bytes.NewReader(bts)), nil, bts)
}
defer clearStorage(s, testModName, tc.strVersions)
dp := New(&Opts{s, nil, &listerMock{versions: tc.goVersions, err: tc.goErr}, nil, Strict})
pkg/download/protocol_test.go
+3 -3
View File
@@ -165,7 +165,7 @@ func TestListMode(t *testing.T) {
networkMode: tc.networkmode,
}
for _, tag := range tc.storageTags {
err := strg.Save(ctx, tc.path, tag, []byte("mod"), bytes.NewReader([]byte("zip")), []byte("info"))
err := strg.Save(ctx, tc.path, tag, []byte("mod"), bytes.NewReader([]byte("zip")), nil, []byte("info"))
require.NoError(t, err)
}
t.Run(tc.name, func(t *testing.T) {
@@ -429,7 +429,7 @@ func TestDownloadProtocolWhenFetchFails(t *testing.T) {
}
fakeMod := testMod{"github.com/athens-artifacts/samplelib", "v1.0.0"}
bts := []byte(fakeMod.mod + "@" + fakeMod.ver)
err = s.Save(context.Background(), fakeMod.mod, fakeMod.ver, bts, io.NopCloser(bytes.NewReader(bts)), bts)
err = s.Save(context.Background(), fakeMod.mod, fakeMod.ver, bts, io.NopCloser(bytes.NewReader(bts)), nil, bts)
if err != nil {
t.Fatal(err)
}
@@ -472,7 +472,7 @@ type mockStasher struct {
}
func (ms *mockStasher) Stash(ctx context.Context, mod string, ver string) (string, error) {
err := ms.s.Save(ctx, mod, ver, []byte("mod"), strings.NewReader("zip"), []byte("info"))
err := ms.s.Save(ctx, mod, ver, []byte("mod"), strings.NewReader("zip"), nil, []byte("info"))
ms.ch <- true // signal async stashing is done
return ver, err
}
pkg/module/go_get_fetcher.go
+23
View File
@@ -3,8 +3,10 @@ package module
import (
"bytes"
"context"
"crypto/md5" //nolint:gosec
"encoding/json"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
@@ -96,6 +98,26 @@ func (g *goGetFetcher) Fetch(ctx context.Context, mod, ver string) (*storage.Ver
}
storageVer.Mod = gomod
zipMD5, err := func() ([]byte, error) {
// Perform in a separate function to ensure file is closed
zipForChecksum, err := g.fs.Open(m.Zip)
if err != nil {
return nil, errors.E(op, err)
}
defer zipForChecksum.Close()
//nolint:gosec
hash := md5.New()
if _, err := io.Copy(hash, zipForChecksum); err != nil {
return nil, errors.E(op, err)
}
return hash.Sum(nil), nil
}()
if err != nil {
return nil, err
}
zip, err := g.fs.Open(m.Zip)
if err != nil {
return nil, errors.E(op, err)
@@ -105,6 +127,7 @@ func (g *goGetFetcher) Fetch(ctx context.Context, mod, ver string) (*storage.Ver
// if we close, then the caller will panic, and the alternative to make this work is
// that we read into memory and return an io.ReadCloser that reads out of memory
storageVer.Zip = &zipReadCloser{zip, g.fs, goPathRoot}
storageVer.ZipMD5 = zipMD5
return &storageVer, nil
}
pkg/stash/stasher.go
+1 -1
View File
@@ -71,7 +71,7 @@ func (s *stasher) Stash(ctx context.Context, mod, ver string) (string, error) {
return v.Semver, nil
}
}
err = s.storage.Save(ctx, mod, v.Semver, v.Mod, v.Zip, v.Info)
err = s.storage.Save(ctx, mod, v.Semver, v.Mod, v.Zip, v.ZipMD5, v.Info)
if err != nil {
return "", errors.E(op, err)
}
pkg/stash/stasher_test.go
+1 -1
View File
@@ -84,7 +84,7 @@ type mockStorage struct {
existsResponse bool
}
func (ms *mockStorage) Save(ctx context.Context, module, version string, mod []byte, zip io.Reader, info []byte) error {
func (ms *mockStorage) Save(ctx context.Context, module, version string, mod []byte, zip io.Reader, zipMD5 []byte, info []byte) error {
ms.saveCalled = true
ms.givenVersion = version
return nil
pkg/stash/with_azureblob_test.go
+1
View File
@@ -73,6 +73,7 @@ func (ms *mockAzureBlobStasher) Stash(ctx context.Context, mod, ver string) (str
ver,
[]byte("mod file"),
strings.NewReader("zip file"),
nil,
[]byte("info file"),
)
if err != nil {
pkg/stash/with_gcs_test.go
+2 -1
View File
@@ -119,7 +119,7 @@ func TestWithGCSPartialFailure(t *testing.T) {
}
s := gs(ms)
// We simulate a failure by manually passing an io.Reader that will fail.
err = ms.strg.Save(ctx, "stashmod", "v1.0.0", []byte(ms.content), fr, []byte(ms.content))
err = ms.strg.Save(ctx, "stashmod", "v1.0.0", []byte(ms.content), fr, nil, []byte(ms.content))
if err == nil {
// We *want* to fail.
t.Fatal(err)
@@ -172,6 +172,7 @@ func (ms *mockGCPStasher) Stash(ctx context.Context, mod, ver string) (string, e
ver,
[]byte(ms.content),
strings.NewReader(ms.content),
nil,
[]byte(ms.content),
)
return "", err
pkg/stash/with_redis_test.go
+1
View File
@@ -210,6 +210,7 @@ func (ms *mockRedisStasher) Stash(ctx context.Context, mod, ver string) (string,
ver,
[]byte("mod file"),
strings.NewReader("zip file"),
nil,
[]byte("info file"),
)
if err != nil {
pkg/storage/azureblob/saver.go
+1 -1
View File
@@ -11,7 +11,7 @@ import (
)
// Save implements the (./pkg/storage).Saver interface.
func (s *Storage) Save(ctx context.Context, module, version string, mod []byte, zip io.Reader, info []byte) error {
func (s *Storage) Save(ctx context.Context, module, version string, mod []byte, zip io.Reader, zipMD5, info []byte) error {
const op errors.Op = "azureblob.Save"
ctx, span := observ.StartSpan(ctx, op.String())
defer span.End()
pkg/storage/compliance/benchmarks.go
+4 -2
View File
@@ -33,6 +33,7 @@ func benchList(b *testing.B, s storage.Backend, reset func() error) {
version,
mock.Mod,
mock.Zip,
mock.ZipMD5,
mock.Info,
)
require.NoError(b, err, "save for storage failed")
@@ -65,6 +66,7 @@ func benchSave(b *testing.B, s storage.Backend, reset func() error) {
version,
mock.Mod,
bytes.NewReader(zipBts),
mock.ZipMD5,
mock.Info,
)
require.NoError(b, err)
@@ -87,7 +89,7 @@ func benchDelete(b *testing.B, s storage.Backend, reset func() error) {
b.Run("delete", func(b *testing.B) {
for i := 0; i < b.N; i++ {
name := fmt.Sprintf("del-%s-%d", module, i)
err := s.Save(ctx, name, version, mock.Mod, bytes.NewReader(zipBts), mock.Info)
err := s.Save(ctx, name, version, mock.Mod, bytes.NewReader(zipBts), mock.ZipMD5, mock.Info)
require.NoError(b, err, "saving %s for storage failed", name)
err = s.Delete(ctx, name, version)
require.NoError(b, err, "delete failed: %s", name)
@@ -104,7 +106,7 @@ func benchExists(b *testing.B, s storage.Backend, reset func() error) {
mock := getMockModule()
ctx := context.Background()
err := s.Save(ctx, module, version, mock.Mod, mock.Zip, mock.Info)
err := s.Save(ctx, module, version, mock.Mod, mock.Zip, mock.ZipMD5, mock.Info)
require.NoError(b, err)
b.Run("exists", func(b *testing.B) {
pkg/storage/compliance/tests.go
+11 -7
View File
@@ -3,6 +3,7 @@ package compliance
import (
"bytes"
"context"
"crypto/md5"
"fmt"
"io"
"math/rand"
@@ -76,6 +77,7 @@ func testListSuffix(t *testing.T, b storage.Backend) {
version,
mock.Mod,
mock.Zip,
mock.ZipMD5,
mock.Info,
)
require.NoError(t, err, "Save for storage failed")
@@ -114,6 +116,7 @@ func testList(t *testing.T, b storage.Backend) {
version,
mock.Mod,
mock.Zip,
mock.ZipMD5,
mock.Info,
)
require.NoError(t, err, "Save for storage failed")
@@ -135,7 +138,7 @@ func testGet(t *testing.T, b storage.Backend) {
ver := "v1.2.3"
mock := getMockModule()
zipBts, _ := io.ReadAll(mock.Zip)
b.Save(ctx, modname, ver, mock.Mod, bytes.NewReader(zipBts), mock.Info)
b.Save(ctx, modname, ver, mock.Mod, bytes.NewReader(zipBts), mock.ZipMD5, mock.Info)
defer b.Delete(ctx, modname, ver)
info, err := b.Info(ctx, modname, ver)
@@ -160,7 +163,7 @@ func testExists(t *testing.T, b storage.Backend) {
ver := "v1.2.3"
mock := getMockModule()
zipBts, _ := io.ReadAll(mock.Zip)
b.Save(ctx, modname, ver, mock.Mod, bytes.NewReader(zipBts), mock.Info)
b.Save(ctx, modname, ver, mock.Mod, bytes.NewReader(zipBts), mock.ZipMD5, mock.Info)
defer b.Delete(ctx, modname, ver)
checker := storage.WithChecker(b)
exists, err := checker.Exists(ctx, modname, ver)
@@ -174,7 +177,7 @@ func testShouldNotExist(t *testing.T, b storage.Backend) {
ver := "v1.2.3-pre.1"
mock := getMockModule()
zipBts, _ := io.ReadAll(mock.Zip)
err := b.Save(ctx, mod, ver, mock.Mod, bytes.NewReader(zipBts), mock.Info)
err := b.Save(ctx, mod, ver, mock.Mod, bytes.NewReader(zipBts), mock.ZipMD5, mock.Info)
require.NoError(t, err, "should successfully safe a mock module")
defer b.Delete(ctx, mod, ver)
@@ -196,7 +199,7 @@ func testDelete(t *testing.T, b storage.Backend) {
version := fmt.Sprintf("%s%d", "delete", rand.Int())
mock := getMockModule()
err := b.Save(ctx, modname, version, mock.Mod, mock.Zip, mock.Info)
err := b.Save(ctx, modname, version, mock.Mod, mock.Zip, mock.ZipMD5, mock.Info)
require.NoError(t, err)
err = b.Delete(ctx, modname, version)
@@ -209,8 +212,9 @@ func testDelete(t *testing.T, b storage.Backend) {
func getMockModule() *storage.Version {
return &storage.Version{
Info: []byte("123"),
Mod: []byte("456"),
Zip: io.NopCloser(bytes.NewReader([]byte("789"))),
Info: []byte("123"),
Mod: []byte("456"),
Zip: io.NopCloser(bytes.NewReader([]byte("789"))),
ZipMD5: md5.New().Sum([]byte("789")),
}
}
pkg/storage/external/client.go
Vendored
+1 -1
View File
@@ -81,7 +81,7 @@ func (s *service) Zip(ctx context.Context, mod, ver string) (storage.SizeReadClo
return storage.NewSizer(body, size), nil
}
func (s *service) Save(ctx context.Context, mod, ver string, modFile []byte, zip io.Reader, info []byte) error {
func (s *service) Save(ctx context.Context, mod, ver string, modFile []byte, zip io.Reader, zipMD5, info []byte) error {
const op errors.Op = "external.Save"
var err error
mod, err = module.EscapePath(mod)
pkg/storage/external/server.go
Vendored
+1 -1
View File
@@ -109,7 +109,7 @@ func NewServer(strg storage.Backend) http.Handler {
return
}
defer func() { _ = modZ.Close() }()
err = strg.Save(r.Context(), params.Module, params.Version, modFile, modZ, info)
err = strg.Save(r.Context(), params.Module, params.Version, modFile, modZ, nil, info)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
pkg/storage/fs/saver.go
+1 -1
View File
@@ -11,7 +11,7 @@ import (
"github.com/spf13/afero"
)
func (s *storageImpl) Save(ctx context.Context, module, version string, mod []byte, zip io.Reader, info []byte) error {
func (s *storageImpl) Save(ctx context.Context, module, version string, mod []byte, zip io.Reader, zipMD5, info []byte) error {
const op errors.Op = "fs.Save"
_, span := observ.StartSpan(ctx, op.String())
defer span.End()
pkg/storage/gcp/saver.go
+11 -7
View File
@@ -20,11 +20,11 @@ import (
//
// Uploaded files are publicly accessible in the storage bucket as per
// an ACL rule.
func (s *Storage) Save(ctx context.Context, module, version string, mod []byte, zip io.Reader, info []byte) error {
func (s *Storage) Save(ctx context.Context, module, version string, mod []byte, zip io.Reader, zipMD5, info []byte) error {
const op errors.Op = "gcp.save"
ctx, span := observ.StartSpan(ctx, op.String())
defer span.End()
err := s.save(ctx, module, version, mod, zip, info)
err := s.save(ctx, module, version, mod, zip, zipMD5, info)
if err != nil {
return errors.E(op, err)
}
@@ -37,26 +37,26 @@ func (s *Storage) SetStaleThreshold(threshold time.Duration) {
s.staleThreshold = threshold
}
func (s *Storage) save(ctx context.Context, module, version string, mod []byte, zip io.Reader, info []byte) error {
func (s *Storage) save(ctx context.Context, module, version string, mod []byte, zip io.Reader, zipMD5, info []byte) error {
const op errors.Op = "gcp.save"
ctx, span := observ.StartSpan(ctx, op.String())
defer span.End()
gomodPath := config.PackageVersionedName(module, version, "mod")
err := s.upload(ctx, gomodPath, bytes.NewReader(mod), false)
err := s.upload(ctx, gomodPath, bytes.NewReader(mod), nil, false)
// KindAlreadyExists means the file is uploaded (somewhere else) successfully.
if err != nil && !errors.Is(err, errors.KindAlreadyExists) {
return errors.E(op, err)
}
zipPath := config.PackageVersionedName(module, version, "zip")
err = s.upload(ctx, zipPath, zip, true)
err = s.upload(ctx, zipPath, zip, zipMD5, true)
if err != nil && !errors.Is(err, errors.KindAlreadyExists) {
return errors.E(op, err)
}
infoPath := config.PackageVersionedName(module, version, "info")
err = s.upload(ctx, infoPath, bytes.NewReader(info), false)
err = s.upload(ctx, infoPath, bytes.NewReader(info), nil, false)
if err != nil && !errors.Is(err, errors.KindAlreadyExists) {
return errors.E(op, err)
}
@@ -64,7 +64,7 @@ func (s *Storage) save(ctx context.Context, module, version string, mod []byte,
return nil
}
func (s *Storage) upload(ctx context.Context, path string, stream io.Reader, checkBefore bool) error {
func (s *Storage) upload(ctx context.Context, path string, stream io.Reader, md5 []byte, checkBefore bool) error {
const op errors.Op = "gcp.upload"
ctx, span := observ.StartSpan(ctx, op.String())
defer span.End()
@@ -90,6 +90,10 @@ func (s *Storage) upload(ctx context.Context, path string, stream io.Reader, che
DoesNotExist: true,
}).NewWriter(cancelCtx)
if len(md5) > 0 {
wc.MD5 = md5
}
// NOTE: content type is auto detected on GCP side and ACL defaults to public
// Once we support private storage buckets this may need refactoring
// unless there is a way to set the default perms in the project.
pkg/storage/minio/saver.go
+1 -1
View File
@@ -13,7 +13,7 @@ import (
minio "github.com/minio/minio-go/v6"
)
func (s *storageImpl) Save(ctx context.Context, module, vsn string, mod []byte, zip io.Reader, info []byte) error {
func (s *storageImpl) Save(ctx context.Context, module, vsn string, mod []byte, zip io.Reader, zipMD5, info []byte) error {
const op errors.Op = "storage.minio.Save"
_, span := observ.StartSpan(ctx, op.String())
defer span.End()
pkg/storage/mongo/mongo_test.go
+2 -2
View File
@@ -59,7 +59,7 @@ func TestQueryModuleVersionExists(t *testing.T) {
backend := getStorage(t)
zipBts, _ := io.ReadAll(mock.Zip)
backend.Save(ctx, modname, ver, mock.Mod, bytes.NewReader(zipBts), mock.Info)
backend.Save(ctx, modname, ver, mock.Mod, bytes.NewReader(zipBts), mock.ZipMD5, mock.Info)
defer backend.Delete(ctx, modname, ver)
info, err := query(ctx, backend, modname, ver)
@@ -80,7 +80,7 @@ func TestQueryKindNotFoundErrorCases(t *testing.T) {
backend := getStorage(t)
zipBts, _ := io.ReadAll(mock.Zip)
backend.Save(ctx, modname, ver, mock.Mod, bytes.NewReader(zipBts), mock.Info)
backend.Save(ctx, modname, ver, mock.Mod, bytes.NewReader(zipBts), nil, mock.Info)
defer backend.Delete(ctx, modname, ver)
testCases := []struct {
pkg/storage/mongo/saver.go
+1 -1
View File
@@ -13,7 +13,7 @@ import (
)
// Save stores a module in mongo storage.
func (s *ModuleStore) Save(ctx context.Context, module, version string, mod []byte, zip io.Reader, info []byte) error {
func (s *ModuleStore) Save(ctx context.Context, module, version string, mod []byte, zip io.Reader, zipMD5, info []byte) error {
const op errors.Op = "mongo.Save"
ctx, span := observ.StartSpan(ctx, op.String())
defer span.End()
pkg/storage/s3/saver.go
+1 -1
View File
@@ -13,7 +13,7 @@ import (
)
// Save implements the (github.com/gomods/athens/pkg/storage).Saver interface.
func (s *Storage) Save(ctx context.Context, module, version string, mod []byte, zip io.Reader, info []byte) error {
func (s *Storage) Save(ctx context.Context, module, version string, mod []byte, zip io.Reader, zipMD5, info []byte) error {
const op errors.Op = "s3.Save"
ctx, span := observ.StartSpan(ctx, op.String())
defer span.End()
pkg/storage/saver.go
+5 -1
View File
@@ -7,5 +7,9 @@ import (
// Saver saves module metadata and its source to underlying storage.
type Saver interface {
Save(ctx context.Context, module, version string, mod []byte, zip io.Reader, info []byte) error
// Save saves the module metadata and its source to the storage.
//
// The caller MAY call zipMD5 with a nil value if the checksum is not available.
// The storage implementation MAY use the zipMD5 to verify the integrity of the zip file.
Save(ctx context.Context, module, version string, mod []byte, zip io.Reader, zipMD5, info []byte) error
}
pkg/storage/version.go
+1
View File
@@ -6,6 +6,7 @@ import "io"
type Version struct {
Mod []byte
Zip io.ReadCloser
ZipMD5 []byte
Info []byte
Semver string
}