primal/athens
1
0
Fork 0
mirror of https://github.com/gomods/athens synced 2026-02-03 08:40:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

Upgrade to AWS SDK v2 (#1938) (#1950)

Browse Source 
Upgrades the AWS SDK to v2. AWS S3 bucket urls will now error if they are not prefixed with a schema (example: https://).
This commit is contained in:
Vanes Angelo
2024-04-29 11:38:42 +07:00
committed by GitHub
parent c35b50f2f3
commit 900cb4f7c1
37 changed files with 274 additions and 177 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/config/azureblob.go
+3 -3
View File
@@ -2,7 +2,7 @@ package config
// AzureBlobConfig specifies the properties required to use Azure as the storage backend.
type AzureBlobConfig struct {
AccountName string `validate:"required" envconfig:"ATHENS_AZURE_ACCOUNT_NAME"`
AccountKey string `validate:"required" envconfig:"ATHENS_AZURE_ACCOUNT_KEY"`
ContainerName string `validate:"required" envconfig:"ATHENS_AZURE_CONTAINER_NAME"`
AccountName string `envconfig:"ATHENS_AZURE_ACCOUNT_NAME" validate:"required"`
AccountKey string `envconfig:"ATHENS_AZURE_ACCOUNT_KEY" validate:"required"`
ContainerName string `envconfig:"ATHENS_AZURE_CONTAINER_NAME" validate:"required"`
}
pkg/config/config.go
+10 -10
View File
@@ -21,22 +21,22 @@ const defaultConfigFile = "athens.toml"
// Config provides configuration values for all components.
type Config struct {
TimeoutConf
GoEnv string `validate:"required" envconfig:"GO_ENV"`
GoBinary string `validate:"required" envconfig:"GO_BINARY_PATH"`
GoEnv string `envconfig:"GO_ENV" validate:"required"`
GoBinary string `envconfig:"GO_BINARY_PATH" validate:"required"`
GoBinaryEnvVars EnvList `envconfig:"ATHENS_GO_BINARY_ENV_VARS"`
GoGetWorkers int `validate:"required" envconfig:"ATHENS_GOGET_WORKERS"`
GoGetWorkers int `envconfig:"ATHENS_GOGET_WORKERS" validate:"required"`
GoGetDir string `envconfig:"ATHENS_GOGET_DIR"`
ProtocolWorkers int `validate:"required" envconfig:"ATHENS_PROTOCOL_WORKERS"`
LogLevel string `validate:"required" envconfig:"ATHENS_LOG_LEVEL"`
LogFormat string `validate:"oneof='' 'json' 'plain'" envconfig:"ATHENS_LOG_FORMAT"`
CloudRuntime string `validate:"required_without=LogFormat" envconfig:"ATHENS_CLOUD_RUNTIME"`
ProtocolWorkers int `envconfig:"ATHENS_PROTOCOL_WORKERS" validate:"required"`
LogLevel string `envconfig:"ATHENS_LOG_LEVEL" validate:"required"`
LogFormat string `envconfig:"ATHENS_LOG_FORMAT" validate:"oneof='' 'json' 'plain'"`
CloudRuntime string `envconfig:"ATHENS_CLOUD_RUNTIME" validate:"required_without=LogFormat"`
EnablePprof bool `envconfig:"ATHENS_ENABLE_PPROF"`
PprofPort string `envconfig:"ATHENS_PPROF_PORT"`
FilterFile string `envconfig:"ATHENS_FILTER_FILE"`
TraceExporterURL string `envconfig:"ATHENS_TRACE_EXPORTER_URL"`
TraceExporter string `envconfig:"ATHENS_TRACE_EXPORTER"`
StatsExporter string `envconfig:"ATHENS_STATS_EXPORTER"`
StorageType string `validate:"required" envconfig:"ATHENS_STORAGE_TYPE"`
StorageType string `envconfig:"ATHENS_STORAGE_TYPE" validate:"required"`
GlobalEndpoint string `envconfig:"ATHENS_GLOBAL_ENDPOINT"` // This feature is not yet implemented
Port string `envconfig:"ATHENS_PORT"`
UnixSocket string `envconfig:"ATHENS_UNIX_SOCKET"`
@@ -55,11 +55,11 @@ type Config struct {
NoSumPatterns []string `envconfig:"ATHENS_GONOSUM_PATTERNS"`
DownloadMode mode.Mode `envconfig:"ATHENS_DOWNLOAD_MODE"`
DownloadURL string `envconfig:"ATHENS_DOWNLOAD_URL"`
NetworkMode string `validate:"oneof=strict offline fallback" envconfig:"ATHENS_NETWORK_MODE"`
NetworkMode string `envconfig:"ATHENS_NETWORK_MODE" validate:"oneof=strict offline fallback"`
SingleFlightType string `envconfig:"ATHENS_SINGLE_FLIGHT_TYPE"`
RobotsFile string `envconfig:"ATHENS_ROBOTS_FILE"`
IndexType string `envconfig:"ATHENS_INDEX_TYPE"`
ShutdownTimeout int `validate:"min=0" envconfig:"ATHENS_SHUTDOWN_TIMEOUT"`
ShutdownTimeout int `envconfig:"ATHENS_SHUTDOWN_TIMEOUT" validate:"min=0"`
SingleFlight *SingleFlight
Storage *Storage
Index *Index
pkg/config/config_test.go
+4 -4
View File
@@ -16,7 +16,7 @@ import (
func testConfigFile(t *testing.T) (testConfigFile string) {
testConfigFile = filepath.Join("..", "..", "config.dev.toml")
if err := os.Chmod(testConfigFile, 0700); err != nil {
if err := os.Chmod(testConfigFile, 0o700); err != nil {
t.Fatalf("%s\n", err)
}
return testConfigFile
@@ -108,6 +108,7 @@ func TestEnvOverrides(t *testing.T) {
if err != nil {
t.Fatalf("Env override failed: %v", err)
}
compareConfigs(conf, expConf, t, Storage{}, SingleFlight{})
}
@@ -305,7 +306,6 @@ func TestParseExampleConfig(t *testing.T) {
}
func getEnvMap(config *Config) map[string]string {
envVars := map[string]string{
"GO_ENV": config.GoEnv,
"GO_BINARY_PATH": config.GoBinary,
@@ -414,7 +414,7 @@ func Test_checkFilePerms(t *testing.T) {
}
incorrectPerms := []os.FileMode{0o777, 0o610, 0o660}
var incorrectFiles = make([]string, len(incorrectPerms))
incorrectFiles := make([]string, len(incorrectPerms))
for i := range incorrectPerms {
f, err := tempFile(incorrectPerms[i])
@@ -426,7 +426,7 @@ func Test_checkFilePerms(t *testing.T) {
}
correctPerms := []os.FileMode{0o600, 0o400, 0o644}
var correctFiles = make([]string, len(correctPerms))
correctFiles := make([]string, len(correctPerms))
for i := range correctPerms {
f, err := tempFile(correctPerms[i])
pkg/config/disk.go
+1 -1
View File
@@ -2,5 +2,5 @@ package config
// DiskConfig specifies the properties required to use Disk as the storage backend.
type DiskConfig struct {
RootPath string `validate:"required" envconfig:"ATHENS_DISK_STORAGE_ROOT"`
RootPath string `envconfig:"ATHENS_DISK_STORAGE_ROOT" validate:"required"`
}
pkg/config/external.go
+1 -1
View File
@@ -2,5 +2,5 @@ package config
// External specifies configuration for an external http storage.
type External struct {
URL string `validate:"required" envconfig:"ATHENS_EXTERNAL_STORAGE_URL"`
URL string `envconfig:"ATHENS_EXTERNAL_STORAGE_URL" validate:"required"`
}
pkg/config/gcp.go
+1 -1
View File
@@ -3,6 +3,6 @@ package config
// GCPConfig specifies the properties required to use GCP as the storage backend.
type GCPConfig struct {
ProjectID string `envconfig:"GOOGLE_CLOUD_PROJECT"`
Bucket string `validate:"required" envconfig:"ATHENS_STORAGE_GCP_BUCKET"`
Bucket string `envconfig:"ATHENS_STORAGE_GCP_BUCKET" validate:"required"`
JSONKey string `envconfig:"ATHENS_STORAGE_GCP_JSON_KEY"`
}
pkg/config/minio.go
+4 -4
View File
@@ -3,10 +3,10 @@ package config
// MinioConfig specifies the properties required to use Minio or DigitalOcean Spaces
// as the storage backend.
type MinioConfig struct {
Endpoint string `validate:"required" envconfig:"ATHENS_MINIO_ENDPOINT"`
Key string `validate:"required" envconfig:"ATHENS_MINIO_ACCESS_KEY_ID"`
Secret string `validate:"required" envconfig:"ATHENS_MINIO_SECRET_ACCESS_KEY"`
Bucket string `validate:"required" envconfig:"ATHENS_MINIO_BUCKET_NAME"`
Endpoint string `envconfig:"ATHENS_MINIO_ENDPOINT" validate:"required"`
Key string `envconfig:"ATHENS_MINIO_ACCESS_KEY_ID" validate:"required"`
Secret string `envconfig:"ATHENS_MINIO_SECRET_ACCESS_KEY" validate:"required"`
Bucket string `envconfig:"ATHENS_MINIO_BUCKET_NAME" validate:"required"`
Region string `envconfig:"ATHENS_MINIO_REGION"`
EnableSSL bool `envconfig:"ATHENS_MINIO_USE_SSL"`
}
pkg/config/mongo.go
+3 -3
View File
@@ -2,9 +2,9 @@ package config
// MongoConfig specifies the properties required to use MongoDB as the storage backend.
type MongoConfig struct {
URL string `validate:"required" envconfig:"ATHENS_MONGO_STORAGE_URL"`
DefaultDBName string `envconfig:"ATHENS_MONGO_DEFAULT_DATABASE" default:"athens"`
DefaultCollectionName string `envconfig:"ATHENS_MONGO_DEFAULT_COLLECTION" default:"modules"`
URL string `envconfig:"ATHENS_MONGO_STORAGE_URL" validate:"required"`
DefaultDBName string `default:"athens" envconfig:"ATHENS_MONGO_DEFAULT_DATABASE"`
DefaultCollectionName string `default:"modules" envconfig:"ATHENS_MONGO_DEFAULT_COLLECTION"`
CertPath string `envconfig:"ATHENS_MONGO_CERT_PATH"`
InsecureConn bool `envconfig:"ATHENS_MONGO_INSECURE"`
}
pkg/config/mysql.go
+7 -7
View File
@@ -2,11 +2,11 @@ package config
// MySQL config.
type MySQL struct {
Protocol string `validate:"required" envconfig:"ATHENS_INDEX_MYSQL_PROTOCOL"`
Host string `validate:"required" envconfig:"ATHENS_INDEX_MYSQL_HOST"`
Port int `validate:"" envconfig:"ATHENS_INDEX_MYSQL_PORT"`
User string `validate:"required" envconfig:"ATHENS_INDEX_MYSQL_USER"`
Password string `validate:"" envconfig:"ATHENS_INDEX_MYSQL_PASSWORD"`
Database string `validate:"required" envconfig:"ATHENS_INDEX_MYSQL_DATABASE"`
Params map[string]string `validate:"required" envconfig:"ATHENS_INDEX_MYSQL_PARAMS"`
Protocol string `envconfig:"ATHENS_INDEX_MYSQL_PROTOCOL" validate:"required"`
Host string `envconfig:"ATHENS_INDEX_MYSQL_HOST" validate:"required"`
Port int `envconfig:"ATHENS_INDEX_MYSQL_PORT" validate:""`
User string `envconfig:"ATHENS_INDEX_MYSQL_USER" validate:"required"`
Password string `envconfig:"ATHENS_INDEX_MYSQL_PASSWORD" validate:""`
Database string `envconfig:"ATHENS_INDEX_MYSQL_DATABASE" validate:"required"`
Params map[string]string `envconfig:"ATHENS_INDEX_MYSQL_PARAMS" validate:"required"`
}
pkg/config/postgres.go
+6 -6
View File
@@ -2,10 +2,10 @@ package config
// Postgres config.
type Postgres struct {
Host string `validate:"required" envconfig:"ATHENS_INDEX_POSTGRES_HOST"`
Port int `validate:"required" envconfig:"ATHENS_INDEX_POSTGRES_PORT"`
User string `validate:"required" envconfig:"ATHENS_INDEX_POSTGRES_USER"`
Password string `validate:"" envconfig:"ATHENS_INDEX_POSTGRES_PASSWORD"`
Database string `validate:"required" envconfig:"ATHENS_INDEX_POSTGRES_DATABASE"`
Params map[string]string `validate:"required" envconfig:"ATHENS_INDEX_POSTGRES_PARAMS"`
Host string `envconfig:"ATHENS_INDEX_POSTGRES_HOST" validate:"required"`
Port int `envconfig:"ATHENS_INDEX_POSTGRES_PORT" validate:"required"`
User string `envconfig:"ATHENS_INDEX_POSTGRES_USER" validate:"required"`
Password string `envconfig:"ATHENS_INDEX_POSTGRES_PASSWORD" validate:""`
Database string `envconfig:"ATHENS_INDEX_POSTGRES_DATABASE" validate:"required"`
Params map[string]string `envconfig:"ATHENS_INDEX_POSTGRES_PARAMS" validate:"required"`
}
pkg/config/s3.go
+2 -2
View File
@@ -2,11 +2,11 @@ package config
// S3Config specifies the properties required to use S3 as the storage backend.
type S3Config struct {
Region string `validate:"required" envconfig:"AWS_REGION"`
Region string `envconfig:"AWS_REGION" validate:"required"`
Key string `envconfig:"AWS_ACCESS_KEY_ID"`
Secret string `envconfig:"AWS_SECRET_ACCESS_KEY"`
Token string `envconfig:"AWS_SESSION_TOKEN"`
Bucket string `validate:"required" envconfig:"ATHENS_S3_BUCKET_NAME"`
Bucket string `envconfig:"ATHENS_S3_BUCKET_NAME" validate:"required"`
UseDefaultConfiguration bool `envconfig:"AWS_USE_DEFAULT_CONFIGURATION"`
ForcePathStyle bool `envconfig:"AWS_FORCE_PATH_STYLE"`
CredentialsEndpoint string `envconfig:"AWS_CREDENTIALS_ENDPOINT"`
pkg/download/list_merge_test.go
+4 -2
View File
@@ -13,8 +13,10 @@ import (
"github.com/stretchr/testify/require"
)
const testOp athenserr.Op = "vcsLister.List"
const testModName = "happy tags"
const (
testOp athenserr.Op = "vcsLister.List"
testModName = "happy tags"
)
type listMergeTest struct {
name string
pkg/download/mode/mode_test.go
+2 -1
View File
@@ -84,7 +84,8 @@ var testCases = []struct {
{
Pattern: "github.com/gomods/*",
Mode: AsyncRedirect,
DownloadURL: "gomods.io"},
DownloadURL: "gomods.io",
},
},
},
input: "github.com/gomods/athens",
pkg/download/protocol_test.go
+2 -4
View File
@@ -28,9 +28,7 @@ import (
"golang.org/x/sync/errgroup"
)
var (
testConfigPath = filepath.Join("..", "..", "config.dev.toml")
)
var testConfigPath = filepath.Join("..", "..", "config.dev.toml")
func getDP(t *testing.T) Protocol {
t.Helper()
@@ -235,7 +233,7 @@ var latestTests = []latestTest{
path: "github.com/athens-artifacts/happy-path",
info: &storage.RevInfo{
Version: "v0.0.3",
Time: time.Date(2018, 8, 3, 17, 16, 00, 0, time.UTC),
Time: time.Date(2018, 8, 3, 17, 16, 0o0, 0, time.UTC),
},
},
}
pkg/middleware/middleware_test.go
+1 -1
View File
@@ -23,7 +23,7 @@ const (
func testConfigFile(t *testing.T) (testConfigFile string) {
testConfigFile = filepath.Join("..", "..", "config.dev.toml")
if err := os.Chmod(testConfigFile, 0700); err != nil {
if err := os.Chmod(testConfigFile, 0o700); err != nil {
t.Fatalf("%s\n", err)
}
return testConfigFile
pkg/module/filter_test.go
+4 -6
View File
@@ -10,7 +10,7 @@ import (
func testConfigFile(t *testing.T) (testConfigFile string) {
testConfigFile = filepath.Join("..", "..", "config.dev.toml")
if err := os.Chmod(testConfigFile, 0700); err != nil {
if err := os.Chmod(testConfigFile, 0o700); err != nil {
t.Fatalf("%s\n", err)
}
return testConfigFile
@@ -40,7 +40,6 @@ func (t *FilterTests) Test_NewFilter() {
mf, err = NewFilter(filter)
r.Equal(filter, mf.filePath)
r.NoError(err)
}
func (t *FilterTests) Test_IgnoreSimple() {
@@ -198,7 +197,6 @@ func (t *FilterTests) Test_versionFilterRobust() {
r.Equal(Exclude, f.Rule("github.com/a/b", "a"))
r.Equal(Exclude, f.Rule("github.com/c/d", "fg"))
}
func (t *FilterTests) Test_initFromConfig() {
@@ -207,20 +205,20 @@ func (t *FilterTests) Test_initFromConfig() {
defer os.Remove(filterFile)
goodInput := []byte("+ github.com/a/b\n\n# some comment\n- github.com/c/d\n\nD github.com/x")
os.WriteFile(filterFile, goodInput, 0644)
os.WriteFile(filterFile, goodInput, 0o644)
f, err := initFromConfig(filterFile)
r.NotNil(f)
r.NoError(err)
badInput := []byte("+ github.com/a/b\n\n# some comment\n\n- github.com/c/d\n\nD github.com/x\nsome_random_line")
os.WriteFile(filterFile, badInput, 0644)
os.WriteFile(filterFile, badInput, 0o644)
f, err = initFromConfig(filterFile)
r.Nil(f)
r.Error(err)
versionInput := []byte("+ github.com/a/b\n\n# some comment\n\n- github.com/c/d v1,v2.3.4,v3.2.*\n\nD github.com/x\n")
os.WriteFile(filterFile, versionInput, 0644)
os.WriteFile(filterFile, versionInput, 0o644)
f, err = initFromConfig(filterFile)
r.NotNil(f)
r.NoError(err)
pkg/storage/minio/minio.go
+11 -1
View File
@@ -2,6 +2,7 @@ package minio
import (
"fmt"
"strings"
"time"
"github.com/gomods/athens/pkg/config"
@@ -24,7 +25,7 @@ func (s *storageImpl) versionLocation(module, version string) string {
// that implements storage.Backend.
func NewStorage(conf *config.MinioConfig, timeout time.Duration) (storage.Backend, error) {
const op errors.Op = "minio.NewStorage"
endpoint := conf.Endpoint
endpoint := TrimHTTP(conf.Endpoint)
accessKeyID := conf.Key
secretAccessKey := conf.Secret
bucketName := conf.Bucket
@@ -53,3 +54,12 @@ func NewStorage(conf *config.MinioConfig, timeout time.Duration) (storage.Backen
}
return &storageImpl{minioClient, minioCore, bucketName}, nil
}
// TrimHTTP trims "http://" or "https://" prefix from input string.
// Minio doesn't need to specify protocol in the URL so it should be trimmed.
// Related issue: https://github.com/gomods/athens/issues/1938#issuecomment-2067590653
func TrimHTTP(s string) string {
s = strings.TrimPrefix(s, "http://")
s = strings.TrimPrefix(s, "https://")
return s
}
pkg/storage/minio/minio_test.go
+3 -3
View File
@@ -15,7 +15,7 @@ func TestBackend(t *testing.T) {
// TestNewStorageExists tests the logic around MakeBucket and BucketExists
func TestNewStorageExists(t *testing.T) {
url := os.Getenv("ATHENS_MINIO_ENDPOINT")
url := TrimHTTP(os.Getenv("ATHENS_MINIO_ENDPOINT"))
if url == "" {
t.SkipNow()
}
@@ -51,7 +51,7 @@ func TestNewStorageExists(t *testing.T) {
// To ensure both paths are tested, there is a strict path error using the
// "_" and a non strict error using less than 3 characters
func TestNewStorageError(t *testing.T) {
url := os.Getenv("ATHENS_MINIO_ENDPOINT")
url := TrimHTTP(os.Getenv("ATHENS_MINIO_ENDPOINT"))
if url == "" {
t.SkipNow()
}
@@ -92,7 +92,7 @@ func (s *storageImpl) clear() error {
}
func getStorage(t testing.TB) *storageImpl {
url := os.Getenv("ATHENS_MINIO_ENDPOINT")
url := TrimHTTP(os.Getenv("ATHENS_MINIO_ENDPOINT"))
if url == "" {
t.SkipNow()
}
pkg/storage/mongo/mongo_test.go
+5 -3
View File
@@ -3,13 +3,14 @@ package mongo
import (
"bytes"
"context"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"io"
"os"
"testing"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"github.com/gomods/athens/pkg/config"
"github.com/gomods/athens/pkg/errors"
"github.com/gomods/athens/pkg/storage"
@@ -145,6 +146,7 @@ func TestQueryKindUnexpectedErrorCases(t *testing.T) {
require.Equal(t, errors.KindUnexpected, errors.Kind(err))
}
}
func TestNewStorageWithDefaultOverrides(t *testing.T) {
url := os.Getenv("ATHENS_MONGO_STORAGE_URL")
pkg/storage/s3/cataloger.go
+6 -5
View File
@@ -5,8 +5,9 @@ import (
"fmt"
"strings"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/gomods/athens/pkg/config"
"github.com/gomods/athens/pkg/errors"
"github.com/gomods/athens/pkg/observ"
@@ -28,7 +29,7 @@ func (s *Storage) Catalog(ctx context.Context, token string, pageSize int) ([]pa
Marker: &queryToken,
}
loo, err := s.s3API.ListObjectsWithContext(ctx, lsParams)
loo, err := s.s3API.ListObjects(ctx, lsParams)
if err != nil {
return nil, "", errors.E(op, err)
}
@@ -49,7 +50,7 @@ func (s *Storage) Catalog(ctx context.Context, token string, pageSize int) ([]pa
return res, queryToken, nil
}
func fetchModsAndVersions(objects []*s3.Object, elementsNum int) ([]paths.AllPathParams, string) {
func fetchModsAndVersions(objects []types.Object, elementsNum int) ([]paths.AllPathParams, string) {
res := make([]paths.AllPathParams, 0)
lastKey := ""
for _, o := range objects {
@@ -72,7 +73,7 @@ func fetchModsAndVersions(objects []*s3.Object, elementsNum int) ([]paths.AllPat
return res, lastKey
}
func parseS3Key(o *s3.Object) (paths.AllPathParams, error) {
func parseS3Key(o types.Object) (paths.AllPathParams, error) {
const op errors.Op = "s3.parseS3Key"
m, v := config.ModuleVersionFromPath(*o.Key)
pkg/storage/s3/checker.go
+6 -6
View File
@@ -4,9 +4,9 @@ import (
"context"
"sync"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/smithy-go"
"github.com/gomods/athens/pkg/config"
"github.com/gomods/athens/pkg/errors"
"github.com/gomods/athens/pkg/observ"
@@ -28,7 +28,7 @@ func (s *Storage) Exists(ctx context.Context, module, version string) (bool, err
wg.Add(1)
go func(file string) {
defer wg.Done()
_, err := s.s3API.HeadObjectWithContext(
_, err := s.s3API.HeadObject(
cancelingCtx,
&s3.HeadObjectInput{
Bucket: aws.String(s.bucket),
@@ -44,8 +44,8 @@ func (s *Storage) Exists(ctx context.Context, module, version string) (bool, err
if err == nil {
continue
}
var aerr awserr.Error
if errors.AsErr(err, &aerr) && aerr.Code() == "NotFound" {
var aerr smithy.APIError
if errors.AsErr(err, &aerr) && aerr.ErrorCode() == "NotFound" {
err = nil
exists = false
}
pkg/storage/s3/deleter.go
+4 -3
View File
@@ -3,8 +3,8 @@ package s3
import (
"context"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/gomods/athens/pkg/errors"
"github.com/gomods/athens/pkg/observ"
modupl "github.com/gomods/athens/pkg/storage/module"
@@ -32,12 +32,13 @@ func (s *Storage) remove(ctx context.Context, path string) error {
const op errors.Op = "s3.Delete"
ctx, span := observ.StartSpan(ctx, op.String())
defer span.End()
delParams := &s3.DeleteObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(path),
}
if _, err := s.s3API.DeleteObjectWithContext(ctx, delParams); err != nil {
if _, err := s.s3API.DeleteObject(ctx, delParams); err != nil {
return errors.E(op, err)
}
pkg/storage/s3/getter.go
+12 -12
View File
@@ -5,9 +5,9 @@ import (
"fmt"
"io"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/gomods/athens/pkg/config"
"github.com/gomods/athens/pkg/errors"
"github.com/gomods/athens/pkg/observ"
@@ -22,8 +22,8 @@ func (s *Storage) Info(ctx context.Context, module, version string) ([]byte, err
infoReader, err := s.open(ctx, config.PackageVersionedName(module, version, "info"))
if err != nil {
var aerr awserr.Error
if errors.AsErr(err, &aerr) && aerr.Code() == s3.ErrCodeNoSuchKey {
var nsk *types.NoSuchKey
if errors.AsErr(err, &nsk) {
return nil, errors.E(op, errors.M(module), errors.V(version), errors.KindNotFound)
}
return nil, errors.E(op, err, errors.M(module), errors.V(version))
@@ -45,8 +45,8 @@ func (s *Storage) GoMod(ctx context.Context, module, version string) ([]byte, er
modReader, err := s.open(ctx, config.PackageVersionedName(module, version, "mod"))
if err != nil {
var aerr awserr.Error
if errors.AsErr(err, &aerr) && aerr.Code() == s3.ErrCodeNoSuchKey {
var nsk *types.NoSuchKey
if errors.AsErr(err, &nsk) {
return nil, errors.E(op, errors.M(module), errors.V(version), errors.KindNotFound)
}
return nil, errors.E(op, err, errors.M(module), errors.V(version))
@@ -69,8 +69,8 @@ func (s *Storage) Zip(ctx context.Context, module, version string) (storage.Size
zipReader, err := s.open(ctx, config.PackageVersionedName(module, version, "zip"))
if err != nil {
var aerr awserr.Error
if errors.AsErr(err, &aerr) && aerr.Code() == s3.ErrCodeNoSuchKey {
var nsk *types.NoSuchKey
if errors.AsErr(err, &nsk) {
return nil, errors.E(op, errors.M(module), errors.V(version), errors.KindNotFound)
}
return nil, errors.E(op, err, errors.M(module), errors.V(version))
@@ -88,10 +88,10 @@ func (s *Storage) open(ctx context.Context, path string) (storage.SizeReadCloser
Key: aws.String(path),
}
goo, err := s.s3API.GetObjectWithContext(ctx, getParams)
goo, err := s.s3API.GetObject(ctx, getParams)
if err != nil {
var aerr awserr.Error
if errors.AsErr(err, &aerr) && aerr.Code() == s3.ErrCodeNoSuchKey {
var nsk *types.NoSuchKey
if errors.AsErr(err, &nsk) {
return nil, errors.E(op, errors.KindNotFound)
}
return nil, errors.E(op, err)
pkg/storage/s3/lister.go
+5 -4
View File
@@ -4,8 +4,9 @@ import (
"context"
"strings"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/gomods/athens/pkg/errors"
"github.com/gomods/athens/pkg/observ"
)
@@ -23,7 +24,7 @@ func (s *Storage) List(ctx context.Context, module string) ([]string, error) {
Prefix: aws.String(modulePrefix),
}
loo, err := s.s3API.ListObjectsWithContext(ctx, lsParams)
loo, err := s.s3API.ListObjects(ctx, lsParams)
if err != nil {
return nil, errors.E(op, err, errors.M(module))
}
@@ -31,7 +32,7 @@ func (s *Storage) List(ctx context.Context, module string) ([]string, error) {
return extractVersions(loo.Contents), nil
}
func extractVersions(objects []*s3.Object) []string {
func extractVersions(objects []types.Object) []string {
var versions []string
for _, o := range objects {
pkg/storage/s3/s3.go
+60 -46
View File
@@ -1,16 +1,16 @@
package s3
import (
"context"
"fmt"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/credentials/endpointcreds"
"github.com/aws/aws-sdk-go/aws/defaults"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3/s3iface"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"github.com/aws/aws-sdk-go/service/s3/s3manager/s3manageriface"
"github.com/aws/aws-sdk-go-v2/aws"
awscfg "github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/credentials/endpointcreds"
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/gomods/athens/pkg/config"
"github.com/gomods/athens/pkg/errors"
)
@@ -26,8 +26,8 @@ import (
// For information how to get your keyId and access key turn to official aws docs: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/setting-up.html.
type Storage struct {
bucket string
uploader s3manageriface.UploaderAPI
s3API s3iface.S3API
uploader *manager.Uploader
s3API *s3.Client
timeout time.Duration
}
@@ -35,50 +35,44 @@ type Storage struct {
func New(s3Conf *config.S3Config, timeout time.Duration, options ...func(*aws.Config)) (*Storage, error) {
const op errors.Op = "s3.New"
awsConfig := defaults.Config()
// Remove anonymous credentials from the default config so that
// session.NewSession can auto-resolve credentials from role, profile, env etc.
awsConfig.Credentials = nil
awsConfig.Region = aws.String(s3Conf.Region)
for _, o := range options {
o(awsConfig)
}
if !s3Conf.UseDefaultConfiguration {
credProviders := defaults.CredProviders(awsConfig, defaults.Handlers())
endpointcreds := []credentials.Provider{
endpointcreds.NewProviderClient(*awsConfig, defaults.Handlers(), endpointFrom(s3Conf.CredentialsEndpoint, s3Conf.AwsContainerCredentialsRelativeURI)),
&credentials.StaticProvider{
Value: credentials.Value{
AccessKeyID: s3Conf.Key,
SecretAccessKey: s3Conf.Secret,
SessionToken: s3Conf.Token,
},
},
}
credProviders = append(endpointcreds, credProviders...)
awsConfig.Credentials = credentials.NewChainCredentials(credProviders)
}
awsConfig.S3ForcePathStyle = aws.Bool(s3Conf.ForcePathStyle)
awsConfig.CredentialsChainVerboseErrors = aws.Bool(true)
if s3Conf.Endpoint != "" {
awsConfig.Endpoint = aws.String(s3Conf.Endpoint)
}
// Create a session with creds.
sess, err := session.NewSession(awsConfig)
awsConfig, err := awscfg.LoadDefaultConfig(context.TODO(), awscfg.WithRegion(s3Conf.Region))
if err != nil {
return nil, errors.E(op, err)
}
uploader := s3manager.NewUploader(sess)
// Remove anonymous credentials from the default config so that
// session.NewSession can auto-resolve credentials from role, profile, env etc.
awsConfig.Credentials = nil
for _, o := range options {
o(&awsConfig)
}
if !s3Conf.UseDefaultConfiguration {
// credProviders := defaults.CredProviders(awsConfig, defaults.Handlers())
endpointCreds := []aws.CredentialsProvider{
endpointcreds.New(endpointFrom(s3Conf.CredentialsEndpoint, s3Conf.AwsContainerCredentialsRelativeURI)),
credentials.NewStaticCredentialsProvider(s3Conf.Key, s3Conf.Secret, s3Conf.Token),
}
// credProviders = append(endpointCreds, credProviders...)
awsConfig.Credentials = newChainCredentials(endpointCreds...)
}
// Create a session with creds.
sess := s3.NewFromConfig(awsConfig, func(o *s3.Options) {
o.UsePathStyle = s3Conf.ForcePathStyle
if s3Conf.Endpoint != "" {
o.BaseEndpoint = aws.String(s3Conf.Endpoint)
}
})
uploader := manager.NewUploader(sess)
return &Storage{
bucket: s3Conf.Bucket,
uploader: uploader,
s3API: uploader.S3,
s3API: sess,
timeout: timeout,
}, nil
}
@@ -86,3 +80,23 @@ func New(s3Conf *config.S3Config, timeout time.Duration, options ...func(*aws.Co
func endpointFrom(credentialsEndpoint, relativeURI string) string {
return credentialsEndpoint + relativeURI
}
// newChainCredentials is based on old credentials.NewChainCredentials in v1.
func newChainCredentials(providers ...aws.CredentialsProvider) aws.CredentialsProvider {
return aws.NewCredentialsCache(
aws.CredentialsProviderFunc(func(ctx context.Context) (aws.Credentials, error) {
var errs []error
for _, p := range providers {
creds, err := p.Retrieve(ctx)
if err == nil {
return creds, nil
}
errs = append(errs, err)
}
return aws.Credentials{}, fmt.Errorf("no valid providers in chain: %s", errs)
}),
)
}
pkg/storage/s3/s3_test.go
+27 -20
View File
@@ -4,10 +4,14 @@ import (
"context"
"os"
"testing"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/aws/smithy-go"
"github.com/gomods/athens/pkg/errors"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/gomods/athens/pkg/config"
"github.com/gomods/athens/pkg/storage/compliance"
)
@@ -26,7 +30,7 @@ func (s *Storage) clear() error {
ctx, cancel := context.WithTimeout(context.Background(), s.timeout)
defer cancel()
objects, err := s.s3API.ListObjectsWithContext(ctx, &s3.ListObjectsInput{Bucket: aws.String(s.bucket)})
objects, err := s.s3API.ListObjects(ctx, &s3.ListObjectsInput{Bucket: aws.String(s.bucket)})
if err != nil {
return err
}
@@ -37,7 +41,7 @@ func (s *Storage) clear() error {
Key: o.Key,
}
_, err := s.s3API.DeleteObjectWithContext(ctx, delParams)
_, err := s.s3API.DeleteObject(ctx, delParams)
if err != nil {
return err
}
@@ -49,23 +53,26 @@ func (s *Storage) createBucket() error {
ctx, cancel := context.WithTimeout(context.Background(), s.timeout)
defer cancel()
if _, err := s.s3API.CreateBucketWithContext(ctx, &s3.CreateBucketInput{Bucket: aws.String(s.bucket)}); err != nil {
aerr, ok := err.(awserr.Error)
if !ok {
return err
if _, err := s.s3API.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: aws.String(s.bucket)}); err != nil {
var aerr smithy.APIError
if errors.AsErr(err, &aerr) {
switch aerr.(type) {
case *types.BucketAlreadyOwnedByYou:
return nil
case *types.BucketAlreadyExists:
return nil
default:
return aerr
}
}
switch aerr.Code() {
case s3.ErrCodeBucketAlreadyOwnedByYou:
return nil
case s3.ErrCodeBucketAlreadyExists:
return nil
default:
return aerr
}
return err
}
return s.s3API.WaitUntilBucketExistsWithContext(ctx, &s3.HeadBucketInput{Bucket: aws.String(s.bucket)})
waiter := s3.NewBucketExistsWaiter(s.s3API)
return waiter.Wait(ctx, &s3.HeadBucketInput{Bucket: aws.String(s.bucket)}, 10*time.Minute)
}
func getStorage(t testing.TB) *Storage {
@@ -75,9 +82,9 @@ func getStorage(t testing.TB) *Storage {
}
options := func(conf *aws.Config) {
conf.Endpoint = aws.String(url)
conf.DisableSSL = aws.Bool(true)
conf.BaseEndpoint = aws.String(url)
}
backend, err := New(
&config.S3Config{
Key: "minio",
pkg/storage/s3/saver.go
+5 -4
View File
@@ -5,8 +5,8 @@ import (
"context"
"io"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/gomods/athens/pkg/errors"
"github.com/gomods/athens/pkg/observ"
moduploader "github.com/gomods/athens/pkg/storage/module"
@@ -31,14 +31,15 @@ func (s *Storage) upload(ctx context.Context, path, contentType string, stream i
const op errors.Op = "s3.upload"
ctx, span := observ.StartSpan(ctx, op.String())
defer span.End()
upParams := &s3manager.UploadInput{
upParams := &s3.PutObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(path),
Body: stream,
ContentType: aws.String(contentType),
}
if _, err := s.uploader.UploadWithContext(ctx, upParams); err != nil {
if _, err := s.uploader.Upload(ctx, upParams); err != nil {
return errors.E(op, err)
}