primal/athens
1
0
Fork 0
mirror of https://github.com/gomods/athens synced 2026-02-03 07:30:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

Upgrade to AWS SDK v2 (#1938) (#1950)

Browse Source 
Upgrades the AWS SDK to v2. AWS S3 bucket urls will now error if they are not prefixed with a schema (example: https://).
This commit is contained in:
Vanes Angelo
2024-04-29 11:38:42 +07:00
committed by GitHub
parent c35b50f2f3
commit 900cb4f7c1
37 changed files with 274 additions and 177 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/ci.yml
+1 -1
View File
@@ -30,7 +30,7 @@ jobs:
build:
env:
ATHENS_MONGO_STORAGE_URL: mongodb://localhost:27017
ATHENS_MINIO_ENDPOINT: localhost:9000
ATHENS_MINIO_ENDPOINT: http://localhost:9000
REDIS_TEST_ENDPOINT: localhost:6379
REDIS_SENTINEL_TEST_ENDPOINT: localhost:26379
REDIS_SENTINEL_TEST_MASTER_NAME: redis-1
cmd/proxy/actions/app_proxy_test.go
+1 -2
View File
@@ -48,7 +48,7 @@ func TestProxyRoutes(t *testing.T) {
tmp, err := template.New("home").Parse(homepage)
assert.NoError(t, err)
var templateData = make(map[string]string)
templateData := make(map[string]string)
templateData["Host"] = req.Host
@@ -109,5 +109,4 @@ func TestProxyRoutes(t *testing.T) {
tc.test(t, req, w.Result())
})
}
}
cmd/proxy/actions/index_test.go
-1
View File
@@ -107,7 +107,6 @@ func TestIndexHandler(t *testing.T) {
if w.Code != tc.code {
t.Fatalf("expected response code to be %d but got %d", tc.code, w.Code)
}
})
}
}
cmd/proxy/actions/storage.go
+1 -1
View File
@@ -6,7 +6,7 @@ import (
"net/http"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/gomods/athens/pkg/config"
"github.com/gomods/athens/pkg/errors"
"github.com/gomods/athens/pkg/storage"
docker-compose.yml
+1 -1
View File
@@ -24,7 +24,7 @@ services:
command: ["./scripts/test_unit.sh"]
environment:
- GO_ENV=test
- ATHENS_MINIO_ENDPOINT=minio:9000
- ATHENS_MINIO_ENDPOINT=http://minio:9000
- ATHENS_MONGO_STORAGE_URL=mongodb://mongo:27017
- TIMEOUT=20 # in case the mongo dependency takes longer to start up
- ATHENS_STORAGE_TYPE=mongo
e2etests/all_test.go
+3 -3
View File
@@ -62,9 +62,9 @@ func (m *E2eSuite) SetupSuite() {
func (m *E2eSuite) TearDownSuite() {
m.stopAthens()
chmodR(m.goPath, 0777)
chmodR(m.goPath, 0o777)
os.RemoveAll(m.goPath)
chmodR(m.sampleRepoPath, 0777)
chmodR(m.sampleRepoPath, 0o777)
os.RemoveAll(m.sampleRepoPath)
}
@@ -73,7 +73,7 @@ func TestE2E(t *testing.T) {
}
func (m *E2eSuite) SetupTest() {
chmodR(m.goPath, 0777)
chmodR(m.goPath, 0o777)
err := cleanGoCache(m.getEnv())
if err != nil {
m.Fail("Failed to clear go cache", err)
go.mod
+20 -1
View File
@@ -10,7 +10,12 @@ require (
github.com/Azure/azure-storage-blob-go v0.10.0
github.com/BurntSushi/toml v1.0.0
github.com/DataDog/opencensus-go-exporter-datadog v0.0.0-20180917103902-e6c7f767dc57
github.com/aws/aws-sdk-go v1.51.21
github.com/aws/aws-sdk-go-v2 v1.26.1
github.com/aws/aws-sdk-go-v2/config v1.27.11
github.com/aws/aws-sdk-go-v2/credentials v1.17.11
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.15
github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1
github.com/aws/smithy-go v1.20.2
github.com/bsm/redislock v0.7.2
github.com/fatih/color v1.13.0
github.com/go-playground/validator/v10 v10.19.0
@@ -55,6 +60,20 @@ require (
github.com/agext/levenshtein v1.2.1 // indirect
github.com/ajg/form v0.0.0-20160822230020-523a5da1a92f // indirect
github.com/apparentlymart/go-textseg v1.0.0 // indirect
github.com/aws/aws-sdk-go v1.44.220 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cenkalti/backoff/v4 v4.1.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
go.sum
+46 -2
View File
@@ -97,8 +97,46 @@ github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kd
github.com/apparentlymart/go-dump v0.0.0-20180507223929-23540a00eaa3/go.mod h1:oL81AME2rN47vu18xqj1S1jPIPuN7afo62yKTNn3XMM=
github.com/apparentlymart/go-textseg v1.0.0 h1:rRmlIsPEEhUTIKQb7T++Nz/A5Q6C9IuX2wFoYVvnCs0=
github.com/apparentlymart/go-textseg v1.0.0/go.mod h1:z96Txxhf3xSFMPmb5X/1W05FF/Nj9VFpLOpjS5yuumk=
github.com/aws/aws-sdk-go v1.51.21 h1:UrT6JC9R9PkYYXDZBV0qDKTualMr+bfK2eboTknMgbs=
github.com/aws/aws-sdk-go v1.51.21/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
github.com/aws/aws-sdk-go v1.44.220 h1:yAj99qAt0Htjle9Up3DglgHfOP77lmFPrElA4jKnrBo=
github.com/aws/aws-sdk-go v1.44.220/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
github.com/aws/aws-sdk-go-v2 v1.26.1 h1:5554eUqIYVWpU0YmeeYZ0wU64H2VLBs8TlhRB2L+EkA=
github.com/aws/aws-sdk-go-v2 v1.26.1/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 h1:x6xsQXGSmW6frevwDA+vi/wqhp1ct18mVXYN08/93to=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2/go.mod h1:lPprDr1e6cJdyYeGXnRaJoP4Md+cDBvi2eOj00BlGmg=
github.com/aws/aws-sdk-go-v2/config v1.27.11 h1:f47rANd2LQEYHda2ddSCKYId18/8BhSRM4BULGmfgNA=
github.com/aws/aws-sdk-go-v2/config v1.27.11/go.mod h1:SMsV78RIOYdve1vf36z8LmnszlRWkwMQtomCAI0/mIE=
github.com/aws/aws-sdk-go-v2/credentials v1.17.11 h1:YuIB1dJNf1Re822rriUOTxopaHHvIq0l/pX3fwO+Tzs=
github.com/aws/aws-sdk-go-v2/credentials v1.17.11/go.mod h1:AQtFPsDH9bI2O+71anW6EKL+NcD7LG3dpKGMV4SShgo=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 h1:FVJ0r5XTHSmIHJV6KuDmdYhEpvlHpiSd38RQWhut5J4=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1/go.mod h1:zusuAeqezXzAB24LGuzuekqMAEgWkVYukBec3kr3jUg=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.15 h1:7Zwtt/lP3KNRkeZre7soMELMGNoBrutx8nobg1jKWmo=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.15/go.mod h1:436h2adoHb57yd+8W+gYPrrA9U/R/SuAuOO42Ushzhw=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 h1:aw39xVGeRWlWx9EzGVnhOR4yOjQDHPQ6o6NmBlscyQg=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5/go.mod h1:FSaRudD0dXiMPK2UjknVwwTYyZMRsHv3TtkabsZih5I=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 h1:PG1F3OD1szkuQPzDw3CIQsRIrtTlUC3lP84taWzHlq0=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5/go.mod h1:jU1li6RFryMz+so64PpKtudI+QzbKoIEivqdf6LNpOc=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5 h1:81KE7vaZzrl7yHBYHVEzYB8sypz11NMOZ40YlWvPxsU=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5/go.mod h1:LIt2rg7Mcgn09Ygbdh/RdIm0rQ+3BNkbP1gyVMFtRK0=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 h1:Ji0DY1xUsUr3I8cHps0G+XM3WWU16lP6yG8qu1GAZAs=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2/go.mod h1:5CsjAbs3NlGQyZNFACh+zztPDI7fU6eW9QsxjfnuBKg=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7 h1:ZMeFZ5yk+Ek+jNr1+uwCd2tG89t6oTS5yVWpa6yy2es=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7/go.mod h1:mxV05U+4JiHqIpGqqYXOHLPKUC6bDXC44bsUhNjOEwY=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 h1:ogRAwT1/gxJBcSWDMZlgyFUM962F51A5CRhDLbxLdmo=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7/go.mod h1:YCsIZhXfRPLFFCl5xxY+1T9RKzOKjCut+28JSX2DnAk=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5 h1:f9RyWNtS8oH7cZlbn+/JNPpjUk5+5fLd5lM9M0i49Ys=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5/go.mod h1:h5CoMZV2VF297/VLhRhO1WF+XYWOzXo+4HsObA4HjBQ=
github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1 h1:6cnno47Me9bRykw9AEv9zkXE+5or7jz8TsskTTccbgc=
github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1/go.mod h1:qmdkIIAC+GCLASF7R2whgNrJADz0QZPX+Seiw/i4S3o=
github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 h1:vN8hEbpRnL7+Hopy9dzmRle1xmDc7o8tmY0klsr175w=
github.com/aws/aws-sdk-go-v2/service/sso v1.20.5/go.mod h1:qGzynb/msuZIE8I75DVRCUXw3o3ZyBmUvMwQ2t/BrGM=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 h1:Jux+gDDyi1Lruk+KHF91tK2KCuY61kzoCpvtvJJBtOE=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4/go.mod h1:mUYPBhaF2lGiukDEjJX2BLRRKTmoUSitGDUgM4tRxak=
github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 h1:cwIxeBttqPN3qkaAjcEcsh8NYr8n2HZPkcKgPAi1phU=
github.com/aws/aws-sdk-go-v2/service/sts v1.28.6/go.mod h1:FZf1/nKNEkHdGGJP/cI2MoIMquumuRK6ol3QQJNDxmw=
github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q=
github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
@@ -649,6 +687,9 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b
golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc=
golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -732,10 +773,12 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -745,6 +788,7 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
pkg/config/azureblob.go
+3 -3
View File
@@ -2,7 +2,7 @@ package config
// AzureBlobConfig specifies the properties required to use Azure as the storage backend.
type AzureBlobConfig struct {
AccountName string `validate:"required" envconfig:"ATHENS_AZURE_ACCOUNT_NAME"`
AccountKey string `validate:"required" envconfig:"ATHENS_AZURE_ACCOUNT_KEY"`
ContainerName string `validate:"required" envconfig:"ATHENS_AZURE_CONTAINER_NAME"`
AccountName string `envconfig:"ATHENS_AZURE_ACCOUNT_NAME" validate:"required"`
AccountKey string `envconfig:"ATHENS_AZURE_ACCOUNT_KEY" validate:"required"`
ContainerName string `envconfig:"ATHENS_AZURE_CONTAINER_NAME" validate:"required"`
}
pkg/config/config.go
+10 -10
View File
@@ -21,22 +21,22 @@ const defaultConfigFile = "athens.toml"
// Config provides configuration values for all components.
type Config struct {
TimeoutConf
GoEnv string `validate:"required" envconfig:"GO_ENV"`
GoBinary string `validate:"required" envconfig:"GO_BINARY_PATH"`
GoEnv string `envconfig:"GO_ENV" validate:"required"`
GoBinary string `envconfig:"GO_BINARY_PATH" validate:"required"`
GoBinaryEnvVars EnvList `envconfig:"ATHENS_GO_BINARY_ENV_VARS"`
GoGetWorkers int `validate:"required" envconfig:"ATHENS_GOGET_WORKERS"`
GoGetWorkers int `envconfig:"ATHENS_GOGET_WORKERS" validate:"required"`
GoGetDir string `envconfig:"ATHENS_GOGET_DIR"`
ProtocolWorkers int `validate:"required" envconfig:"ATHENS_PROTOCOL_WORKERS"`
LogLevel string `validate:"required" envconfig:"ATHENS_LOG_LEVEL"`
LogFormat string `validate:"oneof='' 'json' 'plain'" envconfig:"ATHENS_LOG_FORMAT"`
CloudRuntime string `validate:"required_without=LogFormat" envconfig:"ATHENS_CLOUD_RUNTIME"`
ProtocolWorkers int `envconfig:"ATHENS_PROTOCOL_WORKERS" validate:"required"`
LogLevel string `envconfig:"ATHENS_LOG_LEVEL" validate:"required"`
LogFormat string `envconfig:"ATHENS_LOG_FORMAT" validate:"oneof='' 'json' 'plain'"`
CloudRuntime string `envconfig:"ATHENS_CLOUD_RUNTIME" validate:"required_without=LogFormat"`
EnablePprof bool `envconfig:"ATHENS_ENABLE_PPROF"`
PprofPort string `envconfig:"ATHENS_PPROF_PORT"`
FilterFile string `envconfig:"ATHENS_FILTER_FILE"`
TraceExporterURL string `envconfig:"ATHENS_TRACE_EXPORTER_URL"`
TraceExporter string `envconfig:"ATHENS_TRACE_EXPORTER"`
StatsExporter string `envconfig:"ATHENS_STATS_EXPORTER"`
StorageType string `validate:"required" envconfig:"ATHENS_STORAGE_TYPE"`
StorageType string `envconfig:"ATHENS_STORAGE_TYPE" validate:"required"`
GlobalEndpoint string `envconfig:"ATHENS_GLOBAL_ENDPOINT"` // This feature is not yet implemented
Port string `envconfig:"ATHENS_PORT"`
UnixSocket string `envconfig:"ATHENS_UNIX_SOCKET"`
@@ -55,11 +55,11 @@ type Config struct {
NoSumPatterns []string `envconfig:"ATHENS_GONOSUM_PATTERNS"`
DownloadMode mode.Mode `envconfig:"ATHENS_DOWNLOAD_MODE"`
DownloadURL string `envconfig:"ATHENS_DOWNLOAD_URL"`
NetworkMode string `validate:"oneof=strict offline fallback" envconfig:"ATHENS_NETWORK_MODE"`
NetworkMode string `envconfig:"ATHENS_NETWORK_MODE" validate:"oneof=strict offline fallback"`
SingleFlightType string `envconfig:"ATHENS_SINGLE_FLIGHT_TYPE"`
RobotsFile string `envconfig:"ATHENS_ROBOTS_FILE"`
IndexType string `envconfig:"ATHENS_INDEX_TYPE"`
ShutdownTimeout int `validate:"min=0" envconfig:"ATHENS_SHUTDOWN_TIMEOUT"`
ShutdownTimeout int `envconfig:"ATHENS_SHUTDOWN_TIMEOUT" validate:"min=0"`
SingleFlight *SingleFlight
Storage *Storage
Index *Index
pkg/config/config_test.go
+4 -4
View File
@@ -16,7 +16,7 @@ import (
func testConfigFile(t *testing.T) (testConfigFile string) {
testConfigFile = filepath.Join("..", "..", "config.dev.toml")
if err := os.Chmod(testConfigFile, 0700); err != nil {
if err := os.Chmod(testConfigFile, 0o700); err != nil {
t.Fatalf("%s\n", err)
}
return testConfigFile
@@ -108,6 +108,7 @@ func TestEnvOverrides(t *testing.T) {
if err != nil {
t.Fatalf("Env override failed: %v", err)
}
compareConfigs(conf, expConf, t, Storage{}, SingleFlight{})
}
@@ -305,7 +306,6 @@ func TestParseExampleConfig(t *testing.T) {
}
func getEnvMap(config *Config) map[string]string {
envVars := map[string]string{
"GO_ENV": config.GoEnv,
"GO_BINARY_PATH": config.GoBinary,
@@ -414,7 +414,7 @@ func Test_checkFilePerms(t *testing.T) {
}
incorrectPerms := []os.FileMode{0o777, 0o610, 0o660}
var incorrectFiles = make([]string, len(incorrectPerms))
incorrectFiles := make([]string, len(incorrectPerms))
for i := range incorrectPerms {
f, err := tempFile(incorrectPerms[i])
@@ -426,7 +426,7 @@ func Test_checkFilePerms(t *testing.T) {
}
correctPerms := []os.FileMode{0o600, 0o400, 0o644}
var correctFiles = make([]string, len(correctPerms))
correctFiles := make([]string, len(correctPerms))
for i := range correctPerms {
f, err := tempFile(correctPerms[i])
pkg/config/disk.go
+1 -1
View File
@@ -2,5 +2,5 @@ package config
// DiskConfig specifies the properties required to use Disk as the storage backend.
type DiskConfig struct {
RootPath string `validate:"required" envconfig:"ATHENS_DISK_STORAGE_ROOT"`
RootPath string `envconfig:"ATHENS_DISK_STORAGE_ROOT" validate:"required"`
}
pkg/config/external.go
+1 -1
View File
@@ -2,5 +2,5 @@ package config
// External specifies configuration for an external http storage.
type External struct {
URL string `validate:"required" envconfig:"ATHENS_EXTERNAL_STORAGE_URL"`
URL string `envconfig:"ATHENS_EXTERNAL_STORAGE_URL" validate:"required"`
}
pkg/config/gcp.go
+1 -1
View File
@@ -3,6 +3,6 @@ package config
// GCPConfig specifies the properties required to use GCP as the storage backend.
type GCPConfig struct {
ProjectID string `envconfig:"GOOGLE_CLOUD_PROJECT"`
Bucket string `validate:"required" envconfig:"ATHENS_STORAGE_GCP_BUCKET"`
Bucket string `envconfig:"ATHENS_STORAGE_GCP_BUCKET" validate:"required"`
JSONKey string `envconfig:"ATHENS_STORAGE_GCP_JSON_KEY"`
}
pkg/config/minio.go
+4 -4
View File
@@ -3,10 +3,10 @@ package config
// MinioConfig specifies the properties required to use Minio or DigitalOcean Spaces
// as the storage backend.
type MinioConfig struct {
Endpoint string `validate:"required" envconfig:"ATHENS_MINIO_ENDPOINT"`
Key string `validate:"required" envconfig:"ATHENS_MINIO_ACCESS_KEY_ID"`
Secret string `validate:"required" envconfig:"ATHENS_MINIO_SECRET_ACCESS_KEY"`
Bucket string `validate:"required" envconfig:"ATHENS_MINIO_BUCKET_NAME"`
Endpoint string `envconfig:"ATHENS_MINIO_ENDPOINT" validate:"required"`
Key string `envconfig:"ATHENS_MINIO_ACCESS_KEY_ID" validate:"required"`
Secret string `envconfig:"ATHENS_MINIO_SECRET_ACCESS_KEY" validate:"required"`
Bucket string `envconfig:"ATHENS_MINIO_BUCKET_NAME" validate:"required"`
Region string `envconfig:"ATHENS_MINIO_REGION"`
EnableSSL bool `envconfig:"ATHENS_MINIO_USE_SSL"`
}
pkg/config/mongo.go
+3 -3
View File
@@ -2,9 +2,9 @@ package config
// MongoConfig specifies the properties required to use MongoDB as the storage backend.
type MongoConfig struct {
URL string `validate:"required" envconfig:"ATHENS_MONGO_STORAGE_URL"`
DefaultDBName string `envconfig:"ATHENS_MONGO_DEFAULT_DATABASE" default:"athens"`
DefaultCollectionName string `envconfig:"ATHENS_MONGO_DEFAULT_COLLECTION" default:"modules"`
URL string `envconfig:"ATHENS_MONGO_STORAGE_URL" validate:"required"`
DefaultDBName string `default:"athens" envconfig:"ATHENS_MONGO_DEFAULT_DATABASE"`
DefaultCollectionName string `default:"modules" envconfig:"ATHENS_MONGO_DEFAULT_COLLECTION"`
CertPath string `envconfig:"ATHENS_MONGO_CERT_PATH"`
InsecureConn bool `envconfig:"ATHENS_MONGO_INSECURE"`
}
pkg/config/mysql.go
+7 -7
View File
@@ -2,11 +2,11 @@ package config
// MySQL config.
type MySQL struct {
Protocol string `validate:"required" envconfig:"ATHENS_INDEX_MYSQL_PROTOCOL"`
Host string `validate:"required" envconfig:"ATHENS_INDEX_MYSQL_HOST"`
Port int `validate:"" envconfig:"ATHENS_INDEX_MYSQL_PORT"`
User string `validate:"required" envconfig:"ATHENS_INDEX_MYSQL_USER"`
Password string `validate:"" envconfig:"ATHENS_INDEX_MYSQL_PASSWORD"`
Database string `validate:"required" envconfig:"ATHENS_INDEX_MYSQL_DATABASE"`
Params map[string]string `validate:"required" envconfig:"ATHENS_INDEX_MYSQL_PARAMS"`
Protocol string `envconfig:"ATHENS_INDEX_MYSQL_PROTOCOL" validate:"required"`
Host string `envconfig:"ATHENS_INDEX_MYSQL_HOST" validate:"required"`
Port int `envconfig:"ATHENS_INDEX_MYSQL_PORT" validate:""`
User string `envconfig:"ATHENS_INDEX_MYSQL_USER" validate:"required"`
Password string `envconfig:"ATHENS_INDEX_MYSQL_PASSWORD" validate:""`
Database string `envconfig:"ATHENS_INDEX_MYSQL_DATABASE" validate:"required"`
Params map[string]string `envconfig:"ATHENS_INDEX_MYSQL_PARAMS" validate:"required"`
}
pkg/config/postgres.go
+6 -6
View File
@@ -2,10 +2,10 @@ package config
// Postgres config.
type Postgres struct {
Host string `validate:"required" envconfig:"ATHENS_INDEX_POSTGRES_HOST"`
Port int `validate:"required" envconfig:"ATHENS_INDEX_POSTGRES_PORT"`
User string `validate:"required" envconfig:"ATHENS_INDEX_POSTGRES_USER"`
Password string `validate:"" envconfig:"ATHENS_INDEX_POSTGRES_PASSWORD"`
Database string `validate:"required" envconfig:"ATHENS_INDEX_POSTGRES_DATABASE"`
Params map[string]string `validate:"required" envconfig:"ATHENS_INDEX_POSTGRES_PARAMS"`
Host string `envconfig:"ATHENS_INDEX_POSTGRES_HOST" validate:"required"`
Port int `envconfig:"ATHENS_INDEX_POSTGRES_PORT" validate:"required"`
User string `envconfig:"ATHENS_INDEX_POSTGRES_USER" validate:"required"`
Password string `envconfig:"ATHENS_INDEX_POSTGRES_PASSWORD" validate:""`
Database string `envconfig:"ATHENS_INDEX_POSTGRES_DATABASE" validate:"required"`
Params map[string]string `envconfig:"ATHENS_INDEX_POSTGRES_PARAMS" validate:"required"`
}
pkg/config/s3.go
+2 -2
View File
@@ -2,11 +2,11 @@ package config
// S3Config specifies the properties required to use S3 as the storage backend.
type S3Config struct {
Region string `validate:"required" envconfig:"AWS_REGION"`
Region string `envconfig:"AWS_REGION" validate:"required"`
Key string `envconfig:"AWS_ACCESS_KEY_ID"`
Secret string `envconfig:"AWS_SECRET_ACCESS_KEY"`
Token string `envconfig:"AWS_SESSION_TOKEN"`
Bucket string `validate:"required" envconfig:"ATHENS_S3_BUCKET_NAME"`
Bucket string `envconfig:"ATHENS_S3_BUCKET_NAME" validate:"required"`
UseDefaultConfiguration bool `envconfig:"AWS_USE_DEFAULT_CONFIGURATION"`
ForcePathStyle bool `envconfig:"AWS_FORCE_PATH_STYLE"`
CredentialsEndpoint string `envconfig:"AWS_CREDENTIALS_ENDPOINT"`
pkg/download/list_merge_test.go
+4 -2
View File
@@ -13,8 +13,10 @@ import (
"github.com/stretchr/testify/require"
)
const testOp athenserr.Op = "vcsLister.List"
const testModName = "happy tags"
const (
testOp athenserr.Op = "vcsLister.List"
testModName = "happy tags"
)
type listMergeTest struct {
name string
pkg/download/mode/mode_test.go
+2 -1
View File
@@ -84,7 +84,8 @@ var testCases = []struct {
{
Pattern: "github.com/gomods/*",
Mode: AsyncRedirect,
DownloadURL: "gomods.io"},
DownloadURL: "gomods.io",
},
},
},
input: "github.com/gomods/athens",
pkg/download/protocol_test.go
+2 -4
View File
@@ -28,9 +28,7 @@ import (
"golang.org/x/sync/errgroup"
)
var (
testConfigPath = filepath.Join("..", "..", "config.dev.toml")
)
var testConfigPath = filepath.Join("..", "..", "config.dev.toml")
func getDP(t *testing.T) Protocol {
t.Helper()
@@ -235,7 +233,7 @@ var latestTests = []latestTest{
path: "github.com/athens-artifacts/happy-path",
info: &storage.RevInfo{
Version: "v0.0.3",
Time: time.Date(2018, 8, 3, 17, 16, 00, 0, time.UTC),
Time: time.Date(2018, 8, 3, 17, 16, 0o0, 0, time.UTC),
},
},
}
pkg/middleware/middleware_test.go
+1 -1
View File
@@ -23,7 +23,7 @@ const (
func testConfigFile(t *testing.T) (testConfigFile string) {
testConfigFile = filepath.Join("..", "..", "config.dev.toml")
if err := os.Chmod(testConfigFile, 0700); err != nil {
if err := os.Chmod(testConfigFile, 0o700); err != nil {
t.Fatalf("%s\n", err)
}
return testConfigFile
pkg/module/filter_test.go
+4 -6
View File
@@ -10,7 +10,7 @@ import (
func testConfigFile(t *testing.T) (testConfigFile string) {
testConfigFile = filepath.Join("..", "..", "config.dev.toml")
if err := os.Chmod(testConfigFile, 0700); err != nil {
if err := os.Chmod(testConfigFile, 0o700); err != nil {
t.Fatalf("%s\n", err)
}
return testConfigFile
@@ -40,7 +40,6 @@ func (t *FilterTests) Test_NewFilter() {
mf, err = NewFilter(filter)
r.Equal(filter, mf.filePath)
r.NoError(err)
}
func (t *FilterTests) Test_IgnoreSimple() {
@@ -198,7 +197,6 @@ func (t *FilterTests) Test_versionFilterRobust() {
r.Equal(Exclude, f.Rule("github.com/a/b", "a"))
r.Equal(Exclude, f.Rule("github.com/c/d", "fg"))
}
func (t *FilterTests) Test_initFromConfig() {
@@ -207,20 +205,20 @@ func (t *FilterTests) Test_initFromConfig() {
defer os.Remove(filterFile)
goodInput := []byte("+ github.com/a/b\n\n# some comment\n- github.com/c/d\n\nD github.com/x")
os.WriteFile(filterFile, goodInput, 0644)
os.WriteFile(filterFile, goodInput, 0o644)
f, err := initFromConfig(filterFile)
r.NotNil(f)
r.NoError(err)
badInput := []byte("+ github.com/a/b\n\n# some comment\n\n- github.com/c/d\n\nD github.com/x\nsome_random_line")
os.WriteFile(filterFile, badInput, 0644)
os.WriteFile(filterFile, badInput, 0o644)
f, err = initFromConfig(filterFile)
r.Nil(f)
r.Error(err)
versionInput := []byte("+ github.com/a/b\n\n# some comment\n\n- github.com/c/d v1,v2.3.4,v3.2.*\n\nD github.com/x\n")
os.WriteFile(filterFile, versionInput, 0644)
os.WriteFile(filterFile, versionInput, 0o644)
f, err = initFromConfig(filterFile)
r.NotNil(f)
r.NoError(err)
pkg/storage/minio/minio.go
+11 -1
View File
@@ -2,6 +2,7 @@ package minio
import (
"fmt"
"strings"
"time"
"github.com/gomods/athens/pkg/config"
@@ -24,7 +25,7 @@ func (s *storageImpl) versionLocation(module, version string) string {
// that implements storage.Backend.
func NewStorage(conf *config.MinioConfig, timeout time.Duration) (storage.Backend, error) {
const op errors.Op = "minio.NewStorage"
endpoint := conf.Endpoint
endpoint := TrimHTTP(conf.Endpoint)
accessKeyID := conf.Key
secretAccessKey := conf.Secret
bucketName := conf.Bucket
@@ -53,3 +54,12 @@ func NewStorage(conf *config.MinioConfig, timeout time.Duration) (storage.Backen
}
return &storageImpl{minioClient, minioCore, bucketName}, nil
}
// TrimHTTP trims "http://" or "https://" prefix from input string.
// Minio doesn't need to specify protocol in the URL so it should be trimmed.
// Related issue: https://github.com/gomods/athens/issues/1938#issuecomment-2067590653
func TrimHTTP(s string) string {
s = strings.TrimPrefix(s, "http://")
s = strings.TrimPrefix(s, "https://")
return s
}
pkg/storage/minio/minio_test.go
+3 -3
View File
@@ -15,7 +15,7 @@ func TestBackend(t *testing.T) {
// TestNewStorageExists tests the logic around MakeBucket and BucketExists
func TestNewStorageExists(t *testing.T) {
url := os.Getenv("ATHENS_MINIO_ENDPOINT")
url := TrimHTTP(os.Getenv("ATHENS_MINIO_ENDPOINT"))
if url == "" {
t.SkipNow()
}
@@ -51,7 +51,7 @@ func TestNewStorageExists(t *testing.T) {
// To ensure both paths are tested, there is a strict path error using the
// "_" and a non strict error using less than 3 characters
func TestNewStorageError(t *testing.T) {
url := os.Getenv("ATHENS_MINIO_ENDPOINT")
url := TrimHTTP(os.Getenv("ATHENS_MINIO_ENDPOINT"))
if url == "" {
t.SkipNow()
}
@@ -92,7 +92,7 @@ func (s *storageImpl) clear() error {
}
func getStorage(t testing.TB) *storageImpl {
url := os.Getenv("ATHENS_MINIO_ENDPOINT")
url := TrimHTTP(os.Getenv("ATHENS_MINIO_ENDPOINT"))
if url == "" {
t.SkipNow()
}
pkg/storage/mongo/mongo_test.go
+5 -3
View File
@@ -3,13 +3,14 @@ package mongo
import (
"bytes"
"context"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"io"
"os"
"testing"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"github.com/gomods/athens/pkg/config"
"github.com/gomods/athens/pkg/errors"
"github.com/gomods/athens/pkg/storage"
@@ -145,6 +146,7 @@ func TestQueryKindUnexpectedErrorCases(t *testing.T) {
require.Equal(t, errors.KindUnexpected, errors.Kind(err))
}
}
func TestNewStorageWithDefaultOverrides(t *testing.T) {
url := os.Getenv("ATHENS_MONGO_STORAGE_URL")
pkg/storage/s3/cataloger.go
+6 -5
View File
@@ -5,8 +5,9 @@ import (
"fmt"
"strings"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/gomods/athens/pkg/config"
"github.com/gomods/athens/pkg/errors"
"github.com/gomods/athens/pkg/observ"
@@ -28,7 +29,7 @@ func (s *Storage) Catalog(ctx context.Context, token string, pageSize int) ([]pa
Marker: &queryToken,
}
loo, err := s.s3API.ListObjectsWithContext(ctx, lsParams)
loo, err := s.s3API.ListObjects(ctx, lsParams)
if err != nil {
return nil, "", errors.E(op, err)
}
@@ -49,7 +50,7 @@ func (s *Storage) Catalog(ctx context.Context, token string, pageSize int) ([]pa
return res, queryToken, nil
}
func fetchModsAndVersions(objects []*s3.Object, elementsNum int) ([]paths.AllPathParams, string) {
func fetchModsAndVersions(objects []types.Object, elementsNum int) ([]paths.AllPathParams, string) {
res := make([]paths.AllPathParams, 0)
lastKey := ""
for _, o := range objects {
@@ -72,7 +73,7 @@ func fetchModsAndVersions(objects []*s3.Object, elementsNum int) ([]paths.AllPat
return res, lastKey
}
func parseS3Key(o *s3.Object) (paths.AllPathParams, error) {
func parseS3Key(o types.Object) (paths.AllPathParams, error) {
const op errors.Op = "s3.parseS3Key"
m, v := config.ModuleVersionFromPath(*o.Key)
pkg/storage/s3/checker.go
+6 -6
View File
@@ -4,9 +4,9 @@ import (
"context"
"sync"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/smithy-go"
"github.com/gomods/athens/pkg/config"
"github.com/gomods/athens/pkg/errors"
"github.com/gomods/athens/pkg/observ"
@@ -28,7 +28,7 @@ func (s *Storage) Exists(ctx context.Context, module, version string) (bool, err
wg.Add(1)
go func(file string) {
defer wg.Done()
_, err := s.s3API.HeadObjectWithContext(
_, err := s.s3API.HeadObject(
cancelingCtx,
&s3.HeadObjectInput{
Bucket: aws.String(s.bucket),
@@ -44,8 +44,8 @@ func (s *Storage) Exists(ctx context.Context, module, version string) (bool, err
if err == nil {
continue
}
var aerr awserr.Error
if errors.AsErr(err, &aerr) && aerr.Code() == "NotFound" {
var aerr smithy.APIError
if errors.AsErr(err, &aerr) && aerr.ErrorCode() == "NotFound" {
err = nil
exists = false
}
pkg/storage/s3/deleter.go
+4 -3
View File
@@ -3,8 +3,8 @@ package s3
import (
"context"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/gomods/athens/pkg/errors"
"github.com/gomods/athens/pkg/observ"
modupl "github.com/gomods/athens/pkg/storage/module"
@@ -32,12 +32,13 @@ func (s *Storage) remove(ctx context.Context, path string) error {
const op errors.Op = "s3.Delete"
ctx, span := observ.StartSpan(ctx, op.String())
defer span.End()
delParams := &s3.DeleteObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(path),
}
if _, err := s.s3API.DeleteObjectWithContext(ctx, delParams); err != nil {
if _, err := s.s3API.DeleteObject(ctx, delParams); err != nil {
return errors.E(op, err)
}
pkg/storage/s3/getter.go
+12 -12
View File
@@ -5,9 +5,9 @@ import (
"fmt"
"io"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/gomods/athens/pkg/config"
"github.com/gomods/athens/pkg/errors"
"github.com/gomods/athens/pkg/observ"
@@ -22,8 +22,8 @@ func (s *Storage) Info(ctx context.Context, module, version string) ([]byte, err
infoReader, err := s.open(ctx, config.PackageVersionedName(module, version, "info"))
if err != nil {
var aerr awserr.Error
if errors.AsErr(err, &aerr) && aerr.Code() == s3.ErrCodeNoSuchKey {
var nsk *types.NoSuchKey
if errors.AsErr(err, &nsk) {
return nil, errors.E(op, errors.M(module), errors.V(version), errors.KindNotFound)
}
return nil, errors.E(op, err, errors.M(module), errors.V(version))
@@ -45,8 +45,8 @@ func (s *Storage) GoMod(ctx context.Context, module, version string) ([]byte, er
modReader, err := s.open(ctx, config.PackageVersionedName(module, version, "mod"))
if err != nil {
var aerr awserr.Error
if errors.AsErr(err, &aerr) && aerr.Code() == s3.ErrCodeNoSuchKey {
var nsk *types.NoSuchKey
if errors.AsErr(err, &nsk) {
return nil, errors.E(op, errors.M(module), errors.V(version), errors.KindNotFound)
}
return nil, errors.E(op, err, errors.M(module), errors.V(version))
@@ -69,8 +69,8 @@ func (s *Storage) Zip(ctx context.Context, module, version string) (storage.Size
zipReader, err := s.open(ctx, config.PackageVersionedName(module, version, "zip"))
if err != nil {
var aerr awserr.Error
if errors.AsErr(err, &aerr) && aerr.Code() == s3.ErrCodeNoSuchKey {
var nsk *types.NoSuchKey
if errors.AsErr(err, &nsk) {
return nil, errors.E(op, errors.M(module), errors.V(version), errors.KindNotFound)
}
return nil, errors.E(op, err, errors.M(module), errors.V(version))
@@ -88,10 +88,10 @@ func (s *Storage) open(ctx context.Context, path string) (storage.SizeReadCloser
Key: aws.String(path),
}
goo, err := s.s3API.GetObjectWithContext(ctx, getParams)
goo, err := s.s3API.GetObject(ctx, getParams)
if err != nil {
var aerr awserr.Error
if errors.AsErr(err, &aerr) && aerr.Code() == s3.ErrCodeNoSuchKey {
var nsk *types.NoSuchKey
if errors.AsErr(err, &nsk) {
return nil, errors.E(op, errors.KindNotFound)
}
return nil, errors.E(op, err)
pkg/storage/s3/lister.go
+5 -4
View File
@@ -4,8 +4,9 @@ import (
"context"
"strings"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/gomods/athens/pkg/errors"
"github.com/gomods/athens/pkg/observ"
)
@@ -23,7 +24,7 @@ func (s *Storage) List(ctx context.Context, module string) ([]string, error) {
Prefix: aws.String(modulePrefix),
}
loo, err := s.s3API.ListObjectsWithContext(ctx, lsParams)
loo, err := s.s3API.ListObjects(ctx, lsParams)
if err != nil {
return nil, errors.E(op, err, errors.M(module))
}
@@ -31,7 +32,7 @@ func (s *Storage) List(ctx context.Context, module string) ([]string, error) {
return extractVersions(loo.Contents), nil
}
func extractVersions(objects []*s3.Object) []string {
func extractVersions(objects []types.Object) []string {
var versions []string
for _, o := range objects {
pkg/storage/s3/s3.go
+60 -46
View File
@@ -1,16 +1,16 @@
package s3
import (
"context"
"fmt"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/credentials/endpointcreds"
"github.com/aws/aws-sdk-go/aws/defaults"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3/s3iface"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"github.com/aws/aws-sdk-go/service/s3/s3manager/s3manageriface"
"github.com/aws/aws-sdk-go-v2/aws"
awscfg "github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/credentials/endpointcreds"
"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/gomods/athens/pkg/config"
"github.com/gomods/athens/pkg/errors"
)
@@ -26,8 +26,8 @@ import (
// For information how to get your keyId and access key turn to official aws docs: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/setting-up.html.
type Storage struct {
bucket string
uploader s3manageriface.UploaderAPI
s3API s3iface.S3API
uploader *manager.Uploader
s3API *s3.Client
timeout time.Duration
}
@@ -35,50 +35,44 @@ type Storage struct {
func New(s3Conf *config.S3Config, timeout time.Duration, options ...func(*aws.Config)) (*Storage, error) {
const op errors.Op = "s3.New"
awsConfig := defaults.Config()
// Remove anonymous credentials from the default config so that
// session.NewSession can auto-resolve credentials from role, profile, env etc.
awsConfig.Credentials = nil
awsConfig.Region = aws.String(s3Conf.Region)
for _, o := range options {
o(awsConfig)
}
if !s3Conf.UseDefaultConfiguration {
credProviders := defaults.CredProviders(awsConfig, defaults.Handlers())
endpointcreds := []credentials.Provider{
endpointcreds.NewProviderClient(*awsConfig, defaults.Handlers(), endpointFrom(s3Conf.CredentialsEndpoint, s3Conf.AwsContainerCredentialsRelativeURI)),
&credentials.StaticProvider{
Value: credentials.Value{
AccessKeyID: s3Conf.Key,
SecretAccessKey: s3Conf.Secret,
SessionToken: s3Conf.Token,
},
},
}
credProviders = append(endpointcreds, credProviders...)
awsConfig.Credentials = credentials.NewChainCredentials(credProviders)
}
awsConfig.S3ForcePathStyle = aws.Bool(s3Conf.ForcePathStyle)
awsConfig.CredentialsChainVerboseErrors = aws.Bool(true)
if s3Conf.Endpoint != "" {
awsConfig.Endpoint = aws.String(s3Conf.Endpoint)
}
// Create a session with creds.
sess, err := session.NewSession(awsConfig)
awsConfig, err := awscfg.LoadDefaultConfig(context.TODO(), awscfg.WithRegion(s3Conf.Region))
if err != nil {
return nil, errors.E(op, err)
}
uploader := s3manager.NewUploader(sess)
// Remove anonymous credentials from the default config so that
// session.NewSession can auto-resolve credentials from role, profile, env etc.
awsConfig.Credentials = nil
for _, o := range options {
o(&awsConfig)
}
if !s3Conf.UseDefaultConfiguration {
// credProviders := defaults.CredProviders(awsConfig, defaults.Handlers())
endpointCreds := []aws.CredentialsProvider{
endpointcreds.New(endpointFrom(s3Conf.CredentialsEndpoint, s3Conf.AwsContainerCredentialsRelativeURI)),
credentials.NewStaticCredentialsProvider(s3Conf.Key, s3Conf.Secret, s3Conf.Token),
}
// credProviders = append(endpointCreds, credProviders...)
awsConfig.Credentials = newChainCredentials(endpointCreds...)
}
// Create a session with creds.
sess := s3.NewFromConfig(awsConfig, func(o *s3.Options) {
o.UsePathStyle = s3Conf.ForcePathStyle
if s3Conf.Endpoint != "" {
o.BaseEndpoint = aws.String(s3Conf.Endpoint)
}
})
uploader := manager.NewUploader(sess)
return &Storage{
bucket: s3Conf.Bucket,
uploader: uploader,
s3API: uploader.S3,
s3API: sess,
timeout: timeout,
}, nil
}
@@ -86,3 +80,23 @@ func New(s3Conf *config.S3Config, timeout time.Duration, options ...func(*aws.Co
func endpointFrom(credentialsEndpoint, relativeURI string) string {
return credentialsEndpoint + relativeURI
}
// newChainCredentials is based on old credentials.NewChainCredentials in v1.
func newChainCredentials(providers ...aws.CredentialsProvider) aws.CredentialsProvider {
return aws.NewCredentialsCache(
aws.CredentialsProviderFunc(func(ctx context.Context) (aws.Credentials, error) {
var errs []error
for _, p := range providers {
creds, err := p.Retrieve(ctx)
if err == nil {
return creds, nil
}
errs = append(errs, err)
}
return aws.Credentials{}, fmt.Errorf("no valid providers in chain: %s", errs)
}),
)
}
pkg/storage/s3/s3_test.go
+23 -16
View File
@@ -4,10 +4,14 @@ import (
"context"
"os"
"testing"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/aws/smithy-go"
"github.com/gomods/athens/pkg/errors"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/gomods/athens/pkg/config"
"github.com/gomods/athens/pkg/storage/compliance"
)
@@ -26,7 +30,7 @@ func (s *Storage) clear() error {
ctx, cancel := context.WithTimeout(context.Background(), s.timeout)
defer cancel()
objects, err := s.s3API.ListObjectsWithContext(ctx, &s3.ListObjectsInput{Bucket: aws.String(s.bucket)})
objects, err := s.s3API.ListObjects(ctx, &s3.ListObjectsInput{Bucket: aws.String(s.bucket)})
if err != nil {
return err
}
@@ -37,7 +41,7 @@ func (s *Storage) clear() error {
Key: o.Key,
}
_, err := s.s3API.DeleteObjectWithContext(ctx, delParams)
_, err := s.s3API.DeleteObject(ctx, delParams)
if err != nil {
return err
}
@@ -49,23 +53,26 @@ func (s *Storage) createBucket() error {
ctx, cancel := context.WithTimeout(context.Background(), s.timeout)
defer cancel()
if _, err := s.s3API.CreateBucketWithContext(ctx, &s3.CreateBucketInput{Bucket: aws.String(s.bucket)}); err != nil {
aerr, ok := err.(awserr.Error)
if !ok {
return err
}
if _, err := s.s3API.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: aws.String(s.bucket)}); err != nil {
var aerr smithy.APIError
switch aerr.Code() {
case s3.ErrCodeBucketAlreadyOwnedByYou:
if errors.AsErr(err, &aerr) {
switch aerr.(type) {
case *types.BucketAlreadyOwnedByYou:
return nil
case s3.ErrCodeBucketAlreadyExists:
case *types.BucketAlreadyExists:
return nil
default:
return aerr
}
}
return s.s3API.WaitUntilBucketExistsWithContext(ctx, &s3.HeadBucketInput{Bucket: aws.String(s.bucket)})
return err
}
waiter := s3.NewBucketExistsWaiter(s.s3API)
return waiter.Wait(ctx, &s3.HeadBucketInput{Bucket: aws.String(s.bucket)}, 10*time.Minute)
}
func getStorage(t testing.TB) *Storage {
@@ -75,9 +82,9 @@ func getStorage(t testing.TB) *Storage {
}
options := func(conf *aws.Config) {
conf.Endpoint = aws.String(url)
conf.DisableSSL = aws.Bool(true)
conf.BaseEndpoint = aws.String(url)
}
backend, err := New(
&config.S3Config{
Key: "minio",
pkg/storage/s3/saver.go
+5 -4
View File
@@ -5,8 +5,8 @@ import (
"context"
"io"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/gomods/athens/pkg/errors"
"github.com/gomods/athens/pkg/observ"
moduploader "github.com/gomods/athens/pkg/storage/module"
@@ -31,14 +31,15 @@ func (s *Storage) upload(ctx context.Context, path, contentType string, stream i
const op errors.Op = "s3.upload"
ctx, span := observ.StartSpan(ctx, op.String())
defer span.End()
upParams := &s3manager.UploadInput{
upParams := &s3.PutObjectInput{
Bucket: aws.String(s.bucket),
Key: aws.String(path),
Body: stream,
ContentType: aws.String(contentType),
}
if _, err := s.uploader.UploadWithContext(ctx, upParams); err != nil {
if _, err := s.uploader.Upload(ctx, upParams); err != nil {
return errors.E(op, err)
}
scripts/ps/test_unit.ps1
+1 -1
View File
@@ -3,7 +3,7 @@
if (!(Test-Path env:GO_ENV)) {$env:GO_ENV = "test"}
if (!(Test-Path env:ATHENS_MINIO_ENDPOINT)) {
$env:ATHENS_MINIO_ENDPOINT = "127.0.0.1:9001"
$env:ATHENS_MINIO_ENDPOINT = "http://127.0.0.1:9001"
}
if (!(Test-Path env:ATHENS_MONGO_STORAGE_URL)) {
scripts/test_unit.sh
+1 -1
View File
@@ -7,7 +7,7 @@ if [ -z ${GO_ENV} ]; then
fi
if [ -z ${ATHENS_MINIO_ENDPOINT} ]; then
export ATHENS_MINIO_ENDPOINT="127.0.0.1:9001"
export ATHENS_MINIO_ENDPOINT="http://127.0.0.1:9001"
fi
if [ -z ${ATHENS_MONGO_STORAGE_URL} ]; then