primal/athens
1
0
Fork 0
mirror of https://github.com/gomods/athens synced 2026-02-03 12:10:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

Upgrade to AWS SDK v2 (#1938) (#1950)

Browse Source 
Upgrades the AWS SDK to v2. AWS S3 bucket urls will now error if they are not prefixed with a schema (example: https://).
This commit is contained in:
Vanes Angelo
2024-04-29 11:38:42 +07:00
committed by GitHub
parent c35b50f2f3
commit 900cb4f7c1
37 changed files with 274 additions and 177 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/ci.yml
+1 -1
View File
@@ -30,7 +30,7 @@ jobs:
build: build:
env: env:
ATHENS_MONGO_STORAGE_URL: mongodb://localhost:27017 ATHENS_MONGO_STORAGE_URL: mongodb://localhost:27017
ATHENS_MINIO_ENDPOINT: localhost:9000 ATHENS_MINIO_ENDPOINT: http://localhost:9000
REDIS_TEST_ENDPOINT: localhost:6379 REDIS_TEST_ENDPOINT: localhost:6379
REDIS_SENTINEL_TEST_ENDPOINT: localhost:26379 REDIS_SENTINEL_TEST_ENDPOINT: localhost:26379
REDIS_SENTINEL_TEST_MASTER_NAME: redis-1 REDIS_SENTINEL_TEST_MASTER_NAME: redis-1
cmd/proxy/actions/app_proxy_test.go
+1 -2
View File
@@ -48,7 +48,7 @@ func TestProxyRoutes(t *testing.T) {
tmp, err := template.New("home").Parse(homepage) tmp, err := template.New("home").Parse(homepage)
assert.NoError(t, err) assert.NoError(t, err)
var templateData = make(map[string]string) templateData := make(map[string]string)
templateData["Host"] = req.Host templateData["Host"] = req.Host
@@ -109,5 +109,4 @@ func TestProxyRoutes(t *testing.T) {
tc.test(t, req, w.Result()) tc.test(t, req, w.Result())
}) })
} }
} }
cmd/proxy/actions/index_test.go
-1
View File
@@ -107,7 +107,6 @@ func TestIndexHandler(t *testing.T) {
if w.Code != tc.code { if w.Code != tc.code {
t.Fatalf("expected response code to be %d but got %d", tc.code, w.Code) t.Fatalf("expected response code to be %d but got %d", tc.code, w.Code)
} }
}) })
} }
} }
cmd/proxy/actions/storage.go
+1 -1
View File
@@ -6,7 +6,7 @@ import (
"net/http" "net/http"
"time" "time"
"github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go-v2/aws"
"github.com/gomods/athens/pkg/config" "github.com/gomods/athens/pkg/config"
"github.com/gomods/athens/pkg/errors" "github.com/gomods/athens/pkg/errors"
"github.com/gomods/athens/pkg/storage" "github.com/gomods/athens/pkg/storage"
docker-compose.yml
+1 -1
View File
@@ -24,7 +24,7 @@ services:
command: ["./scripts/test_unit.sh"] command: ["./scripts/test_unit.sh"]
environment: environment:
- GO_ENV=test - GO_ENV=test
- ATHENS_MINIO_ENDPOINT=minio:9000 - ATHENS_MINIO_ENDPOINT=http://minio:9000
- ATHENS_MONGO_STORAGE_URL=mongodb://mongo:27017 - ATHENS_MONGO_STORAGE_URL=mongodb://mongo:27017
- TIMEOUT=20 # in case the mongo dependency takes longer to start up - TIMEOUT=20 # in case the mongo dependency takes longer to start up
- ATHENS_STORAGE_TYPE=mongo - ATHENS_STORAGE_TYPE=mongo
e2etests/all_test.go
+3 -3
View File
@@ -62,9 +62,9 @@ func (m *E2eSuite) SetupSuite() {
func (m *E2eSuite) TearDownSuite() { func (m *E2eSuite) TearDownSuite() {
m.stopAthens() m.stopAthens()
chmodR(m.goPath, 0777) chmodR(m.goPath, 0o777)
os.RemoveAll(m.goPath) os.RemoveAll(m.goPath)
chmodR(m.sampleRepoPath, 0777) chmodR(m.sampleRepoPath, 0o777)
os.RemoveAll(m.sampleRepoPath) os.RemoveAll(m.sampleRepoPath)
} }
@@ -73,7 +73,7 @@ func TestE2E(t *testing.T) {
} }
func (m *E2eSuite) SetupTest() { func (m *E2eSuite) SetupTest() {
chmodR(m.goPath, 0777) chmodR(m.goPath, 0o777)
err := cleanGoCache(m.getEnv()) err := cleanGoCache(m.getEnv())
if err != nil { if err != nil {
m.Fail("Failed to clear go cache", err) m.Fail("Failed to clear go cache", err)
go.mod
+20 -1
View File
@@ -10,7 +10,12 @@ require (
github.com/Azure/azure-storage-blob-go v0.10.0 github.com/Azure/azure-storage-blob-go v0.10.0
github.com/BurntSushi/toml v1.0.0 github.com/BurntSushi/toml v1.0.0
github.com/DataDog/opencensus-go-exporter-datadog v0.0.0-20180917103902-e6c7f767dc57 github.com/DataDog/opencensus-go-exporter-datadog v0.0.0-20180917103902-e6c7f767dc57
github.com/aws/aws-sdk-go v1.51.21 github.com/aws/aws-sdk-go-v2 v1.26.1
github.com/aws/aws-sdk-go-v2/config v1.27.11
github.com/aws/aws-sdk-go-v2/credentials v1.17.11
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.15
github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1
github.com/aws/smithy-go v1.20.2
github.com/bsm/redislock v0.7.2 github.com/bsm/redislock v0.7.2
github.com/fatih/color v1.13.0 github.com/fatih/color v1.13.0
github.com/go-playground/validator/v10 v10.19.0 github.com/go-playground/validator/v10 v10.19.0
@@ -55,6 +60,20 @@ require (
github.com/agext/levenshtein v1.2.1 // indirect github.com/agext/levenshtein v1.2.1 // indirect
github.com/ajg/form v0.0.0-20160822230020-523a5da1a92f // indirect github.com/ajg/form v0.0.0-20160822230020-523a5da1a92f // indirect
github.com/apparentlymart/go-textseg v1.0.0 // indirect github.com/apparentlymart/go-textseg v1.0.0 // indirect
github.com/aws/aws-sdk-go v1.44.220 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 // indirect
github.com/beorn7/perks v1.0.1 // indirect github.com/beorn7/perks v1.0.1 // indirect
github.com/cenkalti/backoff/v4 v4.1.1 // indirect github.com/cenkalti/backoff/v4 v4.1.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect
go.sum
+46 -2
View File
@@ -97,8 +97,46 @@ github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kd
github.com/apparentlymart/go-dump v0.0.0-20180507223929-23540a00eaa3/go.mod h1:oL81AME2rN47vu18xqj1S1jPIPuN7afo62yKTNn3XMM= github.com/apparentlymart/go-dump v0.0.0-20180507223929-23540a00eaa3/go.mod h1:oL81AME2rN47vu18xqj1S1jPIPuN7afo62yKTNn3XMM=
github.com/apparentlymart/go-textseg v1.0.0 h1:rRmlIsPEEhUTIKQb7T++Nz/A5Q6C9IuX2wFoYVvnCs0= github.com/apparentlymart/go-textseg v1.0.0 h1:rRmlIsPEEhUTIKQb7T++Nz/A5Q6C9IuX2wFoYVvnCs0=
github.com/apparentlymart/go-textseg v1.0.0/go.mod h1:z96Txxhf3xSFMPmb5X/1W05FF/Nj9VFpLOpjS5yuumk= github.com/apparentlymart/go-textseg v1.0.0/go.mod h1:z96Txxhf3xSFMPmb5X/1W05FF/Nj9VFpLOpjS5yuumk=
github.com/aws/aws-sdk-go v1.51.21 h1:UrT6JC9R9PkYYXDZBV0qDKTualMr+bfK2eboTknMgbs= github.com/aws/aws-sdk-go v1.44.220 h1:yAj99qAt0Htjle9Up3DglgHfOP77lmFPrElA4jKnrBo=
github.com/aws/aws-sdk-go v1.51.21/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/aws/aws-sdk-go v1.44.220/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
github.com/aws/aws-sdk-go-v2 v1.26.1 h1:5554eUqIYVWpU0YmeeYZ0wU64H2VLBs8TlhRB2L+EkA=
github.com/aws/aws-sdk-go-v2 v1.26.1/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 h1:x6xsQXGSmW6frevwDA+vi/wqhp1ct18mVXYN08/93to=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2/go.mod h1:lPprDr1e6cJdyYeGXnRaJoP4Md+cDBvi2eOj00BlGmg=
github.com/aws/aws-sdk-go-v2/config v1.27.11 h1:f47rANd2LQEYHda2ddSCKYId18/8BhSRM4BULGmfgNA=
github.com/aws/aws-sdk-go-v2/config v1.27.11/go.mod h1:SMsV78RIOYdve1vf36z8LmnszlRWkwMQtomCAI0/mIE=
github.com/aws/aws-sdk-go-v2/credentials v1.17.11 h1:YuIB1dJNf1Re822rriUOTxopaHHvIq0l/pX3fwO+Tzs=
github.com/aws/aws-sdk-go-v2/credentials v1.17.11/go.mod h1:AQtFPsDH9bI2O+71anW6EKL+NcD7LG3dpKGMV4SShgo=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 h1:FVJ0r5XTHSmIHJV6KuDmdYhEpvlHpiSd38RQWhut5J4=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1/go.mod h1:zusuAeqezXzAB24LGuzuekqMAEgWkVYukBec3kr3jUg=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.15 h1:7Zwtt/lP3KNRkeZre7soMELMGNoBrutx8nobg1jKWmo=
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.15/go.mod h1:436h2adoHb57yd+8W+gYPrrA9U/R/SuAuOO42Ushzhw=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 h1:aw39xVGeRWlWx9EzGVnhOR4yOjQDHPQ6o6NmBlscyQg=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5/go.mod h1:FSaRudD0dXiMPK2UjknVwwTYyZMRsHv3TtkabsZih5I=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 h1:PG1F3OD1szkuQPzDw3CIQsRIrtTlUC3lP84taWzHlq0=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5/go.mod h1:jU1li6RFryMz+so64PpKtudI+QzbKoIEivqdf6LNpOc=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5 h1:81KE7vaZzrl7yHBYHVEzYB8sypz11NMOZ40YlWvPxsU=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5/go.mod h1:LIt2rg7Mcgn09Ygbdh/RdIm0rQ+3BNkbP1gyVMFtRK0=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 h1:Ji0DY1xUsUr3I8cHps0G+XM3WWU16lP6yG8qu1GAZAs=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2/go.mod h1:5CsjAbs3NlGQyZNFACh+zztPDI7fU6eW9QsxjfnuBKg=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7 h1:ZMeFZ5yk+Ek+jNr1+uwCd2tG89t6oTS5yVWpa6yy2es=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7/go.mod h1:mxV05U+4JiHqIpGqqYXOHLPKUC6bDXC44bsUhNjOEwY=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 h1:ogRAwT1/gxJBcSWDMZlgyFUM962F51A5CRhDLbxLdmo=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7/go.mod h1:YCsIZhXfRPLFFCl5xxY+1T9RKzOKjCut+28JSX2DnAk=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5 h1:f9RyWNtS8oH7cZlbn+/JNPpjUk5+5fLd5lM9M0i49Ys=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5/go.mod h1:h5CoMZV2VF297/VLhRhO1WF+XYWOzXo+4HsObA4HjBQ=
github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1 h1:6cnno47Me9bRykw9AEv9zkXE+5or7jz8TsskTTccbgc=
github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1/go.mod h1:qmdkIIAC+GCLASF7R2whgNrJADz0QZPX+Seiw/i4S3o=
github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 h1:vN8hEbpRnL7+Hopy9dzmRle1xmDc7o8tmY0klsr175w=
github.com/aws/aws-sdk-go-v2/service/sso v1.20.5/go.mod h1:qGzynb/msuZIE8I75DVRCUXw3o3ZyBmUvMwQ2t/BrGM=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 h1:Jux+gDDyi1Lruk+KHF91tK2KCuY61kzoCpvtvJJBtOE=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4/go.mod h1:mUYPBhaF2lGiukDEjJX2BLRRKTmoUSitGDUgM4tRxak=
github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 h1:cwIxeBttqPN3qkaAjcEcsh8NYr8n2HZPkcKgPAi1phU=
github.com/aws/aws-sdk-go-v2/service/sts v1.28.6/go.mod h1:FZf1/nKNEkHdGGJP/cI2MoIMquumuRK6ol3QQJNDxmw=
github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q=
github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E=
github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
@@ -649,6 +687,9 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b
golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc=
golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -732,10 +773,12 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -745,6 +788,7 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
pkg/config/azureblob.go
+3 -3
View File
@@ -2,7 +2,7 @@ package config
// AzureBlobConfig specifies the properties required to use Azure as the storage backend. // AzureBlobConfig specifies the properties required to use Azure as the storage backend.
type AzureBlobConfig struct { type AzureBlobConfig struct {
AccountName string `validate:"required" envconfig:"ATHENS_AZURE_ACCOUNT_NAME"` AccountName string `envconfig:"ATHENS_AZURE_ACCOUNT_NAME" validate:"required"`
AccountKey string `validate:"required" envconfig:"ATHENS_AZURE_ACCOUNT_KEY"` AccountKey string `envconfig:"ATHENS_AZURE_ACCOUNT_KEY" validate:"required"`
ContainerName string `validate:"required" envconfig:"ATHENS_AZURE_CONTAINER_NAME"` ContainerName string `envconfig:"ATHENS_AZURE_CONTAINER_NAME" validate:"required"`
} }
pkg/config/config.go
+10 -10
View File
@@ -21,22 +21,22 @@ const defaultConfigFile = "athens.toml"
// Config provides configuration values for all components. // Config provides configuration values for all components.
type Config struct { type Config struct {
TimeoutConf TimeoutConf
GoEnv string `validate:"required" envconfig:"GO_ENV"` GoEnv string `envconfig:"GO_ENV" validate:"required"`
GoBinary string `validate:"required" envconfig:"GO_BINARY_PATH"` GoBinary string `envconfig:"GO_BINARY_PATH" validate:"required"`
GoBinaryEnvVars EnvList `envconfig:"ATHENS_GO_BINARY_ENV_VARS"` GoBinaryEnvVars EnvList `envconfig:"ATHENS_GO_BINARY_ENV_VARS"`
GoGetWorkers int `validate:"required" envconfig:"ATHENS_GOGET_WORKERS"` GoGetWorkers int `envconfig:"ATHENS_GOGET_WORKERS" validate:"required"`
GoGetDir string `envconfig:"ATHENS_GOGET_DIR"` GoGetDir string `envconfig:"ATHENS_GOGET_DIR"`
ProtocolWorkers int `validate:"required" envconfig:"ATHENS_PROTOCOL_WORKERS"` ProtocolWorkers int `envconfig:"ATHENS_PROTOCOL_WORKERS" validate:"required"`
LogLevel string `validate:"required" envconfig:"ATHENS_LOG_LEVEL"` LogLevel string `envconfig:"ATHENS_LOG_LEVEL" validate:"required"`
LogFormat string `validate:"oneof='' 'json' 'plain'" envconfig:"ATHENS_LOG_FORMAT"` LogFormat string `envconfig:"ATHENS_LOG_FORMAT" validate:"oneof='' 'json' 'plain'"`
CloudRuntime string `validate:"required_without=LogFormat" envconfig:"ATHENS_CLOUD_RUNTIME"` CloudRuntime string `envconfig:"ATHENS_CLOUD_RUNTIME" validate:"required_without=LogFormat"`
EnablePprof bool `envconfig:"ATHENS_ENABLE_PPROF"` EnablePprof bool `envconfig:"ATHENS_ENABLE_PPROF"`
PprofPort string `envconfig:"ATHENS_PPROF_PORT"` PprofPort string `envconfig:"ATHENS_PPROF_PORT"`
FilterFile string `envconfig:"ATHENS_FILTER_FILE"` FilterFile string `envconfig:"ATHENS_FILTER_FILE"`
TraceExporterURL string `envconfig:"ATHENS_TRACE_EXPORTER_URL"` TraceExporterURL string `envconfig:"ATHENS_TRACE_EXPORTER_URL"`
TraceExporter string `envconfig:"ATHENS_TRACE_EXPORTER"` TraceExporter string `envconfig:"ATHENS_TRACE_EXPORTER"`
StatsExporter string `envconfig:"ATHENS_STATS_EXPORTER"` StatsExporter string `envconfig:"ATHENS_STATS_EXPORTER"`
StorageType string `validate:"required" envconfig:"ATHENS_STORAGE_TYPE"` StorageType string `envconfig:"ATHENS_STORAGE_TYPE" validate:"required"`
GlobalEndpoint string `envconfig:"ATHENS_GLOBAL_ENDPOINT"` // This feature is not yet implemented GlobalEndpoint string `envconfig:"ATHENS_GLOBAL_ENDPOINT"` // This feature is not yet implemented
Port string `envconfig:"ATHENS_PORT"` Port string `envconfig:"ATHENS_PORT"`
UnixSocket string `envconfig:"ATHENS_UNIX_SOCKET"` UnixSocket string `envconfig:"ATHENS_UNIX_SOCKET"`
@@ -55,11 +55,11 @@ type Config struct {
NoSumPatterns []string `envconfig:"ATHENS_GONOSUM_PATTERNS"` NoSumPatterns []string `envconfig:"ATHENS_GONOSUM_PATTERNS"`
DownloadMode mode.Mode `envconfig:"ATHENS_DOWNLOAD_MODE"` DownloadMode mode.Mode `envconfig:"ATHENS_DOWNLOAD_MODE"`
DownloadURL string `envconfig:"ATHENS_DOWNLOAD_URL"` DownloadURL string `envconfig:"ATHENS_DOWNLOAD_URL"`
NetworkMode string `validate:"oneof=strict offline fallback" envconfig:"ATHENS_NETWORK_MODE"` NetworkMode string `envconfig:"ATHENS_NETWORK_MODE" validate:"oneof=strict offline fallback"`
SingleFlightType string `envconfig:"ATHENS_SINGLE_FLIGHT_TYPE"` SingleFlightType string `envconfig:"ATHENS_SINGLE_FLIGHT_TYPE"`
RobotsFile string `envconfig:"ATHENS_ROBOTS_FILE"` RobotsFile string `envconfig:"ATHENS_ROBOTS_FILE"`
IndexType string `envconfig:"ATHENS_INDEX_TYPE"` IndexType string `envconfig:"ATHENS_INDEX_TYPE"`
ShutdownTimeout int `validate:"min=0" envconfig:"ATHENS_SHUTDOWN_TIMEOUT"` ShutdownTimeout int `envconfig:"ATHENS_SHUTDOWN_TIMEOUT" validate:"min=0"`
SingleFlight *SingleFlight SingleFlight *SingleFlight
Storage *Storage Storage *Storage
Index *Index Index *Index
pkg/config/config_test.go
+4 -4
View File
@@ -16,7 +16,7 @@ import (
func testConfigFile(t *testing.T) (testConfigFile string) { func testConfigFile(t *testing.T) (testConfigFile string) {
testConfigFile = filepath.Join("..", "..", "config.dev.toml") testConfigFile = filepath.Join("..", "..", "config.dev.toml")
if err := os.Chmod(testConfigFile, 0700); err != nil { if err := os.Chmod(testConfigFile, 0o700); err != nil {
t.Fatalf("%s\n", err) t.Fatalf("%s\n", err)
} }
return testConfigFile return testConfigFile
@@ -108,6 +108,7 @@ func TestEnvOverrides(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("Env override failed: %v", err) t.Fatalf("Env override failed: %v", err)
} }
compareConfigs(conf, expConf, t, Storage{}, SingleFlight{}) compareConfigs(conf, expConf, t, Storage{}, SingleFlight{})
} }
@@ -305,7 +306,6 @@ func TestParseExampleConfig(t *testing.T) {
} }
func getEnvMap(config *Config) map[string]string { func getEnvMap(config *Config) map[string]string {
envVars := map[string]string{ envVars := map[string]string{
"GO_ENV": config.GoEnv, "GO_ENV": config.GoEnv,
"GO_BINARY_PATH": config.GoBinary, "GO_BINARY_PATH": config.GoBinary,
@@ -414,7 +414,7 @@ func Test_checkFilePerms(t *testing.T) {
} }
incorrectPerms := []os.FileMode{0o777, 0o610, 0o660} incorrectPerms := []os.FileMode{0o777, 0o610, 0o660}
var incorrectFiles = make([]string, len(incorrectPerms)) incorrectFiles := make([]string, len(incorrectPerms))
for i := range incorrectPerms { for i := range incorrectPerms {
f, err := tempFile(incorrectPerms[i]) f, err := tempFile(incorrectPerms[i])
@@ -426,7 +426,7 @@ func Test_checkFilePerms(t *testing.T) {
} }
correctPerms := []os.FileMode{0o600, 0o400, 0o644} correctPerms := []os.FileMode{0o600, 0o400, 0o644}
var correctFiles = make([]string, len(correctPerms)) correctFiles := make([]string, len(correctPerms))
for i := range correctPerms { for i := range correctPerms {
f, err := tempFile(correctPerms[i]) f, err := tempFile(correctPerms[i])
pkg/config/disk.go
+1 -1
View File
@@ -2,5 +2,5 @@ package config
// DiskConfig specifies the properties required to use Disk as the storage backend. // DiskConfig specifies the properties required to use Disk as the storage backend.
type DiskConfig struct { type DiskConfig struct {
RootPath string `validate:"required" envconfig:"ATHENS_DISK_STORAGE_ROOT"` RootPath string `envconfig:"ATHENS_DISK_STORAGE_ROOT" validate:"required"`
} }
pkg/config/external.go
+1 -1
View File
@@ -2,5 +2,5 @@ package config
// External specifies configuration for an external http storage. // External specifies configuration for an external http storage.
type External struct { type External struct {
URL string `validate:"required" envconfig:"ATHENS_EXTERNAL_STORAGE_URL"` URL string `envconfig:"ATHENS_EXTERNAL_STORAGE_URL" validate:"required"`
} }
pkg/config/gcp.go
+1 -1
View File
@@ -3,6 +3,6 @@ package config
// GCPConfig specifies the properties required to use GCP as the storage backend. // GCPConfig specifies the properties required to use GCP as the storage backend.
type GCPConfig struct { type GCPConfig struct {
ProjectID string `envconfig:"GOOGLE_CLOUD_PROJECT"` ProjectID string `envconfig:"GOOGLE_CLOUD_PROJECT"`
Bucket string `validate:"required" envconfig:"ATHENS_STORAGE_GCP_BUCKET"` Bucket string `envconfig:"ATHENS_STORAGE_GCP_BUCKET" validate:"required"`
JSONKey string `envconfig:"ATHENS_STORAGE_GCP_JSON_KEY"` JSONKey string `envconfig:"ATHENS_STORAGE_GCP_JSON_KEY"`
} }
pkg/config/minio.go
+4 -4
View File
@@ -3,10 +3,10 @@ package config
// MinioConfig specifies the properties required to use Minio or DigitalOcean Spaces // MinioConfig specifies the properties required to use Minio or DigitalOcean Spaces
// as the storage backend. // as the storage backend.
type MinioConfig struct { type MinioConfig struct {
Endpoint string `validate:"required" envconfig:"ATHENS_MINIO_ENDPOINT"` Endpoint string `envconfig:"ATHENS_MINIO_ENDPOINT" validate:"required"`
Key string `validate:"required" envconfig:"ATHENS_MINIO_ACCESS_KEY_ID"` Key string `envconfig:"ATHENS_MINIO_ACCESS_KEY_ID" validate:"required"`
Secret string `validate:"required" envconfig:"ATHENS_MINIO_SECRET_ACCESS_KEY"` Secret string `envconfig:"ATHENS_MINIO_SECRET_ACCESS_KEY" validate:"required"`
Bucket string `validate:"required" envconfig:"ATHENS_MINIO_BUCKET_NAME"` Bucket string `envconfig:"ATHENS_MINIO_BUCKET_NAME" validate:"required"`
Region string `envconfig:"ATHENS_MINIO_REGION"` Region string `envconfig:"ATHENS_MINIO_REGION"`
EnableSSL bool `envconfig:"ATHENS_MINIO_USE_SSL"` EnableSSL bool `envconfig:"ATHENS_MINIO_USE_SSL"`
} }
pkg/config/mongo.go
+3 -3
View File
@@ -2,9 +2,9 @@ package config
// MongoConfig specifies the properties required to use MongoDB as the storage backend. // MongoConfig specifies the properties required to use MongoDB as the storage backend.
type MongoConfig struct { type MongoConfig struct {
URL string `validate:"required" envconfig:"ATHENS_MONGO_STORAGE_URL"` URL string `envconfig:"ATHENS_MONGO_STORAGE_URL" validate:"required"`
DefaultDBName string `envconfig:"ATHENS_MONGO_DEFAULT_DATABASE" default:"athens"` DefaultDBName string `default:"athens" envconfig:"ATHENS_MONGO_DEFAULT_DATABASE"`
DefaultCollectionName string `envconfig:"ATHENS_MONGO_DEFAULT_COLLECTION" default:"modules"` DefaultCollectionName string `default:"modules" envconfig:"ATHENS_MONGO_DEFAULT_COLLECTION"`
CertPath string `envconfig:"ATHENS_MONGO_CERT_PATH"` CertPath string `envconfig:"ATHENS_MONGO_CERT_PATH"`
InsecureConn bool `envconfig:"ATHENS_MONGO_INSECURE"` InsecureConn bool `envconfig:"ATHENS_MONGO_INSECURE"`
} }
pkg/config/mysql.go
+7 -7
View File
@@ -2,11 +2,11 @@ package config
// MySQL config. // MySQL config.
type MySQL struct { type MySQL struct {
Protocol string `validate:"required" envconfig:"ATHENS_INDEX_MYSQL_PROTOCOL"` Protocol string `envconfig:"ATHENS_INDEX_MYSQL_PROTOCOL" validate:"required"`
Host string `validate:"required" envconfig:"ATHENS_INDEX_MYSQL_HOST"` Host string `envconfig:"ATHENS_INDEX_MYSQL_HOST" validate:"required"`
Port int `validate:"" envconfig:"ATHENS_INDEX_MYSQL_PORT"` Port int `envconfig:"ATHENS_INDEX_MYSQL_PORT" validate:""`
User string `validate:"required" envconfig:"ATHENS_INDEX_MYSQL_USER"` User string `envconfig:"ATHENS_INDEX_MYSQL_USER" validate:"required"`
Password string `validate:"" envconfig:"ATHENS_INDEX_MYSQL_PASSWORD"` Password string `envconfig:"ATHENS_INDEX_MYSQL_PASSWORD" validate:""`
Database string `validate:"required" envconfig:"ATHENS_INDEX_MYSQL_DATABASE"` Database string `envconfig:"ATHENS_INDEX_MYSQL_DATABASE" validate:"required"`
Params map[string]string `validate:"required" envconfig:"ATHENS_INDEX_MYSQL_PARAMS"` Params map[string]string `envconfig:"ATHENS_INDEX_MYSQL_PARAMS" validate:"required"`
} }
pkg/config/postgres.go
+6 -6
View File
@@ -2,10 +2,10 @@ package config
// Postgres config. // Postgres config.
type Postgres struct { type Postgres struct {
Host string `validate:"required" envconfig:"ATHENS_INDEX_POSTGRES_HOST"` Host string `envconfig:"ATHENS_INDEX_POSTGRES_HOST" validate:"required"`
Port int `validate:"required" envconfig:"ATHENS_INDEX_POSTGRES_PORT"` Port int `envconfig:"ATHENS_INDEX_POSTGRES_PORT" validate:"required"`
User string `validate:"required" envconfig:"ATHENS_INDEX_POSTGRES_USER"` User string `envconfig:"ATHENS_INDEX_POSTGRES_USER" validate:"required"`
Password string `validate:"" envconfig:"ATHENS_INDEX_POSTGRES_PASSWORD"` Password string `envconfig:"ATHENS_INDEX_POSTGRES_PASSWORD" validate:""`
Database string `validate:"required" envconfig:"ATHENS_INDEX_POSTGRES_DATABASE"` Database string `envconfig:"ATHENS_INDEX_POSTGRES_DATABASE" validate:"required"`
Params map[string]string `validate:"required" envconfig:"ATHENS_INDEX_POSTGRES_PARAMS"` Params map[string]string `envconfig:"ATHENS_INDEX_POSTGRES_PARAMS" validate:"required"`
} }
pkg/config/s3.go
+2 -2
View File
@@ -2,11 +2,11 @@ package config
// S3Config specifies the properties required to use S3 as the storage backend. // S3Config specifies the properties required to use S3 as the storage backend.
type S3Config struct { type S3Config struct {
Region string `validate:"required" envconfig:"AWS_REGION"` Region string `envconfig:"AWS_REGION" validate:"required"`
Key string `envconfig:"AWS_ACCESS_KEY_ID"` Key string `envconfig:"AWS_ACCESS_KEY_ID"`
Secret string `envconfig:"AWS_SECRET_ACCESS_KEY"` Secret string `envconfig:"AWS_SECRET_ACCESS_KEY"`
Token string `envconfig:"AWS_SESSION_TOKEN"` Token string `envconfig:"AWS_SESSION_TOKEN"`
Bucket string `validate:"required" envconfig:"ATHENS_S3_BUCKET_NAME"` Bucket string `envconfig:"ATHENS_S3_BUCKET_NAME" validate:"required"`
UseDefaultConfiguration bool `envconfig:"AWS_USE_DEFAULT_CONFIGURATION"` UseDefaultConfiguration bool `envconfig:"AWS_USE_DEFAULT_CONFIGURATION"`
ForcePathStyle bool `envconfig:"AWS_FORCE_PATH_STYLE"` ForcePathStyle bool `envconfig:"AWS_FORCE_PATH_STYLE"`
CredentialsEndpoint string `envconfig:"AWS_CREDENTIALS_ENDPOINT"` CredentialsEndpoint string `envconfig:"AWS_CREDENTIALS_ENDPOINT"`
pkg/download/list_merge_test.go
+4 -2
View File
@@ -13,8 +13,10 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
const testOp athenserr.Op = "vcsLister.List" const (
const testModName = "happy tags" testOp athenserr.Op = "vcsLister.List"
testModName = "happy tags"
)
type listMergeTest struct { type listMergeTest struct {
name string name string
pkg/download/mode/mode_test.go
+2 -1
View File
@@ -84,7 +84,8 @@ var testCases = []struct {
{ {
Pattern: "github.com/gomods/*", Pattern: "github.com/gomods/*",
Mode: AsyncRedirect, Mode: AsyncRedirect,
DownloadURL: "gomods.io"}, DownloadURL: "gomods.io",
},
}, },
}, },
input: "github.com/gomods/athens", input: "github.com/gomods/athens",
pkg/download/protocol_test.go
+2 -4
View File
@@ -28,9 +28,7 @@ import (
"golang.org/x/sync/errgroup" "golang.org/x/sync/errgroup"
) )
var ( var testConfigPath = filepath.Join("..", "..", "config.dev.toml")
testConfigPath = filepath.Join("..", "..", "config.dev.toml")
)
func getDP(t *testing.T) Protocol { func getDP(t *testing.T) Protocol {
t.Helper() t.Helper()
@@ -235,7 +233,7 @@ var latestTests = []latestTest{
path: "github.com/athens-artifacts/happy-path", path: "github.com/athens-artifacts/happy-path",
info: &storage.RevInfo{ info: &storage.RevInfo{
Version: "v0.0.3", Version: "v0.0.3",
Time: time.Date(2018, 8, 3, 17, 16, 00, 0, time.UTC), Time: time.Date(2018, 8, 3, 17, 16, 0o0, 0, time.UTC),
}, },
}, },
} }
pkg/middleware/middleware_test.go
+1 -1
View File
@@ -23,7 +23,7 @@ const (
func testConfigFile(t *testing.T) (testConfigFile string) { func testConfigFile(t *testing.T) (testConfigFile string) {
testConfigFile = filepath.Join("..", "..", "config.dev.toml") testConfigFile = filepath.Join("..", "..", "config.dev.toml")
if err := os.Chmod(testConfigFile, 0700); err != nil { if err := os.Chmod(testConfigFile, 0o700); err != nil {
t.Fatalf("%s\n", err) t.Fatalf("%s\n", err)
} }
return testConfigFile return testConfigFile
pkg/module/filter_test.go
+4 -6
View File
@@ -10,7 +10,7 @@ import (
func testConfigFile(t *testing.T) (testConfigFile string) { func testConfigFile(t *testing.T) (testConfigFile string) {
testConfigFile = filepath.Join("..", "..", "config.dev.toml") testConfigFile = filepath.Join("..", "..", "config.dev.toml")
if err := os.Chmod(testConfigFile, 0700); err != nil { if err := os.Chmod(testConfigFile, 0o700); err != nil {
t.Fatalf("%s\n", err) t.Fatalf("%s\n", err)
} }
return testConfigFile return testConfigFile
@@ -40,7 +40,6 @@ func (t *FilterTests) Test_NewFilter() {
mf, err = NewFilter(filter) mf, err = NewFilter(filter)
r.Equal(filter, mf.filePath) r.Equal(filter, mf.filePath)
r.NoError(err) r.NoError(err)
} }
func (t *FilterTests) Test_IgnoreSimple() { func (t *FilterTests) Test_IgnoreSimple() {
@@ -198,7 +197,6 @@ func (t *FilterTests) Test_versionFilterRobust() {
r.Equal(Exclude, f.Rule("github.com/a/b", "a")) r.Equal(Exclude, f.Rule("github.com/a/b", "a"))
r.Equal(Exclude, f.Rule("github.com/c/d", "fg")) r.Equal(Exclude, f.Rule("github.com/c/d", "fg"))
} }
func (t *FilterTests) Test_initFromConfig() { func (t *FilterTests) Test_initFromConfig() {
@@ -207,20 +205,20 @@ func (t *FilterTests) Test_initFromConfig() {
defer os.Remove(filterFile) defer os.Remove(filterFile)
goodInput := []byte("+ github.com/a/b\n\n# some comment\n- github.com/c/d\n\nD github.com/x") goodInput := []byte("+ github.com/a/b\n\n# some comment\n- github.com/c/d\n\nD github.com/x")
os.WriteFile(filterFile, goodInput, 0644) os.WriteFile(filterFile, goodInput, 0o644)
f, err := initFromConfig(filterFile) f, err := initFromConfig(filterFile)
r.NotNil(f) r.NotNil(f)
r.NoError(err) r.NoError(err)
badInput := []byte("+ github.com/a/b\n\n# some comment\n\n- github.com/c/d\n\nD github.com/x\nsome_random_line") badInput := []byte("+ github.com/a/b\n\n# some comment\n\n- github.com/c/d\n\nD github.com/x\nsome_random_line")
os.WriteFile(filterFile, badInput, 0644) os.WriteFile(filterFile, badInput, 0o644)
f, err = initFromConfig(filterFile) f, err = initFromConfig(filterFile)
r.Nil(f) r.Nil(f)
r.Error(err) r.Error(err)
versionInput := []byte("+ github.com/a/b\n\n# some comment\n\n- github.com/c/d v1,v2.3.4,v3.2.*\n\nD github.com/x\n") versionInput := []byte("+ github.com/a/b\n\n# some comment\n\n- github.com/c/d v1,v2.3.4,v3.2.*\n\nD github.com/x\n")
os.WriteFile(filterFile, versionInput, 0644) os.WriteFile(filterFile, versionInput, 0o644)
f, err = initFromConfig(filterFile) f, err = initFromConfig(filterFile)
r.NotNil(f) r.NotNil(f)
r.NoError(err) r.NoError(err)
pkg/storage/minio/minio.go
+11 -1
View File
@@ -2,6 +2,7 @@ package minio
import ( import (
"fmt" "fmt"
"strings"
"time" "time"
"github.com/gomods/athens/pkg/config" "github.com/gomods/athens/pkg/config"
@@ -24,7 +25,7 @@ func (s *storageImpl) versionLocation(module, version string) string {
// that implements storage.Backend. // that implements storage.Backend.
func NewStorage(conf *config.MinioConfig, timeout time.Duration) (storage.Backend, error) { func NewStorage(conf *config.MinioConfig, timeout time.Duration) (storage.Backend, error) {
const op errors.Op = "minio.NewStorage" const op errors.Op = "minio.NewStorage"
endpoint := conf.Endpoint endpoint := TrimHTTP(conf.Endpoint)
accessKeyID := conf.Key accessKeyID := conf.Key
secretAccessKey := conf.Secret secretAccessKey := conf.Secret
bucketName := conf.Bucket bucketName := conf.Bucket
@@ -53,3 +54,12 @@ func NewStorage(conf *config.MinioConfig, timeout time.Duration) (storage.Backen
} }
return &storageImpl{minioClient, minioCore, bucketName}, nil return &storageImpl{minioClient, minioCore, bucketName}, nil
} }
// TrimHTTP trims "http://" or "https://" prefix from input string.
// Minio doesn't need to specify protocol in the URL so it should be trimmed.
// Related issue: https://github.com/gomods/athens/issues/1938#issuecomment-2067590653
func TrimHTTP(s string) string {
s = strings.TrimPrefix(s, "http://")
s = strings.TrimPrefix(s, "https://")
return s
}
pkg/storage/minio/minio_test.go
+3 -3
View File
@@ -15,7 +15,7 @@ func TestBackend(t *testing.T) {
// TestNewStorageExists tests the logic around MakeBucket and BucketExists // TestNewStorageExists tests the logic around MakeBucket and BucketExists
func TestNewStorageExists(t *testing.T) { func TestNewStorageExists(t *testing.T) {
url := os.Getenv("ATHENS_MINIO_ENDPOINT") url := TrimHTTP(os.Getenv("ATHENS_MINIO_ENDPOINT"))
if url == "" { if url == "" {
t.SkipNow() t.SkipNow()
} }
@@ -51,7 +51,7 @@ func TestNewStorageExists(t *testing.T) {
// To ensure both paths are tested, there is a strict path error using the // To ensure both paths are tested, there is a strict path error using the
// "_" and a non strict error using less than 3 characters // "_" and a non strict error using less than 3 characters
func TestNewStorageError(t *testing.T) { func TestNewStorageError(t *testing.T) {
url := os.Getenv("ATHENS_MINIO_ENDPOINT") url := TrimHTTP(os.Getenv("ATHENS_MINIO_ENDPOINT"))
if url == "" { if url == "" {
t.SkipNow() t.SkipNow()
} }
@@ -92,7 +92,7 @@ func (s *storageImpl) clear() error {
} }
func getStorage(t testing.TB) *storageImpl { func getStorage(t testing.TB) *storageImpl {
url := os.Getenv("ATHENS_MINIO_ENDPOINT") url := TrimHTTP(os.Getenv("ATHENS_MINIO_ENDPOINT"))
if url == "" { if url == "" {
t.SkipNow() t.SkipNow()
} }
pkg/storage/mongo/mongo_test.go
+5 -3
View File
@@ -3,13 +3,14 @@ package mongo
import ( import (
"bytes" "bytes"
"context" "context"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"io" "io"
"os" "os"
"testing" "testing"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
"github.com/gomods/athens/pkg/config" "github.com/gomods/athens/pkg/config"
"github.com/gomods/athens/pkg/errors" "github.com/gomods/athens/pkg/errors"
"github.com/gomods/athens/pkg/storage" "github.com/gomods/athens/pkg/storage"
@@ -145,6 +146,7 @@ func TestQueryKindUnexpectedErrorCases(t *testing.T) {
require.Equal(t, errors.KindUnexpected, errors.Kind(err)) require.Equal(t, errors.KindUnexpected, errors.Kind(err))
} }
} }
func TestNewStorageWithDefaultOverrides(t *testing.T) { func TestNewStorageWithDefaultOverrides(t *testing.T) {
url := os.Getenv("ATHENS_MONGO_STORAGE_URL") url := os.Getenv("ATHENS_MONGO_STORAGE_URL")
pkg/storage/s3/cataloger.go
+6 -5
View File
@@ -5,8 +5,9 @@ import (
"fmt" "fmt"
"strings" "strings"
"github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/gomods/athens/pkg/config" "github.com/gomods/athens/pkg/config"
"github.com/gomods/athens/pkg/errors" "github.com/gomods/athens/pkg/errors"
"github.com/gomods/athens/pkg/observ" "github.com/gomods/athens/pkg/observ"
@@ -28,7 +29,7 @@ func (s *Storage) Catalog(ctx context.Context, token string, pageSize int) ([]pa
Marker: &queryToken, Marker: &queryToken,
} }
loo, err := s.s3API.ListObjectsWithContext(ctx, lsParams) loo, err := s.s3API.ListObjects(ctx, lsParams)
if err != nil { if err != nil {
return nil, "", errors.E(op, err) return nil, "", errors.E(op, err)
} }
@@ -49,7 +50,7 @@ func (s *Storage) Catalog(ctx context.Context, token string, pageSize int) ([]pa
return res, queryToken, nil return res, queryToken, nil
} }
func fetchModsAndVersions(objects []*s3.Object, elementsNum int) ([]paths.AllPathParams, string) { func fetchModsAndVersions(objects []types.Object, elementsNum int) ([]paths.AllPathParams, string) {
res := make([]paths.AllPathParams, 0) res := make([]paths.AllPathParams, 0)
lastKey := "" lastKey := ""
for _, o := range objects { for _, o := range objects {
@@ -72,7 +73,7 @@ func fetchModsAndVersions(objects []*s3.Object, elementsNum int) ([]paths.AllPat
return res, lastKey return res, lastKey
} }
func parseS3Key(o *s3.Object) (paths.AllPathParams, error) { func parseS3Key(o types.Object) (paths.AllPathParams, error) {
const op errors.Op = "s3.parseS3Key" const op errors.Op = "s3.parseS3Key"
m, v := config.ModuleVersionFromPath(*o.Key) m, v := config.ModuleVersionFromPath(*o.Key)
pkg/storage/s3/checker.go
+6 -6
View File
@@ -4,9 +4,9 @@ import (
"context" "context"
"sync" "sync"
"github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go/service/s3" "github.com/aws/smithy-go"
"github.com/gomods/athens/pkg/config" "github.com/gomods/athens/pkg/config"
"github.com/gomods/athens/pkg/errors" "github.com/gomods/athens/pkg/errors"
"github.com/gomods/athens/pkg/observ" "github.com/gomods/athens/pkg/observ"
@@ -28,7 +28,7 @@ func (s *Storage) Exists(ctx context.Context, module, version string) (bool, err
wg.Add(1) wg.Add(1)
go func(file string) { go func(file string) {
defer wg.Done() defer wg.Done()
_, err := s.s3API.HeadObjectWithContext( _, err := s.s3API.HeadObject(
cancelingCtx, cancelingCtx,
&s3.HeadObjectInput{ &s3.HeadObjectInput{
Bucket: aws.String(s.bucket), Bucket: aws.String(s.bucket),
@@ -44,8 +44,8 @@ func (s *Storage) Exists(ctx context.Context, module, version string) (bool, err
if err == nil { if err == nil {
continue continue
} }
var aerr awserr.Error var aerr smithy.APIError
if errors.AsErr(err, &aerr) && aerr.Code() == "NotFound" { if errors.AsErr(err, &aerr) && aerr.ErrorCode() == "NotFound" {
err = nil err = nil
exists = false exists = false
} }
pkg/storage/s3/deleter.go
+4 -3
View File
@@ -3,8 +3,8 @@ package s3
import ( import (
"context" "context"
"github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/gomods/athens/pkg/errors" "github.com/gomods/athens/pkg/errors"
"github.com/gomods/athens/pkg/observ" "github.com/gomods/athens/pkg/observ"
modupl "github.com/gomods/athens/pkg/storage/module" modupl "github.com/gomods/athens/pkg/storage/module"
@@ -32,12 +32,13 @@ func (s *Storage) remove(ctx context.Context, path string) error {
const op errors.Op = "s3.Delete" const op errors.Op = "s3.Delete"
ctx, span := observ.StartSpan(ctx, op.String()) ctx, span := observ.StartSpan(ctx, op.String())
defer span.End() defer span.End()
delParams := &s3.DeleteObjectInput{ delParams := &s3.DeleteObjectInput{
Bucket: aws.String(s.bucket), Bucket: aws.String(s.bucket),
Key: aws.String(path), Key: aws.String(path),
} }
if _, err := s.s3API.DeleteObjectWithContext(ctx, delParams); err != nil { if _, err := s.s3API.DeleteObject(ctx, delParams); err != nil {
return errors.E(op, err) return errors.E(op, err)
} }
pkg/storage/s3/getter.go
+12 -12
View File
@@ -5,9 +5,9 @@ import (
"fmt" "fmt"
"io" "io"
"github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/gomods/athens/pkg/config" "github.com/gomods/athens/pkg/config"
"github.com/gomods/athens/pkg/errors" "github.com/gomods/athens/pkg/errors"
"github.com/gomods/athens/pkg/observ" "github.com/gomods/athens/pkg/observ"
@@ -22,8 +22,8 @@ func (s *Storage) Info(ctx context.Context, module, version string) ([]byte, err
infoReader, err := s.open(ctx, config.PackageVersionedName(module, version, "info")) infoReader, err := s.open(ctx, config.PackageVersionedName(module, version, "info"))
if err != nil { if err != nil {
var aerr awserr.Error var nsk *types.NoSuchKey
if errors.AsErr(err, &aerr) && aerr.Code() == s3.ErrCodeNoSuchKey { if errors.AsErr(err, &nsk) {
return nil, errors.E(op, errors.M(module), errors.V(version), errors.KindNotFound) return nil, errors.E(op, errors.M(module), errors.V(version), errors.KindNotFound)
} }
return nil, errors.E(op, err, errors.M(module), errors.V(version)) return nil, errors.E(op, err, errors.M(module), errors.V(version))
@@ -45,8 +45,8 @@ func (s *Storage) GoMod(ctx context.Context, module, version string) ([]byte, er
modReader, err := s.open(ctx, config.PackageVersionedName(module, version, "mod")) modReader, err := s.open(ctx, config.PackageVersionedName(module, version, "mod"))
if err != nil { if err != nil {
var aerr awserr.Error var nsk *types.NoSuchKey
if errors.AsErr(err, &aerr) && aerr.Code() == s3.ErrCodeNoSuchKey { if errors.AsErr(err, &nsk) {
return nil, errors.E(op, errors.M(module), errors.V(version), errors.KindNotFound) return nil, errors.E(op, errors.M(module), errors.V(version), errors.KindNotFound)
} }
return nil, errors.E(op, err, errors.M(module), errors.V(version)) return nil, errors.E(op, err, errors.M(module), errors.V(version))
@@ -69,8 +69,8 @@ func (s *Storage) Zip(ctx context.Context, module, version string) (storage.Size
zipReader, err := s.open(ctx, config.PackageVersionedName(module, version, "zip")) zipReader, err := s.open(ctx, config.PackageVersionedName(module, version, "zip"))
if err != nil { if err != nil {
var aerr awserr.Error var nsk *types.NoSuchKey
if errors.AsErr(err, &aerr) && aerr.Code() == s3.ErrCodeNoSuchKey { if errors.AsErr(err, &nsk) {
return nil, errors.E(op, errors.M(module), errors.V(version), errors.KindNotFound) return nil, errors.E(op, errors.M(module), errors.V(version), errors.KindNotFound)
} }
return nil, errors.E(op, err, errors.M(module), errors.V(version)) return nil, errors.E(op, err, errors.M(module), errors.V(version))
@@ -88,10 +88,10 @@ func (s *Storage) open(ctx context.Context, path string) (storage.SizeReadCloser
Key: aws.String(path), Key: aws.String(path),
} }
goo, err := s.s3API.GetObjectWithContext(ctx, getParams) goo, err := s.s3API.GetObject(ctx, getParams)
if err != nil { if err != nil {
var aerr awserr.Error var nsk *types.NoSuchKey
if errors.AsErr(err, &aerr) && aerr.Code() == s3.ErrCodeNoSuchKey { if errors.AsErr(err, &nsk) {
return nil, errors.E(op, errors.KindNotFound) return nil, errors.E(op, errors.KindNotFound)
} }
return nil, errors.E(op, err) return nil, errors.E(op, err)
pkg/storage/s3/lister.go
+5 -4
View File
@@ -4,8 +4,9 @@ import (
"context" "context"
"strings" "strings"
"github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/gomods/athens/pkg/errors" "github.com/gomods/athens/pkg/errors"
"github.com/gomods/athens/pkg/observ" "github.com/gomods/athens/pkg/observ"
) )
@@ -23,7 +24,7 @@ func (s *Storage) List(ctx context.Context, module string) ([]string, error) {
Prefix: aws.String(modulePrefix), Prefix: aws.String(modulePrefix),
} }
loo, err := s.s3API.ListObjectsWithContext(ctx, lsParams) loo, err := s.s3API.ListObjects(ctx, lsParams)
if err != nil { if err != nil {
return nil, errors.E(op, err, errors.M(module)) return nil, errors.E(op, err, errors.M(module))
} }
@@ -31,7 +32,7 @@ func (s *Storage) List(ctx context.Context, module string) ([]string, error) {
return extractVersions(loo.Contents), nil return extractVersions(loo.Contents), nil
} }
func extractVersions(objects []*s3.Object) []string { func extractVersions(objects []types.Object) []string {
var versions []string var versions []string
for _, o := range objects { for _, o := range objects {
pkg/storage/s3/s3.go
+60 -46
View File
@@ -1,16 +1,16 @@
package s3 package s3
import ( import (
"context"
"fmt"
"time" "time"
"github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go/aws/credentials" awscfg "github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go/aws/credentials/endpointcreds" "github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go/aws/defaults" "github.com/aws/aws-sdk-go-v2/credentials/endpointcreds"
"github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go-v2/feature/s3/manager"
"github.com/aws/aws-sdk-go/service/s3/s3iface" "github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"github.com/aws/aws-sdk-go/service/s3/s3manager/s3manageriface"
"github.com/gomods/athens/pkg/config" "github.com/gomods/athens/pkg/config"
"github.com/gomods/athens/pkg/errors" "github.com/gomods/athens/pkg/errors"
) )
@@ -26,8 +26,8 @@ import (
// For information how to get your keyId and access key turn to official aws docs: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/setting-up.html. // For information how to get your keyId and access key turn to official aws docs: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/setting-up.html.
type Storage struct { type Storage struct {
bucket string bucket string
uploader s3manageriface.UploaderAPI uploader *manager.Uploader
s3API s3iface.S3API s3API *s3.Client
timeout time.Duration timeout time.Duration
} }
@@ -35,50 +35,44 @@ type Storage struct {
func New(s3Conf *config.S3Config, timeout time.Duration, options ...func(*aws.Config)) (*Storage, error) { func New(s3Conf *config.S3Config, timeout time.Duration, options ...func(*aws.Config)) (*Storage, error) {
const op errors.Op = "s3.New" const op errors.Op = "s3.New"
awsConfig := defaults.Config() awsConfig, err := awscfg.LoadDefaultConfig(context.TODO(), awscfg.WithRegion(s3Conf.Region))
// Remove anonymous credentials from the default config so that
// session.NewSession can auto-resolve credentials from role, profile, env etc.
awsConfig.Credentials = nil
awsConfig.Region = aws.String(s3Conf.Region)
for _, o := range options {
o(awsConfig)
}
if !s3Conf.UseDefaultConfiguration {
credProviders := defaults.CredProviders(awsConfig, defaults.Handlers())
endpointcreds := []credentials.Provider{
endpointcreds.NewProviderClient(*awsConfig, defaults.Handlers(), endpointFrom(s3Conf.CredentialsEndpoint, s3Conf.AwsContainerCredentialsRelativeURI)),
&credentials.StaticProvider{
Value: credentials.Value{
AccessKeyID: s3Conf.Key,
SecretAccessKey: s3Conf.Secret,
SessionToken: s3Conf.Token,
},
},
}
credProviders = append(endpointcreds, credProviders...)
awsConfig.Credentials = credentials.NewChainCredentials(credProviders)
}
awsConfig.S3ForcePathStyle = aws.Bool(s3Conf.ForcePathStyle)
awsConfig.CredentialsChainVerboseErrors = aws.Bool(true)
if s3Conf.Endpoint != "" {
awsConfig.Endpoint = aws.String(s3Conf.Endpoint)
}
// Create a session with creds.
sess, err := session.NewSession(awsConfig)
if err != nil { if err != nil {
return nil, errors.E(op, err) return nil, errors.E(op, err)
} }
uploader := s3manager.NewUploader(sess) // Remove anonymous credentials from the default config so that
// session.NewSession can auto-resolve credentials from role, profile, env etc.
awsConfig.Credentials = nil
for _, o := range options {
o(&awsConfig)
}
if !s3Conf.UseDefaultConfiguration {
// credProviders := defaults.CredProviders(awsConfig, defaults.Handlers())
endpointCreds := []aws.CredentialsProvider{
endpointcreds.New(endpointFrom(s3Conf.CredentialsEndpoint, s3Conf.AwsContainerCredentialsRelativeURI)),
credentials.NewStaticCredentialsProvider(s3Conf.Key, s3Conf.Secret, s3Conf.Token),
}
// credProviders = append(endpointCreds, credProviders...)
awsConfig.Credentials = newChainCredentials(endpointCreds...)
}
// Create a session with creds.
sess := s3.NewFromConfig(awsConfig, func(o *s3.Options) {
o.UsePathStyle = s3Conf.ForcePathStyle
if s3Conf.Endpoint != "" {
o.BaseEndpoint = aws.String(s3Conf.Endpoint)
}
})
uploader := manager.NewUploader(sess)
return &Storage{ return &Storage{
bucket: s3Conf.Bucket, bucket: s3Conf.Bucket,
uploader: uploader, uploader: uploader,
s3API: uploader.S3, s3API: sess,
timeout: timeout, timeout: timeout,
}, nil }, nil
} }
@@ -86,3 +80,23 @@ func New(s3Conf *config.S3Config, timeout time.Duration, options ...func(*aws.Co
func endpointFrom(credentialsEndpoint, relativeURI string) string { func endpointFrom(credentialsEndpoint, relativeURI string) string {
return credentialsEndpoint + relativeURI return credentialsEndpoint + relativeURI
} }
// newChainCredentials is based on old credentials.NewChainCredentials in v1.
func newChainCredentials(providers ...aws.CredentialsProvider) aws.CredentialsProvider {
return aws.NewCredentialsCache(
aws.CredentialsProviderFunc(func(ctx context.Context) (aws.Credentials, error) {
var errs []error
for _, p := range providers {
creds, err := p.Retrieve(ctx)
if err == nil {
return creds, nil
}
errs = append(errs, err)
}
return aws.Credentials{}, fmt.Errorf("no valid providers in chain: %s", errs)
}),
)
}
pkg/storage/s3/s3_test.go
+27 -20
View File
@@ -4,10 +4,14 @@ import (
"context" "context"
"os" "os"
"testing" "testing"
"time"
"github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/smithy-go"
"github.com/aws/aws-sdk-go/service/s3" "github.com/gomods/athens/pkg/errors"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/gomods/athens/pkg/config" "github.com/gomods/athens/pkg/config"
"github.com/gomods/athens/pkg/storage/compliance" "github.com/gomods/athens/pkg/storage/compliance"
) )
@@ -26,7 +30,7 @@ func (s *Storage) clear() error {
ctx, cancel := context.WithTimeout(context.Background(), s.timeout) ctx, cancel := context.WithTimeout(context.Background(), s.timeout)
defer cancel() defer cancel()
objects, err := s.s3API.ListObjectsWithContext(ctx, &s3.ListObjectsInput{Bucket: aws.String(s.bucket)}) objects, err := s.s3API.ListObjects(ctx, &s3.ListObjectsInput{Bucket: aws.String(s.bucket)})
if err != nil { if err != nil {
return err return err
} }
@@ -37,7 +41,7 @@ func (s *Storage) clear() error {
Key: o.Key, Key: o.Key,
} }
_, err := s.s3API.DeleteObjectWithContext(ctx, delParams) _, err := s.s3API.DeleteObject(ctx, delParams)
if err != nil { if err != nil {
return err return err
} }
@@ -49,23 +53,26 @@ func (s *Storage) createBucket() error {
ctx, cancel := context.WithTimeout(context.Background(), s.timeout) ctx, cancel := context.WithTimeout(context.Background(), s.timeout)
defer cancel() defer cancel()
if _, err := s.s3API.CreateBucketWithContext(ctx, &s3.CreateBucketInput{Bucket: aws.String(s.bucket)}); err != nil { if _, err := s.s3API.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: aws.String(s.bucket)}); err != nil {
aerr, ok := err.(awserr.Error) var aerr smithy.APIError
if !ok {
return err if errors.AsErr(err, &aerr) {
switch aerr.(type) {
case *types.BucketAlreadyOwnedByYou:
return nil
case *types.BucketAlreadyExists:
return nil
default:
return aerr
}
} }
switch aerr.Code() { return err
case s3.ErrCodeBucketAlreadyOwnedByYou:
return nil
case s3.ErrCodeBucketAlreadyExists:
return nil
default:
return aerr
}
} }
return s.s3API.WaitUntilBucketExistsWithContext(ctx, &s3.HeadBucketInput{Bucket: aws.String(s.bucket)}) waiter := s3.NewBucketExistsWaiter(s.s3API)
return waiter.Wait(ctx, &s3.HeadBucketInput{Bucket: aws.String(s.bucket)}, 10*time.Minute)
} }
func getStorage(t testing.TB) *Storage { func getStorage(t testing.TB) *Storage {
@@ -75,9 +82,9 @@ func getStorage(t testing.TB) *Storage {
} }
options := func(conf *aws.Config) { options := func(conf *aws.Config) {
conf.Endpoint = aws.String(url) conf.BaseEndpoint = aws.String(url)
conf.DisableSSL = aws.Bool(true)
} }
backend, err := New( backend, err := New(
&config.S3Config{ &config.S3Config{
Key: "minio", Key: "minio",
pkg/storage/s3/saver.go
+5 -4
View File
@@ -5,8 +5,8 @@ import (
"context" "context"
"io" "io"
"github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go/service/s3/s3manager" "github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/gomods/athens/pkg/errors" "github.com/gomods/athens/pkg/errors"
"github.com/gomods/athens/pkg/observ" "github.com/gomods/athens/pkg/observ"
moduploader "github.com/gomods/athens/pkg/storage/module" moduploader "github.com/gomods/athens/pkg/storage/module"
@@ -31,14 +31,15 @@ func (s *Storage) upload(ctx context.Context, path, contentType string, stream i
const op errors.Op = "s3.upload" const op errors.Op = "s3.upload"
ctx, span := observ.StartSpan(ctx, op.String()) ctx, span := observ.StartSpan(ctx, op.String())
defer span.End() defer span.End()
upParams := &s3manager.UploadInput{
upParams := &s3.PutObjectInput{
Bucket: aws.String(s.bucket), Bucket: aws.String(s.bucket),
Key: aws.String(path), Key: aws.String(path),
Body: stream, Body: stream,
ContentType: aws.String(contentType), ContentType: aws.String(contentType),
} }
if _, err := s.uploader.UploadWithContext(ctx, upParams); err != nil { if _, err := s.uploader.Upload(ctx, upParams); err != nil {
return errors.E(op, err) return errors.E(op, err)
} }
scripts/ps/test_unit.ps1
+1 -1
View File
@@ -3,7 +3,7 @@
if (!(Test-Path env:GO_ENV)) {$env:GO_ENV = "test"} if (!(Test-Path env:GO_ENV)) {$env:GO_ENV = "test"}
if (!(Test-Path env:ATHENS_MINIO_ENDPOINT)) { if (!(Test-Path env:ATHENS_MINIO_ENDPOINT)) {
$env:ATHENS_MINIO_ENDPOINT = "127.0.0.1:9001" $env:ATHENS_MINIO_ENDPOINT = "http://127.0.0.1:9001"
} }
if (!(Test-Path env:ATHENS_MONGO_STORAGE_URL)) { if (!(Test-Path env:ATHENS_MONGO_STORAGE_URL)) {
scripts/test_unit.sh
+1 -1
View File
@@ -7,7 +7,7 @@ if [ -z ${GO_ENV} ]; then
fi fi
if [ -z ${ATHENS_MINIO_ENDPOINT} ]; then if [ -z ${ATHENS_MINIO_ENDPOINT} ]; then
export ATHENS_MINIO_ENDPOINT="127.0.0.1:9001" export ATHENS_MINIO_ENDPOINT="http://127.0.0.1:9001"
fi fi
if [ -z ${ATHENS_MONGO_STORAGE_URL} ]; then if [ -z ${ATHENS_MONGO_STORAGE_URL} ]; then