From 4530a5835978c2804fcbf031b41a9d40a8adf679 Mon Sep 17 00:00:00 2001 From: vapod Date: Tue, 11 Feb 2020 03:03:25 +0300 Subject: [PATCH] Add forcepathstyle config for s3 (#1533) * Add forcepathstyle config for s3 * Bump chart version * Fix typos in s3 config struct --- charts/athens-proxy/Chart.yaml | 2 +- charts/athens-proxy/templates/deployment.yaml | 2 ++ config.dev.toml | 4 ++++ docs/content/configuration/storage.md | 4 ++++ pkg/config/config_test.go | 1 + pkg/config/s3.go | 3 ++- pkg/storage/s3/doc.go | 1 + pkg/storage/s3/s3.go | 2 ++ pkg/storage/s3/s3_test.go | 2 +- 9 files changed, 18 insertions(+), 3 deletions(-) diff --git a/charts/athens-proxy/Chart.yaml b/charts/athens-proxy/Chart.yaml index 4d9b9962..b16ae5a7 100644 --- a/charts/athens-proxy/Chart.yaml +++ b/charts/athens-proxy/Chart.yaml @@ -1,5 +1,5 @@ name: athens-proxy -version: 0.4.2 +version: 0.4.3 appVersion: 0.7.0 description: The proxy server for Go modules icon: https://raw.githubusercontent.com/gomods/athens/master/docs/static/banner.png diff --git a/charts/athens-proxy/templates/deployment.yaml b/charts/athens-proxy/templates/deployment.yaml index d3f931be..e2d5a384 100644 --- a/charts/athens-proxy/templates/deployment.yaml +++ b/charts/athens-proxy/templates/deployment.yaml @@ -86,6 +86,8 @@ spec: value: {{ .Values.storage.s3.bucket | quote }} - name: AWS_USE_DEFAULT_CONFIGURATION value: {{ .Values.storage.s3.useDefaultConfiguration | quote }} + - name: AWS_FORCE_PATH_STYLE + value: {{ .Values.storage.s3.ForcePathStyle | quote }} {{- if .Values.storage.s3.access_key_id }} - name: AWS_ACCESS_KEY_ID value: {{ .Values.storage.s3.access_key_id | quote }} diff --git a/config.dev.toml b/config.dev.toml index 38c28cc0..cdb1f29c 100755 --- a/config.dev.toml +++ b/config.dev.toml @@ -407,6 +407,10 @@ SingleFlightType = "memory" # Env override: ATHENS_S3_BUCKET_NAME Bucket = "MY_S3_BUCKET_NAME" + # If true then path style url for s3 endpoint will be used + # Env override: AWS_FORCE_PATH_STYLE + ForcePathStyle = false + # If true then the default aws configuration will be used. This will # attempt to find credentials in the environment, in the shared # configuration (~/.aws/credentials) and from ec2 instance role diff --git a/docs/content/configuration/storage.md b/docs/content/configuration/storage.md index 383151f7..c036989d 100644 --- a/docs/content/configuration/storage.md +++ b/docs/content/configuration/storage.md @@ -167,6 +167,10 @@ After this you can pass your credentials inside `config.toml` file. If the acce # S3 Bucket to use for storage # Env override: ATHENS_S3_BUCKET_NAME Bucket = "MY_S3_BUCKET_NAME" + + # If true then path style url for s3 endpoint will be used + # Env override: AWS_FORCE_PATH_STYLE + ForcePathStyle = false # If true then the default aws configuration will be used. This will # attempt to find credentials in the environment, in the shared diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 631c7df3..3c06a9da 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -362,6 +362,7 @@ func getEnvMap(config *Config) map[string]string { envVars["AWS_ACCESS_KEY_ID"] = storage.S3.Key envVars["AWS_SECRET_ACCESS_KEY"] = storage.S3.Secret envVars["AWS_SESSION_TOKEN"] = storage.S3.Token + envVars["AWS_FORCE_PATH_STYLE"] = strconv.FormatBool(storage.S3.ForcePathStyle) envVars["ATHENS_S3_BUCKET_NAME"] = storage.S3.Bucket } } diff --git a/pkg/config/s3.go b/pkg/config/s3.go index 82de1f7b..bc644408 100644 --- a/pkg/config/s3.go +++ b/pkg/config/s3.go @@ -8,7 +8,8 @@ type S3Config struct { Token string `envconfig:"AWS_SESSION_TOKEN"` Bucket string `validate:"required" envconfig:"ATHENS_S3_BUCKET_NAME"` UseDefaultConfiguration bool `envconfig:"AWS_USE_DEFAULT_CONFIGURATION"` + ForcePathStyle bool `envconfig:"AWS_FORCE_PATH_STYLE"` CredentialsEndpoint string `envconfig:"AWS_CREDENTIALS_ENDPOINT"` AwsContainerCredentialsRelativeURI string `envconfig:"AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"` - Endpoint string `evnconfig:"AWS_ENDPOINT"` + Endpoint string `envconfig:"AWS_ENDPOINT"` } diff --git a/pkg/storage/s3/doc.go b/pkg/storage/s3/doc.go index 2907c33e..03e66db8 100644 --- a/pkg/storage/s3/doc.go +++ b/pkg/storage/s3/doc.go @@ -9,6 +9,7 @@ Environment variables: AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN // [optional] + AWS_FORCE_PATH_STYLE // [optional] ATHENS_S3_BUCKET_NAME For information how to get your keyId and access key turn to official aws docs: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/setting-up.html diff --git a/pkg/storage/s3/s3.go b/pkg/storage/s3/s3.go index be05c02e..cfa80fb1 100644 --- a/pkg/storage/s3/s3.go +++ b/pkg/storage/s3/s3.go @@ -22,6 +22,7 @@ import ( // - AWS_ACCESS_KEY_ID - [optional] // - AWS_SECRET_ACCESS_KEY - [optional] // - AWS_SESSION_TOKEN - [optional] +// - AWS_FORCE_PATH_STYLE - [optional] // For information how to get your keyId and access key turn to official aws docs: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/setting-up.html type Storage struct { bucket string @@ -57,6 +58,7 @@ func New(s3Conf *config.S3Config, timeout time.Duration, options ...func(*aws.Co credProviders = append(endpointcreds, credProviders...) } + awsConfig.S3ForcePathStyle = aws.Bool(s3Conf.ForcePathStyle) awsConfig.Credentials = credentials.NewChainCredentials(credProviders) awsConfig.CredentialsChainVerboseErrors = aws.Bool(true) if s3Conf.Endpoint != "" { diff --git a/pkg/storage/s3/s3_test.go b/pkg/storage/s3/s3_test.go index 0a5ead97..f24b40e8 100644 --- a/pkg/storage/s3/s3_test.go +++ b/pkg/storage/s3/s3_test.go @@ -77,7 +77,6 @@ func getStorage(t testing.TB) *Storage { options := func(conf *aws.Config) { conf.Endpoint = aws.String(url) conf.DisableSSL = aws.Bool(true) - conf.S3ForcePathStyle = aws.Bool(true) } backend, err := New( &config.S3Config{ @@ -85,6 +84,7 @@ func getStorage(t testing.TB) *Storage { Secret: "minio123", Bucket: "gomodsaws", Region: "us-west-1", + ForcePathStyle: true, }, config.GetTimeoutDuration(300), options,