From 5b194688a0aa341e0beed2799d836c926b49314f Mon Sep 17 00:00:00 2001 From: Aleksandr Razumov Date: Fri, 6 Dec 2019 00:04:03 +0300 Subject: [PATCH] storage/s3: add custom endpoint url support (#1467) --- config.dev.toml | 8 +++++++- docs/content/configuration/storage.md | 7 +++++++ pkg/config/s3.go | 1 + pkg/storage/s3/s3.go | 3 +++ 4 files changed, 18 insertions(+), 1 deletion(-) diff --git a/config.dev.toml b/config.dev.toml index f2c72e4b..3699ee3e 100755 --- a/config.dev.toml +++ b/config.dev.toml @@ -361,7 +361,6 @@ SingleFlightType = "memory" Insecure = false [Storage.S3] - ### The authentication model is as below for S3 in the following order ### If AWS_CREDENTIALS_ENDPOINT is specified and it returns valid results, then it is used ### If config variables are specified and they are valid, then they return valid results, then it is used @@ -415,6 +414,13 @@ SingleFlightType = "memory" # Ref: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-metadata-endpoint-v2.html AwsContainerCredentialsRelativeURI = "" + # An optional endpoint URL (hostname only or fully qualified URI) + # that overrides the default generated endpoint for S3 storage client. + # + # You must still provide a `Region` value when specifying an endpoint. + # Env override: AWS_ENDPOINT + Endpoint = "" + [Storage.AzureBlob] # Storage Account name for Azure Blob # Env override: ATHENS_AZURE_ACCOUNT_NAME diff --git a/docs/content/configuration/storage.md b/docs/content/configuration/storage.md index 4a60c339..b60b109a 100644 --- a/docs/content/configuration/storage.md +++ b/docs/content/configuration/storage.md @@ -177,6 +177,13 @@ After this you can pass your credentials inside `config.toml` file. If the acce # Ref: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-metadata-endpoint-v2.html AwsContainerCredentialsRelativeURI = "" + # An optional endpoint URL (hostname only or fully qualified URI) + # that overrides the default generated endpoint for S3 storage client. + # + # You must still provide a `Region` value when specifying an endpoint. + # Env override: AWS_ENDPOINT + Endpoint = "" + ## Minio [Minio](https://www.minio.io/) is an open source object storage server that provides an interface for S3 compatible block storages. If you have never used minio, you can read this [quick start guide](https://docs.minio.io/). Any S3 compatible object storage is supported by Athens through the minio interface. Below, you can find different configuration options we provide for Minio. Example configuration for Digital Ocean and Alibaba OSS block storages are provided below. diff --git a/pkg/config/s3.go b/pkg/config/s3.go index baa0701a..82de1f7b 100644 --- a/pkg/config/s3.go +++ b/pkg/config/s3.go @@ -10,4 +10,5 @@ type S3Config struct { UseDefaultConfiguration bool `envconfig:"AWS_USE_DEFAULT_CONFIGURATION"` CredentialsEndpoint string `envconfig:"AWS_CREDENTIALS_ENDPOINT"` AwsContainerCredentialsRelativeURI string `envconfig:"AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"` + Endpoint string `evnconfig:"AWS_ENDPOINT"` } diff --git a/pkg/storage/s3/s3.go b/pkg/storage/s3/s3.go index d01b1436..be05c02e 100644 --- a/pkg/storage/s3/s3.go +++ b/pkg/storage/s3/s3.go @@ -59,6 +59,9 @@ func New(s3Conf *config.S3Config, timeout time.Duration, options ...func(*aws.Co awsConfig.Credentials = credentials.NewChainCredentials(credProviders) awsConfig.CredentialsChainVerboseErrors = aws.Bool(true) + if s3Conf.Endpoint != "" { + awsConfig.Endpoint = aws.String(s3Conf.Endpoint) + } // create a session with creds sess, err := session.NewSession(awsConfig)