storage/s3: add custom endpoint url support (#1467)

config.dev.toml

@@ -361,7 +361,6 @@ SingleFlightType = "memory"
         Insecure = false
 
     [Storage.S3]
-
         ### The authentication model is as below for S3 in the following order
         ### If AWS_CREDENTIALS_ENDPOINT is specified and it returns valid results, then it is used
         ### If config variables are specified and they are valid, then they return valid results, then it is used
@@ -415,6 +414,13 @@ SingleFlightType = "memory"
         # Ref: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-metadata-endpoint-v2.html
         AwsContainerCredentialsRelativeURI = ""
 
+        # An optional endpoint URL (hostname only or fully qualified URI)
+        # that overrides the default generated endpoint for S3 storage client.
+        #
+        # You must still provide a `Region` value when specifying an endpoint.
+        # Env override: AWS_ENDPOINT
+        Endpoint = ""
+
    [Storage.AzureBlob]
         # Storage Account name for Azure Blob
         # Env override: ATHENS_AZURE_ACCOUNT_NAME

docs/content/configuration/storage.md

@@ -177,6 +177,13 @@ After this you can pass your credentials inside `config.toml` file.  If the acce
             # Ref: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-metadata-endpoint-v2.html
             AwsContainerCredentialsRelativeURI = ""
 
+            # An optional endpoint URL (hostname only or fully qualified URI)
+            # that overrides the default generated endpoint for S3 storage client.
+            #
+            # You must still provide a `Region` value when specifying an endpoint.
+            # Env override: AWS_ENDPOINT
+            Endpoint = ""
+
 ## Minio
 
 [Minio](https://www.minio.io/) is an open source object storage server that provides an interface for S3 compatible block storages. If you have never used minio, you can read this [quick start guide](https://docs.minio.io/).  Any S3 compatible object storage is supported by Athens through the minio interface. Below, you can find different configuration options we provide for Minio. Example configuration for Digital Ocean and Alibaba OSS block storages are provided below.

pkg/config/s3.go

@@ -10,4 +10,5 @@ type S3Config struct {
 	UseDefaultConfiguration            bool   `envconfig:"AWS_USE_DEFAULT_CONFIGURATION"`
 	CredentialsEndpoint                string `envconfig:"AWS_CREDENTIALS_ENDPOINT"`
 	AwsContainerCredentialsRelativeURI string `envconfig:"AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"`
+	Endpoint                           string `evnconfig:"AWS_ENDPOINT"`
 }

pkg/storage/s3/s3.go

@@ -59,6 +59,9 @@ func New(s3Conf *config.S3Config, timeout time.Duration, options ...func(*aws.Co
 
 	awsConfig.Credentials = credentials.NewChainCredentials(credProviders)
 	awsConfig.CredentialsChainVerboseErrors = aws.Bool(true)
+	if s3Conf.Endpoint != "" {
+		awsConfig.Endpoint = aws.String(s3Conf.Endpoint)
+	}
 
 	// create a session with creds
 	sess, err := session.NewSession(awsConfig)