endpoint creds (#1291)

* endpoint creds

* gofmt

* Add credential chain

* error handling

* Fix things

* add docs

* f

* asd

* fix

config.dev.toml

@@ -328,6 +328,21 @@ SingleFlightType = "memory"
         Insecure = false
 
     [Storage.S3]
+
+        ### The authentication model is as below for S3 in the following order
+        ### If AWS_CREDENTIALS_ENDPOINT is specified and it returns valid results, then it is used
+        ### If config variables are specified and they are valid, then they return valid results, then it is used
+        ### Otherwise, it will default to default configurations which is as follows
+        # attempt to find credentials in the environment, in the shared
+        # configuration (~/.aws/credentials) and from ec2 instance role
+        # credentials. See
+        # https://godoc.org/github.com/aws/aws-sdk-go#hdr-Configuring_Credentials
+        # and
+        # https://godoc.org/github.com/aws/aws-sdk-go/aws/session#hdr-Environment_Variables
+        # for environment variables that will affect the aws configuration.
+        # Setting UseDefaultConfiguration would only use default configuration. It will be deprecated in future releases 
+        # and is recommended not to use it.
+
         # Region for S3 storage
         # Env override: AWS_REGION
         Region = "MY_AWS_REGION"
@@ -357,6 +372,10 @@ SingleFlightType = "memory"
         # https://godoc.org/github.com/aws/aws-sdk-go/aws/session#hdr-Environment_Variables
         # for environment variables that will affect the aws configuration.
         UseDefaultConfiguration = false
+
+        # https://docs.aws.amazon.com/sdk-for-go/api/aws/credentials/endpointcreds/
+        CredentialsEndpoint = ""
+
    [Storage.AzureBlob]
         # Storage Account name for Azure Blob
         # Env override: ATHENS_AZURE_ACCOUNT_NAME

pkg/config/s3.go

@@ -8,4 +8,5 @@ type S3Config struct {
 	Token                   string `envconfig:"AWS_SESSION_TOKEN"`
 	Bucket                  string `validate:"required" envconfig:"ATHENS_S3_BUCKET_NAME"`
 	UseDefaultConfiguration bool   `envconfig:"AWS_USE_DEFAULT_CONFIGURATION"`
+	CredentialsEndpoint     string `envconfig:"AWS_CREDENTIALS_ENDPOINT"`
 }

pkg/storage/s3/s3.go

@@ -5,6 +5,8 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/credentials/endpointcreds"
+	"github.com/aws/aws-sdk-go/aws/defaults"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3/s3iface"
 	"github.com/aws/aws-sdk-go/service/s3/s3manager"
@@ -32,22 +34,38 @@ type Storage struct {
 func New(s3Conf *config.S3Config, timeout time.Duration, options ...func(*aws.Config)) (*Storage, error) {
 	const op errors.Op = "s3.New"
 
-	creds := buildAWSCredentials(s3Conf)
+	awsConfig := aws.NewConfig()
-
+	awsConfig.Region = aws.String(s3Conf.Region)
-	awsConfig := &aws.Config{
-		Credentials: creds,
-		Region:      aws.String(s3Conf.Region),
-	}
-
 	for _, o := range options {
 		o(awsConfig)
 	}
 
-	// create a session
+	credProviders := defaults.CredProviders(awsConfig, defaults.Handlers())
+
+	if !s3Conf.UseDefaultConfiguration {
+		endpointcreds := []credentials.Provider{
+			endpointcreds.NewProviderClient(*awsConfig, defaults.Handlers(), s3Conf.CredentialsEndpoint),
+			&credentials.StaticProvider{
+				Value: credentials.Value{
+					AccessKeyID:     s3Conf.Key,
+					SecretAccessKey: s3Conf.Secret,
+					SessionToken:    s3Conf.Token,
+				},
+			},
+		}
+
+		credProviders = append(endpointcreds, credProviders...)
+	}
+
+	awsConfig.Credentials = credentials.NewChainCredentials(credProviders)
+	awsConfig.CredentialsChainVerboseErrors = aws.Bool(true)
+
+	// create a session with creds
 	sess, err := session.NewSession(awsConfig)
 	if err != nil {
 		return nil, errors.E(op, err)
 	}
+
 	uploader := s3manager.NewUploader(sess)
 
 	return &Storage{
@@ -57,21 +75,3 @@ func New(s3Conf *config.S3Config, timeout time.Duration, options ...func(*aws.Co
 		timeout:  timeout,
 	}, nil
 }
-
-// buildAWSCredentials builds the credentials required to create a new AWS
-// session.  It will prefer the access key ID and secret access key if specified
-// in the S3Config unless UseDefaultConfiguration is true. If the key ID and
-// secret access key are unspecified or UseDefaultConfiguration is true, then
-// the default aws configuration will be used. This will attempt to find
-// credentials in the environment, in the shared configuration
-// (~/.aws/credentials) and from ec2 instance role credentials. See
-// https://godoc.org/github.com/aws/aws-sdk-go#hdr-Configuring_Credentials and
-// https://godoc.org/github.com/aws/aws-sdk-go/aws/session#hdr-Environment_Variables
-// for environment variables that will affect the aws configuration.
-func buildAWSCredentials(s3Conf *config.S3Config) *credentials.Credentials {
-	if !s3Conf.UseDefaultConfiguration && s3Conf.Key != "" && s3Conf.Secret != "" {
-		return credentials.NewStaticCredentials(s3Conf.Key, s3Conf.Secret, s3Conf.Token)
-	}
-
-	return nil
-}