diff --git a/charts/athens-proxy/templates/deployment.yaml b/charts/athens-proxy/templates/deployment.yaml index c421531f..953200ff 100644 --- a/charts/athens-proxy/templates/deployment.yaml +++ b/charts/athens-proxy/templates/deployment.yaml @@ -95,6 +95,15 @@ spec: - name: AWS_SESSION_TOKEN value: {{ .Values.storage.s3.session_token | quote }} {{- end }} + {{- else if eq .Values.storage.type "gcp"}} + - name: GOOGLE_CLOUD_PROJECT + value: {{ .Values.storage.gcp.projectID | quote }} + - name: ATHENS_STORAGE_GCP_BUCKET + value: {{ .Values.storage.gcp.bucket | quote }} + {{- if .Values.storage.gcp.serviceAccount }} + - name: ATHENS_STORAGE_GCP_SERVICE_ACCOUNT + value: {{ .Values.storage.gcp.serviceAccount | b64enc | quote }} + {{- end }} {{- else if eq .Values.storage.type "minio" }} {{- if .Values.storage.minio.endpoint }} - name: ATHENS_MINIO_ENDPOINT diff --git a/charts/athens-proxy/values.yaml b/charts/athens-proxy/values.yaml index 3ca5cd0e..98a4a7df 100644 --- a/charts/athens-proxy/values.yaml +++ b/charts/athens-proxy/values.yaml @@ -64,6 +64,16 @@ storage: accessKey: "" secretKey: "" bucket: "" + gcp: + # For more information, see: + # https://docs.gomods.io/install/install-on-kubernetes/#google-cloud-storage + # you must set gcp projectID and bucket when running 'helm install' + projectID: "" + bucket: "" + # set serviceAccount to a key which has read/write access to the GCS bucket. + # If you are running Athens inside GCP, you will most likely not need this + # as GCP figures out internal authentication between products for you. + serviceAccount: "" # Extra environment variables to be passed # You can add any new ones at the bottom diff --git a/config.dev.toml b/config.dev.toml index fb1ef96d..d0b6e96c 100755 --- a/config.dev.toml +++ b/config.dev.toml @@ -23,16 +23,16 @@ GoEnv = "development" # Env override: GOPROXY GoProxy = "direct" -# GoBinaryEnvVars are environment variables that you'd like -# to pass directly to the Go command that Athens runs under the -# hood. Athens primarily runs two Go commands: +# GoBinaryEnvVars are environment variables that you'd like +# to pass directly to the Go command that Athens runs under the +# hood. Athens primarily runs two Go commands: # 1. `go mod download -json @` # 2. `go list -m -json @latest` -# The go command accepts multiple environment variables that -# can affect the two processes above such as GONOSUMDB and GOPROXY. +# The go command accepts multiple environment variables that +# can affect the two processes above such as GONOSUMDB and GOPROXY. # Note that athens passes the NoSumPatterns to the two commands above, # but the existence of GONOSUMDB in this configuration takes precedence. -# Although you can pass any key=value to the Go command here, you can see +# Although you can pass any key=value to the Go command here, you can see # the list of possible env vars by running `go env`. # Env override: ATHENS_GO_BINARY_ENV_VARS GoBinaryEnvVars = ["GOPROXY=direct"] @@ -42,7 +42,7 @@ GoBinaryEnvVars = ["GOPROXY=direct"] # can manage go get more sanely and not run out of disk or memory. # One way to think about is: the number of GoGetWorkers corresponds # directly to how many parallel "git clones" your underlying machine -# can handle. +# can handle. # Env override: ATHENS_GOGET_WORKERS GoGetWorkers = 10 @@ -78,7 +78,7 @@ PprofPort = ":3001" # # To point Athens to an upstream proxy to fetch modules, # set GlobalEndpoint to "https://" -# and also ensure that FilterFile is set to a fully qualified file name +# and also ensure that FilterFile is set to a fully qualified file name # that contains the letter `D` (for "Direct Access") in the first line. FilterFile = "" @@ -119,15 +119,15 @@ Port = ":3000" # # To point Athens to an upstream proxy to fetch modules, # set GlobalEndpoint to "https://" -# and also ensure that FilterFile is set to a fully qualified file name +# and also ensure that FilterFile is set to a fully qualified file name # that contains the letter `D` (for "Direct Access") in the first line. GlobalEndpoint = "http://localhost:3001" -# BASIC AUTH OPTIONS +# BASIC AUTH OPTIONS # ================== -# PLASE NOTE THAT THIS IS A BAD HACK AROUND -# THE FACT THAT GO DOES NOT SUPPORT PROPER AUTHENTICATION -# YET! YOUR BASIC AUTH CREDENTIALS CAN EASILY LEAK +# PLASE NOTE THAT THIS IS A BAD HACK AROUND +# THE FACT THAT GO DOES NOT SUPPORT PROPER AUTHENTICATION +# YET! YOUR BASIC AUTH CREDENTIALS CAN EASILY LEAK # IN ATHENS LOGS AS WELL AS GO COMMAND LOGS. # THIS WILL BE ADDRESSED IN 1.13. # SEE https://github.com/golang/go/issues/30610 @@ -200,15 +200,15 @@ TraceExporterURL = "http://localhost:14268" # Env override: ATHENS_STATS_EXPORTER StatsExporter = "prometheus" -# SumDBs specifies a list of fully qualified URLs that Athens will proxy +# SumDBs specifies a list of fully qualified URLs that Athens will proxy # so that the go command can use as a checksum verifier. -# See NoSumDB for enforcing the go command to use +# See NoSumDB for enforcing the go command to use # GONOSUMDB. # Env override: ATHENS_SUM_DBS SumDBs = ["https://sum.golang.org"] -# NoSumPatterns specifies a list of patterns that will make the -# Sum DB proxy return a 403 if any of those patterns match. +# NoSumPatterns specifies a list of patterns that will make the +# Sum DB proxy return a 403 if any of those patterns match. # This will enforce the client to run GONOSUMDB # Example pattern: NoSumPatterns = ["github.com/mycompany/*"] # Env override: ATHENS_GONOSUM_PATTERNS @@ -227,37 +227,37 @@ NoSumPatterns = [] # 5. "none": return 404 if a module is not found and do nothing. # 6. "file:": will point to an HCL file that specifies # any of the 5 options above based on different import paths. -# 7. "custom:" is the same as option 6 -# but the file is fully encoded in the option. This is -# useful for using an environment variable in serverless +# 7. "custom:" is the same as option 6 +# but the file is fully encoded in the option. This is +# useful for using an environment variable in serverless # deployments. # Env override: ATHENS_DOWNLOAD_MODE DownloadMode = "sync" -# DownloadURL is the URL that will be used if -# DownloadMode is set to "redirect" +# DownloadURL is the URL that will be used if +# DownloadMode is set to "redirect" # Env override: ATHENS_DOWNLOAD_URL DownloadURL = "" -# SingleFlightType determines what mechanism Athens uses -# to manage concurrency flowing into the Athens Backend. -# This is important for the following scenario: if two concurrent requests -# were made to Athens that wanted to store the same module at the same time, -# we want to make sure only the first request gets to store the module, -# and the second request will wait for the first one to finish so that -# it doesn't override the storage. +# SingleFlightType determines what mechanism Athens uses +# to manage concurrency flowing into the Athens Backend. +# This is important for the following scenario: if two concurrent requests +# were made to Athens that wanted to store the same module at the same time, +# we want to make sure only the first request gets to store the module, +# and the second request will wait for the first one to finish so that +# it doesn't override the storage. # Options are ["memory", "etcd", "redis", "gcp", "azureblob"] -# The default option is "memory" which means that only one instance of Athens +# The default option is "memory" which means that only one instance of Athens # should be used. -# The "gcp" single flight will assume that you have a "gcp" StorageType -# and therefore it will use its strong-consistency features to ensure -# that only one module is ever written even when concurrent saves happen +# The "gcp" single flight will assume that you have a "gcp" StorageType +# and therefore it will use its strong-consistency features to ensure +# that only one module is ever written even when concurrent saves happen # at the same time. -# The "azureblob" single flight will assume that you have a "azureblob" StorageType -# and therefore it will use its strong-consistency features to ensure -# that only one module is ever written even when concurrent saves happen +# The "azureblob" single flight will assume that you have a "azureblob" StorageType +# and therefore it will use its strong-consistency features to ensure +# that only one module is ever written even when concurrent saves happen # at the same time. # Env override: ATHENS_SINGLE_FLIGHT_TYPE SingleFlightType = "memory" @@ -265,7 +265,7 @@ SingleFlightType = "memory" [SingleFlight] [SingleFlight.Etcd] # Endpoints are comma separated URLs that determine all distributed etcd servers. - # It defaults to what we have in our docker-compose file. It is by default ignored + # It defaults to what we have in our docker-compose file. It is by default ignored # as the SingleFlightType field above determines whether Etcd is used or not. # Env override: ATHENS_ETCD_ENDPOINTS Endpoints = "localhost:2379,localhost:22379,localhost:32379" @@ -301,14 +301,14 @@ SingleFlightType = "memory" # Env override: ATHENS_STORAGE_GCP_BUCKET Bucket = "MY_GCP_BUCKET" - # JSONKey is a base64 encoded service account + # ServiceAccount is a base64 encoded service account # key that allows Athens to be run outside of GCP - # but still be able to access GCS. If you are + # but still be able to access GCS. If you are # running Athens inside GCP, you will most - # likely not need this as GCP figures out - # internal authentication between products for you. + # likely not need this as GCP figures out + # internal authentication between products for you. # Env override: ATHENS_STORAGE_GCP_JSON_KEY - JSONKey = "" + ServiceAccount = "" [Storage.Minio] # Endpoint for Minio storage @@ -368,7 +368,7 @@ SingleFlightType = "memory" # and # https://godoc.org/github.com/aws/aws-sdk-go/aws/session#hdr-Environment_Variables # for environment variables that will affect the aws configuration. - # Setting UseDefaultConfiguration would only use default configuration. It will be deprecated in future releases + # Setting UseDefaultConfiguration would only use default configuration. It will be deprecated in future releases # and is recommended not to use it. # Region for S3 storage diff --git a/docs/content/install/install-on-kubernetes.md b/docs/content/install/install-on-kubernetes.md index 0a175753..919bc395 100644 --- a/docs/content/install/install-on-kubernetes.md +++ b/docs/content/install/install-on-kubernetes.md @@ -177,6 +177,18 @@ Last athens need authentication credentials for your minio in `storage.minio.acc helm install gomods/athens-proxy -n athens --namespace athens --set storage.type=minio --set storage.minio.endpoint= --set storage.minio.bucket= --set storage.minio.accessKey= --set storage.minio.secretKey= ``` +#### Google Cloud Storage + + To use Google Cloud Storage storage with Athens, set `storage.type` to `gcp`. You need to set `storage.gcp.projectID` and `storage.gcp.bucket` to the + desired GCP project and bucket name, respectively. + + Depending on your deployment environment you will also need to set `storage.gcp.serviceAccount` to a key which has read/write access to + the GCS bucket. If you are running Athens inside GCP, you will most likely not need this as GCP figures out internal authentication between products for you. + + ``` + helm install gomods/athens-proxy -n athens --namespace athens --set storage.type=gcp --set storage.gcp.projectID= --set storage.gcp.bucket= + ``` + ### Kubernetes Service By default, a Kubernetes `ClusterIP` service is created for the Athens proxy. "ClusterIP" is sufficient in the case when the Athens proxy will be used from within the cluster. To expose Athens outside of the cluster, consider using a "NodePort" or "LoadBalancer" service. This can be changed by setting the `service.type` value when installing the chart. For example, to deploy Athens using a NodePort service, the following command could be used: diff --git a/pkg/config/gcp.go b/pkg/config/gcp.go index 36321a12..81d41cfc 100644 --- a/pkg/config/gcp.go +++ b/pkg/config/gcp.go @@ -2,7 +2,7 @@ package config // GCPConfig specifies the properties required to use GCP as the storage backend type GCPConfig struct { - ProjectID string `envconfig:"GOOGLE_CLOUD_PROJECT"` - Bucket string `validate:"required" envconfig:"ATHENS_STORAGE_GCP_BUCKET"` - JSONKey string `envconfig:"ATHENS_STORAGE_GCP_JSON_KEY"` + ProjectID string `envconfig:"GOOGLE_CLOUD_PROJECT"` + Bucket string `validate:"required" envconfig:"ATHENS_STORAGE_GCP_BUCKET"` + ServiceAccount string `envconfig:"ATHENS_STORAGE_GCP_SERVICE_ACCOUNT"` } diff --git a/pkg/stash/with_gcs_test.go b/pkg/stash/with_gcs_test.go index 89b97da1..57b8a564 100644 --- a/pkg/stash/with_gcs_test.go +++ b/pkg/stash/with_gcs_test.go @@ -121,7 +121,7 @@ func getTestConfig() *config.GCPConfig { return nil } return &config.GCPConfig{ - Bucket: "athens_drone_stash_bucket", - JSONKey: creds, + Bucket: "athens_drone_stash_bucket", + ServiceAccount: creds, } } diff --git a/pkg/storage/gcp/gcp.go b/pkg/storage/gcp/gcp.go index e7429845..a2524f00 100644 --- a/pkg/storage/gcp/gcp.go +++ b/pkg/storage/gcp/gcp.go @@ -49,8 +49,8 @@ func New(ctx context.Context, gcpConf *config.GCPConfig, timeout time.Duration) func newClient(ctx context.Context, gcpConf *config.GCPConfig, timeout time.Duration) (*Storage, error) { const op errors.Op = "gcp.newClient" opts := []option.ClientOption{} - if gcpConf.JSONKey != "" { - key, err := base64.StdEncoding.DecodeString(gcpConf.JSONKey) + if gcpConf.ServiceAccount != "" { + key, err := base64.StdEncoding.DecodeString(gcpConf.ServiceAccount) if err != nil { return nil, errors.E(op, fmt.Errorf("could not decode base64 json key: %v", err)) } diff --git a/pkg/storage/gcp/gcp_test.go b/pkg/storage/gcp/gcp_test.go index 1d80b68d..a1db8698 100644 --- a/pkg/storage/gcp/gcp_test.go +++ b/pkg/storage/gcp/gcp_test.go @@ -72,9 +72,9 @@ func getTestConfig(bucket string) *config.GCPConfig { return nil } return &config.GCPConfig{ - Bucket: bucket, - JSONKey: creds, - ProjectID: os.Getenv("GCS_PROJECT_ID"), + Bucket: bucket, + ServiceAccount: creds, + ProjectID: os.Getenv("GCS_PROJECT_ID"), } }