cmd/proxy/actions: pass NoSumPatterns to GoGetFetcher (#1391)

* cmd/proxy/actions: pass NoSumPatterns to GoGetFetcher

* add gosum unit tests

* upgrade appveyor to 1.13

* fix test typo

* check appveyor version

* AppVeyor has no 1.13

* use logger

appveyor.yml

@@ -6,8 +6,9 @@ environment:
   GOPATH: c:\gopath
   GO111MODULE: on
   GOPROXY: https://proxy.golang.org
+  SKIP_UNTIL_113: true
 
-stack: go 1.12
+stack: go 1.13
 
 test_script:
   - go test ./...

cmd/proxy/actions/app_proxy.go

@@ -74,12 +74,23 @@ func addProxyRoutes(
 	// 3. The stashpool manages limiting concurrent requests and passes them to stash.
 	// 4. The plain stash.New just takes a request from upstream and saves it into storage.
 	fs := afero.NewOsFs()
-	mf, err := module.NewGoGetFetcher(c.GoBinary, c.GoProxy, fs)
+
+	// TODO: remove before we release v0.7.0
+	if c.GoProxy != "direct" && c.GoProxy != "" {
+		l.Error("GoProxy is deprecated, please use GoBinaryEnvVars")
+	}
+	if !c.GoBinaryEnvVars.HasKey("GONOSUMDB") {
+		c.GoBinaryEnvVars.Add("GONOSUMDB", strings.Join(c.NoSumPatterns, ","))
+	}
+	if err := c.GoBinaryEnvVars.Validate(); err != nil {
+		return err
+	}
+	mf, err := module.NewGoGetFetcher(c.GoBinary, c.GoBinaryEnvVars, fs)
 	if err != nil {
 		return err
 	}
 
-	lister := module.NewVCSLister(c.GoBinary, c.GoProxy, fs)
+	lister := module.NewVCSLister(c.GoBinary, c.GoBinaryEnvVars, fs)
 
 	withSingleFlight, err := getSingleFlight(c, s)
 	if err != nil {

config.dev.toml

@@ -15,6 +15,7 @@ GoBinary = "go"
 # Env override: GO_ENV
 GoEnv = "development"
 
+# GoProxy IS DEPRECATED and will be removed in v0.7.0: PLEASE USE GoBinaryEnvVars
 # GoProxy specifies GOPROXY env for go list or mod download inside athens
 # which can be configured totally same with GOPROXY of Go Command.
 # Notes that the comma-separated GOPROXY (e.g. <proxy1>,<proxy2>,direct) is only available in Go 1.13 or higher,
@@ -22,6 +23,20 @@ GoEnv = "development"
 # Env override: GOPROXY
 GoProxy = "direct"
 
+# GoBinaryEnvVars are environment variables that you'd like 
+# to pass directly to the Go command that Athens runs under the 
+# hood. Athens primarily runs two Go commands: 
+# 1. `go mod download -json <module>@<version>`
+# 2. `go list -m -json <module>@latest`
+# The go command accepts multiple environment variables that 
+# can affect the two processes above such as GONOSUMDB and GOPROXY. 
+# Note that athens passes the NoSumPatterns to the two commands above,
+# but the existence of GONOSUMDB in this configuration takes precedence.
+# Although you can pass any key=value to the Go command here, you can see 
+# the list of possible env vars by running `go env`.
+# Env override: ATHENS_GO_BINARY_ENV_VARS
+GoBinaryEnvVars = ["GOPROXY=direct"]
+
 # GoGetWorkers specifies how many times you can concurrently
 # go mod download, this is so that low performance instances
 # can manage go get more sanely and not run out of disk or memory.

go.mod

@@ -6,6 +6,7 @@ require (
 	cloud.google.com/go v0.26.0
 	contrib.go.opencensus.io/exporter/stackdriver v0.6.0
 	github.com/Azure/azure-storage-blob-go v0.7.0
+	github.com/Azure/go-autorest/autorest/adal v0.6.0 // indirect
 	github.com/BurntSushi/toml v0.3.1
 	github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895 // indirect
 	github.com/DataDog/opencensus-go-exporter-datadog v0.0.0-20180917103902-e6c7f767dc57

go.sum

@@ -4,14 +4,26 @@ contrib.go.opencensus.io/exporter/stackdriver v0.6.0 h1:U0FQWsZU3aO8W+BrZc88T8fd
 contrib.go.opencensus.io/exporter/stackdriver v0.6.0/go.mod h1:QeFzMJDAw8TXt5+aRaSuE8l5BwaMIOIlaVkBOPRuMuw=
 git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999 h1:OR8VhtwhcAI3U48/rzBsVOuHi0zDPzYI1xASVcdSgR8=
 git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg=
-github.com/Azure/azure-pipeline-go v0.1.8 h1:KmVRa8oFMaargVesEuuEoiLCQ4zCCwQ8QX/xg++KS20=
-github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg=
 github.com/Azure/azure-pipeline-go v0.2.1 h1:OLBdZJ3yvOn2MezlWvbrBMTEUQC72zAftRZOMdj5HYo=
 github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4=
-github.com/Azure/azure-storage-blob-go v0.0.0-20181022225951-5152f14ace1c h1:Y5ueznoCekgCWBytF1Q9lTpZ3tJeX37dQtCcGjMCLYI=
-github.com/Azure/azure-storage-blob-go v0.0.0-20181022225951-5152f14ace1c/go.mod h1:oGfmITT1V6x//CswqY2gtAHND+xIP64/qL7a5QJix0Y=
 github.com/Azure/azure-storage-blob-go v0.7.0 h1:MuueVOYkufCxJw5YZzF842DY2MBsp+hLuh2apKY0mck=
 github.com/Azure/azure-storage-blob-go v0.7.0/go.mod h1:f9YQKtsG1nMisotuTPpO0tjNuEjKRYAcJU8/ydDI++4=
+github.com/Azure/go-autorest/autorest v0.9.0 h1:MRvx8gncNaXJqOoLmhNjUAKh33JJF8LyxPhomEtOsjs=
+github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI=
+github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0=
+github.com/Azure/go-autorest/autorest/adal v0.6.0 h1:UCTq22yE3RPgbU/8u4scfnnzuCW6pwQ9n+uBtV78ouo=
+github.com/Azure/go-autorest/autorest/adal v0.6.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc=
+github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA=
+github.com/Azure/go-autorest/autorest/date v0.2.0 h1:yW+Zlqf26583pE43KhfnhFcdmSWlm5Ew6bxipnr/tbM=
+github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g=
+github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
+github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
+github.com/Azure/go-autorest/autorest/mocks v0.3.0 h1:qJumjCaCudz+OcqE9/XtEPfvtOjOmKaui4EOpFI6zZc=
+github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM=
+github.com/Azure/go-autorest/logger v0.1.0 h1:ruG4BSDXONFRrZZJ2GUXDiUyVpayPmb1GnWeHDdaNKY=
+github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc=
+github.com/Azure/go-autorest/tracing v0.5.0 h1:TRn4WjSnkcSy5AEG3pnbtFSwNtwzjr4VYyQflFE619k=
+github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895 h1:dmc/C8bpE5VkQn65PNbbyACDC8xw8Hpp/NEurdPmQDQ=
@@ -64,6 +76,7 @@ github.com/go-redis/redis v6.15.2+incompatible h1:9SpNVG76gr6InJGxoZ6IuuxaCOQwDA
 github.com/go-redis/redis v6.15.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
 github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/go-test/deep v1.0.1 h1:UQhStjbkDClarlmv0am7OXXO4/GaPdCGiUiMTvi28sg=
 github.com/go-test/deep v1.0.1/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
 github.com/gobuffalo/envy v1.6.7 h1:XMZGuFqTupAXhZTriQ+qO38QvNOSU/0rl3hEPCFci/4=
 github.com/gobuffalo/envy v1.6.7/go.mod h1:N+GkhhZ/93bGZc6ZKhJLP6+m+tCNPKwgSpH9kaifseQ=
@@ -165,9 +178,11 @@ github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7/go.mod h1:ZX
 github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
 github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.4.2 h1:3mYCb7aPxS/RU7TI1y4rkEn1oKmPRjNJLNEXgw7MH2I=
 github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
 github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ=
@@ -205,6 +220,7 @@ github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B
 github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/pflag v1.0.1 h1:aCvUg6QPl3ibpQUxyLkrEkCHtPqYJL4x9AuhqVqFis4=
 github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.2 h1:Fy0orTDgHdbnzHcsOgfCN4LtHf0ec3wwtiwJqwvf3Gc=
 github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
@@ -249,8 +265,6 @@ go.uber.org/zap v1.9.1 h1:XCJQEf3W6eZaVwhRBof6ImoYGJSITeKWsyeh3HFu/5o=
 go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 golang.org/x/crypto v0.0.0-20180608092829-8ac0e0d97ce4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20181029103014-dab2b1051b5d h1:5JyY8HlzxzYI+qHOOciM8s2lJbIEaefMUdtYt7dRDrg=
-golang.org/x/crypto v0.0.0-20181029103014-dab2b1051b5d/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 h1:p/H982KKEjUnLJkM3tt/LemDnOc1GiZL5FCVlORJ5zo=
 golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -262,8 +276,6 @@ golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20180811021610-c39426892332/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd h1:nTDtHvHSdCn1m6ITfMRqtOd/9+7a3s8RBNOZ3eYZzJA=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181029044818-c44066c5c816 h1:mVFkLpejdFLXVUv9E42f3XJVfMdqd0IVLVIVLjZWn5o=
-golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190502183928-7f726cade0ab h1:9RfW3ktsOZxgo9YNbBAjq1FWzc/igwEcUzZz8IXgSbk=
 golang.org/x/net v0.0.0-20190502183928-7f726cade0ab/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
@@ -276,8 +288,6 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e h1:o3PsSEY8E4eXWkXrIP9YJALUkVZqzHJT5DOasTyn8Vs=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181031143558-9b800f95dbbc h1:SdCq5U4J+PpbSDIl9bM0V1e1Ug1jsnBkAFvTs1htn7U=
-golang.org/x/sys v0.0.0-20181031143558-9b800f95dbbc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190502175342-a43fa875dd82 h1:vsphBvatvfbhlb4PO1BYSr9dzugGxJ/SQHoNufZJq1w=
@@ -303,7 +313,6 @@ gopkg.in/DataDog/dd-trace-go.v1 v1.10.0 h1:aKIe93NsKAn5Gm/A4nNO4hlPuKTnhaf+khqu0
 gopkg.in/DataDog/dd-trace-go.v1 v1.10.0/go.mod h1:DVp8HmDh8PuTu2Z0fVVlBsyWaC++fzwVCaGWylTe3tg=
 gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=

pkg/config/config.go

@@ -7,6 +7,7 @@ import (
 	"path/filepath"
 	"runtime"
 	"strconv"
+	"strings"
 
 	"github.com/BurntSushi/toml"
 	"github.com/gomods/athens/pkg/download/mode"
@@ -23,6 +24,7 @@ type Config struct {
 	GoEnv            string    `validate:"required" envconfig:"GO_ENV"`
 	GoBinary         string    `validate:"required" envconfig:"GO_BINARY_PATH"`
 	GoProxy          string    `envconfig:"GOPROXY"`
+	GoBinaryEnvVars  EnvList   `envconfig:"ATHENS_GO_BINARY_ENV_VARS"`
 	GoGetWorkers     int       `validate:"required" envconfig:"ATHENS_GOGET_WORKERS"`
 	ProtocolWorkers  int       `validate:"required" envconfig:"ATHENS_PROTOCOL_WORKERS"`
 	LogLevel         string    `validate:"required" envconfig:"ATHENS_LOG_LEVEL"`
@@ -56,6 +58,40 @@ type Config struct {
 	Storage          *StorageConfig
 }
 
+// EnvList is a list of key-value environment
+// variables that are passed to the Go command
+type EnvList []string
+
+// HasKey returns whether a key-value entry
+// is present by only checking the left of
+// key=value
+func (el EnvList) HasKey(key string) bool {
+	for _, env := range el {
+		if strings.HasPrefix(env, key+"=") {
+			return true
+		}
+	}
+	return false
+}
+
+// Add adds a key=value entry to the environment
+// list
+func (el *EnvList) Add(key, value string) {
+	*el = append(*el, key+"="+value)
+}
+
+// Validate validates that all strings inside the
+// list are of the key=value format
+func (el EnvList) Validate() error {
+	const op errors.Op = "EnvList.Validate"
+	for _, env := range el {
+		if strings.Count(env, "=") != 1 {
+			return errors.E(op, fmt.Errorf("incorrect env format: %v", env))
+		}
+	}
+	return nil
+}
+
 // Load loads the config from a file.
 // If file is not present returns default config
 func Load(configFile string) (*Config, error) {
@@ -78,6 +114,7 @@ func Load(configFile string) (*Config, error) {
 func defaultConfig() *Config {
 	return &Config{
 		GoBinary:         "go",
+		GoBinaryEnvVars:  EnvList{"GOPROXY=direct"},
 		GoEnv:            "development",
 		GoProxy:          "direct",
 		GoGetWorkers:     10,

pkg/config/config_test.go

@@ -6,6 +6,7 @@ import (
 	"path/filepath"
 	"runtime"
 	"strconv"
+	"strings"
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
@@ -76,21 +77,22 @@ func TestEnvOverrides(t *testing.T) {
 		TimeoutConf: TimeoutConf{
 			Timeout: 30,
 		},
-		StorageType:    "minio",
+		StorageType:     "minio",
-		GlobalEndpoint: "mytikas.gomods.io",
+		GlobalEndpoint:  "mytikas.gomods.io",
-		Port:           ":7000",
+		Port:            ":7000",
-		EnablePprof:    false,
+		EnablePprof:     false,
-		PprofPort:      ":3001",
+		PprofPort:       ":3001",
-		BasicAuthUser:  "testuser",
+		BasicAuthUser:   "testuser",
-		BasicAuthPass:  "testpass",
+		BasicAuthPass:   "testpass",
-		ForceSSL:       true,
+		ForceSSL:        true,
-		ValidatorHook:  "testhook.io",
+		ValidatorHook:   "testhook.io",
-		PathPrefix:     "prefix",
+		PathPrefix:      "prefix",
-		NETRCPath:      "/test/path/.netrc",
+		NETRCPath:       "/test/path/.netrc",
-		HGRCPath:       "/test/path/.hgrc",
+		HGRCPath:        "/test/path/.hgrc",
-		Storage:        &StorageConfig{},
+		Storage:         &StorageConfig{},
-		SingleFlight:   &SingleFlight{},
+		GoBinaryEnvVars: []string{"GOPROXY=direct"},
-		RobotsFile:     "robots.txt",
+		SingleFlight:    &SingleFlight{},
+		RobotsFile:      "robots.txt",
 	}
 
 	envVars := getEnvMap(expConf)
@@ -279,6 +281,7 @@ func TestParseExampleConfig(t *testing.T) {
 		TraceExporter:    "",
 		StatsExporter:    "prometheus",
 		SingleFlightType: "memory",
+		GoBinaryEnvVars:  []string{"GOPROXY=direct"},
 		SingleFlight:     &SingleFlight{},
 		SumDBs:           []string{"https://sum.golang.org"},
 		NoSumPatterns:    []string{},
@@ -325,6 +328,7 @@ func getEnvMap(config *Config) map[string]string {
 	envVars["ATHENS_NETRC_PATH"] = config.NETRCPath
 	envVars["ATHENS_HGRC_PATH"] = config.HGRCPath
 	envVars["ATHENS_ROBOTS_FILE"] = config.RobotsFile
+	envVars["ATHENS_GO_BINARY_ENV_VARS"] = strings.Join(config.GoBinaryEnvVars, ",")
 
 	storage := config.Storage
 	if storage != nil {
@@ -477,3 +481,24 @@ func TestDefaultConfigMatchesConfigFile(t *testing.T) {
 		t.Errorf("Default values from the config file: %v should equal to the default values returned in case the config file isn't provided %v", parsedConf, defConf)
 	}
 }
+
+func TestEnvList(t *testing.T) {
+	el := EnvList{"KEY=VALUE"}
+	if !el.HasKey("KEY") {
+		t.Fatal("expected KEY to be present")
+	}
+	if el.HasKey("KEY=") {
+		t.Fatal("expected KEY= to not be found")
+	}
+	el.Add("HELLO", "WORLD")
+	if !el.HasKey("HELLO") {
+		t.Fatal("expected HELLO key to be found")
+	}
+	if err := el.Validate(); err != nil {
+		t.Fatalf("expected err to be nil but got %v", err)
+	}
+	el = EnvList{"HELLO"}
+	if err := el.Validate(); err == nil {
+		t.Fatal("expected a validation error for incorrect formatting but got nil")
+	}
+}

pkg/download/protocol_test.go

@@ -35,9 +35,8 @@ func getDP(t *testing.T) Protocol {
 		t.Fatalf("Unable to parse config file: %s", err.Error())
 	}
 	goBin := conf.GoBinary
-	goProxy := conf.GoProxy
 	fs := afero.NewOsFs()
-	mf, err := module.NewGoGetFetcher(goBin, goProxy, fs)
+	mf, err := module.NewGoGetFetcher(goBin, conf.GoBinaryEnvVars, fs)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -46,7 +45,7 @@ func getDP(t *testing.T) Protocol {
 		t.Fatal(err)
 	}
 	st := stash.New(mf, s)
-	return New(&Opts{s, st, module.NewVCSLister(goBin, goProxy, fs), nil})
+	return New(&Opts{s, st, module.NewVCSLister(goBin, conf.GoBinaryEnvVars, fs), nil})
 }
 
 type listTest struct {

pkg/module/all_test.go

@@ -19,7 +19,7 @@ type ModuleSuite struct {
 	suite.Suite
 	fs           afero.Fs
 	goBinaryName string
-	goProxy      string
+	env          []string
 }
 
 func (m *ModuleSuite) SetupTest() {
@@ -28,5 +28,5 @@ func (m *ModuleSuite) SetupTest() {
 
 func TestModules(t *testing.T) {
 	goBinaryPath := envy.Get("GO_BINARY_PATH", "go")
-	suite.Run(t, &ModuleSuite{goBinaryName: goBinaryPath, goProxy: "direct"})
+	suite.Run(t, &ModuleSuite{goBinaryName: goBinaryPath, env: []string{"GOPROXY=direct"}})
 }

pkg/module/go_get_fetcher.go

@@ -19,7 +19,7 @@ import (
 type goGetFetcher struct {
 	fs           afero.Fs
 	goBinaryName string
-	goProxy      string
+	envVars      []string
 }
 
 type goModule struct {
@@ -35,7 +35,7 @@ type goModule struct {
 }
 
 // NewGoGetFetcher creates fetcher which uses go get tool to fetch modules
-func NewGoGetFetcher(goBinaryName string, goProxy string, fs afero.Fs) (Fetcher, error) {
+func NewGoGetFetcher(goBinaryName string, envVars []string, fs afero.Fs) (Fetcher, error) {
 	const op errors.Op = "module.NewGoGetFetcher"
 	if err := validGoBinary(goBinaryName); err != nil {
 		return nil, errors.E(op, err)
@@ -43,7 +43,7 @@ func NewGoGetFetcher(goBinaryName string, goProxy string, fs afero.Fs) (Fetcher,
 	return &goGetFetcher{
 		fs:           fs,
 		goBinaryName: goBinaryName,
-		goProxy:      goProxy,
+		envVars:      envVars,
 	}, nil
 }
 
@@ -66,7 +66,7 @@ func (g *goGetFetcher) Fetch(ctx context.Context, mod, ver string) (*storage.Ver
 		return nil, errors.E(op, err)
 	}
 
-	m, err := downloadModule(g.goBinaryName, g.goProxy, g.fs, goPathRoot, modPath, mod, ver)
+	m, err := downloadModule(g.goBinaryName, g.envVars, g.fs, goPathRoot, modPath, mod, ver)
 	if err != nil {
 		clearFiles(g.fs, goPathRoot)
 		return nil, errors.E(op, err)
@@ -101,13 +101,13 @@ func (g *goGetFetcher) Fetch(ctx context.Context, mod, ver string) (*storage.Ver
 
 // given a filesystem, gopath, repository root, module and version, runs 'go mod download -json'
 // on module@version from the repoRoot with GOPATH=gopath, and returns a non-nil error if anything went wrong.
-func downloadModule(goBinaryName, goProxy string, fs afero.Fs, gopath, repoRoot, module, version string) (goModule, error) {
+func downloadModule(goBinaryName string, envVars []string, fs afero.Fs, gopath, repoRoot, module, version string) (goModule, error) {
 	const op errors.Op = "module.downloadModule"
 	uri := strings.TrimSuffix(module, "/")
 	fullURI := fmt.Sprintf("%s@%s", uri, version)
 
 	cmd := exec.Command(goBinaryName, "mod", "download", "-json", fullURI)
-	cmd.Env = prepareEnv(gopath, goProxy)
+	cmd.Env = prepareEnv(gopath, envVars)
 	cmd.Dir = repoRoot
 	stdout := &bytes.Buffer{}
 	stderr := &bytes.Buffer{}

pkg/module/go_get_fetcher_test.go

@@ -3,6 +3,9 @@ package module
 import (
 	"context"
 	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"os"
 	"runtime"
 
 	"github.com/gomods/athens/pkg/errors"
@@ -14,14 +17,14 @@ var ctx = context.Background()
 
 func (s *ModuleSuite) TestNewGoGetFetcher() {
 	r := s.Require()
-	fetcher, err := NewGoGetFetcher(s.goBinaryName, s.goProxy, s.fs)
+	fetcher, err := NewGoGetFetcher(s.goBinaryName, s.env, s.fs)
 	r.NoError(err)
 	_, ok := fetcher.(*goGetFetcher)
 	r.True(ok)
 }
 
 func (s *ModuleSuite) TestGoGetFetcherError() {
-	fetcher, err := NewGoGetFetcher("invalidpath", "", afero.NewOsFs())
+	fetcher, err := NewGoGetFetcher("invalidpath", s.env, afero.NewOsFs())
 
 	assert.Nil(s.T(), fetcher)
 	if runtime.GOOS == "windows" {
@@ -35,7 +38,7 @@ func (s *ModuleSuite) TestGoGetFetcherFetch() {
 	r := s.Require()
 	// we need to use an OS filesystem because fetch executes vgo on the command line, which
 	// always writes to the filesystem
-	fetcher, err := NewGoGetFetcher(s.goBinaryName, s.goProxy, afero.NewOsFs())
+	fetcher, err := NewGoGetFetcher(s.goBinaryName, s.env, afero.NewOsFs())
 	r.NoError(err)
 	ver, err := fetcher.Fetch(ctx, repoURI, version)
 	r.NoError(err)
@@ -55,7 +58,7 @@ func (s *ModuleSuite) TestGoGetFetcherFetch() {
 
 func (s *ModuleSuite) TestNotFoundFetches() {
 	r := s.Require()
-	fetcher, err := NewGoGetFetcher(s.goBinaryName, s.goProxy, afero.NewOsFs())
+	fetcher, err := NewGoGetFetcher(s.goBinaryName, s.env, afero.NewOsFs())
 	r.NoError(err)
 	// when someone buys laks47dfjoijskdvjxuyyd.com, and implements
 	// a git server on top of it, this test will fail :)
@@ -67,3 +70,48 @@ func (s *ModuleSuite) TestNotFoundFetches() {
 		s.Failf("incorrect error kind", "expected a not found error but got %v", errors.Kind(err))
 	}
 }
+
+func (s *ModuleSuite) TestGoGetFetcherSumDB() {
+	if os.Getenv("SKIP_UNTIL_113") != "" {
+		return
+	}
+	r := s.Require()
+	zipBytes, err := ioutil.ReadFile("test_data/mockmod.xyz@v1.2.3.zip")
+	r.NoError(err)
+	mp := &mockProxy{paths: map[string][]byte{
+		"/mockmod.xyz/@v/v1.2.3.info": []byte(`{"Version":"v1.2.3"}`),
+		"/mockmod.xyz/@v/v1.2.3.mod":  []byte(`{"module mod}`),
+		"/mockmod.xyz/@v/v1.2.3.zip":  zipBytes,
+	}}
+	proxyAddr, close := s.getProxy(mp)
+	defer close()
+
+	fetcher, err := NewGoGetFetcher(s.goBinaryName, []string{"GOPROXY=" + proxyAddr}, afero.NewOsFs())
+	r.NoError(err)
+	_, err = fetcher.Fetch(ctx, "mockmod.xyz", "v1.2.3")
+	if err == nil {
+		s.T().Fatal("expected a gosum error but got nil")
+	}
+	fetcher, err = NewGoGetFetcher(s.goBinaryName, []string{"GONOSUMDB=mockmod.xyz", "GOPROXY=" + proxyAddr}, afero.NewOsFs())
+	r.NoError(err)
+	_, err = fetcher.Fetch(ctx, "mockmod.xyz", "v1.2.3")
+	r.NoError(err, "expected the go sum to not be consulted but got an error")
+}
+
+func (s *ModuleSuite) getProxy(h http.Handler) (addr string, close func()) {
+	srv := httptest.NewServer(h)
+	return srv.URL, srv.Close
+}
+
+type mockProxy struct {
+	paths map[string][]byte
+}
+
+func (m *mockProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	resp, ok := m.paths[r.URL.Path]
+	if !ok {
+		w.WriteHeader(404)
+		return
+	}
+	w.Write(resp)
+}

pkg/module/go_vcs_lister.go

@@ -24,7 +24,7 @@ type listResp struct {
 
 type vcsLister struct {
 	goBinPath string
-	goProxy   string
+	env       []string
 	fs        afero.Fs
 }
 
@@ -55,7 +55,7 @@ func (l *vcsLister) List(ctx context.Context, mod string) (*storage.RevInfo, []s
 		return nil, nil, errors.E(op, err)
 	}
 	defer clearFiles(l.fs, gopath)
-	cmd.Env = prepareEnv(gopath, l.goProxy)
+	cmd.Env = prepareEnv(gopath, l.env)
 
 	err = cmd.Run()
 	if err != nil {
@@ -83,6 +83,6 @@ func (l *vcsLister) List(ctx context.Context, mod string) (*storage.RevInfo, []s
 }
 
 // NewVCSLister creates an UpstreamLister which uses VCS to fetch a list of available versions
-func NewVCSLister(goBinPath, goProxy string, fs afero.Fs) UpstreamLister {
+func NewVCSLister(goBinPath string, env []string, fs afero.Fs) UpstreamLister {
-	return &vcsLister{goBinPath: goBinPath, goProxy: goProxy, fs: fs}
+	return &vcsLister{goBinPath: goBinPath, env: env, fs: fs}
 }

pkg/module/prepare_env.go

@@ -10,7 +10,7 @@ import (
 // prepareEnv will return all the appropriate
 // environment variables for a Go Command to run
 // successfully (such as GOPATH, GOCACHE, PATH etc)
-func prepareEnv(gopath, goProxy string) []string {
+func prepareEnv(gopath string, envVars []string) []string {
 	pathEnv := fmt.Sprintf("PATH=%s", os.Getenv("PATH"))
 	homeEnv := fmt.Sprintf("HOME=%s", os.Getenv("HOME"))
 	httpProxy := fmt.Sprintf("HTTP_PROXY=%s", os.Getenv("HTTP_PROXY"))
@@ -21,7 +21,6 @@ func prepareEnv(gopath, goProxy string) []string {
 	httpsProxyLower := fmt.Sprintf("https_proxy=%s", os.Getenv("https_proxy"))
 	noProxyLower := fmt.Sprintf("no_proxy=%s", os.Getenv("no_proxy"))
 	gopathEnv := fmt.Sprintf("GOPATH=%s", gopath)
-	goProxyEnv := fmt.Sprintf("GOPROXY=%s", goProxy)
 	cacheEnv := fmt.Sprintf("GOCACHE=%s", filepath.Join(gopath, "cache"))
 	gitSSH := fmt.Sprintf("GIT_SSH=%s", os.Getenv("GIT_SSH"))
 	gitSSHCmd := fmt.Sprintf("GIT_SSH_COMMAND=%s", os.Getenv("GIT_SSH_COMMAND"))
@@ -31,7 +30,6 @@ func prepareEnv(gopath, goProxy string) []string {
 		pathEnv,
 		homeEnv,
 		gopathEnv,
-		goProxyEnv,
 		cacheEnv,
 		disableCgo,
 		enableGoModules,
@@ -44,6 +42,7 @@ func prepareEnv(gopath, goProxy string) []string {
 		gitSSH,
 		gitSSHCmd,
 	}
+	cmdEnv = append(cmdEnv, envVars...)
 
 	if sshAuthSockVal, hasSSHAuthSock := os.LookupEnv("SSH_AUTH_SOCK"); hasSSHAuthSock {
 		// Verify that the ssh agent unix socket exists and is a unix socket.