downloadProtocol: support multi-proxy environments with DownloadFile (#1230)

* downloadProtocol: support multi-proxy environments with DownloadFile

* remove debugging lines

* update config tests

* download/mode: add tests for DownloadFile and friends

* add documentation to Download File

cmd/proxy/actions/app_proxy.go

@@ -10,6 +10,7 @@ import (
 	"github.com/gomods/athens/pkg/config"
 	"github.com/gomods/athens/pkg/download"
 	"github.com/gomods/athens/pkg/download/addons"
+	"github.com/gomods/athens/pkg/download/mode"
 	"github.com/gomods/athens/pkg/log"
 	"github.com/gomods/athens/pkg/module"
 	"github.com/gomods/athens/pkg/stash"
@@ -85,14 +86,21 @@ func addProxyRoutes(
 	}
 	st := stash.New(mf, s, stash.WithPool(c.GoGetWorkers), withSingleFlight)
 
-	dpOpts := &download.Opts{
+	df, err := mode.NewFile(c.DownloadMode, c.DownloadURL)
-		Storage: s,
+	if err != nil {
-		Stasher: st,
+		return err
-		Lister:  lister,
 	}
+
+	dpOpts := &download.Opts{
+		Storage:      s,
+		Stasher:      st,
+		Lister:       lister,
+		DownloadFile: df,
+	}
+
 	dp := download.New(dpOpts, addons.WithPool(c.ProtocolWorkers))
 
-	handlerOpts := &download.HandlerOpts{Protocol: dp, Logger: l}
+	handlerOpts := &download.HandlerOpts{Protocol: dp, Logger: l, DownloadFile: df}
 	download.RegisterHandlers(r, handlerOpts)
 
 	return nil

config.dev.toml

@@ -81,7 +81,7 @@ Port = ":3000"
 #
 # To point Athens to an upstream proxy to fetch modules,
 # set GlobalEndpoint to "https://<url_to_upstream>"
-# and also ensure that FilterFile is  set to a fully qualified file name 
+# and also ensure that FilterFile is set to a fully qualified file name 
 # that contains the letter `D` (for "Direct Access") in the first line.
 GlobalEndpoint = "http://localhost:3001"
 
@@ -176,6 +176,31 @@ SumDBs = ["https://sum.golang.org"]
 # Env override: ATHENS_GONOSUM_PATTERNS
 NoSumPatterns = []
 
+# DownloadMode defines how Athens behaves when a module@version
+# is not found in storage. There are 4 options:
+# 1. "sync" (default): download the module synchronously and
+# return the results to the client.
+# 2. "async": return 404, but asynchronously store the module
+# in the storage backend.
+# 3. "redirect": return a 301 redirect status to the client
+# with the base URL as the DownloadRedirectURL from below.
+# 4. "async_redirect": same as option number 3 but it will
+# asynchronously store the module to the backend.
+# 5. "none": return 404 if a module is not found and do nothing.
+# 6. "file:<path>": will point to an HCL file that specifies
+# any of the 5 options above based on different import paths.
+# 7. "custom:<base64-encoded-hcl>" is the same as option 6 
+# but the file is fully encoded in the option. This is 
+# useful for using an environment variable in serverless 
+# deployments.
+# Env override: ATHENS_DOWNLOAD_MODE
+DownloadMode = "sync"
+
+# DownloadURL is the URL that will be used if 
+# DownloadMode is set to "redirect" 
+# Env override: ATHENS_DOWNLOAD_URL
+DownloadURL = ""
+
 # SingleFlightType determines what mechanism Athens uses 
 # to manage concurrency flowing into the Athens Backend. 
 # This is important for the following scenario: if two concurrent requests 

docs/content/configuration/download.md

@@ -0,0 +1,62 @@
+---
+title: Download Mode
+description: What to do when a module is not in storage
+weight: 2
+---
+
+Athens accepts an HCL formatted Download File that serves as the source of truth for answering the following question: 
+
+### What should Athens do when a module@version is not found in storage? 
+
+Say a client sends an HTTP request with the path `/github.com/pkg/errors/@v/v0.8.1` and Athens
+does not have this module in storage. Athens will look at the Download File for one of the following Modes:
+
+1. **`sync`**: Synchronously download the module from VCS via `go mod download`, persist it to the Athens storage, and serve it back to the user immediately. Note that this is the default behavior. 
+2. **`async`**: Return a 404 to the client, and asynchronously download and persist the module@version to storage.
+3. **`none`**: Return a 404 and do nothing. 
+4. **`redirect`**: Redirect to an upstream proxy (such as proxy.golang.org) and do nothing after.
+5. **`async_redirect`**: Redirect to an upstream proxy (such as proxy.golang.org) and asynchronously download and persist the module@version to storage. 
+
+Furthermore, the Download File can describe any of the above behavior for different modules and module patterns alike using [path.Match](https://golang.org/pkg/path/#Match). Take a look at the following example:
+
+```javascript
+downloadURL = "https://proxy.golang.org"
+
+mode = "async_redirect"
+
+download "github.com/gomods/*" {
+    mode = "sync"
+}
+
+download "golang.org/x/*" {
+    mode = "none"
+}
+
+download "github.com/pkg/*" {
+    mode = "redirect"
+    downloadURL = "https://gocenter.io"
+}
+```
+
+The first two lines describe the behavior and the destination of all packages: redirect to `https://proxy.golang.org` and asynchronously persist the module to storage. 
+
+The following two blocks describe what to do if the requested module matches the given pattern:
+
+Any module that matches "github.com/gomods/*" such as "github.com/gomods/athens", will be synchronously fetched, stored, and returned to the user. 
+
+Any module that matches "golang.org/x/*" such as "golang.org/x/text" will just return a 404. Note that this behavior allows the client to set GOPROXY to multiple comma separated proxies so that the Go command can move into the second argument. 
+
+Any module that matches "github.com/pkg/*" such as "github.com/pkg/errors" will be redirected to https://gocenter.io (and not proxy.golang.org) and will also never persist the module to the Athens storage.
+
+
+## Use cases
+
+So why would you want to use the Download File to configure the behavior above? Here are a few use cases where it might make sense for you to do so: 
+
+**Limited storage:**
+
+If you have limited storage, then it might be a good idea to only persist some moduels (such as private ones) and then redirect to a public proxy for everything else. 
+
+**Limited resources:**
+
+If you are running Athens with low memory/cpu, then you can redirect all public modules to proxy.golang.org but asynchronously fetch them so that the client does not timeout. At the same time, you can return a 404 for private modules through the `none` mode and let the client (the Go command) fetch private modules directly through `GOPROXY=<athens-url>,direct` 

download.example.hcl

@@ -0,0 +1,16 @@
+downloadURL = "https://proxy.golang.org"
+
+mode = "async_redirect"
+
+download "github.com/gomods/*" {
+    mode = "sync"
+}
+
+download "golang.org/x/*" {
+    mode = "none"
+}
+
+download "github.com/pkg/*" {
+    mode = "redirect"
+    downloadURL = "https://gocenter.io"
+}

go.mod

@@ -26,10 +26,10 @@ require (
 	github.com/gopherjs/gopherjs v0.0.0-20180825215210-0210a2f0f73c // indirect
 	github.com/gorilla/mux v1.6.2
 	github.com/hashicorp/go-multierror v1.0.0
+	github.com/hashicorp/hcl2 v0.0.0-20190503213020-640445e16309
 	github.com/jtolds/gls v4.2.1+incompatible // indirect
 	github.com/kelseyhightower/envconfig v1.3.0
 	github.com/konsorten/go-windows-terminal-sequences v1.0.1 // indirect
-	github.com/kr/pretty v0.1.0 // indirect
 	github.com/minio/minio-go v6.0.5+incompatible
 	github.com/mitchellh/go-homedir v1.0.0
 	github.com/philhofer/fwd v1.0.0 // indirect
@@ -48,15 +48,11 @@ require (
 	go.etcd.io/etcd v0.0.0-20190215181705-784daa04988c
 	go.mongodb.org/mongo-driver v1.0.0
 	go.opencensus.io v0.17.0
-	golang.org/x/crypto v0.0.0-20181029103014-dab2b1051b5d // indirect
-	golang.org/x/net v0.0.0-20181029044818-c44066c5c816 // indirect
 	golang.org/x/oauth2 v0.0.0-20180620175406-ef147856a6dd
-	golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f
+	golang.org/x/sync v0.0.0-20190423024810-112230192c58
-	golang.org/x/sys v0.0.0-20181031143558-9b800f95dbbc // indirect
 	google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf
 	google.golang.org/appengine v1.3.0 // indirect
 	gopkg.in/DataDog/dd-trace-go.v1 v1.10.0 // indirect
-	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
 	gopkg.in/go-playground/assert.v1 v1.2.1 // indirect
 	gopkg.in/go-playground/validator.v9 v9.20.2
 )

go.sum

@@ -14,13 +14,19 @@ github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895 h1:dmc/C8bpE5Vk
 github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
 github.com/DataDog/opencensus-go-exporter-datadog v0.0.0-20180917103902-e6c7f767dc57 h1:V1H8VVVxLALfaPLvAFCPoa0AN5nVPAqEu2UvH+QP3Vc=
 github.com/DataDog/opencensus-go-exporter-datadog v0.0.0-20180917103902-e6c7f767dc57/go.mod h1:gMGUEe16aZh0QN941HgDjwrdjU4iTthPoz2/AtDRADE=
+github.com/agext/levenshtein v1.2.1 h1:QmvMAjj2aEICytGiWzmxoE0x2KZvE0fvmqMOfy2tjT8=
+github.com/agext/levenshtein v1.2.1/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558=
 github.com/ajg/form v0.0.0-20160822230020-523a5da1a92f h1:zvClvFQwU++UpIUBGC8YmDlfhUrweEy1R1Fj1gu5iIM=
 github.com/ajg/form v0.0.0-20160822230020-523a5da1a92f/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
+github.com/apparentlymart/go-dump v0.0.0-20180507223929-23540a00eaa3/go.mod h1:oL81AME2rN47vu18xqj1S1jPIPuN7afo62yKTNn3XMM=
+github.com/apparentlymart/go-textseg v1.0.0 h1:rRmlIsPEEhUTIKQb7T++Nz/A5Q6C9IuX2wFoYVvnCs0=
+github.com/apparentlymart/go-textseg v1.0.0/go.mod h1:z96Txxhf3xSFMPmb5X/1W05FF/Nj9VFpLOpjS5yuumk=
 github.com/aws/aws-sdk-go v1.15.24 h1:xLAdTA/ore6xdPAljzZRed7IGqQgC+nY+ERS5vaj4Ro=
 github.com/aws/aws-sdk-go v1.15.24/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/bsm/go-vlq v0.0.0-20150828105119-ec6e8d4f5f4e/go.mod h1:N+BjUcTjSxc2mtRGSCPsat1kze3CUtvJN3/jTXlp29k=
 github.com/bsm/redis-lock v8.0.0+incompatible h1:QgB0J2pNG8hUfndTIvpPh38F5XsUTTvO7x8Sls++9Mk=
 github.com/bsm/redis-lock v8.0.0+incompatible/go.mod h1:8dGkQ5GimBCahwF2R67tqGCJbyDZSp0gzO7wq3pDrik=
 github.com/codegangsta/negroni v1.0.0 h1:+aYywywx4bnKXWvoWtRfJ91vC59NbEhEY03sZjQhbVY=
@@ -54,6 +60,7 @@ github.com/go-redis/redis v6.15.2+incompatible h1:9SpNVG76gr6InJGxoZ6IuuxaCOQwDA
 github.com/go-redis/redis v6.15.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
 github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/go-test/deep v1.0.1/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
 github.com/gobuffalo/envy v1.6.7 h1:XMZGuFqTupAXhZTriQ+qO38QvNOSU/0rl3hEPCFci/4=
 github.com/gobuffalo/envy v1.6.7/go.mod h1:N+GkhhZ/93bGZc6ZKhJLP6+m+tCNPKwgSpH9kaifseQ=
 github.com/gobuffalo/httptest v1.0.4 h1:P0uKaPEjti1bbJmuBILE3QQ7iU1cS7oIkxVba5HbcVE=
@@ -65,6 +72,7 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekf
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 h1:LbsanbbD6LieFkXbj9YNNBupiGHJgFeLpO0j0Fza1h8=
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
@@ -99,13 +107,18 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92Bcuy
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.4.1 h1:pX7cnDwSSmG0dR9yNjCQSSpmsJOqFdT7SzVp5Yl9uVw=
 github.com/grpc-ecosystem/grpc-gateway v1.4.1/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
+github.com/hashicorp/errwrap v0.0.0-20180715044906-d6c0cd880357/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-multierror v0.0.0-20180717150148-3d5d8f294aa0/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I=
 github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o=
 github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
+github.com/hashicorp/hcl2 v0.0.0-20190503213020-640445e16309 h1:VBvyXC+b6Ix/MXMGIrOHjq+Ew1IRP52EzoQXf1KpwZo=
+github.com/hashicorp/hcl2 v0.0.0-20190503213020-640445e16309/go.mod h1:4oI94iqF3GB10QScn46WqbG0kgTUpha97SAzzg2+2ec=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8 h1:12VvqtR6Aowv3l/EQUlocDHW2Cp4G9WJVH7uyH8QFJE=
 github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc=
@@ -125,6 +138,8 @@ github.com/kr/pty v1.0.0/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4=
+github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
 github.com/markbates/hmax v1.0.0 h1:yo2N0gBoCnUMKhV/VRLHomT6Y9wUm+oQQENuWJqCdlM=
 github.com/markbates/hmax v1.0.0/go.mod h1:cOkR9dktiESxIMu+65oc/r/bdY4bE8zZw3OLhLx0X2c=
 github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4=
@@ -139,11 +154,15 @@ github.com/minio/minio-go v6.0.5+incompatible h1:qxQQW40lV2vuE9i6yYmt90GSJlT1YrM
 github.com/minio/minio-go v6.0.5+incompatible/go.mod h1:7guKYtitv8dktvNUGrhzmNlA5wrAABTQXCoesZdFQO8=
 github.com/mitchellh/go-homedir v1.0.0 h1:vKb8ShqSby24Yrqr/yDYkuFz8d0WUjys40rvnGC8aR0=
 github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7 h1:DpOJ2HYzCv8LZP15IdmG+YdwD2luVPHITV96TkirNBM=
+github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo=
 github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
 github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.4.2 h1:3mYCb7aPxS/RU7TI1y4rkEn1oKmPRjNJLNEXgw7MH2I=
 github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
 github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ=
 github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
@@ -164,6 +183,8 @@ github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7q
 github.com/prometheus/procfs v0.0.0-20180612222113-7d6f385de8be/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273 h1:agujYaXJSxSo18YNX3jzl+4G6Bstwt+kqv47GS12uL0=
 github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
+github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
 github.com/sirupsen/logrus v1.1.1 h1:VzGj7lhU7KEB9e9gMpAV/v5XT2NVSvLJhJLCWbnkgXg=
 github.com/sirupsen/logrus v1.1.1/go.mod h1:zrgwTnHtNr00buQ1vSptGe8m1f/BbgsPukg8qsT7A+A=
@@ -178,6 +199,7 @@ github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B
 github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/pflag v1.0.1 h1:aCvUg6QPl3ibpQUxyLkrEkCHtPqYJL4x9AuhqVqFis4=
 github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
@@ -196,12 +218,15 @@ github.com/ugorji/go v1.1.1/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJ
 github.com/unrolled/secure v0.0.0-20181221173256-0d6b5bb13069 h1:RKeYksgIwGE8zFJTvXI1WWx09QPrGyaVFMy0vpU7j/o=
 github.com/unrolled/secure v0.0.0-20181221173256-0d6b5bb13069/go.mod h1:mnPT77IAdsi/kV7+Es7y+pXALeV3h7G6dQF6mNYjcLA=
 github.com/urfave/cli v1.18.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
+github.com/vmihailenco/msgpack v3.3.3+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk=
 github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk=
 github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
 github.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0=
 github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
+github.com/zclconf/go-cty v0.0.0-20190426224007-b18a157db9e2 h1:Ai1LhlYNEqE39zGU07qHDNJ41iZVPZfZr1dSCoXrp1w=
+github.com/zclconf/go-cty v0.0.0-20190426224007-b18a157db9e2/go.mod h1:xnAOWiHeOqg2nWS62VtQ7pbOu17FtxJNW8RLEih+O3s=
 go.etcd.io/bbolt v1.3.2 h1:Z/90sZLPOeCy2PwprqkFa25PdkusRzaj9P8zm/KNyvk=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/etcd v0.0.0-20190215181705-784daa04988c h1:pkiZ418C7QN/HIps1lDF1+lzZhdgMpvFN4kDcxrYhD0=
@@ -220,26 +245,43 @@ golang.org/x/crypto v0.0.0-20180608092829-8ac0e0d97ce4/go.mod h1:6SG95UA2DQfeDnf
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181029103014-dab2b1051b5d h1:5JyY8HlzxzYI+qHOOciM8s2lJbIEaefMUdtYt7dRDrg=
 golang.org/x/crypto v0.0.0-20181029103014-dab2b1051b5d/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 h1:p/H982KKEjUnLJkM3tt/LemDnOc1GiZL5FCVlORJ5zo=
+golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180811021610-c39426892332/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd h1:nTDtHvHSdCn1m6ITfMRqtOd/9+7a3s8RBNOZ3eYZzJA=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181029044818-c44066c5c816 h1:mVFkLpejdFLXVUv9E42f3XJVfMdqd0IVLVIVLjZWn5o=
 golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190502183928-7f726cade0ab h1:9RfW3ktsOZxgo9YNbBAjq1FWzc/igwEcUzZz8IXgSbk=
+golang.org/x/net v0.0.0-20190502183928-7f726cade0ab/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/oauth2 v0.0.0-20180620175406-ef147856a6dd h1:QQhib242ErYDSMitlBm8V7wYCm/1a25hV8qMadIKLPA=
 golang.org/x/oauth2 v0.0.0-20180620175406-ef147856a6dd/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e h1:o3PsSEY8E4eXWkXrIP9YJALUkVZqzHJT5DOasTyn8Vs=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181031143558-9b800f95dbbc h1:SdCq5U4J+PpbSDIl9bM0V1e1Ug1jsnBkAFvTs1htn7U=
 golang.org/x/sys v0.0.0-20181031143558-9b800f95dbbc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190502175342-a43fa875dd82 h1:vsphBvatvfbhlb4PO1BYSr9dzugGxJ/SQHoNufZJq1w=
+golang.org/x/sys v0.0.0-20190502175342-a43fa875dd82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 h1:+DCIGbF/swA92ohVg0//6X2IVY3KZs6p9mix0ziNYJM=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf h1:rjxqQmxjyqerRKEj+tZW+MCm4LgpFXu18bsEoCMgDsk=
 google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.3.0 h1:FBSsiFRMz3LBeXIomRnVzrQwSDj4ibvcRexLG0LZGQk=
 google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/genproto v0.0.0-20180608181217-32ee49c4dd80/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
@@ -267,3 +309,4 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0=

pkg/config/config.go

@@ -8,6 +8,7 @@ import (
 	"runtime"
 
 	"github.com/BurntSushi/toml"
+	"github.com/gomods/athens/pkg/download/mode"
 	"github.com/gomods/athens/pkg/errors"
 	"github.com/kelseyhightower/envconfig"
 	validator "gopkg.in/go-playground/validator.v9"
@@ -18,32 +19,34 @@ const defaultConfigFile = "athens.toml"
 // Config provides configuration values for all components
 type Config struct {
 	TimeoutConf
-	GoEnv            string   `validate:"required" envconfig:"GO_ENV"`
+	GoEnv            string    `validate:"required" envconfig:"GO_ENV"`
-	GoBinary         string   `validate:"required" envconfig:"GO_BINARY_PATH"`
+	GoBinary         string    `validate:"required" envconfig:"GO_BINARY_PATH"`
-	GoGetWorkers     int      `validate:"required" envconfig:"ATHENS_GOGET_WORKERS"`
+	GoGetWorkers     int       `validate:"required" envconfig:"ATHENS_GOGET_WORKERS"`
-	ProtocolWorkers  int      `validate:"required" envconfig:"ATHENS_PROTOCOL_WORKERS"`
+	ProtocolWorkers  int       `validate:"required" envconfig:"ATHENS_PROTOCOL_WORKERS"`
-	LogLevel         string   `validate:"required" envconfig:"ATHENS_LOG_LEVEL"`
+	LogLevel         string    `validate:"required" envconfig:"ATHENS_LOG_LEVEL"`
-	CloudRuntime     string   `validate:"required" envconfig:"ATHENS_CLOUD_RUNTIME"`
+	CloudRuntime     string    `validate:"required" envconfig:"ATHENS_CLOUD_RUNTIME"`
-	FilterFile       string   `envconfig:"ATHENS_FILTER_FILE"`
+	FilterFile       string    `envconfig:"ATHENS_FILTER_FILE"`
-	TraceExporterURL string   `envconfig:"ATHENS_TRACE_EXPORTER_URL"`
+	TraceExporterURL string    `envconfig:"ATHENS_TRACE_EXPORTER_URL"`
-	TraceExporter    string   `envconfig:"ATHENS_TRACE_EXPORTER"`
+	TraceExporter    string    `envconfig:"ATHENS_TRACE_EXPORTER"`
-	StatsExporter    string   `envconfig:"ATHENS_STATS_EXPORTER"`
+	StatsExporter    string    `envconfig:"ATHENS_STATS_EXPORTER"`
-	StorageType      string   `validate:"required" envconfig:"ATHENS_STORAGE_TYPE"`
+	StorageType      string    `validate:"required" envconfig:"ATHENS_STORAGE_TYPE"`
-	GlobalEndpoint   string   `envconfig:"ATHENS_GLOBAL_ENDPOINT"` // This feature is not yet implemented
+	GlobalEndpoint   string    `envconfig:"ATHENS_GLOBAL_ENDPOINT"` // This feature is not yet implemented
-	Port             string   `envconfig:"ATHENS_PORT"`
+	Port             string    `envconfig:"ATHENS_PORT"`
-	BasicAuthUser    string   `envconfig:"BASIC_AUTH_USER"`
+	BasicAuthUser    string    `envconfig:"BASIC_AUTH_USER"`
-	BasicAuthPass    string   `envconfig:"BASIC_AUTH_PASS"`
+	BasicAuthPass    string    `envconfig:"BASIC_AUTH_PASS"`
-	ForceSSL         bool     `envconfig:"PROXY_FORCE_SSL"`
+	ForceSSL         bool      `envconfig:"PROXY_FORCE_SSL"`
-	ValidatorHook    string   `envconfig:"ATHENS_PROXY_VALIDATOR"`
+	ValidatorHook    string    `envconfig:"ATHENS_PROXY_VALIDATOR"`
-	PathPrefix       string   `envconfig:"ATHENS_PATH_PREFIX"`
+	PathPrefix       string    `envconfig:"ATHENS_PATH_PREFIX"`
-	NETRCPath        string   `envconfig:"ATHENS_NETRC_PATH"`
+	NETRCPath        string    `envconfig:"ATHENS_NETRC_PATH"`
-	GithubToken      string   `envconfig:"ATHENS_GITHUB_TOKEN"`
+	GithubToken      string    `envconfig:"ATHENS_GITHUB_TOKEN"`
-	HGRCPath         string   `envconfig:"ATHENS_HGRC_PATH"`
+	HGRCPath         string    `envconfig:"ATHENS_HGRC_PATH"`
-	TLSCertFile      string   `envconfig:"ATHENS_TLSCERT_FILE"`
+	TLSCertFile      string    `envconfig:"ATHENS_TLSCERT_FILE"`
-	TLSKeyFile       string   `envconfig:"ATHENS_TLSKEY_FILE"`
+	TLSKeyFile       string    `envconfig:"ATHENS_TLSKEY_FILE"`
-	SumDBs           []string `envconfig:"ATHENS_SUM_DBS"`
+	SumDBs           []string  `envconfig:"ATHENS_SUM_DBS"`
-	NoSumPatterns    []string `envconfig:"ATHENS_GONOSUM_PATTERNS"`
+	NoSumPatterns    []string  `envconfig:"ATHENS_GONOSUM_PATTERNS"`
-	SingleFlightType string   `envconfig:"ATHENS_SINGLE_FLIGHT_TYPE"`
+	DownloadMode     mode.Mode `envconfig:"ATHENS_DOWNLOAD_MODE"`
+	DownloadURL      string    `envconfig:"ATHENS_DOWNLOAD_URL"`
+	SingleFlightType string    `envconfig:"ATHENS_SINGLE_FLIGHT_TYPE"`
 	SingleFlight     *SingleFlight
 	Storage          *StorageConfig
 }
@@ -84,6 +87,8 @@ func defaultConfig() *Config {
 		TraceExporterURL: "http://localhost:14268",
 		SumDBs:           []string{"https://sum.golang.org"},
 		NoSumPatterns:    []string{},
+		DownloadMode:     "sync",
+		DownloadURL:      "",
 		SingleFlight: &SingleFlight{
 			Etcd:  &Etcd{"localhost:2379,localhost:22379,localhost:32379"},
 			Redis: &Redis{"127.0.0.1:6379"},

pkg/config/config_test.go

@@ -266,6 +266,7 @@ func TestParseExampleConfig(t *testing.T) {
 		SingleFlight:     &SingleFlight{},
 		SumDBs:           []string{"https://sum.golang.org"},
 		NoSumPatterns:    []string{},
+		DownloadMode:     "sync",
 	}
 
 	absPath, err := filepath.Abs(testConfigFile(t))

pkg/download/handler.go

@@ -2,7 +2,9 @@ package download
 
 import (
 	"net/http"
+	"net/url"
 
+	"github.com/gomods/athens/pkg/download/mode"
 	"github.com/gomods/athens/pkg/log"
 	"github.com/gomods/athens/pkg/middleware"
 	"github.com/gorilla/mux"
@@ -10,13 +12,14 @@ import (
 
 // ProtocolHandler is a function that takes all that it needs to return
 // a ready-to-go http handler that serves up cmd/go's download protocol.
-type ProtocolHandler func(dp Protocol, lggr log.Entry) http.Handler
+type ProtocolHandler func(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler
 
 // HandlerOpts are the generic options
 // for a ProtocolHandler
 type HandlerOpts struct {
-	Protocol Protocol
+	Protocol     Protocol
-	Logger   *log.Logger
+	Logger       *log.Logger
+	DownloadFile *mode.DownloadFile
 }
 
 // LogEntryHandler pulls a log entry from the request context. Thanks to the
@@ -26,7 +29,7 @@ type HandlerOpts struct {
 func LogEntryHandler(ph ProtocolHandler, opts *HandlerOpts) http.Handler {
 	f := func(w http.ResponseWriter, r *http.Request) {
 		ent := log.EntryFromContext(r.Context())
-		handler := ph(opts.Protocol, ent)
+		handler := ph(opts.Protocol, ent, opts.DownloadFile)
 		handler.ServeHTTP(w, r)
 	}
 	return http.HandlerFunc(f)
@@ -51,3 +54,9 @@ func RegisterHandlers(r *mux.Router, opts *HandlerOpts) {
 	r.Handle(PathVersionModule, LogEntryHandler(ModuleHandler, opts)).Methods(http.MethodGet)
 	r.Handle(PathVersionZip, LogEntryHandler(ZipHandler, opts)).Methods(http.MethodGet)
 }
+
+func getRedirectURL(base, path string) string {
+	url, _ := url.Parse(base)
+	url.Path = path
+	return url.String()
+}

pkg/download/handler_test.go

@@ -0,0 +1,45 @@
+package download
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/gomods/athens/pkg/download/mode"
+	"github.com/gomods/athens/pkg/errors"
+	"github.com/gomods/athens/pkg/log"
+	"github.com/gorilla/mux"
+)
+
+func TestRedirect(t *testing.T) {
+	r := mux.NewRouter()
+	RegisterHandlers(r, &HandlerOpts{
+		Protocol: &mockProtocol{},
+		Logger:   log.NoOpLogger(),
+		DownloadFile: &mode.DownloadFile{
+			Mode:        mode.Redirect,
+			DownloadURL: "https://gomods.io",
+		},
+	})
+	req := httptest.NewRequest("GET", "/github.com/gomods/athens/@v/v0.4.0.info", nil)
+	w := httptest.NewRecorder()
+	r.ServeHTTP(w, req)
+	if w.Code != http.StatusMovedPermanently {
+		t.Fatalf("expected a redirect status (301) but got %v", w.Code)
+	}
+	expectedRedirect := "https://gomods.io/github.com/gomods/athens/@v/v0.4.0.info"
+	givenRedirect := w.HeaderMap.Get("location")
+	if expectedRedirect != givenRedirect {
+		t.Fatalf("expected the handler to redirect to %q but got %q", expectedRedirect, givenRedirect)
+	}
+}
+
+type mockProtocol struct {
+	Protocol
+}
+
+func (mp *mockProtocol) Info(ctx context.Context, mod, ver string) ([]byte, error) {
+	const op errors.Op = "mockProtocol.Info"
+	return nil, errors.E(op, "not found", errors.KindRedirect)
+}

pkg/download/latest.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"net/http"
 
+	"github.com/gomods/athens/pkg/download/mode"
 	"github.com/gomods/athens/pkg/errors"
 	"github.com/gomods/athens/pkg/log"
 	"github.com/gomods/athens/pkg/paths"
@@ -13,7 +14,7 @@ import (
 const PathLatest = "/{module:.+}/@latest"
 
 // LatestHandler implements GET baseURL/module/@latest
-func LatestHandler(dp Protocol, lggr log.Entry) http.Handler {
+func LatestHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler {
 	const op errors.Op = "download.LatestHandler"
 	f := func(w http.ResponseWriter, r *http.Request) {
 		mod, err := paths.GetModule(r)

pkg/download/list.go

@@ -5,6 +5,7 @@ import (
 	"net/http"
 	"strings"
 
+	"github.com/gomods/athens/pkg/download/mode"
 	"github.com/gomods/athens/pkg/errors"
 	"github.com/gomods/athens/pkg/log"
 	"github.com/gomods/athens/pkg/paths"
@@ -14,7 +15,7 @@ import (
 const PathList = "/{module:.+}/@v/list"
 
 // ListHandler implements GET baseURL/module/@v/list
-func ListHandler(dp Protocol, lggr log.Entry) http.Handler {
+func ListHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler {
 	const op errors.Op = "download.ListHandler"
 	f := func(w http.ResponseWriter, r *http.Request) {
 		mod, err := paths.GetModule(r)

pkg/download/list_merge_test.go

@@ -128,7 +128,7 @@ func TestListMerge(t *testing.T) {
 				s.Save(ctx, testModName, v, bts, ioutil.NopCloser(bytes.NewReader(bts)), bts)
 			}
 			defer clearStorage(s, testModName, tc.strVersions)
-			dp := New(&Opts{s, nil, &listerMock{versions: tc.goVersions, err: tc.goErr}})
+			dp := New(&Opts{s, nil, &listerMock{versions: tc.goVersions, err: tc.goErr}, nil})
 			list, err := dp.List(ctx, testModName)
 
 			if ok := testErrEq(tc.expectedErr, err); !ok {

pkg/download/mode/mode.go

@@ -0,0 +1,132 @@
+package mode
+
+import (
+	"encoding/base64"
+	"fmt"
+	"io/ioutil"
+	"path"
+	"strings"
+
+	"github.com/gomods/athens/pkg/errors"
+	"github.com/hashicorp/hcl2/gohcl"
+	"github.com/hashicorp/hcl2/hclparse"
+)
+
+// Mode specifies the behavior of what to do
+// when a module is not found in storage.
+type Mode string
+
+// DownloadMode constants. For more information see config.dev.toml
+const (
+	Sync          Mode = "sync"
+	Async         Mode = "async"
+	Redirect      Mode = "redirect"
+	AsyncRedirect Mode = "async_redirect"
+	None          Mode = "none"
+)
+
+// DownloadFile represents a custom HCL format of
+// how to handle module@version requests that are
+// not found in storage.
+type DownloadFile struct {
+	Mode        Mode            `hcl:"mode"`
+	DownloadURL string          `hcl:"downloadURL"`
+	Paths       []*DownloadPath `hcl:"download,block"`
+}
+
+// DownloadPath represents a custom Mode for
+// a matching path.
+type DownloadPath struct {
+	Pattern     string `hcl:"pattern,label"`
+	Mode        Mode   `hcl:"mode"`
+	DownloadURL string `hcl:"downloadURL,optional"`
+}
+
+// NewFile takes a mode and returns a DownloadFile.
+// Mode can be one of the constants declared above
+// or a custom HCL file. To pass a custom HCL file,
+// you can either point to a file path by passing
+// file:/path/to/file OR custom:<base64-encoded-hcl>
+// directly.
+func NewFile(m Mode, downloadURL string) (*DownloadFile, error) {
+	const op errors.Op = "downloadMode.NewFile"
+	if strings.HasPrefix(string(m), "file:") {
+		filePath := string(m[5:])
+		bts, err := ioutil.ReadFile(filePath)
+		if err != nil {
+			return nil, err
+		}
+		return parseFile(bts)
+	} else if strings.HasPrefix(string(m), "custom:") {
+		bts, err := base64.StdEncoding.DecodeString(string(m[7:]))
+		if err != nil {
+			return nil, err
+		}
+		return parseFile(bts)
+	}
+	switch m {
+	case Sync, Async, Redirect, AsyncRedirect, None:
+		return &DownloadFile{Mode: m, DownloadURL: downloadURL}, nil
+	default:
+		return nil, errors.E(op, "unrecognized download mode: "+m, errors.KindBadRequest)
+	}
+}
+
+// parseFile parses an HCL file according to the
+// DownloadMode spec.
+func parseFile(file []byte) (*DownloadFile, error) {
+	const op errors.Op = "downloadmode.parseFile"
+	f, dig := hclparse.NewParser().ParseHCL(file, "config.hcl")
+	if dig.HasErrors() {
+		return nil, errors.E(op, dig.Error())
+	}
+	var df DownloadFile
+	dig = gohcl.DecodeBody(f.Body, nil, &df)
+	if dig.HasErrors() {
+		return nil, errors.E(op, dig.Error())
+	}
+	if err := df.validate(); err != nil {
+		return nil, errors.E(op, err)
+	}
+	return &df, nil
+}
+
+func (d *DownloadFile) validate() error {
+	const op errors.Op = "downloadMode.validate"
+	for _, p := range d.Paths {
+		switch p.Mode {
+		case Sync, Async, Redirect, AsyncRedirect, None:
+		default:
+			return errors.E(op, fmt.Errorf("unrecognized mode for %v: %v", p.Pattern, p.Mode))
+		}
+	}
+	return nil
+}
+
+// Match returns the Mode that matches the given
+// module. A pattern is prioritized by order in
+// which it appears in the HCL file, while the
+// default Mode will be returned if no patterns
+// exist or match.
+func (d *DownloadFile) Match(mod string) Mode {
+	for _, p := range d.Paths {
+		if hasMatch, err := path.Match(p.Pattern, mod); hasMatch && err == nil {
+			return p.Mode
+		}
+	}
+	return d.Mode
+}
+
+// URL returns the redirect URL that applies
+// to the given module. If no pattern matches,
+// the top level downloadURL is returned.
+func (d *DownloadFile) URL(mod string) string {
+	for _, p := range d.Paths {
+		if hasMatch, err := path.Match(p.Pattern, mod); hasMatch && err == nil {
+			if p.DownloadURL != "" {
+				return p.DownloadURL
+			}
+		}
+	}
+	return d.DownloadURL
+}

pkg/download/mode/mode_test.go

@@ -0,0 +1,96 @@
+package mode
+
+import (
+	"testing"
+)
+
+var testCases = []struct {
+	name         string
+	file         *DownloadFile
+	input        string
+	expectedMode Mode
+	expectedURL  string
+}{
+	{
+		name:         "sync",
+		file:         &DownloadFile{Mode: Sync},
+		input:        "github.com/gomods/athens",
+		expectedMode: Sync,
+	},
+	{
+		name:         "redirect",
+		file:         &DownloadFile{Mode: Redirect, DownloadURL: "gomods.io"},
+		input:        "github.com/gomods/athens",
+		expectedMode: Redirect,
+		expectedURL:  "gomods.io",
+	},
+	{
+		name: "pattern match",
+		file: &DownloadFile{
+			Mode: Sync,
+			Paths: []*DownloadPath{
+				{Pattern: "github.com/gomods/*", Mode: None},
+			},
+		},
+		input:        "github.com/gomods/athens",
+		expectedMode: None,
+	},
+	{
+		name: "pattern fallback",
+		file: &DownloadFile{
+			Mode: Sync,
+			Paths: []*DownloadPath{
+				{Pattern: "github.com/gomods/*", Mode: None},
+			},
+		},
+		input:        "github.com/athens-artifacts/maturelib",
+		expectedMode: Sync,
+	},
+	{
+		name: "pattern redirect",
+		file: &DownloadFile{
+			Mode: Sync,
+			Paths: []*DownloadPath{
+				{
+					Pattern:     "github.com/gomods/*",
+					Mode:        AsyncRedirect,
+					DownloadURL: "gomods.io"},
+			},
+		},
+		input:        "github.com/gomods/athens",
+		expectedMode: AsyncRedirect,
+		expectedURL:  "gomods.io",
+	},
+	{
+		name: "redirect fallback",
+		file: &DownloadFile{
+			Mode:        Redirect,
+			DownloadURL: "proxy.golang.org",
+			Paths: []*DownloadPath{
+				{
+					Pattern:     "github.com/gomods/*",
+					Mode:        AsyncRedirect,
+					DownloadURL: "gomods.io",
+				},
+			},
+		},
+		input:        "github.com/athens-artifacts/maturelib",
+		expectedMode: Redirect,
+		expectedURL:  "proxy.golang.org",
+	},
+}
+
+func TestMode(t *testing.T) {
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			givenMode := tc.file.Match(tc.input)
+			if givenMode != tc.expectedMode {
+				t.Fatalf("expected matched mode to be %q but got %q", tc.expectedMode, givenMode)
+			}
+			givenURL := tc.file.URL(tc.input)
+			if givenURL != tc.expectedURL {
+				t.Fatalf("expected matched DownloadURL to be %q but got %q", tc.expectedURL, givenURL)
+			}
+		})
+	}
+}

pkg/download/protocol.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/gomods/athens/pkg/download/mode"
 	"github.com/gomods/athens/pkg/errors"
 	"github.com/gomods/athens/pkg/observ"
 	"github.com/gomods/athens/pkg/stash"
@@ -37,9 +38,10 @@ type Wrapper func(Protocol) Protocol
 
 // Opts specifies download protocol options to avoid long func signature.
 type Opts struct {
-	Storage storage.Backend
+	Storage      storage.Backend
-	Stasher stash.Stasher
+	Stasher      stash.Stasher
-	Lister  UpstreamLister
+	Lister       UpstreamLister
+	DownloadFile *mode.DownloadFile
 }
 
 // New returns a full implementation of the download.Protocol
@@ -48,7 +50,10 @@ type Opts struct {
 // The wrappers are applied in order, meaning the last wrapper
 // passed is the Protocol that gets hit first.
 func New(opts *Opts, wrappers ...Wrapper) Protocol {
-	var p Protocol = &protocol{opts.Storage, opts.Stasher, opts.Lister}
+	if opts.DownloadFile == nil {
+		opts.DownloadFile = &mode.DownloadFile{Mode: mode.Sync}
+	}
+	var p Protocol = &protocol{opts.DownloadFile, opts.Storage, opts.Stasher, opts.Lister}
 	for _, w := range wrappers {
 		p = w(p)
 	}
@@ -57,6 +62,7 @@ func New(opts *Opts, wrappers ...Wrapper) Protocol {
 }
 
 type protocol struct {
+	df      *mode.DownloadFile
 	storage storage.Backend
 	stasher stash.Stasher
 	lister  UpstreamLister
@@ -151,13 +157,11 @@ func (p *protocol) Info(ctx context.Context, mod, ver string) ([]byte, error) {
 	ctx, span := observ.StartSpan(ctx, op.String())
 	defer span.End()
 	info, err := p.storage.Info(ctx, mod, ver)
-	var newVer string
 	if errors.IsNotFoundErr(err) {
-		newVer, err = p.stasher.Stash(ctx, mod, ver)
+		err = p.processDownload(ctx, mod, ver, func(newVer string) error {
-		if err != nil {
+			info, err = p.storage.Info(ctx, mod, newVer)
-			return nil, errors.E(op, err)
+			return err
-		}
+		})
-		info, err = p.storage.Info(ctx, mod, newVer)
 	}
 	if err != nil {
 		return nil, errors.E(op, err)
@@ -171,18 +175,15 @@ func (p *protocol) GoMod(ctx context.Context, mod, ver string) ([]byte, error) {
 	ctx, span := observ.StartSpan(ctx, op.String())
 	defer span.End()
 	goMod, err := p.storage.GoMod(ctx, mod, ver)
-	var newVer string
 	if errors.IsNotFoundErr(err) {
-		newVer, err = p.stasher.Stash(ctx, mod, ver)
+		err = p.processDownload(ctx, mod, ver, func(newVer string) error {
-		if err != nil {
+			goMod, err = p.storage.GoMod(ctx, mod, newVer)
-			return nil, errors.E(op, err)
+			return err
-		}
+		})
-		goMod, err = p.storage.GoMod(ctx, mod, newVer)
 	}
 	if err != nil {
 		return nil, errors.E(op, err)
 	}
-
 	return goMod, nil
 }
 
@@ -191,13 +192,11 @@ func (p *protocol) Zip(ctx context.Context, mod, ver string) (io.ReadCloser, err
 	ctx, span := observ.StartSpan(ctx, op.String())
 	defer span.End()
 	zip, err := p.storage.Zip(ctx, mod, ver)
-	var newVer string
 	if errors.IsNotFoundErr(err) {
-		newVer, err = p.stasher.Stash(ctx, mod, ver)
+		err = p.processDownload(ctx, mod, ver, func(newVer string) error {
-		if err != nil {
+			zip, err = p.storage.Zip(ctx, mod, newVer)
-			return nil, errors.E(op, err)
+			return err
-		}
+		})
-		zip, err = p.storage.Zip(ctx, mod, newVer)
 	}
 	if err != nil {
 		return nil, errors.E(op, err)
@@ -206,6 +205,29 @@ func (p *protocol) Zip(ctx context.Context, mod, ver string) (io.ReadCloser, err
 	return zip, nil
 }
 
+func (p *protocol) processDownload(ctx context.Context, mod, ver string, f func(newVer string) error) error {
+	const op errors.Op = "protocol.processDownload"
+	switch p.df.Match(mod) {
+	case mode.Sync:
+		newVer, err := p.stasher.Stash(ctx, mod, ver)
+		if err != nil {
+			return errors.E(op, err)
+		}
+		return f(newVer)
+	case mode.Async:
+		go p.stasher.Stash(ctx, mod, ver)
+		return errors.E(op, "async: module not found", errors.KindNotFound)
+	case mode.Redirect:
+		return errors.E(op, "redirect", errors.KindRedirect)
+	case mode.AsyncRedirect:
+		go p.stasher.Stash(ctx, mod, ver)
+		return errors.E(op, "async_redirect: module not found", errors.KindRedirect)
+	case mode.None:
+		return errors.E(op, "none", errors.KindNotFound)
+	}
+	return nil
+}
+
 // union concatenates two version lists and removes duplicates
 func union(list1, list2 []string) []string {
 	if list1 == nil {

pkg/download/protocol_test.go

@@ -13,6 +13,7 @@ import (
 	"time"
 
 	"github.com/gomods/athens/pkg/config"
+	"github.com/gomods/athens/pkg/download/mode"
 	"github.com/gomods/athens/pkg/errors"
 	"github.com/gomods/athens/pkg/module"
 	"github.com/gomods/athens/pkg/stash"
@@ -44,7 +45,7 @@ func getDP(t *testing.T) Protocol {
 		t.Fatal(err)
 	}
 	st := stash.New(mf, s)
-	return New(&Opts{s, st, NewVCSLister(goBin, fs)})
+	return New(&Opts{s, st, NewVCSLister(goBin, fs), nil})
 }
 
 type listTest struct {
@@ -280,7 +281,7 @@ func TestDownloadProtocol(t *testing.T) {
 	}
 	mp := &mockFetcher{}
 	st := stash.New(mp, s)
-	dp := New(&Opts{s, st, nil})
+	dp := New(&Opts{s, st, nil, nil})
 	ctx := context.Background()
 
 	var eg errgroup.Group
@@ -332,7 +333,7 @@ func TestDownloadProtocolWhenFetchFails(t *testing.T) {
 	}
 	mp := &notFoundFetcher{}
 	st := stash.New(mp, s)
-	dp := New(&Opts{s, st, nil})
+	dp := New(&Opts{s, st, nil, nil})
 	ctx := context.Background()
 	_, err = dp.GoMod(ctx, fakeMod.mod, fakeMod.ver)
 	if err != nil {
@@ -340,6 +341,40 @@ func TestDownloadProtocolWhenFetchFails(t *testing.T) {
 	}
 }
 
+func TestAsyncRedirect(t *testing.T) {
+	s, err := mem.NewStorage()
+	require.NoError(t, err)
+	ms := &mockStasher{s, make(chan bool)}
+	dp := New(&Opts{
+		Stasher: ms,
+		Storage: s,
+		DownloadFile: &mode.DownloadFile{
+			Mode:        mode.Async,
+			DownloadURL: "https://gomods.io",
+		},
+	})
+	mod, ver := "github.com/athens-artifacts/happy-path", "v0.0.1"
+	_, err = dp.Info(context.Background(), mod, ver)
+	if errors.Kind(err) != errors.KindNotFound {
+		t.Fatalf("expected async_redirect to enforce a 404 but got %v", errors.Kind(err))
+	}
+	<-ms.ch
+	info, err := dp.Info(context.Background(), mod, ver)
+	require.NoError(t, err)
+	require.Equal(t, string(info), "info", "expected async fetch to be successful")
+}
+
+type mockStasher struct {
+	s  storage.Backend
+	ch chan bool
+}
+
+func (ms *mockStasher) Stash(ctx context.Context, mod string, ver string) (string, error) {
+	err := ms.s.Save(ctx, mod, ver, []byte("mod"), strings.NewReader("zip"), []byte("info"))
+	ms.ch <- true // signal async stashing is done
+	return ver, err
+}
+
 type notFoundFetcher struct{}
 
 func (m *notFoundFetcher) Fetch(ctx context.Context, mod, ver string) (*storage.Version, error) {

pkg/download/version_info.go

@@ -3,6 +3,7 @@ package download
 import (
 	"net/http"
 
+	"github.com/gomods/athens/pkg/download/mode"
 	"github.com/gomods/athens/pkg/errors"
 	"github.com/gomods/athens/pkg/log"
 )
@@ -11,7 +12,7 @@ import (
 const PathVersionInfo = "/{module:.+}/@v/{version}.info"
 
 // InfoHandler implements GET baseURL/module/@v/version.info
-func InfoHandler(dp Protocol, lggr log.Entry) http.Handler {
+func InfoHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler {
 	const op errors.Op = "download.InfoHandler"
 	f := func(w http.ResponseWriter, r *http.Request) {
 		mod, ver, err := getModuleParams(r, op)
@@ -23,6 +24,10 @@ func InfoHandler(dp Protocol, lggr log.Entry) http.Handler {
 		info, err := dp.Info(r.Context(), mod, ver)
 		if err != nil {
 			lggr.SystemErr(errors.E(op, err, errors.M(mod), errors.V(ver)))
+			if errors.Kind(err) == errors.KindRedirect {
+				http.Redirect(w, r, getRedirectURL(df.URL(mod), r.URL.Path), errors.KindRedirect)
+				return
+			}
 			w.WriteHeader(errors.Kind(err))
 		}
 

pkg/download/version_module.go

@@ -3,6 +3,7 @@ package download
 import (
 	"net/http"
 
+	"github.com/gomods/athens/pkg/download/mode"
 	"github.com/gomods/athens/pkg/errors"
 	"github.com/gomods/athens/pkg/log"
 )
@@ -11,12 +12,16 @@ import (
 const PathVersionModule = "/{module:.+}/@v/{version}.mod"
 
 // ModuleHandler implements GET baseURL/module/@v/version.mod
-func ModuleHandler(dp Protocol, lggr log.Entry) http.Handler {
+func ModuleHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler {
 	const op errors.Op = "download.VersionModuleHandler"
 	f := func(w http.ResponseWriter, r *http.Request) {
 		mod, ver, err := getModuleParams(r, op)
 		if err != nil {
 			lggr.SystemErr(err)
+			if errors.Kind(err) == errors.KindRedirect {
+				http.Redirect(w, r, getRedirectURL(df.URL(mod), r.URL.Path), errors.KindRedirect)
+				return
+			}
 			w.WriteHeader(errors.Kind(err))
 			return
 		}

pkg/download/version_zip.go

@@ -4,6 +4,7 @@ import (
 	"io"
 	"net/http"
 
+	"github.com/gomods/athens/pkg/download/mode"
 	"github.com/gomods/athens/pkg/errors"
 	"github.com/gomods/athens/pkg/log"
 )
@@ -12,7 +13,7 @@ import (
 const PathVersionZip = "/{module:.+}/@v/{version}.zip"
 
 // ZipHandler implements GET baseURL/module/@v/version.zip
-func ZipHandler(dp Protocol, lggr log.Entry) http.Handler {
+func ZipHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler {
 	const op errors.Op = "download.ZipHandler"
 	f := func(w http.ResponseWriter, r *http.Request) {
 		mod, ver, err := getModuleParams(r, op)
@@ -24,6 +25,10 @@ func ZipHandler(dp Protocol, lggr log.Entry) http.Handler {
 		zip, err := dp.Zip(r.Context(), mod, ver)
 		if err != nil {
 			lggr.SystemErr(err)
+			if errors.Kind(err) == errors.KindRedirect {
+				http.Redirect(w, r, getRedirectURL(df.URL(mod), r.URL.Path), errors.KindRedirect)
+				return
+			}
 			w.WriteHeader(errors.Kind(err))
 			return
 		}

pkg/errors/errors.go

@@ -17,6 +17,7 @@ const (
 	KindAlreadyExists  = http.StatusConflict
 	KindRateLimit      = http.StatusTooManyRequests
 	KindNotImplemented = http.StatusNotImplemented
+	KindRedirect       = http.StatusMovedPermanently
 )
 
 // Error is an Athens system error.