mirror of
https://github.com/gomods/athens
synced 2026-02-03 07:30:32 +00:00
downloadProtocol: support multi-proxy environments with DownloadFile (#1230)
* downloadProtocol: support multi-proxy environments with DownloadFile * remove debugging lines * update config tests * download/mode: add tests for DownloadFile and friends * add documentation to Download File
This commit is contained in:
Marwan Sulaiman
@@ -10,6 +10,7 @@ import (
|
||||
"github.com/gomods/athens/pkg/config"
|
||||
"github.com/gomods/athens/pkg/download"
|
||||
"github.com/gomods/athens/pkg/download/addons"
|
||||
"github.com/gomods/athens/pkg/download/mode"
|
||||
"github.com/gomods/athens/pkg/log"
|
||||
"github.com/gomods/athens/pkg/module"
|
||||
"github.com/gomods/athens/pkg/stash"
|
||||
@@ -85,14 +86,21 @@ func addProxyRoutes(
|
||||
}
|
||||
st := stash.New(mf, s, stash.WithPool(c.GoGetWorkers), withSingleFlight)
|
||||
|
||||
df, err := mode.NewFile(c.DownloadMode, c.DownloadURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
dpOpts := &download.Opts{
|
||||
Storage: s,
|
||||
Stasher: st,
|
||||
Lister: lister,
|
||||
DownloadFile: df,
|
||||
}
|
||||
|
||||
dp := download.New(dpOpts, addons.WithPool(c.ProtocolWorkers))
|
||||
|
||||
handlerOpts := &download.HandlerOpts{Protocol: dp, Logger: l}
|
||||
handlerOpts := &download.HandlerOpts{Protocol: dp, Logger: l, DownloadFile: df}
|
||||
download.RegisterHandlers(r, handlerOpts)
|
||||
|
||||
return nil
|
||||
|
||||
@@ -176,6 +176,31 @@ SumDBs = ["https://sum.golang.org"]
|
||||
# Env override: ATHENS_GONOSUM_PATTERNS
|
||||
NoSumPatterns = []
|
||||
|
||||
# DownloadMode defines how Athens behaves when a module@version
|
||||
# is not found in storage. There are 4 options:
|
||||
# 1. "sync" (default): download the module synchronously and
|
||||
# return the results to the client.
|
||||
# 2. "async": return 404, but asynchronously store the module
|
||||
# in the storage backend.
|
||||
# 3. "redirect": return a 301 redirect status to the client
|
||||
# with the base URL as the DownloadRedirectURL from below.
|
||||
# 4. "async_redirect": same as option number 3 but it will
|
||||
# asynchronously store the module to the backend.
|
||||
# 5. "none": return 404 if a module is not found and do nothing.
|
||||
# 6. "file:<path>": will point to an HCL file that specifies
|
||||
# any of the 5 options above based on different import paths.
|
||||
# 7. "custom:<base64-encoded-hcl>" is the same as option 6
|
||||
# but the file is fully encoded in the option. This is
|
||||
# useful for using an environment variable in serverless
|
||||
# deployments.
|
||||
# Env override: ATHENS_DOWNLOAD_MODE
|
||||
DownloadMode = "sync"
|
||||
|
||||
# DownloadURL is the URL that will be used if
|
||||
# DownloadMode is set to "redirect"
|
||||
# Env override: ATHENS_DOWNLOAD_URL
|
||||
DownloadURL = ""
|
||||
|
||||
# SingleFlightType determines what mechanism Athens uses
|
||||
# to manage concurrency flowing into the Athens Backend.
|
||||
# This is important for the following scenario: if two concurrent requests
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
---
|
||||
title: Download Mode
|
||||
description: What to do when a module is not in storage
|
||||
weight: 2
|
||||
---
|
||||
|
||||
Athens accepts an HCL formatted Download File that serves as the source of truth for answering the following question:
|
||||
|
||||
### What should Athens do when a module@version is not found in storage?
|
||||
|
||||
Say a client sends an HTTP request with the path `/github.com/pkg/errors/@v/v0.8.1` and Athens
|
||||
does not have this module in storage. Athens will look at the Download File for one of the following Modes:
|
||||
|
||||
1. **`sync`**: Synchronously download the module from VCS via `go mod download`, persist it to the Athens storage, and serve it back to the user immediately. Note that this is the default behavior.
|
||||
2. **`async`**: Return a 404 to the client, and asynchronously download and persist the module@version to storage.
|
||||
3. **`none`**: Return a 404 and do nothing.
|
||||
4. **`redirect`**: Redirect to an upstream proxy (such as proxy.golang.org) and do nothing after.
|
||||
5. **`async_redirect`**: Redirect to an upstream proxy (such as proxy.golang.org) and asynchronously download and persist the module@version to storage.
|
||||
|
||||
Furthermore, the Download File can describe any of the above behavior for different modules and module patterns alike using [path.Match](https://golang.org/pkg/path/#Match). Take a look at the following example:
|
||||
|
||||
```javascript
|
||||
downloadURL = "https://proxy.golang.org"
|
||||
|
||||
mode = "async_redirect"
|
||||
|
||||
download "github.com/gomods/*" {
|
||||
mode = "sync"
|
||||
}
|
||||
|
||||
download "golang.org/x/*" {
|
||||
mode = "none"
|
||||
}
|
||||
|
||||
download "github.com/pkg/*" {
|
||||
mode = "redirect"
|
||||
downloadURL = "https://gocenter.io"
|
||||
}
|
||||
```
|
||||
|
||||
The first two lines describe the behavior and the destination of all packages: redirect to `https://proxy.golang.org` and asynchronously persist the module to storage.
|
||||
|
||||
The following two blocks describe what to do if the requested module matches the given pattern:
|
||||
|
||||
Any module that matches "github.com/gomods/*" such as "github.com/gomods/athens", will be synchronously fetched, stored, and returned to the user.
|
||||
|
||||
Any module that matches "golang.org/x/*" such as "golang.org/x/text" will just return a 404. Note that this behavior allows the client to set GOPROXY to multiple comma separated proxies so that the Go command can move into the second argument.
|
||||
|
||||
Any module that matches "github.com/pkg/*" such as "github.com/pkg/errors" will be redirected to https://gocenter.io (and not proxy.golang.org) and will also never persist the module to the Athens storage.
|
||||
|
||||
|
||||
## Use cases
|
||||
|
||||
So why would you want to use the Download File to configure the behavior above? Here are a few use cases where it might make sense for you to do so:
|
||||
|
||||
**Limited storage:**
|
||||
|
||||
If you have limited storage, then it might be a good idea to only persist some moduels (such as private ones) and then redirect to a public proxy for everything else.
|
||||
|
||||
**Limited resources:**
|
||||
|
||||
If you are running Athens with low memory/cpu, then you can redirect all public modules to proxy.golang.org but asynchronously fetch them so that the client does not timeout. At the same time, you can return a 404 for private modules through the `none` mode and let the client (the Go command) fetch private modules directly through `GOPROXY=<athens-url>,direct`
|
||||
@@ -0,0 +1,16 @@
|
||||
downloadURL = "https://proxy.golang.org"
|
||||
|
||||
mode = "async_redirect"
|
||||
|
||||
download "github.com/gomods/*" {
|
||||
mode = "sync"
|
||||
}
|
||||
|
||||
download "golang.org/x/*" {
|
||||
mode = "none"
|
||||
}
|
||||
|
||||
download "github.com/pkg/*" {
|
||||
mode = "redirect"
|
||||
downloadURL = "https://gocenter.io"
|
||||
}
|
||||
@@ -26,10 +26,10 @@ require (
|
||||
github.com/gopherjs/gopherjs v0.0.0-20180825215210-0210a2f0f73c // indirect
|
||||
github.com/gorilla/mux v1.6.2
|
||||
github.com/hashicorp/go-multierror v1.0.0
|
||||
github.com/hashicorp/hcl2 v0.0.0-20190503213020-640445e16309
|
||||
github.com/jtolds/gls v4.2.1+incompatible // indirect
|
||||
github.com/kelseyhightower/envconfig v1.3.0
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1 // indirect
|
||||
github.com/kr/pretty v0.1.0 // indirect
|
||||
github.com/minio/minio-go v6.0.5+incompatible
|
||||
github.com/mitchellh/go-homedir v1.0.0
|
||||
github.com/philhofer/fwd v1.0.0 // indirect
|
||||
@@ -48,15 +48,11 @@ require (
|
||||
go.etcd.io/etcd v0.0.0-20190215181705-784daa04988c
|
||||
go.mongodb.org/mongo-driver v1.0.0
|
||||
go.opencensus.io v0.17.0
|
||||
golang.org/x/crypto v0.0.0-20181029103014-dab2b1051b5d // indirect
|
||||
golang.org/x/net v0.0.0-20181029044818-c44066c5c816 // indirect
|
||||
golang.org/x/oauth2 v0.0.0-20180620175406-ef147856a6dd
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f
|
||||
golang.org/x/sys v0.0.0-20181031143558-9b800f95dbbc // indirect
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58
|
||||
google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf
|
||||
google.golang.org/appengine v1.3.0 // indirect
|
||||
gopkg.in/DataDog/dd-trace-go.v1 v1.10.0 // indirect
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
|
||||
gopkg.in/go-playground/assert.v1 v1.2.1 // indirect
|
||||
gopkg.in/go-playground/validator.v9 v9.20.2
|
||||
)
|
||||
|
||||
@@ -14,13 +14,19 @@ github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895 h1:dmc/C8bpE5Vk
|
||||
github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
|
||||
github.com/DataDog/opencensus-go-exporter-datadog v0.0.0-20180917103902-e6c7f767dc57 h1:V1H8VVVxLALfaPLvAFCPoa0AN5nVPAqEu2UvH+QP3Vc=
|
||||
github.com/DataDog/opencensus-go-exporter-datadog v0.0.0-20180917103902-e6c7f767dc57/go.mod h1:gMGUEe16aZh0QN941HgDjwrdjU4iTthPoz2/AtDRADE=
|
||||
github.com/agext/levenshtein v1.2.1 h1:QmvMAjj2aEICytGiWzmxoE0x2KZvE0fvmqMOfy2tjT8=
|
||||
github.com/agext/levenshtein v1.2.1/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558=
|
||||
github.com/ajg/form v0.0.0-20160822230020-523a5da1a92f h1:zvClvFQwU++UpIUBGC8YmDlfhUrweEy1R1Fj1gu5iIM=
|
||||
github.com/ajg/form v0.0.0-20160822230020-523a5da1a92f/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
|
||||
github.com/apparentlymart/go-dump v0.0.0-20180507223929-23540a00eaa3/go.mod h1:oL81AME2rN47vu18xqj1S1jPIPuN7afo62yKTNn3XMM=
|
||||
github.com/apparentlymart/go-textseg v1.0.0 h1:rRmlIsPEEhUTIKQb7T++Nz/A5Q6C9IuX2wFoYVvnCs0=
|
||||
github.com/apparentlymart/go-textseg v1.0.0/go.mod h1:z96Txxhf3xSFMPmb5X/1W05FF/Nj9VFpLOpjS5yuumk=
|
||||
github.com/aws/aws-sdk-go v1.15.24 h1:xLAdTA/ore6xdPAljzZRed7IGqQgC+nY+ERS5vaj4Ro=
|
||||
github.com/aws/aws-sdk-go v1.15.24/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0=
|
||||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
|
||||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
|
||||
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
|
||||
github.com/bsm/go-vlq v0.0.0-20150828105119-ec6e8d4f5f4e/go.mod h1:N+BjUcTjSxc2mtRGSCPsat1kze3CUtvJN3/jTXlp29k=
|
||||
github.com/bsm/redis-lock v8.0.0+incompatible h1:QgB0J2pNG8hUfndTIvpPh38F5XsUTTvO7x8Sls++9Mk=
|
||||
github.com/bsm/redis-lock v8.0.0+incompatible/go.mod h1:8dGkQ5GimBCahwF2R67tqGCJbyDZSp0gzO7wq3pDrik=
|
||||
github.com/codegangsta/negroni v1.0.0 h1:+aYywywx4bnKXWvoWtRfJ91vC59NbEhEY03sZjQhbVY=
|
||||
@@ -54,6 +60,7 @@ github.com/go-redis/redis v6.15.2+incompatible h1:9SpNVG76gr6InJGxoZ6IuuxaCOQwDA
|
||||
github.com/go-redis/redis v6.15.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
|
||||
github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
|
||||
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
||||
github.com/go-test/deep v1.0.1/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
|
||||
github.com/gobuffalo/envy v1.6.7 h1:XMZGuFqTupAXhZTriQ+qO38QvNOSU/0rl3hEPCFci/4=
|
||||
github.com/gobuffalo/envy v1.6.7/go.mod h1:N+GkhhZ/93bGZc6ZKhJLP6+m+tCNPKwgSpH9kaifseQ=
|
||||
github.com/gobuffalo/httptest v1.0.4 h1:P0uKaPEjti1bbJmuBILE3QQ7iU1cS7oIkxVba5HbcVE=
|
||||
@@ -65,6 +72,7 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekf
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
||||
github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 h1:LbsanbbD6LieFkXbj9YNNBupiGHJgFeLpO0j0Fza1h8=
|
||||
github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
|
||||
@@ -99,13 +107,18 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92Bcuy
|
||||
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.4.1 h1:pX7cnDwSSmG0dR9yNjCQSSpmsJOqFdT7SzVp5Yl9uVw=
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.4.1/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
|
||||
github.com/hashicorp/errwrap v0.0.0-20180715044906-d6c0cd880357/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
|
||||
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
github.com/hashicorp/go-multierror v0.0.0-20180717150148-3d5d8f294aa0/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I=
|
||||
github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o=
|
||||
github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
|
||||
github.com/hashicorp/hcl2 v0.0.0-20190503213020-640445e16309 h1:VBvyXC+b6Ix/MXMGIrOHjq+Ew1IRP52EzoQXf1KpwZo=
|
||||
github.com/hashicorp/hcl2 v0.0.0-20190503213020-640445e16309/go.mod h1:4oI94iqF3GB10QScn46WqbG0kgTUpha97SAzzg2+2ec=
|
||||
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
|
||||
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
|
||||
github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8 h1:12VvqtR6Aowv3l/EQUlocDHW2Cp4G9WJVH7uyH8QFJE=
|
||||
github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
|
||||
github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc=
|
||||
@@ -125,6 +138,8 @@ github.com/kr/pty v1.0.0/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4=
|
||||
github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
|
||||
github.com/markbates/hmax v1.0.0 h1:yo2N0gBoCnUMKhV/VRLHomT6Y9wUm+oQQENuWJqCdlM=
|
||||
github.com/markbates/hmax v1.0.0/go.mod h1:cOkR9dktiESxIMu+65oc/r/bdY4bE8zZw3OLhLx0X2c=
|
||||
github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4=
|
||||
@@ -139,11 +154,15 @@ github.com/minio/minio-go v6.0.5+incompatible h1:qxQQW40lV2vuE9i6yYmt90GSJlT1YrM
|
||||
github.com/minio/minio-go v6.0.5+incompatible/go.mod h1:7guKYtitv8dktvNUGrhzmNlA5wrAABTQXCoesZdFQO8=
|
||||
github.com/mitchellh/go-homedir v1.0.0 h1:vKb8ShqSby24Yrqr/yDYkuFz8d0WUjys40rvnGC8aR0=
|
||||
github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||
github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7 h1:DpOJ2HYzCv8LZP15IdmG+YdwD2luVPHITV96TkirNBM=
|
||||
github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo=
|
||||
github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
|
||||
github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw=
|
||||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/gomega v1.4.2 h1:3mYCb7aPxS/RU7TI1y4rkEn1oKmPRjNJLNEXgw7MH2I=
|
||||
github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
||||
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
||||
github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
|
||||
github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ=
|
||||
github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
|
||||
@@ -164,6 +183,8 @@ github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7q
|
||||
github.com/prometheus/procfs v0.0.0-20180612222113-7d6f385de8be/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273 h1:agujYaXJSxSo18YNX3jzl+4G6Bstwt+kqv47GS12uL0=
|
||||
github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
|
||||
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
|
||||
github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
|
||||
github.com/sirupsen/logrus v1.1.1 h1:VzGj7lhU7KEB9e9gMpAV/v5XT2NVSvLJhJLCWbnkgXg=
|
||||
github.com/sirupsen/logrus v1.1.1/go.mod h1:zrgwTnHtNr00buQ1vSptGe8m1f/BbgsPukg8qsT7A+A=
|
||||
@@ -178,6 +199,7 @@ github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B
|
||||
github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
|
||||
github.com/spf13/pflag v1.0.1 h1:aCvUg6QPl3ibpQUxyLkrEkCHtPqYJL4x9AuhqVqFis4=
|
||||
github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
|
||||
github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
@@ -196,12 +218,15 @@ github.com/ugorji/go v1.1.1/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJ
|
||||
github.com/unrolled/secure v0.0.0-20181221173256-0d6b5bb13069 h1:RKeYksgIwGE8zFJTvXI1WWx09QPrGyaVFMy0vpU7j/o=
|
||||
github.com/unrolled/secure v0.0.0-20181221173256-0d6b5bb13069/go.mod h1:mnPT77IAdsi/kV7+Es7y+pXALeV3h7G6dQF6mNYjcLA=
|
||||
github.com/urfave/cli v1.18.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
|
||||
github.com/vmihailenco/msgpack v3.3.3+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk=
|
||||
github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk=
|
||||
github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
|
||||
github.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0=
|
||||
github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
|
||||
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8=
|
||||
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
|
||||
github.com/zclconf/go-cty v0.0.0-20190426224007-b18a157db9e2 h1:Ai1LhlYNEqE39zGU07qHDNJ41iZVPZfZr1dSCoXrp1w=
|
||||
github.com/zclconf/go-cty v0.0.0-20190426224007-b18a157db9e2/go.mod h1:xnAOWiHeOqg2nWS62VtQ7pbOu17FtxJNW8RLEih+O3s=
|
||||
go.etcd.io/bbolt v1.3.2 h1:Z/90sZLPOeCy2PwprqkFa25PdkusRzaj9P8zm/KNyvk=
|
||||
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
|
||||
go.etcd.io/etcd v0.0.0-20190215181705-784daa04988c h1:pkiZ418C7QN/HIps1lDF1+lzZhdgMpvFN4kDcxrYhD0=
|
||||
@@ -220,26 +245,43 @@ golang.org/x/crypto v0.0.0-20180608092829-8ac0e0d97ce4/go.mod h1:6SG95UA2DQfeDnf
|
||||
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20181029103014-dab2b1051b5d h1:5JyY8HlzxzYI+qHOOciM8s2lJbIEaefMUdtYt7dRDrg=
|
||||
golang.org/x/crypto v0.0.0-20181029103014-dab2b1051b5d/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 h1:p/H982KKEjUnLJkM3tt/LemDnOc1GiZL5FCVlORJ5zo=
|
||||
golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180811021610-c39426892332/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd h1:nTDtHvHSdCn1m6ITfMRqtOd/9+7a3s8RBNOZ3eYZzJA=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181029044818-c44066c5c816 h1:mVFkLpejdFLXVUv9E42f3XJVfMdqd0IVLVIVLjZWn5o=
|
||||
golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190502183928-7f726cade0ab h1:9RfW3ktsOZxgo9YNbBAjq1FWzc/igwEcUzZz8IXgSbk=
|
||||
golang.org/x/net v0.0.0-20190502183928-7f726cade0ab/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/oauth2 v0.0.0-20180620175406-ef147856a6dd h1:QQhib242ErYDSMitlBm8V7wYCm/1a25hV8qMadIKLPA=
|
||||
golang.org/x/oauth2 v0.0.0-20180620175406-ef147856a6dd/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e h1:o3PsSEY8E4eXWkXrIP9YJALUkVZqzHJT5DOasTyn8Vs=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181031143558-9b800f95dbbc h1:SdCq5U4J+PpbSDIl9bM0V1e1Ug1jsnBkAFvTs1htn7U=
|
||||
golang.org/x/sys v0.0.0-20181031143558-9b800f95dbbc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190502175342-a43fa875dd82 h1:vsphBvatvfbhlb4PO1BYSr9dzugGxJ/SQHoNufZJq1w=
|
||||
golang.org/x/sys v0.0.0-20190502175342-a43fa875dd82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 h1:+DCIGbF/swA92ohVg0//6X2IVY3KZs6p9mix0ziNYJM=
|
||||
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf h1:rjxqQmxjyqerRKEj+tZW+MCm4LgpFXu18bsEoCMgDsk=
|
||||
google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.3.0 h1:FBSsiFRMz3LBeXIomRnVzrQwSDj4ibvcRexLG0LZGQk=
|
||||
google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/genproto v0.0.0-20180608181217-32ee49c4dd80/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
@@ -267,3 +309,4 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD
|
||||
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0=
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"runtime"
|
||||
|
||||
"github.com/BurntSushi/toml"
|
||||
"github.com/gomods/athens/pkg/download/mode"
|
||||
"github.com/gomods/athens/pkg/errors"
|
||||
"github.com/kelseyhightower/envconfig"
|
||||
validator "gopkg.in/go-playground/validator.v9"
|
||||
@@ -43,6 +44,8 @@ type Config struct {
|
||||
TLSKeyFile string `envconfig:"ATHENS_TLSKEY_FILE"`
|
||||
SumDBs []string `envconfig:"ATHENS_SUM_DBS"`
|
||||
NoSumPatterns []string `envconfig:"ATHENS_GONOSUM_PATTERNS"`
|
||||
DownloadMode mode.Mode `envconfig:"ATHENS_DOWNLOAD_MODE"`
|
||||
DownloadURL string `envconfig:"ATHENS_DOWNLOAD_URL"`
|
||||
SingleFlightType string `envconfig:"ATHENS_SINGLE_FLIGHT_TYPE"`
|
||||
SingleFlight *SingleFlight
|
||||
Storage *StorageConfig
|
||||
@@ -84,6 +87,8 @@ func defaultConfig() *Config {
|
||||
TraceExporterURL: "http://localhost:14268",
|
||||
SumDBs: []string{"https://sum.golang.org"},
|
||||
NoSumPatterns: []string{},
|
||||
DownloadMode: "sync",
|
||||
DownloadURL: "",
|
||||
SingleFlight: &SingleFlight{
|
||||
Etcd: &Etcd{"localhost:2379,localhost:22379,localhost:32379"},
|
||||
Redis: &Redis{"127.0.0.1:6379"},
|
||||
|
||||
@@ -266,6 +266,7 @@ func TestParseExampleConfig(t *testing.T) {
|
||||
SingleFlight: &SingleFlight{},
|
||||
SumDBs: []string{"https://sum.golang.org"},
|
||||
NoSumPatterns: []string{},
|
||||
DownloadMode: "sync",
|
||||
}
|
||||
|
||||
absPath, err := filepath.Abs(testConfigFile(t))
|
||||
|
||||
+11
-2
@@ -2,7 +2,9 @@ package download
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/url"
|
||||
|
||||
"github.com/gomods/athens/pkg/download/mode"
|
||||
"github.com/gomods/athens/pkg/log"
|
||||
"github.com/gomods/athens/pkg/middleware"
|
||||
"github.com/gorilla/mux"
|
||||
@@ -10,13 +12,14 @@ import (
|
||||
|
||||
// ProtocolHandler is a function that takes all that it needs to return
|
||||
// a ready-to-go http handler that serves up cmd/go's download protocol.
|
||||
type ProtocolHandler func(dp Protocol, lggr log.Entry) http.Handler
|
||||
type ProtocolHandler func(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler
|
||||
|
||||
// HandlerOpts are the generic options
|
||||
// for a ProtocolHandler
|
||||
type HandlerOpts struct {
|
||||
Protocol Protocol
|
||||
Logger *log.Logger
|
||||
DownloadFile *mode.DownloadFile
|
||||
}
|
||||
|
||||
// LogEntryHandler pulls a log entry from the request context. Thanks to the
|
||||
@@ -26,7 +29,7 @@ type HandlerOpts struct {
|
||||
func LogEntryHandler(ph ProtocolHandler, opts *HandlerOpts) http.Handler {
|
||||
f := func(w http.ResponseWriter, r *http.Request) {
|
||||
ent := log.EntryFromContext(r.Context())
|
||||
handler := ph(opts.Protocol, ent)
|
||||
handler := ph(opts.Protocol, ent, opts.DownloadFile)
|
||||
handler.ServeHTTP(w, r)
|
||||
}
|
||||
return http.HandlerFunc(f)
|
||||
@@ -51,3 +54,9 @@ func RegisterHandlers(r *mux.Router, opts *HandlerOpts) {
|
||||
r.Handle(PathVersionModule, LogEntryHandler(ModuleHandler, opts)).Methods(http.MethodGet)
|
||||
r.Handle(PathVersionZip, LogEntryHandler(ZipHandler, opts)).Methods(http.MethodGet)
|
||||
}
|
||||
|
||||
func getRedirectURL(base, path string) string {
|
||||
url, _ := url.Parse(base)
|
||||
url.Path = path
|
||||
return url.String()
|
||||
}
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
package download
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/gomods/athens/pkg/download/mode"
|
||||
"github.com/gomods/athens/pkg/errors"
|
||||
"github.com/gomods/athens/pkg/log"
|
||||
"github.com/gorilla/mux"
|
||||
)
|
||||
|
||||
func TestRedirect(t *testing.T) {
|
||||
r := mux.NewRouter()
|
||||
RegisterHandlers(r, &HandlerOpts{
|
||||
Protocol: &mockProtocol{},
|
||||
Logger: log.NoOpLogger(),
|
||||
DownloadFile: &mode.DownloadFile{
|
||||
Mode: mode.Redirect,
|
||||
DownloadURL: "https://gomods.io",
|
||||
},
|
||||
})
|
||||
req := httptest.NewRequest("GET", "/github.com/gomods/athens/@v/v0.4.0.info", nil)
|
||||
w := httptest.NewRecorder()
|
||||
r.ServeHTTP(w, req)
|
||||
if w.Code != http.StatusMovedPermanently {
|
||||
t.Fatalf("expected a redirect status (301) but got %v", w.Code)
|
||||
}
|
||||
expectedRedirect := "https://gomods.io/github.com/gomods/athens/@v/v0.4.0.info"
|
||||
givenRedirect := w.HeaderMap.Get("location")
|
||||
if expectedRedirect != givenRedirect {
|
||||
t.Fatalf("expected the handler to redirect to %q but got %q", expectedRedirect, givenRedirect)
|
||||
}
|
||||
}
|
||||
|
||||
type mockProtocol struct {
|
||||
Protocol
|
||||
}
|
||||
|
||||
func (mp *mockProtocol) Info(ctx context.Context, mod, ver string) ([]byte, error) {
|
||||
const op errors.Op = "mockProtocol.Info"
|
||||
return nil, errors.E(op, "not found", errors.KindRedirect)
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
|
||||
"github.com/gomods/athens/pkg/download/mode"
|
||||
"github.com/gomods/athens/pkg/errors"
|
||||
"github.com/gomods/athens/pkg/log"
|
||||
"github.com/gomods/athens/pkg/paths"
|
||||
@@ -13,7 +14,7 @@ import (
|
||||
const PathLatest = "/{module:.+}/@latest"
|
||||
|
||||
// LatestHandler implements GET baseURL/module/@latest
|
||||
func LatestHandler(dp Protocol, lggr log.Entry) http.Handler {
|
||||
func LatestHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler {
|
||||
const op errors.Op = "download.LatestHandler"
|
||||
f := func(w http.ResponseWriter, r *http.Request) {
|
||||
mod, err := paths.GetModule(r)
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/gomods/athens/pkg/download/mode"
|
||||
"github.com/gomods/athens/pkg/errors"
|
||||
"github.com/gomods/athens/pkg/log"
|
||||
"github.com/gomods/athens/pkg/paths"
|
||||
@@ -14,7 +15,7 @@ import (
|
||||
const PathList = "/{module:.+}/@v/list"
|
||||
|
||||
// ListHandler implements GET baseURL/module/@v/list
|
||||
func ListHandler(dp Protocol, lggr log.Entry) http.Handler {
|
||||
func ListHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler {
|
||||
const op errors.Op = "download.ListHandler"
|
||||
f := func(w http.ResponseWriter, r *http.Request) {
|
||||
mod, err := paths.GetModule(r)
|
||||
|
||||
@@ -128,7 +128,7 @@ func TestListMerge(t *testing.T) {
|
||||
s.Save(ctx, testModName, v, bts, ioutil.NopCloser(bytes.NewReader(bts)), bts)
|
||||
}
|
||||
defer clearStorage(s, testModName, tc.strVersions)
|
||||
dp := New(&Opts{s, nil, &listerMock{versions: tc.goVersions, err: tc.goErr}})
|
||||
dp := New(&Opts{s, nil, &listerMock{versions: tc.goVersions, err: tc.goErr}, nil})
|
||||
list, err := dp.List(ctx, testModName)
|
||||
|
||||
if ok := testErrEq(tc.expectedErr, err); !ok {
|
||||
|
||||
@@ -0,0 +1,132 @@
|
||||
package mode
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"github.com/gomods/athens/pkg/errors"
|
||||
"github.com/hashicorp/hcl2/gohcl"
|
||||
"github.com/hashicorp/hcl2/hclparse"
|
||||
)
|
||||
|
||||
// Mode specifies the behavior of what to do
|
||||
// when a module is not found in storage.
|
||||
type Mode string
|
||||
|
||||
// DownloadMode constants. For more information see config.dev.toml
|
||||
const (
|
||||
Sync Mode = "sync"
|
||||
Async Mode = "async"
|
||||
Redirect Mode = "redirect"
|
||||
AsyncRedirect Mode = "async_redirect"
|
||||
None Mode = "none"
|
||||
)
|
||||
|
||||
// DownloadFile represents a custom HCL format of
|
||||
// how to handle module@version requests that are
|
||||
// not found in storage.
|
||||
type DownloadFile struct {
|
||||
Mode Mode `hcl:"mode"`
|
||||
DownloadURL string `hcl:"downloadURL"`
|
||||
Paths []*DownloadPath `hcl:"download,block"`
|
||||
}
|
||||
|
||||
// DownloadPath represents a custom Mode for
|
||||
// a matching path.
|
||||
type DownloadPath struct {
|
||||
Pattern string `hcl:"pattern,label"`
|
||||
Mode Mode `hcl:"mode"`
|
||||
DownloadURL string `hcl:"downloadURL,optional"`
|
||||
}
|
||||
|
||||
// NewFile takes a mode and returns a DownloadFile.
|
||||
// Mode can be one of the constants declared above
|
||||
// or a custom HCL file. To pass a custom HCL file,
|
||||
// you can either point to a file path by passing
|
||||
// file:/path/to/file OR custom:<base64-encoded-hcl>
|
||||
// directly.
|
||||
func NewFile(m Mode, downloadURL string) (*DownloadFile, error) {
|
||||
const op errors.Op = "downloadMode.NewFile"
|
||||
if strings.HasPrefix(string(m), "file:") {
|
||||
filePath := string(m[5:])
|
||||
bts, err := ioutil.ReadFile(filePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return parseFile(bts)
|
||||
} else if strings.HasPrefix(string(m), "custom:") {
|
||||
bts, err := base64.StdEncoding.DecodeString(string(m[7:]))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return parseFile(bts)
|
||||
}
|
||||
switch m {
|
||||
case Sync, Async, Redirect, AsyncRedirect, None:
|
||||
return &DownloadFile{Mode: m, DownloadURL: downloadURL}, nil
|
||||
default:
|
||||
return nil, errors.E(op, "unrecognized download mode: "+m, errors.KindBadRequest)
|
||||
}
|
||||
}
|
||||
|
||||
// parseFile parses an HCL file according to the
|
||||
// DownloadMode spec.
|
||||
func parseFile(file []byte) (*DownloadFile, error) {
|
||||
const op errors.Op = "downloadmode.parseFile"
|
||||
f, dig := hclparse.NewParser().ParseHCL(file, "config.hcl")
|
||||
if dig.HasErrors() {
|
||||
return nil, errors.E(op, dig.Error())
|
||||
}
|
||||
var df DownloadFile
|
||||
dig = gohcl.DecodeBody(f.Body, nil, &df)
|
||||
if dig.HasErrors() {
|
||||
return nil, errors.E(op, dig.Error())
|
||||
}
|
||||
if err := df.validate(); err != nil {
|
||||
return nil, errors.E(op, err)
|
||||
}
|
||||
return &df, nil
|
||||
}
|
||||
|
||||
func (d *DownloadFile) validate() error {
|
||||
const op errors.Op = "downloadMode.validate"
|
||||
for _, p := range d.Paths {
|
||||
switch p.Mode {
|
||||
case Sync, Async, Redirect, AsyncRedirect, None:
|
||||
default:
|
||||
return errors.E(op, fmt.Errorf("unrecognized mode for %v: %v", p.Pattern, p.Mode))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Match returns the Mode that matches the given
|
||||
// module. A pattern is prioritized by order in
|
||||
// which it appears in the HCL file, while the
|
||||
// default Mode will be returned if no patterns
|
||||
// exist or match.
|
||||
func (d *DownloadFile) Match(mod string) Mode {
|
||||
for _, p := range d.Paths {
|
||||
if hasMatch, err := path.Match(p.Pattern, mod); hasMatch && err == nil {
|
||||
return p.Mode
|
||||
}
|
||||
}
|
||||
return d.Mode
|
||||
}
|
||||
|
||||
// URL returns the redirect URL that applies
|
||||
// to the given module. If no pattern matches,
|
||||
// the top level downloadURL is returned.
|
||||
func (d *DownloadFile) URL(mod string) string {
|
||||
for _, p := range d.Paths {
|
||||
if hasMatch, err := path.Match(p.Pattern, mod); hasMatch && err == nil {
|
||||
if p.DownloadURL != "" {
|
||||
return p.DownloadURL
|
||||
}
|
||||
}
|
||||
}
|
||||
return d.DownloadURL
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
package mode
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
var testCases = []struct {
|
||||
name string
|
||||
file *DownloadFile
|
||||
input string
|
||||
expectedMode Mode
|
||||
expectedURL string
|
||||
}{
|
||||
{
|
||||
name: "sync",
|
||||
file: &DownloadFile{Mode: Sync},
|
||||
input: "github.com/gomods/athens",
|
||||
expectedMode: Sync,
|
||||
},
|
||||
{
|
||||
name: "redirect",
|
||||
file: &DownloadFile{Mode: Redirect, DownloadURL: "gomods.io"},
|
||||
input: "github.com/gomods/athens",
|
||||
expectedMode: Redirect,
|
||||
expectedURL: "gomods.io",
|
||||
},
|
||||
{
|
||||
name: "pattern match",
|
||||
file: &DownloadFile{
|
||||
Mode: Sync,
|
||||
Paths: []*DownloadPath{
|
||||
{Pattern: "github.com/gomods/*", Mode: None},
|
||||
},
|
||||
},
|
||||
input: "github.com/gomods/athens",
|
||||
expectedMode: None,
|
||||
},
|
||||
{
|
||||
name: "pattern fallback",
|
||||
file: &DownloadFile{
|
||||
Mode: Sync,
|
||||
Paths: []*DownloadPath{
|
||||
{Pattern: "github.com/gomods/*", Mode: None},
|
||||
},
|
||||
},
|
||||
input: "github.com/athens-artifacts/maturelib",
|
||||
expectedMode: Sync,
|
||||
},
|
||||
{
|
||||
name: "pattern redirect",
|
||||
file: &DownloadFile{
|
||||
Mode: Sync,
|
||||
Paths: []*DownloadPath{
|
||||
{
|
||||
Pattern: "github.com/gomods/*",
|
||||
Mode: AsyncRedirect,
|
||||
DownloadURL: "gomods.io"},
|
||||
},
|
||||
},
|
||||
input: "github.com/gomods/athens",
|
||||
expectedMode: AsyncRedirect,
|
||||
expectedURL: "gomods.io",
|
||||
},
|
||||
{
|
||||
name: "redirect fallback",
|
||||
file: &DownloadFile{
|
||||
Mode: Redirect,
|
||||
DownloadURL: "proxy.golang.org",
|
||||
Paths: []*DownloadPath{
|
||||
{
|
||||
Pattern: "github.com/gomods/*",
|
||||
Mode: AsyncRedirect,
|
||||
DownloadURL: "gomods.io",
|
||||
},
|
||||
},
|
||||
},
|
||||
input: "github.com/athens-artifacts/maturelib",
|
||||
expectedMode: Redirect,
|
||||
expectedURL: "proxy.golang.org",
|
||||
},
|
||||
}
|
||||
|
||||
func TestMode(t *testing.T) {
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
givenMode := tc.file.Match(tc.input)
|
||||
if givenMode != tc.expectedMode {
|
||||
t.Fatalf("expected matched mode to be %q but got %q", tc.expectedMode, givenMode)
|
||||
}
|
||||
givenURL := tc.file.URL(tc.input)
|
||||
if givenURL != tc.expectedURL {
|
||||
t.Fatalf("expected matched DownloadURL to be %q but got %q", tc.expectedURL, givenURL)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+39
-17
@@ -7,6 +7,7 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/gomods/athens/pkg/download/mode"
|
||||
"github.com/gomods/athens/pkg/errors"
|
||||
"github.com/gomods/athens/pkg/observ"
|
||||
"github.com/gomods/athens/pkg/stash"
|
||||
@@ -40,6 +41,7 @@ type Opts struct {
|
||||
Storage storage.Backend
|
||||
Stasher stash.Stasher
|
||||
Lister UpstreamLister
|
||||
DownloadFile *mode.DownloadFile
|
||||
}
|
||||
|
||||
// New returns a full implementation of the download.Protocol
|
||||
@@ -48,7 +50,10 @@ type Opts struct {
|
||||
// The wrappers are applied in order, meaning the last wrapper
|
||||
// passed is the Protocol that gets hit first.
|
||||
func New(opts *Opts, wrappers ...Wrapper) Protocol {
|
||||
var p Protocol = &protocol{opts.Storage, opts.Stasher, opts.Lister}
|
||||
if opts.DownloadFile == nil {
|
||||
opts.DownloadFile = &mode.DownloadFile{Mode: mode.Sync}
|
||||
}
|
||||
var p Protocol = &protocol{opts.DownloadFile, opts.Storage, opts.Stasher, opts.Lister}
|
||||
for _, w := range wrappers {
|
||||
p = w(p)
|
||||
}
|
||||
@@ -57,6 +62,7 @@ func New(opts *Opts, wrappers ...Wrapper) Protocol {
|
||||
}
|
||||
|
||||
type protocol struct {
|
||||
df *mode.DownloadFile
|
||||
storage storage.Backend
|
||||
stasher stash.Stasher
|
||||
lister UpstreamLister
|
||||
@@ -151,13 +157,11 @@ func (p *protocol) Info(ctx context.Context, mod, ver string) ([]byte, error) {
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
defer span.End()
|
||||
info, err := p.storage.Info(ctx, mod, ver)
|
||||
var newVer string
|
||||
if errors.IsNotFoundErr(err) {
|
||||
newVer, err = p.stasher.Stash(ctx, mod, ver)
|
||||
if err != nil {
|
||||
return nil, errors.E(op, err)
|
||||
}
|
||||
err = p.processDownload(ctx, mod, ver, func(newVer string) error {
|
||||
info, err = p.storage.Info(ctx, mod, newVer)
|
||||
return err
|
||||
})
|
||||
}
|
||||
if err != nil {
|
||||
return nil, errors.E(op, err)
|
||||
@@ -171,18 +175,15 @@ func (p *protocol) GoMod(ctx context.Context, mod, ver string) ([]byte, error) {
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
defer span.End()
|
||||
goMod, err := p.storage.GoMod(ctx, mod, ver)
|
||||
var newVer string
|
||||
if errors.IsNotFoundErr(err) {
|
||||
newVer, err = p.stasher.Stash(ctx, mod, ver)
|
||||
if err != nil {
|
||||
return nil, errors.E(op, err)
|
||||
}
|
||||
err = p.processDownload(ctx, mod, ver, func(newVer string) error {
|
||||
goMod, err = p.storage.GoMod(ctx, mod, newVer)
|
||||
return err
|
||||
})
|
||||
}
|
||||
if err != nil {
|
||||
return nil, errors.E(op, err)
|
||||
}
|
||||
|
||||
return goMod, nil
|
||||
}
|
||||
|
||||
@@ -191,13 +192,11 @@ func (p *protocol) Zip(ctx context.Context, mod, ver string) (io.ReadCloser, err
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
defer span.End()
|
||||
zip, err := p.storage.Zip(ctx, mod, ver)
|
||||
var newVer string
|
||||
if errors.IsNotFoundErr(err) {
|
||||
newVer, err = p.stasher.Stash(ctx, mod, ver)
|
||||
if err != nil {
|
||||
return nil, errors.E(op, err)
|
||||
}
|
||||
err = p.processDownload(ctx, mod, ver, func(newVer string) error {
|
||||
zip, err = p.storage.Zip(ctx, mod, newVer)
|
||||
return err
|
||||
})
|
||||
}
|
||||
if err != nil {
|
||||
return nil, errors.E(op, err)
|
||||
@@ -206,6 +205,29 @@ func (p *protocol) Zip(ctx context.Context, mod, ver string) (io.ReadCloser, err
|
||||
return zip, nil
|
||||
}
|
||||
|
||||
func (p *protocol) processDownload(ctx context.Context, mod, ver string, f func(newVer string) error) error {
|
||||
const op errors.Op = "protocol.processDownload"
|
||||
switch p.df.Match(mod) {
|
||||
case mode.Sync:
|
||||
newVer, err := p.stasher.Stash(ctx, mod, ver)
|
||||
if err != nil {
|
||||
return errors.E(op, err)
|
||||
}
|
||||
return f(newVer)
|
||||
case mode.Async:
|
||||
go p.stasher.Stash(ctx, mod, ver)
|
||||
return errors.E(op, "async: module not found", errors.KindNotFound)
|
||||
case mode.Redirect:
|
||||
return errors.E(op, "redirect", errors.KindRedirect)
|
||||
case mode.AsyncRedirect:
|
||||
go p.stasher.Stash(ctx, mod, ver)
|
||||
return errors.E(op, "async_redirect: module not found", errors.KindRedirect)
|
||||
case mode.None:
|
||||
return errors.E(op, "none", errors.KindNotFound)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// union concatenates two version lists and removes duplicates
|
||||
func union(list1, list2 []string) []string {
|
||||
if list1 == nil {
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/gomods/athens/pkg/config"
|
||||
"github.com/gomods/athens/pkg/download/mode"
|
||||
"github.com/gomods/athens/pkg/errors"
|
||||
"github.com/gomods/athens/pkg/module"
|
||||
"github.com/gomods/athens/pkg/stash"
|
||||
@@ -44,7 +45,7 @@ func getDP(t *testing.T) Protocol {
|
||||
t.Fatal(err)
|
||||
}
|
||||
st := stash.New(mf, s)
|
||||
return New(&Opts{s, st, NewVCSLister(goBin, fs)})
|
||||
return New(&Opts{s, st, NewVCSLister(goBin, fs), nil})
|
||||
}
|
||||
|
||||
type listTest struct {
|
||||
@@ -280,7 +281,7 @@ func TestDownloadProtocol(t *testing.T) {
|
||||
}
|
||||
mp := &mockFetcher{}
|
||||
st := stash.New(mp, s)
|
||||
dp := New(&Opts{s, st, nil})
|
||||
dp := New(&Opts{s, st, nil, nil})
|
||||
ctx := context.Background()
|
||||
|
||||
var eg errgroup.Group
|
||||
@@ -332,7 +333,7 @@ func TestDownloadProtocolWhenFetchFails(t *testing.T) {
|
||||
}
|
||||
mp := ¬FoundFetcher{}
|
||||
st := stash.New(mp, s)
|
||||
dp := New(&Opts{s, st, nil})
|
||||
dp := New(&Opts{s, st, nil, nil})
|
||||
ctx := context.Background()
|
||||
_, err = dp.GoMod(ctx, fakeMod.mod, fakeMod.ver)
|
||||
if err != nil {
|
||||
@@ -340,6 +341,40 @@ func TestDownloadProtocolWhenFetchFails(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAsyncRedirect(t *testing.T) {
|
||||
s, err := mem.NewStorage()
|
||||
require.NoError(t, err)
|
||||
ms := &mockStasher{s, make(chan bool)}
|
||||
dp := New(&Opts{
|
||||
Stasher: ms,
|
||||
Storage: s,
|
||||
DownloadFile: &mode.DownloadFile{
|
||||
Mode: mode.Async,
|
||||
DownloadURL: "https://gomods.io",
|
||||
},
|
||||
})
|
||||
mod, ver := "github.com/athens-artifacts/happy-path", "v0.0.1"
|
||||
_, err = dp.Info(context.Background(), mod, ver)
|
||||
if errors.Kind(err) != errors.KindNotFound {
|
||||
t.Fatalf("expected async_redirect to enforce a 404 but got %v", errors.Kind(err))
|
||||
}
|
||||
<-ms.ch
|
||||
info, err := dp.Info(context.Background(), mod, ver)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, string(info), "info", "expected async fetch to be successful")
|
||||
}
|
||||
|
||||
type mockStasher struct {
|
||||
s storage.Backend
|
||||
ch chan bool
|
||||
}
|
||||
|
||||
func (ms *mockStasher) Stash(ctx context.Context, mod string, ver string) (string, error) {
|
||||
err := ms.s.Save(ctx, mod, ver, []byte("mod"), strings.NewReader("zip"), []byte("info"))
|
||||
ms.ch <- true // signal async stashing is done
|
||||
return ver, err
|
||||
}
|
||||
|
||||
type notFoundFetcher struct{}
|
||||
|
||||
func (m *notFoundFetcher) Fetch(ctx context.Context, mod, ver string) (*storage.Version, error) {
|
||||
|
||||
@@ -3,6 +3,7 @@ package download
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/gomods/athens/pkg/download/mode"
|
||||
"github.com/gomods/athens/pkg/errors"
|
||||
"github.com/gomods/athens/pkg/log"
|
||||
)
|
||||
@@ -11,7 +12,7 @@ import (
|
||||
const PathVersionInfo = "/{module:.+}/@v/{version}.info"
|
||||
|
||||
// InfoHandler implements GET baseURL/module/@v/version.info
|
||||
func InfoHandler(dp Protocol, lggr log.Entry) http.Handler {
|
||||
func InfoHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler {
|
||||
const op errors.Op = "download.InfoHandler"
|
||||
f := func(w http.ResponseWriter, r *http.Request) {
|
||||
mod, ver, err := getModuleParams(r, op)
|
||||
@@ -23,6 +24,10 @@ func InfoHandler(dp Protocol, lggr log.Entry) http.Handler {
|
||||
info, err := dp.Info(r.Context(), mod, ver)
|
||||
if err != nil {
|
||||
lggr.SystemErr(errors.E(op, err, errors.M(mod), errors.V(ver)))
|
||||
if errors.Kind(err) == errors.KindRedirect {
|
||||
http.Redirect(w, r, getRedirectURL(df.URL(mod), r.URL.Path), errors.KindRedirect)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(errors.Kind(err))
|
||||
}
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ package download
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/gomods/athens/pkg/download/mode"
|
||||
"github.com/gomods/athens/pkg/errors"
|
||||
"github.com/gomods/athens/pkg/log"
|
||||
)
|
||||
@@ -11,12 +12,16 @@ import (
|
||||
const PathVersionModule = "/{module:.+}/@v/{version}.mod"
|
||||
|
||||
// ModuleHandler implements GET baseURL/module/@v/version.mod
|
||||
func ModuleHandler(dp Protocol, lggr log.Entry) http.Handler {
|
||||
func ModuleHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler {
|
||||
const op errors.Op = "download.VersionModuleHandler"
|
||||
f := func(w http.ResponseWriter, r *http.Request) {
|
||||
mod, ver, err := getModuleParams(r, op)
|
||||
if err != nil {
|
||||
lggr.SystemErr(err)
|
||||
if errors.Kind(err) == errors.KindRedirect {
|
||||
http.Redirect(w, r, getRedirectURL(df.URL(mod), r.URL.Path), errors.KindRedirect)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(errors.Kind(err))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
|
||||
"github.com/gomods/athens/pkg/download/mode"
|
||||
"github.com/gomods/athens/pkg/errors"
|
||||
"github.com/gomods/athens/pkg/log"
|
||||
)
|
||||
@@ -12,7 +13,7 @@ import (
|
||||
const PathVersionZip = "/{module:.+}/@v/{version}.zip"
|
||||
|
||||
// ZipHandler implements GET baseURL/module/@v/version.zip
|
||||
func ZipHandler(dp Protocol, lggr log.Entry) http.Handler {
|
||||
func ZipHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler {
|
||||
const op errors.Op = "download.ZipHandler"
|
||||
f := func(w http.ResponseWriter, r *http.Request) {
|
||||
mod, ver, err := getModuleParams(r, op)
|
||||
@@ -24,6 +25,10 @@ func ZipHandler(dp Protocol, lggr log.Entry) http.Handler {
|
||||
zip, err := dp.Zip(r.Context(), mod, ver)
|
||||
if err != nil {
|
||||
lggr.SystemErr(err)
|
||||
if errors.Kind(err) == errors.KindRedirect {
|
||||
http.Redirect(w, r, getRedirectURL(df.URL(mod), r.URL.Path), errors.KindRedirect)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(errors.Kind(err))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -17,6 +17,7 @@ const (
|
||||
KindAlreadyExists = http.StatusConflict
|
||||
KindRateLimit = http.StatusTooManyRequests
|
||||
KindNotImplemented = http.StatusNotImplemented
|
||||
KindRedirect = http.StatusMovedPermanently
|
||||
)
|
||||
|
||||
// Error is an Athens system error.
|
||||
|
||||
Reference in New Issue
Block a user