diff --git a/cmd/proxy/actions/app_proxy.go b/cmd/proxy/actions/app_proxy.go index 85cc7cd1..4f758afd 100644 --- a/cmd/proxy/actions/app_proxy.go +++ b/cmd/proxy/actions/app_proxy.go @@ -10,6 +10,7 @@ import ( "github.com/gomods/athens/pkg/config" "github.com/gomods/athens/pkg/download" "github.com/gomods/athens/pkg/download/addons" + "github.com/gomods/athens/pkg/download/mode" "github.com/gomods/athens/pkg/log" "github.com/gomods/athens/pkg/module" "github.com/gomods/athens/pkg/stash" @@ -85,14 +86,21 @@ func addProxyRoutes( } st := stash.New(mf, s, stash.WithPool(c.GoGetWorkers), withSingleFlight) - dpOpts := &download.Opts{ - Storage: s, - Stasher: st, - Lister: lister, + df, err := mode.NewFile(c.DownloadMode, c.DownloadURL) + if err != nil { + return err } + + dpOpts := &download.Opts{ + Storage: s, + Stasher: st, + Lister: lister, + DownloadFile: df, + } + dp := download.New(dpOpts, addons.WithPool(c.ProtocolWorkers)) - handlerOpts := &download.HandlerOpts{Protocol: dp, Logger: l} + handlerOpts := &download.HandlerOpts{Protocol: dp, Logger: l, DownloadFile: df} download.RegisterHandlers(r, handlerOpts) return nil diff --git a/config.dev.toml b/config.dev.toml index 3e017dab..e623dd2d 100755 --- a/config.dev.toml +++ b/config.dev.toml @@ -81,7 +81,7 @@ Port = ":3000" # # To point Athens to an upstream proxy to fetch modules, # set GlobalEndpoint to "https://" -# and also ensure that FilterFile is set to a fully qualified file name +# and also ensure that FilterFile is set to a fully qualified file name # that contains the letter `D` (for "Direct Access") in the first line. GlobalEndpoint = "http://localhost:3001" @@ -176,6 +176,31 @@ SumDBs = ["https://sum.golang.org"] # Env override: ATHENS_GONOSUM_PATTERNS NoSumPatterns = [] +# DownloadMode defines how Athens behaves when a module@version +# is not found in storage. There are 4 options: +# 1. "sync" (default): download the module synchronously and +# return the results to the client. +# 2. "async": return 404, but asynchronously store the module +# in the storage backend. +# 3. "redirect": return a 301 redirect status to the client +# with the base URL as the DownloadRedirectURL from below. +# 4. "async_redirect": same as option number 3 but it will +# asynchronously store the module to the backend. +# 5. "none": return 404 if a module is not found and do nothing. +# 6. "file:": will point to an HCL file that specifies +# any of the 5 options above based on different import paths. +# 7. "custom:" is the same as option 6 +# but the file is fully encoded in the option. This is +# useful for using an environment variable in serverless +# deployments. +# Env override: ATHENS_DOWNLOAD_MODE +DownloadMode = "sync" + +# DownloadURL is the URL that will be used if +# DownloadMode is set to "redirect" +# Env override: ATHENS_DOWNLOAD_URL +DownloadURL = "" + # SingleFlightType determines what mechanism Athens uses # to manage concurrency flowing into the Athens Backend. # This is important for the following scenario: if two concurrent requests diff --git a/docs/content/configuration/download.md b/docs/content/configuration/download.md new file mode 100644 index 00000000..c5d800b7 --- /dev/null +++ b/docs/content/configuration/download.md @@ -0,0 +1,62 @@ +--- +title: Download Mode +description: What to do when a module is not in storage +weight: 2 +--- + +Athens accepts an HCL formatted Download File that serves as the source of truth for answering the following question: + +### What should Athens do when a module@version is not found in storage? + +Say a client sends an HTTP request with the path `/github.com/pkg/errors/@v/v0.8.1` and Athens +does not have this module in storage. Athens will look at the Download File for one of the following Modes: + +1. **`sync`**: Synchronously download the module from VCS via `go mod download`, persist it to the Athens storage, and serve it back to the user immediately. Note that this is the default behavior. +2. **`async`**: Return a 404 to the client, and asynchronously download and persist the module@version to storage. +3. **`none`**: Return a 404 and do nothing. +4. **`redirect`**: Redirect to an upstream proxy (such as proxy.golang.org) and do nothing after. +5. **`async_redirect`**: Redirect to an upstream proxy (such as proxy.golang.org) and asynchronously download and persist the module@version to storage. + +Furthermore, the Download File can describe any of the above behavior for different modules and module patterns alike using [path.Match](https://golang.org/pkg/path/#Match). Take a look at the following example: + +```javascript +downloadURL = "https://proxy.golang.org" + +mode = "async_redirect" + +download "github.com/gomods/*" { + mode = "sync" +} + +download "golang.org/x/*" { + mode = "none" +} + +download "github.com/pkg/*" { + mode = "redirect" + downloadURL = "https://gocenter.io" +} +``` + +The first two lines describe the behavior and the destination of all packages: redirect to `https://proxy.golang.org` and asynchronously persist the module to storage. + +The following two blocks describe what to do if the requested module matches the given pattern: + +Any module that matches "github.com/gomods/*" such as "github.com/gomods/athens", will be synchronously fetched, stored, and returned to the user. + +Any module that matches "golang.org/x/*" such as "golang.org/x/text" will just return a 404. Note that this behavior allows the client to set GOPROXY to multiple comma separated proxies so that the Go command can move into the second argument. + +Any module that matches "github.com/pkg/*" such as "github.com/pkg/errors" will be redirected to https://gocenter.io (and not proxy.golang.org) and will also never persist the module to the Athens storage. + + +## Use cases + +So why would you want to use the Download File to configure the behavior above? Here are a few use cases where it might make sense for you to do so: + +**Limited storage:** + +If you have limited storage, then it might be a good idea to only persist some moduels (such as private ones) and then redirect to a public proxy for everything else. + +**Limited resources:** + +If you are running Athens with low memory/cpu, then you can redirect all public modules to proxy.golang.org but asynchronously fetch them so that the client does not timeout. At the same time, you can return a 404 for private modules through the `none` mode and let the client (the Go command) fetch private modules directly through `GOPROXY=,direct` \ No newline at end of file diff --git a/download.example.hcl b/download.example.hcl new file mode 100644 index 00000000..a2e3275c --- /dev/null +++ b/download.example.hcl @@ -0,0 +1,16 @@ +downloadURL = "https://proxy.golang.org" + +mode = "async_redirect" + +download "github.com/gomods/*" { + mode = "sync" +} + +download "golang.org/x/*" { + mode = "none" +} + +download "github.com/pkg/*" { + mode = "redirect" + downloadURL = "https://gocenter.io" +} diff --git a/go.mod b/go.mod index 4c402959..3cffdff5 100644 --- a/go.mod +++ b/go.mod @@ -26,10 +26,10 @@ require ( github.com/gopherjs/gopherjs v0.0.0-20180825215210-0210a2f0f73c // indirect github.com/gorilla/mux v1.6.2 github.com/hashicorp/go-multierror v1.0.0 + github.com/hashicorp/hcl2 v0.0.0-20190503213020-640445e16309 github.com/jtolds/gls v4.2.1+incompatible // indirect github.com/kelseyhightower/envconfig v1.3.0 github.com/konsorten/go-windows-terminal-sequences v1.0.1 // indirect - github.com/kr/pretty v0.1.0 // indirect github.com/minio/minio-go v6.0.5+incompatible github.com/mitchellh/go-homedir v1.0.0 github.com/philhofer/fwd v1.0.0 // indirect @@ -48,15 +48,11 @@ require ( go.etcd.io/etcd v0.0.0-20190215181705-784daa04988c go.mongodb.org/mongo-driver v1.0.0 go.opencensus.io v0.17.0 - golang.org/x/crypto v0.0.0-20181029103014-dab2b1051b5d // indirect - golang.org/x/net v0.0.0-20181029044818-c44066c5c816 // indirect golang.org/x/oauth2 v0.0.0-20180620175406-ef147856a6dd - golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f - golang.org/x/sys v0.0.0-20181031143558-9b800f95dbbc // indirect + golang.org/x/sync v0.0.0-20190423024810-112230192c58 google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf google.golang.org/appengine v1.3.0 // indirect gopkg.in/DataDog/dd-trace-go.v1 v1.10.0 // indirect - gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect gopkg.in/go-playground/assert.v1 v1.2.1 // indirect gopkg.in/go-playground/validator.v9 v9.20.2 ) diff --git a/go.sum b/go.sum index b9f75e37..6881904d 100644 --- a/go.sum +++ b/go.sum @@ -14,13 +14,19 @@ github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895 h1:dmc/C8bpE5Vk github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/DataDog/opencensus-go-exporter-datadog v0.0.0-20180917103902-e6c7f767dc57 h1:V1H8VVVxLALfaPLvAFCPoa0AN5nVPAqEu2UvH+QP3Vc= github.com/DataDog/opencensus-go-exporter-datadog v0.0.0-20180917103902-e6c7f767dc57/go.mod h1:gMGUEe16aZh0QN941HgDjwrdjU4iTthPoz2/AtDRADE= +github.com/agext/levenshtein v1.2.1 h1:QmvMAjj2aEICytGiWzmxoE0x2KZvE0fvmqMOfy2tjT8= +github.com/agext/levenshtein v1.2.1/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= github.com/ajg/form v0.0.0-20160822230020-523a5da1a92f h1:zvClvFQwU++UpIUBGC8YmDlfhUrweEy1R1Fj1gu5iIM= github.com/ajg/form v0.0.0-20160822230020-523a5da1a92f/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= +github.com/apparentlymart/go-dump v0.0.0-20180507223929-23540a00eaa3/go.mod h1:oL81AME2rN47vu18xqj1S1jPIPuN7afo62yKTNn3XMM= +github.com/apparentlymart/go-textseg v1.0.0 h1:rRmlIsPEEhUTIKQb7T++Nz/A5Q6C9IuX2wFoYVvnCs0= +github.com/apparentlymart/go-textseg v1.0.0/go.mod h1:z96Txxhf3xSFMPmb5X/1W05FF/Nj9VFpLOpjS5yuumk= github.com/aws/aws-sdk-go v1.15.24 h1:xLAdTA/ore6xdPAljzZRed7IGqQgC+nY+ERS5vaj4Ro= github.com/aws/aws-sdk-go v1.15.24/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bsm/go-vlq v0.0.0-20150828105119-ec6e8d4f5f4e/go.mod h1:N+BjUcTjSxc2mtRGSCPsat1kze3CUtvJN3/jTXlp29k= github.com/bsm/redis-lock v8.0.0+incompatible h1:QgB0J2pNG8hUfndTIvpPh38F5XsUTTvO7x8Sls++9Mk= github.com/bsm/redis-lock v8.0.0+incompatible/go.mod h1:8dGkQ5GimBCahwF2R67tqGCJbyDZSp0gzO7wq3pDrik= github.com/codegangsta/negroni v1.0.0 h1:+aYywywx4bnKXWvoWtRfJ91vC59NbEhEY03sZjQhbVY= @@ -54,6 +60,7 @@ github.com/go-redis/redis v6.15.2+incompatible h1:9SpNVG76gr6InJGxoZ6IuuxaCOQwDA github.com/go-redis/redis v6.15.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-test/deep v1.0.1/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/gobuffalo/envy v1.6.7 h1:XMZGuFqTupAXhZTriQ+qO38QvNOSU/0rl3hEPCFci/4= github.com/gobuffalo/envy v1.6.7/go.mod h1:N+GkhhZ/93bGZc6ZKhJLP6+m+tCNPKwgSpH9kaifseQ= github.com/gobuffalo/httptest v1.0.4 h1:P0uKaPEjti1bbJmuBILE3QQ7iU1cS7oIkxVba5HbcVE= @@ -65,6 +72,7 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekf github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 h1:LbsanbbD6LieFkXbj9YNNBupiGHJgFeLpO0j0Fza1h8= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= @@ -99,13 +107,18 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92Bcuy github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.4.1 h1:pX7cnDwSSmG0dR9yNjCQSSpmsJOqFdT7SzVp5Yl9uVw= github.com/grpc-ecosystem/grpc-gateway v1.4.1/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= +github.com/hashicorp/errwrap v0.0.0-20180715044906-d6c0cd880357/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v0.0.0-20180717150148-3d5d8f294aa0/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/hcl2 v0.0.0-20190503213020-640445e16309 h1:VBvyXC+b6Ix/MXMGIrOHjq+Ew1IRP52EzoQXf1KpwZo= +github.com/hashicorp/hcl2 v0.0.0-20190503213020-640445e16309/go.mod h1:4oI94iqF3GB10QScn46WqbG0kgTUpha97SAzzg2+2ec= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8 h1:12VvqtR6Aowv3l/EQUlocDHW2Cp4G9WJVH7uyH8QFJE= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= @@ -125,6 +138,8 @@ github.com/kr/pty v1.0.0/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4= +github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/markbates/hmax v1.0.0 h1:yo2N0gBoCnUMKhV/VRLHomT6Y9wUm+oQQENuWJqCdlM= github.com/markbates/hmax v1.0.0/go.mod h1:cOkR9dktiESxIMu+65oc/r/bdY4bE8zZw3OLhLx0X2c= github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= @@ -139,11 +154,15 @@ github.com/minio/minio-go v6.0.5+incompatible h1:qxQQW40lV2vuE9i6yYmt90GSJlT1YrM github.com/minio/minio-go v6.0.5+incompatible/go.mod h1:7guKYtitv8dktvNUGrhzmNlA5wrAABTQXCoesZdFQO8= github.com/mitchellh/go-homedir v1.0.0 h1:vKb8ShqSby24Yrqr/yDYkuFz8d0WUjys40rvnGC8aR0= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7 h1:DpOJ2HYzCv8LZP15IdmG+YdwD2luVPHITV96TkirNBM= +github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.4.2 h1:3mYCb7aPxS/RU7TI1y4rkEn1oKmPRjNJLNEXgw7MH2I= github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= @@ -164,6 +183,8 @@ github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7q github.com/prometheus/procfs v0.0.0-20180612222113-7d6f385de8be/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273 h1:agujYaXJSxSo18YNX3jzl+4G6Bstwt+kqv47GS12uL0= github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.1.1 h1:VzGj7lhU7KEB9e9gMpAV/v5XT2NVSvLJhJLCWbnkgXg= github.com/sirupsen/logrus v1.1.1/go.mod h1:zrgwTnHtNr00buQ1vSptGe8m1f/BbgsPukg8qsT7A+A= @@ -178,6 +199,7 @@ github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/pflag v1.0.1 h1:aCvUg6QPl3ibpQUxyLkrEkCHtPqYJL4x9AuhqVqFis4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -196,12 +218,15 @@ github.com/ugorji/go v1.1.1/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJ github.com/unrolled/secure v0.0.0-20181221173256-0d6b5bb13069 h1:RKeYksgIwGE8zFJTvXI1WWx09QPrGyaVFMy0vpU7j/o= github.com/unrolled/secure v0.0.0-20181221173256-0d6b5bb13069/go.mod h1:mnPT77IAdsi/kV7+Es7y+pXALeV3h7G6dQF6mNYjcLA= github.com/urfave/cli v1.18.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/vmihailenco/msgpack v3.3.3+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0= github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/zclconf/go-cty v0.0.0-20190426224007-b18a157db9e2 h1:Ai1LhlYNEqE39zGU07qHDNJ41iZVPZfZr1dSCoXrp1w= +github.com/zclconf/go-cty v0.0.0-20190426224007-b18a157db9e2/go.mod h1:xnAOWiHeOqg2nWS62VtQ7pbOu17FtxJNW8RLEih+O3s= go.etcd.io/bbolt v1.3.2 h1:Z/90sZLPOeCy2PwprqkFa25PdkusRzaj9P8zm/KNyvk= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd v0.0.0-20190215181705-784daa04988c h1:pkiZ418C7QN/HIps1lDF1+lzZhdgMpvFN4kDcxrYhD0= @@ -220,26 +245,43 @@ golang.org/x/crypto v0.0.0-20180608092829-8ac0e0d97ce4/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029103014-dab2b1051b5d h1:5JyY8HlzxzYI+qHOOciM8s2lJbIEaefMUdtYt7dRDrg= golang.org/x/crypto v0.0.0-20181029103014-dab2b1051b5d/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 h1:p/H982KKEjUnLJkM3tt/LemDnOc1GiZL5FCVlORJ5zo= +golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180811021610-c39426892332/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd h1:nTDtHvHSdCn1m6ITfMRqtOd/9+7a3s8RBNOZ3eYZzJA= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181029044818-c44066c5c816 h1:mVFkLpejdFLXVUv9E42f3XJVfMdqd0IVLVIVLjZWn5o= golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190502183928-7f726cade0ab h1:9RfW3ktsOZxgo9YNbBAjq1FWzc/igwEcUzZz8IXgSbk= +golang.org/x/net v0.0.0-20190502183928-7f726cade0ab/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/oauth2 v0.0.0-20180620175406-ef147856a6dd h1:QQhib242ErYDSMitlBm8V7wYCm/1a25hV8qMadIKLPA= golang.org/x/oauth2 v0.0.0-20180620175406-ef147856a6dd/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e h1:o3PsSEY8E4eXWkXrIP9YJALUkVZqzHJT5DOasTyn8Vs= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181031143558-9b800f95dbbc h1:SdCq5U4J+PpbSDIl9bM0V1e1Ug1jsnBkAFvTs1htn7U= golang.org/x/sys v0.0.0-20181031143558-9b800f95dbbc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502175342-a43fa875dd82 h1:vsphBvatvfbhlb4PO1BYSr9dzugGxJ/SQHoNufZJq1w= +golang.org/x/sys v0.0.0-20190502175342-a43fa875dd82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 h1:+DCIGbF/swA92ohVg0//6X2IVY3KZs6p9mix0ziNYJM= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf h1:rjxqQmxjyqerRKEj+tZW+MCm4LgpFXu18bsEoCMgDsk= google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.3.0 h1:FBSsiFRMz3LBeXIomRnVzrQwSDj4ibvcRexLG0LZGQk= google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180608181217-32ee49c4dd80/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= @@ -267,3 +309,4 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= diff --git a/pkg/config/config.go b/pkg/config/config.go index 23b1a921..874a88d0 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -8,6 +8,7 @@ import ( "runtime" "github.com/BurntSushi/toml" + "github.com/gomods/athens/pkg/download/mode" "github.com/gomods/athens/pkg/errors" "github.com/kelseyhightower/envconfig" validator "gopkg.in/go-playground/validator.v9" @@ -18,32 +19,34 @@ const defaultConfigFile = "athens.toml" // Config provides configuration values for all components type Config struct { TimeoutConf - GoEnv string `validate:"required" envconfig:"GO_ENV"` - GoBinary string `validate:"required" envconfig:"GO_BINARY_PATH"` - GoGetWorkers int `validate:"required" envconfig:"ATHENS_GOGET_WORKERS"` - ProtocolWorkers int `validate:"required" envconfig:"ATHENS_PROTOCOL_WORKERS"` - LogLevel string `validate:"required" envconfig:"ATHENS_LOG_LEVEL"` - CloudRuntime string `validate:"required" envconfig:"ATHENS_CLOUD_RUNTIME"` - FilterFile string `envconfig:"ATHENS_FILTER_FILE"` - TraceExporterURL string `envconfig:"ATHENS_TRACE_EXPORTER_URL"` - TraceExporter string `envconfig:"ATHENS_TRACE_EXPORTER"` - StatsExporter string `envconfig:"ATHENS_STATS_EXPORTER"` - StorageType string `validate:"required" envconfig:"ATHENS_STORAGE_TYPE"` - GlobalEndpoint string `envconfig:"ATHENS_GLOBAL_ENDPOINT"` // This feature is not yet implemented - Port string `envconfig:"ATHENS_PORT"` - BasicAuthUser string `envconfig:"BASIC_AUTH_USER"` - BasicAuthPass string `envconfig:"BASIC_AUTH_PASS"` - ForceSSL bool `envconfig:"PROXY_FORCE_SSL"` - ValidatorHook string `envconfig:"ATHENS_PROXY_VALIDATOR"` - PathPrefix string `envconfig:"ATHENS_PATH_PREFIX"` - NETRCPath string `envconfig:"ATHENS_NETRC_PATH"` - GithubToken string `envconfig:"ATHENS_GITHUB_TOKEN"` - HGRCPath string `envconfig:"ATHENS_HGRC_PATH"` - TLSCertFile string `envconfig:"ATHENS_TLSCERT_FILE"` - TLSKeyFile string `envconfig:"ATHENS_TLSKEY_FILE"` - SumDBs []string `envconfig:"ATHENS_SUM_DBS"` - NoSumPatterns []string `envconfig:"ATHENS_GONOSUM_PATTERNS"` - SingleFlightType string `envconfig:"ATHENS_SINGLE_FLIGHT_TYPE"` + GoEnv string `validate:"required" envconfig:"GO_ENV"` + GoBinary string `validate:"required" envconfig:"GO_BINARY_PATH"` + GoGetWorkers int `validate:"required" envconfig:"ATHENS_GOGET_WORKERS"` + ProtocolWorkers int `validate:"required" envconfig:"ATHENS_PROTOCOL_WORKERS"` + LogLevel string `validate:"required" envconfig:"ATHENS_LOG_LEVEL"` + CloudRuntime string `validate:"required" envconfig:"ATHENS_CLOUD_RUNTIME"` + FilterFile string `envconfig:"ATHENS_FILTER_FILE"` + TraceExporterURL string `envconfig:"ATHENS_TRACE_EXPORTER_URL"` + TraceExporter string `envconfig:"ATHENS_TRACE_EXPORTER"` + StatsExporter string `envconfig:"ATHENS_STATS_EXPORTER"` + StorageType string `validate:"required" envconfig:"ATHENS_STORAGE_TYPE"` + GlobalEndpoint string `envconfig:"ATHENS_GLOBAL_ENDPOINT"` // This feature is not yet implemented + Port string `envconfig:"ATHENS_PORT"` + BasicAuthUser string `envconfig:"BASIC_AUTH_USER"` + BasicAuthPass string `envconfig:"BASIC_AUTH_PASS"` + ForceSSL bool `envconfig:"PROXY_FORCE_SSL"` + ValidatorHook string `envconfig:"ATHENS_PROXY_VALIDATOR"` + PathPrefix string `envconfig:"ATHENS_PATH_PREFIX"` + NETRCPath string `envconfig:"ATHENS_NETRC_PATH"` + GithubToken string `envconfig:"ATHENS_GITHUB_TOKEN"` + HGRCPath string `envconfig:"ATHENS_HGRC_PATH"` + TLSCertFile string `envconfig:"ATHENS_TLSCERT_FILE"` + TLSKeyFile string `envconfig:"ATHENS_TLSKEY_FILE"` + SumDBs []string `envconfig:"ATHENS_SUM_DBS"` + NoSumPatterns []string `envconfig:"ATHENS_GONOSUM_PATTERNS"` + DownloadMode mode.Mode `envconfig:"ATHENS_DOWNLOAD_MODE"` + DownloadURL string `envconfig:"ATHENS_DOWNLOAD_URL"` + SingleFlightType string `envconfig:"ATHENS_SINGLE_FLIGHT_TYPE"` SingleFlight *SingleFlight Storage *StorageConfig } @@ -84,6 +87,8 @@ func defaultConfig() *Config { TraceExporterURL: "http://localhost:14268", SumDBs: []string{"https://sum.golang.org"}, NoSumPatterns: []string{}, + DownloadMode: "sync", + DownloadURL: "", SingleFlight: &SingleFlight{ Etcd: &Etcd{"localhost:2379,localhost:22379,localhost:32379"}, Redis: &Redis{"127.0.0.1:6379"}, diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 2839f722..94c8ca32 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -266,6 +266,7 @@ func TestParseExampleConfig(t *testing.T) { SingleFlight: &SingleFlight{}, SumDBs: []string{"https://sum.golang.org"}, NoSumPatterns: []string{}, + DownloadMode: "sync", } absPath, err := filepath.Abs(testConfigFile(t)) diff --git a/pkg/download/handler.go b/pkg/download/handler.go index 1a474fa2..cdbc4ab9 100644 --- a/pkg/download/handler.go +++ b/pkg/download/handler.go @@ -2,7 +2,9 @@ package download import ( "net/http" + "net/url" + "github.com/gomods/athens/pkg/download/mode" "github.com/gomods/athens/pkg/log" "github.com/gomods/athens/pkg/middleware" "github.com/gorilla/mux" @@ -10,13 +12,14 @@ import ( // ProtocolHandler is a function that takes all that it needs to return // a ready-to-go http handler that serves up cmd/go's download protocol. -type ProtocolHandler func(dp Protocol, lggr log.Entry) http.Handler +type ProtocolHandler func(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler // HandlerOpts are the generic options // for a ProtocolHandler type HandlerOpts struct { - Protocol Protocol - Logger *log.Logger + Protocol Protocol + Logger *log.Logger + DownloadFile *mode.DownloadFile } // LogEntryHandler pulls a log entry from the request context. Thanks to the @@ -26,7 +29,7 @@ type HandlerOpts struct { func LogEntryHandler(ph ProtocolHandler, opts *HandlerOpts) http.Handler { f := func(w http.ResponseWriter, r *http.Request) { ent := log.EntryFromContext(r.Context()) - handler := ph(opts.Protocol, ent) + handler := ph(opts.Protocol, ent, opts.DownloadFile) handler.ServeHTTP(w, r) } return http.HandlerFunc(f) @@ -51,3 +54,9 @@ func RegisterHandlers(r *mux.Router, opts *HandlerOpts) { r.Handle(PathVersionModule, LogEntryHandler(ModuleHandler, opts)).Methods(http.MethodGet) r.Handle(PathVersionZip, LogEntryHandler(ZipHandler, opts)).Methods(http.MethodGet) } + +func getRedirectURL(base, path string) string { + url, _ := url.Parse(base) + url.Path = path + return url.String() +} diff --git a/pkg/download/handler_test.go b/pkg/download/handler_test.go new file mode 100644 index 00000000..41c09e99 --- /dev/null +++ b/pkg/download/handler_test.go @@ -0,0 +1,45 @@ +package download + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + "github.com/gomods/athens/pkg/download/mode" + "github.com/gomods/athens/pkg/errors" + "github.com/gomods/athens/pkg/log" + "github.com/gorilla/mux" +) + +func TestRedirect(t *testing.T) { + r := mux.NewRouter() + RegisterHandlers(r, &HandlerOpts{ + Protocol: &mockProtocol{}, + Logger: log.NoOpLogger(), + DownloadFile: &mode.DownloadFile{ + Mode: mode.Redirect, + DownloadURL: "https://gomods.io", + }, + }) + req := httptest.NewRequest("GET", "/github.com/gomods/athens/@v/v0.4.0.info", nil) + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != http.StatusMovedPermanently { + t.Fatalf("expected a redirect status (301) but got %v", w.Code) + } + expectedRedirect := "https://gomods.io/github.com/gomods/athens/@v/v0.4.0.info" + givenRedirect := w.HeaderMap.Get("location") + if expectedRedirect != givenRedirect { + t.Fatalf("expected the handler to redirect to %q but got %q", expectedRedirect, givenRedirect) + } +} + +type mockProtocol struct { + Protocol +} + +func (mp *mockProtocol) Info(ctx context.Context, mod, ver string) ([]byte, error) { + const op errors.Op = "mockProtocol.Info" + return nil, errors.E(op, "not found", errors.KindRedirect) +} diff --git a/pkg/download/latest.go b/pkg/download/latest.go index 5855a689..bd5b1bed 100644 --- a/pkg/download/latest.go +++ b/pkg/download/latest.go @@ -4,6 +4,7 @@ import ( "encoding/json" "net/http" + "github.com/gomods/athens/pkg/download/mode" "github.com/gomods/athens/pkg/errors" "github.com/gomods/athens/pkg/log" "github.com/gomods/athens/pkg/paths" @@ -13,7 +14,7 @@ import ( const PathLatest = "/{module:.+}/@latest" // LatestHandler implements GET baseURL/module/@latest -func LatestHandler(dp Protocol, lggr log.Entry) http.Handler { +func LatestHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler { const op errors.Op = "download.LatestHandler" f := func(w http.ResponseWriter, r *http.Request) { mod, err := paths.GetModule(r) diff --git a/pkg/download/list.go b/pkg/download/list.go index 393fe84f..58ed6d2c 100644 --- a/pkg/download/list.go +++ b/pkg/download/list.go @@ -5,6 +5,7 @@ import ( "net/http" "strings" + "github.com/gomods/athens/pkg/download/mode" "github.com/gomods/athens/pkg/errors" "github.com/gomods/athens/pkg/log" "github.com/gomods/athens/pkg/paths" @@ -14,7 +15,7 @@ import ( const PathList = "/{module:.+}/@v/list" // ListHandler implements GET baseURL/module/@v/list -func ListHandler(dp Protocol, lggr log.Entry) http.Handler { +func ListHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler { const op errors.Op = "download.ListHandler" f := func(w http.ResponseWriter, r *http.Request) { mod, err := paths.GetModule(r) diff --git a/pkg/download/list_merge_test.go b/pkg/download/list_merge_test.go index 40caf24f..051622a7 100644 --- a/pkg/download/list_merge_test.go +++ b/pkg/download/list_merge_test.go @@ -128,7 +128,7 @@ func TestListMerge(t *testing.T) { s.Save(ctx, testModName, v, bts, ioutil.NopCloser(bytes.NewReader(bts)), bts) } defer clearStorage(s, testModName, tc.strVersions) - dp := New(&Opts{s, nil, &listerMock{versions: tc.goVersions, err: tc.goErr}}) + dp := New(&Opts{s, nil, &listerMock{versions: tc.goVersions, err: tc.goErr}, nil}) list, err := dp.List(ctx, testModName) if ok := testErrEq(tc.expectedErr, err); !ok { diff --git a/pkg/download/mode/mode.go b/pkg/download/mode/mode.go new file mode 100644 index 00000000..0e736d67 --- /dev/null +++ b/pkg/download/mode/mode.go @@ -0,0 +1,132 @@ +package mode + +import ( + "encoding/base64" + "fmt" + "io/ioutil" + "path" + "strings" + + "github.com/gomods/athens/pkg/errors" + "github.com/hashicorp/hcl2/gohcl" + "github.com/hashicorp/hcl2/hclparse" +) + +// Mode specifies the behavior of what to do +// when a module is not found in storage. +type Mode string + +// DownloadMode constants. For more information see config.dev.toml +const ( + Sync Mode = "sync" + Async Mode = "async" + Redirect Mode = "redirect" + AsyncRedirect Mode = "async_redirect" + None Mode = "none" +) + +// DownloadFile represents a custom HCL format of +// how to handle module@version requests that are +// not found in storage. +type DownloadFile struct { + Mode Mode `hcl:"mode"` + DownloadURL string `hcl:"downloadURL"` + Paths []*DownloadPath `hcl:"download,block"` +} + +// DownloadPath represents a custom Mode for +// a matching path. +type DownloadPath struct { + Pattern string `hcl:"pattern,label"` + Mode Mode `hcl:"mode"` + DownloadURL string `hcl:"downloadURL,optional"` +} + +// NewFile takes a mode and returns a DownloadFile. +// Mode can be one of the constants declared above +// or a custom HCL file. To pass a custom HCL file, +// you can either point to a file path by passing +// file:/path/to/file OR custom: +// directly. +func NewFile(m Mode, downloadURL string) (*DownloadFile, error) { + const op errors.Op = "downloadMode.NewFile" + if strings.HasPrefix(string(m), "file:") { + filePath := string(m[5:]) + bts, err := ioutil.ReadFile(filePath) + if err != nil { + return nil, err + } + return parseFile(bts) + } else if strings.HasPrefix(string(m), "custom:") { + bts, err := base64.StdEncoding.DecodeString(string(m[7:])) + if err != nil { + return nil, err + } + return parseFile(bts) + } + switch m { + case Sync, Async, Redirect, AsyncRedirect, None: + return &DownloadFile{Mode: m, DownloadURL: downloadURL}, nil + default: + return nil, errors.E(op, "unrecognized download mode: "+m, errors.KindBadRequest) + } +} + +// parseFile parses an HCL file according to the +// DownloadMode spec. +func parseFile(file []byte) (*DownloadFile, error) { + const op errors.Op = "downloadmode.parseFile" + f, dig := hclparse.NewParser().ParseHCL(file, "config.hcl") + if dig.HasErrors() { + return nil, errors.E(op, dig.Error()) + } + var df DownloadFile + dig = gohcl.DecodeBody(f.Body, nil, &df) + if dig.HasErrors() { + return nil, errors.E(op, dig.Error()) + } + if err := df.validate(); err != nil { + return nil, errors.E(op, err) + } + return &df, nil +} + +func (d *DownloadFile) validate() error { + const op errors.Op = "downloadMode.validate" + for _, p := range d.Paths { + switch p.Mode { + case Sync, Async, Redirect, AsyncRedirect, None: + default: + return errors.E(op, fmt.Errorf("unrecognized mode for %v: %v", p.Pattern, p.Mode)) + } + } + return nil +} + +// Match returns the Mode that matches the given +// module. A pattern is prioritized by order in +// which it appears in the HCL file, while the +// default Mode will be returned if no patterns +// exist or match. +func (d *DownloadFile) Match(mod string) Mode { + for _, p := range d.Paths { + if hasMatch, err := path.Match(p.Pattern, mod); hasMatch && err == nil { + return p.Mode + } + } + return d.Mode +} + +// URL returns the redirect URL that applies +// to the given module. If no pattern matches, +// the top level downloadURL is returned. +func (d *DownloadFile) URL(mod string) string { + for _, p := range d.Paths { + if hasMatch, err := path.Match(p.Pattern, mod); hasMatch && err == nil { + if p.DownloadURL != "" { + return p.DownloadURL + } + } + } + return d.DownloadURL +} diff --git a/pkg/download/mode/mode_test.go b/pkg/download/mode/mode_test.go new file mode 100644 index 00000000..d6beb313 --- /dev/null +++ b/pkg/download/mode/mode_test.go @@ -0,0 +1,96 @@ +package mode + +import ( + "testing" +) + +var testCases = []struct { + name string + file *DownloadFile + input string + expectedMode Mode + expectedURL string +}{ + { + name: "sync", + file: &DownloadFile{Mode: Sync}, + input: "github.com/gomods/athens", + expectedMode: Sync, + }, + { + name: "redirect", + file: &DownloadFile{Mode: Redirect, DownloadURL: "gomods.io"}, + input: "github.com/gomods/athens", + expectedMode: Redirect, + expectedURL: "gomods.io", + }, + { + name: "pattern match", + file: &DownloadFile{ + Mode: Sync, + Paths: []*DownloadPath{ + {Pattern: "github.com/gomods/*", Mode: None}, + }, + }, + input: "github.com/gomods/athens", + expectedMode: None, + }, + { + name: "pattern fallback", + file: &DownloadFile{ + Mode: Sync, + Paths: []*DownloadPath{ + {Pattern: "github.com/gomods/*", Mode: None}, + }, + }, + input: "github.com/athens-artifacts/maturelib", + expectedMode: Sync, + }, + { + name: "pattern redirect", + file: &DownloadFile{ + Mode: Sync, + Paths: []*DownloadPath{ + { + Pattern: "github.com/gomods/*", + Mode: AsyncRedirect, + DownloadURL: "gomods.io"}, + }, + }, + input: "github.com/gomods/athens", + expectedMode: AsyncRedirect, + expectedURL: "gomods.io", + }, + { + name: "redirect fallback", + file: &DownloadFile{ + Mode: Redirect, + DownloadURL: "proxy.golang.org", + Paths: []*DownloadPath{ + { + Pattern: "github.com/gomods/*", + Mode: AsyncRedirect, + DownloadURL: "gomods.io", + }, + }, + }, + input: "github.com/athens-artifacts/maturelib", + expectedMode: Redirect, + expectedURL: "proxy.golang.org", + }, +} + +func TestMode(t *testing.T) { + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + givenMode := tc.file.Match(tc.input) + if givenMode != tc.expectedMode { + t.Fatalf("expected matched mode to be %q but got %q", tc.expectedMode, givenMode) + } + givenURL := tc.file.URL(tc.input) + if givenURL != tc.expectedURL { + t.Fatalf("expected matched DownloadURL to be %q but got %q", tc.expectedURL, givenURL) + } + }) + } +} diff --git a/pkg/download/protocol.go b/pkg/download/protocol.go index e0159115..b3e9a87a 100644 --- a/pkg/download/protocol.go +++ b/pkg/download/protocol.go @@ -7,6 +7,7 @@ import ( "strings" "sync" + "github.com/gomods/athens/pkg/download/mode" "github.com/gomods/athens/pkg/errors" "github.com/gomods/athens/pkg/observ" "github.com/gomods/athens/pkg/stash" @@ -37,9 +38,10 @@ type Wrapper func(Protocol) Protocol // Opts specifies download protocol options to avoid long func signature. type Opts struct { - Storage storage.Backend - Stasher stash.Stasher - Lister UpstreamLister + Storage storage.Backend + Stasher stash.Stasher + Lister UpstreamLister + DownloadFile *mode.DownloadFile } // New returns a full implementation of the download.Protocol @@ -48,7 +50,10 @@ type Opts struct { // The wrappers are applied in order, meaning the last wrapper // passed is the Protocol that gets hit first. func New(opts *Opts, wrappers ...Wrapper) Protocol { - var p Protocol = &protocol{opts.Storage, opts.Stasher, opts.Lister} + if opts.DownloadFile == nil { + opts.DownloadFile = &mode.DownloadFile{Mode: mode.Sync} + } + var p Protocol = &protocol{opts.DownloadFile, opts.Storage, opts.Stasher, opts.Lister} for _, w := range wrappers { p = w(p) } @@ -57,6 +62,7 @@ func New(opts *Opts, wrappers ...Wrapper) Protocol { } type protocol struct { + df *mode.DownloadFile storage storage.Backend stasher stash.Stasher lister UpstreamLister @@ -151,13 +157,11 @@ func (p *protocol) Info(ctx context.Context, mod, ver string) ([]byte, error) { ctx, span := observ.StartSpan(ctx, op.String()) defer span.End() info, err := p.storage.Info(ctx, mod, ver) - var newVer string if errors.IsNotFoundErr(err) { - newVer, err = p.stasher.Stash(ctx, mod, ver) - if err != nil { - return nil, errors.E(op, err) - } - info, err = p.storage.Info(ctx, mod, newVer) + err = p.processDownload(ctx, mod, ver, func(newVer string) error { + info, err = p.storage.Info(ctx, mod, newVer) + return err + }) } if err != nil { return nil, errors.E(op, err) @@ -171,18 +175,15 @@ func (p *protocol) GoMod(ctx context.Context, mod, ver string) ([]byte, error) { ctx, span := observ.StartSpan(ctx, op.String()) defer span.End() goMod, err := p.storage.GoMod(ctx, mod, ver) - var newVer string if errors.IsNotFoundErr(err) { - newVer, err = p.stasher.Stash(ctx, mod, ver) - if err != nil { - return nil, errors.E(op, err) - } - goMod, err = p.storage.GoMod(ctx, mod, newVer) + err = p.processDownload(ctx, mod, ver, func(newVer string) error { + goMod, err = p.storage.GoMod(ctx, mod, newVer) + return err + }) } if err != nil { return nil, errors.E(op, err) } - return goMod, nil } @@ -191,13 +192,11 @@ func (p *protocol) Zip(ctx context.Context, mod, ver string) (io.ReadCloser, err ctx, span := observ.StartSpan(ctx, op.String()) defer span.End() zip, err := p.storage.Zip(ctx, mod, ver) - var newVer string if errors.IsNotFoundErr(err) { - newVer, err = p.stasher.Stash(ctx, mod, ver) - if err != nil { - return nil, errors.E(op, err) - } - zip, err = p.storage.Zip(ctx, mod, newVer) + err = p.processDownload(ctx, mod, ver, func(newVer string) error { + zip, err = p.storage.Zip(ctx, mod, newVer) + return err + }) } if err != nil { return nil, errors.E(op, err) @@ -206,6 +205,29 @@ func (p *protocol) Zip(ctx context.Context, mod, ver string) (io.ReadCloser, err return zip, nil } +func (p *protocol) processDownload(ctx context.Context, mod, ver string, f func(newVer string) error) error { + const op errors.Op = "protocol.processDownload" + switch p.df.Match(mod) { + case mode.Sync: + newVer, err := p.stasher.Stash(ctx, mod, ver) + if err != nil { + return errors.E(op, err) + } + return f(newVer) + case mode.Async: + go p.stasher.Stash(ctx, mod, ver) + return errors.E(op, "async: module not found", errors.KindNotFound) + case mode.Redirect: + return errors.E(op, "redirect", errors.KindRedirect) + case mode.AsyncRedirect: + go p.stasher.Stash(ctx, mod, ver) + return errors.E(op, "async_redirect: module not found", errors.KindRedirect) + case mode.None: + return errors.E(op, "none", errors.KindNotFound) + } + return nil +} + // union concatenates two version lists and removes duplicates func union(list1, list2 []string) []string { if list1 == nil { diff --git a/pkg/download/protocol_test.go b/pkg/download/protocol_test.go index 656e7d10..438aec0d 100644 --- a/pkg/download/protocol_test.go +++ b/pkg/download/protocol_test.go @@ -13,6 +13,7 @@ import ( "time" "github.com/gomods/athens/pkg/config" + "github.com/gomods/athens/pkg/download/mode" "github.com/gomods/athens/pkg/errors" "github.com/gomods/athens/pkg/module" "github.com/gomods/athens/pkg/stash" @@ -44,7 +45,7 @@ func getDP(t *testing.T) Protocol { t.Fatal(err) } st := stash.New(mf, s) - return New(&Opts{s, st, NewVCSLister(goBin, fs)}) + return New(&Opts{s, st, NewVCSLister(goBin, fs), nil}) } type listTest struct { @@ -280,7 +281,7 @@ func TestDownloadProtocol(t *testing.T) { } mp := &mockFetcher{} st := stash.New(mp, s) - dp := New(&Opts{s, st, nil}) + dp := New(&Opts{s, st, nil, nil}) ctx := context.Background() var eg errgroup.Group @@ -332,7 +333,7 @@ func TestDownloadProtocolWhenFetchFails(t *testing.T) { } mp := ¬FoundFetcher{} st := stash.New(mp, s) - dp := New(&Opts{s, st, nil}) + dp := New(&Opts{s, st, nil, nil}) ctx := context.Background() _, err = dp.GoMod(ctx, fakeMod.mod, fakeMod.ver) if err != nil { @@ -340,6 +341,40 @@ func TestDownloadProtocolWhenFetchFails(t *testing.T) { } } +func TestAsyncRedirect(t *testing.T) { + s, err := mem.NewStorage() + require.NoError(t, err) + ms := &mockStasher{s, make(chan bool)} + dp := New(&Opts{ + Stasher: ms, + Storage: s, + DownloadFile: &mode.DownloadFile{ + Mode: mode.Async, + DownloadURL: "https://gomods.io", + }, + }) + mod, ver := "github.com/athens-artifacts/happy-path", "v0.0.1" + _, err = dp.Info(context.Background(), mod, ver) + if errors.Kind(err) != errors.KindNotFound { + t.Fatalf("expected async_redirect to enforce a 404 but got %v", errors.Kind(err)) + } + <-ms.ch + info, err := dp.Info(context.Background(), mod, ver) + require.NoError(t, err) + require.Equal(t, string(info), "info", "expected async fetch to be successful") +} + +type mockStasher struct { + s storage.Backend + ch chan bool +} + +func (ms *mockStasher) Stash(ctx context.Context, mod string, ver string) (string, error) { + err := ms.s.Save(ctx, mod, ver, []byte("mod"), strings.NewReader("zip"), []byte("info")) + ms.ch <- true // signal async stashing is done + return ver, err +} + type notFoundFetcher struct{} func (m *notFoundFetcher) Fetch(ctx context.Context, mod, ver string) (*storage.Version, error) { diff --git a/pkg/download/version_info.go b/pkg/download/version_info.go index c9c38833..abef94ea 100644 --- a/pkg/download/version_info.go +++ b/pkg/download/version_info.go @@ -3,6 +3,7 @@ package download import ( "net/http" + "github.com/gomods/athens/pkg/download/mode" "github.com/gomods/athens/pkg/errors" "github.com/gomods/athens/pkg/log" ) @@ -11,7 +12,7 @@ import ( const PathVersionInfo = "/{module:.+}/@v/{version}.info" // InfoHandler implements GET baseURL/module/@v/version.info -func InfoHandler(dp Protocol, lggr log.Entry) http.Handler { +func InfoHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler { const op errors.Op = "download.InfoHandler" f := func(w http.ResponseWriter, r *http.Request) { mod, ver, err := getModuleParams(r, op) @@ -23,6 +24,10 @@ func InfoHandler(dp Protocol, lggr log.Entry) http.Handler { info, err := dp.Info(r.Context(), mod, ver) if err != nil { lggr.SystemErr(errors.E(op, err, errors.M(mod), errors.V(ver))) + if errors.Kind(err) == errors.KindRedirect { + http.Redirect(w, r, getRedirectURL(df.URL(mod), r.URL.Path), errors.KindRedirect) + return + } w.WriteHeader(errors.Kind(err)) } diff --git a/pkg/download/version_module.go b/pkg/download/version_module.go index f4bec86a..bb0bcb2a 100644 --- a/pkg/download/version_module.go +++ b/pkg/download/version_module.go @@ -3,6 +3,7 @@ package download import ( "net/http" + "github.com/gomods/athens/pkg/download/mode" "github.com/gomods/athens/pkg/errors" "github.com/gomods/athens/pkg/log" ) @@ -11,12 +12,16 @@ import ( const PathVersionModule = "/{module:.+}/@v/{version}.mod" // ModuleHandler implements GET baseURL/module/@v/version.mod -func ModuleHandler(dp Protocol, lggr log.Entry) http.Handler { +func ModuleHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler { const op errors.Op = "download.VersionModuleHandler" f := func(w http.ResponseWriter, r *http.Request) { mod, ver, err := getModuleParams(r, op) if err != nil { lggr.SystemErr(err) + if errors.Kind(err) == errors.KindRedirect { + http.Redirect(w, r, getRedirectURL(df.URL(mod), r.URL.Path), errors.KindRedirect) + return + } w.WriteHeader(errors.Kind(err)) return } diff --git a/pkg/download/version_zip.go b/pkg/download/version_zip.go index d468d11d..9c8702b5 100644 --- a/pkg/download/version_zip.go +++ b/pkg/download/version_zip.go @@ -4,6 +4,7 @@ import ( "io" "net/http" + "github.com/gomods/athens/pkg/download/mode" "github.com/gomods/athens/pkg/errors" "github.com/gomods/athens/pkg/log" ) @@ -12,7 +13,7 @@ import ( const PathVersionZip = "/{module:.+}/@v/{version}.zip" // ZipHandler implements GET baseURL/module/@v/version.zip -func ZipHandler(dp Protocol, lggr log.Entry) http.Handler { +func ZipHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler { const op errors.Op = "download.ZipHandler" f := func(w http.ResponseWriter, r *http.Request) { mod, ver, err := getModuleParams(r, op) @@ -24,6 +25,10 @@ func ZipHandler(dp Protocol, lggr log.Entry) http.Handler { zip, err := dp.Zip(r.Context(), mod, ver) if err != nil { lggr.SystemErr(err) + if errors.Kind(err) == errors.KindRedirect { + http.Redirect(w, r, getRedirectURL(df.URL(mod), r.URL.Path), errors.KindRedirect) + return + } w.WriteHeader(errors.Kind(err)) return } diff --git a/pkg/errors/errors.go b/pkg/errors/errors.go index f47e2e5a..6e67073c 100644 --- a/pkg/errors/errors.go +++ b/pkg/errors/errors.go @@ -17,6 +17,7 @@ const ( KindAlreadyExists = http.StatusConflict KindRateLimit = http.StatusTooManyRequests KindNotImplemented = http.StatusNotImplemented + KindRedirect = http.StatusMovedPermanently ) // Error is an Athens system error.