From be84140e6abfa7e05a2ee35004ae203d4f82112e Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Fri, 6 Dec 2019 13:48:00 -0800 Subject: [PATCH 01/23] Adding a "next step" button to our contributing pages (#1486) * Adding a "next step" button to our contributing pages Fixes https://github.com/gomods/athens/issues/1485 * a little more text --- docs/content/contributing/community/_index.md | 12 +++++++++--- docs/content/contributing/new/_index.md | 7 ++++--- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/docs/content/contributing/community/_index.md b/docs/content/contributing/community/_index.md index 0c90cfc6..5fcd1f98 100644 --- a/docs/content/contributing/community/_index.md +++ b/docs/content/contributing/community/_index.md @@ -6,8 +6,14 @@ weight: 1 --- -### Contributing - # The Athens Community -The following guide is designed to help members participate within the community by becoming a contributor, maintainer, or even just following the `#athens` slack channel or attending the developer meetings +Welcome, Athenian! We've put together this section to help you get involved with the Athens community. + +Before we go further, we want you to know two things: + +1. You can contribute in many ways, including documentation, testing, writing code, reporting bugs, reviewing code, technical writing, and more +2. [Absolutely everybody is welcome](https://arschles.com/blog/absolutely-everybody-is-welcome/). You are welcome in our community, regardless of your level of programming experience, number of years writing Go, race, religion, sexual orientation, gender identity, or anything else. If you want to be here, we will do everything we can to help you feel welcome and involved + + +Ready to join us? Head over to the [guide on how to participate](./participating) to get started! diff --git a/docs/content/contributing/new/_index.md b/docs/content/contributing/new/_index.md index 693d23c6..e7479b49 100644 --- a/docs/content/contributing/new/_index.md +++ b/docs/content/contributing/new/_index.md @@ -6,8 +6,9 @@ weight: 2 --- -### Contributing +# Welcome, New Contributors! -# New Contributors -The following guide is designed to help contributors new to open source. +This section is all about helping you get started with open source and Athens. + +Let's get started with [learning how to use `git`](./git) From 6c47f8bcbfa7f7e20332cd11fc98812f85c3b15f Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Mon, 9 Dec 2019 11:01:53 -0800 Subject: [PATCH 02/23] Adding a header to the contributing doc (#1483) --- CONTRIBUTING.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 16537533..eb021610 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,3 +1,4 @@ +# Contributing to Athens Hurray! We are glad that you want to contribute to our project! 👍 If this is your first contribution, not to worry! We have a great [tutorial](https://www.youtube.com/watch?v=bgSDcTyysRc) to help you get started, and you can always ask us for help in the `#athens` channel in the [gopher slack](https://invite.slack.golangbridge.org/). We'll give you whatever guidance you need. Another great resource for first time contributors can be found [here](https://github.com/firstcontributions/first-contributions/blob/master/README.md). From c695a41fefb34e2e1d5a59c903ef783991255aad Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Thu, 19 Dec 2019 13:00:52 -0800 Subject: [PATCH 03/23] Making 'make build' not cd into the cmd/proxy directory (#1497) Fixes https://github.com/gomods/athens/issues/1495 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index babc4cae..45e13717 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ endif .PHONY: build build: ## build the athens proxy - cd cmd/proxy && go build + go build -o ./cmd/proxy/proxy ./cmd/proxy .PHONY: build-ver build-ver: ## build the athens proxy with version number From 0b649557bb4ba707f0ed2f769471030a6715dffc Mon Sep 17 00:00:00 2001 From: Marwan Sulaiman Date: Thu, 9 Jan 2020 18:15:37 -0500 Subject: [PATCH 04/23] stasher: log when a stash event occurs (#1518) * stasher: log when a stash event occurs * rm space --- pkg/stash/stasher.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/stash/stasher.go b/pkg/stash/stasher.go index 3937b546..7dbb8607 100644 --- a/pkg/stash/stasher.go +++ b/pkg/stash/stasher.go @@ -5,6 +5,7 @@ import ( "time" "github.com/gomods/athens/pkg/errors" + "github.com/gomods/athens/pkg/log" "github.com/gomods/athens/pkg/module" "github.com/gomods/athens/pkg/observ" "github.com/gomods/athens/pkg/storage" @@ -44,6 +45,7 @@ func (s *stasher) Stash(ctx context.Context, mod, ver string) (string, error) { const op errors.Op = "stasher.Stash" _, span := observ.StartSpan(ctx, op.String()) defer span.End() + log.EntryFromContext(ctx).Debugf("saving %s@%s to storage...", mod, ver) // create a new context that ditches whatever deadline the caller passed // but keep the tracing info so that we can properly trace the whole thing. From 519a69adf997b6a1f6d34c5b9ff97601c6e9d3c7 Mon Sep 17 00:00:00 2001 From: Marwan Sulaiman Date: Fri, 10 Jan 2020 11:18:38 -0500 Subject: [PATCH 05/23] go mod tidy (#1519) --- go.sum | 64 +++++++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 57 insertions(+), 7 deletions(-) diff --git a/go.sum b/go.sum index 0f57637b..1a3d52ce 100644 --- a/go.sum +++ b/go.sum @@ -11,15 +11,21 @@ github.com/Azure/azure-pipeline-go v0.2.1 h1:OLBdZJ3yvOn2MezlWvbrBMTEUQC72zAftRZ github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= github.com/Azure/azure-storage-blob-go v0.7.0 h1:MuueVOYkufCxJw5YZzF842DY2MBsp+hLuh2apKY0mck= github.com/Azure/azure-storage-blob-go v0.7.0/go.mod h1:f9YQKtsG1nMisotuTPpO0tjNuEjKRYAcJU8/ydDI++4= +github.com/Azure/go-autorest/autorest v0.9.0 h1:MRvx8gncNaXJqOoLmhNjUAKh33JJF8LyxPhomEtOsjs= github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= +github.com/Azure/go-autorest/autorest/adal v0.6.0 h1:UCTq22yE3RPgbU/8u4scfnnzuCW6pwQ9n+uBtV78ouo= github.com/Azure/go-autorest/autorest/adal v0.6.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc= github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= +github.com/Azure/go-autorest/autorest/date v0.2.0 h1:yW+Zlqf26583pE43KhfnhFcdmSWlm5Ew6bxipnr/tbM= github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g= github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.3.0 h1:qJumjCaCudz+OcqE9/XtEPfvtOjOmKaui4EOpFI6zZc= github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= +github.com/Azure/go-autorest/logger v0.1.0 h1:ruG4BSDXONFRrZZJ2GUXDiUyVpayPmb1GnWeHDdaNKY= github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= +github.com/Azure/go-autorest/tracing v0.5.0 h1:TRn4WjSnkcSy5AEG3pnbtFSwNtwzjr4VYyQflFE619k= github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -49,7 +55,9 @@ github.com/bsm/go-vlq v0.0.0-20150828105119-ec6e8d4f5f4e/go.mod h1:N+BjUcTjSxc2m github.com/bsm/redis-lock v8.0.0+incompatible h1:QgB0J2pNG8hUfndTIvpPh38F5XsUTTvO7x8Sls++9Mk= github.com/bsm/redis-lock v8.0.0+incompatible/go.mod h1:8dGkQ5GimBCahwF2R67tqGCJbyDZSp0gzO7wq3pDrik= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/codegangsta/negroni v1.0.0 h1:+aYywywx4bnKXWvoWtRfJ91vC59NbEhEY03sZjQhbVY= github.com/codegangsta/negroni v1.0.0/go.mod h1:v0y3T5G7Y1UlFfyxFn/QLRU4a2EuNau2iZY63YTKWo0= +github.com/coreos/go-semver v0.2.0 h1:3Jm3tLmsgAYcjC+4Up7hJrFBPr+n7rAqYeSw/SZazuY= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7 h1:u9SHYsPQNyt5tgDm3YN7+9dYrpK96E5wFilTFWIDZOM= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= @@ -58,14 +66,19 @@ github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfc github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-ini/ini v1.25.4 h1:Mujh4R/dH6YL8bxuISne3xX2+qcQ9p0IxKAP6ExWoUo= github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= @@ -79,6 +92,7 @@ github.com/go-redis/redis v6.15.2+incompatible h1:9SpNVG76gr6InJGxoZ6IuuxaCOQwDA github.com/go-redis/redis v6.15.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-test/deep v1.0.1 h1:UQhStjbkDClarlmv0am7OXXO4/GaPdCGiUiMTvi28sg= github.com/go-test/deep v1.0.1/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/gobuffalo/envy v1.6.7 h1:XMZGuFqTupAXhZTriQ+qO38QvNOSU/0rl3hEPCFci/4= github.com/gobuffalo/envy v1.6.7/go.mod h1:N+GkhhZ/93bGZc6ZKhJLP6+m+tCNPKwgSpH9kaifseQ= @@ -86,8 +100,11 @@ github.com/gobuffalo/httptest v1.0.4 h1:P0uKaPEjti1bbJmuBILE3QQ7iU1cS7oIkxVba5Hb github.com/gobuffalo/httptest v1.0.4/go.mod h1:7T1IbSrg60ankme0aDLVnEY0h056g9M1/ZvpVThtB7E= github.com/gogo/protobuf v1.0.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.0 h1:xU6/SpYbvkNYiptHJYEDRseDLvYE7wSqhYYNy0QSUzI= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 h1:LbsanbbD6LieFkXbj9YNNBupiGHJgFeLpO0j0Fza1h8= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -97,25 +114,35 @@ github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a h1:ZJu5NB1Bk5ms4vw0Xu4i+jD32SE9jQXyfnOvwhHqlT0= github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go v2.0.0+incompatible h1:j0GKcs05QVmm7yesiZq2+9cxHkNK9YM6zKx4D2qucQU= github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.6.2 h1:Pgr17XVTNXAk3q/r4CpKzC5xBM/qW1uVLV+IhRZpIIk= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= +github.com/gorilla/sessions v1.1.3 h1:uXoZdcdA5XdXF3QzuSlheVRUvjl+1rKY7zBXL68L9RU= github.com/gorilla/sessions v1.1.3/go.mod h1:8KCfur6+4Mqcc6S0FEfKuN15Vl5MgXW92AE8ovaJD0w= +github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c h1:Lh2aW+HnU2Nbe1gqD9SOJLJxW1jBMmQOktN2acDyJk8= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 h1:Iju5GlWwrvL6UBg4zJJt3btmonfrMlCDdsejg4CZE7c= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.4.1 h1:pX7cnDwSSmG0dR9yNjCQSSpmsJOqFdT7SzVp5Yl9uVw= github.com/grpc-ecosystem/grpc-gateway v1.4.1/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= github.com/hashicorp/errwrap v0.0.0-20180715044906-d6c0cd880357/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= @@ -128,44 +155,47 @@ github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl2 v0.0.0-20190503213020-640445e16309 h1:VBvyXC+b6Ix/MXMGIrOHjq+Ew1IRP52EzoQXf1KpwZo= github.com/hashicorp/hcl2 v0.0.0-20190503213020-640445e16309/go.mod h1:4oI94iqF3GB10QScn46WqbG0kgTUpha97SAzzg2+2ec= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8 h1:12VvqtR6Aowv3l/EQUlocDHW2Cp4G9WJVH7uyH8QFJE= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kelseyhightower/envconfig v1.3.0 h1:IvRS4f2VcIQy6j4ORGIf9145T/AsUB+oY8LyvN8BXNM= github.com/kelseyhightower/envconfig v1.3.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.0.0/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/markbates/hmax v1.0.0 h1:yo2N0gBoCnUMKhV/VRLHomT6Y9wUm+oQQENuWJqCdlM= github.com/markbates/hmax v1.0.0/go.mod h1:cOkR9dktiESxIMu+65oc/r/bdY4bE8zZw3OLhLx0X2c= +github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149 h1:HfxbT6/JcvIljmERptWhwa8XzP7H3T+Z2N26gTsaDaA= github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= +github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/matttproud/golang_protobuf_extensions v1.0.0/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/minio/minio-go/v6 v6.0.36 h1:JnMQ3zP0fDzjUtTBVvUbO/0JotET2SuaRoa5b+l9Qcs= -github.com/minio/minio-go/v6 v6.0.36/go.mod h1:moIK68hwsWbeT999vT68nHNu2Kmq8Esjlucw02WpwME= -github.com/minio/minio-go/v6 v6.0.37 h1:rqot4cO9+mLpf56q+yumA0xZlncbkFpqa4A8jw1Y2XE= -github.com/minio/minio-go/v6 v6.0.37/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg= -github.com/minio/minio-go/v6 v6.0.38 h1:zd3yagckaBVAMJT+HsbpURx9ndqYQp/N/udc1UVS72E= -github.com/minio/minio-go/v6 v6.0.38/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg= github.com/minio/minio-go/v6 v6.0.43 h1:D7c6Kx0ZB5U8EXJ6SQVOqPzapaLK/qpxQIktCnPHp/o= github.com/minio/minio-go/v6 v6.0.43/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg= -github.com/minio/sha256-simd v0.1.0/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= @@ -175,13 +205,16 @@ github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7/go.mod h1:ZX github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -204,18 +237,23 @@ github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1 h1:/K3IL0Z1quvmJ7X0A1AwNEK7CRkVK3YwfOU/QAL4WGg= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.2 h1:Fy0orTDgHdbnzHcsOgfCN4LtHf0ec3wwtiwJqwvf3Gc= github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -224,10 +262,13 @@ github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/technosophos/moniker v0.0.0-20180509230615-a5dbd03a2245 h1:DNVk+NIkGS0RbLkjQOLCJb/759yfCysThkMbl7EXxyY= github.com/technosophos/moniker v0.0.0-20180509230615-a5dbd03a2245/go.mod h1:O1c8HleITsZqzNZDjSNzirUGsMT0oGu9LhHKoJrqO+A= +github.com/tidwall/pretty v0.0.0-20180105212114-65a9db5fad51 h1:BP2bjP495BBPaBcS5rmqviTfrOkN5rO5ceKAMRZCRFc= github.com/tidwall/pretty v0.0.0-20180105212114-65a9db5fad51/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tinylib/msgp v1.0.2 h1:DfdQrzQa7Yh2es9SuLkixqxuXS2SxsdYn0KbdrOGWD8= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= +github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8 h1:ndzgwNDnKIqyCvHTXaCqh9KlOWKvBry6nuXMJmonVsE= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/ugorji/go v1.1.1 h1:gmervu+jDMvXTbcHQ0pd2wee85nEoE0BsVyEuzkfK8w= github.com/ugorji/go v1.1.1/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ= github.com/unrolled/secure v0.0.0-20181221173256-0d6b5bb13069 h1:RKeYksgIwGE8zFJTvXI1WWx09QPrGyaVFMy0vpU7j/o= github.com/unrolled/secure v0.0.0-20181221173256-0d6b5bb13069/go.mod h1:mnPT77IAdsi/kV7+Es7y+pXALeV3h7G6dQF6mNYjcLA= @@ -237,9 +278,11 @@ github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0= github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/zclconf/go-cty v0.0.0-20190426224007-b18a157db9e2 h1:Ai1LhlYNEqE39zGU07qHDNJ41iZVPZfZr1dSCoXrp1w= github.com/zclconf/go-cty v0.0.0-20190426224007-b18a157db9e2/go.mod h1:xnAOWiHeOqg2nWS62VtQ7pbOu17FtxJNW8RLEih+O3s= +go.etcd.io/bbolt v1.3.2 h1:Z/90sZLPOeCy2PwprqkFa25PdkusRzaj9P8zm/KNyvk= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd v0.0.0-20190215181705-784daa04988c h1:pkiZ418C7QN/HIps1lDF1+lzZhdgMpvFN4kDcxrYhD0= go.etcd.io/etcd v0.0.0-20190215181705-784daa04988c/go.mod h1:RutfZdQAP913VY0GI8/Mjwf50+IZ7Mpg2zt3SDs17/g= @@ -307,6 +350,7 @@ golang.org/x/sys v0.0.0-20190502175342-a43fa875dd82/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 h1:+DCIGbF/swA92ohVg0//6X2IVY3KZs6p9mix0ziNYJM= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -319,6 +363,7 @@ google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMt google.golang.org/api v0.3.2 h1:iTp+3yyl/KOtxa/d1/JUE0GGSoR6FuW5udver22iwpw= google.golang.org/api v0.3.2/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180608181217-32ee49c4dd80/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= @@ -335,17 +380,22 @@ gopkg.in/DataDog/dd-trace-go.v1 v1.10.0/go.mod h1:DVp8HmDh8PuTu2Z0fVVlBsyWaC++fz gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= +gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= gopkg.in/go-playground/validator.v9 v9.20.2 h1:6AVDyt8bk0FDiSYSeWivUfzqEjHyVSCMRkpTr6ZCIgk= gopkg.in/go-playground/validator.v9 v9.20.2/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= gopkg.in/ini.v1 v1.42.0 h1:7N3gPTt50s8GuLortA00n8AqRTk75qOP98+mTPpgzRk= gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From 951089317702da717603f409aa4ea202167fd2b6 Mon Sep 17 00:00:00 2001 From: Marwan Sulaiman Date: Fri, 10 Jan 2020 11:42:29 -0500 Subject: [PATCH 06/23] ci/cd: create liveness probe for go mod download (#1520) Co-authored-by: Aaron Schlesinger <70865+arschles@users.noreply.github.com> --- .drone.yml | 5 +-- scripts/liveness_probe/go.mod | 3 ++ scripts/liveness_probe/main.go | 59 ++++++++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 2 deletions(-) create mode 100644 scripts/liveness_probe/go.mod create mode 100644 scripts/liveness_probe/main.go diff --git a/.drone.yml b/.drone.yml index 149c5208..9beb1aaf 100644 --- a/.drone.yml +++ b/.drone.yml @@ -7,7 +7,9 @@ steps: image: golang:1.13 commands: # wait for services to be ready. - - sleep 10 + - cd scripts/liveness_probe + - go run . + - cd ../.. - go mod download - go mod vendor # this is for when the Dockerfile gets built environment: @@ -17,7 +19,6 @@ steps: - name: cache path: /go - - name: build image: golang:1.13 commands: diff --git a/scripts/liveness_probe/go.mod b/scripts/liveness_probe/go.mod new file mode 100644 index 00000000..690cee89 --- /dev/null +++ b/scripts/liveness_probe/go.mod @@ -0,0 +1,3 @@ +module liveness_probe + +go 1.13 diff --git a/scripts/liveness_probe/main.go b/scripts/liveness_probe/main.go new file mode 100644 index 00000000..ff57a2ff --- /dev/null +++ b/scripts/liveness_probe/main.go @@ -0,0 +1,59 @@ +// Package main is a simple script for our CI/CD workflow +// that ensures our sidecar proxy is running before proceeding +package main + +import ( + "context" + "errors" + "fmt" + "net" + "net/http" + "os" + "time" +) + +var goproxy = os.Getenv("GOPROXY") + +func main() { + timeout := time.After(time.Minute) + for { + select { + case <-timeout: + fmt.Println("liveness probe timed out") + os.Exit(1) + default: + } + isLive, err := probe() + if err != nil { + shouldPrintErr := true + // connection-refused errors are expected, don't print them + var opErr *net.OpError + if errors.As(err, &opErr) && opErr.Op == "read" { + shouldPrintErr = false + } + if shouldPrintErr { + fmt.Println(err) + } + } + if isLive { + fmt.Println("proxy is live") + return + } + time.Sleep(time.Second) + } +} + +func probe() (bool, error) { + req, err := http.NewRequest(http.MethodGet, goproxy, nil) + if err != nil { + return false, err + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + req = req.WithContext(ctx) + resp, err := http.DefaultClient.Do(req) + if err != nil { + return false, err + } + return resp.StatusCode == http.StatusOK, nil +} From 3edd2428482b8be99696cd55755bd3d9742a00f6 Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Mon, 13 Jan 2020 13:28:59 -0800 Subject: [PATCH 07/23] Revamping the download mode file (#1508) Also doing a few things in the process: - Deprecating the documentation that references the filter file - Changing the order of the configuration documentation to put the deprecated documents at the bottom of the config section - Adding a note to the disk storage documentation about pre-filling the disk cache Fixes https://github.com/gomods/athens/issues/1501 --- docs/content/configuration/authentication.md | 4 +- docs/content/configuration/download.md | 81 +++++++++++++------ docs/content/configuration/filter.md | 8 +- .../configuration/prefill-disk-cache.md | 4 +- docs/content/configuration/storage.md | 34 +++++--- docs/content/configuration/sumdb.md | 6 +- docs/content/configuration/upstream.md | 10 ++- docs/content/design/_index.md | 2 +- 8 files changed, 100 insertions(+), 49 deletions(-) diff --git a/docs/content/configuration/authentication.md b/docs/content/configuration/authentication.md index 2dd84f72..9ede38a9 100644 --- a/docs/content/configuration/authentication.md +++ b/docs/content/configuration/authentication.md @@ -1,7 +1,7 @@ --- -title: Configuring Authentication +title: Authentication to private repositories description: Configuring Authentication on Athens -weight: 1 +weight: 2 --- ## Authentication diff --git a/docs/content/configuration/download.md b/docs/content/configuration/download.md index 775469ef..6bb28f21 100644 --- a/docs/content/configuration/download.md +++ b/docs/content/configuration/download.md @@ -1,28 +1,42 @@ --- -title: Download Mode +title: The download mode file description: What to do when a module is not in storage -weight: 2 +weight: 1 --- -Athens accepts an HCL formatted Download File that has instructions for how it should behave when a module@version isn't found in its storage. +Athens accepts an [HCL](https://github.com/hashicorp/hcl) formatted file that has instructions for how it should behave when a module@version isn't found in its storage. This functionality gives Athens the flexibility configure Athens to fit your organization's needs. The most popular uses of this download file are: -You configure this download file by setting the `ATHENS_DOWNLOAD_MODE` environment variable in one of two ways: +- Configure Athens to never download or serve a module or group of modules +- Redirect to a different module proxy for a module or group of modules -1. Set its value to `file:$FILE_PATH`, where `FILE_PATH` is the path to the HCL file -1. Set its value to `custom$BASE_64` where `BASE_64` is the base64 encoded HCL file +This document will outline how to use this file - called the download mode file - to accomplish these tasks and more. -## What should Athens do when a module@version is not found in storage? +>Please see the "Use cases" section below for more details on how to enable these behaviors and more. -Say a client sends an HTTP request with the path `/github.com/pkg/errors/@v/v0.8.1` and Athens -does not have this module in storage. Athens will look at the Download File for one of the following Modes: +## Configuration -1. **`sync`**: Synchronously download the module from VCS via `go mod download`, persist it to the Athens storage, and serve it back to the user immediately. Note that this is the default behavior. +First, once you've created your download mode file, you tell Athens to use it by setting the `DownloadMode` configuration parameter in the `config.toml` file, or setting the `ATHENS_DOWNLOAD_MODE` environment variable. You can set this configuration value to one of two values to tell Athens to use your file: + +1. Set its value to `file:$FILE_PATH`, where `$FILE_PATH` is the path to the HCL file +2. Set its value to `custom$BASE_64` where `$BASE_64` is the base64 encoded HCL file + +>Instead of one of the above two values, you can set this configuration to `sync`, `async`, `none`, `redirect`, or `async_redirect`. If you do, the download mode will be set globally rather than for specific sub-groups of modules. See below for what each of these values mean. + +## Download mode keywords + +If Athens receives a request for the module `github.com/pkg/errors` at version `v0.8.1`, and it doesn't have that module and version in its storage, it will consult the download mode file for specific instructions on what action to take: + +1. **`sync`**: Synchronously download the module from VCS via `go mod download`, persist it to the Athens storage, and serve it back to the user immediately. Note that this is the default behavior. 2. **`async`**: Return a 404 to the client, and asynchronously download and persist the module@version to storage. -3. **`none`**: Return a 404 and do nothing. +3. **`none`**: Return a 404 and do nothing. 4. **`redirect`**: Redirect to an upstream proxy (such as proxy.golang.org) and do nothing after. -5. **`async_redirect`**: Redirect to an upstream proxy (such as proxy.golang.org) and asynchronously download and persist the module@version to storage. +5. **`async_redirect`**: Redirect to an upstream proxy (such as proxy.golang.org) and asynchronously download and persist the module@version to storage. -Furthermore, the Download File can describe any of the above behavior for different modules and module patterns alike using [path.Match](https://golang.org/pkg/path/#Match). Take a look at the following example: +Athens expects these keywords to be used in conjunction with module patterns (`github.com/pkg/*`, for example). You combine the keyword and the pattern to specify behavior for a specific group of modules. + +>Athens uses the Go [path.Match](https://golang.org/pkg/path/#Match) function to parse module patterns. + +Below is an example download mode file. ```javascript downloadURL = "https://proxy.golang.org" @@ -43,25 +57,46 @@ download "github.com/pkg/*" { } ``` -The first two lines describe the behavior and the destination of all packages: redirect to `https://proxy.golang.org` and asynchronously persist the module to storage. +The first two lines describe the _default_ behavior for all modules. This behavior is overridden for select module groups below. In this case, the default behavior is: -The following two blocks describe what to do if the requested module matches the given pattern: +- Immediatley redirect all requests to `https://proxy.golang.org` +- In the background, download the module from the version control system (VCS) and store it -Any module that matches "github.com/gomods/*" such as "github.com/gomods/athens", will be synchronously fetched, stored, and returned to the user. +The rest of the file contains `download` blocks. These override the default behavior for specific groups of modules. -Any module that matches "golang.org/x/*" such as "golang.org/x/text" will just return a 404. Note that this behavior allows the client to set GOPROXY to multiple comma separated proxies so that the Go command can move into the second argument. +The first block specifies that any module matching `github.com/gomods/*` (such as `github.com/gomods/athens`) will be downloaded from GitHub, stored, and then returned to the user. -Any module that matches "github.com/pkg/*" such as "github.com/pkg/errors" will be redirected to https://gocenter.io (and not proxy.golang.org) and will also never persist the module to the Athens storage. +The second block specifies that any module matching `golang.org/x/*` (such as `golang.org/x/text`) will always return a HTTP 404 response code. This behavior ensures that Athens will _never_ store or serve any module names starting with `golang.org/x`. +If a user has their `GOPROXY` environment variable set with a comma separated list, their `go` command line tool will always try the option next in the list. For example, if a user has their `GOPROXY` environment variable set to `https://athens.azurefd.net,direct`, and then runs `go get golang.org/x/text`, they will still download `golang.org/x/text` to their machine. The module just won't come from Athens. + +The last block specifies that any module matching `github.com/pkg/*` (such as `github.com/pkg/errors`) will always redirect the `go` tool to https://gocenter.io. In this case, Athens will never persist the given module to its storage. ## Use cases -So why would you want to use the Download File to configure the behavior above? Here are a few use cases where it might make sense for you to do so: +The download mode file is versatile and allows you to configure Athens in a large variety of different ways. Below are some of the mode common. -**Limited storage:** +## Blocking certain modules -If you have limited storage, then it might be a good idea to only persist some moduels (such as private ones) and then redirect to a public proxy for everything else. +If you're running Athens to serve a team of Go developers, it might be useful to ensure that the team doesn't use a specific group or groups of modules (for example, because of licensing or security issues). -**Limited resources:** +In this case, you would write this in your file: -If you are running Athens with low memory/cpu, then you can redirect all public modules to proxy.golang.org but asynchronously fetch them so that the client does not timeout. At the same time, you can return a 404 for private modules through the `none` mode and let the client (the Go command) fetch private modules directly through `GOPROXY=,direct` +```hcl +download "bad/module/repo/*" { + mode = "none" +} +``` + +### Preventing storage overflow + +If you are running Athens using a [storage backend](./storage) that has limited space, you may want to prevent Athens from storing certain groups of modules that take up a lot of space. To avoid exhausting Athens storage, while still ensuring that the users of your Athens server still get access to the modules you can't store, you would use a `redirect` directive, as shown below: + +```hcl +download "very/large/*" { + mode = "redirect" + url = "https://reliable.proxy.com" +} +``` + +>If you use the `redirect` mode, make sure that you specify a `url` value that points to a reliable proxy. diff --git a/docs/content/configuration/filter.md b/docs/content/configuration/filter.md index f0708a97..ec279235 100644 --- a/docs/content/configuration/filter.md +++ b/docs/content/configuration/filter.md @@ -1,9 +1,11 @@ --- -title: Filtering modules +title: Filtering modules (deprecated) description: Configuring modules that are stored on the proxy -weight: 1 +weight: 6 --- +>Note: the filter file that this page documents is deprecated. Please instead see ["Filtering with the download mode file"](./download) for updated instructions on how to filter modules in Athens. + The proxy supports the following three use cases 1. Fetches a module directly from the source (upstream proxy) @@ -90,4 +92,4 @@ The currently supported modifiers are * `<1.2.3` will enable all versions lower than 1.2.3 (e.g. 1.2.2, 1.0.0 and 0.58.9) * Formally, `x.y.z` where `x <= 1`, `y < = 2` and `z < 3` -This kind of modifiers will work only if a three parts semantic version is specified. For example, `~4.5.6` will work while `~4.5` won't. \ No newline at end of file +This kind of modifiers will work only if a three parts semantic version is specified. For example, `~4.5.6` will work while `~4.5` won't. diff --git a/docs/content/configuration/prefill-disk-cache.md b/docs/content/configuration/prefill-disk-cache.md index eea17aff..db39925d 100644 --- a/docs/content/configuration/prefill-disk-cache.md +++ b/docs/content/configuration/prefill-disk-cache.md @@ -1,7 +1,7 @@ --- -title: Pre-filling the Disk Cache +title: Pre-filling disk storage description: How to pre-fill the disk cache -weight: 4 +weight: 5 --- One of the popular features of Athens is that it can be run completely cut off from the internet. In this case, though, it can't reach out to an upstream (e.g. a VCS or another module proxy) to fetch modules that it doesn't have in storage. So, we need to manually fill up the disk partition that Athens uses with the dependencies that we need. diff --git a/docs/content/configuration/storage.md b/docs/content/configuration/storage.md index b60b109a..383151f7 100644 --- a/docs/content/configuration/storage.md +++ b/docs/content/configuration/storage.md @@ -1,22 +1,32 @@ --- title: Configuring Storage description: Configuring Storage in Athens +weight: 3 --- ## Storage The Athens proxy supports many storage types: -1. [Memory](#memory) -1. [Disk](#disk) -1. [Mongo](#mongo) -1. [Google Cloud Storage](#google-cloud-storage) -1. [AWS S3](#aws-s3) -1. [Minio](#minio) - 1. [DigitalOcean Spaces](#digitalocean-spaces) - 1. [Alibaba OSS](#alibaba-oss) - 1. and other S3 / Minio compatible interfaces -1. [Azure Blob Storage](#azure-blob-storage) +- [Storage](#storage) +- [Memory](#memory) + - [Configuration:](#configuration) +- [Disk](#disk) + - [Configuration:](#configuration-1) +- [Mongo](#mongo) + - [Configuration:](#configuration-2) +- [Google Cloud Storage](#google-cloud-storage) + - [Configuration:](#configuration-3) +- [AWS S3](#aws-s3) + - [Configuration:](#configuration-4) +- [Minio](#minio) + - [Configuration:](#configuration-5) + - [DigitalOcean Spaces](#digitalocean-spaces) + - [Configuration:](#configuration-6) + - [Alibaba OSS](#alibaba-oss) + - [Configuration:](#configuration-7) +- [Azure Blob Storage](#azure-blob-storage) + - [Configuration:](#configuration-8) All of them can be configured using `config.toml` file. You need to set a valid driver in `StorageType` value or you can set it in environment variable `ATHENS_STORAGE_TYPE` on your server. Also for most of the drivers you need to provide additional configuration data which will be described below. @@ -37,6 +47,8 @@ This storage doesn't need any specific configuration and it's also used by defau Disk storage allows modules to be stored on a file system. The location on disk where modules will be stored can be configured. +>You can pre-fill disk-based storage to enable Athens deployments that have no access to the internet. See [here](./prefill-disk-cache) for instructions on how to do that. + ##### Configuration: # StorageType sets the type of storage backend the proxy will use. @@ -318,4 +330,4 @@ It assumes that you already have the following: # Name of container in the blob storage # Env override: ATHENS_AZURE_CONTAINER_NAME - ContainerName = "MY_AZURE_BLOB_CONTAINER_NAME" \ No newline at end of file + ContainerName = "MY_AZURE_BLOB_CONTAINER_NAME" diff --git a/docs/content/configuration/sumdb.md b/docs/content/configuration/sumdb.md index a7d91c80..1c72ca23 100644 --- a/docs/content/configuration/sumdb.md +++ b/docs/content/configuration/sumdb.md @@ -1,7 +1,7 @@ --- -title: Checksum DB -description: Proxying A Checksum DB API -weight: 2 +title: Proxying a checksum database API +description: How to configure Athens to proxy a checksum database API, and why you might want to. +weight: 4 --- ## Proxying A Checksum DB diff --git a/docs/content/configuration/upstream.md b/docs/content/configuration/upstream.md index 5b7b18f3..c0e91650 100644 --- a/docs/content/configuration/upstream.md +++ b/docs/content/configuration/upstream.md @@ -1,10 +1,12 @@ --- -title: Configuring Upstream Proxy to use an Upstream Go Modules Repository +title: Using an upstream Go modules repository (deprecated) description: How to Configure Athens to Fetch Missing Modules From an Upstream Module Repository Like GoCenter, or Another Athens Server -weight: 1 +weight: 7 --- -By default, Athens fetches module code from an upstream VCS like github.com, but this can be configured to use a Go modules repository like GoCenter or another Athens Server. +>Note: the filter file that this page documents is deprecated. Please instead see ["Filtering with the download mode file"](./download) for updated instructions on how to set upstream repositories in Athens. + +By default, Athens fetches module code from an upstream version control system (VCS) like github.com, but this can be configured to use a Go modules repository like GoCenter or another Athens Server. 1. Create a filter file (e.g ```/usr/local/lib/FilterForGoCenter```) with letter `D` (stands for "direct access") in first line. For more details, please refer to documentation on - [Filtering Modules](/configuration/filter) @@ -12,7 +14,7 @@ By default, Athens fetches module code from an upstream VCS like github.com, but # FilterFile for fetching modules directly from upstream D ``` -1. If you are not using a config file, create a new config file (based on the sample config.dev.toml) and edit values to match your environment). +2. If you are not using a config file, create a new config file (based on the sample config.dev.toml) and edit values to match your environment). Additionally in the current or new config file, set the following parameters as suggested: ``` diff --git a/docs/content/design/_index.md b/docs/content/design/_index.md index 643b1756..aa689556 100644 --- a/docs/content/design/_index.md +++ b/docs/content/design/_index.md @@ -1,7 +1,7 @@ --- title: "The Design of Athens" date: 2018-09-20T15:37:49-07:00 -weight: 5 +weight: 3 --- This section of the documentation details the design of Athens. You can read the code and ask plenty of questions (which we're always happy to answer!), but we want to take some time here to give you a head start by describing how Athens is designed in words and diagrams, rather than code. From 15733395eb9d7bf1e156e0e702173d8cb1198f67 Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Mon, 13 Jan 2020 13:53:47 -0800 Subject: [PATCH 08/23] Fixing filter link (#1524) --- docs/content/configuration/filter.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/configuration/filter.md b/docs/content/configuration/filter.md index ec279235..45678448 100644 --- a/docs/content/configuration/filter.md +++ b/docs/content/configuration/filter.md @@ -4,7 +4,7 @@ description: Configuring modules that are stored on the proxy weight: 6 --- ->Note: the filter file that this page documents is deprecated. Please instead see ["Filtering with the download mode file"](./download) for updated instructions on how to filter modules in Athens. +>Note: the filter file that this page documents is deprecated. Please instead see ["Filtering with the download mode file"](/configuration/download) for updated instructions on how to filter modules in Athens. The proxy supports the following three use cases From 24f10b8a254edc3df554955029da9748634d1abd Mon Sep 17 00:00:00 2001 From: Ted Wexler Date: Sun, 19 Jan 2020 15:12:48 -0500 Subject: [PATCH 09/23] Adds detailed docker usage instructions (#1527) * Adds detailed docker usage instructions * Fixes a couple of typos * Add link to release image description * Describe the canary tag --- docs/content/install/using-docker.md | 70 ++++++++++++++++++++++++++++ docs/content/walkthrough.md | 14 ++---- 2 files changed, 73 insertions(+), 11 deletions(-) create mode 100644 docs/content/install/using-docker.md diff --git a/docs/content/install/using-docker.md b/docs/content/install/using-docker.md new file mode 100644 index 00000000..8c698a78 --- /dev/null +++ b/docs/content/install/using-docker.md @@ -0,0 +1,70 @@ +--- +title: Using the Athens Docker images +description: Information about Athens' Docker images +weight: 1 +--- + +Whether setting Athens up using [Kubernetes](./install-on-kubernetes) or using the [Walkthrough](/Walkthrough), you'll most likely be using one of the images that the Athens project produces. This document details what images are available, and has a recap from the Walkthrough of how to use them on their own. + +--- + +## Available Docker images + +The Athens project produces two docker images, available via [Docker Hub](https://hub.docker.com/) + +1. A release version as [`gomods/athens`](https://hub.docker.com/gomods/athens), each tag corresponds with an Athens [release](https://github.com/gomods/athens/releases), e.g. `v0.7.1`. Additionally, a `canary` tag is available and tracks each commit to `master` +2. A tip version, as [`gomods/athens-dev`](https://hub.docker.com/r/gomods/athens-dev), tagged with every commit to `master`, e.g. `1573339` + +For a detailed tags list, check each image's Docker Hub + +## Running Athens as a Docker image + +This is a quick recap of the [Walkthrough](/walkthrough) + +### Using the `docker` cli + +In order to run the Athens Proxy using docker, we need first to create a directory that will store the persitant modules. +In the example below, the new directory is named `athens-storage` and is located in our userspace (i.e. `$HOME`). +Then we need to set the `ATHENS_STORAGE_TYPE` and `ATHENS_DISK_STORAGE_ROOT` environment variables when we run the Docker container. + +**Bash** +```bash +export ATHENS_STORAGE=$HOME/athens-storage +mkdir -p $ATHENS_STORAGE +docker run -d -v $ATHENS_STORAGE:/var/lib/athens \ + -e ATHENS_DISK_STORAGE_ROOT=/var/lib/athens \ + -e ATHENS_STORAGE_TYPE=disk \ + --name athens-proxy \ + --restart always \ + -p 3000:3000 \ + gomods/athens:latest +``` + +**PowerShell** +```PowerShell +$env:ATHENS_STORAGE = "$(Join-Path $HOME athens-storage)" +md -Path $env:ATHENS_STORAGE +docker run -d -v "$($env:ATHENS_STORAGE):/var/lib/athens" ` + -e ATHENS_DISK_STORAGE_ROOT=/var/lib/athens ` + -e ATHENS_STORAGE_TYPE=disk ` + --name athens-proxy ` + --restart always ` + -p 3000:3000 ` + gomods/athens:latest +``` + + +## Troubleshooting Athens in Docker + +### `init` issues + +The Athens docker image uses [tini](https://github.com/krallin/tini) so that defunct processes get reaped. +Docker 1.13 and greater includes `tini` and lets you enable it by passing the `--init` flag to `docker run` or by configuring the docker deamon with `"init": true`. When running in this mode. you may see a warning like this: + +```console +[WARN tini (6)] Tini is not running as PID 1 and isn't registered as a child subreaper. + Zombie processes will not be re-parented to Tini, so zombie reaping won't work. + To fix the problem, use the -s option or set the environment variable TINI_SUBREAPER to register Tini as a child subreaper, or run Tini as PID 1. +``` +This is the "Athens-tini" complaining that it's not running as PID 1. +There is no harm in that, since the zombie processes will be reaped by the `tini` included in Docker. diff --git a/docs/content/walkthrough.md b/docs/content/walkthrough.md index b51dc100..1afdd07a 100644 --- a/docs/content/walkthrough.md +++ b/docs/content/walkthrough.md @@ -86,6 +86,9 @@ for a short period of time, as you will quickly run out of memory and the storag doesn't persist between restarts. ### With Docker + +For more details on running Athens in docker, take a look at the [install documentation](/install/using-docker) + In order to run the Athens Proxy using docker, we need first to create a directory that will store the persitant modules. In the example below, the new directory is named `athens-storage` and is located in our userspace (i.e. `$HOME`). Then we need to set the `ATHENS_STORAGE_TYPE` and `ATHENS_DISK_STORAGE_ROOT` environment variables when we run the Docker container. @@ -116,17 +119,6 @@ docker run -d -v "$($env:ATHENS_STORAGE):/var/lib/athens" ` gomods/athens:latest ``` -Athens docker image uses [tini](https://github.com/krallin/tini) so that defunct processes get reaped. -Since Docker 1.13 and greater includes `tini` and lets you enable it by passing the `--init` flag to `docker run` or by configuring the docker deamon with `"init": true`, you may see a warning like this: - -```console -[WARN tini (6)] Tini is not running as PID 1 and isn't registered as a child subreaper. - Zombie processes will not be re-parented to Tini, so zombie reaping won't work. - To fix the problem, use the -s option or set the environment variable TINI_SUBREAPER to register Tini as a child subreaper, or run Tini as PID 1. -``` -This is the "Athens-tini" complaining that it's not running as PID 1. -There is no harm in that, since the zombie processes will be reaped by the `tini` included in Docker. - Next, you will need to enable the [Go Modules](https://github.com/golang/go/wiki/Modules) feature and configure Go to use the Athens proxy! From 1345d1f9a6d10eaa562267e2de05f76991573e33 Mon Sep 17 00:00:00 2001 From: Michael D Henderson Date: Sun, 19 Jan 2020 16:36:13 -0700 Subject: [PATCH 10/23] fix typo in Atlassian name (#1515) Co-authored-by: Rob j Loranger --- docs/content/configuration/authentication.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/configuration/authentication.md b/docs/content/configuration/authentication.md index 9ede38a9..646ad327 100644 --- a/docs/content/configuration/authentication.md +++ b/docs/content/configuration/authentication.md @@ -183,7 +183,7 @@ weight: 2 gomods/athens:latest ``` -## Altassian Bitbucket and SSH-secured git VCS's +## Atlassian Bitbucket and SSH-secured git VCS's This section was originally written to describe configuring the Athens git client to fetch specific Go imports over SSH instead of From 86da1590bcc49c416f92de8aa3d8e976e23c2072 Mon Sep 17 00:00:00 2001 From: Marwan Sulaiman Date: Sun, 19 Jan 2020 19:14:49 -0500 Subject: [PATCH 11/23] Dockerfile: ensure right perms for production setting (#1530) --- cmd/proxy/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/proxy/Dockerfile b/cmd/proxy/Dockerfile index b10c98cd..b922042f 100644 --- a/cmd/proxy/Dockerfile +++ b/cmd/proxy/Dockerfile @@ -24,6 +24,8 @@ COPY --from=builder /bin/athens-proxy /bin/athens-proxy COPY --from=builder /go/src/github.com/gomods/athens/config.dev.toml /config/config.toml COPY --from=builder /usr/local/go/bin/go /bin/go +RUN chmod 700 /config/config.toml + # Add tini, see https://github.com/gomods/athens/issues/1155 for details. RUN apk add --update bzr git git-lfs mercurial openssh-client subversion procps fossil tini && \ mkdir -p /usr/local/go From 90d213f0afd082a1ae07a30710aa7409b783680d Mon Sep 17 00:00:00 2001 From: Marwan Sulaiman Date: Sun, 19 Jan 2020 19:38:31 -0500 Subject: [PATCH 12/23] config: remove incorrect documentation (#1525) --- config.dev.toml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/config.dev.toml b/config.dev.toml index 3699ee3e..11c34c8d 100755 --- a/config.dev.toml +++ b/config.dev.toml @@ -307,10 +307,6 @@ SingleFlightType = "memory" # running Athens inside GCP, you will most # likely not need this as GCP figures out # internal authentication between products for you. - # - # NOTE: This config value is deprecated in favor of - # ServiceAccount above. Athens will check for it, - # but please do not rely on it being available forever. # # Env override: ATHENS_STORAGE_GCP_JSON_KEY JSONKey = "" From b456c5c189864a2496186d46d6b99f32e61f7371 Mon Sep 17 00:00:00 2001 From: Zhongpeng Lin Date: Sun, 19 Jan 2020 16:58:22 -0800 Subject: [PATCH 13/23] Escaping underscore (#1516) Co-authored-by: Rob j Loranger --- docs/content/design/from-vcs2user.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/design/from-vcs2user.md b/docs/content/design/from-vcs2user.md index b66e7334..764997c2 100644 --- a/docs/content/design/from-vcs2user.md +++ b/docs/content/design/from-vcs2user.md @@ -32,7 +32,7 @@ if err != nil { return err } ``` -_app_proxy.go_ +_app\_proxy.go_ When a request for a new module comes, the `Fetch` function is invoked. From f76ca00665944ca9ec868ebac20e527a6e6bdf32 Mon Sep 17 00:00:00 2001 From: Mark Dodgson Date: Mon, 20 Jan 2020 18:26:17 +0000 Subject: [PATCH 14/23] Display warning log message if validation fails (#1517) * Display a warning log message if the validation response fails with a valid reason * provide a reason and description for failed validation * added unknown to description * Allow a successful response to carry a message * tidy up * formatting * Simplified the message response * updated based on latest comments * no need to export struct * suggested method rename Co-authored-by: Marwan Sulaiman --- pkg/middleware/validation.go | 45 +++++++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 11 deletions(-) diff --git a/pkg/middleware/validation.go b/pkg/middleware/validation.go index 0e7514c3..feb6fc60 100644 --- a/pkg/middleware/validation.go +++ b/pkg/middleware/validation.go @@ -2,7 +2,9 @@ package middleware import ( "bytes" + "context" "encoding/json" + "io/ioutil" "net/http" "github.com/gomods/athens/pkg/errors" @@ -27,7 +29,7 @@ func NewValidationMiddleware(validatorHook string) mux.MiddlewareFunc { // i.e. list requests path is like /{module:.+}/@v/list with no version parameter version, _ := paths.GetVersion(r) if version != "" { - valid, err := validate(validatorHook, mod, version) + response, err := validate(validatorHook, mod, version) if err != nil { entry := log.EntryFromContext(r.Context()) entry.SystemErr(err) @@ -35,7 +37,9 @@ func NewValidationMiddleware(validatorHook string) mux.MiddlewareFunc { return } - if !valid { + maybeLogValidationReason(r.Context(), string(response.Message), mod, version) + + if !response.Valid { w.WriteHeader(http.StatusForbidden) return } @@ -45,31 +49,50 @@ func NewValidationMiddleware(validatorHook string) mux.MiddlewareFunc { } } +func maybeLogValidationReason(context context.Context, message string, mod string, version string) { + if len(message) > 0 { + entry := log.EntryFromContext(context) + entry.Warnf("error validating %s@%s %s", mod, version, message) + } +} + type validationParams struct { Module string Version string } -func validate(hook, mod, ver string) (bool, error) { +type validationResponse struct { + Valid bool + Message []byte +} + +func validate(hook, mod, ver string) (validationResponse, error) { const op errors.Op = "actions.validate" toVal := &validationParams{mod, ver} jsonVal, err := json.Marshal(toVal) if err != nil { - return false, errors.E(op, err) + return validationResponse{Valid: false}, errors.E(op, err) } resp, err := http.Post(hook, "application/json", bytes.NewBuffer(jsonVal)) if err != nil { - return false, errors.E(op, err) + return validationResponse{Valid: false}, errors.E(op, err) } - switch { - case resp.StatusCode == http.StatusOK: - return true, nil - case resp.StatusCode == http.StatusForbidden: - return false, nil + switch resp.StatusCode { + case http.StatusOK: + return validationResponseFromRequest(resp), nil + case http.StatusForbidden: + return validationResponseFromRequest(resp), nil default: - return false, errors.E(op, "Unexpected status code ", resp.StatusCode) + return validationResponse{Valid: false}, errors.E(op, "Unexpected status code ", resp.StatusCode) } } + +func validationResponseFromRequest(resp *http.Response) validationResponse { + defer resp.Body.Close() + + body, _ := ioutil.ReadAll(resp.Body) + return validationResponse{Valid: resp.StatusCode == http.StatusOK, Message: body} +} From 71067bb2ca444d37eb145913c271abecceaefd48 Mon Sep 17 00:00:00 2001 From: Luke Ross Date: Mon, 20 Jan 2020 22:37:22 +0000 Subject: [PATCH 15/23] Helm Chart - Allow setting extra annotations on the athens pod (#1523) * Allow setting annotations on Athens pods * Bump chart version Co-authored-by: Marwan Sulaiman --- charts/athens-proxy/Chart.yaml | 2 +- charts/athens-proxy/templates/deployment.yaml | 3 +++ charts/athens-proxy/values.yaml | 3 +++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/charts/athens-proxy/Chart.yaml b/charts/athens-proxy/Chart.yaml index 1365d6d5..4d9b9962 100644 --- a/charts/athens-proxy/Chart.yaml +++ b/charts/athens-proxy/Chart.yaml @@ -1,5 +1,5 @@ name: athens-proxy -version: 0.4.1 +version: 0.4.2 appVersion: 0.7.0 description: The proxy server for Go modules icon: https://raw.githubusercontent.com/gomods/athens/master/docs/static/banner.png diff --git a/charts/athens-proxy/templates/deployment.yaml b/charts/athens-proxy/templates/deployment.yaml index 8aeb109f..d3f931be 100644 --- a/charts/athens-proxy/templates/deployment.yaml +++ b/charts/athens-proxy/templates/deployment.yaml @@ -24,6 +24,9 @@ spec: checksum/upstream: {{ include (print $.Template.BasePath "/config-upstream.yaml") . | sha256sum }} checksum/ssh-config: {{ include (print $.Template.BasePath "/config-ssh-git-servers.yaml") . | sha256sum }} checksum/ssh-secret: {{ include (print $.Template.BasePath "/secret-ssh-git-servers.yaml") . | sha256sum }} + {{- if .Values.annotations }} +{{ toYaml .Values.annotations | indent 8 }} + {{- end }} spec: {{- if .Values.sshGitServers }} initContainers: diff --git a/charts/athens-proxy/values.yaml b/charts/athens-proxy/values.yaml index 98a4a7df..17b03088 100644 --- a/charts/athens-proxy/values.yaml +++ b/charts/athens-proxy/values.yaml @@ -79,6 +79,9 @@ storage: # You can add any new ones at the bottom configEnvVars: {} +# Extra annotations to be added to the athens pods +annotations: {} + # HTTP basic auth basicAuth: enabled: false From 34f55365a2dfe932b18cd0cb1521e9fe7e7ebd85 Mon Sep 17 00:00:00 2001 From: Marwan Sulaiman Date: Mon, 20 Jan 2020 18:25:48 -0500 Subject: [PATCH 16/23] always use request logging middleware (#1522) * always use request logging middleware * reads better * rm http-url as it is redundant * gopls being redundant * add color name --- cmd/proxy/actions/app.go | 4 +-- go.mod | 3 ++ go.sum | 6 ++++ pkg/log/format.go | 48 +++++++++++++++++++++++++++++++- pkg/middleware/log_entry.go | 1 - pkg/middleware/log_entry_test.go | 2 +- pkg/middleware/request.go | 22 +++++---------- 7 files changed, 65 insertions(+), 21 deletions(-) diff --git a/cmd/proxy/actions/app.go b/cmd/proxy/actions/app.go index c1955d56..2c6a06a0 100644 --- a/cmd/proxy/actions/app.go +++ b/cmd/proxy/actions/app.go @@ -56,10 +56,8 @@ func App(conf *config.Config) (http.Handler, error) { lggr := log.New(conf.CloudRuntime, logLvl) r := mux.NewRouter() - if conf.GoEnv == "development" { - r.Use(mw.RequestLogger) - } r.Use(mw.LogEntryMiddleware(lggr)) + r.Use(mw.RequestLogger) r.Use(secure.New(secure.Options{ SSLRedirect: conf.ForceSSL, SSLProxyHeaders: map[string]string{"X-Forwarded-Proto": "https"}, diff --git a/go.mod b/go.mod index ff52647f..5d00f562 100644 --- a/go.mod +++ b/go.mod @@ -30,11 +30,14 @@ require ( github.com/hashicorp/go-multierror v1.0.0 github.com/hashicorp/hcl2 v0.0.0-20190503213020-640445e16309 github.com/kelseyhightower/envconfig v1.3.0 + github.com/kylelemons/godebug v1.1.0 // indirect github.com/minio/minio-go/v6 v6.0.43 github.com/mitchellh/go-homedir v1.1.0 github.com/philhofer/fwd v1.0.0 // indirect + github.com/pkg/errors v0.8.1 // indirect github.com/sirupsen/logrus v1.4.2 github.com/spf13/afero v1.1.2 + github.com/spf13/pflag v1.0.3 // indirect github.com/stretchr/testify v1.3.0 github.com/technosophos/moniker v0.0.0-20180509230615-a5dbd03a2245 github.com/tidwall/pretty v0.0.0-20180105212114-65a9db5fad51 // indirect diff --git a/go.sum b/go.sum index 1a3d52ce..5685aacf 100644 --- a/go.sum +++ b/go.sum @@ -182,6 +182,8 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/markbates/hmax v1.0.0 h1:yo2N0gBoCnUMKhV/VRLHomT6Y9wUm+oQQENuWJqCdlM= github.com/markbates/hmax v1.0.0/go.mod h1:cOkR9dktiESxIMu+65oc/r/bdY4bE8zZw3OLhLx0X2c= github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= @@ -216,6 +218,8 @@ github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -255,6 +259,8 @@ github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3 github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.2 h1:Fy0orTDgHdbnzHcsOgfCN4LtHf0ec3wwtiwJqwvf3Gc= github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= diff --git a/pkg/log/format.go b/pkg/log/format.go index 01cdc01e..f1e69d06 100644 --- a/pkg/log/format.go +++ b/pkg/log/format.go @@ -1,6 +1,13 @@ package log import ( + "bytes" + "fmt" + "sort" + "strings" + "time" + + "github.com/fatih/color" "github.com/sirupsen/logrus" ) @@ -15,7 +22,46 @@ func getGCPFormatter() logrus.Formatter { } func getDevFormatter() logrus.Formatter { - return &logrus.TextFormatter{} + return devFormatter{} +} + +type devFormatter struct{} + +const lightGrey = 0xffccc + +func (devFormatter) Format(e *logrus.Entry) ([]byte, error) { + var buf bytes.Buffer + var sprintf func(format string, a ...interface{}) string + switch e.Level { + case logrus.DebugLevel: + sprintf = color.New(lightGrey).Sprintf + case logrus.WarnLevel: + sprintf = color.YellowString + case logrus.ErrorLevel: + sprintf = color.RedString + default: + sprintf = color.CyanString + } + lvl := strings.ToUpper(e.Level.String()) + buf.WriteString(sprintf(lvl)) + buf.WriteString("[" + e.Time.Format(time.Kitchen) + "]") + buf.WriteString(": ") + buf.WriteString(e.Message) + buf.WriteByte('\t') + for _, k := range sortFields(e.Data) { + fmt.Fprintf(&buf, "%s=%s ", color.MagentaString(k), e.Data[k]) + } + buf.WriteByte('\n') + return buf.Bytes(), nil +} + +func sortFields(data logrus.Fields) []string { + keys := []string{} + for k := range data { + keys = append(keys, k) + } + sort.Strings(keys) + return keys } func getDefaultFormatter() logrus.Formatter { diff --git a/pkg/middleware/log_entry.go b/pkg/middleware/log_entry.go index 77b03066..4d2acee6 100644 --- a/pkg/middleware/log_entry.go +++ b/pkg/middleware/log_entry.go @@ -16,7 +16,6 @@ func LogEntryMiddleware(lggr *log.Logger) mux.MiddlewareFunc { ent := lggr.WithFields(logrus.Fields{ "http-method": r.Method, "http-path": r.URL.Path, - "http-url": r.URL.String(), }) ctx := log.SetEntryInContext(r.Context(), ent) diff --git a/pkg/middleware/log_entry_test.go b/pkg/middleware/log_entry_test.go index c2825e7e..14e88d8b 100644 --- a/pkg/middleware/log_entry_test.go +++ b/pkg/middleware/log_entry_test.go @@ -34,6 +34,6 @@ func TestLogContext(t *testing.T) { req, _ := http.NewRequest("GET", "/test", nil) r.ServeHTTP(w, req) - expected := `{"http-method":"GET","http-path":"/test","http-url":"/test","level":"info","msg":"test"}` + expected := `{"http-method":"GET","http-path":"/test","level":"info","msg":"test"}` assert.True(t, strings.Contains(buf.String(), expected), fmt.Sprintf("%s should contain: %s", buf.String(), expected)) } diff --git a/pkg/middleware/request.go b/pkg/middleware/request.go index 6948cb3c..1bff12e6 100644 --- a/pkg/middleware/request.go +++ b/pkg/middleware/request.go @@ -5,6 +5,8 @@ import ( "net/http" "github.com/fatih/color" + "github.com/gomods/athens/pkg/log" + logrus "github.com/sirupsen/logrus" ) type responseWriter struct { @@ -23,18 +25,14 @@ func RequestLogger(h http.Handler) http.Handler { f := func(w http.ResponseWriter, r *http.Request) { rw := &responseWriter{w, 0} h.ServeHTTP(rw, r) - fmt.Println( - fmtRequest( - r.Method, - r.URL.Path, - rw.statusCode, - ), - ) + log.EntryFromContext(r.Context()).WithFields(logrus.Fields{ + "http-status": fmtResponseCode(rw.statusCode), + }).Infof("incoming request") } return http.HandlerFunc(f) } -func fmtRequest(method, path string, statusCode int) string { +func fmtResponseCode(statusCode int) string { if statusCode == 0 { statusCode = 200 } @@ -47,11 +45,5 @@ func fmtRequest(method, path string, statusCode int) string { default: status = color.HiRedString("%v", status) } - return fmt.Sprintf( - "%v %v %v [%v]", - color.CyanString("handler:"), - method, - path, - status, - ) + return status } From ba40b8f76daff911ac293bbdd0eb4d3b3b6acc23 Mon Sep 17 00:00:00 2001 From: Marwan Sulaiman Date: Fri, 24 Jan 2020 10:20:37 -0500 Subject: [PATCH 17/23] =?UTF-8?q?config:=20use=20semi-colon=20instead=20of?= =?UTF-8?q?=20comma=20to=20separate=20ATHENS=5FGO=5FBINARY=5F=E2=80=A6=20(?= =?UTF-8?q?#1531)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * config: use semi-colon instead of comma to separate ATHENS_GO_BINARY_ENV_VARS * document behavior --- config.dev.toml | 16 +++++ pkg/config/config.go | 25 ++++++++ pkg/config/config_test.go | 119 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 160 insertions(+) diff --git a/config.dev.toml b/config.dev.toml index 11c34c8d..38c28cc0 100755 --- a/config.dev.toml +++ b/config.dev.toml @@ -35,6 +35,22 @@ GoProxy = "direct" # Although you can pass any key=value to the Go command here, you can see # the list of possible env vars by running `go env`. # Env override: ATHENS_GO_BINARY_ENV_VARS +# +# IMPORTANT note about using the env var to override this config: +# +# You must use a semi-colon (;) to separate multiple env vars +# within ATHENS_GO_BINARY_ENV_VARS. For example: +# ATHENS_GO_BINARY_ENV_VARS='GOPROXY=proxy.golang.org,direct; GOPRIVATE=github.com/gomods/*' +# The semi-colon is here used instead of the comma (,) because the comma is a valid value to +# separate arguments in certain go env vars such as GOPROXY and GOPRIVATE +# +# NOTE that if you use the env var, then whatever you have in this config file will be overridden +# and NOT appended/merged. In other words, if the config file value is +# GoBinaryEnvVars = ["GOPROXY=direct"] +# And you pass the following env var: +# ATHENS_GO_BINARY_ENV_VARS='GODEBUG=true' +# Then the final value that the Go binary will receive is [GOBINARY=true] and NOT ["GOPROXY=direct", "GOBINARY=true"] +# Therefore, whether you use the config file or the env var, make sure you have all the values you need there. GoBinaryEnvVars = ["GOPROXY=direct"] # GoGetWorkers specifies how many times you can concurrently diff --git a/pkg/config/config.go b/pkg/config/config.go index 7bfa4e9d..ad108fd1 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -80,6 +80,31 @@ func (el *EnvList) Add(key, value string) { *el = append(*el, key+"="+value) } +// Decode implements envconfig.Decoder. Please see the below link for more information on +// that interface: +// +// https://github.com/kelseyhightower/envconfig#custom-decoders +// +// We are doing this to allow for very long lists of assignments to be set inside of +// a single environment variable. For example: +// +// ATHENS_GO_BINARY_ENV_VARS="GOPRIVATE=*.corp.example.com,rsc.io/private; GOPROXY=direct" +// +// See the below link for more information: +// https://github.com/gomods/athens/issues/1404 +func (el *EnvList) Decode(value string) error { + const op errors.Op = "envList.Decode" + if value == "" { + return nil + } + *el = EnvList{} // env vars must override config file + assignments := strings.Split(value, ";") + for _, assignment := range assignments { + *el = append(*el, strings.TrimSpace(assignment)) + } + return el.Validate() +} + // Validate validates that all strings inside the // list are of the key=value format func (el EnvList) Validate() error { diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 07b6fd6a..631c7df3 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -11,6 +11,8 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" + "github.com/kelseyhightower/envconfig" + "github.com/stretchr/testify/require" ) func testConfigFile(t *testing.T) (testConfigFile string) { @@ -512,3 +514,120 @@ func TestEnvList(t *testing.T) { t.Fatalf("expected err to be nil but got %v", err) } } + +type decodeTestCase struct { + name string + pre EnvList + given string + valid bool + expected EnvList +} + +var envListDecodeTests = []decodeTestCase{ + { + name: "empty", + pre: EnvList{}, + given: "", + valid: true, + expected: EnvList{}, + }, + { + name: "unchanged", + pre: EnvList{"GOPROXY=direct"}, + given: "", + valid: true, + expected: EnvList{"GOPROXY=direct"}, + }, + { + name: "must not merge", + pre: EnvList{"GOPROXY=direct"}, + given: "GOPRIVATE=github.com/gomods/*", + valid: true, + expected: EnvList{"GOPRIVATE=github.com/gomods/*"}, + }, + { + name: "must override", + pre: EnvList{"GOPROXY=direct"}, + given: "GOPROXY=https://proxy.golang.org", + valid: true, + expected: EnvList{"GOPROXY=https://proxy.golang.org"}, + }, + { + name: "semi colon separator", + pre: EnvList{"GOPROXY=direct", "GOPRIVATE="}, + given: "GOPROXY=off; GOPRIVATE=marwan.io/*;GONUTS=lol;GODEBUG=dns=true", + valid: true, + expected: EnvList{ + "GOPROXY=off", + "GOPRIVATE=marwan.io/*", + "GONUTS=lol", + "GODEBUG=dns=true", + }, + }, + { + name: "with commas", + pre: EnvList{"GOPROXY=direct", "GOPRIVATE="}, + given: "GOPROXY=proxy.golang.org,direct;GOPRIVATE=marwan.io/*;GONUTS=lol;GODEBUG=dns=true", + valid: true, + expected: EnvList{ + "GOPROXY=proxy.golang.org,direct", + "GOPRIVATE=marwan.io/*", + "GONUTS=lol", + "GODEBUG=dns=true", + }, + }, + { + name: "invalid", + pre: EnvList{}, + given: "GOPROXY=direct; INVALID", + valid: false, + }, + { + name: "accept empty value", + pre: EnvList{"GOPROXY=direct"}, + given: "GOPROXY=; GOPRIVATE=github.com/*", + valid: true, + expected: EnvList{"GOPROXY=", "GOPRIVATE=github.com/*"}, + }, +} + +func TestEnvListDecode(t *testing.T) { + for _, tc := range envListDecodeTests { + t.Run(tc.name, func(t *testing.T) { + testDecode(t, tc) + }) + } + cfg := &Config{ + GoBinaryEnvVars: EnvList{"GOPROXY=direct"}, + } + err := cfg.GoBinaryEnvVars.Decode("GOPROXY=https://proxy.golang.org; GOPRIVATE=github.com/gomods/*") + if err != nil { + t.Fatal(err) + } + cfg.GoBinaryEnvVars.Validate() +} + +func testDecode(t *testing.T, tc decodeTestCase) { + const envKey = "ATHENS_LIST_TEST" + + os.Setenv(envKey, tc.given) + defer func() { + require.NoError(t, os.Unsetenv(envKey)) + }() + + var config struct { + GoBinaryEnvVars EnvList `envconfig:"ATHENS_LIST_TEST"` + } + config.GoBinaryEnvVars = tc.pre + err := envconfig.Process("", &config) + if tc.valid && err != nil { + t.Fatal(err) + } + if !tc.valid { + if err == nil { + t.Fatal("expected an error but got nil") + } + return + } + require.Equal(t, tc.expected, config.GoBinaryEnvVars) +} From 4530a5835978c2804fcbf031b41a9d40a8adf679 Mon Sep 17 00:00:00 2001 From: vapod Date: Tue, 11 Feb 2020 03:03:25 +0300 Subject: [PATCH 18/23] Add forcepathstyle config for s3 (#1533) * Add forcepathstyle config for s3 * Bump chart version * Fix typos in s3 config struct --- charts/athens-proxy/Chart.yaml | 2 +- charts/athens-proxy/templates/deployment.yaml | 2 ++ config.dev.toml | 4 ++++ docs/content/configuration/storage.md | 4 ++++ pkg/config/config_test.go | 1 + pkg/config/s3.go | 3 ++- pkg/storage/s3/doc.go | 1 + pkg/storage/s3/s3.go | 2 ++ pkg/storage/s3/s3_test.go | 2 +- 9 files changed, 18 insertions(+), 3 deletions(-) diff --git a/charts/athens-proxy/Chart.yaml b/charts/athens-proxy/Chart.yaml index 4d9b9962..b16ae5a7 100644 --- a/charts/athens-proxy/Chart.yaml +++ b/charts/athens-proxy/Chart.yaml @@ -1,5 +1,5 @@ name: athens-proxy -version: 0.4.2 +version: 0.4.3 appVersion: 0.7.0 description: The proxy server for Go modules icon: https://raw.githubusercontent.com/gomods/athens/master/docs/static/banner.png diff --git a/charts/athens-proxy/templates/deployment.yaml b/charts/athens-proxy/templates/deployment.yaml index d3f931be..e2d5a384 100644 --- a/charts/athens-proxy/templates/deployment.yaml +++ b/charts/athens-proxy/templates/deployment.yaml @@ -86,6 +86,8 @@ spec: value: {{ .Values.storage.s3.bucket | quote }} - name: AWS_USE_DEFAULT_CONFIGURATION value: {{ .Values.storage.s3.useDefaultConfiguration | quote }} + - name: AWS_FORCE_PATH_STYLE + value: {{ .Values.storage.s3.ForcePathStyle | quote }} {{- if .Values.storage.s3.access_key_id }} - name: AWS_ACCESS_KEY_ID value: {{ .Values.storage.s3.access_key_id | quote }} diff --git a/config.dev.toml b/config.dev.toml index 38c28cc0..cdb1f29c 100755 --- a/config.dev.toml +++ b/config.dev.toml @@ -407,6 +407,10 @@ SingleFlightType = "memory" # Env override: ATHENS_S3_BUCKET_NAME Bucket = "MY_S3_BUCKET_NAME" + # If true then path style url for s3 endpoint will be used + # Env override: AWS_FORCE_PATH_STYLE + ForcePathStyle = false + # If true then the default aws configuration will be used. This will # attempt to find credentials in the environment, in the shared # configuration (~/.aws/credentials) and from ec2 instance role diff --git a/docs/content/configuration/storage.md b/docs/content/configuration/storage.md index 383151f7..c036989d 100644 --- a/docs/content/configuration/storage.md +++ b/docs/content/configuration/storage.md @@ -167,6 +167,10 @@ After this you can pass your credentials inside `config.toml` file. If the acce # S3 Bucket to use for storage # Env override: ATHENS_S3_BUCKET_NAME Bucket = "MY_S3_BUCKET_NAME" + + # If true then path style url for s3 endpoint will be used + # Env override: AWS_FORCE_PATH_STYLE + ForcePathStyle = false # If true then the default aws configuration will be used. This will # attempt to find credentials in the environment, in the shared diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 631c7df3..3c06a9da 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -362,6 +362,7 @@ func getEnvMap(config *Config) map[string]string { envVars["AWS_ACCESS_KEY_ID"] = storage.S3.Key envVars["AWS_SECRET_ACCESS_KEY"] = storage.S3.Secret envVars["AWS_SESSION_TOKEN"] = storage.S3.Token + envVars["AWS_FORCE_PATH_STYLE"] = strconv.FormatBool(storage.S3.ForcePathStyle) envVars["ATHENS_S3_BUCKET_NAME"] = storage.S3.Bucket } } diff --git a/pkg/config/s3.go b/pkg/config/s3.go index 82de1f7b..bc644408 100644 --- a/pkg/config/s3.go +++ b/pkg/config/s3.go @@ -8,7 +8,8 @@ type S3Config struct { Token string `envconfig:"AWS_SESSION_TOKEN"` Bucket string `validate:"required" envconfig:"ATHENS_S3_BUCKET_NAME"` UseDefaultConfiguration bool `envconfig:"AWS_USE_DEFAULT_CONFIGURATION"` + ForcePathStyle bool `envconfig:"AWS_FORCE_PATH_STYLE"` CredentialsEndpoint string `envconfig:"AWS_CREDENTIALS_ENDPOINT"` AwsContainerCredentialsRelativeURI string `envconfig:"AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"` - Endpoint string `evnconfig:"AWS_ENDPOINT"` + Endpoint string `envconfig:"AWS_ENDPOINT"` } diff --git a/pkg/storage/s3/doc.go b/pkg/storage/s3/doc.go index 2907c33e..03e66db8 100644 --- a/pkg/storage/s3/doc.go +++ b/pkg/storage/s3/doc.go @@ -9,6 +9,7 @@ Environment variables: AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN // [optional] + AWS_FORCE_PATH_STYLE // [optional] ATHENS_S3_BUCKET_NAME For information how to get your keyId and access key turn to official aws docs: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/setting-up.html diff --git a/pkg/storage/s3/s3.go b/pkg/storage/s3/s3.go index be05c02e..cfa80fb1 100644 --- a/pkg/storage/s3/s3.go +++ b/pkg/storage/s3/s3.go @@ -22,6 +22,7 @@ import ( // - AWS_ACCESS_KEY_ID - [optional] // - AWS_SECRET_ACCESS_KEY - [optional] // - AWS_SESSION_TOKEN - [optional] +// - AWS_FORCE_PATH_STYLE - [optional] // For information how to get your keyId and access key turn to official aws docs: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/setting-up.html type Storage struct { bucket string @@ -57,6 +58,7 @@ func New(s3Conf *config.S3Config, timeout time.Duration, options ...func(*aws.Co credProviders = append(endpointcreds, credProviders...) } + awsConfig.S3ForcePathStyle = aws.Bool(s3Conf.ForcePathStyle) awsConfig.Credentials = credentials.NewChainCredentials(credProviders) awsConfig.CredentialsChainVerboseErrors = aws.Bool(true) if s3Conf.Endpoint != "" { diff --git a/pkg/storage/s3/s3_test.go b/pkg/storage/s3/s3_test.go index 0a5ead97..f24b40e8 100644 --- a/pkg/storage/s3/s3_test.go +++ b/pkg/storage/s3/s3_test.go @@ -77,7 +77,6 @@ func getStorage(t testing.TB) *Storage { options := func(conf *aws.Config) { conf.Endpoint = aws.String(url) conf.DisableSSL = aws.Bool(true) - conf.S3ForcePathStyle = aws.Bool(true) } backend, err := New( &config.S3Config{ @@ -85,6 +84,7 @@ func getStorage(t testing.TB) *Storage { Secret: "minio123", Bucket: "gomodsaws", Region: "us-west-1", + ForcePathStyle: true, }, config.GetTimeoutDuration(300), options, From 27f36834161274dd3b98f8547a1aea1f536f643d Mon Sep 17 00:00:00 2001 From: Federico Paolinelli Date: Thu, 20 Feb 2020 00:38:08 +0100 Subject: [PATCH 19/23] Replace the current e2e script with a test suite running e2e tests. (#1514) * Replace the current e2e script with a test suite running e2e tests. * Add a build tag to skip e2e while running unit tests. We want e2e tests to be skipped while running normal unit tests. --- Makefile | 2 +- appveyor.yml | 3 +- e2etests/all_test.go | 144 +++++++++++++++++++++++++++++++++++++++++ e2etests/filesystem.go | 39 +++++++++++ e2etests/run_athens.go | 61 +++++++++++++++++ scripts/test_e2e.sh | 69 -------------------- 6 files changed, 247 insertions(+), 71 deletions(-) create mode 100644 e2etests/all_test.go create mode 100644 e2etests/filesystem.go create mode 100644 e2etests/run_athens.go delete mode 100755 scripts/test_e2e.sh diff --git a/Makefile b/Makefile index 45e13717..2730a530 100644 --- a/Makefile +++ b/Makefile @@ -63,7 +63,7 @@ test-unit-docker: ## run unit tests with docker .PHONY: test-e2e test-e2e: - ./scripts/test_e2e.sh + cd e2etests && go test --tags e2etests .PHONY: test-e2e-docker test-e2e-docker: diff --git a/appveyor.yml b/appveyor.yml index bb5775df..5159492d 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -11,4 +11,5 @@ environment: stack: go 1.13 test_script: - - go test ./... \ No newline at end of file + - go test ./... + diff --git a/e2etests/all_test.go b/e2etests/all_test.go new file mode 100644 index 00000000..92ac1579 --- /dev/null +++ b/e2etests/all_test.go @@ -0,0 +1,144 @@ +// +build e2etests + +package e2etests + +import ( + "context" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/gobuffalo/envy" + "github.com/stretchr/testify/suite" +) + +type E2eSuite struct { + suite.Suite + goBinaryPath string + env []string + goPath string + sampleRepoPath string + stopAthens context.CancelFunc +} + +type catalogRes struct { + Modules []struct { + Module string `json:"module"` + Version string `json:"version"` + } `json:"modules"` +} + +func (m *E2eSuite) SetupSuite() { + var err error + m.goPath, err = ioutil.TempDir("/tmp", "gopath") + if err != nil { + m.Fail("Failed to make temp dir", err) + } + + m.sampleRepoPath, err = ioutil.TempDir("/tmp", "repopath") + if err != nil { + m.Fail("Failed to make temp dir for sample repo", err) + } + + m.goBinaryPath = envy.Get("GO_BINARY_PATH", "go") + + athensBin, err := buildAthens(m.goBinaryPath, m.goPath, m.env) + if err != nil { + m.Fail("Failed to build athens ", err) + } + stopAthens() // in case a dangling instance was around. + // ignoring error as if no athens is running it fails. + + ctx := context.Background() + ctx, m.stopAthens = context.WithCancel(ctx) + runAthensAndWait(ctx, athensBin, m.getEnv()) + setupTestRepo(m.sampleRepoPath, "https://github.com/athens-artifacts/happy-path.git") +} + +func (m *E2eSuite) TearDownSuite() { + m.stopAthens() + chmodR(m.goPath, 0777) + os.RemoveAll(m.goPath) + chmodR(m.sampleRepoPath, 0777) + os.RemoveAll(m.sampleRepoPath) +} + +func TestE2E(t *testing.T) { + suite.Run(t, &E2eSuite{}) +} + +func (m *E2eSuite) SetupTest() { + chmodR(m.goPath, 0777) + err := cleanGoCache(m.getEnv()) + if err != nil { + m.Fail("Failed to clear go cache", err) + } +} + +func (m *E2eSuite) TestNoGoProxy() { + cmd := exec.Command("go", "run", ".") + cmd.Env = m.env + cmd.Dir = m.sampleRepoPath + + err := cmd.Run() + if err != nil { + m.Fail("go run failed on test repo", err) + } +} + +func (m *E2eSuite) TestGoProxy() { + cmd := exec.Command("go", "run", ".") + cmd.Env = m.getEnvGoProxy(m.goPath) + cmd.Dir = m.sampleRepoPath + err := cmd.Run() + if err != nil { + m.Fail("go run failed on test repo", err) + } + resp, err := http.Get("http://localhost:3000/catalog") + if err != nil { + m.Fail("failed to read catalog", err) + } + + var catalog catalogRes + err = json.NewDecoder(resp.Body).Decode(&catalog) + if err != nil { + m.Fail("failed to decode catalog res", err) + } + m.Assert().Equal(len(catalog.Modules), 1) + m.Assert().Equal(catalog.Modules[0].Module, "github.com/athens-artifacts/no-tags") +} + +func (m *E2eSuite) TestWrongGoProxy() { + cmd := exec.Command("go", "run", ".") + cmd.Env = m.getEnvWrongGoProxy(m.goPath) + cmd.Dir = m.sampleRepoPath + err := cmd.Run() + m.Assert().NotNil(err, "Wrong proxy should fail") +} + +func (m *E2eSuite) getEnv() []string { + res := []string{ + fmt.Sprintf("GOPATH=%s", m.goPath), + "GO111MODULE=on", + fmt.Sprintf("PATH=%s", os.Getenv("PATH")), + fmt.Sprintf("GOCACHE=%s", filepath.Join(m.goPath, "cache")), + } + return res +} + +func (m *E2eSuite) getEnvGoProxy(gopath string) []string { + res := m.getEnv() + res = append(res, "GOPROXY=http://localhost:3000") + return res +} + +func (m *E2eSuite) getEnvWrongGoProxy(gopath string) []string { + res := m.getEnv() + res = append(res, "GOPROXY=http://localhost:3001") + return res +} diff --git a/e2etests/filesystem.go b/e2etests/filesystem.go new file mode 100644 index 00000000..7ff059c9 --- /dev/null +++ b/e2etests/filesystem.go @@ -0,0 +1,39 @@ +// +build e2etests + +package e2etests + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" +) + +func setupTestRepo(repoPath, repoURL string) { + os.RemoveAll(repoPath) + cmd := exec.Command("git", + "clone", + repoURL, + repoPath) + + cmd.Run() +} + +func chmodR(path string, mode os.FileMode) error { + return filepath.Walk(path, func(name string, info os.FileInfo, err error) error { + if err == nil { + os.Chmod(name, mode) + } + return err + }) +} + +func cleanGoCache(env []string) error { + cmd := exec.Command("go", "clean", "--modcache") + cmd.Env = env + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("Failed to clear go cache: %v - %s", err, string(output)) + } + return nil +} diff --git a/e2etests/run_athens.go b/e2etests/run_athens.go new file mode 100644 index 00000000..5546a0fe --- /dev/null +++ b/e2etests/run_athens.go @@ -0,0 +1,61 @@ +// +build e2etests + +package e2etests + +import ( + "context" + "fmt" + "net/http" + "os/exec" + "path" + "path/filepath" + "time" +) + +func buildAthens(goBin, destPath string, env []string) (string, error) { + target := path.Join(destPath, "athens-proxy") + binFolder, err := filepath.Abs("../cmd/proxy") + if err != nil { + return "", fmt.Errorf("Failed to get athens source path %v", err) + } + + cmd := exec.Command(goBin, "build", "-o", target, binFolder) + cmd.Env = env + output, err := cmd.CombinedOutput() + if err != nil { + return "", fmt.Errorf("Failed to build athens: %v - %s", err, string(output)) + } + return target, nil +} + +func stopAthens() error { + cmd := exec.Command("pkill", "athens-proxy") + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("Failed to stop athens: %v - %s", err, string(output)) + } + return err +} + +func runAthensAndWait(ctx context.Context, athensBin string, env []string) error { + cmd := exec.CommandContext(ctx, athensBin) + cmd.Env = env + + cmd.Start() + + ticker := time.NewTicker(time.Second) + timeout := time.After(20 * time.Second) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + resp, _ := http.Get("http://localhost:3000/readyz") + if resp.StatusCode == 200 { + return nil + } + case <-timeout: + return fmt.Errorf("Failed to run athens") + } + } +} diff --git a/scripts/test_e2e.sh b/scripts/test_e2e.sh deleted file mode 100755 index fbed5341..00000000 --- a/scripts/test_e2e.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash - -# test_e2e.sh -# Execute end-to-end (e2e) tests to verify that everything is working right -# from the end user perspective -set -xeuo pipefail - -REPO_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )/.." - -OGOPATH=${GOPATH:-} -OGO111MODULE=${GO111MODULE:-} -OGOPROXY=${GOPROXY:-} -export GO_BINARY_PATH=${GO_BINARY_PATH:-go} -TMPDIR=$(mktemp -d) -export GOPATH=$TMPDIR -GOMOD_CACHE=$TMPDIR/pkg/mod -export PATH=${REPO_DIR}/bin:${PATH} - -clearGoModCache () { - chmod -R 0770 ${GOMOD_CACHE} - rm -fr ${GOMOD_CACHE} -} - -teardown () { - # Cleanup after our tests - [[ -z "${OGOPATH}" ]] && unset GOPATH || export GOPATH=$OGOPATH - [[ -z "${OGO111MODULE}" ]] && unset GO111MODULE || export GO111MODULE=$OGO111MODULE - [[ -z "${OGOPROXY}" ]] && unset GOPROXY || export GOPROXY=$OGOPROXY - - clearGoModCache - pkill athens-proxy || true - rm $REPO_DIR/cmd/proxy/athens-proxy || true - rm -fr ${TMPDIR} - popd 2> /dev/null || true -} -trap teardown EXIT - -export GO111MODULE=on - -# Start the proxy in the background and wait for it to be ready -cd $REPO_DIR/cmd/proxy -pkill athens-proxy || true # cleanup proxy if it is running -go build -o athens-proxy && ./athens-proxy & -while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:3000/readyz)" != "200" ]]; do sleep 1; done - -# Clone our test repo -TEST_SOURCE=${TMPDIR}/happy-path -rm -fr ${TEST_SOURCE} 2> /dev/null || true -git clone https://github.com/athens-artifacts/happy-path.git ${TEST_SOURCE} -pushd ${TEST_SOURCE} - -# Make sure that our test repo works without the GOPROXY first -unset GOPROXY -$GO_BINARY_PATH run . - -# clear cache so that go uses the proxy -clearGoModCache - -# Verify that the test works against the proxy -export GOPROXY=http://localhost:3000 -$GO_BINARY_PATH run . - -CATALOG_RES=$(curl localhost:3000/catalog) -CATALOG_EXPECTED='{"modules":[{"module":"github.com/athens-artifacts/no-tags","version":"v0.0.0-20180803171426-1a540c5d67ab"}]}' - -if [[ "$CATALOG_RES" != "$CATALOG_EXPECTED" ]]; then - echo ERROR: catalog endpoint failed - exit 1 # terminate and indicate error -fi From d7db2d4f1104e5799dc50971865c10799c278a51 Mon Sep 17 00:00:00 2001 From: Sebastian Heid Date: Thu, 20 Feb 2020 01:39:12 +0100 Subject: [PATCH 20/23] Add installation instructions for BOSH (#1535) * Add installation instructions for BOSH * Fix weight and add link to try-out document Co-authored-by: Aaron Schlesinger <70865+arschles@users.noreply.github.com> --- docs/content/install/install-on-bosh.md | 95 +++++++++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 docs/content/install/install-on-bosh.md diff --git a/docs/content/install/install-on-bosh.md b/docs/content/install/install-on-bosh.md new file mode 100644 index 00000000..0aafca3d --- /dev/null +++ b/docs/content/install/install-on-bosh.md @@ -0,0 +1,95 @@ +--- +title: Install Athens with BOSH +description: Installing an Athens Instance with BOSH +weight: 8 +--- + +Athens can be deployed in many ways. The following guide explains how to use [BOSH](https://bosh.io), a deployment and lifecycle tool, in order to deploy an Athens server on virtual machines (VMs) on any infrastructure as a service (IaaS) that is supported by BOSH. + +--- + +## Prerequisites + +* Install [BOSH](#install-bosh) +* Setup the [infrastructure](#setup-the-infrastructure) + +### Install BOSH + +Make sure to have the [BOSH CLI](https://bosh.io/docs/cli-v2-install/) installed and set up a [BOSH Director](https://bosh.io/docs/quick-start/) on an infrastructure of your choice. + +### Setup the Infrastructure + +If you choose to deploy on a IaaS provider, there are a few prerequisites that need to be set up before starting with the deployment. Depending on which IaaS you will be deploying, you may need to create: + +* **Public IP**: a public IP address for association with the Athens VM. +* **Firewall Rules**: the following ingress ports must be allowed + + - `3000/tcp` - Athens proxy port (if you specify a different port than the default port 3000 in the job properties, adapt this rule accordingly). + + Egress traffic should be restricted depending on your requirements. + +#### Amazon Web Services (AWS) + +AWS requires additional settings that should be added to a `credentials.yml` file using the following template: + +```yaml +# security group IDs to apply to the VM +athens_security_groups: [sg-0123456abcdefgh] + +# VPC subnet to deploy Athens to +athens_subnet_id: subnet-0123456789abcdefgh + +# a specific, elastic IP address for the VM +external_ip: 3.123.200.100 +``` + +The credentials need to be added to the `deploy` command, i.e. + +``` +-o manifests/operations/aws-ops.yml +``` + +#### VirtualBox + +The fastest way to install Athens using BOSH is probably a Director VM running on VirtualBox which is sufficient for development or testing purposes. If you follow the bosh-lite [installation guide](https://bosh.cloudfoundry.org/docs/bosh-lite), no further preparation is required to deploy Athens. + + +## Deployment + +A deployment manifest contains all the information for managing and updating a BOSH deployment. To aid in the deployment of Athens on BOSH, the [athens-bosh-release](https://github.com/s4heid/athens-bosh-release) repository provides manifests for basic deployment configurations inside the `manifests` directory. For quickly creating a standalone Athens server, clone the release repository and `cd` into it: + +```console +git clone --recursive https://github.com/s4heid/athens-bosh-release.git +cd athens-bosh-release +``` + +Once the [infrastructure](#setup-the-infrastructure) has been prepared and the BOSH Director is running, make sure that a [stemcell](https://bosh.cloudfoundry.org/docs/stemcell/) has been uploaded. If this has not been done yet, choose a stemcell from the [stemcells section of bosh.io](https://bosh.io/stemcells), and upload it via the command line. Additionally, a [cloud config](https://bosh.cloudfoundry.org/docs/cloud-config/) is required for IaaS specific configuration used by the Director and the Athens deployment. The `manifests` directory also contains an example cloud config, which can be uploaded to the Director via + +```console +bosh update-config --type=cloud --name=athens \ + --vars-file=credentials.yml manifests/cloud-config.yml +``` + +Execute the `deploy` command which can be extended with ops/vars files depending on which IaaS you will be deploying to. + +```console +bosh -d athens deploy manifests/athens.yml # add extra arguments +``` + +For example, when using AWS the deploy command for an Athens Proxy with disk storage would look like + +```console +bosh -d athens deploy \ + -o manifests/operations/aws-ops.yml \ + -o manifests/operations/with-persistent-disk.yml \ + -v disk_size=1024 \ + --vars-file=credentials.yml manifests/athens.yml +``` + +This will deploy a single Athens instance in the `athens` deployment with a persistent disk of 1024MB. The IP address of that instance can be obtained with + +```console +bosh -d athens instances +``` + +which is useful for targeting Athens, e.g. with the `GOPROXY` variable. You can follow this [quickstart guide](/try-out) for more information. \ No newline at end of file From ed703ee63dcf0418d9ce662baf8dee78097dbb1e Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Wed, 19 Feb 2020 16:52:24 -0800 Subject: [PATCH 21/23] Adding docs for setting the GONOSUMDB env var on the Athens side (#1504) * Adding docs for setting the GONOSUMDB env var on the Athens side Ref https://github.com/gomods/athens/issues/1363 * Updating with multiple repos, and adding a note * Update docs/content/configuration/sumdb.md Co-Authored-By: Ted Wexler * Qualifying "user" Co-authored-by: Ted Wexler --- docs/content/configuration/sumdb.md | 36 +++++++++++++++++------------ 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/docs/content/configuration/sumdb.md b/docs/content/configuration/sumdb.md index 1c72ca23..fdb6de49 100644 --- a/docs/content/configuration/sumdb.md +++ b/docs/content/configuration/sumdb.md @@ -5,41 +5,47 @@ weight: 4 --- ## Proxying A Checksum DB + The Athens Proxy has the ability to proxy a Checksum Database as defined by [this proposal](https://go.googlesource.com/proposal/+/master/design/25530-sumdb.md) by the Go team. -Athens by default will accept proxying `https://sum.golang.org`. However, if you'd like to override that behavior or proxy more Checksum DBs you can do so through the `SumDBs` config or its equivalent Environment Variable: `ATHENS_SUM_DBS` +Athens by default will accept proxying `https://sum.golang.org`. However, if you'd like to override that behavior or proxy more Checksum DBs you can do so through the `SumDBs` config or its equivalent Environment Variable: `ATHENS_SUM_DBS`. -So for example, if you run the following command: +So for example, if you run the following command: ```bash GOPROXY= go build ``` -The Go command will proxy requests to `sum.golang.org` like this: `/sumdb/sum.golang.org`. Feel free to read the linked proposal above for the exact requests that makes Athens successfully proxy Checksum DB APIs. +The Go command will proxy requests to `sum.golang.org` like this: `/sumdb/sum.golang.org`. Feel free to read the linked proposal above for the exact requests that makes Athens successfully proxy Checksum DB APIs. Note that as of this documentation (May 2019), you need to explicitly set `GOSUMDB=https://sum.golang.org`, but the Go team is planning on enabling this by default. -### Why a Checksum DB? - -The reasons for needing a Checksum DB is explained in the linked proposal above. However, the reasons for proxying a Checksum DB are more explained below. - -### Why Proxy a Checksum DB? +### Why Proxy a Checksum DB? This is quite important. Say you are a company that is running an Athens instance, and you don't want the world to know about where your repositories live. For example, say you have a private repo under `github.com/mycompany/secret-repo`. In order to ensure that the Go client -does not send a request to `https://sum.golang.org/lookup/github.com/mycompany/secret-repo@v1.0.0` and therefore leaking your private import path to the public, you need to ensure that you tell Go to skip particular import paths as such: +does not send a request to `https://sum.golang.org/lookup/github.com/mycompany/secret-repo@v1.0.0` and therefore leaking your private import path to the public, you need to ensure that you tell Go to skip particular import paths by setting the `GONOSUMDB` environment variable: ``` GONOSUMDB=github.com/mycompany/* go build ``` -This will make sure that Go does not send any requests to the Checksum DB for your private import paths. -However, how can you ensure that all of your employees are building private code with the right configuration? +This will make sure that Go does not send any requests to the Checksum DB for your private import paths. However, how can you ensure that all of your employees are building private code with the right configuration? -Athens, in this case can help ensure that all private code flowing through it never goes to the Checksum DB. So as long as your employees are using Athens, then they will get a helpful reminder to ensure Their GONOSUMDB is rightly configured. +Athens, in this case can help ensure that all private code flowing through it is never checked against the checksum DB. That means that as long as your employees are using Athens, then they will get a helpful reminder to ensure Their GONOSUMDB is rightly configured. -As the Athens company maintainer, you can run Athens with the following configuration: +Athens allows the operator running the server to specify a list of patterns to not lookup via the upstream sum provider: -`NoSumPatterns = ["github.com/mycompany/*] # or comma separted env var: ATHENS_GONOSUM_PATTERNS` +```toml +NoSumPatterns = ["github.com/mycompany/*", "github.com/secret/*"] +``` -This will ensure that when Go sends a request to ``, Athens will return a 403 and failing the build ensuring that the client knows something is not configured correctly and also never leaking those import paths +Or you can do it with an environment variable: + +```console +$ export ATHENS_GONOSUM_PATTERNS="github.com/mycompany/*,github.com/secret/*" +``` + +>In both of the above configuration examples, there are two patterns specified, and they are separated by a comma (`,`) in both cases. When you are using the environment variable (`export ATHENS_GONOSUM_PATTERNS`), make sure you don't use brackes (`[` and `]`). + +Either way, this configuration will ensure that when Go sends a request to `/sumdb/sum.golang.org/github.com/mycompany/secret-repo@v1.0.0`, Athens will return a 403 and failing the build ensuring that the client knows something is not configured correctly and also never leaking those import paths. From 9624953236f5a8b45ecdefd0887b1682c7f815e0 Mon Sep 17 00:00:00 2001 From: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Date: Fri, 21 Feb 2020 14:25:07 -0800 Subject: [PATCH 22/23] Updating the sum database documentation (#1494) * Updating the sum database documentation --- docs/content/configuration/sumdb.md | 64 ++++++++++++++++++----------- 1 file changed, 41 insertions(+), 23 deletions(-) diff --git a/docs/content/configuration/sumdb.md b/docs/content/configuration/sumdb.md index fdb6de49..9f54d18c 100644 --- a/docs/content/configuration/sumdb.md +++ b/docs/content/configuration/sumdb.md @@ -4,48 +4,66 @@ description: How to configure Athens to proxy a checksum database API, and why y weight: 4 --- -## Proxying A Checksum DB +If you run `go get github.com/mycompany/secret-repo@v1.0.0` and that module version is not yet in your `go.sum` file, Go will by default send a request to `https://sum.golang.org/lookup/github.com/mycompany/secret-repo@v1.0.0`. That request will fail because the Go tool requires a checksum, but `sum.golang.org` doesn't have access to your private code. -The Athens Proxy has the ability to proxy a Checksum Database as defined by [this proposal](https://go.googlesource.com/proposal/+/master/design/25530-sumdb.md) by the Go team. +The result is that *(1) your build will fail*, and *(2) your private module names have been sent over the internet to an opaque public server that you don't control*. -Athens by default will accept proxying `https://sum.golang.org`. However, if you'd like to override that behavior or proxy more Checksum DBs you can do so through the `SumDBs` config or its equivalent Environment Variable: `ATHENS_SUM_DBS`. +>You can read more about this `sum.golang.org` service [here](https://go.googlesource.com/proposal/+/master/design/25530-sumdb.md) -So for example, if you run the following command: +## Proxying a checksum DB + +Many companies use Athens to host their private code, but Athens is not only a module proxy. It's also a checksum database proxy. That means that anyone inside of your company can configure `go` to send these checksum requests to Athens instead of the public `sum.golang.org` server. + +If the Athens server is configured with checksum filters, then you can prevent these problems. + +If you run the below command using Go 1.13 or later: ```bash -GOPROXY= go build +$ GOPROXY= go build . ``` -The Go command will proxy requests to `sum.golang.org` like this: `/sumdb/sum.golang.org`. Feel free to read the linked proposal above for the exact requests that makes Athens successfully proxy Checksum DB APIs. +... then the Go tool will automatically send all checksum requests to `/sumdb/sum.golang.org` instead of `https://sum.golang.org`. -Note that as of this documentation (May 2019), you need to explicitly set `GOSUMDB=https://sum.golang.org`, but the Go team is planning on enabling this by default. +By default, when Athens receives a `/sumdb/...` request, it automatically proxies it to `https://sum.golang.org`, even if it's a private module that `sum.golang.org` doesn't and can't know about. So if you are working with private modules, you'll want to change the default behavior. -### Why Proxy a Checksum DB? +>If you want Athens to _not_ send some module names up to the global checksum database, set those module names in the `NoSumPatterns` value in `config.toml` or using the `ATHENS_GONOSUM_PATTERNS` environment variable. -This is quite important. Say you are a company that is running an Athens instance, and you don't want the world to know about where your -repositories live. For example, say you have a private repo under `github.com/mycompany/secret-repo`. In order to ensure that the Go client -does not send a request to `https://sum.golang.org/lookup/github.com/mycompany/secret-repo@v1.0.0` and therefore leaking your private import path to the public, you need to ensure that you tell Go to skip particular import paths by setting the `GONOSUMDB` environment variable: +The following sections will go into more detail on how checksum databases work, how Athens fits in, and how this all impacts your workflow. -``` -GONOSUMDB=github.com/mycompany/* go build -``` +## How to set this all up -This will make sure that Go does not send any requests to the Checksum DB for your private import paths. However, how can you ensure that all of your employees are building private code with the right configuration? - -Athens, in this case can help ensure that all private code flowing through it is never checked against the checksum DB. That means that as long as your employees are using Athens, then they will get a helpful reminder to ensure Their GONOSUMDB is rightly configured. - -Athens allows the operator running the server to specify a list of patterns to not lookup via the upstream sum provider: +Before you begin, you'll need to run Athens with configuration values that tell it to not proxy certain modules. If you're using `config.toml`, use this configuration: ```toml NoSumPatterns = ["github.com/mycompany/*", "github.com/secret/*"] ``` -Or you can do it with an environment variable: +And if you're using an environment variable, use this configuration: -```console +```bash $ export ATHENS_GONOSUM_PATTERNS="github.com/mycompany/*,github.com/secret/*" ``` ->In both of the above configuration examples, there are two patterns specified, and they are separated by a comma (`,`) in both cases. When you are using the environment variable (`export ATHENS_GONOSUM_PATTERNS`), make sure you don't use brackes (`[` and `]`). +>You can use any string compatible with [`path.Match`](https://pkg.go.dev/path?tab=doc#Match) in these environment variables -Either way, this configuration will ensure that when Go sends a request to `/sumdb/sum.golang.org/github.com/mycompany/secret-repo@v1.0.0`, Athens will return a 403 and failing the build ensuring that the client knows something is not configured correctly and also never leaking those import paths. +After you start Athens up with this configuration, all checksum requests for modules that start with `github.com/mycompany` or `github.com/secret` will not be forwarded, and Athens will return an error to the `go` CLI tool. + +This behavior will ensure that none of your private module names leak to the public internet, but your builds will still fail. To fix that problem, set another environment variable on your machine (that you run your `go` commands) + +```bash +$ export GONOSUMDB="github.com/mycompany/*,github.com/secret/*" +``` + +Now, your builds will work and you won't be sending information about your private codebase to the internet. + +## I'm confused, why is this hard? + +When the Go tool has to download _new_ code that isn't currently in the project's `go.sum` file, it tries its hardest to get a checksum from a server it trusts, and compare it to the checksum in the actual code it downloads. It does all of this to ensure _provenance_. That is, to ensure that the code you just downloaded wasn't tampered with. + +The trusted checksums are all stored in `sum.golang.org`, and that server is centrally controlled. + +>These build failures and potential privacy leaks can only happen when you try to get a module version that is _not_ already in your `go.sum` file. + +Athens does its best to respect and use the trusted checksums while also ensuring that your private names don't get leaked to the public server. In some cases, it has to choose whether to fail your build or leak information, so it chooses to fail your build. That's why everybody using that Athens server needs to set up their `GONOSUMDB` environment variable. + +We believe that along with good documentation - which we hope this is! - we have struck the right balance between convenience and privacy. From 71c57ef7eeed364c77cb25aec89654b5518a5be4 Mon Sep 17 00:00:00 2001 From: Matthew Plachter Date: Tue, 25 Feb 2020 17:18:04 -0700 Subject: [PATCH 23/23] add config options for redis password (#1545) * add config options for redis password * redis password test and failure test * changed redis name and made a minimal redis.conf add drone volume volume fix remove volumes from drone. repoint redis path Co-authored-by: Aaron Schlesinger <70865+arschles@users.noreply.github.com> Co-authored-by: Marwan Sulaiman --- .drone.yml | 9 +++++ cmd/proxy/actions/app_proxy.go | 2 +- config.dev.toml | 4 +++ docker-compose.yml | 9 ++++- pkg/config/config.go | 2 +- pkg/config/singleflight.go | 1 + pkg/stash/with_redis.go | 7 ++-- pkg/stash/with_redis_test.go | 60 +++++++++++++++++++++++++++++++++- test/redis.conf | 3 ++ 9 files changed, 90 insertions(+), 7 deletions(-) create mode 100644 test/redis.conf diff --git a/.drone.yml b/.drone.yml index 9beb1aaf..66f9e825 100644 --- a/.drone.yml +++ b/.drone.yml @@ -46,6 +46,8 @@ steps: ATHENS_MONGO_STORAGE_URL: mongodb://mongo:27017 ATHENS_MINIO_ENDPOINT: minio:9000 REDIS_TEST_ENDPOINT: redis:6379 + PROTECTED_REDIS_TEST_ENDPOINT: protectedredis:6380 + ATHENS_PROTECTED_REDIS_PASSWORD: AthensPass1 GCS_SERVICE_ACCOUNT: from_secret: GCS_SERVICE_ACCOUNT GCS_PROJECT_ID: @@ -154,6 +156,13 @@ services: image: redis ports: - 6379 +- name: protectedredis + image: redis + ports: + - 6380 + commands: + - "redis-server ./test/redis.conf" + - name: athens-proxy image: gomods/athens:canary pull: always diff --git a/cmd/proxy/actions/app_proxy.go b/cmd/proxy/actions/app_proxy.go index 1a705837..454695dd 100644 --- a/cmd/proxy/actions/app_proxy.go +++ b/cmd/proxy/actions/app_proxy.go @@ -132,7 +132,7 @@ func getSingleFlight(c *config.Config, checker storage.Checker) (stash.Wrapper, if c.SingleFlight == nil || c.SingleFlight.Redis == nil { return nil, fmt.Errorf("Redis config must be present") } - return stash.WithRedisLock(c.SingleFlight.Redis.Endpoint, checker) + return stash.WithRedisLock(c.SingleFlight.Redis.Endpoint, c.SingleFlight.Redis.Password, checker) case "gcp": if c.StorageType != "gcp" { return nil, fmt.Errorf("gcp SingleFlight only works with a gcp storage type and not: %v", c.StorageType) diff --git a/config.dev.toml b/config.dev.toml index cdb1f29c..8eb7d610 100755 --- a/config.dev.toml +++ b/config.dev.toml @@ -291,6 +291,10 @@ SingleFlightType = "memory" # Env override: ATHENS_REDIS_ENDPOINT Endpoint = "127.0.0.1:6379" + # Password is the password for a redis SingleFlight lock. + # Env override: ATHENS_REDIS_PASSWORD + Password = "" + [Storage] # Only storage backends that are specified in Proxy.StorageType are required here [Storage.CDN] diff --git a/docker-compose.yml b/docker-compose.yml index 8b04efcb..4a55d408 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -79,8 +79,15 @@ services: - 16686:16686 redis: image: redis - ports: + ports: - 6379:6379 + protectedredis: + image: redis + ports: + - "6380:6380" + volumes: + - "./test/redis.conf:/usr/local/etc/redis/redis.conf" + entrypoint: ["redis-server", "/usr/local/etc/redis/redis.conf"] etcd0: image: quay.io/coreos/etcd ports: diff --git a/pkg/config/config.go b/pkg/config/config.go index ad108fd1..2599a0b8 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -163,7 +163,7 @@ func defaultConfig() *Config { RobotsFile: "robots.txt", SingleFlight: &SingleFlight{ Etcd: &Etcd{"localhost:2379,localhost:22379,localhost:32379"}, - Redis: &Redis{"127.0.0.1:6379"}, + Redis: &Redis{"127.0.0.1:6379", ""}, }, } } diff --git a/pkg/config/singleflight.go b/pkg/config/singleflight.go index 361f4bc5..52e76a21 100644 --- a/pkg/config/singleflight.go +++ b/pkg/config/singleflight.go @@ -19,4 +19,5 @@ type Etcd struct { // to connect to redis as a SingleFlight implementation. type Redis struct { Endpoint string `envconfig:"ATHENS_REDIS_ENDPOINT"` + Password string `envconfig:"ATHENS_REDIS_PASSWORD"` } diff --git a/pkg/stash/with_redis.go b/pkg/stash/with_redis.go index 2316c21b..bf89f8c2 100644 --- a/pkg/stash/with_redis.go +++ b/pkg/stash/with_redis.go @@ -14,11 +14,12 @@ import ( // WithRedisLock returns a distributed singleflight // using an redis cluster. If it cannot connect, it will return an error. -func WithRedisLock(endpoint string, checker storage.Checker) (Wrapper, error) { +func WithRedisLock(endpoint string, password string, checker storage.Checker) (Wrapper, error) { const op errors.Op = "stash.WithRedisLock" client := redis.NewClient(&redis.Options{ - Network: "tcp", - Addr: endpoint, + Network: "tcp", + Addr: endpoint, + Password: password, }) _, err := client.Ping().Result() if err != nil { diff --git a/pkg/stash/with_redis_test.go b/pkg/stash/with_redis_test.go index 8ec50a0e..88d18357 100644 --- a/pkg/stash/with_redis_test.go +++ b/pkg/stash/with_redis_test.go @@ -19,6 +19,7 @@ import ( // and therefore all 5 responses should have no error. func TestWithRedisLock(t *testing.T) { endpoint := os.Getenv("REDIS_TEST_ENDPOINT") + password := os.Getenv("ATHENS_REDIS_PASSWORD") if len(endpoint) == 0 { t.SkipNow() } @@ -27,7 +28,7 @@ func TestWithRedisLock(t *testing.T) { t.Fatal(err) } ms := &mockRedisStasher{strg: strg} - wrapper, err := WithRedisLock(endpoint, strg) + wrapper, err := WithRedisLock(endpoint, password, strg) if err != nil { t.Fatal(err) } @@ -49,6 +50,63 @@ func TestWithRedisLock(t *testing.T) { } } +// Verify with WithRedisLock working with password protected redis +// Same logic as the TestWithRedisLock test. +func TestWithRedisLockWithPassword(t *testing.T) { + endpoint := os.Getenv("PROTECTED_REDIS_TEST_ENDPOINT") + password := os.Getenv("ATHENS_PROTECTED_REDIS_PASSWORD") + if len(endpoint) == 0 { + t.SkipNow() + } + strg, err := mem.NewStorage() + if err != nil { + t.Fatal(err) + } + ms := &mockRedisStasher{strg: strg} + wrapper, err := WithRedisLock(endpoint, password, strg) + if err != nil { + t.Fatal(err) + } + s := wrapper(ms) + + var eg errgroup.Group + for i := 0; i < 5; i++ { + eg.Go(func() error { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + _, err := s.Stash(ctx, "mod", "ver") + return err + }) + } + + err = eg.Wait() + if err != nil { + t.Fatal(err) + } +} + +// Verify the WithRedisLock fails with the correct error when trying +// to connect with the wrong password. +func TestWithRedisLockWithWrongPassword(t *testing.T) { + endpoint := os.Getenv("PROTECTED_REDIS_TEST_ENDPOINT") + password := "" + if len(endpoint) == 0 { + t.SkipNow() + } + strg, err := mem.NewStorage() + if err != nil { + t.Fatal(err) + } + _, err = WithRedisLock(endpoint, password, strg) + if err == nil { + t.Fatal("Expected Connection Error") + } + + if err.Error() != "NOAUTH Authentication required." { + t.Fatalf("Wrong error was thrown %s\n", err.Error()) + } +} + // mockRedisStasher is like mockStasher // but leverages in memory storage // so that redis can determine diff --git a/test/redis.conf b/test/redis.conf new file mode 100644 index 00000000..83cee356 --- /dev/null +++ b/test/redis.conf @@ -0,0 +1,3 @@ +# protectedredis testing configuration file. +port 6380 +requirepass AthensPass1 \ No newline at end of file