From 7de77889ad9a010798627d956c83cd9847b47513 Mon Sep 17 00:00:00 2001 From: Yi Tang Date: Tue, 16 Jul 2019 02:07:24 +0800 Subject: [PATCH] support goproxy for list/download command (#1304) --- cmd/proxy/actions/app_proxy.go | 4 ++-- config.dev.toml | 7 +++++++ pkg/config/config.go | 2 ++ pkg/config/config_test.go | 3 +++ pkg/download/protocol_test.go | 5 +++-- pkg/download/upstream_lister.go | 7 ++++--- pkg/module/all_test.go | 3 ++- pkg/module/go_get_fetcher.go | 14 +++++++++----- pkg/module/go_get_fetcher_test.go | 8 ++++---- 9 files changed, 36 insertions(+), 17 deletions(-) diff --git a/cmd/proxy/actions/app_proxy.go b/cmd/proxy/actions/app_proxy.go index 4f758afd..4906b4fb 100644 --- a/cmd/proxy/actions/app_proxy.go +++ b/cmd/proxy/actions/app_proxy.go @@ -73,12 +73,12 @@ func addProxyRoutes( // 3. The stashpool manages limiting concurrent requests and passes them to stash. // 4. The plain stash.New just takes a request from upstream and saves it into storage. fs := afero.NewOsFs() - mf, err := module.NewGoGetFetcher(c.GoBinary, fs) + mf, err := module.NewGoGetFetcher(c.GoBinary, c.GoProxy, fs) if err != nil { return err } - lister := download.NewVCSLister(c.GoBinary, fs) + lister := download.NewVCSLister(c.GoBinary, c.GoProxy, fs) withSingleFlight, err := getSingleFlight(c, s) if err != nil { diff --git a/config.dev.toml b/config.dev.toml index 1b7bb289..eae0b459 100755 --- a/config.dev.toml +++ b/config.dev.toml @@ -15,6 +15,13 @@ GoBinary = "go" # Env override: GO_ENV GoEnv = "development" +# GoProxy specifies GOPROXY env for go list or mod download inside athens +# which can be configured totally same with GOPROXY of Go Command. +# Notes that the comma-separated GOPROXY (e.g. ,,direct) is only available in Go 1.13 or higher, +# otherwise only single proxy URL can be set. +# Env override: GOPROXY +GoProxy = "direct" + # GoGetWorkers specifies how many times you can concurrently # go mod download, this is so that low performance instances # can manage go get more sanely and not run out of disk or memory. diff --git a/pkg/config/config.go b/pkg/config/config.go index 34586fa8..2e7d9519 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -21,6 +21,7 @@ type Config struct { TimeoutConf GoEnv string `validate:"required" envconfig:"GO_ENV"` GoBinary string `validate:"required" envconfig:"GO_BINARY_PATH"` + GoProxy string `envconfig:"GOPROXY"` GoGetWorkers int `validate:"required" envconfig:"ATHENS_GOGET_WORKERS"` ProtocolWorkers int `validate:"required" envconfig:"ATHENS_PROTOCOL_WORKERS"` LogLevel string `validate:"required" envconfig:"ATHENS_LOG_LEVEL"` @@ -76,6 +77,7 @@ func defaultConfig() *Config { return &Config{ GoBinary: "go", GoEnv: "development", + GoProxy: "direct", GoGetWorkers: 10, ProtocolWorkers: 30, LogLevel: "debug", diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 4528a4ed..f0fbd508 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -71,6 +71,7 @@ func TestEnvOverrides(t *testing.T) { ProtocolWorkers: 10, LogLevel: "info", GoBinary: "go11", + GoProxy: "direct", CloudRuntime: "gcp", TimeoutConf: TimeoutConf{ Timeout: 30, @@ -249,6 +250,7 @@ func TestParseExampleConfig(t *testing.T) { GoEnv: "development", LogLevel: "debug", GoBinary: "go", + GoProxy: "direct", GoGetWorkers: 10, ProtocolWorkers: 30, CloudRuntime: "none", @@ -291,6 +293,7 @@ func getEnvMap(config *Config) map[string]string { envVars := map[string]string{ "GO_ENV": config.GoEnv, "GO_BINARY_PATH": config.GoBinary, + "GOPROXY": config.GoProxy, "ATHENS_GOGET_WORKERS": strconv.Itoa(config.GoGetWorkers), "ATHENS_PROTOCOL_WORKERS": strconv.Itoa(config.ProtocolWorkers), "ATHENS_LOG_LEVEL": config.LogLevel, diff --git a/pkg/download/protocol_test.go b/pkg/download/protocol_test.go index 438aec0d..74ad937f 100644 --- a/pkg/download/protocol_test.go +++ b/pkg/download/protocol_test.go @@ -35,8 +35,9 @@ func getDP(t *testing.T) Protocol { t.Fatalf("Unable to parse config file: %s", err.Error()) } goBin := conf.GoBinary + goProxy := conf.GoProxy fs := afero.NewOsFs() - mf, err := module.NewGoGetFetcher(goBin, fs) + mf, err := module.NewGoGetFetcher(goBin, goProxy, fs) if err != nil { t.Fatal(err) } @@ -45,7 +46,7 @@ func getDP(t *testing.T) Protocol { t.Fatal(err) } st := stash.New(mf, s) - return New(&Opts{s, st, NewVCSLister(goBin, fs), nil}) + return New(&Opts{s, st, NewVCSLister(goBin, goProxy, fs), nil}) } type listTest struct { diff --git a/pkg/download/upstream_lister.go b/pkg/download/upstream_lister.go index 23f7e13b..7d625647 100644 --- a/pkg/download/upstream_lister.go +++ b/pkg/download/upstream_lister.go @@ -31,6 +31,7 @@ type listResp struct { type vcsLister struct { goBinPath string + goProxy string fs afero.Fs } @@ -61,7 +62,7 @@ func (l *vcsLister) List(ctx context.Context, mod string) (*storage.RevInfo, []s return nil, nil, errors.E(op, err) } defer module.ClearFiles(l.fs, gopath) - cmd.Env = module.PrepareEnv(gopath) + cmd.Env = module.PrepareEnv(gopath, l.goProxy) err = cmd.Run() if err != nil { @@ -89,6 +90,6 @@ func (l *vcsLister) List(ctx context.Context, mod string) (*storage.RevInfo, []s } // NewVCSLister creates an UpstreamLister which uses VCS to fetch a list of available versions -func NewVCSLister(goBinPath string, fs afero.Fs) UpstreamLister { - return &vcsLister{goBinPath: goBinPath, fs: fs} +func NewVCSLister(goBinPath, goProxy string, fs afero.Fs) UpstreamLister { + return &vcsLister{goBinPath: goBinPath, goProxy: goProxy, fs: fs} } diff --git a/pkg/module/all_test.go b/pkg/module/all_test.go index 57415fdd..56354f93 100644 --- a/pkg/module/all_test.go +++ b/pkg/module/all_test.go @@ -19,6 +19,7 @@ type ModuleSuite struct { suite.Suite fs afero.Fs goBinaryName string + goProxy string } func (m *ModuleSuite) SetupTest() { @@ -27,5 +28,5 @@ func (m *ModuleSuite) SetupTest() { func TestModules(t *testing.T) { goBinaryPath := envy.Get("GO_BINARY_PATH", "go") - suite.Run(t, &ModuleSuite{goBinaryName: goBinaryPath}) + suite.Run(t, &ModuleSuite{goBinaryName: goBinaryPath, goProxy: "direct"}) } diff --git a/pkg/module/go_get_fetcher.go b/pkg/module/go_get_fetcher.go index 6d6e1c7a..4f83baf7 100644 --- a/pkg/module/go_get_fetcher.go +++ b/pkg/module/go_get_fetcher.go @@ -20,6 +20,7 @@ import ( type goGetFetcher struct { fs afero.Fs goBinaryName string + goProxy string } type goModule struct { @@ -35,7 +36,7 @@ type goModule struct { } // NewGoGetFetcher creates fetcher which uses go get tool to fetch modules -func NewGoGetFetcher(goBinaryName string, fs afero.Fs) (Fetcher, error) { +func NewGoGetFetcher(goBinaryName string, goProxy string, fs afero.Fs) (Fetcher, error) { const op errors.Op = "module.NewGoGetFetcher" if err := validGoBinary(goBinaryName); err != nil { return nil, errors.E(op, err) @@ -43,6 +44,7 @@ func NewGoGetFetcher(goBinaryName string, fs afero.Fs) (Fetcher, error) { return &goGetFetcher{ fs: fs, goBinaryName: goBinaryName, + goProxy: goProxy, }, nil } @@ -65,7 +67,7 @@ func (g *goGetFetcher) Fetch(ctx context.Context, mod, ver string) (*storage.Ver return nil, errors.E(op, err) } - m, err := downloadModule(g.goBinaryName, g.fs, goPathRoot, modPath, mod, ver) + m, err := downloadModule(g.goBinaryName, g.goProxy, g.fs, goPathRoot, modPath, mod, ver) if err != nil { ClearFiles(g.fs, goPathRoot) return nil, errors.E(op, err) @@ -100,13 +102,13 @@ func (g *goGetFetcher) Fetch(ctx context.Context, mod, ver string) (*storage.Ver // given a filesystem, gopath, repository root, module and version, runs 'go mod download -json' // on module@version from the repoRoot with GOPATH=gopath, and returns a non-nil error if anything went wrong. -func downloadModule(goBinaryName string, fs afero.Fs, gopath, repoRoot, module, version string) (goModule, error) { +func downloadModule(goBinaryName, goProxy string, fs afero.Fs, gopath, repoRoot, module, version string) (goModule, error) { const op errors.Op = "module.downloadModule" uri := strings.TrimSuffix(module, "/") fullURI := fmt.Sprintf("%s@%s", uri, version) cmd := exec.Command(goBinaryName, "mod", "download", "-json", fullURI) - cmd.Env = PrepareEnv(gopath) + cmd.Env = PrepareEnv(gopath, goProxy) cmd.Dir = repoRoot stdout := &bytes.Buffer{} stderr := &bytes.Buffer{} @@ -140,7 +142,7 @@ func downloadModule(goBinaryName string, fs afero.Fs, gopath, repoRoot, module, // PrepareEnv will return all the appropriate // environment variables for a Go Command to run // successfully (such as GOPATH, GOCACHE, PATH etc) -func PrepareEnv(gopath string) []string { +func PrepareEnv(gopath, goProxy string) []string { pathEnv := fmt.Sprintf("PATH=%s", os.Getenv("PATH")) homeEnv := fmt.Sprintf("HOME=%s", os.Getenv("HOME")) httpProxy := fmt.Sprintf("HTTP_PROXY=%s", os.Getenv("HTTP_PROXY")) @@ -151,6 +153,7 @@ func PrepareEnv(gopath string) []string { httpsProxyLower := fmt.Sprintf("https_proxy=%s", os.Getenv("https_proxy")) noProxyLower := fmt.Sprintf("no_proxy=%s", os.Getenv("no_proxy")) gopathEnv := fmt.Sprintf("GOPATH=%s", gopath) + goProxyEnv := fmt.Sprintf("GOPROXY=%s", goProxy) cacheEnv := fmt.Sprintf("GOCACHE=%s", filepath.Join(gopath, "cache")) gitSSH := fmt.Sprintf("GIT_SSH=%s", os.Getenv("GIT_SSH")) gitSSHCmd := fmt.Sprintf("GIT_SSH_COMMAND=%s", os.Getenv("GIT_SSH_COMMAND")) @@ -160,6 +163,7 @@ func PrepareEnv(gopath string) []string { pathEnv, homeEnv, gopathEnv, + goProxyEnv, cacheEnv, disableCgo, enableGoModules, diff --git a/pkg/module/go_get_fetcher_test.go b/pkg/module/go_get_fetcher_test.go index 5ccee759..5cdb5f43 100644 --- a/pkg/module/go_get_fetcher_test.go +++ b/pkg/module/go_get_fetcher_test.go @@ -14,14 +14,14 @@ var ctx = context.Background() func (s *ModuleSuite) TestNewGoGetFetcher() { r := s.Require() - fetcher, err := NewGoGetFetcher(s.goBinaryName, s.fs) + fetcher, err := NewGoGetFetcher(s.goBinaryName, s.goProxy, s.fs) r.NoError(err) _, ok := fetcher.(*goGetFetcher) r.True(ok) } func (s *ModuleSuite) TestGoGetFetcherError() { - fetcher, err := NewGoGetFetcher("invalidpath", afero.NewOsFs()) + fetcher, err := NewGoGetFetcher("invalidpath", "", afero.NewOsFs()) assert.Nil(s.T(), fetcher) if runtime.GOOS == "windows" { @@ -35,7 +35,7 @@ func (s *ModuleSuite) TestGoGetFetcherFetch() { r := s.Require() // we need to use an OS filesystem because fetch executes vgo on the command line, which // always writes to the filesystem - fetcher, err := NewGoGetFetcher(s.goBinaryName, afero.NewOsFs()) + fetcher, err := NewGoGetFetcher(s.goBinaryName, s.goProxy, afero.NewOsFs()) r.NoError(err) ver, err := fetcher.Fetch(ctx, repoURI, version) r.NoError(err) @@ -55,7 +55,7 @@ func (s *ModuleSuite) TestGoGetFetcherFetch() { func (s *ModuleSuite) TestNotFoundFetches() { r := s.Require() - fetcher, err := NewGoGetFetcher(s.goBinaryName, afero.NewOsFs()) + fetcher, err := NewGoGetFetcher(s.goBinaryName, s.goProxy, afero.NewOsFs()) r.NoError(err) // when someone buys laks47dfjoijskdvjxuyyd.com, and implements // a git server on top of it, this test will fail :)