primal/athens
1
0
Fork 0
mirror of https://github.com/gomods/athens synced 2026-02-03 12:10:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

support goproxy for list/download command (#1304)

Browse Source
This commit is contained in:
Yi Tang
2019-07-16 02:07:24 +08:00
committed by Marwan Sulaiman
parent 8e8e886f84
commit 7de77889ad
9 changed files with 36 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/proxy/actions/app_proxy.go
+2 -2
View File
@@ -73,12 +73,12 @@ func addProxyRoutes(
// 3. The stashpool manages limiting concurrent requests and passes them to stash.
// 4. The plain stash.New just takes a request from upstream and saves it into storage.
fs := afero.NewOsFs()
mf, err := module.NewGoGetFetcher(c.GoBinary, fs)
mf, err := module.NewGoGetFetcher(c.GoBinary, c.GoProxy, fs)
if err != nil {
return err
}
lister := download.NewVCSLister(c.GoBinary, fs)
lister := download.NewVCSLister(c.GoBinary, c.GoProxy, fs)
withSingleFlight, err := getSingleFlight(c, s)
if err != nil {
config.dev.toml
+7
View File
@@ -15,6 +15,13 @@ GoBinary = "go"
# Env override: GO_ENV
GoEnv = "development"
# GoProxy specifies GOPROXY env for go list or mod download inside athens
# which can be configured totally same with GOPROXY of Go Command.
# Notes that the comma-separated GOPROXY (e.g. <proxy1>,<proxy2>,direct) is only available in Go 1.13 or higher,
# otherwise only single proxy URL can be set.
# Env override: GOPROXY
GoProxy = "direct"
# GoGetWorkers specifies how many times you can concurrently
# go mod download, this is so that low performance instances
# can manage go get more sanely and not run out of disk or memory.
pkg/config/config.go
+2
View File
@@ -21,6 +21,7 @@ type Config struct {
TimeoutConf
GoEnv string `validate:"required" envconfig:"GO_ENV"`
GoBinary string `validate:"required" envconfig:"GO_BINARY_PATH"`
GoProxy string `envconfig:"GOPROXY"`
GoGetWorkers int `validate:"required" envconfig:"ATHENS_GOGET_WORKERS"`
ProtocolWorkers int `validate:"required" envconfig:"ATHENS_PROTOCOL_WORKERS"`
LogLevel string `validate:"required" envconfig:"ATHENS_LOG_LEVEL"`
@@ -76,6 +77,7 @@ func defaultConfig() *Config {
return &Config{
GoBinary: "go",
GoEnv: "development",
GoProxy: "direct",
GoGetWorkers: 10,
ProtocolWorkers: 30,
LogLevel: "debug",
pkg/config/config_test.go
+3
View File
@@ -71,6 +71,7 @@ func TestEnvOverrides(t *testing.T) {
ProtocolWorkers: 10,
LogLevel: "info",
GoBinary: "go11",
GoProxy: "direct",
CloudRuntime: "gcp",
TimeoutConf: TimeoutConf{
Timeout: 30,
@@ -249,6 +250,7 @@ func TestParseExampleConfig(t *testing.T) {
GoEnv: "development",
LogLevel: "debug",
GoBinary: "go",
GoProxy: "direct",
GoGetWorkers: 10,
ProtocolWorkers: 30,
CloudRuntime: "none",
@@ -291,6 +293,7 @@ func getEnvMap(config *Config) map[string]string {
envVars := map[string]string{
"GO_ENV": config.GoEnv,
"GO_BINARY_PATH": config.GoBinary,
"GOPROXY": config.GoProxy,
"ATHENS_GOGET_WORKERS": strconv.Itoa(config.GoGetWorkers),
"ATHENS_PROTOCOL_WORKERS": strconv.Itoa(config.ProtocolWorkers),
"ATHENS_LOG_LEVEL": config.LogLevel,
pkg/download/protocol_test.go
+3 -2
View File
@@ -35,8 +35,9 @@ func getDP(t *testing.T) Protocol {
t.Fatalf("Unable to parse config file: %s", err.Error())
}
goBin := conf.GoBinary
goProxy := conf.GoProxy
fs := afero.NewOsFs()
mf, err := module.NewGoGetFetcher(goBin, fs)
mf, err := module.NewGoGetFetcher(goBin, goProxy, fs)
if err != nil {
t.Fatal(err)
}
@@ -45,7 +46,7 @@ func getDP(t *testing.T) Protocol {
t.Fatal(err)
}
st := stash.New(mf, s)
return New(&Opts{s, st, NewVCSLister(goBin, fs), nil})
return New(&Opts{s, st, NewVCSLister(goBin, goProxy, fs), nil})
}
type listTest struct {
pkg/download/upstream_lister.go
+4 -3
View File
@@ -31,6 +31,7 @@ type listResp struct {
type vcsLister struct {
goBinPath string
goProxy string
fs afero.Fs
}
@@ -61,7 +62,7 @@ func (l *vcsLister) List(ctx context.Context, mod string) (*storage.RevInfo, []s
return nil, nil, errors.E(op, err)
}
defer module.ClearFiles(l.fs, gopath)
cmd.Env = module.PrepareEnv(gopath)
cmd.Env = module.PrepareEnv(gopath, l.goProxy)
err = cmd.Run()
if err != nil {
@@ -89,6 +90,6 @@ func (l *vcsLister) List(ctx context.Context, mod string) (*storage.RevInfo, []s
}
// NewVCSLister creates an UpstreamLister which uses VCS to fetch a list of available versions
func NewVCSLister(goBinPath string, fs afero.Fs) UpstreamLister {
return &vcsLister{goBinPath: goBinPath, fs: fs}
func NewVCSLister(goBinPath, goProxy string, fs afero.Fs) UpstreamLister {
return &vcsLister{goBinPath: goBinPath, goProxy: goProxy, fs: fs}
}
pkg/module/all_test.go
+2 -1
View File
@@ -19,6 +19,7 @@ type ModuleSuite struct {
suite.Suite
fs afero.Fs
goBinaryName string
goProxy string
}
func (m *ModuleSuite) SetupTest() {
@@ -27,5 +28,5 @@ func (m *ModuleSuite) SetupTest() {
func TestModules(t *testing.T) {
goBinaryPath := envy.Get("GO_BINARY_PATH", "go")
suite.Run(t, &ModuleSuite{goBinaryName: goBinaryPath})
suite.Run(t, &ModuleSuite{goBinaryName: goBinaryPath, goProxy: "direct"})
}
pkg/module/go_get_fetcher.go
+9 -5
View File
@@ -20,6 +20,7 @@ import (
type goGetFetcher struct {
fs afero.Fs
goBinaryName string
goProxy string
}
type goModule struct {
@@ -35,7 +36,7 @@ type goModule struct {
}
// NewGoGetFetcher creates fetcher which uses go get tool to fetch modules
func NewGoGetFetcher(goBinaryName string, fs afero.Fs) (Fetcher, error) {
func NewGoGetFetcher(goBinaryName string, goProxy string, fs afero.Fs) (Fetcher, error) {
const op errors.Op = "module.NewGoGetFetcher"
if err := validGoBinary(goBinaryName); err != nil {
return nil, errors.E(op, err)
@@ -43,6 +44,7 @@ func NewGoGetFetcher(goBinaryName string, fs afero.Fs) (Fetcher, error) {
return &goGetFetcher{
fs: fs,
goBinaryName: goBinaryName,
goProxy: goProxy,
}, nil
}
@@ -65,7 +67,7 @@ func (g *goGetFetcher) Fetch(ctx context.Context, mod, ver string) (*storage.Ver
return nil, errors.E(op, err)
}
m, err := downloadModule(g.goBinaryName, g.fs, goPathRoot, modPath, mod, ver)
m, err := downloadModule(g.goBinaryName, g.goProxy, g.fs, goPathRoot, modPath, mod, ver)
if err != nil {
ClearFiles(g.fs, goPathRoot)
return nil, errors.E(op, err)
@@ -100,13 +102,13 @@ func (g *goGetFetcher) Fetch(ctx context.Context, mod, ver string) (*storage.Ver
// given a filesystem, gopath, repository root, module and version, runs 'go mod download -json'
// on module@version from the repoRoot with GOPATH=gopath, and returns a non-nil error if anything went wrong.
func downloadModule(goBinaryName string, fs afero.Fs, gopath, repoRoot, module, version string) (goModule, error) {
func downloadModule(goBinaryName, goProxy string, fs afero.Fs, gopath, repoRoot, module, version string) (goModule, error) {
const op errors.Op = "module.downloadModule"
uri := strings.TrimSuffix(module, "/")
fullURI := fmt.Sprintf("%s@%s", uri, version)
cmd := exec.Command(goBinaryName, "mod", "download", "-json", fullURI)
cmd.Env = PrepareEnv(gopath)
cmd.Env = PrepareEnv(gopath, goProxy)
cmd.Dir = repoRoot
stdout := &bytes.Buffer{}
stderr := &bytes.Buffer{}
@@ -140,7 +142,7 @@ func downloadModule(goBinaryName string, fs afero.Fs, gopath, repoRoot, module,
// PrepareEnv will return all the appropriate
// environment variables for a Go Command to run
// successfully (such as GOPATH, GOCACHE, PATH etc)
func PrepareEnv(gopath string) []string {
func PrepareEnv(gopath, goProxy string) []string {
pathEnv := fmt.Sprintf("PATH=%s", os.Getenv("PATH"))
homeEnv := fmt.Sprintf("HOME=%s", os.Getenv("HOME"))
httpProxy := fmt.Sprintf("HTTP_PROXY=%s", os.Getenv("HTTP_PROXY"))
@@ -151,6 +153,7 @@ func PrepareEnv(gopath string) []string {
httpsProxyLower := fmt.Sprintf("https_proxy=%s", os.Getenv("https_proxy"))
noProxyLower := fmt.Sprintf("no_proxy=%s", os.Getenv("no_proxy"))
gopathEnv := fmt.Sprintf("GOPATH=%s", gopath)
goProxyEnv := fmt.Sprintf("GOPROXY=%s", goProxy)
cacheEnv := fmt.Sprintf("GOCACHE=%s", filepath.Join(gopath, "cache"))
gitSSH := fmt.Sprintf("GIT_SSH=%s", os.Getenv("GIT_SSH"))
gitSSHCmd := fmt.Sprintf("GIT_SSH_COMMAND=%s", os.Getenv("GIT_SSH_COMMAND"))
@@ -160,6 +163,7 @@ func PrepareEnv(gopath string) []string {
pathEnv,
homeEnv,
gopathEnv,
goProxyEnv,
cacheEnv,
disableCgo,
enableGoModules,
pkg/module/go_get_fetcher_test.go
+4 -4
View File
@@ -14,14 +14,14 @@ var ctx = context.Background()
func (s *ModuleSuite) TestNewGoGetFetcher() {
r := s.Require()
fetcher, err := NewGoGetFetcher(s.goBinaryName, s.fs)
fetcher, err := NewGoGetFetcher(s.goBinaryName, s.goProxy, s.fs)
r.NoError(err)
_, ok := fetcher.(*goGetFetcher)
r.True(ok)
}
func (s *ModuleSuite) TestGoGetFetcherError() {
fetcher, err := NewGoGetFetcher("invalidpath", afero.NewOsFs())
fetcher, err := NewGoGetFetcher("invalidpath", "", afero.NewOsFs())
assert.Nil(s.T(), fetcher)
if runtime.GOOS == "windows" {
@@ -35,7 +35,7 @@ func (s *ModuleSuite) TestGoGetFetcherFetch() {
r := s.Require()
// we need to use an OS filesystem because fetch executes vgo on the command line, which
// always writes to the filesystem
fetcher, err := NewGoGetFetcher(s.goBinaryName, afero.NewOsFs())
fetcher, err := NewGoGetFetcher(s.goBinaryName, s.goProxy, afero.NewOsFs())
r.NoError(err)
ver, err := fetcher.Fetch(ctx, repoURI, version)
r.NoError(err)
@@ -55,7 +55,7 @@ func (s *ModuleSuite) TestGoGetFetcherFetch() {
func (s *ModuleSuite) TestNotFoundFetches() {
r := s.Require()
fetcher, err := NewGoGetFetcher(s.goBinaryName, afero.NewOsFs())
fetcher, err := NewGoGetFetcher(s.goBinaryName, s.goProxy, afero.NewOsFs())
r.NoError(err)
// when someone buys laks47dfjoijskdvjxuyyd.com, and implements
// a git server on top of it, this test will fail :)