diff --git a/cmd/proxy/actions/app_proxy.go b/cmd/proxy/actions/app_proxy.go index 3d1e3b7f..99e8fecb 100644 --- a/cmd/proxy/actions/app_proxy.go +++ b/cmd/proxy/actions/app_proxy.go @@ -96,7 +96,7 @@ func addProxyRoutes( if err := c.GoBinaryEnvVars.Validate(); err != nil { return err } - mf, err := module.NewGoGetFetcher(c.GoBinary, c.GoBinaryEnvVars, fs) + mf, err := module.NewGoGetFetcher(c.GoBinary, c.GoGetDir, c.GoBinaryEnvVars, fs) if err != nil { return err } diff --git a/config.dev.toml b/config.dev.toml index cf579ecb..426f0b88 100755 --- a/config.dev.toml +++ b/config.dev.toml @@ -62,6 +62,16 @@ GoBinaryEnvVars = ["GOPROXY=direct"] # Env override: ATHENS_GOGET_WORKERS GoGetWorkers = 10 +# GoGetDir specifies the temporary directory that Athens +# will use to fetch modules from VCS prior to persisting +# them to a storage backend. This is useful if you are in a +# Kubernetes environment where a specific path is volumed into +# a directory that has larger disk resources. If the value is +# empty, Athens will use the default OS temporary directory. +# +# Env override: ATHENS_GOGOET_DIR +GoGetDir = "" + # ProtocolWorkers specifies how many concurrent # requests can you handle at a time for all # download protocol paths. This is different from diff --git a/pkg/config/config.go b/pkg/config/config.go index 108c0a6a..656ff597 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -26,6 +26,7 @@ type Config struct { GoProxy string `envconfig:"GOPROXY"` GoBinaryEnvVars EnvList `envconfig:"ATHENS_GO_BINARY_ENV_VARS"` GoGetWorkers int `validate:"required" envconfig:"ATHENS_GOGET_WORKERS"` + GoGetDir string `envconfig:"ATHENS_GOGOET_DIR"` ProtocolWorkers int `validate:"required" envconfig:"ATHENS_PROTOCOL_WORKERS"` LogLevel string `validate:"required" envconfig:"ATHENS_LOG_LEVEL"` CloudRuntime string `validate:"required" envconfig:"ATHENS_CLOUD_RUNTIME"` diff --git a/pkg/download/protocol_test.go b/pkg/download/protocol_test.go index 3c307190..5703f6b8 100644 --- a/pkg/download/protocol_test.go +++ b/pkg/download/protocol_test.go @@ -37,7 +37,7 @@ func getDP(t *testing.T) Protocol { } goBin := conf.GoBinary fs := afero.NewOsFs() - mf, err := module.NewGoGetFetcher(goBin, conf.GoBinaryEnvVars, fs) + mf, err := module.NewGoGetFetcher(goBin, conf.GoGetDir, conf.GoBinaryEnvVars, fs) if err != nil { t.Fatal(err) } diff --git a/pkg/module/go_get_fetcher.go b/pkg/module/go_get_fetcher.go index 3f92a020..9055ae1a 100644 --- a/pkg/module/go_get_fetcher.go +++ b/pkg/module/go_get_fetcher.go @@ -20,6 +20,7 @@ type goGetFetcher struct { fs afero.Fs goBinaryName string envVars []string + gogetDir string } type goModule struct { @@ -35,7 +36,7 @@ type goModule struct { } // NewGoGetFetcher creates fetcher which uses go get tool to fetch modules -func NewGoGetFetcher(goBinaryName string, envVars []string, fs afero.Fs) (Fetcher, error) { +func NewGoGetFetcher(goBinaryName, gogetDir string, envVars []string, fs afero.Fs) (Fetcher, error) { const op errors.Op = "module.NewGoGetFetcher" if err := validGoBinary(goBinaryName); err != nil { return nil, errors.E(op, err) @@ -44,6 +45,7 @@ func NewGoGetFetcher(goBinaryName string, envVars []string, fs afero.Fs) (Fetche fs: fs, goBinaryName: goBinaryName, envVars: envVars, + gogetDir: gogetDir, }, nil } @@ -55,7 +57,7 @@ func (g *goGetFetcher) Fetch(ctx context.Context, mod, ver string) (*storage.Ver defer span.End() // setup the GOPATH - goPathRoot, err := afero.TempDir(g.fs, "", "athens") + goPathRoot, err := afero.TempDir(g.fs, g.gogetDir, "athens") if err != nil { return nil, errors.E(op, err) } diff --git a/pkg/module/go_get_fetcher_test.go b/pkg/module/go_get_fetcher_test.go index 22009b7e..37273c17 100644 --- a/pkg/module/go_get_fetcher_test.go +++ b/pkg/module/go_get_fetcher_test.go @@ -17,14 +17,14 @@ var ctx = context.Background() func (s *ModuleSuite) TestNewGoGetFetcher() { r := s.Require() - fetcher, err := NewGoGetFetcher(s.goBinaryName, s.env, s.fs) + fetcher, err := NewGoGetFetcher(s.goBinaryName, "", s.env, s.fs) r.NoError(err) _, ok := fetcher.(*goGetFetcher) r.True(ok) } func (s *ModuleSuite) TestGoGetFetcherError() { - fetcher, err := NewGoGetFetcher("invalidpath", s.env, afero.NewOsFs()) + fetcher, err := NewGoGetFetcher("invalidpath", "", s.env, afero.NewOsFs()) assert.Nil(s.T(), fetcher) if runtime.GOOS == "windows" { @@ -38,7 +38,7 @@ func (s *ModuleSuite) TestGoGetFetcherFetch() { r := s.Require() // we need to use an OS filesystem because fetch executes vgo on the command line, which // always writes to the filesystem - fetcher, err := NewGoGetFetcher(s.goBinaryName, s.env, afero.NewOsFs()) + fetcher, err := NewGoGetFetcher(s.goBinaryName, "", s.env, afero.NewOsFs()) r.NoError(err) ver, err := fetcher.Fetch(ctx, repoURI, version) r.NoError(err) @@ -58,7 +58,7 @@ func (s *ModuleSuite) TestGoGetFetcherFetch() { func (s *ModuleSuite) TestNotFoundFetches() { r := s.Require() - fetcher, err := NewGoGetFetcher(s.goBinaryName, s.env, afero.NewOsFs()) + fetcher, err := NewGoGetFetcher(s.goBinaryName, "", s.env, afero.NewOsFs()) r.NoError(err) // when someone buys laks47dfjoijskdvjxuyyd.com, and implements // a git server on top of it, this test will fail :) @@ -86,18 +86,41 @@ func (s *ModuleSuite) TestGoGetFetcherSumDB() { proxyAddr, close := s.getProxy(mp) defer close() - fetcher, err := NewGoGetFetcher(s.goBinaryName, []string{"GOPROXY=" + proxyAddr}, afero.NewOsFs()) + fetcher, err := NewGoGetFetcher(s.goBinaryName, "", []string{"GOPROXY=" + proxyAddr}, afero.NewOsFs()) r.NoError(err) _, err = fetcher.Fetch(ctx, "mockmod.xyz", "v1.2.3") if err == nil { s.T().Fatal("expected a gosum error but got nil") } - fetcher, err = NewGoGetFetcher(s.goBinaryName, []string{"GONOSUMDB=mockmod.xyz", "GOPROXY=" + proxyAddr}, afero.NewOsFs()) + fetcher, err = NewGoGetFetcher(s.goBinaryName, "", []string{"GONOSUMDB=mockmod.xyz", "GOPROXY=" + proxyAddr}, afero.NewOsFs()) r.NoError(err) _, err = fetcher.Fetch(ctx, "mockmod.xyz", "v1.2.3") r.NoError(err, "expected the go sum to not be consulted but got an error") } +func (s *ModuleSuite) TestGoGetDir() { + r := s.Require() + t := s.T() + dir, err := ioutil.TempDir("", "nested") + r.NoError(err) + t.Cleanup(func() { + os.RemoveAll(dir) + }) + fetcher, err := NewGoGetFetcher(s.goBinaryName, dir, s.env, afero.NewOsFs()) + r.NoError(err) + + ver, err := fetcher.Fetch(ctx, repoURI, version) + r.NoError(err) + defer ver.Zip.Close() + + dirInfo, err := ioutil.ReadDir(dir) + r.NoError(err) + + if len(dirInfo) <= 0 { + t.Fatalf("expected the directory %q to have eat least one sub directory but it was empty", dir) + } +} + func (s *ModuleSuite) getProxy(h http.Handler) (addr string, close func()) { srv := httptest.NewServer(h) return srv.URL, srv.Close