Allow users to specify the module fetcher's temporary directory (#1636)

cmd/proxy/actions/app_proxy.go

@@ -96,7 +96,7 @@ func addProxyRoutes(
 	if err := c.GoBinaryEnvVars.Validate(); err != nil {
 		return err
 	}
-	mf, err := module.NewGoGetFetcher(c.GoBinary, c.GoBinaryEnvVars, fs)
+	mf, err := module.NewGoGetFetcher(c.GoBinary, c.GoGetDir, c.GoBinaryEnvVars, fs)
 	if err != nil {
 		return err
 	}

config.dev.toml

@@ -62,6 +62,16 @@ GoBinaryEnvVars = ["GOPROXY=direct"]
 # Env override: ATHENS_GOGET_WORKERS
 GoGetWorkers = 10
 
+# GoGetDir specifies the temporary directory that Athens
+# will use to fetch modules from VCS prior to persisting
+# them to a storage backend. This is useful if you are in a
+# Kubernetes environment where a specific path is volumed into 
+# a directory that has larger disk resources. If the value is
+# empty, Athens will use the default OS temporary directory.
+# 
+# Env override: ATHENS_GOGOET_DIR
+GoGetDir = ""
+
 # ProtocolWorkers specifies how many concurrent
 # requests can you handle at a time for all
 # download protocol paths. This is different from

pkg/config/config.go

@@ -26,6 +26,7 @@ type Config struct {
 	GoProxy          string    `envconfig:"GOPROXY"`
 	GoBinaryEnvVars  EnvList   `envconfig:"ATHENS_GO_BINARY_ENV_VARS"`
 	GoGetWorkers     int       `validate:"required" envconfig:"ATHENS_GOGET_WORKERS"`
+	GoGetDir         string    `envconfig:"ATHENS_GOGOET_DIR"`
 	ProtocolWorkers  int       `validate:"required" envconfig:"ATHENS_PROTOCOL_WORKERS"`
 	LogLevel         string    `validate:"required" envconfig:"ATHENS_LOG_LEVEL"`
 	CloudRuntime     string    `validate:"required" envconfig:"ATHENS_CLOUD_RUNTIME"`

pkg/download/protocol_test.go

@@ -37,7 +37,7 @@ func getDP(t *testing.T) Protocol {
 	}
 	goBin := conf.GoBinary
 	fs := afero.NewOsFs()
-	mf, err := module.NewGoGetFetcher(goBin, conf.GoBinaryEnvVars, fs)
+	mf, err := module.NewGoGetFetcher(goBin, conf.GoGetDir, conf.GoBinaryEnvVars, fs)
 	if err != nil {
 		t.Fatal(err)
 	}

pkg/module/go_get_fetcher.go

@@ -20,6 +20,7 @@ type goGetFetcher struct {
 	fs           afero.Fs
 	goBinaryName string
 	envVars      []string
+	gogetDir     string
 }
 
 type goModule struct {
@@ -35,7 +36,7 @@ type goModule struct {
 }
 
 // NewGoGetFetcher creates fetcher which uses go get tool to fetch modules
-func NewGoGetFetcher(goBinaryName string, envVars []string, fs afero.Fs) (Fetcher, error) {
+func NewGoGetFetcher(goBinaryName, gogetDir string, envVars []string, fs afero.Fs) (Fetcher, error) {
 	const op errors.Op = "module.NewGoGetFetcher"
 	if err := validGoBinary(goBinaryName); err != nil {
 		return nil, errors.E(op, err)
@@ -44,6 +45,7 @@ func NewGoGetFetcher(goBinaryName string, envVars []string, fs afero.Fs) (Fetche
 		fs:           fs,
 		goBinaryName: goBinaryName,
 		envVars:      envVars,
+		gogetDir:     gogetDir,
 	}, nil
 }
 
@@ -55,7 +57,7 @@ func (g *goGetFetcher) Fetch(ctx context.Context, mod, ver string) (*storage.Ver
 	defer span.End()
 
 	// setup the GOPATH
-	goPathRoot, err := afero.TempDir(g.fs, "", "athens")
+	goPathRoot, err := afero.TempDir(g.fs, g.gogetDir, "athens")
 	if err != nil {
 		return nil, errors.E(op, err)
 	}

pkg/module/go_get_fetcher_test.go

@@ -17,14 +17,14 @@ var ctx = context.Background()
 
 func (s *ModuleSuite) TestNewGoGetFetcher() {
 	r := s.Require()
-	fetcher, err := NewGoGetFetcher(s.goBinaryName, s.env, s.fs)
+	fetcher, err := NewGoGetFetcher(s.goBinaryName, "", s.env, s.fs)
 	r.NoError(err)
 	_, ok := fetcher.(*goGetFetcher)
 	r.True(ok)
 }
 
 func (s *ModuleSuite) TestGoGetFetcherError() {
-	fetcher, err := NewGoGetFetcher("invalidpath", s.env, afero.NewOsFs())
+	fetcher, err := NewGoGetFetcher("invalidpath", "", s.env, afero.NewOsFs())
 
 	assert.Nil(s.T(), fetcher)
 	if runtime.GOOS == "windows" {
@@ -38,7 +38,7 @@ func (s *ModuleSuite) TestGoGetFetcherFetch() {
 	r := s.Require()
 	// we need to use an OS filesystem because fetch executes vgo on the command line, which
 	// always writes to the filesystem
-	fetcher, err := NewGoGetFetcher(s.goBinaryName, s.env, afero.NewOsFs())
+	fetcher, err := NewGoGetFetcher(s.goBinaryName, "", s.env, afero.NewOsFs())
 	r.NoError(err)
 	ver, err := fetcher.Fetch(ctx, repoURI, version)
 	r.NoError(err)
@@ -58,7 +58,7 @@ func (s *ModuleSuite) TestGoGetFetcherFetch() {
 
 func (s *ModuleSuite) TestNotFoundFetches() {
 	r := s.Require()
-	fetcher, err := NewGoGetFetcher(s.goBinaryName, s.env, afero.NewOsFs())
+	fetcher, err := NewGoGetFetcher(s.goBinaryName, "", s.env, afero.NewOsFs())
 	r.NoError(err)
 	// when someone buys laks47dfjoijskdvjxuyyd.com, and implements
 	// a git server on top of it, this test will fail :)
@@ -86,18 +86,41 @@ func (s *ModuleSuite) TestGoGetFetcherSumDB() {
 	proxyAddr, close := s.getProxy(mp)
 	defer close()
 
-	fetcher, err := NewGoGetFetcher(s.goBinaryName, []string{"GOPROXY=" + proxyAddr}, afero.NewOsFs())
+	fetcher, err := NewGoGetFetcher(s.goBinaryName, "", []string{"GOPROXY=" + proxyAddr}, afero.NewOsFs())
 	r.NoError(err)
 	_, err = fetcher.Fetch(ctx, "mockmod.xyz", "v1.2.3")
 	if err == nil {
 		s.T().Fatal("expected a gosum error but got nil")
 	}
-	fetcher, err = NewGoGetFetcher(s.goBinaryName, []string{"GONOSUMDB=mockmod.xyz", "GOPROXY=" + proxyAddr}, afero.NewOsFs())
+	fetcher, err = NewGoGetFetcher(s.goBinaryName, "", []string{"GONOSUMDB=mockmod.xyz", "GOPROXY=" + proxyAddr}, afero.NewOsFs())
 	r.NoError(err)
 	_, err = fetcher.Fetch(ctx, "mockmod.xyz", "v1.2.3")
 	r.NoError(err, "expected the go sum to not be consulted but got an error")
 }
 
+func (s *ModuleSuite) TestGoGetDir() {
+	r := s.Require()
+	t := s.T()
+	dir, err := ioutil.TempDir("", "nested")
+	r.NoError(err)
+	t.Cleanup(func() {
+		os.RemoveAll(dir)
+	})
+	fetcher, err := NewGoGetFetcher(s.goBinaryName, dir, s.env, afero.NewOsFs())
+	r.NoError(err)
+
+	ver, err := fetcher.Fetch(ctx, repoURI, version)
+	r.NoError(err)
+	defer ver.Zip.Close()
+
+	dirInfo, err := ioutil.ReadDir(dir)
+	r.NoError(err)
+
+	if len(dirInfo) <= 0 {
+		t.Fatalf("expected the directory %q to have eat least one sub directory but it was empty", dir)
+	}
+}
+
 func (s *ModuleSuite) getProxy(h http.Handler) (addr string, close func()) {
 	srv := httptest.NewServer(h)
 	return srv.URL, srv.Close