Moving the fetch logic into 'pkg/module' (#291)

* add module fetch and fetchers

* Breaking module ref out of the fetcher

* disk ref in progress

* very broken go get fetcher

* refactoring ref and fixing up the disk ref

* adding a test for the disk ref

* doing test setup right

* passing version

* fixing syntax err

* removing raw format string

in favor of a function that does the same thing

* removing obsolete godoc

* removing the switch in favor of nested if statements

hopefully this is easier to read

* passing data into the format strings

* refactoring the closer

* creating and looking for files in the root

* genericFetcher => goGetFetcher

* remove the clear method on go get fetcher

the ref returned from fetch does it

* remove unnecessary call to isVgoInstalled

calls to fetch will fail if vgo is not installed

* remove unneeded dirName

* adding preliminary test for the go get fetcher

* adding a test for fetch

* executing fetch test

* adding docs and simplifying the fetcher

* declaring the storage version up front

* fixing the source path

* creating test files in a separate function

* closing files immediately

instead of aggregating them for later. whaaat was I thinking???

* passing context into storage driver

* not closing the source file

* not putting the module name in quotes anymore

* adding a warning about deleting the package

* adding go binary name

* embedding github API limit error

* check files in the success path

* passing errors up

not just messages

* doh

* using vgo for the go get fetcher tests

* not using multierror because it's unneeded

* less code 😄

* custom error

* removing unused import

* return noop refs

* adding link to issue for the TODO

* simplifying

* simplifying moar!

* using config.FmtModVer

* simplifying

* defer closing the zip

* closing the zip

* adding comment about closing the ver.Zip

* GO_BINARY_NAME => GO_BINARY_PATH

* using errors pkg

* fixing compile err

* removing redundant args

* removing requirement that caller calls Clear on fetch error

* fixing build

* fix

* adding masterminds semver

* returning better error

and checking version format

* removing unused error

* using proper semver function

* fixin compile err

* rm that shizz

pkg/module/go_get_fetcher.go

@@ -0,0 +1,144 @@
+package module
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"github.com/gomods/athens/pkg/errors"
+	pkgerrors "github.com/pkg/errors"
+	"github.com/spf13/afero"
+)
+
+type goGetFetcher struct {
+	fs           afero.Fs
+	goBinaryName string
+}
+
+// NewGoGetFetcher creates fetcher which uses go get tool to fetch modules
+func NewGoGetFetcher(goBinaryName string, fs afero.Fs) (Fetcher, error) {
+	return &goGetFetcher{
+		fs:           fs,
+		goBinaryName: goBinaryName,
+	}, nil
+}
+
+// Fetch downloads the sources and returns path where it can be found. Make sure to call Clear
+// on the returned Ref when you are done with it
+func (g *goGetFetcher) Fetch(mod, ver string) (Ref, error) {
+	ref := noopRef{}
+
+	// setup the GOPATH
+	goPathRoot, err := afero.TempDir(g.fs, "", "athens")
+	if err != nil {
+		// TODO: return a ref for cleaning up the goPathRoot
+		// https://github.com/gomods/athens/issues/329
+		ref.Clear()
+		return ref, err
+	}
+	sourcePath := filepath.Join(goPathRoot, "src")
+	modPath := filepath.Join(sourcePath, getRepoDirName(mod, ver))
+	if err := g.fs.MkdirAll(modPath, os.ModeDir|os.ModePerm); err != nil {
+		// TODO: return a ref for cleaning up the goPathRoot
+		// https://github.com/gomods/athens/issues/329
+		ref.Clear()
+		return ref, err
+	}
+
+	// setup the module with barebones stuff
+	if err := prepareStructure(g.fs, modPath); err != nil {
+		// TODO: return a ref for cleaning up the goPathRoot
+		// https://github.com/gomods/athens/issues/329
+		ref.Clear()
+		return ref, err
+	}
+
+	cachePath, err := getSources(g.goBinaryName, g.fs, goPathRoot, modPath, mod, ver)
+	if err != nil {
+		// TODO: return a ref that cleans up the goPathRoot
+		// https://github.com/gomods/athens/issues/329
+		ref.Clear()
+		return nil, err
+	}
+	// TODO: make sure this ref also cleans up the goPathRoot
+	// https://github.com/gomods/athens/issues/329
+	return newDiskRef(g.fs, cachePath, ver), err
+}
+
+// Hacky thing makes vgo not to complain
+func prepareStructure(fs afero.Fs, repoRoot string) error {
+	// vgo expects go.mod file present with module statement or .go file with import comment
+	gomodPath := filepath.Join(repoRoot, "go.mod")
+	gomodContent := []byte("module mod")
+	if err := afero.WriteFile(fs, gomodPath, gomodContent, 0666); err != nil {
+		return err
+	}
+
+	sourcePath := filepath.Join(repoRoot, "mod.go")
+	sourceContent := []byte("package mod")
+	return afero.WriteFile(fs, sourcePath, sourceContent, 0666)
+}
+
+// given a filesystem, gopath, repository root, module and version, runs 'vgo get'
+// on module@version from the repoRoot with GOPATH=gopath, and returns the location
+// of the module cache. returns a non-nil error if anything went wrong. always returns
+// the location of the module cache so you can delete it if necessary
+func getSources(goBinaryName string, fs afero.Fs, gopath, repoRoot, module, version string) (string, error) {
+	uri := strings.TrimSuffix(module, "/")
+
+	fullURI := fmt.Sprintf("%s@%s", uri, version)
+
+	gopathEnv := fmt.Sprintf("GOPATH=%s", gopath)
+	cacheEnv := fmt.Sprintf("GOCACHE=%s", filepath.Join(gopath, "cache"))
+	disableCgo := "CGO_ENABLED=0"
+
+	cmd := exec.Command(goBinaryName, "get", fullURI)
+	// PATH is needed for vgo to recognize vcs binaries
+	// this breaks windows.
+	cmd.Env = []string{"PATH=" + os.Getenv("PATH"), gopathEnv, cacheEnv, disableCgo}
+	cmd.Dir = repoRoot
+
+	packagePath := filepath.Join(gopath, "src", "mod", "cache", "download", module, "@v")
+
+	o, err := cmd.CombinedOutput()
+	if err != nil {
+		// github quota exceeded
+		if isLimitHit(o) {
+			return packagePath, errors.E("module.getSources", err, errors.KindRateLimit)
+		}
+		// another error in the output
+		return packagePath, err
+	}
+	// make sure the expected files exist
+	return packagePath, checkFiles(fs, packagePath, version)
+}
+
+func checkFiles(fs afero.Fs, path, version string) error {
+	if _, err := fs.Stat(filepath.Join(path, version+".mod")); err != nil {
+		return pkgerrors.WithMessage(err, fmt.Sprintf("%s.mod not found in %s", version, path))
+	}
+
+	if _, err := fs.Stat(filepath.Join(path, version+".zip")); err != nil {
+		return pkgerrors.WithMessage(err, fmt.Sprintf("%s.zip not found in %s", version, path))
+	}
+
+	if _, err := fs.Stat(filepath.Join(path, version+".info")); err != nil {
+		return pkgerrors.WithMessage(err, fmt.Sprintf("%s.info not found in %s", version, path))
+	}
+
+	return nil
+}
+
+func isLimitHit(o []byte) bool {
+	return bytes.Contains(o, []byte("403 response from api.github.com"))
+}
+
+// getRepoDirName takes a raw repository URI and a version and creates a directory name that the
+// repository contents can be put into
+func getRepoDirName(repoURI, version string) string {
+	escapedURI := strings.Replace(repoURI, "/", "-", -1)
+	return fmt.Sprintf("%s-%s", escapedURI, version)
+}