diff --git a/cmd/cli/upload.go b/cmd/cli/upload.go index 9bb38517..9f0c7a94 100644 --- a/cmd/cli/upload.go +++ b/cmd/cli/upload.go @@ -4,17 +4,16 @@ import ( "bytes" "encoding/json" "fmt" - "io/ioutil" "net/http" "net/url" "path" "path/filepath" - "github.com/spf13/cobra" - parser "github.com/gomods/athens/pkg/gomod/file" "github.com/gomods/athens/pkg/module" "github.com/gomods/athens/pkg/payloads" + "github.com/spf13/afero" + "github.com/spf13/cobra" ) type uploadCmd struct { @@ -38,6 +37,7 @@ func newUploadCmd() *cobra.Command { func upload(c *uploadCmd) func(*cobra.Command, []string) error { return func(cmd *cobra.Command, args []string) error { + fs := afero.NewOsFs() dir := args[0] fullDirectory, err := filepath.Abs(dir) @@ -46,18 +46,18 @@ func upload(c *uploadCmd) func(*cobra.Command, []string) error { } cmd.Printf("found directory %s", fullDirectory) modFilePath := filepath.Join(fullDirectory, "go.mod") - modBytes, err := ioutil.ReadFile(modFilePath) + modBytes, err := afero.ReadFile(fs, modFilePath) if err != nil { return fmt.Errorf("couldn't find go.mod file (%s)", err) } - gomodParser := parser.NewFileParser(modFilePath) + gomodParser := parser.NewFileParser(fs, modFilePath) c.moduleName, err = gomodParser.ModuleName() if err != nil { return fmt.Errorf("couldn't parse go.mod file (%s)", err) } - zipBytes, err := module.MakeZip(fullDirectory, c.moduleName, c.version) + zipBytes, err := module.MakeZip(fs, fullDirectory, c.moduleName, c.version) if err != nil { return fmt.Errorf("couldn't make zip (%s)", err) } diff --git a/cmd/proxy/actions/fetch.go b/cmd/proxy/actions/fetch.go index e9f0c575..e68cee1e 100644 --- a/cmd/proxy/actions/fetch.go +++ b/cmd/proxy/actions/fetch.go @@ -2,7 +2,6 @@ package actions import ( "fmt" - "io/ioutil" "net/http" "os" "path/filepath" @@ -13,6 +12,7 @@ import ( "github.com/gomods/athens/pkg/repo/github" "github.com/gomods/athens/pkg/storage" "github.com/pkg/errors" + "github.com/spf13/afero" ) // /admin/fetch/{module:[a-zA-Z./]+}/{owner}/{repo}/{ref}/{version} @@ -22,8 +22,9 @@ func fetchHandler(store storage.Saver) func(c buffalo.Context) error { repo := c.Param("repo") ref := c.Param("ref") version := c.Param("version") + fs := afero.NewOsFs() - git, err := github.NewGitFetcher(owner, repo, ref) + git, err := github.NewGitFetcher(fs, owner, repo, ref) if err != nil { return err } @@ -35,18 +36,18 @@ func fetchHandler(store storage.Saver) func(c buffalo.Context) error { } modFilePath := filepath.Join(path, "go.mod") - modBytes, err := ioutil.ReadFile(modFilePath) + modBytes, err := afero.ReadFile(fs, modFilePath) if err != nil { return fmt.Errorf("couldn't find go.mod file (%s)", err) } - gomodParser := parser.NewFileParser(modFilePath) + gomodParser := parser.NewFileParser(fs, modFilePath) moduleName, err := gomodParser.ModuleName() if err != nil { return fmt.Errorf("couldn't parse go.mod file (%s)", err) } - zipBytes, err := module.MakeZip(path, moduleName, version) + zipBytes, err := module.MakeZip(fs, path, moduleName, version) if err != nil { return fmt.Errorf("couldn't make zip (%s)", err) } diff --git a/pkg/gomod/file/parser.go b/pkg/gomod/file/parser.go index e5678b6b..3680d2ea 100644 --- a/pkg/gomod/file/parser.go +++ b/pkg/gomod/file/parser.go @@ -1,22 +1,22 @@ package file import ( - "os" - "github.com/gomods/athens/pkg/gomod" + "github.com/spf13/afero" ) // NewFileParser creates shorthand for parsing module name out of file specified by filepath -func NewFileParser(filepath string) parser.GomodParser { - return fileParser{filepath: filepath} +func NewFileParser(fs afero.Fs, filepath string) parser.GomodParser { + return fileParser{filepath: filepath, fs: fs} } type fileParser struct { filepath string + fs afero.Fs } func (p fileParser) ModuleName() (string, error) { - file, err := os.Open(p.filepath) + file, err := p.fs.Open(p.filepath) if err != nil { return "", err } diff --git a/pkg/gomod/file/parser_test.go b/pkg/gomod/file/parser_test.go index a9f0707e..a1584f03 100644 --- a/pkg/gomod/file/parser_test.go +++ b/pkg/gomod/file/parser_test.go @@ -4,9 +4,9 @@ import ( "os" "testing" - "github.com/stretchr/testify/assert" - "github.com/gomods/athens/pkg/gomod" + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" ) func TestFileParser_ModuleName(t *testing.T) { @@ -23,10 +23,10 @@ func TestFileParser_ModuleName(t *testing.T) { {"testdata/go.3.mod", "", parser.ErrNotFound}, {"testdata/go.4.mod", "", parser.ErrNotFound}, } - + fs := afero.NewOsFs() for _, tc := range testCases { t.Run(tc.file, func(t *testing.T) { - fp := NewFileParser(tc.file) + fp := NewFileParser(fs, tc.file) actual, actualErr := fp.ModuleName() a.Equal(tc.expected, actual) @@ -36,7 +36,8 @@ func TestFileParser_ModuleName(t *testing.T) { } func TestFileParser_ModuleName_FileNotFound(t *testing.T) { - fp := NewFileParser("/not/exist") + fs := afero.NewOsFs() + fp := NewFileParser(fs, "/not/exist") _, err := fp.ModuleName() assert.True(t, os.IsNotExist(err)) } diff --git a/pkg/gomod/zip/parser_test.go b/pkg/gomod/zip/parser_test.go index 4ae0c4d3..f44c061c 100644 --- a/pkg/gomod/zip/parser_test.go +++ b/pkg/gomod/zip/parser_test.go @@ -4,19 +4,17 @@ import ( "archive/zip" "errors" "io" - "io/ioutil" - "os" "testing" + "github.com/gomods/athens/pkg/gomod" + "github.com/spf13/afero" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - - "github.com/gomods/athens/pkg/gomod" ) func TestZipParser_ModuleName(t *testing.T) { a := assert.New(t) - + fs := afero.NewOsFs() var testCases = []struct { file string dstFileName string @@ -33,7 +31,7 @@ func TestZipParser_ModuleName(t *testing.T) { for _, tc := range testCases { t.Run(tc.file, func(t *testing.T) { - zipfile := zipTestMod(t, tc.file, tc.dstFileName) + zipfile := zipTestMod(t, fs, tc.file, tc.dstFileName) reader, err := zip.OpenReader(zipfile) a.NoError(err) @@ -47,16 +45,16 @@ func TestZipParser_ModuleName(t *testing.T) { } } -func zipTestMod(t *testing.T, src string, dstFileName string) (target string) { +func zipTestMod(t *testing.T, fs afero.Fs, src string, dstFileName string) (target string) { r := require.New(t) - zipfile, err := ioutil.TempFile("", "") + zipfile, err := afero.TempFile(fs, "", "") r.NoError(err, "an error occurred while creating temporary file") defer zipfile.Close() archive := zip.NewWriter(zipfile) defer archive.Close() - srcfile, err := os.Open(src) + srcfile, err := fs.Open(src) r.NoError(err, "an error occurred while opening fixture file") defer srcfile.Close() diff --git a/pkg/module/module.go b/pkg/module/module.go index b526ce4b..a3e17671 100644 --- a/pkg/module/module.go +++ b/pkg/module/module.go @@ -4,12 +4,12 @@ import ( "archive/zip" "bytes" "fmt" - "io/ioutil" "os" "path/filepath" "strings" ignore "github.com/sabhiram/go-gitignore" + "github.com/spf13/afero" ) const ( @@ -23,8 +23,8 @@ type file struct { // MakeZip takes dir and module info and generates vgo valid zip // the dir must end with a "/" -func MakeZip(dir, module, version string) ([]byte, error) { - ignoreParser := getIgnoreParser(dir) +func MakeZip(fs afero.Fs, dir, module, version string) ([]byte, error) { + ignoreParser := getIgnoreParser(fs, dir) buf := new(bytes.Buffer) w := zip.NewWriter(buf) @@ -39,7 +39,7 @@ func MakeZip(dir, module, version string) ([]byte, error) { return nil } - fileContent, err := ioutil.ReadFile(path) + fileContent, err := afero.ReadFile(fs, path) if err != nil { return err } @@ -53,20 +53,29 @@ func MakeZip(dir, module, version string) ([]byte, error) { return err } - err := filepath.Walk(dir, walkFunc) + err := afero.Walk(fs, dir, walkFunc) w.Close() return buf.Bytes(), err } -func getIgnoreParser(dir string) ignore.IgnoreParser { +func getIgnoreParser(fs afero.Fs, dir string) ignore.IgnoreParser { gitFilePath := filepath.Join(dir, gitIgnoreFilename) - gitParser, _ := ignore.CompileIgnoreFileAndLines(gitFilePath, gitIgnoreFilename) + gitParser, _ := compileIgnoreFileAndLines(fs, gitFilePath, gitIgnoreFilename) dsStoreParser := dsStoreIgnoreParser{} return newMultiIgnoreParser(gitParser, dsStoreParser) } +func compileIgnoreFileAndLines(fs afero.Fs, fpath string, lines ...string) (*ignore.GitIgnore, error) { + buffer, err := afero.ReadFile(fs, fpath) + if err != nil { + return nil, err + } + s := strings.Split(string(buffer), "\n") + return ignore.CompileIgnoreLines(append(s, lines...)...) +} + // getFileName composes filename for zip to match standard specified as // module@version/{filename} func getFileName(path, dir, module, version string) string { diff --git a/pkg/repo/github/github.go b/pkg/repo/github/github.go index a4387dec..a1dc0eea 100644 --- a/pkg/repo/github/github.go +++ b/pkg/repo/github/github.go @@ -12,7 +12,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "net/http" "os" "path/filepath" @@ -20,8 +19,8 @@ import ( "github.com/gomods/athens/pkg/gomod/file" "github.com/gomods/athens/pkg/module" - "github.com/gomods/athens/pkg/repo" + "github.com/spf13/afero" ) const ( @@ -31,6 +30,7 @@ const ( ) type gitFetcher struct { + fs afero.Fs owner string repoName string tag string @@ -38,16 +38,18 @@ type gitFetcher struct { } // NewGitFetcher creates a new Fetcher for repositories hosted on github -func NewGitFetcher(owner string, repoName string, tag string) (repo.Fetcher, error) { +func NewGitFetcher(fs afero.Fs, owner string, repoName string, tag string) (repo.Fetcher, error) { if owner == "" || repoName == "" { return nil, errors.New("invalid repository identifier") } - return &gitFetcher{ + gf := &gitFetcher{ owner: owner, repoName: repoName, tag: tag, - }, nil + fs: fs, + } + return gf, nil } // Fetches a tarball of a repo and untars it into a temp dir which is used later in the workflow. @@ -56,44 +58,48 @@ func (g gitFetcher) Fetch() (string, error) { client := http.Client{Timeout: timeout * time.Second} resp, err := client.Get(uri) + if err != nil { return "", err } defer resp.Body.Close() - tmpDir := os.TempDir() - g.dirName, err = untar(resp.Body, tmpDir) + tmpDir, err := afero.TempDir(g.fs, "", "") if err != nil { - os.RemoveAll(tmpDir) + return "", err + } + g.dirName, err = untar(g.fs, resp.Body, tmpDir) + if err != nil { + g.fs.RemoveAll(tmpDir) return "", err } // Get module name from go.mod gomodPath := filepath.Join(g.dirName, "go.mod") - parser := file.NewFileParser(gomodPath) + parser := file.NewFileParser(g.fs, gomodPath) moduleName, err := parser.ModuleName() if err != nil { - os.RemoveAll(tmpDir) + g.fs.RemoveAll(tmpDir) return g.dirName, err } // Generate zip if err := g.generateZip(moduleName); err != nil { - os.RemoveAll(tmpDir) + g.fs.RemoveAll(tmpDir) return g.dirName, err } // Rename go.mod verModPath := filepath.Join(g.dirName, g.tag+".mod") - if err := os.Rename(gomodPath, verModPath); err != nil { - os.RemoveAll(tmpDir) + if err := g.fs.Rename(gomodPath, verModPath); err != nil { + g.fs.RemoveAll(tmpDir) return g.dirName, err } // Generate info if err := g.generateInfo(); err != nil { - os.RemoveAll(tmpDir) + g.fs.RemoveAll(tmpDir) return g.dirName, err } @@ -106,17 +112,17 @@ func (g *gitFetcher) Clear() error { return nil } - return os.RemoveAll(g.dirName) + return g.fs.RemoveAll(g.dirName) } func (g *gitFetcher) generateZip(moduleName string) error { - zipContent, err := module.MakeZip(g.dirName, moduleName, g.tag) + zipContent, err := module.MakeZip(g.fs, g.dirName, moduleName, g.tag) if err != nil { return err } zipPath := filepath.Join(g.dirName, g.tag+".zip") - return ioutil.WriteFile(zipPath, zipContent, os.ModePerm) + return afero.WriteFile(g.fs, zipPath, zipContent, os.ModePerm) } func (g *gitFetcher) generateInfo() error { @@ -131,10 +137,10 @@ func (g *gitFetcher) generateInfo() error { } goinfoPath := filepath.Join(g.dirName, g.tag+".info") - return ioutil.WriteFile(goinfoPath, infoContent, os.ModePerm) + return afero.WriteFile(g.fs, goinfoPath, infoContent, os.ModePerm) } -func untar(content io.Reader, tmpDir string) (string, error) { +func untar(fs afero.Fs, content io.Reader, tmpDir string) (string, error) { gzr, err := gzip.NewReader(content) defer gzr.Close() if err != nil { @@ -167,14 +173,14 @@ func untar(content io.Reader, tmpDir string) (string, error) { dirName = target } - if _, err := os.Stat(target); err != nil { - if err := os.MkdirAll(target, 0755); err != nil { + if _, err := fs.Stat(target); err != nil { + if err := fs.MkdirAll(target, 0755); err != nil { return "", err } } case tar.TypeReg: - f, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(hdr.Mode)) + f, err := fs.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(hdr.Mode)) if err != nil { return "", err } diff --git a/pkg/repo/github/github_test.go b/pkg/repo/github/github_test.go index d3ddffcf..1541aa37 100644 --- a/pkg/repo/github/github_test.go +++ b/pkg/repo/github/github_test.go @@ -1,9 +1,10 @@ package github import ( - "os" "path/filepath" "testing" + + "github.com/spf13/afero" ) func Test_Download(t *testing.T) { @@ -11,7 +12,9 @@ func Test_Download(t *testing.T) { repo := "captainhook" version := "v0.1.8" - fetcher, err := NewGitFetcher(owner, repo, version) + memFs := afero.NewMemMapFs() + + fetcher, err := NewGitFetcher(memFs, owner, repo, version) if err != nil { t.Error(err) } @@ -25,18 +28,17 @@ func Test_Download(t *testing.T) { } t.Log(path) - - if _, err := os.Stat(filepath.Join(path, version+".mod")); err != nil { + if _, err := memFs.Stat(filepath.Join(path, version+".mod")); err != nil { t.Error(err) t.Fail() } - if _, err := os.Stat(filepath.Join(path, version+".zip")); err != nil { + if _, err := memFs.Stat(filepath.Join(path, version+".zip")); err != nil { t.Error(err) t.Fail() } - if _, err := os.Stat(filepath.Join(path, version+".info")); err != nil { + if _, err := memFs.Stat(filepath.Join(path, version+".info")); err != nil { t.Error(err) t.Fail() } diff --git a/pkg/repo/goget.go b/pkg/repo/goget.go index 8135d0ff..ab026559 100644 --- a/pkg/repo/goget.go +++ b/pkg/repo/goget.go @@ -4,11 +4,12 @@ import ( "bytes" "errors" "fmt" - "io/ioutil" "os" "os/exec" "path/filepath" "strings" + + "github.com/spf13/afero" ) const ( @@ -21,6 +22,7 @@ var ( ) type genericFetcher struct { + fs afero.Fs repoURI string version string dirName string @@ -28,7 +30,7 @@ type genericFetcher struct { // NewGenericFetcher creates fetcher which uses go get tool to fetch sources // returns path of directory containing vx.y.z.(zip|info|mod) -func NewGenericFetcher(repoURI, version string) (Fetcher, error) { +func NewGenericFetcher(fs afero.Fs, repoURI, version string) (Fetcher, error) { if !isVgoInstalled() { return nil, errors.New("vgo not installed") } @@ -38,6 +40,7 @@ func NewGenericFetcher(repoURI, version string) (Fetcher, error) { } return &genericFetcher{ + fs: fs, repoURI: repoURI, version: version, }, nil @@ -48,15 +51,15 @@ func (g *genericFetcher) Fetch() (string, error) { escapedURI := strings.Replace(g.repoURI, "/", "-", -1) repoDirName := fmt.Sprintf(tmpRepoDir, escapedURI, g.version) - gopath, repoRoot, err := setupTmp(repoDirName) + gopath, repoRoot, err := setupTmp(g.fs, repoDirName) if err != nil { return "", err } g.dirName = repoRoot - prepareStructure(repoRoot) + prepareStructure(g.fs, repoRoot) - dirName, err := getSources(gopath, repoRoot, g.repoURI, g.version) + dirName, err := getSources(g.fs, gopath, repoRoot, g.repoURI, g.version) return dirName, err } @@ -67,7 +70,7 @@ func (g *genericFetcher) Clear() error { return nil } - return os.RemoveAll(g.dirName) + return g.fs.RemoveAll(g.dirName) } func isVgoInstalled() bool { @@ -82,29 +85,32 @@ func isVgoInstalled() bool { return false } -func setupTmp(repoDirName string) (string, string, error) { - gopathDir := os.TempDir() +func setupTmp(fs afero.Fs, repoDirName string) (string, string, error) { + gopathDir, err := afero.TempDir(fs, "", "") + if err != nil { + return "", "", err + } path := filepath.Join(gopathDir, "src", repoDirName) - return gopathDir, path, os.MkdirAll(path, os.ModeDir|os.ModePerm) + return gopathDir, path, fs.MkdirAll(path, os.ModeDir|os.ModePerm) } // Hacky thing makes vgo not to complain -func prepareStructure(repoRoot string) error { +func prepareStructure(fs afero.Fs, repoRoot string) error { // vgo expects go.mod file present with module statement or .go file with import comment gomodPath := filepath.Join(repoRoot, "go.mod") gomodContent := []byte("module \"mod\"") - if err := ioutil.WriteFile(gomodPath, gomodContent, 0666); err != nil { + if err := afero.WriteFile(fs, gomodPath, gomodContent, 0666); err != nil { return err } sourcePath := filepath.Join(repoRoot, "mod.go") sourceContent := []byte(`package mod // import "mod"`) - return ioutil.WriteFile(sourcePath, sourceContent, 0666) + return afero.WriteFile(fs, sourcePath, sourceContent, 0666) } -func getSources(gopath, repoRoot, repoURI, version string) (string, error) { +func getSources(fs afero.Fs, gopath, repoRoot, repoURI, version string) (string, error) { version = strings.TrimPrefix(version, "@") if !strings.HasPrefix(version, "v") { version = "v" + version @@ -129,7 +135,7 @@ func getSources(gopath, repoRoot, repoURI, version string) (string, error) { case isLimitHit(o): // github quota exceeded return packagePath, ErrLimitExceeded - case checkFiles(packagePath, version) == nil: + case checkFiles(fs, packagePath, version) == nil: // some compilation error return packagePath, nil default: @@ -140,16 +146,16 @@ func getSources(gopath, repoRoot, repoURI, version string) (string, error) { return packagePath, err } -func checkFiles(path, version string) error { - if _, err := os.Stat(filepath.Join(path, version+".mod")); err != nil { +func checkFiles(fs afero.Fs, path, version string) error { + if _, err := fs.Stat(filepath.Join(path, version+".mod")); err != nil { return errors.New("go.mod not found") } - if _, err := os.Stat(filepath.Join(path, version+".zip")); err != nil { + if _, err := fs.Stat(filepath.Join(path, version+".zip")); err != nil { return errors.New("zip package not found") } - if _, err := os.Stat(filepath.Join(path, version+".info")); err != nil { + if _, err := fs.Stat(filepath.Join(path, version+".info")); err != nil { return errors.New("info file not found") } diff --git a/pkg/repo/goget_test.go b/pkg/repo/goget_test.go index e5138194..bba3d6f6 100644 --- a/pkg/repo/goget_test.go +++ b/pkg/repo/goget_test.go @@ -1,16 +1,17 @@ package repo import ( - "os" "path/filepath" "testing" + + "github.com/spf13/afero" ) func Test_Download(t *testing.T) { version := "v0.1.8" gitURI := "github.com/bketelsen/captainhook" - - fetcher, err := NewGenericFetcher(gitURI, version) + fs := afero.NewOsFs() + fetcher, err := NewGenericFetcher(fs, gitURI, version) if err != nil { t.Error(err) t.Fail() @@ -32,17 +33,17 @@ func Test_Download(t *testing.T) { t.Fail() } - if _, err := os.Stat(filepath.Join(path, version+".mod")); err != nil { + if _, err := fs.Stat(filepath.Join(path, version+".mod")); err != nil { t.Error(err) t.Fail() } - if _, err := os.Stat(filepath.Join(path, version+".zip")); err != nil { + if _, err := fs.Stat(filepath.Join(path, version+".zip")); err != nil { t.Error(err) t.Fail() } - if _, err := os.Stat(filepath.Join(path, version+".info")); err != nil { + if _, err := fs.Stat(filepath.Join(path, version+".info")); err != nil { t.Error(err) t.Fail() } diff --git a/pkg/storage/fs/all_test.go b/pkg/storage/fs/all_test.go index dea2aade..1ae2a58a 100644 --- a/pkg/storage/fs/all_test.go +++ b/pkg/storage/fs/all_test.go @@ -5,7 +5,6 @@ import ( "github.com/gomods/athens/pkg/storage" "github.com/spf13/afero" - "github.com/stretchr/testify/suite" ) diff --git a/pkg/storage/fs/getter.go b/pkg/storage/fs/getter.go index 719192fb..0028763a 100644 --- a/pkg/storage/fs/getter.go +++ b/pkg/storage/fs/getter.go @@ -5,9 +5,8 @@ import ( "path/filepath" "time" - "github.com/spf13/afero" - "github.com/gomods/athens/pkg/storage" + "github.com/spf13/afero" ) func (v *storageImpl) Get(module, version string) (*storage.Version, error) {