primal/athens
1
0
Fork 0
mirror of https://github.com/gomods/athens synced 2026-02-03 11:00:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

replace fs operations with afero (#123)

Browse Source 
* replace fs operations with afero

* fix goget test

* fix test

* fix test again...

* remove empty lines

* fix merge

setupTmp - return 3, not 2 values

* just to start new travis build

got The command "go get github.com/golang/lint/golint" failed and exited with 1 during . I hope it was a temp problem.
This commit is contained in:
marpio
2018-05-13 21:40:09 +02:00
committed by Aaron Schlesinger
parent 7cd41ef236
commit 8ceda38ad7
12 changed files with 114 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/cli/upload.go
+6 -6
View File
@@ -4,17 +4,16 @@ import (
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"path"
"path/filepath"
"github.com/spf13/cobra"
parser "github.com/gomods/athens/pkg/gomod/file"
"github.com/gomods/athens/pkg/module"
"github.com/gomods/athens/pkg/payloads"
"github.com/spf13/afero"
"github.com/spf13/cobra"
)
type uploadCmd struct {
@@ -38,6 +37,7 @@ func newUploadCmd() *cobra.Command {
func upload(c *uploadCmd) func(*cobra.Command, []string) error {
return func(cmd *cobra.Command, args []string) error {
fs := afero.NewOsFs()
dir := args[0]
fullDirectory, err := filepath.Abs(dir)
@@ -46,18 +46,18 @@ func upload(c *uploadCmd) func(*cobra.Command, []string) error {
}
cmd.Printf("found directory %s", fullDirectory)
modFilePath := filepath.Join(fullDirectory, "go.mod")
modBytes, err := ioutil.ReadFile(modFilePath)
modBytes, err := afero.ReadFile(fs, modFilePath)
if err != nil {
return fmt.Errorf("couldn't find go.mod file (%s)", err)
}
gomodParser := parser.NewFileParser(modFilePath)
gomodParser := parser.NewFileParser(fs, modFilePath)
c.moduleName, err = gomodParser.ModuleName()
if err != nil {
return fmt.Errorf("couldn't parse go.mod file (%s)", err)
}
zipBytes, err := module.MakeZip(fullDirectory, c.moduleName, c.version)
zipBytes, err := module.MakeZip(fs, fullDirectory, c.moduleName, c.version)
if err != nil {
return fmt.Errorf("couldn't make zip (%s)", err)
}
cmd/proxy/actions/fetch.go
+6 -5
View File
@@ -2,7 +2,6 @@ package actions
import (
"fmt"
"io/ioutil"
"net/http"
"os"
"path/filepath"
@@ -13,6 +12,7 @@ import (
"github.com/gomods/athens/pkg/repo/github"
"github.com/gomods/athens/pkg/storage"
"github.com/pkg/errors"
"github.com/spf13/afero"
)
// /admin/fetch/{module:[a-zA-Z./]+}/{owner}/{repo}/{ref}/{version}
@@ -22,8 +22,9 @@ func fetchHandler(store storage.Saver) func(c buffalo.Context) error {
repo := c.Param("repo")
ref := c.Param("ref")
version := c.Param("version")
fs := afero.NewOsFs()
git, err := github.NewGitFetcher(owner, repo, ref)
git, err := github.NewGitFetcher(fs, owner, repo, ref)
if err != nil {
return err
}
@@ -35,18 +36,18 @@ func fetchHandler(store storage.Saver) func(c buffalo.Context) error {
}
modFilePath := filepath.Join(path, "go.mod")
modBytes, err := ioutil.ReadFile(modFilePath)
modBytes, err := afero.ReadFile(fs, modFilePath)
if err != nil {
return fmt.Errorf("couldn't find go.mod file (%s)", err)
}
gomodParser := parser.NewFileParser(modFilePath)
gomodParser := parser.NewFileParser(fs, modFilePath)
moduleName, err := gomodParser.ModuleName()
if err != nil {
return fmt.Errorf("couldn't parse go.mod file (%s)", err)
}
zipBytes, err := module.MakeZip(path, moduleName, version)
zipBytes, err := module.MakeZip(fs, path, moduleName, version)
if err != nil {
return fmt.Errorf("couldn't make zip (%s)", err)
}
pkg/gomod/file/parser.go
+5 -5
View File
@@ -1,22 +1,22 @@
package file
import (
"os"
"github.com/gomods/athens/pkg/gomod"
"github.com/spf13/afero"
)
// NewFileParser creates shorthand for parsing module name out of file specified by filepath
func NewFileParser(filepath string) parser.GomodParser {
return fileParser{filepath: filepath}
func NewFileParser(fs afero.Fs, filepath string) parser.GomodParser {
return fileParser{filepath: filepath, fs: fs}
}
type fileParser struct {
filepath string
fs afero.Fs
}
func (p fileParser) ModuleName() (string, error) {
file, err := os.Open(p.filepath)
file, err := p.fs.Open(p.filepath)
if err != nil {
return "", err
}
pkg/gomod/file/parser_test.go
+6 -5
View File
@@ -4,9 +4,9 @@ import (
"os"
"testing"
"github.com/stretchr/testify/assert"
"github.com/gomods/athens/pkg/gomod"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
)
func TestFileParser_ModuleName(t *testing.T) {
@@ -23,10 +23,10 @@ func TestFileParser_ModuleName(t *testing.T) {
{"testdata/go.3.mod", "", parser.ErrNotFound},
{"testdata/go.4.mod", "", parser.ErrNotFound},
}
fs := afero.NewOsFs()
for _, tc := range testCases {
t.Run(tc.file, func(t *testing.T) {
fp := NewFileParser(tc.file)
fp := NewFileParser(fs, tc.file)
actual, actualErr := fp.ModuleName()
a.Equal(tc.expected, actual)
@@ -36,7 +36,8 @@ func TestFileParser_ModuleName(t *testing.T) {
}
func TestFileParser_ModuleName_FileNotFound(t *testing.T) {
fp := NewFileParser("/not/exist")
fs := afero.NewOsFs()
fp := NewFileParser(fs, "/not/exist")
_, err := fp.ModuleName()
assert.True(t, os.IsNotExist(err))
}
pkg/gomod/zip/parser_test.go
+7 -9
View File
@@ -4,19 +4,17 @@ import (
"archive/zip"
"errors"
"io"
"io/ioutil"
"os"
"testing"
"github.com/gomods/athens/pkg/gomod"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/gomods/athens/pkg/gomod"
)
func TestZipParser_ModuleName(t *testing.T) {
a := assert.New(t)
fs := afero.NewOsFs()
var testCases = []struct {
file string
dstFileName string
@@ -33,7 +31,7 @@ func TestZipParser_ModuleName(t *testing.T) {
for _, tc := range testCases {
t.Run(tc.file, func(t *testing.T) {
zipfile := zipTestMod(t, tc.file, tc.dstFileName)
zipfile := zipTestMod(t, fs, tc.file, tc.dstFileName)
reader, err := zip.OpenReader(zipfile)
a.NoError(err)
@@ -47,16 +45,16 @@ func TestZipParser_ModuleName(t *testing.T) {
}
}
func zipTestMod(t *testing.T, src string, dstFileName string) (target string) {
func zipTestMod(t *testing.T, fs afero.Fs, src string, dstFileName string) (target string) {
r := require.New(t)
zipfile, err := ioutil.TempFile("", "")
zipfile, err := afero.TempFile(fs, "", "")
r.NoError(err, "an error occurred while creating temporary file")
defer zipfile.Close()
archive := zip.NewWriter(zipfile)
defer archive.Close()
srcfile, err := os.Open(src)
srcfile, err := fs.Open(src)
r.NoError(err, "an error occurred while opening fixture file")
defer srcfile.Close()
pkg/module/module.go
+16 -7
View File
@@ -4,12 +4,12 @@ import (
"archive/zip"
"bytes"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strings"
ignore "github.com/sabhiram/go-gitignore"
"github.com/spf13/afero"
)
const (
@@ -23,8 +23,8 @@ type file struct {
// MakeZip takes dir and module info and generates vgo valid zip
// the dir must end with a "/"
func MakeZip(dir, module, version string) ([]byte, error) {
ignoreParser := getIgnoreParser(dir)
func MakeZip(fs afero.Fs, dir, module, version string) ([]byte, error) {
ignoreParser := getIgnoreParser(fs, dir)
buf := new(bytes.Buffer)
w := zip.NewWriter(buf)
@@ -39,7 +39,7 @@ func MakeZip(dir, module, version string) ([]byte, error) {
return nil
}
fileContent, err := ioutil.ReadFile(path)
fileContent, err := afero.ReadFile(fs, path)
if err != nil {
return err
}
@@ -53,20 +53,29 @@ func MakeZip(dir, module, version string) ([]byte, error) {
return err
}
err := filepath.Walk(dir, walkFunc)
err := afero.Walk(fs, dir, walkFunc)
w.Close()
return buf.Bytes(), err
}
func getIgnoreParser(dir string) ignore.IgnoreParser {
func getIgnoreParser(fs afero.Fs, dir string) ignore.IgnoreParser {
gitFilePath := filepath.Join(dir, gitIgnoreFilename)
gitParser, _ := ignore.CompileIgnoreFileAndLines(gitFilePath, gitIgnoreFilename)
gitParser, _ := compileIgnoreFileAndLines(fs, gitFilePath, gitIgnoreFilename)
dsStoreParser := dsStoreIgnoreParser{}
return newMultiIgnoreParser(gitParser, dsStoreParser)
}
func compileIgnoreFileAndLines(fs afero.Fs, fpath string, lines ...string) (*ignore.GitIgnore, error) {
buffer, err := afero.ReadFile(fs, fpath)
if err != nil {
return nil, err
}
s := strings.Split(string(buffer), "\n")
return ignore.CompileIgnoreLines(append(s, lines...)...)
}
// getFileName composes filename for zip to match standard specified as
// module@version/{filename}
func getFileName(path, dir, module, version string) string {
pkg/repo/github/github.go
+28 -22
View File
@@ -12,7 +12,6 @@ import (
"errors"
"fmt"
"io"
"io/ioutil"
"net/http"
"os"
"path/filepath"
@@ -20,8 +19,8 @@ import (
"github.com/gomods/athens/pkg/gomod/file"
"github.com/gomods/athens/pkg/module"
"github.com/gomods/athens/pkg/repo"
"github.com/spf13/afero"
)
const (
@@ -31,6 +30,7 @@ const (
)
type gitFetcher struct {
fs afero.Fs
owner string
repoName string
tag string
@@ -38,16 +38,18 @@ type gitFetcher struct {
}
// NewGitFetcher creates a new Fetcher for repositories hosted on github
func NewGitFetcher(owner string, repoName string, tag string) (repo.Fetcher, error) {
func NewGitFetcher(fs afero.Fs, owner string, repoName string, tag string) (repo.Fetcher, error) {
if owner == "" || repoName == "" {
return nil, errors.New("invalid repository identifier")
}
return &gitFetcher{
gf := &gitFetcher{
owner: owner,
repoName: repoName,
tag: tag,
}, nil
fs: fs,
}
return gf, nil
}
// Fetches a tarball of a repo and untars it into a temp dir which is used later in the workflow.
@@ -56,44 +58,48 @@ func (g gitFetcher) Fetch() (string, error) {
client := http.Client{Timeout: timeout * time.Second}
resp, err := client.Get(uri)
if err != nil {
return "", err
}
defer resp.Body.Close()
tmpDir := os.TempDir()
g.dirName, err = untar(resp.Body, tmpDir)
tmpDir, err := afero.TempDir(g.fs, "", "")
if err != nil {
os.RemoveAll(tmpDir)
return "", err
}
g.dirName, err = untar(g.fs, resp.Body, tmpDir)
if err != nil {
g.fs.RemoveAll(tmpDir)
return "", err
}
// Get module name from go.mod
gomodPath := filepath.Join(g.dirName, "go.mod")
parser := file.NewFileParser(gomodPath)
parser := file.NewFileParser(g.fs, gomodPath)
moduleName, err := parser.ModuleName()
if err != nil {
os.RemoveAll(tmpDir)
g.fs.RemoveAll(tmpDir)
return g.dirName, err
}
// Generate zip
if err := g.generateZip(moduleName); err != nil {
os.RemoveAll(tmpDir)
g.fs.RemoveAll(tmpDir)
return g.dirName, err
}
// Rename go.mod
verModPath := filepath.Join(g.dirName, g.tag+".mod")
if err := os.Rename(gomodPath, verModPath); err != nil {
os.RemoveAll(tmpDir)
if err := g.fs.Rename(gomodPath, verModPath); err != nil {
g.fs.RemoveAll(tmpDir)
return g.dirName, err
}
// Generate info
if err := g.generateInfo(); err != nil {
os.RemoveAll(tmpDir)
g.fs.RemoveAll(tmpDir)
return g.dirName, err
}
@@ -106,17 +112,17 @@ func (g *gitFetcher) Clear() error {
return nil
}
return os.RemoveAll(g.dirName)
return g.fs.RemoveAll(g.dirName)
}
func (g *gitFetcher) generateZip(moduleName string) error {
zipContent, err := module.MakeZip(g.dirName, moduleName, g.tag)
zipContent, err := module.MakeZip(g.fs, g.dirName, moduleName, g.tag)
if err != nil {
return err
}
zipPath := filepath.Join(g.dirName, g.tag+".zip")
return ioutil.WriteFile(zipPath, zipContent, os.ModePerm)
return afero.WriteFile(g.fs, zipPath, zipContent, os.ModePerm)
}
func (g *gitFetcher) generateInfo() error {
@@ -131,10 +137,10 @@ func (g *gitFetcher) generateInfo() error {
}
goinfoPath := filepath.Join(g.dirName, g.tag+".info")
return ioutil.WriteFile(goinfoPath, infoContent, os.ModePerm)
return afero.WriteFile(g.fs, goinfoPath, infoContent, os.ModePerm)
}
func untar(content io.Reader, tmpDir string) (string, error) {
func untar(fs afero.Fs, content io.Reader, tmpDir string) (string, error) {
gzr, err := gzip.NewReader(content)
defer gzr.Close()
if err != nil {
@@ -167,14 +173,14 @@ func untar(content io.Reader, tmpDir string) (string, error) {
dirName = target
}
if _, err := os.Stat(target); err != nil {
if err := os.MkdirAll(target, 0755); err != nil {
if _, err := fs.Stat(target); err != nil {
if err := fs.MkdirAll(target, 0755); err != nil {
return "", err
}
}
case tar.TypeReg:
f, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(hdr.Mode))
f, err := fs.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(hdr.Mode))
if err != nil {
return "", err
}
pkg/repo/github/github_test.go
+8 -6
View File
@@ -1,9 +1,10 @@
package github
import (
"os"
"path/filepath"
"testing"
"github.com/spf13/afero"
)
func Test_Download(t *testing.T) {
@@ -11,7 +12,9 @@ func Test_Download(t *testing.T) {
repo := "captainhook"
version := "v0.1.8"
fetcher, err := NewGitFetcher(owner, repo, version)
memFs := afero.NewMemMapFs()
fetcher, err := NewGitFetcher(memFs, owner, repo, version)
if err != nil {
t.Error(err)
}
@@ -25,18 +28,17 @@ func Test_Download(t *testing.T) {
}
t.Log(path)
if _, err := os.Stat(filepath.Join(path, version+".mod")); err != nil {
if _, err := memFs.Stat(filepath.Join(path, version+".mod")); err != nil {
t.Error(err)
t.Fail()
}
if _, err := os.Stat(filepath.Join(path, version+".zip")); err != nil {
if _, err := memFs.Stat(filepath.Join(path, version+".zip")); err != nil {
t.Error(err)
t.Fail()
}
if _, err := os.Stat(filepath.Join(path, version+".info")); err != nil {
if _, err := memFs.Stat(filepath.Join(path, version+".info")); err != nil {
t.Error(err)
t.Fail()
}
pkg/repo/goget.go
+24 -18
View File
@@ -4,11 +4,12 @@ import (
"bytes"
"errors"
"fmt"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"strings"
"github.com/spf13/afero"
)
const (
@@ -21,6 +22,7 @@ var (
)
type genericFetcher struct {
fs afero.Fs
repoURI string
version string
dirName string
@@ -28,7 +30,7 @@ type genericFetcher struct {
// NewGenericFetcher creates fetcher which uses go get tool to fetch sources
// returns path of directory containing vx.y.z.(zip|info|mod)
func NewGenericFetcher(repoURI, version string) (Fetcher, error) {
func NewGenericFetcher(fs afero.Fs, repoURI, version string) (Fetcher, error) {
if !isVgoInstalled() {
return nil, errors.New("vgo not installed")
}
@@ -38,6 +40,7 @@ func NewGenericFetcher(repoURI, version string) (Fetcher, error) {
}
return &genericFetcher{
fs: fs,
repoURI: repoURI,
version: version,
}, nil
@@ -48,15 +51,15 @@ func (g *genericFetcher) Fetch() (string, error) {
escapedURI := strings.Replace(g.repoURI, "/", "-", -1)
repoDirName := fmt.Sprintf(tmpRepoDir, escapedURI, g.version)
gopath, repoRoot, err := setupTmp(repoDirName)
gopath, repoRoot, err := setupTmp(g.fs, repoDirName)
if err != nil {
return "", err
}
g.dirName = repoRoot
prepareStructure(repoRoot)
prepareStructure(g.fs, repoRoot)
dirName, err := getSources(gopath, repoRoot, g.repoURI, g.version)
dirName, err := getSources(g.fs, gopath, repoRoot, g.repoURI, g.version)
return dirName, err
}
@@ -67,7 +70,7 @@ func (g *genericFetcher) Clear() error {
return nil
}
return os.RemoveAll(g.dirName)
return g.fs.RemoveAll(g.dirName)
}
func isVgoInstalled() bool {
@@ -82,29 +85,32 @@ func isVgoInstalled() bool {
return false
}
func setupTmp(repoDirName string) (string, string, error) {
gopathDir := os.TempDir()
func setupTmp(fs afero.Fs, repoDirName string) (string, string, error) {
gopathDir, err := afero.TempDir(fs, "", "")
if err != nil {
return "", "", err
}
path := filepath.Join(gopathDir, "src", repoDirName)
return gopathDir, path, os.MkdirAll(path, os.ModeDir|os.ModePerm)
return gopathDir, path, fs.MkdirAll(path, os.ModeDir|os.ModePerm)
}
// Hacky thing makes vgo not to complain
func prepareStructure(repoRoot string) error {
func prepareStructure(fs afero.Fs, repoRoot string) error {
// vgo expects go.mod file present with module statement or .go file with import comment
gomodPath := filepath.Join(repoRoot, "go.mod")
gomodContent := []byte("module \"mod\"")
if err := ioutil.WriteFile(gomodPath, gomodContent, 0666); err != nil {
if err := afero.WriteFile(fs, gomodPath, gomodContent, 0666); err != nil {
return err
}
sourcePath := filepath.Join(repoRoot, "mod.go")
sourceContent := []byte(`package mod // import "mod"`)
return ioutil.WriteFile(sourcePath, sourceContent, 0666)
return afero.WriteFile(fs, sourcePath, sourceContent, 0666)
}
func getSources(gopath, repoRoot, repoURI, version string) (string, error) {
func getSources(fs afero.Fs, gopath, repoRoot, repoURI, version string) (string, error) {
version = strings.TrimPrefix(version, "@")
if !strings.HasPrefix(version, "v") {
version = "v" + version
@@ -129,7 +135,7 @@ func getSources(gopath, repoRoot, repoURI, version string) (string, error) {
case isLimitHit(o):
// github quota exceeded
return packagePath, ErrLimitExceeded
case checkFiles(packagePath, version) == nil:
case checkFiles(fs, packagePath, version) == nil:
// some compilation error
return packagePath, nil
default:
@@ -140,16 +146,16 @@ func getSources(gopath, repoRoot, repoURI, version string) (string, error) {
return packagePath, err
}
func checkFiles(path, version string) error {
if _, err := os.Stat(filepath.Join(path, version+".mod")); err != nil {
func checkFiles(fs afero.Fs, path, version string) error {
if _, err := fs.Stat(filepath.Join(path, version+".mod")); err != nil {
return errors.New("go.mod not found")
}
if _, err := os.Stat(filepath.Join(path, version+".zip")); err != nil {
if _, err := fs.Stat(filepath.Join(path, version+".zip")); err != nil {
return errors.New("zip package not found")
}
if _, err := os.Stat(filepath.Join(path, version+".info")); err != nil {
if _, err := fs.Stat(filepath.Join(path, version+".info")); err != nil {
return errors.New("info file not found")
}
pkg/repo/goget_test.go
+7 -6
View File
@@ -1,16 +1,17 @@
package repo
import (
"os"
"path/filepath"
"testing"
"github.com/spf13/afero"
)
func Test_Download(t *testing.T) {
version := "v0.1.8"
gitURI := "github.com/bketelsen/captainhook"
fetcher, err := NewGenericFetcher(gitURI, version)
fs := afero.NewOsFs()
fetcher, err := NewGenericFetcher(fs, gitURI, version)
if err != nil {
t.Error(err)
t.Fail()
@@ -32,17 +33,17 @@ func Test_Download(t *testing.T) {
t.Fail()
}
if _, err := os.Stat(filepath.Join(path, version+".mod")); err != nil {
if _, err := fs.Stat(filepath.Join(path, version+".mod")); err != nil {
t.Error(err)
t.Fail()
}
if _, err := os.Stat(filepath.Join(path, version+".zip")); err != nil {
if _, err := fs.Stat(filepath.Join(path, version+".zip")); err != nil {
t.Error(err)
t.Fail()
}
if _, err := os.Stat(filepath.Join(path, version+".info")); err != nil {
if _, err := fs.Stat(filepath.Join(path, version+".info")); err != nil {
t.Error(err)
t.Fail()
}
pkg/storage/fs/all_test.go
-1
View File
@@ -5,7 +5,6 @@ import (
"github.com/gomods/athens/pkg/storage"
"github.com/spf13/afero"
"github.com/stretchr/testify/suite"
)
pkg/storage/fs/getter.go
+1 -2
View File
@@ -5,9 +5,8 @@ import (
"path/filepath"
"time"
"github.com/spf13/afero"
"github.com/gomods/athens/pkg/storage"
"github.com/spf13/afero"
)
func (v *storageImpl) Get(module, version string) (*storage.Version, error) {