primal/athens
1
0
Fork 0
mirror of https://github.com/gomods/athens synced 2026-02-03 11:00:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

Ensure diskRef cleans up everything in it's own GOPATH (#354)

Browse Source 
* Ensure diskRef cleans up everything in it's own GOPATH

* cleanup diskref opens. Fix err return in go_get_fetcher

* Remove Noop ref from go_get_fetcher.Fetch()

* handle temp dir cleanup and return nil diskref
This commit is contained in:
Rohan Chakravarthy
2018-07-30 16:14:06 -07:00
committed by Marwan Sulaiman
parent 59d5d5ffac
commit c7d0c0a8e6
4 changed files with 68 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/module/disk_ref.go
+24 -15
View File
@@ -2,7 +2,7 @@ package module
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"github.com/gomods/athens/pkg/storage"
@@ -15,14 +15,16 @@ import (
// Do not create this struct directly. use newDiskRef
type diskRef struct {
root string
module string
fs afero.Fs
version string
}
func newDiskRef(fs afero.Fs, root, version string) *diskRef {
func newDiskRef(fs afero.Fs, root, module, version string) *diskRef {
return &diskRef{
fs: fs,
root: root,
module: module,
version: version,
}
}
@@ -31,6 +33,20 @@ func newDiskRef(fs afero.Fs, root, version string) *diskRef {
//
// You should always call this function after you fetch a module into a DiskRef
func (d *diskRef) Clear() error {
// This is required because vgo ensures dependencies are read-only
// See https://github.com/golang/go/issues/24111 and
// https://go-review.googlesource.com/c/vgo/+/96978
walkFn := func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
return d.fs.Chmod(path, 0770)
}
err := afero.Walk(d.fs, d.root, walkFn)
if err != nil {
return err
}
return d.fs.RemoveAll(d.root)
}
@@ -38,30 +54,23 @@ func (d *diskRef) Clear() error {
func (d *diskRef) Read() (*storage.Version, error) {
var ver storage.Version
infoFile, err := d.fs.Open(filepath.Join(d.root, fmt.Sprintf("%s.info", d.version)))
if err != nil {
return nil, errors.WithStack(err)
}
defer infoFile.Close()
packagePath := getPackagePath(d.root, d.module)
info, err := ioutil.ReadAll(infoFile)
infoFile := filepath.Join(packagePath, fmt.Sprintf("%s.info", d.version))
info, err := afero.ReadFile(d.fs, infoFile)
if err != nil {
return nil, errors.WithStack(err)
}
ver.Info = info
modFile, err := d.fs.Open(filepath.Join(d.root, fmt.Sprintf("%s.mod", d.version)))
if err != nil {
return nil, errors.WithStack(err)
}
defer modFile.Close()
mod, err := ioutil.ReadAll(modFile)
modFile := filepath.Join(packagePath, fmt.Sprintf("%s.mod", d.version))
mod, err := afero.ReadFile(d.fs, modFile)
if err != nil {
return nil, errors.WithStack(err)
}
ver.Mod = mod
sourceFile, err := d.fs.Open(filepath.Join(d.root, fmt.Sprintf("%s.zip", d.version)))
sourceFile, err := d.fs.Open(filepath.Join(packagePath, fmt.Sprintf("%s.zip", d.version)))
if err != nil {
return nil, errors.WithStack(err)
}
pkg/module/disk_ref_test.go
+16 -4
View File
@@ -17,8 +17,9 @@ func (m *ModuleSuite) TestDiskRefReadAndClear() {
)
r := m.Require()
packagePath := getPackagePath(root, mod)
// create a new disk ref using the filesystem
diskRef := newDiskRef(m.fs, root, version)
diskRef := newDiskRef(m.fs, root, mod, version)
// ensure that reading fails, because there are no files
ver, err := diskRef.Read()
@@ -26,9 +27,9 @@ func (m *ModuleSuite) TestDiskRefReadAndClear() {
r.NotNil(err)
// create all the files the disk ref expects
r.NoError(createAndWriteFile(m.fs, filepath.Join(root, version+".info"), info))
r.NoError(createAndWriteFile(m.fs, filepath.Join(root, version+".mod"), mod))
r.NoError(createAndWriteFile(m.fs, filepath.Join(root, version+".zip"), zip))
r.NoError(createAndWriteFile(m.fs, filepath.Join(packagePath, version+".info"), info))
r.NoError(createAndWriteFile(m.fs, filepath.Join(packagePath, version+".mod"), mod))
r.NoError(createAndWriteFile(m.fs, filepath.Join(packagePath, version+".zip"), zip))
// read from the disk ref - this time it should succeed
ver, err = diskRef.Read()
@@ -39,11 +40,22 @@ func (m *ModuleSuite) TestDiskRefReadAndClear() {
r.NoError(err)
r.Equal(zip, string(zipBytes))
// Validate that the root dir still exists
fInfo, err := m.fs.Stat(root)
r.NotNil(fInfo)
r.Nil(err)
// clear the disk ref and expect it to fail again
r.NoError(diskRef.Clear())
ver, err = diskRef.Read()
r.Nil(ver)
r.NotNil(err)
// The root dir should not exist after a clear
fInfo, err = m.fs.Stat(root)
r.Nil(fInfo)
r.NotNil(err)
}
// creates filename with fs, writes data to the file, and closes the file,
pkg/module/go_get_fetcher.go
+26 -33
View File
@@ -29,43 +29,35 @@ func NewGoGetFetcher(goBinaryName string, fs afero.Fs) Fetcher {
// Fetch downloads the sources and returns path where it can be found. Make sure to call Clear
// on the returned Ref when you are done with it
func (g *goGetFetcher) Fetch(mod, ver string) (Ref, error) {
ref := noopRef{}
// setup the GOPATH
goPathRoot, err := afero.TempDir(g.fs, "", "athens")
if err != nil {
// TODO: return a ref for cleaning up the goPathRoot
// https://github.com/gomods/athens/issues/329
ref.Clear()
return ref, err
return nil, err
}
sourcePath := filepath.Join(goPathRoot, "src")
modPath := filepath.Join(sourcePath, getRepoDirName(mod, ver))
if err := g.fs.MkdirAll(modPath, os.ModeDir|os.ModePerm); err != nil {
// TODO: return a ref for cleaning up the goPathRoot
// https://github.com/gomods/athens/issues/329
ref.Clear()
return ref, err
diskRef := newDiskRef(g.fs, goPathRoot, "", "")
diskRef.Clear()
return nil, err
}
// setup the module with barebones stuff
if err := Dummy(g.fs, modPath); err != nil {
// TODO: return a ref for cleaning up the goPathRoot
// https://github.com/gomods/athens/issues/329
ref.Clear()
return ref, err
}
cachePath, err := getSources(g.goBinaryName, g.fs, goPathRoot, modPath, mod, ver)
if err != nil {
// TODO: return a ref that cleans up the goPathRoot
// https://github.com/gomods/athens/issues/329
ref.Clear()
diskRef := newDiskRef(g.fs, goPathRoot, "", "")
diskRef.Clear()
return nil, err
}
// TODO: make sure this ref also cleans up the goPathRoot
// https://github.com/gomods/athens/issues/329
return newDiskRef(g.fs, cachePath, ver), err
err = getSources(g.goBinaryName, g.fs, goPathRoot, modPath, mod, ver)
if err != nil {
diskRef := newDiskRef(g.fs, goPathRoot, "", "")
diskRef.Clear()
return nil, err
}
return newDiskRef(g.fs, goPathRoot, mod, ver), nil
}
// Dummy Hacky thing makes vgo not to complain
@@ -86,10 +78,8 @@ func Dummy(fs afero.Fs, repoRoot string) error {
}
// given a filesystem, gopath, repository root, module and version, runs 'vgo get'
// on module@version from the repoRoot with GOPATH=gopath, and returns the location
// of the module cache. returns a non-nil error if anything went wrong. always returns
// the location of the module cache so you can delete it if necessary
func getSources(goBinaryName string, fs afero.Fs, gopath, repoRoot, module, version string) (string, error) {
// on module@version from the repoRoot with GOPATH=gopath, and returns a non-nil error if anything went wrong.
func getSources(goBinaryName string, fs afero.Fs, gopath, repoRoot, module, version string) error {
uri := strings.TrimSuffix(module, "/")
fullURI := fmt.Sprintf("%s@%s", uri, version)
@@ -104,21 +94,19 @@ func getSources(goBinaryName string, fs afero.Fs, gopath, repoRoot, module, vers
// this breaks windows.
cmd.Env = []string{"PATH=" + os.Getenv("PATH"), gopathEnv, cacheEnv, disableCgo, enableGoModules}
cmd.Dir = repoRoot
packagePath := filepath.Join(gopath, "src", "mod", "cache", "download", module, "@v")
o, err := cmd.CombinedOutput()
if err != nil {
errMsg := fmt.Sprintf("%v : %s", err, o)
// github quota exceeded
if isLimitHit(o) {
return packagePath, errors.E("module.getSources", errMsg, errors.KindRateLimit)
return errors.E("module.getSources", errMsg, errors.KindRateLimit)
}
// another error in the output
return packagePath, errors.E("module.getSources", errMsg)
return errors.E("module.getSources", errMsg)
}
// make sure the expected files exist
return packagePath, checkFiles(fs, packagePath, version)
packagePath := getPackagePath(gopath, module)
return checkFiles(fs, packagePath, version)
}
func checkFiles(fs afero.Fs, path, version string) error {
@@ -147,3 +135,8 @@ func getRepoDirName(repoURI, version string) string {
escapedURI := strings.Replace(repoURI, "/", "-", -1)
return fmt.Sprintf("%s-%s", escapedURI, version)
}
// getPackagePath returns the path to the module cache given the gopath and module name
func getPackagePath(gopath, module string) string {
return filepath.Join(gopath, "src", "mod", "cache", "download", module, "@v")
}
pkg/module/go_get_fetcher_test.go
+2 -1
View File
@@ -1,8 +1,9 @@
package module
import (
"github.com/spf13/afero"
"io/ioutil"
"github.com/spf13/afero"
)
func (s *ModuleSuite) TestNewGoGetFetcher() {