primal/athens
1
0
Fork 0
mirror of https://github.com/gomods/athens synced 2026-02-03 11:00:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

ref: upstream lister and it's impl to module pkg (#1309)

Browse Source
This commit is contained in:
Yi Tang
2019-07-26 08:40:03 +08:00
committed by Marwan Sulaiman
parent 5bff5fe74d
commit ca10728887
8 changed files with 94 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/proxy/actions/app_proxy.go
+1 -1
View File
@@ -78,7 +78,7 @@ func addProxyRoutes(
return err
}
lister := download.NewVCSLister(c.GoBinary, c.GoProxy, fs)
lister := module.NewVCSLister(c.GoBinary, c.GoProxy, fs)
withSingleFlight, err := getSingleFlight(c, s)
if err != nil {
pkg/download/protocol.go
+3 -2
View File
@@ -9,6 +9,7 @@ import (
"github.com/gomods/athens/pkg/download/mode"
"github.com/gomods/athens/pkg/errors"
"github.com/gomods/athens/pkg/module"
"github.com/gomods/athens/pkg/observ"
"github.com/gomods/athens/pkg/stash"
"github.com/gomods/athens/pkg/storage"
@@ -40,7 +41,7 @@ type Wrapper func(Protocol) Protocol
type Opts struct {
Storage storage.Backend
Stasher stash.Stasher
Lister UpstreamLister
Lister module.UpstreamLister
DownloadFile *mode.DownloadFile
}
@@ -65,7 +66,7 @@ type protocol struct {
df *mode.DownloadFile
storage storage.Backend
stasher stash.Stasher
lister UpstreamLister
lister module.UpstreamLister
}
func (p *protocol) List(ctx context.Context, mod string) ([]string, error) {
pkg/download/protocol_test.go
+1 -1
View File
@@ -46,7 +46,7 @@ func getDP(t *testing.T) Protocol {
t.Fatal(err)
}
st := stash.New(mf, s)
return New(&Opts{s, st, NewVCSLister(goBin, goProxy, fs), nil})
return New(&Opts{s, st, module.NewVCSLister(goBin, goProxy, fs), nil})
}
type listTest struct {
pkg/module/go_get_fetcher.go
+3 -63
View File
@@ -8,7 +8,6 @@ import (
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"github.com/gomods/athens/pkg/errors"
@@ -63,13 +62,13 @@ func (g *goGetFetcher) Fetch(ctx context.Context, mod, ver string) (*storage.Ver
sourcePath := filepath.Join(goPathRoot, "src")
modPath := filepath.Join(sourcePath, getRepoDirName(mod, ver))
if err := g.fs.MkdirAll(modPath, os.ModeDir|os.ModePerm); err != nil {
ClearFiles(g.fs, goPathRoot)
clearFiles(g.fs, goPathRoot)
return nil, errors.E(op, err)
}
m, err := downloadModule(g.goBinaryName, g.goProxy, g.fs, goPathRoot, modPath, mod, ver)
if err != nil {
ClearFiles(g.fs, goPathRoot)
clearFiles(g.fs, goPathRoot)
return nil, errors.E(op, err)
}
@@ -108,7 +107,7 @@ func downloadModule(goBinaryName, goProxy string, fs afero.Fs, gopath, repoRoot,
fullURI := fmt.Sprintf("%s@%s", uri, version)
cmd := exec.Command(goBinaryName, "mod", "download", "-json", fullURI)
cmd.Env = PrepareEnv(gopath, goProxy)
cmd.Env = prepareEnv(gopath, goProxy)
cmd.Dir = repoRoot
stdout := &bytes.Buffer{}
stderr := &bytes.Buffer{}
@@ -139,65 +138,6 @@ func downloadModule(goBinaryName, goProxy string, fs afero.Fs, gopath, repoRoot,
return m, nil
}
// PrepareEnv will return all the appropriate
// environment variables for a Go Command to run
// successfully (such as GOPATH, GOCACHE, PATH etc)
func PrepareEnv(gopath, goProxy string) []string {
pathEnv := fmt.Sprintf("PATH=%s", os.Getenv("PATH"))
homeEnv := fmt.Sprintf("HOME=%s", os.Getenv("HOME"))
httpProxy := fmt.Sprintf("HTTP_PROXY=%s", os.Getenv("HTTP_PROXY"))
httpsProxy := fmt.Sprintf("HTTPS_PROXY=%s", os.Getenv("HTTPS_PROXY"))
noProxy := fmt.Sprintf("NO_PROXY=%s", os.Getenv("NO_PROXY"))
// need to also check the lower case version of just these three env variables
httpProxyLower := fmt.Sprintf("http_proxy=%s", os.Getenv("http_proxy"))
httpsProxyLower := fmt.Sprintf("https_proxy=%s", os.Getenv("https_proxy"))
noProxyLower := fmt.Sprintf("no_proxy=%s", os.Getenv("no_proxy"))
gopathEnv := fmt.Sprintf("GOPATH=%s", gopath)
goProxyEnv := fmt.Sprintf("GOPROXY=%s", goProxy)
cacheEnv := fmt.Sprintf("GOCACHE=%s", filepath.Join(gopath, "cache"))
gitSSH := fmt.Sprintf("GIT_SSH=%s", os.Getenv("GIT_SSH"))
gitSSHCmd := fmt.Sprintf("GIT_SSH_COMMAND=%s", os.Getenv("GIT_SSH_COMMAND"))
disableCgo := "CGO_ENABLED=0"
enableGoModules := "GO111MODULE=on"
cmdEnv := []string{
pathEnv,
homeEnv,
gopathEnv,
goProxyEnv,
cacheEnv,
disableCgo,
enableGoModules,
httpProxy,
httpsProxy,
noProxy,
httpProxyLower,
httpsProxyLower,
noProxyLower,
gitSSH,
gitSSHCmd,
}
if sshAuthSockVal, hasSSHAuthSock := os.LookupEnv("SSH_AUTH_SOCK"); hasSSHAuthSock {
// Verify that the ssh agent unix socket exists and is a unix socket.
st, err := os.Stat(sshAuthSockVal)
if err == nil && st.Mode()&os.ModeSocket != 0 {
sshAuthSock := fmt.Sprintf("SSH_AUTH_SOCK=%s", sshAuthSockVal)
cmdEnv = append(cmdEnv, sshAuthSock)
}
}
// add Windows specific ENV VARS
if runtime.GOOS == "windows" {
cmdEnv = append(cmdEnv, fmt.Sprintf("USERPROFILE=%s", os.Getenv("USERPROFILE")))
cmdEnv = append(cmdEnv, fmt.Sprintf("SystemRoot=%s", os.Getenv("SystemRoot")))
cmdEnv = append(cmdEnv, fmt.Sprintf("ALLUSERSPROFILE=%s", os.Getenv("ALLUSERSPROFILE")))
cmdEnv = append(cmdEnv, fmt.Sprintf("HOMEDRIVE=%s", os.Getenv("HOMEDRIVE")))
cmdEnv = append(cmdEnv, fmt.Sprintf("HOMEPATH=%s", os.Getenv("HOMEPATH")))
}
return cmdEnv
}
func isLimitHit(o string) bool {
return strings.Contains(o, "403 response from api.github.com")
}
pkg/download/upstream_lister.go → pkg/module/go_vcs_lister.go
+3 -10
View File
@@ -1,4 +1,4 @@
package download
package module
import (
"bytes"
@@ -10,18 +10,11 @@ import (
"github.com/gomods/athens/pkg/config"
"github.com/gomods/athens/pkg/errors"
"github.com/gomods/athens/pkg/module"
"github.com/gomods/athens/pkg/observ"
"github.com/gomods/athens/pkg/storage"
"github.com/spf13/afero"
)
// UpstreamLister retrieves a list of available module versions from upstream
// i.e. VCS, and a Storage backend.
type UpstreamLister interface {
List(ctx context.Context, mod string) (*storage.RevInfo, []string, error)
}
type listResp struct {
Path string
Version string
@@ -61,8 +54,8 @@ func (l *vcsLister) List(ctx context.Context, mod string) (*storage.RevInfo, []s
if err != nil {
return nil, nil, errors.E(op, err)
}
defer module.ClearFiles(l.fs, gopath)
cmd.Env = module.PrepareEnv(gopath, l.goProxy)
defer clearFiles(l.fs, gopath)
cmd.Env = prepareEnv(gopath, l.goProxy)
err = cmd.Run()
if err != nil {
pkg/module/prepare_env.go
+67
View File
@@ -0,0 +1,67 @@
package module
import (
"fmt"
"os"
"path/filepath"
"runtime"
)
// prepareEnv will return all the appropriate
// environment variables for a Go Command to run
// successfully (such as GOPATH, GOCACHE, PATH etc)
func prepareEnv(gopath, goProxy string) []string {
pathEnv := fmt.Sprintf("PATH=%s", os.Getenv("PATH"))
homeEnv := fmt.Sprintf("HOME=%s", os.Getenv("HOME"))
httpProxy := fmt.Sprintf("HTTP_PROXY=%s", os.Getenv("HTTP_PROXY"))
httpsProxy := fmt.Sprintf("HTTPS_PROXY=%s", os.Getenv("HTTPS_PROXY"))
noProxy := fmt.Sprintf("NO_PROXY=%s", os.Getenv("NO_PROXY"))
// need to also check the lower case version of just these three env variables
httpProxyLower := fmt.Sprintf("http_proxy=%s", os.Getenv("http_proxy"))
httpsProxyLower := fmt.Sprintf("https_proxy=%s", os.Getenv("https_proxy"))
noProxyLower := fmt.Sprintf("no_proxy=%s", os.Getenv("no_proxy"))
gopathEnv := fmt.Sprintf("GOPATH=%s", gopath)
goProxyEnv := fmt.Sprintf("GOPROXY=%s", goProxy)
cacheEnv := fmt.Sprintf("GOCACHE=%s", filepath.Join(gopath, "cache"))
gitSSH := fmt.Sprintf("GIT_SSH=%s", os.Getenv("GIT_SSH"))
gitSSHCmd := fmt.Sprintf("GIT_SSH_COMMAND=%s", os.Getenv("GIT_SSH_COMMAND"))
disableCgo := "CGO_ENABLED=0"
enableGoModules := "GO111MODULE=on"
cmdEnv := []string{
pathEnv,
homeEnv,
gopathEnv,
goProxyEnv,
cacheEnv,
disableCgo,
enableGoModules,
httpProxy,
httpsProxy,
noProxy,
httpProxyLower,
httpsProxyLower,
noProxyLower,
gitSSH,
gitSSHCmd,
}
if sshAuthSockVal, hasSSHAuthSock := os.LookupEnv("SSH_AUTH_SOCK"); hasSSHAuthSock {
// Verify that the ssh agent unix socket exists and is a unix socket.
st, err := os.Stat(sshAuthSockVal)
if err == nil && st.Mode()&os.ModeSocket != 0 {
sshAuthSock := fmt.Sprintf("SSH_AUTH_SOCK=%s", sshAuthSockVal)
cmdEnv = append(cmdEnv, sshAuthSock)
}
}
// add Windows specific ENV VARS
if runtime.GOOS == "windows" {
cmdEnv = append(cmdEnv, fmt.Sprintf("USERPROFILE=%s", os.Getenv("USERPROFILE")))
cmdEnv = append(cmdEnv, fmt.Sprintf("SystemRoot=%s", os.Getenv("SystemRoot")))
cmdEnv = append(cmdEnv, fmt.Sprintf("ALLUSERSPROFILE=%s", os.Getenv("ALLUSERSPROFILE")))
cmdEnv = append(cmdEnv, fmt.Sprintf("HOMEDRIVE=%s", os.Getenv("HOMEDRIVE")))
cmdEnv = append(cmdEnv, fmt.Sprintf("HOMEPATH=%s", os.Getenv("HOMEPATH")))
}
return cmdEnv
}
pkg/module/upstream_lister.go
+13
View File
@@ -0,0 +1,13 @@
package module
import (
"context"
"github.com/gomods/athens/pkg/storage"
)
// UpstreamLister retrieves a list of available module versions from upstream
// i.e. VCS, and a Storage backend.
type UpstreamLister interface {
List(ctx context.Context, mod string) (*storage.RevInfo, []string, error)
}
pkg/module/zip_read_closer.go
+3 -3
View File
@@ -18,16 +18,16 @@ type zipReadCloser struct {
// It is the caller's responsibility to call this method to free up utilized disk space
func (rc *zipReadCloser) Close() error {
rc.zip.Close()
return ClearFiles(rc.fs, rc.goPath)
return clearFiles(rc.fs, rc.goPath)
}
func (rc *zipReadCloser) Read(p []byte) (n int, err error) {
return rc.zip.Read(p)
}
// ClearFiles deletes all data from the given fs at path root
// clearFiles deletes all data from the given fs at path root
// This function must be called when zip is closed to cleanup the entire GOPATH created by the diskref
func ClearFiles(fs afero.Fs, root string) error {
func clearFiles(fs afero.Fs, root string) error {
const op errors.Op = "module.ClearFiles"
// This is required because vgo ensures dependencies are read-only
// See https://github.com/golang/go/issues/24111 and