mirror of
https://github.com/gomods/athens
synced 2026-02-03 07:30:32 +00:00
chore: lint code with golangci-lint (#1828)
* feat: add golangci-lint linting * chore: fix linter issues * feat: add linting into the workflow * docs: update lint docs * fix: cr suggestions * fix: remove old formatting and vetting scripts * fix: add docker make target * fix: action go caching * fix: depreciated actions checkout version * fix: cr suggestion * fix: cr suggestions --------- Co-authored-by: Manu Gupta <manugupt1@gmail.com>
This commit is contained in:
@@ -3,6 +3,18 @@ on:
|
|||||||
- push
|
- push
|
||||||
- pull_request
|
- pull_request
|
||||||
jobs:
|
jobs:
|
||||||
|
lint:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v3
|
||||||
|
- name: Set up Go
|
||||||
|
uses: actions/setup-go@v3
|
||||||
|
with:
|
||||||
|
go-version-file: 'go.mod'
|
||||||
|
cache: true
|
||||||
|
- name: Lint code
|
||||||
|
run: make lint-docker
|
||||||
|
|
||||||
build:
|
build:
|
||||||
env:
|
env:
|
||||||
ATHENS_MONGO_STORAGE_URL: mongodb://localhost:27017
|
ATHENS_MONGO_STORAGE_URL: mongodb://localhost:27017
|
||||||
@@ -53,7 +65,7 @@ jobs:
|
|||||||
with:
|
with:
|
||||||
go-version-file: 'go.mod'
|
go-version-file: 'go.mod'
|
||||||
cache: true
|
cache: true
|
||||||
- name: Run linter
|
- name: Verify changes
|
||||||
run: make verify
|
run: make verify
|
||||||
- name: Unit tests
|
- name: Unit tests
|
||||||
run: go test -v -race ./...
|
run: go test -v -race ./...
|
||||||
|
|||||||
@@ -0,0 +1,80 @@
|
|||||||
|
run:
|
||||||
|
tests: false
|
||||||
|
timeout: 5m
|
||||||
|
|
||||||
|
linters-settings:
|
||||||
|
cyclop:
|
||||||
|
max-complexity: 12
|
||||||
|
skip-tests: true
|
||||||
|
gofumpt:
|
||||||
|
extra-rules: true
|
||||||
|
|
||||||
|
linters:
|
||||||
|
enable-all: true
|
||||||
|
disable:
|
||||||
|
- interfacer # deprecated
|
||||||
|
- scopelint # deprecated
|
||||||
|
- maligned # deprecated
|
||||||
|
- golint # deprecated
|
||||||
|
- structcheck # deprecated
|
||||||
|
- deadcode # deprecated
|
||||||
|
- varcheck # deprecated
|
||||||
|
- nosnakecase # deprecated
|
||||||
|
- ifshort # deprecated
|
||||||
|
- errchkjson
|
||||||
|
- exhaustive
|
||||||
|
- exhaustivestruct
|
||||||
|
- exhaustruct
|
||||||
|
- forcetypeassert
|
||||||
|
- funlen
|
||||||
|
- gochecknoglobals
|
||||||
|
- gochecknoinits
|
||||||
|
- goconst
|
||||||
|
- godox
|
||||||
|
- goerr113
|
||||||
|
- gomnd
|
||||||
|
- ireturn
|
||||||
|
- lll
|
||||||
|
- musttag
|
||||||
|
- nilnil
|
||||||
|
- nlreturn
|
||||||
|
- nonamedreturns
|
||||||
|
- tagliatelle
|
||||||
|
- varnamelen
|
||||||
|
- wrapcheck
|
||||||
|
- wsl
|
||||||
|
- cyclop # TODO: turn this back on later
|
||||||
|
- gocognit # TODO: turn this back on later
|
||||||
|
- forbidigo # TODO: turn this back on later
|
||||||
|
|
||||||
|
issues:
|
||||||
|
exclude-use-default: false
|
||||||
|
exclude:
|
||||||
|
- 'package-comments: should have a package comment'
|
||||||
|
- 'ST1000: at least one file in a package should have a package comment'
|
||||||
|
- 'G204: Subprocess launched with a potential tainted input or cmd arguments'
|
||||||
|
- 'G204: Subprocess launched with variable'
|
||||||
|
- 'G402: TLS MinVersion too low.'
|
||||||
|
- 'const `op` is unused'
|
||||||
|
exclude-rules:
|
||||||
|
- path: cmd/proxy/main.go
|
||||||
|
text: 'G108: Profiling endpoint is automatically exposed on /debug/pprof'
|
||||||
|
- path: pkg/stash/stasher.go
|
||||||
|
linters:
|
||||||
|
- contextcheck
|
||||||
|
- path: pkg/stash/with_azureblob.go # False positive
|
||||||
|
linters:
|
||||||
|
- bodyclose
|
||||||
|
- path: pkg/storage/azureblob/azureblob.go # False positive
|
||||||
|
linters:
|
||||||
|
- bodyclose
|
||||||
|
- path: pkg/storage/compliance/*
|
||||||
|
linters:
|
||||||
|
- thelper
|
||||||
|
- gosec
|
||||||
|
- errcheck
|
||||||
|
- path: pkg/index/compliance/*
|
||||||
|
linters:
|
||||||
|
- thelper
|
||||||
|
- gosec
|
||||||
|
- errcheck
|
||||||
+2
-2
@@ -234,10 +234,10 @@ Then open [http://localhost:1313](http://localhost:1313/).
|
|||||||
|
|
||||||
# Linting
|
# Linting
|
||||||
|
|
||||||
In our CI/CD pass, we use govet, so feel free to run it locally beforehand:
|
In our CI/CD pass, we use golangci-lint, so feel free to run it locally beforehand:
|
||||||
|
|
||||||
```
|
```
|
||||||
go vet ./...
|
make lint
|
||||||
```
|
```
|
||||||
|
|
||||||
# For People Doing a Release
|
# For People Doing a Release
|
||||||
|
|||||||
@@ -1,6 +1,8 @@
|
|||||||
VERSION = "unset"
|
VERSION = "unset"
|
||||||
DATE=$(shell date -u +%Y-%m-%d-%H:%M:%S-%Z)
|
DATE=$(shell date -u +%Y-%m-%d-%H:%M:%S-%Z)
|
||||||
|
|
||||||
|
GOLANGCI_LINT_VERSION=v1.51.2
|
||||||
|
|
||||||
ifndef GOLANG_VERSION
|
ifndef GOLANG_VERSION
|
||||||
override GOLANG_VERSION = 1.19
|
override GOLANG_VERSION = 1.19
|
||||||
endif
|
endif
|
||||||
@@ -43,10 +45,16 @@ docs: ## build the docs docker image
|
|||||||
setup-dev-env:
|
setup-dev-env:
|
||||||
$(MAKE) dev
|
$(MAKE) dev
|
||||||
|
|
||||||
|
.PHONY: lint
|
||||||
|
lint:
|
||||||
|
@golangci-lint run ./...
|
||||||
|
|
||||||
|
.PHONY: lint-docker
|
||||||
|
lint-docker:
|
||||||
|
@docker run -t --rm -v $(CURDIR):/app -w /app golangci/golangci-lint:$(GOLANGCI_LINT_VERSION) golangci-lint run ./...
|
||||||
|
|
||||||
.PHONY: verify
|
.PHONY: verify
|
||||||
verify: ## verify athens codebase
|
verify: ## verify athens codebase
|
||||||
./scripts/check_gofmt.sh
|
|
||||||
./scripts/check_govet.sh
|
|
||||||
./scripts/check_deps.sh
|
./scripts/check_deps.sh
|
||||||
./scripts/check_conflicts.sh
|
./scripts/check_conflicts.sh
|
||||||
|
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ import (
|
|||||||
"go.opencensus.io/plugin/ochttp"
|
"go.opencensus.io/plugin/ochttp"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Service is the name of the service that we want to tag our processes with
|
// Service is the name of the service that we want to tag our processes with.
|
||||||
const Service = "proxy"
|
const Service = "proxy"
|
||||||
|
|
||||||
// App is where all routes and middleware for the proxy
|
// App is where all routes and middleware for the proxy
|
||||||
@@ -126,7 +126,7 @@ func App(conf *config.Config) (http.Handler, error) {
|
|||||||
|
|
||||||
store, err := GetStorage(conf.StorageType, conf.Storage, conf.TimeoutDuration(), client)
|
store, err := GetStorage(conf.StorageType, conf.Storage, conf.TimeoutDuration(), client)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
err = fmt.Errorf("error getting storage configuration (%s)", err)
|
err = fmt.Errorf("error getting storage configuration: %w", err)
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -140,7 +140,7 @@ func App(conf *config.Config) (http.Handler, error) {
|
|||||||
lggr,
|
lggr,
|
||||||
conf,
|
conf,
|
||||||
); err != nil {
|
); err != nil {
|
||||||
err = fmt.Errorf("error adding proxy routes (%s)", err)
|
err = fmt.Errorf("error adding proxy routes: %w", err)
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package actions
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
@@ -54,7 +55,7 @@ func addProxyRoutes(
|
|||||||
}
|
}
|
||||||
supportPath := path.Join("/sumdb", sumdbURL.Host, "/supported")
|
supportPath := path.Join("/sumdb", sumdbURL.Host, "/supported")
|
||||||
r.HandleFunc(supportPath, func(w http.ResponseWriter, r *http.Request) {
|
r.HandleFunc(supportPath, func(w http.ResponseWriter, r *http.Request) {
|
||||||
w.WriteHeader(200)
|
w.WriteHeader(http.StatusOK)
|
||||||
})
|
})
|
||||||
sumHandler := sumdbProxy(sumdbURL, c.NoSumPatterns)
|
sumHandler := sumdbProxy(sumdbURL, c.NoSumPatterns)
|
||||||
pathPrefix := "/sumdb/" + sumdbURL.Host
|
pathPrefix := "/sumdb/" + sumdbURL.Host
|
||||||
@@ -63,7 +64,7 @@ func addProxyRoutes(
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Download Protocol
|
// Download Protocol:
|
||||||
// the download.Protocol and the stash.Stasher interfaces are composable
|
// the download.Protocol and the stash.Stasher interfaces are composable
|
||||||
// in a middleware fashion. Therefore you can separate concerns
|
// in a middleware fashion. Therefore you can separate concerns
|
||||||
// by the functionality: a download.Protocol that just takes care
|
// by the functionality: a download.Protocol that just takes care
|
||||||
@@ -142,13 +143,13 @@ func getSingleFlight(l *log.Logger, c *config.Config, checker storage.Checker) (
|
|||||||
return stash.WithSingleflight, nil
|
return stash.WithSingleflight, nil
|
||||||
case "etcd":
|
case "etcd":
|
||||||
if c.SingleFlight == nil || c.SingleFlight.Etcd == nil {
|
if c.SingleFlight == nil || c.SingleFlight.Etcd == nil {
|
||||||
return nil, fmt.Errorf("Etcd config must be present")
|
return nil, errors.New("etcd config must be present")
|
||||||
}
|
}
|
||||||
endpoints := strings.Split(c.SingleFlight.Etcd.Endpoints, ",")
|
endpoints := strings.Split(c.SingleFlight.Etcd.Endpoints, ",")
|
||||||
return stash.WithEtcd(endpoints, checker)
|
return stash.WithEtcd(endpoints, checker)
|
||||||
case "redis":
|
case "redis":
|
||||||
if c.SingleFlight == nil || c.SingleFlight.Redis == nil {
|
if c.SingleFlight == nil || c.SingleFlight.Redis == nil {
|
||||||
return nil, fmt.Errorf("Redis config must be present")
|
return nil, errors.New("redis config must be present")
|
||||||
}
|
}
|
||||||
return stash.WithRedisLock(
|
return stash.WithRedisLock(
|
||||||
&athensLoggerForRedis{logger: l},
|
&athensLoggerForRedis{logger: l},
|
||||||
@@ -158,7 +159,7 @@ func getSingleFlight(l *log.Logger, c *config.Config, checker storage.Checker) (
|
|||||||
c.SingleFlight.Redis.LockConfig)
|
c.SingleFlight.Redis.LockConfig)
|
||||||
case "redis-sentinel":
|
case "redis-sentinel":
|
||||||
if c.SingleFlight == nil || c.SingleFlight.RedisSentinel == nil {
|
if c.SingleFlight == nil || c.SingleFlight.RedisSentinel == nil {
|
||||||
return nil, fmt.Errorf("Redis config must be present")
|
return nil, errors.New("redis config must be present")
|
||||||
}
|
}
|
||||||
return stash.WithRedisSentinelLock(
|
return stash.WithRedisSentinelLock(
|
||||||
&athensLoggerForRedis{logger: l},
|
&athensLoggerForRedis{logger: l},
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ func initializeAuthFile(path string) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
fileBts, err := os.ReadFile(path)
|
fileBts, err := os.ReadFile(filepath.Clean(path))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatalf("could not read %s: %v", path, err)
|
log.Fatalf("could not read %s: %v", path, err)
|
||||||
}
|
}
|
||||||
@@ -31,7 +31,7 @@ func initializeAuthFile(path string) {
|
|||||||
|
|
||||||
fileName := transformAuthFileName(filepath.Base(path))
|
fileName := transformAuthFileName(filepath.Base(path))
|
||||||
rcp := filepath.Join(hdir, fileName)
|
rcp := filepath.Join(hdir, fileName)
|
||||||
if err := os.WriteFile(rcp, fileBts, 0600); err != nil {
|
if err := os.WriteFile(rcp, fileBts, 0o600); err != nil {
|
||||||
log.Fatalf("could not write to file: %v", err)
|
log.Fatalf("could not write to file: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -45,7 +45,7 @@ func netrcFromToken(tok string) {
|
|||||||
log.Fatalf("netrcFromToken: could not get homedir: %v", err)
|
log.Fatalf("netrcFromToken: could not get homedir: %v", err)
|
||||||
}
|
}
|
||||||
rcp := filepath.Join(hdir, getNETRCFilename())
|
rcp := filepath.Join(hdir, getNETRCFilename())
|
||||||
if err := os.WriteFile(rcp, []byte(fileContent), 0600); err != nil {
|
if err := os.WriteFile(rcp, []byte(fileContent), 0o600); err != nil {
|
||||||
log.Fatalf("netrcFromToken: could not write to file: %v", err)
|
log.Fatalf("netrcFromToken: could not write to file: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,10 +8,8 @@ import (
|
|||||||
"github.com/gorilla/mux"
|
"github.com/gorilla/mux"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
// basicAuthExcludedPaths is a regular expression that matches paths that should not be protected by HTTP basic authentication.
|
||||||
// basicAuthExcludedPaths is a regular expression that matches paths that should not be protected by HTTP basic authentication.
|
var basicAuthExcludedPaths = regexp.MustCompile("^/(health|ready)z$")
|
||||||
basicAuthExcludedPaths = regexp.MustCompile("^/(health|ready)z$")
|
|
||||||
)
|
|
||||||
|
|
||||||
func basicAuth(user, pass string) mux.MiddlewareFunc {
|
func basicAuth(user, pass string) mux.MiddlewareFunc {
|
||||||
return func(h http.Handler) http.Handler {
|
return func(h http.Handler) http.Handler {
|
||||||
@@ -40,9 +38,5 @@ func checkAuth(r *http.Request, user, pass string) bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
isPass := subtle.ConstantTimeCompare([]byte(pass), []byte(givenPass))
|
isPass := subtle.ConstantTimeCompare([]byte(pass), []byte(givenPass))
|
||||||
if isPass != 1 {
|
return isPass == 1
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
return true
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ type catalogRes struct {
|
|||||||
NextPageToken string `json:"next,omitempty"`
|
NextPageToken string `json:"next,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// catalogHandler implements GET baseURL/catalog
|
// catalogHandler implements GET baseURL/catalog.
|
||||||
func catalogHandler(s storage.Backend) http.HandlerFunc {
|
func catalogHandler(s storage.Backend) http.HandlerFunc {
|
||||||
const op errors.Op = "actions.CatalogHandler"
|
const op errors.Op = "actions.CatalogHandler"
|
||||||
cs, isCataloger := s.(storage.Cataloger)
|
cs, isCataloger := s.(storage.Cataloger)
|
||||||
@@ -54,7 +54,7 @@ func catalogHandler(s storage.Backend) http.HandlerFunc {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// getLimitFromParam converts a URL query parameter into an int
|
// getLimitFromParam converts a URL query parameter into an int
|
||||||
// otherwise converts defaultPageSize constant
|
// otherwise converts defaultPageSize constant.
|
||||||
func getLimitFromParam(param string) (int, error) {
|
func getLimitFromParam(param string) (int, error) {
|
||||||
if param == "" {
|
if param == "" {
|
||||||
return defaultPageSize, nil
|
return defaultPageSize, nil
|
||||||
|
|||||||
@@ -5,5 +5,5 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func proxyHomeHandler(w http.ResponseWriter, r *http.Request) {
|
func proxyHomeHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
w.Write([]byte(`"Welcome to The Athens Proxy"`))
|
_, _ = w.Write([]byte(`"Welcome to The Athens Proxy"`))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ import (
|
|||||||
"github.com/sirupsen/logrus"
|
"github.com/sirupsen/logrus"
|
||||||
)
|
)
|
||||||
|
|
||||||
// indexHandler implements GET baseURL/index
|
// indexHandler implements GET baseURL/index.
|
||||||
func indexHandler(index index.Indexer) http.HandlerFunc {
|
func indexHandler(index index.Indexer) http.HandlerFunc {
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
ctx := r.Context()
|
ctx := r.Context()
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ import (
|
|||||||
"github.com/gomods/athens/pkg/config"
|
"github.com/gomods/athens/pkg/config"
|
||||||
)
|
)
|
||||||
|
|
||||||
// robotsHandler implements GET baseURL/robots.txt
|
// robotsHandler implements GET baseURL/robots.txt.
|
||||||
func robotsHandler(c *config.Config) http.HandlerFunc {
|
func robotsHandler(c *config.Config) http.HandlerFunc {
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
http.ServeFile(w, r, c.RobotsFile)
|
http.ServeFile(w, r, c.RobotsFile)
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ import (
|
|||||||
"github.com/spf13/afero"
|
"github.com/spf13/afero"
|
||||||
)
|
)
|
||||||
|
|
||||||
// GetStorage returns storage backend based on env configuration
|
// GetStorage returns storage backend based on env configuration.
|
||||||
func GetStorage(storageType string, storageConfig *config.Storage, timeout time.Duration, client *http.Client) (storage.Backend, error) {
|
func GetStorage(storageType string, storageConfig *config.Storage, timeout time.Duration, client *http.Client) (storage.Backend, error) {
|
||||||
const op errors.Op = "actions.GetStorage"
|
const op errors.Op = "actions.GetStorage"
|
||||||
switch storageType {
|
switch storageType {
|
||||||
|
|||||||
@@ -17,12 +17,12 @@ func sumdbProxy(url *url.URL, nosumPatterns []string) http.Handler {
|
|||||||
req.URL.Host = url.Host
|
req.URL.Host = url.Host
|
||||||
}
|
}
|
||||||
if len(nosumPatterns) > 0 {
|
if len(nosumPatterns) > 0 {
|
||||||
return noSumWrapper(rp, url.Host, nosumPatterns)
|
return noSumWrapper(rp, nosumPatterns)
|
||||||
}
|
}
|
||||||
return rp
|
return rp
|
||||||
}
|
}
|
||||||
|
|
||||||
func noSumWrapper(h http.Handler, host string, patterns []string) http.Handler {
|
func noSumWrapper(h http.Handler, patterns []string) http.Handler {
|
||||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
if strings.HasPrefix(r.URL.Path, "/lookup/") {
|
if strings.HasPrefix(r.URL.Path, "/lookup/") {
|
||||||
for _, p := range patterns {
|
for _, p := range patterns {
|
||||||
|
|||||||
@@ -97,7 +97,7 @@ func TestNoSumPatterns(t *testing.T) {
|
|||||||
for _, tc := range noSumTestCases {
|
for _, tc := range noSumTestCases {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
w := httptest.NewRecorder()
|
w := httptest.NewRecorder()
|
||||||
skipHandler := noSumWrapper(http.HandlerFunc(emptyHandler), "sum.golang.org", tc.patterns)
|
skipHandler := noSumWrapper(http.HandlerFunc(emptyHandler), tc.patterns)
|
||||||
req := httptest.NewRequest("GET", "/lookup/"+tc.given, nil)
|
req := httptest.NewRequest("GET", "/lookup/"+tc.given, nil)
|
||||||
skipHandler.ServeHTTP(w, req)
|
skipHandler.ServeHTTP(w, req)
|
||||||
if tc.status != w.Code {
|
if tc.status != w.Code {
|
||||||
|
|||||||
@@ -8,5 +8,5 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
func versionHandler(w http.ResponseWriter, r *http.Request) {
|
func versionHandler(w http.ResponseWriter, r *http.Request) {
|
||||||
json.NewEncoder(w).Encode(build.Data())
|
_ = json.NewEncoder(w).Encode(build.Data())
|
||||||
}
|
}
|
||||||
|
|||||||
+8
-7
@@ -2,16 +2,16 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"flag"
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log"
|
"log"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
_ "net/http/pprof"
|
||||||
"os"
|
"os"
|
||||||
"os/signal"
|
"os/signal"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
_ "net/http/pprof"
|
|
||||||
|
|
||||||
"github.com/gomods/athens/cmd/proxy/actions"
|
"github.com/gomods/athens/cmd/proxy/actions"
|
||||||
"github.com/gomods/athens/internal/shutdown"
|
"github.com/gomods/athens/internal/shutdown"
|
||||||
"github.com/gomods/athens/pkg/build"
|
"github.com/gomods/athens/pkg/build"
|
||||||
@@ -45,8 +45,9 @@ func main() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
srv := &http.Server{
|
srv := &http.Server{
|
||||||
Addr: conf.Port,
|
Addr: conf.Port,
|
||||||
Handler: handler,
|
Handler: handler,
|
||||||
|
ReadHeaderTimeout: 2 * time.Second,
|
||||||
}
|
}
|
||||||
idleConnsClosed := make(chan struct{})
|
idleConnsClosed := make(chan struct{})
|
||||||
|
|
||||||
@@ -54,7 +55,7 @@ func main() {
|
|||||||
sigint := make(chan os.Signal, 1)
|
sigint := make(chan os.Signal, 1)
|
||||||
signal.Notify(sigint, shutdown.GetSignals()...)
|
signal.Notify(sigint, shutdown.GetSignals()...)
|
||||||
s := <-sigint
|
s := <-sigint
|
||||||
log.Printf("Recived signal (%s): Shutting down server", s)
|
log.Printf("Received signal (%s): Shutting down server", s)
|
||||||
|
|
||||||
// We received an interrupt signal, shut down.
|
// We received an interrupt signal, shut down.
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*time.Duration(conf.ShutdownTimeout))
|
ctx, cancel := context.WithTimeout(context.Background(), time.Second*time.Duration(conf.ShutdownTimeout))
|
||||||
@@ -70,7 +71,7 @@ func main() {
|
|||||||
// pprof to be exposed on a different port than the application for security matters, not to expose profiling data and avoid DoS attacks (profiling slows down the service)
|
// pprof to be exposed on a different port than the application for security matters, not to expose profiling data and avoid DoS attacks (profiling slows down the service)
|
||||||
// https://www.farsightsecurity.com/txt-record/2016/10/28/cmikk-go-remote-profiling/
|
// https://www.farsightsecurity.com/txt-record/2016/10/28/cmikk-go-remote-profiling/
|
||||||
log.Printf("Starting `pprof` at port %v", conf.PprofPort)
|
log.Printf("Starting `pprof` at port %v", conf.PprofPort)
|
||||||
log.Fatal(http.ListenAndServe(conf.PprofPort, nil))
|
log.Fatal(http.ListenAndServe(conf.PprofPort, nil)) //nolint:gosec // This should not be exposed to the world.
|
||||||
}()
|
}()
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -81,7 +82,7 @@ func main() {
|
|||||||
err = srv.ListenAndServe()
|
err = srv.ListenAndServe()
|
||||||
}
|
}
|
||||||
|
|
||||||
if err != http.ErrServerClosed {
|
if !errors.Is(err, http.ErrServerClosed) {
|
||||||
log.Fatal(err)
|
log.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -79,8 +79,6 @@ if ($docs.IsPresent) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if ($verify.IsPresent) {
|
if ($verify.IsPresent) {
|
||||||
execScript "check_gofmt.ps1"
|
|
||||||
execScript "check_govet.ps1"
|
|
||||||
execScript "check_deps.ps1"
|
execScript "check_deps.ps1"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+2
-2
@@ -9,7 +9,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Details represents known data for a given build
|
// Details represents known data for a given build.
|
||||||
type Details struct {
|
type Details struct {
|
||||||
Version string `json:"version,omitempty"`
|
Version string `json:"version,omitempty"`
|
||||||
Date string `json:"date,omitempty"`
|
Date string `json:"date,omitempty"`
|
||||||
@@ -29,7 +29,7 @@ func String() string {
|
|||||||
return fmt.Sprintf("Build Details:\n\tVersion:\t%s\n\tDate:\t\t%s", version, buildDate)
|
return fmt.Sprintf("Build Details:\n\tVersion:\t%s\n\tDate:\t\t%s", version, buildDate)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Data returns build details as a struct
|
// Data returns build details as a struct.
|
||||||
func Data() Details {
|
func Data() Details {
|
||||||
return Details{
|
return Details{
|
||||||
Version: version,
|
Version: version,
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
package config
|
package config
|
||||||
|
|
||||||
// AzureBlobConfig specifies the properties required to use Azure as the storage backend
|
// AzureBlobConfig specifies the properties required to use Azure as the storage backend.
|
||||||
type AzureBlobConfig struct {
|
type AzureBlobConfig struct {
|
||||||
AccountName string `validate:"required" envconfig:"ATHENS_AZURE_ACCOUNT_NAME"`
|
AccountName string `validate:"required" envconfig:"ATHENS_AZURE_ACCOUNT_NAME"`
|
||||||
AccountKey string `validate:"required" envconfig:"ATHENS_AZURE_ACCOUNT_KEY"`
|
AccountKey string `validate:"required" envconfig:"ATHENS_AZURE_ACCOUNT_KEY"`
|
||||||
|
|||||||
+18
-20
@@ -18,7 +18,7 @@ import (
|
|||||||
|
|
||||||
const defaultConfigFile = "athens.toml"
|
const defaultConfigFile = "athens.toml"
|
||||||
|
|
||||||
// Config provides configuration values for all components
|
// Config provides configuration values for all components.
|
||||||
type Config struct {
|
type Config struct {
|
||||||
TimeoutConf
|
TimeoutConf
|
||||||
GoEnv string `validate:"required" envconfig:"GO_ENV"`
|
GoEnv string `validate:"required" envconfig:"GO_ENV"`
|
||||||
@@ -63,12 +63,12 @@ type Config struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// EnvList is a list of key-value environment
|
// EnvList is a list of key-value environment
|
||||||
// variables that are passed to the Go command
|
// variables that are passed to the Go command.
|
||||||
type EnvList []string
|
type EnvList []string
|
||||||
|
|
||||||
// HasKey returns whether a key-value entry
|
// HasKey returns whether a key-value entry
|
||||||
// is present by only checking the left of
|
// is present by only checking the left of
|
||||||
// key=value
|
// key=value.
|
||||||
func (el EnvList) HasKey(key string) bool {
|
func (el EnvList) HasKey(key string) bool {
|
||||||
for _, env := range el {
|
for _, env := range el {
|
||||||
if strings.HasPrefix(env, key+"=") {
|
if strings.HasPrefix(env, key+"=") {
|
||||||
@@ -79,7 +79,7 @@ func (el EnvList) HasKey(key string) bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Add adds a key=value entry to the environment
|
// Add adds a key=value entry to the environment
|
||||||
// list
|
// list.
|
||||||
func (el *EnvList) Add(key, value string) {
|
func (el *EnvList) Add(key, value string) {
|
||||||
*el = append(*el, key+"="+value)
|
*el = append(*el, key+"="+value)
|
||||||
}
|
}
|
||||||
@@ -110,7 +110,7 @@ func (el *EnvList) Decode(value string) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Validate validates that all strings inside the
|
// Validate validates that all strings inside the
|
||||||
// list are of the key=value format
|
// list are of the key=value format.
|
||||||
func (el EnvList) Validate() error {
|
func (el EnvList) Validate() error {
|
||||||
const op errors.Op = "EnvList.Validate"
|
const op errors.Op = "EnvList.Validate"
|
||||||
for _, env := range el {
|
for _, env := range el {
|
||||||
@@ -123,7 +123,7 @@ func (el EnvList) Validate() error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Load loads the config from a file.
|
// Load loads the config from a file.
|
||||||
// If file is not present returns default config
|
// If file is not present returns default config.
|
||||||
func Load(configFile string) (*Config, error) {
|
func Load(configFile string) (*Config, error) {
|
||||||
// User explicitly specified a config file
|
// User explicitly specified a config file
|
||||||
if configFile != "" {
|
if configFile != "" {
|
||||||
@@ -206,7 +206,7 @@ func defaultConfig() *Config {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// BasicAuth returns BasicAuthUser and BasicAuthPassword
|
// BasicAuth returns BasicAuthUser and BasicAuthPassword
|
||||||
// and ok if neither of them are empty
|
// and ok if neither of them are empty.
|
||||||
func (c *Config) BasicAuth() (user, pass string, ok bool) {
|
func (c *Config) BasicAuth() (user, pass string, ok bool) {
|
||||||
user = c.BasicAuthUser
|
user = c.BasicAuthUser
|
||||||
pass = c.BasicAuthPass
|
pass = c.BasicAuthPass
|
||||||
@@ -215,7 +215,7 @@ func (c *Config) BasicAuth() (user, pass string, ok bool) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// TLSCertFiles returns certificate and key files and an error if
|
// TLSCertFiles returns certificate and key files and an error if
|
||||||
// both files doesn't exist and have approperiate file permissions
|
// both files doesn't exist and have approperiate file permissions.
|
||||||
func (c *Config) TLSCertFiles() (cert, key string, err error) {
|
func (c *Config) TLSCertFiles() (cert, key string, err error) {
|
||||||
if c.TLSCertFile == "" && c.TLSKeyFile == "" {
|
if c.TLSCertFile == "" && c.TLSKeyFile == "" {
|
||||||
return "", "", nil
|
return "", "", nil
|
||||||
@@ -223,29 +223,28 @@ func (c *Config) TLSCertFiles() (cert, key string, err error) {
|
|||||||
|
|
||||||
certFile, err := os.Stat(c.TLSCertFile)
|
certFile, err := os.Stat(c.TLSCertFile)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", "", fmt.Errorf("Could not access TLSCertFile: %v", err)
|
return "", "", fmt.Errorf("could not access TLSCertFile: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
keyFile, err := os.Stat(c.TLSKeyFile)
|
keyFile, err := os.Stat(c.TLSKeyFile)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", "", fmt.Errorf("Could not access TLSKeyFile: %v", err)
|
return "", "", fmt.Errorf("could not access TLSKeyFile: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if keyFile.Mode()&077 != 0 && runtime.GOOS != "windows" {
|
if keyFile.Mode()&0o077 != 0 && runtime.GOOS != "windows" {
|
||||||
return "", "", fmt.Errorf("TLSKeyFile should not be accessible by others")
|
return "", "", fmt.Errorf("TLSKeyFile should not be accessible by others")
|
||||||
}
|
}
|
||||||
|
|
||||||
return certFile.Name(), keyFile.Name(), nil
|
return certFile.Name(), keyFile.Name(), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// FilterOff returns true if the FilterFile is empty
|
// FilterOff returns true if the FilterFile is empty.
|
||||||
func (c *Config) FilterOff() bool {
|
func (c *Config) FilterOff() bool {
|
||||||
return c.FilterFile == ""
|
return c.FilterFile == ""
|
||||||
}
|
}
|
||||||
|
|
||||||
// ParseConfigFile parses the given file into an athens config struct
|
// ParseConfigFile parses the given file into an athens config struct.
|
||||||
func ParseConfigFile(configFile string) (*Config, error) {
|
func ParseConfigFile(configFile string) (*Config, error) {
|
||||||
|
|
||||||
var config Config
|
var config Config
|
||||||
// attempt to read the given config file
|
// attempt to read the given config file
|
||||||
if _, err := toml.DecodeFile(configFile, &config); err != nil {
|
if _, err := toml.DecodeFile(configFile, &config); err != nil {
|
||||||
@@ -271,7 +270,7 @@ func ParseConfigFile(configFile string) (*Config, error) {
|
|||||||
return &config, nil
|
return &config, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// envOverride uses Environment variables to override unspecified properties
|
// envOverride uses Environment variables to override unspecified properties.
|
||||||
func envOverride(config *Config) error {
|
func envOverride(config *Config) error {
|
||||||
const defaultPort = ":3000"
|
const defaultPort = ":3000"
|
||||||
err := envconfig.Process("athens", config)
|
err := envconfig.Process("athens", config)
|
||||||
@@ -355,16 +354,16 @@ func validateIndex(validate *validator.Validate, indexType string, config *Index
|
|||||||
func GetConf(path string) (*Config, error) {
|
func GetConf(path string) (*Config, error) {
|
||||||
absPath, err := filepath.Abs(path)
|
absPath, err := filepath.Abs(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("Unable to construct absolute path to test config file")
|
return nil, fmt.Errorf("unable to construct absolute path to test config file")
|
||||||
}
|
}
|
||||||
conf, err := ParseConfigFile(absPath)
|
conf, err := ParseConfigFile(absPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("Unable to parse test config file: %s", err.Error())
|
return nil, fmt.Errorf("unable to parse test config file: %w", err)
|
||||||
}
|
}
|
||||||
return conf, nil
|
return conf, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// checkFilePerms given a list of files
|
// checkFilePerms given a list of files.
|
||||||
func checkFilePerms(files ...string) error {
|
func checkFilePerms(files ...string) error {
|
||||||
const op = "config.checkFilePerms"
|
const op = "config.checkFilePerms"
|
||||||
|
|
||||||
@@ -384,10 +383,9 @@ func checkFilePerms(files ...string) error {
|
|||||||
// Assume unix based system (MacOS and Linux)
|
// Assume unix based system (MacOS and Linux)
|
||||||
// the bit mask is calculated using the umask command which tells which permissions
|
// the bit mask is calculated using the umask command which tells which permissions
|
||||||
// should not be allowed for a particular user, group or world
|
// should not be allowed for a particular user, group or world
|
||||||
if fInfo.Mode()&0077 != 0 && runtime.GOOS != "windows" {
|
if fInfo.Mode()&0o077 != 0 && runtime.GOOS != "windows" {
|
||||||
return errors.E(op, f+" should have at most rwx,-, - (bit mask 077) as permission")
|
return errors.E(op, f+" should have at most rwx,-, - (bit mask 077) as permission")
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
package config
|
package config
|
||||||
|
|
||||||
// DiskConfig specifies the properties required to use Disk as the storage backend
|
// DiskConfig specifies the properties required to use Disk as the storage backend.
|
||||||
type DiskConfig struct {
|
type DiskConfig struct {
|
||||||
RootPath string `validate:"required" envconfig:"ATHENS_DISK_STORAGE_ROOT"`
|
RootPath string `validate:"required" envconfig:"ATHENS_DISK_STORAGE_ROOT"`
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
package config
|
package config
|
||||||
|
|
||||||
// External specifies configuration for an external http storage
|
// External specifies configuration for an external http storage.
|
||||||
type External struct {
|
type External struct {
|
||||||
URL string `validate:"required" envconfig:"ATHENS_EXTERNAL_STORAGE_URL"`
|
URL string `validate:"required" envconfig:"ATHENS_EXTERNAL_STORAGE_URL"`
|
||||||
}
|
}
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
package config
|
package config
|
||||||
|
|
||||||
// GCPConfig specifies the properties required to use GCP as the storage backend
|
// GCPConfig specifies the properties required to use GCP as the storage backend.
|
||||||
type GCPConfig struct {
|
type GCPConfig struct {
|
||||||
ProjectID string `envconfig:"GOOGLE_CLOUD_PROJECT"`
|
ProjectID string `envconfig:"GOOGLE_CLOUD_PROJECT"`
|
||||||
Bucket string `validate:"required" envconfig:"ATHENS_STORAGE_GCP_BUCKET"`
|
Bucket string `validate:"required" envconfig:"ATHENS_STORAGE_GCP_BUCKET"`
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
package config
|
package config
|
||||||
|
|
||||||
// Index is the config for various index storage backends
|
// Index is the config for various index storage backends.
|
||||||
type Index struct {
|
type Index struct {
|
||||||
MySQL *MySQL
|
MySQL *MySQL
|
||||||
Postgres *Postgres
|
Postgres *Postgres
|
||||||
|
|||||||
+1
-1
@@ -1,7 +1,7 @@
|
|||||||
package config
|
package config
|
||||||
|
|
||||||
// MinioConfig specifies the properties required to use Minio or DigitalOcean Spaces
|
// MinioConfig specifies the properties required to use Minio or DigitalOcean Spaces
|
||||||
// as the storage backend
|
// as the storage backend.
|
||||||
type MinioConfig struct {
|
type MinioConfig struct {
|
||||||
Endpoint string `validate:"required" envconfig:"ATHENS_MINIO_ENDPOINT"`
|
Endpoint string `validate:"required" envconfig:"ATHENS_MINIO_ENDPOINT"`
|
||||||
Key string `validate:"required" envconfig:"ATHENS_MINIO_ACCESS_KEY_ID"`
|
Key string `validate:"required" envconfig:"ATHENS_MINIO_ACCESS_KEY_ID"`
|
||||||
|
|||||||
@@ -7,20 +7,20 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// PackageVersionedName return package full name used in storage.
|
// PackageVersionedName return package full name used in storage.
|
||||||
// E.g athens/@v/v1.0.mod
|
// E.g athens/@v/v1.0.mod.
|
||||||
func PackageVersionedName(module, version, ext string) string {
|
func PackageVersionedName(module, version, ext string) string {
|
||||||
return fmt.Sprintf("%s/@v/%s.%s", module, version, ext)
|
return fmt.Sprintf("%s/@v/%s.%s", module, version, ext)
|
||||||
}
|
}
|
||||||
|
|
||||||
// FmtModVer is a helper function that can take
|
// FmtModVer is a helper function that can take
|
||||||
// pkg/a/b and v2.3.1 and returns pkg/a/b@v2.3.1
|
// pkg/a/b and v2.3.1 and returns pkg/a/b@v2.3.1.
|
||||||
func FmtModVer(mod, ver string) string {
|
func FmtModVer(mod, ver string) string {
|
||||||
return fmt.Sprintf("%s@%s", mod, ver)
|
return fmt.Sprintf("%s@%s", mod, ver)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ModuleVersionFromPath returns module and version from a
|
// ModuleVersionFromPath returns module and version from a
|
||||||
// storage path
|
// storage path.
|
||||||
// E.g athens/@v/v1.0.info -> athens and v.1.0
|
// E.g athens/@v/v1.0.info -> athens and v.1.0.
|
||||||
func ModuleVersionFromPath(path string) (string, string) {
|
func ModuleVersionFromPath(path string) (string, string) {
|
||||||
segments := strings.Split(path, "/@v/")
|
segments := strings.Split(path, "/@v/")
|
||||||
if len(segments) != 2 {
|
if len(segments) != 2 {
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
package config
|
package config
|
||||||
|
|
||||||
// MongoConfig specifies the properties required to use MongoDB as the storage backend
|
// MongoConfig specifies the properties required to use MongoDB as the storage backend.
|
||||||
type MongoConfig struct {
|
type MongoConfig struct {
|
||||||
URL string `validate:"required" envconfig:"ATHENS_MONGO_STORAGE_URL"`
|
URL string `validate:"required" envconfig:"ATHENS_MONGO_STORAGE_URL"`
|
||||||
DefaultDBName string `envconfig:"ATHENS_MONGO_DEFAULT_DATABASE" default:"athens"`
|
DefaultDBName string `envconfig:"ATHENS_MONGO_DEFAULT_DATABASE" default:"athens"`
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
package config
|
package config
|
||||||
|
|
||||||
// MySQL config
|
// MySQL config.
|
||||||
type MySQL struct {
|
type MySQL struct {
|
||||||
Protocol string `validate:"required" envconfig:"ATHENS_INDEX_MYSQL_PROTOCOL"`
|
Protocol string `validate:"required" envconfig:"ATHENS_INDEX_MYSQL_PROTOCOL"`
|
||||||
Host string `validate:"required" envconfig:"ATHENS_INDEX_MYSQL_HOST"`
|
Host string `validate:"required" envconfig:"ATHENS_INDEX_MYSQL_HOST"`
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
package config
|
package config
|
||||||
|
|
||||||
// Postgres config
|
// Postgres config.
|
||||||
type Postgres struct {
|
type Postgres struct {
|
||||||
Host string `validate:"required" envconfig:"ATHENS_INDEX_POSTGRES_HOST"`
|
Host string `validate:"required" envconfig:"ATHENS_INDEX_POSTGRES_HOST"`
|
||||||
Port int `validate:"required" envconfig:"ATHENS_INDEX_POSTGRES_PORT"`
|
Port int `validate:"required" envconfig:"ATHENS_INDEX_POSTGRES_PORT"`
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
package config
|
package config
|
||||||
|
|
||||||
// S3Config specifies the properties required to use S3 as the storage backend
|
// S3Config specifies the properties required to use S3 as the storage backend.
|
||||||
type S3Config struct {
|
type S3Config struct {
|
||||||
Region string `validate:"required" envconfig:"AWS_REGION"`
|
Region string `validate:"required" envconfig:"AWS_REGION"`
|
||||||
Key string `envconfig:"AWS_ACCESS_KEY_ID"`
|
Key string `envconfig:"AWS_ACCESS_KEY_ID"`
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ type Redis struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// RedisSentinel is the configuration for using redis with sentinel
|
// RedisSentinel is the configuration for using redis with sentinel
|
||||||
// for SingleFlight
|
// for SingleFlight.
|
||||||
type RedisSentinel struct {
|
type RedisSentinel struct {
|
||||||
Endpoints []string `envconfig:"ATHENS_REDIS_SENTINEL_ENDPOINTS"`
|
Endpoints []string `envconfig:"ATHENS_REDIS_SENTINEL_ENDPOINTS"`
|
||||||
MasterName string `envconfig:"ATHENS_REDIS_SENTINEL_MASTER_NAME"`
|
MasterName string `envconfig:"ATHENS_REDIS_SENTINEL_MASTER_NAME"`
|
||||||
@@ -33,12 +33,14 @@ type RedisSentinel struct {
|
|||||||
LockConfig *RedisLockConfig
|
LockConfig *RedisLockConfig
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// RedisLockConfig is the configuration for redis locking.
|
||||||
type RedisLockConfig struct {
|
type RedisLockConfig struct {
|
||||||
Timeout int `envconfig:"ATHENS_REDIS_LOCK_TIMEOUT"`
|
Timeout int `envconfig:"ATHENS_REDIS_LOCK_TIMEOUT"`
|
||||||
TTL int `envconfig:"ATHENS_REDIS_LOCK_TTL"`
|
TTL int `envconfig:"ATHENS_REDIS_LOCK_TTL"`
|
||||||
MaxRetries int `envconfig:"ATHENS_REDIS_LOCK_MAX_RETRIES"`
|
MaxRetries int `envconfig:"ATHENS_REDIS_LOCK_MAX_RETRIES"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// DefaultRedisLockConfig returns the default redis locking configuration.
|
||||||
func DefaultRedisLockConfig() *RedisLockConfig {
|
func DefaultRedisLockConfig() *RedisLockConfig {
|
||||||
return &RedisLockConfig{
|
return &RedisLockConfig{
|
||||||
TTL: 900,
|
TTL: 900,
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
package config
|
package config
|
||||||
|
|
||||||
// Storage provides configs for various storage backends
|
// Storage provides configs for various storage backends.
|
||||||
type Storage struct {
|
type Storage struct {
|
||||||
Disk *DiskConfig
|
Disk *DiskConfig
|
||||||
GCP *GCPConfig
|
GCP *GCPConfig
|
||||||
|
|||||||
@@ -2,17 +2,17 @@ package config
|
|||||||
|
|
||||||
import "time"
|
import "time"
|
||||||
|
|
||||||
// TimeoutConf is a common struct for anything with a timeout
|
// TimeoutConf is a common struct for anything with a timeout.
|
||||||
type TimeoutConf struct {
|
type TimeoutConf struct {
|
||||||
Timeout int `validate:"required"`
|
Timeout int `validate:"required"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// TimeoutDuration returns the timeout as time.duration
|
// TimeoutDuration returns the timeout as time.duration.
|
||||||
func (t *TimeoutConf) TimeoutDuration() time.Duration {
|
func (t *TimeoutConf) TimeoutDuration() time.Duration {
|
||||||
return GetTimeoutDuration(t.Timeout)
|
return GetTimeoutDuration(t.Timeout)
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetTimeoutDuration returns the timeout as time.duration
|
// GetTimeoutDuration returns the timeout as time.duration.
|
||||||
func GetTimeoutDuration(timeout int) time.Duration {
|
func GetTimeoutDuration(timeout int) time.Duration {
|
||||||
return time.Second * time.Duration(timeout)
|
return time.Second * time.Duration(timeout)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ import (
|
|||||||
"github.com/gomods/athens/pkg/paths"
|
"github.com/gomods/athens/pkg/paths"
|
||||||
)
|
)
|
||||||
|
|
||||||
func getModuleParams(r *http.Request, op errors.Op) (mod string, ver string, err error) {
|
func getModuleParams(r *http.Request, op errors.Op) (mod, ver string, err error) {
|
||||||
params, err := paths.GetAllParams(r)
|
params, err := paths.GetAllParams(r)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", "", errors.E(op, err, errors.KindBadRequest)
|
return "", "", errors.E(op, err, errors.KindBadRequest)
|
||||||
|
|||||||
@@ -15,8 +15,7 @@ import (
|
|||||||
// a ready-to-go http handler that serves up cmd/go's download protocol.
|
// a ready-to-go http handler that serves up cmd/go's download protocol.
|
||||||
type ProtocolHandler func(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler
|
type ProtocolHandler func(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler
|
||||||
|
|
||||||
// HandlerOpts are the generic options
|
// HandlerOpts are the generic options for a ProtocolHandler.
|
||||||
// for a ProtocolHandler
|
|
||||||
type HandlerOpts struct {
|
type HandlerOpts struct {
|
||||||
Protocol Protocol
|
Protocol Protocol
|
||||||
Logger *log.Logger
|
Logger *log.Logger
|
||||||
@@ -26,7 +25,7 @@ type HandlerOpts struct {
|
|||||||
// LogEntryHandler pulls a log entry from the request context. Thanks to the
|
// LogEntryHandler pulls a log entry from the request context. Thanks to the
|
||||||
// LogEntryMiddleware, we should have a log entry stored in the context for each
|
// LogEntryMiddleware, we should have a log entry stored in the context for each
|
||||||
// request with request-specific fields. This will grab the entry and pass it to
|
// request with request-specific fields. This will grab the entry and pass it to
|
||||||
// the protocol handlers
|
// the protocol handlers.
|
||||||
func LogEntryHandler(ph ProtocolHandler, opts *HandlerOpts) http.Handler {
|
func LogEntryHandler(ph ProtocolHandler, opts *HandlerOpts) http.Handler {
|
||||||
f := func(w http.ResponseWriter, r *http.Request) {
|
f := func(w http.ResponseWriter, r *http.Request) {
|
||||||
ent := log.EntryFromContext(r.Context())
|
ent := log.EntryFromContext(r.Context())
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ import (
|
|||||||
// PathLatest URL.
|
// PathLatest URL.
|
||||||
const PathLatest = "/{module:.+}/@latest"
|
const PathLatest = "/{module:.+}/@latest"
|
||||||
|
|
||||||
// LatestHandler implements GET baseURL/module/@latest
|
// LatestHandler implements GET baseURL/module/@latest.
|
||||||
func LatestHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler {
|
func LatestHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler {
|
||||||
const op errors.Op = "download.LatestHandler"
|
const op errors.Op = "download.LatestHandler"
|
||||||
f := func(w http.ResponseWriter, r *http.Request) {
|
f := func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ import (
|
|||||||
// PathList URL.
|
// PathList URL.
|
||||||
const PathList = "/{module:.+}/@v/list"
|
const PathList = "/{module:.+}/@v/list"
|
||||||
|
|
||||||
// ListHandler implements GET baseURL/module/@v/list
|
// ListHandler implements GET baseURL/module/@v/list.
|
||||||
func ListHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler {
|
func ListHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler {
|
||||||
const op errors.Op = "download.ListHandler"
|
const op errors.Op = "download.ListHandler"
|
||||||
f := func(w http.ResponseWriter, r *http.Request) {
|
f := func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"encoding/base64"
|
"encoding/base64"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/gomods/athens/pkg/errors"
|
"github.com/gomods/athens/pkg/errors"
|
||||||
@@ -16,7 +17,7 @@ import (
|
|||||||
// when a module is not found in storage.
|
// when a module is not found in storage.
|
||||||
type Mode string
|
type Mode string
|
||||||
|
|
||||||
// DownloadMode constants. For more information see config.dev.toml
|
// DownloadMode constants. For more information see config.dev.toml.
|
||||||
const (
|
const (
|
||||||
Sync Mode = "sync"
|
Sync Mode = "sync"
|
||||||
Async Mode = "async"
|
Async Mode = "async"
|
||||||
@@ -59,7 +60,7 @@ func NewFile(m Mode, downloadURL string) (*DownloadFile, error) {
|
|||||||
|
|
||||||
if strings.HasPrefix(string(m), "file:") {
|
if strings.HasPrefix(string(m), "file:") {
|
||||||
filePath := string(m[5:])
|
filePath := string(m[5:])
|
||||||
bts, err := os.ReadFile(filePath)
|
bts, err := os.ReadFile(filepath.Clean(filePath))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -48,7 +48,7 @@ type Opts struct {
|
|||||||
NetworkMode string
|
NetworkMode string
|
||||||
}
|
}
|
||||||
|
|
||||||
// NetworkMode constants
|
// NetworkMode constants.
|
||||||
const (
|
const (
|
||||||
Strict = "strict"
|
Strict = "strict"
|
||||||
Offline = "offline"
|
Offline = "offline"
|
||||||
@@ -263,12 +263,12 @@ func (p *protocol) processDownload(ctx context.Context, mod, ver string, f func(
|
|||||||
}
|
}
|
||||||
return f(newVer)
|
return f(newVer)
|
||||||
case mode.Async:
|
case mode.Async:
|
||||||
go p.stasher.Stash(ctx, mod, ver)
|
go func() { _, _ = p.stasher.Stash(ctx, mod, ver) }()
|
||||||
return errors.E(op, "async: module not found", errors.KindNotFound)
|
return errors.E(op, "async: module not found", errors.KindNotFound)
|
||||||
case mode.Redirect:
|
case mode.Redirect:
|
||||||
return errors.E(op, "redirect", errors.KindRedirect)
|
return errors.E(op, "redirect", errors.KindRedirect)
|
||||||
case mode.AsyncRedirect:
|
case mode.AsyncRedirect:
|
||||||
go p.stasher.Stash(ctx, mod, ver)
|
go func() { _, _ = p.stasher.Stash(ctx, mod, ver) }()
|
||||||
return errors.E(op, "async_redirect: module not found", errors.KindRedirect)
|
return errors.E(op, "async_redirect: module not found", errors.KindRedirect)
|
||||||
case mode.None:
|
case mode.None:
|
||||||
return errors.E(op, "none", errors.KindNotFound)
|
return errors.E(op, "none", errors.KindNotFound)
|
||||||
@@ -276,7 +276,7 @@ func (p *protocol) processDownload(ctx context.Context, mod, ver string, f func(
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// union concatenates two version lists and removes duplicates
|
// union concatenates two version lists and removes duplicates.
|
||||||
func union(list1, list2 []string) []string {
|
func union(list1, list2 []string) []string {
|
||||||
if list1 == nil {
|
if list1 == nil {
|
||||||
list1 = []string{}
|
list1 = []string{}
|
||||||
@@ -284,10 +284,10 @@ func union(list1, list2 []string) []string {
|
|||||||
if list2 == nil {
|
if list2 == nil {
|
||||||
list2 = []string{}
|
list2 = []string{}
|
||||||
}
|
}
|
||||||
list := append(list1, list2...)
|
list1 = append(list1, list2...)
|
||||||
unique := []string{}
|
unique := []string{}
|
||||||
m := make(map[string]struct{})
|
m := make(map[string]struct{})
|
||||||
for _, v := range list {
|
for _, v := range list1 {
|
||||||
if _, ok := m[v]; !ok {
|
if _, ok := m[v]; !ok {
|
||||||
unique = append(unique, v)
|
unique = append(unique, v)
|
||||||
m[v] = struct{}{}
|
m[v] = struct{}{}
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ import (
|
|||||||
// PathVersionInfo URL.
|
// PathVersionInfo URL.
|
||||||
const PathVersionInfo = "/{module:.+}/@v/{version}.info"
|
const PathVersionInfo = "/{module:.+}/@v/{version}.info"
|
||||||
|
|
||||||
// InfoHandler implements GET baseURL/module/@v/version.info
|
// InfoHandler implements GET baseURL/module/@v/version.info.
|
||||||
func InfoHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler {
|
func InfoHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler {
|
||||||
const op errors.Op = "download.InfoHandler"
|
const op errors.Op = "download.InfoHandler"
|
||||||
f := func(w http.ResponseWriter, r *http.Request) {
|
f := func(w http.ResponseWriter, r *http.Request) {
|
||||||
@@ -38,7 +38,7 @@ func InfoHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handle
|
|||||||
w.WriteHeader(errors.Kind(err))
|
w.WriteHeader(errors.Kind(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
w.Write(info)
|
_, _ = w.Write(info)
|
||||||
}
|
}
|
||||||
return http.HandlerFunc(f)
|
return http.HandlerFunc(f)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ import (
|
|||||||
// PathVersionModule URL.
|
// PathVersionModule URL.
|
||||||
const PathVersionModule = "/{module:.+}/@v/{version}.mod"
|
const PathVersionModule = "/{module:.+}/@v/{version}.mod"
|
||||||
|
|
||||||
// ModuleHandler implements GET baseURL/module/@v/version.mod
|
// ModuleHandler implements GET baseURL/module/@v/version.mod.
|
||||||
func ModuleHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler {
|
func ModuleHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler {
|
||||||
const op errors.Op = "download.VersionModuleHandler"
|
const op errors.Op = "download.VersionModuleHandler"
|
||||||
f := func(w http.ResponseWriter, r *http.Request) {
|
f := func(w http.ResponseWriter, r *http.Request) {
|
||||||
@@ -42,7 +42,7 @@ func ModuleHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Hand
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
w.Write(modBts)
|
_, _ = w.Write(modBts)
|
||||||
}
|
}
|
||||||
return http.HandlerFunc(f)
|
return http.HandlerFunc(f)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ import (
|
|||||||
// PathVersionZip URL.
|
// PathVersionZip URL.
|
||||||
const PathVersionZip = "/{module:.+}/@v/{version}.zip"
|
const PathVersionZip = "/{module:.+}/@v/{version}.zip"
|
||||||
|
|
||||||
// ZipHandler implements GET baseURL/module/@v/version.zip
|
// ZipHandler implements GET baseURL/module/@v/version.zip.
|
||||||
func ZipHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler {
|
func ZipHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler {
|
||||||
const op errors.Op = "download.ZipHandler"
|
const op errors.Op = "download.ZipHandler"
|
||||||
f := func(w http.ResponseWriter, r *http.Request) {
|
f := func(w http.ResponseWriter, r *http.Request) {
|
||||||
@@ -41,7 +41,7 @@ func ZipHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler
|
|||||||
w.WriteHeader(errors.Kind(err))
|
w.WriteHeader(errors.Kind(err))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
defer zip.Close()
|
defer func() { _ = zip.Close() }()
|
||||||
|
|
||||||
w.Header().Set("Content-Type", "application/zip")
|
w.Header().Set("Content-Type", "application/zip")
|
||||||
size := zip.Size()
|
size := zip.Size()
|
||||||
|
|||||||
+17
-6
@@ -9,7 +9,7 @@ import (
|
|||||||
"github.com/sirupsen/logrus"
|
"github.com/sirupsen/logrus"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Kind enums
|
// Kind enums.
|
||||||
const (
|
const (
|
||||||
KindNotFound = http.StatusNotFound
|
KindNotFound = http.StatusNotFound
|
||||||
KindBadRequest = http.StatusBadRequest
|
KindBadRequest = http.StatusBadRequest
|
||||||
@@ -55,6 +55,16 @@ func Is(err error, kind int) bool {
|
|||||||
return Kind(err) == kind
|
return Kind(err) == kind
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// IsErr is a convenience wrapper around the std library errors.Is.
|
||||||
|
func IsErr(err, target error) bool {
|
||||||
|
return errors.Is(err, target)
|
||||||
|
}
|
||||||
|
|
||||||
|
// AsErr is a convenience wrapper around the std library errors.As.
|
||||||
|
func AsErr(err error, target any) bool {
|
||||||
|
return errors.As(err, target)
|
||||||
|
}
|
||||||
|
|
||||||
// Op describes any independent function or
|
// Op describes any independent function or
|
||||||
// method in Athens. A series of operations
|
// method in Athens. A series of operations
|
||||||
// forms a more readable stack trace.
|
// forms a more readable stack trace.
|
||||||
@@ -69,7 +79,7 @@ func (o Op) String() string {
|
|||||||
// a module from a regular error string or version.
|
// a module from a regular error string or version.
|
||||||
type M string
|
type M string
|
||||||
|
|
||||||
// V represents a module version in an error
|
// V represents a module version in an error.
|
||||||
type V string
|
type V string
|
||||||
|
|
||||||
// E is a helper function to construct an Error type
|
// E is a helper function to construct an Error type
|
||||||
@@ -114,8 +124,8 @@ func E(op Op, args ...interface{}) error {
|
|||||||
// if none exists, then the level is Error because
|
// if none exists, then the level is Error because
|
||||||
// it is an unexpected.
|
// it is an unexpected.
|
||||||
func Severity(err error) logrus.Level {
|
func Severity(err error) logrus.Level {
|
||||||
e, ok := err.(Error)
|
var e Error
|
||||||
if !ok {
|
if !errors.As(err, &e) {
|
||||||
return logrus.ErrorLevel
|
return logrus.ErrorLevel
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -145,8 +155,8 @@ func Expect(err error, kinds ...int) logrus.Level {
|
|||||||
// Kind recursively searches for the
|
// Kind recursively searches for the
|
||||||
// first error kind it finds.
|
// first error kind it finds.
|
||||||
func Kind(err error) int {
|
func Kind(err error) int {
|
||||||
e, ok := err.(Error)
|
var e Error
|
||||||
if !ok {
|
if !errors.As(err, &e) {
|
||||||
return KindUnexpected
|
return KindUnexpected
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -173,6 +183,7 @@ func KindText(err error) string {
|
|||||||
func Ops(err Error) []Op {
|
func Ops(err Error) []Op {
|
||||||
ops := []Op{err.Op}
|
ops := []Op{err.Op}
|
||||||
for {
|
for {
|
||||||
|
//nolint:errorlint // We iterate the errors anyway.
|
||||||
embeddedErr, ok := err.Err.(Error)
|
embeddedErr, ok := err.Err.(Error)
|
||||||
if !ok {
|
if !ok {
|
||||||
break
|
break
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
package errors
|
package errors
|
||||||
|
|
||||||
// IsNotFoundErr helper function for KindNotFound
|
// IsNotFoundErr helper function for KindNotFound.
|
||||||
func IsNotFoundErr(err error) bool {
|
func IsNotFoundErr(err error) bool {
|
||||||
return Kind(err) == KindNotFound
|
return Kind(err) == KindNotFound
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ func RunTests(t *testing.T, indexer index.Indexer, clearIndex func() error) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
var tests = []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
desc string
|
desc string
|
||||||
limit int
|
limit int
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ type Line struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Indexer is an interface that can process new module@versions
|
// Indexer is an interface that can process new module@versions
|
||||||
// and also retrieve 'limit' module@versions that were indexed after 'since'
|
// and also retrieve 'limit' module@versions that were indexed after 'since'.
|
||||||
type Indexer interface {
|
type Indexer interface {
|
||||||
// Index stores the module@version into the index backend.
|
// Index stores the module@version into the index backend.
|
||||||
// Implementer must create the Timestamp at the time and set it
|
// Implementer must create the Timestamp at the time and set it
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ import (
|
|||||||
"github.com/gomods/athens/pkg/index"
|
"github.com/gomods/athens/pkg/index"
|
||||||
)
|
)
|
||||||
|
|
||||||
// New returns a new in-memory indexer
|
// New returns a new in-memory indexer.
|
||||||
func New() index.Indexer {
|
func New() index.Indexer {
|
||||||
return &indexer{}
|
return &indexer{}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -83,8 +83,8 @@ func (i *indexer) Lines(ctx context.Context, since time.Time, limit int) ([]*ind
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.E(op, err)
|
return nil, errors.E(op, err)
|
||||||
}
|
}
|
||||||
defer rows.Close()
|
defer func() { _ = rows.Close() }()
|
||||||
lines := []*index.Line{}
|
var lines []*index.Line
|
||||||
for rows.Next() {
|
for rows.Next() {
|
||||||
var line index.Line
|
var line index.Line
|
||||||
err = rows.Scan(&line.Path, &line.Version, &line.Timestamp)
|
err = rows.Scan(&line.Path, &line.Version, &line.Timestamp)
|
||||||
@@ -108,13 +108,14 @@ func getMySQLSource(cfg *config.MySQL) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func getKind(err error) int {
|
func getKind(err error) int {
|
||||||
mysqlErr, ok := err.(*mysql.MySQLError)
|
mysqlErr := &mysql.MySQLError{}
|
||||||
if !ok {
|
if !errors.AsErr(err, &mysqlErr) {
|
||||||
return errors.KindUnexpected
|
return errors.KindUnexpected
|
||||||
}
|
}
|
||||||
switch mysqlErr.Number {
|
switch mysqlErr.Number {
|
||||||
case 1062:
|
case 1062:
|
||||||
return errors.KindAlreadyExists
|
return errors.KindAlreadyExists
|
||||||
|
default:
|
||||||
|
return errors.KindUnexpected
|
||||||
}
|
}
|
||||||
return errors.KindUnexpected
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ import (
|
|||||||
"github.com/gomods/athens/pkg/index"
|
"github.com/gomods/athens/pkg/index"
|
||||||
)
|
)
|
||||||
|
|
||||||
// New returns a no-op Indexer
|
// New returns a no-op Indexer.
|
||||||
func New() index.Indexer {
|
func New() index.Indexer {
|
||||||
return indexer{}
|
return indexer{}
|
||||||
}
|
}
|
||||||
@@ -17,6 +17,7 @@ type indexer struct{}
|
|||||||
func (indexer) Index(ctx context.Context, mod, ver string) error {
|
func (indexer) Index(ctx context.Context, mod, ver string) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (indexer) Lines(ctx context.Context, since time.Time, limit int) ([]*index.Line, error) {
|
func (indexer) Lines(ctx context.Context, since time.Time, limit int) ([]*index.Line, error) {
|
||||||
return []*index.Line{}, nil
|
return []*index.Line{}, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,17 +7,15 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
// register the driver with database/sql
|
|
||||||
"github.com/lib/pq"
|
|
||||||
|
|
||||||
"github.com/gomods/athens/pkg/config"
|
"github.com/gomods/athens/pkg/config"
|
||||||
"github.com/gomods/athens/pkg/errors"
|
"github.com/gomods/athens/pkg/errors"
|
||||||
"github.com/gomods/athens/pkg/index"
|
"github.com/gomods/athens/pkg/index"
|
||||||
|
"github.com/lib/pq"
|
||||||
)
|
)
|
||||||
|
|
||||||
// New returns a new Indexer with a PostgreSQL implementation.
|
// New returns a new Indexer with a PostgreSQL implementation.
|
||||||
// It attempts to connect to the DB and create the index table
|
// It attempts to connect to the DB and create the index table
|
||||||
// if it doesn ot already exist.
|
// if it does not already exist.
|
||||||
func New(cfg *config.Postgres) (index.Indexer, error) {
|
func New(cfg *config.Postgres) (index.Indexer, error) {
|
||||||
dataSource := getPostgresSource(cfg)
|
dataSource := getPostgresSource(cfg)
|
||||||
db, err := sql.Open("postgres", dataSource)
|
db, err := sql.Open("postgres", dataSource)
|
||||||
@@ -42,7 +40,7 @@ var schema = [...]string{
|
|||||||
id SERIAL PRIMARY KEY,
|
id SERIAL PRIMARY KEY,
|
||||||
path VARCHAR(255) NOT NULL,
|
path VARCHAR(255) NOT NULL,
|
||||||
version VARCHAR(255) NOT NULL,
|
version VARCHAR(255) NOT NULL,
|
||||||
timestamp timestamp NOT NULL
|
timestamp TIMESTAMP NOT NULL
|
||||||
)
|
)
|
||||||
`,
|
`,
|
||||||
`
|
`
|
||||||
@@ -82,8 +80,8 @@ func (i *indexer) Lines(ctx context.Context, since time.Time, limit int) ([]*ind
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.E(op, err)
|
return nil, errors.E(op, err)
|
||||||
}
|
}
|
||||||
defer rows.Close()
|
defer func() { _ = rows.Close() }()
|
||||||
lines := []*index.Line{}
|
var lines []*index.Line
|
||||||
for rows.Next() {
|
for rows.Next() {
|
||||||
var line index.Line
|
var line index.Line
|
||||||
err = rows.Scan(&line.Path, &line.Version, &line.Timestamp)
|
err = rows.Scan(&line.Path, &line.Version, &line.Timestamp)
|
||||||
@@ -96,7 +94,7 @@ func (i *indexer) Lines(ctx context.Context, since time.Time, limit int) ([]*ind
|
|||||||
}
|
}
|
||||||
|
|
||||||
func getPostgresSource(cfg *config.Postgres) string {
|
func getPostgresSource(cfg *config.Postgres) string {
|
||||||
args := []string{}
|
args := make([]string, 0, 5+len(cfg.Params))
|
||||||
args = append(args, "host="+cfg.Host)
|
args = append(args, "host="+cfg.Host)
|
||||||
args = append(args, "port=", strconv.Itoa(cfg.Port))
|
args = append(args, "port=", strconv.Itoa(cfg.Port))
|
||||||
args = append(args, "user=", cfg.User)
|
args = append(args, "user=", cfg.User)
|
||||||
@@ -109,13 +107,14 @@ func getPostgresSource(cfg *config.Postgres) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func getKind(err error) int {
|
func getKind(err error) int {
|
||||||
pqerr, ok := err.(*pq.Error)
|
pqerr := &pq.Error{}
|
||||||
if !ok {
|
if !errors.AsErr(err, &pqerr) {
|
||||||
return errors.KindUnexpected
|
return errors.KindUnexpected
|
||||||
}
|
}
|
||||||
switch pqerr.Code {
|
switch pqerr.Code {
|
||||||
case "23505":
|
case "23505":
|
||||||
return errors.KindAlreadyExists
|
return errors.KindAlreadyExists
|
||||||
|
default:
|
||||||
|
return errors.KindUnexpected
|
||||||
}
|
}
|
||||||
return errors.KindUnexpected
|
|
||||||
}
|
}
|
||||||
|
|||||||
+2
-2
@@ -35,8 +35,8 @@ func (e *entry) WithFields(fields map[string]interface{}) Entry {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (e *entry) SystemErr(err error) {
|
func (e *entry) SystemErr(err error) {
|
||||||
athensErr, ok := err.(errors.Error)
|
var athensErr errors.Error
|
||||||
if !ok {
|
if !errors.IsErr(err, athensErr) {
|
||||||
e.Error(err)
|
e.Error(err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|||||||
+1
-1
@@ -41,7 +41,7 @@ func (l *Logger) WithFields(fields map[string]interface{}) Entry {
|
|||||||
return &entry{e}
|
return &entry{e}
|
||||||
}
|
}
|
||||||
|
|
||||||
// NoOpLogger provides a Logger that does nothing
|
// NoOpLogger provides a Logger that does nothing.
|
||||||
func NoOpLogger() *Logger {
|
func NoOpLogger() *Logger {
|
||||||
return &Logger{
|
return &Logger{
|
||||||
Logger: &logrus.Logger{},
|
Logger: &logrus.Logger{},
|
||||||
|
|||||||
@@ -8,14 +8,14 @@ type ctxKey string
|
|||||||
|
|
||||||
const logEntryKey ctxKey = "log-entry-context-key"
|
const logEntryKey ctxKey = "log-entry-context-key"
|
||||||
|
|
||||||
// SetEntryInContext stores an Entry in the request context
|
// SetEntryInContext stores an Entry in the request context.
|
||||||
func SetEntryInContext(ctx context.Context, e Entry) context.Context {
|
func SetEntryInContext(ctx context.Context, e Entry) context.Context {
|
||||||
return context.WithValue(ctx, logEntryKey, e)
|
return context.WithValue(ctx, logEntryKey, e)
|
||||||
}
|
}
|
||||||
|
|
||||||
// EntryFromContext returns an Entry that has been stored in the request context.
|
// EntryFromContext returns an Entry that has been stored in the request context.
|
||||||
// If there is no value for the key or the type assertion fails, it returns a new
|
// If there is no value for the key or the type assertion fails, it returns a new
|
||||||
// entry from the provided logger
|
// entry from the provided logger.
|
||||||
func EntryFromContext(ctx context.Context) Entry {
|
func EntryFromContext(ctx context.Context) Entry {
|
||||||
e, ok := ctx.Value(logEntryKey).(Entry)
|
e, ok := ctx.Value(logEntryKey).(Entry)
|
||||||
if !ok || e == nil {
|
if !ok || e == nil {
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ContentType writes writes an application/json
|
// ContentType writes an application/json
|
||||||
// Content-Type header.
|
// Content-Type header.
|
||||||
func ContentType(h http.Handler) http.Handler {
|
func ContentType(h http.Handler) http.Handler {
|
||||||
f := func(w http.ResponseWriter, r *http.Request) {
|
f := func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// LogEntryMiddleware builds a log.Entry, setting the request fields
|
// LogEntryMiddleware builds a log.Entry, setting the request fields
|
||||||
// and storing it in the context to be used throughout the stack
|
// and storing it in the context to be used throughout the stack.
|
||||||
func LogEntryMiddleware(lggr *log.Logger) mux.MiddlewareFunc {
|
func LogEntryMiddleware(lggr *log.Logger) mux.MiddlewareFunc {
|
||||||
return func(h http.Handler) http.Handler {
|
return func(h http.Handler) http.Handler {
|
||||||
f := func(w http.ResponseWriter, r *http.Request) {
|
f := func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ import (
|
|||||||
|
|
||||||
// WithRequestID ensures a request id is in the
|
// WithRequestID ensures a request id is in the
|
||||||
// request context by either the incoming header
|
// request context by either the incoming header
|
||||||
// or creating a new one
|
// or creating a new one.
|
||||||
func WithRequestID(h http.Handler) http.Handler {
|
func WithRequestID(h http.Handler) http.Handler {
|
||||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
requestID := r.Header.Get(requestid.HeaderKey)
|
requestID := r.Header.Get(requestid.HeaderKey)
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// NewValidationMiddleware builds a middleware function that performs validation checks by calling
|
// NewValidationMiddleware builds a middleware function that performs validation checks by calling
|
||||||
// an external webhook
|
// an external webhook.
|
||||||
func NewValidationMiddleware(client *http.Client, validatorHook string) mux.MiddlewareFunc {
|
func NewValidationMiddleware(client *http.Client, validatorHook string) mux.MiddlewareFunc {
|
||||||
const op errors.Op = "actions.NewValidationMiddleware"
|
const op errors.Op = "actions.NewValidationMiddleware"
|
||||||
return func(h http.Handler) http.Handler {
|
return func(h http.Handler) http.Handler {
|
||||||
@@ -50,7 +50,7 @@ func NewValidationMiddleware(client *http.Client, validatorHook string) mux.Midd
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func maybeLogValidationReason(context context.Context, message string, mod string, version string) {
|
func maybeLogValidationReason(context context.Context, message, mod, version string) {
|
||||||
if len(message) > 0 {
|
if len(message) > 0 {
|
||||||
entry := log.EntryFromContext(context)
|
entry := log.EntryFromContext(context)
|
||||||
entry.Warnf("error validating %s@%s %s", mod, version, message)
|
entry.Warnf("error validating %s@%s %s", mod, version, message)
|
||||||
@@ -85,6 +85,10 @@ func validate(ctx context.Context, client *http.Client, hook, mod, ver string) (
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return validationResponse{Valid: false}, errors.E(op, err)
|
return validationResponse{Valid: false}, errors.E(op, err)
|
||||||
}
|
}
|
||||||
|
defer func() {
|
||||||
|
_, _ = io.Copy(io.Discard, resp.Body)
|
||||||
|
_ = resp.Body.Close()
|
||||||
|
}()
|
||||||
|
|
||||||
switch resp.StatusCode {
|
switch resp.StatusCode {
|
||||||
case http.StatusOK:
|
case http.StatusOK:
|
||||||
@@ -97,8 +101,6 @@ func validate(ctx context.Context, client *http.Client, hook, mod, ver string) (
|
|||||||
}
|
}
|
||||||
|
|
||||||
func validationResponseFromRequest(resp *http.Response) validationResponse {
|
func validationResponseFromRequest(resp *http.Response) validationResponse {
|
||||||
defer resp.Body.Close()
|
|
||||||
|
|
||||||
body, _ := io.ReadAll(resp.Body)
|
body, _ := io.ReadAll(resp.Body)
|
||||||
return validationResponse{Valid: resp.StatusCode == http.StatusOK, Message: body}
|
return validationResponse{Valid: resp.StatusCode == http.StatusOK, Message: body}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ import (
|
|||||||
"github.com/gomods/athens/pkg/storage"
|
"github.com/gomods/athens/pkg/storage"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Fetcher fetches module from an upstream source
|
// Fetcher fetches module from an upstream source.
|
||||||
type Fetcher interface {
|
type Fetcher interface {
|
||||||
// Fetch downloads the sources from an upstream and returns the corresponding
|
// Fetch downloads the sources from an upstream and returns the corresponding
|
||||||
// .info, .mod, and .zip files.
|
// .info, .mod, and .zip files.
|
||||||
|
|||||||
+15
-16
@@ -3,6 +3,7 @@ package module
|
|||||||
import (
|
import (
|
||||||
"bufio"
|
"bufio"
|
||||||
"os"
|
"os"
|
||||||
|
"path/filepath"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
@@ -14,15 +15,15 @@ var (
|
|||||||
versionSeparator = "."
|
versionSeparator = "."
|
||||||
)
|
)
|
||||||
|
|
||||||
// Filter is a filter of modules
|
// Filter is a filter of modules.
|
||||||
type Filter struct {
|
type Filter struct {
|
||||||
root ruleNode
|
root ruleNode
|
||||||
filePath string
|
filePath string
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewFilter creates new filter based on rules defined in a configuration file
|
// NewFilter creates new filter based on rules defined in a configuration file.
|
||||||
// WARNING: this is not concurrently safe
|
// WARNING: this is not concurrently safe.
|
||||||
// Configuration consists of two operations: + for include and - for exclude
|
// Configuration consists of two operations: + for include and - for exclude:
|
||||||
// e.g.
|
// e.g.
|
||||||
// - github.com/a
|
// - github.com/a
|
||||||
// - github.com/a/b
|
// - github.com/a/b
|
||||||
@@ -33,7 +34,7 @@ type Filter struct {
|
|||||||
// -
|
// -
|
||||||
// + github.com/a
|
// + github.com/a
|
||||||
//
|
//
|
||||||
// will exclude all items from communication except github.com/a
|
// will exclude all items from communication except github.com/a.
|
||||||
func NewFilter(filterFilePath string) (*Filter, error) {
|
func NewFilter(filterFilePath string) (*Filter, error) {
|
||||||
// Do not return an error if the file path is empty
|
// Do not return an error if the file path is empty
|
||||||
// Do not attempt to parse it as well.
|
// Do not attempt to parse it as well.
|
||||||
@@ -42,10 +43,9 @@ func NewFilter(filterFilePath string) (*Filter, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
return initFromConfig(filterFilePath)
|
return initFromConfig(filterFilePath)
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// AddRule adds rule for specified path
|
// AddRule adds rule for specified path.
|
||||||
func (f *Filter) AddRule(path string, qualifiers []string, rule FilterRule) {
|
func (f *Filter) AddRule(path string, qualifiers []string, rule FilterRule) {
|
||||||
f.ensurePath(path)
|
f.ensurePath(path)
|
||||||
|
|
||||||
@@ -70,7 +70,7 @@ func (f *Filter) AddRule(path string, qualifiers []string, rule FilterRule) {
|
|||||||
latest.next[last] = rn
|
latest.next[last] = rn
|
||||||
}
|
}
|
||||||
|
|
||||||
// Rule returns the filter rule to be applied to the given path
|
// Rule returns the filter rule to be applied to the given path.
|
||||||
func (f *Filter) Rule(path, version string) FilterRule {
|
func (f *Filter) Rule(path, version string) FilterRule {
|
||||||
segs := getPathSegments(path)
|
segs := getPathSegments(path)
|
||||||
rule := f.getAssociatedRule(version, segs...)
|
rule := f.getAssociatedRule(version, segs...)
|
||||||
@@ -145,7 +145,6 @@ func initFromConfig(filePath string) (*Filter, error) {
|
|||||||
f.root = rn
|
f.root = rn
|
||||||
|
|
||||||
for idx, line := range lines {
|
for idx, line := range lines {
|
||||||
|
|
||||||
// Ignore newline
|
// Ignore newline
|
||||||
if len(line) == 0 {
|
if len(line) == 0 {
|
||||||
continue
|
continue
|
||||||
@@ -160,7 +159,7 @@ func initFromConfig(filePath string) (*Filter, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
ruleSign := strings.TrimSpace(split[0])
|
ruleSign := strings.TrimSpace(split[0])
|
||||||
rule := Default
|
var rule FilterRule
|
||||||
switch ruleSign {
|
switch ruleSign {
|
||||||
case "+":
|
case "+":
|
||||||
rule = Include
|
rule = Include
|
||||||
@@ -195,11 +194,11 @@ func initFromConfig(filePath string) (*Filter, error) {
|
|||||||
|
|
||||||
// matches checks if the given version matches the given qualifier.
|
// matches checks if the given version matches the given qualifier.
|
||||||
// Qualifiers can be:
|
// Qualifiers can be:
|
||||||
// - plain versions
|
// - plain versions.
|
||||||
// - v1.2.3 enables v1.2.3
|
// - v1.2.3 enables v1.2.3.
|
||||||
// - ~1.2.3: enables 1.2.x which are at least 1.2.3
|
// - ~1.2.3: enables 1.2.x which are at least 1.2.3.
|
||||||
// - ^1.2.3: enables 1.x.x which are at least 1.2.3
|
// - ^1.2.3: enables 1.x.x which are at least 1.2.3.
|
||||||
// - <1.2.3: enables everything lower than 1.2.3 includes 1.2.2 and 0.58.9 as well
|
// - <1.2.3: enables everything lower than 1.2.3 includes 1.2.2 and 0.58.9 as well.
|
||||||
func matches(version, qualifier string) bool {
|
func matches(version, qualifier string) bool {
|
||||||
if len(qualifier) < 2 || len(version) < 1 {
|
if len(qualifier) < 2 || len(version) < 1 {
|
||||||
return false
|
return false
|
||||||
@@ -297,7 +296,7 @@ func newRule(r FilterRule) ruleNode {
|
|||||||
func getConfigLines(filterFile string) ([]string, error) {
|
func getConfigLines(filterFile string) ([]string, error) {
|
||||||
const op errors.Op = "module.getConfigLines"
|
const op errors.Op = "module.getConfigLines"
|
||||||
|
|
||||||
f, err := os.Open(filterFile)
|
f, err := os.Open(filepath.Clean(filterFile))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.E(op, err)
|
return nil, errors.E(op, err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,16 +1,16 @@
|
|||||||
package module
|
package module
|
||||||
|
|
||||||
// FilterRule defines behavior of module communication
|
// FilterRule defines behavior of module communication.
|
||||||
type FilterRule int
|
type FilterRule int
|
||||||
|
|
||||||
const (
|
const (
|
||||||
// Default filter rule does not alter default/parent behavior
|
// Default filter rule does not alter default/parent behavior.
|
||||||
Default FilterRule = iota
|
Default FilterRule = iota
|
||||||
// Include treats modules the usual way
|
// Include treats modules the usual way.
|
||||||
// Used for reverting Exclude of parent path
|
// Used for reverting Exclude of parent path.
|
||||||
Include
|
Include
|
||||||
// Exclude filter rule excludes package and its children from communication
|
// Exclude filter rule excludes package and its children from communication.
|
||||||
Exclude
|
Exclude
|
||||||
// Direct filter rule forces the package to be fetched directly from upstream proxy
|
// Direct filter rule forces the package to be fetched directly from upstream proxy.
|
||||||
Direct
|
Direct
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ type goModule struct {
|
|||||||
GoModSum string `json:"goModSum"` // checksum for go.mod (as in go.sum)
|
GoModSum string `json:"goModSum"` // checksum for go.mod (as in go.sum)
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewGoGetFetcher creates fetcher which uses go get tool to fetch modules
|
// NewGoGetFetcher creates fetcher which uses go get tool to fetch modules.
|
||||||
func NewGoGetFetcher(goBinaryName, gogetDir string, envVars []string, fs afero.Fs) (Fetcher, error) {
|
func NewGoGetFetcher(goBinaryName, gogetDir string, envVars []string, fs afero.Fs) (Fetcher, error) {
|
||||||
const op errors.Op = "module.NewGoGetFetcher"
|
const op errors.Op = "module.NewGoGetFetcher"
|
||||||
if err := validGoBinary(goBinaryName); err != nil {
|
if err := validGoBinary(goBinaryName); err != nil {
|
||||||
@@ -64,7 +64,7 @@ func (g *goGetFetcher) Fetch(ctx context.Context, mod, ver string) (*storage.Ver
|
|||||||
sourcePath := filepath.Join(goPathRoot, "src")
|
sourcePath := filepath.Join(goPathRoot, "src")
|
||||||
modPath := filepath.Join(sourcePath, getRepoDirName(mod, ver))
|
modPath := filepath.Join(sourcePath, getRepoDirName(mod, ver))
|
||||||
if err := g.fs.MkdirAll(modPath, os.ModeDir|os.ModePerm); err != nil {
|
if err := g.fs.MkdirAll(modPath, os.ModeDir|os.ModePerm); err != nil {
|
||||||
clearFiles(g.fs, goPathRoot)
|
_ = clearFiles(g.fs, goPathRoot)
|
||||||
return nil, errors.E(op, err)
|
return nil, errors.E(op, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -72,14 +72,13 @@ func (g *goGetFetcher) Fetch(ctx context.Context, mod, ver string) (*storage.Ver
|
|||||||
ctx,
|
ctx,
|
||||||
g.goBinaryName,
|
g.goBinaryName,
|
||||||
g.envVars,
|
g.envVars,
|
||||||
g.fs,
|
|
||||||
goPathRoot,
|
goPathRoot,
|
||||||
modPath,
|
modPath,
|
||||||
mod,
|
mod,
|
||||||
ver,
|
ver,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
clearFiles(g.fs, goPathRoot)
|
_ = clearFiles(g.fs, goPathRoot)
|
||||||
return nil, errors.E(op, err)
|
return nil, errors.E(op, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -116,7 +115,6 @@ func downloadModule(
|
|||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
goBinaryName string,
|
goBinaryName string,
|
||||||
envVars []string,
|
envVars []string,
|
||||||
fs afero.Fs,
|
|
||||||
gopath,
|
gopath,
|
||||||
repoRoot,
|
repoRoot,
|
||||||
module,
|
module,
|
||||||
@@ -137,7 +135,7 @@ func downloadModule(
|
|||||||
|
|
||||||
err := cmd.Run()
|
err := cmd.Run()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
err = fmt.Errorf("%v: %s", err, stderr)
|
err = fmt.Errorf("%w: %s", err, stderr)
|
||||||
var m goModule
|
var m goModule
|
||||||
if jsonErr := json.NewDecoder(stdout).Decode(&m); jsonErr != nil {
|
if jsonErr := json.NewDecoder(stdout).Decode(&m); jsonErr != nil {
|
||||||
return goModule{}, errors.E(op, err)
|
return goModule{}, errors.E(op, err)
|
||||||
@@ -165,17 +163,17 @@ func isLimitHit(o string) bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// getRepoDirName takes a raw repository URI and a version and creates a directory name that the
|
// getRepoDirName takes a raw repository URI and a version and creates a directory name that the
|
||||||
// repository contents can be put into
|
// repository contents can be put into.
|
||||||
func getRepoDirName(repoURI, version string) string {
|
func getRepoDirName(repoURI, version string) string {
|
||||||
escapedURI := strings.Replace(repoURI, "/", "-", -1)
|
escapedURI := strings.ReplaceAll(repoURI, "/", "-")
|
||||||
return fmt.Sprintf("%s-%s", escapedURI, version)
|
return fmt.Sprintf("%s-%s", escapedURI, version)
|
||||||
}
|
}
|
||||||
|
|
||||||
func validGoBinary(name string) error {
|
func validGoBinary(name string) error {
|
||||||
const op errors.Op = "module.validGoBinary"
|
const op errors.Op = "module.validGoBinary"
|
||||||
err := exec.Command(name).Run()
|
err := exec.Command(name).Run()
|
||||||
_, ok := err.(*exec.ExitError)
|
eErr := &exec.ExitError{}
|
||||||
if err != nil && !ok {
|
if err != nil && !errors.AsErr(err, &eErr) {
|
||||||
return errors.E(op, err)
|
return errors.E(op, err)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ type vcsLister struct {
|
|||||||
fs afero.Fs
|
fs afero.Fs
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewVCSLister creates an UpstreamLister which uses VCS to fetch a list of available versions
|
// NewVCSLister creates an UpstreamLister which uses VCS to fetch a list of available versions.
|
||||||
func NewVCSLister(goBinPath string, env []string, fs afero.Fs) UpstreamLister {
|
func NewVCSLister(goBinPath string, env []string, fs afero.Fs) UpstreamLister {
|
||||||
return &vcsLister{
|
return &vcsLister{
|
||||||
goBinPath: goBinPath,
|
goBinPath: goBinPath,
|
||||||
@@ -39,13 +39,13 @@ func NewVCSLister(goBinPath string, env []string, fs afero.Fs) UpstreamLister {
|
|||||||
|
|
||||||
func (l *vcsLister) List(ctx context.Context, module string) (*storage.RevInfo, []string, error) {
|
func (l *vcsLister) List(ctx context.Context, module string) (*storage.RevInfo, []string, error) {
|
||||||
const op errors.Op = "vcsLister.List"
|
const op errors.Op = "vcsLister.List"
|
||||||
ctx, span := observ.StartSpan(ctx, op.String())
|
_, span := observ.StartSpan(ctx, op.String())
|
||||||
defer span.End()
|
defer span.End()
|
||||||
tmpDir, err := afero.TempDir(l.fs, "", "go-list")
|
tmpDir, err := afero.TempDir(l.fs, "", "go-list")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, errors.E(op, err)
|
return nil, nil, errors.E(op, err)
|
||||||
}
|
}
|
||||||
defer l.fs.RemoveAll(tmpDir)
|
defer func() { _ = l.fs.RemoveAll(tmpDir) }()
|
||||||
|
|
||||||
cmd := exec.Command(
|
cmd := exec.Command(
|
||||||
l.goBinPath,
|
l.goBinPath,
|
||||||
@@ -62,12 +62,12 @@ func (l *vcsLister) List(ctx context.Context, module string) (*storage.RevInfo,
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, errors.E(op, err)
|
return nil, nil, errors.E(op, err)
|
||||||
}
|
}
|
||||||
defer clearFiles(l.fs, gopath)
|
defer func() { _ = clearFiles(l.fs, gopath) }()
|
||||||
cmd.Env = prepareEnv(gopath, l.env)
|
cmd.Env = prepareEnv(gopath, l.env)
|
||||||
|
|
||||||
err = cmd.Run()
|
err = cmd.Run()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
err = fmt.Errorf("%v: %s", err, stderr)
|
err = fmt.Errorf("%w: %s", err, stderr)
|
||||||
// as of now, we can't recognize between a true NotFound
|
// as of now, we can't recognize between a true NotFound
|
||||||
// and an unexpected error, so we choose the more
|
// and an unexpected error, so we choose the more
|
||||||
// hopeful path of NotFound. This way the Go command
|
// hopeful path of NotFound. This way the Go command
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ import (
|
|||||||
|
|
||||||
// prepareEnv will return all the appropriate
|
// prepareEnv will return all the appropriate
|
||||||
// environment variables for a Go Command to run
|
// environment variables for a Go Command to run
|
||||||
// successfully (such as GOPATH, GOCACHE, PATH etc)
|
// successfully (such as GOPATH, GOCACHE, PATH etc).
|
||||||
func prepareEnv(gopath string, envVars []string) []string {
|
func prepareEnv(gopath string, envVars []string) []string {
|
||||||
gopathEnv := fmt.Sprintf("GOPATH=%s", gopath)
|
gopathEnv := fmt.Sprintf("GOPATH=%s", gopath)
|
||||||
cacheEnv := fmt.Sprintf("GOCACHE=%s", filepath.Join(gopath, "cache"))
|
cacheEnv := fmt.Sprintf("GOCACHE=%s", filepath.Join(gopath, "cache"))
|
||||||
|
|||||||
@@ -14,10 +14,10 @@ type zipReadCloser struct {
|
|||||||
goPath string
|
goPath string
|
||||||
}
|
}
|
||||||
|
|
||||||
// Close closes the zip file handle and clears up disk space used by the underlying disk ref
|
// Close closes the zip file handle and clears up disk space used by the underlying disk ref.
|
||||||
// It is the caller's responsibility to call this method to free up utilized disk space
|
// It is the caller's responsibility to call this method to free up utilized disk space.
|
||||||
func (rc *zipReadCloser) Close() error {
|
func (rc *zipReadCloser) Close() error {
|
||||||
rc.zip.Close()
|
_ = rc.zip.Close()
|
||||||
return clearFiles(rc.fs, rc.goPath)
|
return clearFiles(rc.fs, rc.goPath)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -25,8 +25,8 @@ func (rc *zipReadCloser) Read(p []byte) (n int, err error) {
|
|||||||
return rc.zip.Read(p)
|
return rc.zip.Read(p)
|
||||||
}
|
}
|
||||||
|
|
||||||
// clearFiles deletes all data from the given fs at path root
|
// clearFiles deletes all data from the given fs at path root.
|
||||||
// This function must be called when zip is closed to cleanup the entire GOPATH created by the diskref
|
// This function must be called when zip is closed to cleanup the entire GOPATH created by the diskref.
|
||||||
func clearFiles(fs afero.Fs, root string) error {
|
func clearFiles(fs afero.Fs, root string) error {
|
||||||
const op errors.Op = "module.ClearFiles"
|
const op errors.Op = "module.ClearFiles"
|
||||||
// This is required because vgo ensures dependencies are read-only
|
// This is required because vgo ensures dependencies are read-only
|
||||||
@@ -36,7 +36,7 @@ func clearFiles(fs afero.Fs, root string) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
return fs.Chmod(path, 0770)
|
return fs.Chmod(path, 0o770)
|
||||||
}
|
}
|
||||||
err := afero.Walk(fs, root, walkFn)
|
err := afero.Walk(fs, root, walkFn)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
+18
-18
@@ -13,16 +13,16 @@ import (
|
|||||||
|
|
||||||
// RegisterExporter determines the type of TraceExporter service for exporting traces from opencensus
|
// RegisterExporter determines the type of TraceExporter service for exporting traces from opencensus
|
||||||
// User can choose from multiple tracing services (datadog, jaegar)
|
// User can choose from multiple tracing services (datadog, jaegar)
|
||||||
// RegisterExporter returns the 'Flush' function for that particular tracing service
|
// RegisterExporter returns the 'Flush' function for that particular tracing service.
|
||||||
func RegisterExporter(traceExporter, URL, service, ENV string) (func(), error) {
|
func RegisterExporter(traceExporter, url, service, env string) (func(), error) {
|
||||||
const op errors.Op = "observ.RegisterExporter"
|
const op errors.Op = "observ.RegisterExporter"
|
||||||
switch traceExporter {
|
switch traceExporter {
|
||||||
case "jaeger":
|
case "jaeger":
|
||||||
return registerJaegerExporter(URL, service, ENV)
|
return registerJaegerExporter(url, service, env)
|
||||||
case "datadog":
|
case "datadog":
|
||||||
return registerDatadogExporter(URL, service, ENV)
|
return registerDatadogExporter(url, service, env)
|
||||||
case "stackdriver":
|
case "stackdriver":
|
||||||
return registerStackdriverExporter(URL, ENV)
|
return registerStackdriverExporter(url, env)
|
||||||
case "":
|
case "":
|
||||||
return nil, errors.E(op, "Exporter not specified. Traces won't be exported")
|
return nil, errors.E(op, "Exporter not specified. Traces won't be exported")
|
||||||
default:
|
default:
|
||||||
@@ -32,14 +32,14 @@ func RegisterExporter(traceExporter, URL, service, ENV string) (func(), error) {
|
|||||||
|
|
||||||
// registerJaegerExporter creates a jaeger exporter for exporting traces to opencensus.
|
// registerJaegerExporter creates a jaeger exporter for exporting traces to opencensus.
|
||||||
// Currently uses the 'TraceExporter' variable in the config file.
|
// Currently uses the 'TraceExporter' variable in the config file.
|
||||||
// It should in the future have a nice sampling rate defined
|
// It should in the future have a nice sampling rate defined.
|
||||||
func registerJaegerExporter(URL, service, ENV string) (func(), error) {
|
func registerJaegerExporter(url, service, env string) (func(), error) {
|
||||||
const op errors.Op = "observ.registerJaegarExporter"
|
const op errors.Op = "observ.registerJaegarExporter"
|
||||||
if URL == "" {
|
if url == "" {
|
||||||
return nil, errors.E(op, "Exporter URL is empty. Traces won't be exported")
|
return nil, errors.E(op, "Exporter URL is empty. Traces won't be exported")
|
||||||
}
|
}
|
||||||
ex, err := jaeger.NewExporter(jaeger.Options{
|
ex, err := jaeger.NewExporter(jaeger.Options{
|
||||||
Endpoint: URL,
|
Endpoint: url,
|
||||||
Process: jaeger.Process{
|
Process: jaeger.Process{
|
||||||
ServiceName: service,
|
ServiceName: service,
|
||||||
Tags: []jaeger.Tag{
|
Tags: []jaeger.Tag{
|
||||||
@@ -53,41 +53,41 @@ func registerJaegerExporter(URL, service, ENV string) (func(), error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.E(op, err)
|
return nil, errors.E(op, err)
|
||||||
}
|
}
|
||||||
traceRegisterExporter(ex, ENV)
|
traceRegisterExporter(ex, env)
|
||||||
return ex.Flush, nil
|
return ex.Flush, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func traceRegisterExporter(exporter trace.Exporter, ENV string) {
|
func traceRegisterExporter(exporter trace.Exporter, env string) {
|
||||||
trace.RegisterExporter(exporter)
|
trace.RegisterExporter(exporter)
|
||||||
if ENV == "development" {
|
if env == "development" {
|
||||||
trace.ApplyConfig(trace.Config{DefaultSampler: trace.AlwaysSample()})
|
trace.ApplyConfig(trace.Config{DefaultSampler: trace.AlwaysSample()})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// registerDatadogTracerExporter creates a datadog exporter.
|
// registerDatadogTracerExporter creates a datadog exporter.
|
||||||
// Currently uses the 'TraceExporter' variable in the config file.
|
// Currently uses the 'TraceExporter' variable in the config file.
|
||||||
func registerDatadogExporter(URL, service, ENV string) (func(), error) {
|
func registerDatadogExporter(url, service, env string) (func(), error) {
|
||||||
ex := datadog.NewExporter(
|
ex := datadog.NewExporter(
|
||||||
datadog.Options{
|
datadog.Options{
|
||||||
TraceAddr: URL,
|
TraceAddr: url,
|
||||||
Service: service,
|
Service: service,
|
||||||
})
|
})
|
||||||
traceRegisterExporter(ex, ENV)
|
traceRegisterExporter(ex, env)
|
||||||
return ex.Stop, nil
|
return ex.Stop, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func registerStackdriverExporter(projectID, ENV string) (func(), error) {
|
func registerStackdriverExporter(projectID, env string) (func(), error) {
|
||||||
const op errors.Op = "observ.registerStackdriverExporter"
|
const op errors.Op = "observ.registerStackdriverExporter"
|
||||||
ex, err := stackdriver.NewExporter(stackdriver.Options{ProjectID: projectID})
|
ex, err := stackdriver.NewExporter(stackdriver.Options{ProjectID: projectID})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.E(op, err)
|
return nil, errors.E(op, err)
|
||||||
}
|
}
|
||||||
traceRegisterExporter(ex, ENV)
|
traceRegisterExporter(ex, env)
|
||||||
return ex.Flush, nil
|
return ex.Flush, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// StartSpan takes in a Context Interface and opName and starts a span. It returns the new attached ObserverContext
|
// StartSpan takes in a Context Interface and opName and starts a span. It returns the new attached ObserverContext
|
||||||
// and span
|
// and span.
|
||||||
func StartSpan(ctx context.Context, op string) (context.Context, *trace.Span) {
|
func StartSpan(ctx context.Context, op string) (context.Context, *trace.Span) {
|
||||||
return trace.StartSpan(ctx, op)
|
return trace.StartSpan(ctx, op)
|
||||||
}
|
}
|
||||||
|
|||||||
+4
-4
@@ -15,10 +15,10 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// RegisterStatsExporter determines the type of StatsExporter service for exporting stats from Opencensus
|
// RegisterStatsExporter determines the type of StatsExporter service for exporting stats from Opencensus
|
||||||
// Currently it supports: prometheus
|
// Currently it supports: prometheus.
|
||||||
func RegisterStatsExporter(r *mux.Router, statsExporter, service string) (func(), error) {
|
func RegisterStatsExporter(r *mux.Router, statsExporter, service string) (func(), error) {
|
||||||
const op errors.Op = "observ.RegisterStatsExporter"
|
const op errors.Op = "observ.RegisterStatsExporter"
|
||||||
var stop = func() {}
|
stop := func() {}
|
||||||
var err error
|
var err error
|
||||||
switch statsExporter {
|
switch statsExporter {
|
||||||
case "prometheus":
|
case "prometheus":
|
||||||
@@ -38,7 +38,7 @@ func RegisterStatsExporter(r *mux.Router, statsExporter, service string) (func()
|
|||||||
default:
|
default:
|
||||||
return nil, errors.E(op, fmt.Sprintf("StatsExporter %s not supported. Please open PR or an issue at github.com/gomods/athens", statsExporter))
|
return nil, errors.E(op, fmt.Sprintf("StatsExporter %s not supported. Please open PR or an issue at github.com/gomods/athens", statsExporter))
|
||||||
}
|
}
|
||||||
if err := registerViews(); err != nil {
|
if err = registerViews(); err != nil {
|
||||||
return nil, errors.E(op, err)
|
return nil, errors.E(op, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -89,7 +89,7 @@ func registerStatsStackDriverExporter(projectID string) (func(), error) {
|
|||||||
return sd.Flush, nil
|
return sd.Flush, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// registerViews register stats which should be collected in Athens
|
// registerViews register stats which should be collected in Athens.
|
||||||
func registerViews() error {
|
func registerViews() error {
|
||||||
const op errors.Op = "observ.registerViews"
|
const op errors.Op = "observ.registerViews"
|
||||||
if err := view.Register(
|
if err := view.Register(
|
||||||
|
|||||||
+2
-1
@@ -19,8 +19,9 @@ func DecodePath(encoding string) (path string, err error) {
|
|||||||
return path, nil
|
return path, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Ripped from cmd/go
|
// Ripped from cmd/go.
|
||||||
func decodeString(encoding string) (string, bool) {
|
func decodeString(encoding string) (string, bool) {
|
||||||
|
//nolint:prealloc
|
||||||
var buf []byte
|
var buf []byte
|
||||||
|
|
||||||
bang := false
|
bang := false
|
||||||
|
|||||||
+5
-5
@@ -9,7 +9,7 @@ import (
|
|||||||
"github.com/gorilla/mux"
|
"github.com/gorilla/mux"
|
||||||
)
|
)
|
||||||
|
|
||||||
// GetModule gets the module from the path of a ?go-get=1 request
|
// GetModule gets the module from the path of a ?go-get=1 request.
|
||||||
func GetModule(r *http.Request) (string, error) {
|
func GetModule(r *http.Request) (string, error) {
|
||||||
const op errors.Op = "paths.GetModule"
|
const op errors.Op = "paths.GetModule"
|
||||||
module := mux.Vars(r)["module"]
|
module := mux.Vars(r)["module"]
|
||||||
@@ -19,7 +19,7 @@ func GetModule(r *http.Request) (string, error) {
|
|||||||
return DecodePath(module)
|
return DecodePath(module)
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetVersion gets the version from the path of a ?go-get=1 request
|
// GetVersion gets the version from the path of a ?go-get=1 request.
|
||||||
func GetVersion(r *http.Request) (string, error) {
|
func GetVersion(r *http.Request) (string, error) {
|
||||||
const op errors.Op = "paths.GetVersion"
|
const op errors.Op = "paths.GetVersion"
|
||||||
|
|
||||||
@@ -31,13 +31,13 @@ func GetVersion(r *http.Request) (string, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// AllPathParams holds the module and version in the path of a ?go-get=1
|
// AllPathParams holds the module and version in the path of a ?go-get=1
|
||||||
// request
|
// request.
|
||||||
type AllPathParams struct {
|
type AllPathParams struct {
|
||||||
Module string `json:"module"`
|
Module string `json:"module"`
|
||||||
Version string `json:"version"`
|
Version string `json:"version"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetAllParams fetches the path params from r and returns them
|
// GetAllParams fetches the path params from r and returns them.
|
||||||
func GetAllParams(r *http.Request) (*AllPathParams, error) {
|
func GetAllParams(r *http.Request) (*AllPathParams, error) {
|
||||||
const op errors.Op = "paths.GetAllParams"
|
const op errors.Op = "paths.GetAllParams"
|
||||||
mod, err := GetModule(r)
|
mod, err := GetModule(r)
|
||||||
@@ -54,7 +54,7 @@ func GetAllParams(r *http.Request) (*AllPathParams, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// MatchesPattern reports whether the path prefix of target matches
|
// MatchesPattern reports whether the path prefix of target matches
|
||||||
// pattern (as defined by path.Match)
|
// pattern (as defined by path.Match).
|
||||||
//
|
//
|
||||||
// This tries to keep the same behavior as GOPRIVATE/GONOPROXY/GONOSUMDB,
|
// This tries to keep the same behavior as GOPRIVATE/GONOPROXY/GONOSUMDB,
|
||||||
// and is adopted from:
|
// and is adopted from:
|
||||||
|
|||||||
@@ -3,18 +3,18 @@ package requestid
|
|||||||
import "context"
|
import "context"
|
||||||
|
|
||||||
// HeaderKey is the header key that athens uses
|
// HeaderKey is the header key that athens uses
|
||||||
// to pass request ids into logs and outbound requests
|
// to pass request ids into logs and outbound requests.
|
||||||
const HeaderKey = "Athens-Request-ID"
|
const HeaderKey = "Athens-Request-ID"
|
||||||
|
|
||||||
type key struct{}
|
type key struct{}
|
||||||
|
|
||||||
// SetInContext sets the given requestID into the context
|
// SetInContext sets the given requestID into the context.
|
||||||
func SetInContext(ctx context.Context, id string) context.Context {
|
func SetInContext(ctx context.Context, id string) context.Context {
|
||||||
return context.WithValue(ctx, key{}, id)
|
return context.WithValue(ctx, key{}, id)
|
||||||
}
|
}
|
||||||
|
|
||||||
// FromContext returns a requestID from the context or an empty
|
// FromContext returns a requestID from the context or an empty
|
||||||
// string if not found
|
// string if not found.
|
||||||
func FromContext(ctx context.Context) string {
|
func FromContext(ctx context.Context) string {
|
||||||
id, _ := ctx.Value(key{}).(string)
|
id, _ := ctx.Value(key{}).(string)
|
||||||
return id
|
return id
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ import (
|
|||||||
// what was requested, this is helpful if what was requested
|
// what was requested, this is helpful if what was requested
|
||||||
// was a descriptive version such as a branch name or a full commit sha.
|
// was a descriptive version such as a branch name or a full commit sha.
|
||||||
type Stasher interface {
|
type Stasher interface {
|
||||||
Stash(ctx context.Context, mod string, ver string) (string, error)
|
Stash(ctx context.Context, mod, ver string) (string, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Wrapper helps extend the main stasher's functionality with addons.
|
// Wrapper helps extend the main stasher's functionality with addons.
|
||||||
@@ -46,7 +46,7 @@ type stasher struct {
|
|||||||
|
|
||||||
func (s *stasher) Stash(ctx context.Context, mod, ver string) (string, error) {
|
func (s *stasher) Stash(ctx context.Context, mod, ver string) (string, error) {
|
||||||
const op errors.Op = "stasher.Stash"
|
const op errors.Op = "stasher.Stash"
|
||||||
_, span := observ.StartSpan(ctx, op.String())
|
ctx, span := observ.StartSpan(ctx, op.String())
|
||||||
defer span.End()
|
defer span.End()
|
||||||
log.EntryFromContext(ctx).Debugf("saving %s@%s to storage...", mod, ver)
|
log.EntryFromContext(ctx).Debugf("saving %s@%s to storage...", mod, ver)
|
||||||
|
|
||||||
@@ -58,7 +58,7 @@ func (s *stasher) Stash(ctx context.Context, mod, ver string) (string, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return "", errors.E(op, err)
|
return "", errors.E(op, err)
|
||||||
}
|
}
|
||||||
defer v.Zip.Close()
|
defer func() { _ = v.Zip.Close() }()
|
||||||
if v.Semver != ver {
|
if v.Semver != ver {
|
||||||
exists, err := s.checker.Exists(ctx, mod, v.Semver)
|
exists, err := s.checker.Exists(ctx, mod, v.Semver)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -132,8 +132,8 @@ func (s *azblobLock) acquireLease(ctx context.Context, blobURL azblob.BlockBlobU
|
|||||||
_, err := blobURL.Upload(tctx, bytes.NewReader([]byte{1}), azblob.BlobHTTPHeaders{}, nil, azblob.BlobAccessConditions{})
|
_, err := blobURL.Upload(tctx, bytes.NewReader([]byte{1}), azblob.BlobHTTPHeaders{}, nil, azblob.BlobAccessConditions{})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// if the blob is already leased we will get http.StatusPreconditionFailed while writing to that blob
|
// if the blob is already leased we will get http.StatusPreconditionFailed while writing to that blob
|
||||||
stgErr, ok := err.(azblob.StorageError)
|
var stgErr azblob.StorageError
|
||||||
if !ok || stgErr.Response().StatusCode != http.StatusPreconditionFailed {
|
if !errors.AsErr(err, &stgErr) || stgErr.Response().StatusCode != http.StatusPreconditionFailed {
|
||||||
return "", errors.E(op, err)
|
return "", errors.E(op, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -147,7 +147,8 @@ func (s *azblobLock) acquireLease(ctx context.Context, blobURL azblob.BlockBlobU
|
|||||||
res, err := blobURL.AcquireLease(tctx, leaseID.String(), 15, azblob.ModifiedAccessConditions{})
|
res, err := blobURL.AcquireLease(tctx, leaseID.String(), 15, azblob.ModifiedAccessConditions{})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// if the blob is already leased we will get http.StatusConflict - wait and try again
|
// if the blob is already leased we will get http.StatusConflict - wait and try again
|
||||||
if stgErr, ok := err.(azblob.StorageError); ok && stgErr.Response().StatusCode == http.StatusConflict {
|
var stgErr azblob.StorageError
|
||||||
|
if ok := errors.AsErr(err, &stgErr); ok && stgErr.Response().StatusCode == http.StatusConflict {
|
||||||
select {
|
select {
|
||||||
case <-time.After(1 * time.Second):
|
case <-time.After(1 * time.Second):
|
||||||
continue
|
continue
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ import (
|
|||||||
"github.com/gomods/athens/pkg/errors"
|
"github.com/gomods/athens/pkg/errors"
|
||||||
"github.com/gomods/athens/pkg/observ"
|
"github.com/gomods/athens/pkg/observ"
|
||||||
"github.com/gomods/athens/pkg/storage"
|
"github.com/gomods/athens/pkg/storage"
|
||||||
"go.etcd.io/etcd/client/v3"
|
clientv3 "go.etcd.io/etcd/client/v3"
|
||||||
"go.etcd.io/etcd/client/v3/concurrency"
|
"go.etcd.io/etcd/client/v3/concurrency"
|
||||||
"golang.org/x/sync/errgroup"
|
"golang.org/x/sync/errgroup"
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -13,14 +13,14 @@ import (
|
|||||||
"github.com/gomods/athens/pkg/storage"
|
"github.com/gomods/athens/pkg/storage"
|
||||||
)
|
)
|
||||||
|
|
||||||
// RedisLogger mirrors github.com/go-redis/redis/v8/internal.Logging
|
// RedisLogger mirrors github.com/go-redis/redis/v8/internal.Logging.
|
||||||
type RedisLogger interface {
|
type RedisLogger interface {
|
||||||
Printf(ctx context.Context, format string, v ...interface{})
|
Printf(ctx context.Context, format string, v ...interface{})
|
||||||
}
|
}
|
||||||
|
|
||||||
// WithRedisLock returns a distributed singleflight
|
// WithRedisLock returns a distributed singleflight
|
||||||
// using a redis cluster. If it cannot connect, it will return an error.
|
// using a redis cluster. If it cannot connect, it will return an error.
|
||||||
func WithRedisLock(l RedisLogger, endpoint string, password string, checker storage.Checker, lockConfig *config.RedisLockConfig) (Wrapper, error) {
|
func WithRedisLock(l RedisLogger, endpoint, password string, checker storage.Checker, lockConfig *config.RedisLockConfig) (Wrapper, error) {
|
||||||
redis.SetLogger(l)
|
redis.SetLogger(l)
|
||||||
|
|
||||||
const op errors.Op = "stash.WithRedisLock"
|
const op errors.Op = "stash.WithRedisLock"
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// WithRedisSentinelLock returns a distributed singleflight
|
// WithRedisSentinelLock returns a distributed singleflight
|
||||||
// with a redis cluster that utilizes sentinel for quorum and failover
|
// with a redis cluster that utilizes sentinel for quorum and failover.
|
||||||
func WithRedisSentinelLock(l RedisLogger, endpoints []string, master, password string, checker storage.Checker, lockConfig *config.RedisLockConfig) (Wrapper, error) {
|
func WithRedisSentinelLock(l RedisLogger, endpoints []string, master, password string, checker storage.Checker, lockConfig *config.RedisLockConfig) (Wrapper, error) {
|
||||||
redis.SetLogger(l)
|
redis.SetLogger(l)
|
||||||
|
|
||||||
|
|||||||
@@ -15,14 +15,6 @@ import (
|
|||||||
"github.com/gomods/athens/pkg/storage"
|
"github.com/gomods/athens/pkg/storage"
|
||||||
)
|
)
|
||||||
|
|
||||||
type client interface {
|
|
||||||
UploadWithContext(ctx context.Context, path, contentType string, content io.Reader) error
|
|
||||||
BlobExists(ctx context.Context, path string) (bool, error)
|
|
||||||
ReadBlob(ctx context.Context, path string) (io.ReadCloser, error)
|
|
||||||
ListBlobs(ctx context.Context, prefix string) ([]string, error)
|
|
||||||
DeleteBlob(ctx context.Context, path string) error
|
|
||||||
}
|
|
||||||
|
|
||||||
type azureBlobStoreClient struct {
|
type azureBlobStoreClient struct {
|
||||||
containerURL *azblob.ContainerURL
|
containerURL *azblob.ContainerURL
|
||||||
}
|
}
|
||||||
@@ -45,13 +37,13 @@ func newBlobStoreClient(accountURL *url.URL, accountName, accountKey, containerN
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Storage implements (github.com/gomods/athens/pkg/storage).Saver and
|
// Storage implements (github.com/gomods/athens/pkg/storage).Saver and
|
||||||
// also provides a function to fetch the location of a module
|
// also provides a function to fetch the location of a module.
|
||||||
type Storage struct {
|
type Storage struct {
|
||||||
client *azureBlobStoreClient
|
client *azureBlobStoreClient
|
||||||
timeout time.Duration
|
timeout time.Duration
|
||||||
}
|
}
|
||||||
|
|
||||||
// New creates a new azure blobs storage
|
// New creates a new azure blobs storage.
|
||||||
func New(conf *config.AzureBlobConfig, timeout time.Duration) (*Storage, error) {
|
func New(conf *config.AzureBlobConfig, timeout time.Duration) (*Storage, error) {
|
||||||
const op errors.Op = "azureblob.New"
|
const op errors.Op = "azureblob.New"
|
||||||
u, err := url.Parse(fmt.Sprintf("https://%s.blob.core.windows.net", conf.AccountName))
|
u, err := url.Parse(fmt.Sprintf("https://%s.blob.core.windows.net", conf.AccountName))
|
||||||
@@ -65,7 +57,7 @@ func New(conf *config.AzureBlobConfig, timeout time.Duration) (*Storage, error)
|
|||||||
return &Storage{client: cl, timeout: timeout}, nil
|
return &Storage{client: cl, timeout: timeout}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// BlobExists checks if a particular blob exists in the container
|
// BlobExists checks if a particular blob exists in the container.
|
||||||
func (c *azureBlobStoreClient) BlobExists(ctx context.Context, path string) (bool, error) {
|
func (c *azureBlobStoreClient) BlobExists(ctx context.Context, path string) (bool, error) {
|
||||||
const op errors.Op = "azureblob.BlobExists"
|
const op errors.Op = "azureblob.BlobExists"
|
||||||
// TODO: Any better way of doing this ?
|
// TODO: Any better way of doing this ?
|
||||||
@@ -73,10 +65,8 @@ func (c *azureBlobStoreClient) BlobExists(ctx context.Context, path string) (boo
|
|||||||
_, err := blobURL.GetProperties(ctx, azblob.BlobAccessConditions{})
|
_, err := blobURL.GetProperties(ctx, azblob.BlobAccessConditions{})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
var serr azblob.StorageError
|
var serr azblob.StorageError
|
||||||
var ok bool
|
if !errors.AsErr(err, &serr) {
|
||||||
|
return false, errors.E(op, fmt.Errorf("error in casting to azure error type %w", err))
|
||||||
if serr, ok = err.(azblob.StorageError); !ok {
|
|
||||||
return false, errors.E(op, fmt.Errorf("Error in casting to azure error type %v", err))
|
|
||||||
}
|
}
|
||||||
if serr.Response().StatusCode == http.StatusNotFound {
|
if serr.Response().StatusCode == http.StatusNotFound {
|
||||||
return false, nil
|
return false, nil
|
||||||
@@ -85,10 +75,9 @@ func (c *azureBlobStoreClient) BlobExists(ctx context.Context, path string) (boo
|
|||||||
return false, errors.E(op, err)
|
return false, errors.E(op, err)
|
||||||
}
|
}
|
||||||
return true, nil
|
return true, nil
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ReadBlob returns a storage.SizeReadCloser for the contents of a blob
|
// ReadBlob returns a storage.SizeReadCloser for the contents of a blob.
|
||||||
func (c *azureBlobStoreClient) ReadBlob(ctx context.Context, path string) (storage.SizeReadCloser, error) {
|
func (c *azureBlobStoreClient) ReadBlob(ctx context.Context, path string) (storage.SizeReadCloser, error) {
|
||||||
const op errors.Op = "azureblob.ReadBlob"
|
const op errors.Op = "azureblob.ReadBlob"
|
||||||
blobURL := c.containerURL.NewBlockBlobURL(path)
|
blobURL := c.containerURL.NewBlockBlobURL(path)
|
||||||
@@ -101,7 +90,7 @@ func (c *azureBlobStoreClient) ReadBlob(ctx context.Context, path string) (stora
|
|||||||
return storage.NewSizer(rc, size), nil
|
return storage.NewSizer(rc, size), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// ListBlobs will list all blobs which has the given prefix
|
// ListBlobs will list all blobs which has the given prefix.
|
||||||
func (c *azureBlobStoreClient) ListBlobs(ctx context.Context, prefix string) ([]string, error) {
|
func (c *azureBlobStoreClient) ListBlobs(ctx context.Context, prefix string) ([]string, error) {
|
||||||
const op errors.Op = "azureblob.ListBlobs"
|
const op errors.Op = "azureblob.ListBlobs"
|
||||||
var blobs []string
|
var blobs []string
|
||||||
@@ -122,7 +111,7 @@ func (c *azureBlobStoreClient) ListBlobs(ctx context.Context, prefix string) ([]
|
|||||||
return blobs, nil
|
return blobs, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// DeleteBlob deletes the blob with the given path
|
// DeleteBlob deletes the blob with the given path.
|
||||||
func (c *azureBlobStoreClient) DeleteBlob(ctx context.Context, path string) error {
|
func (c *azureBlobStoreClient) DeleteBlob(ctx context.Context, path string) error {
|
||||||
const op errors.Op = "azureblob.DeleteBlob"
|
const op errors.Op = "azureblob.DeleteBlob"
|
||||||
blobURL := c.containerURL.NewBlockBlobURL(path)
|
blobURL := c.containerURL.NewBlockBlobURL(path)
|
||||||
@@ -133,7 +122,7 @@ func (c *azureBlobStoreClient) DeleteBlob(ctx context.Context, path string) erro
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// UploadWithContext uploads a blob to the container
|
// UploadWithContext uploads a blob to the container.
|
||||||
func (c *azureBlobStoreClient) UploadWithContext(ctx context.Context, path, contentType string, content io.Reader) error {
|
func (c *azureBlobStoreClient) UploadWithContext(ctx context.Context, path, contentType string, content io.Reader) error {
|
||||||
const op errors.Op = "azureblob.UploadWithContext"
|
const op errors.Op = "azureblob.UploadWithContext"
|
||||||
ctx, span := observ.StartSpan(ctx, op.String())
|
ctx, span := observ.StartSpan(ctx, op.String())
|
||||||
|
|||||||
@@ -12,8 +12,8 @@ import (
|
|||||||
"github.com/gomods/athens/pkg/paths"
|
"github.com/gomods/athens/pkg/paths"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Catalog implements the (./pkg/storage).Catalog interface
|
// Catalog implements the (./pkg/storage).Catalog interface.
|
||||||
// It returns a list of versions, if any, for a given module
|
// It returns a list of versions, if any, for a given module.
|
||||||
func (s *Storage) Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error) {
|
func (s *Storage) Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error) {
|
||||||
const op errors.Op = "azblob.Catalog"
|
const op errors.Op = "azblob.Catalog"
|
||||||
ctx, span := observ.StartSpan(ctx, op.String())
|
ctx, span := observ.StartSpan(ctx, op.String())
|
||||||
|
|||||||
@@ -9,8 +9,8 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// Exists implements the (./pkg/storage).Checker interface
|
// Exists implements the (./pkg/storage).Checker interface
|
||||||
// returning true if the module at version exists in storage
|
// returning true if the module at version exists in storage.
|
||||||
func (s *Storage) Exists(ctx context.Context, module string, version string) (bool, error) {
|
func (s *Storage) Exists(ctx context.Context, module, version string) (bool, error) {
|
||||||
const op errors.Op = "azureblob.Exists"
|
const op errors.Op = "azureblob.Exists"
|
||||||
ctx, span := observ.StartSpan(ctx, op.String())
|
ctx, span := observ.StartSpan(ctx, op.String())
|
||||||
defer span.End()
|
defer span.End()
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ import (
|
|||||||
// Delete implements the (./pkg/storage).Deleter interface and
|
// Delete implements the (./pkg/storage).Deleter interface and
|
||||||
// removes a version of a module from storage. Returning ErrNotFound
|
// removes a version of a module from storage. Returning ErrNotFound
|
||||||
// if the version does not exist.
|
// if the version does not exist.
|
||||||
func (s *Storage) Delete(ctx context.Context, module string, version string) error {
|
func (s *Storage) Delete(ctx context.Context, module, version string) error {
|
||||||
const op errors.Op = "azureblob.Delete"
|
const op errors.Op = "azureblob.Delete"
|
||||||
ctx, span := observ.StartSpan(ctx, op.String())
|
ctx, span := observ.StartSpan(ctx, op.String())
|
||||||
defer span.End()
|
defer span.End()
|
||||||
|
|||||||
@@ -11,8 +11,8 @@ import (
|
|||||||
"github.com/gomods/athens/pkg/storage"
|
"github.com/gomods/athens/pkg/storage"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Info implements the (./pkg/storage).Getter interface
|
// Info implements the (./pkg/storage).Getter interface.
|
||||||
func (s *Storage) Info(ctx context.Context, module string, version string) ([]byte, error) {
|
func (s *Storage) Info(ctx context.Context, module, version string) ([]byte, error) {
|
||||||
const op errors.Op = "azureblob.Info"
|
const op errors.Op = "azureblob.Info"
|
||||||
ctx, span := observ.StartSpan(ctx, op.String())
|
ctx, span := observ.StartSpan(ctx, op.String())
|
||||||
defer span.End()
|
defer span.End()
|
||||||
@@ -43,8 +43,8 @@ func (s *Storage) Info(ctx context.Context, module string, version string) ([]by
|
|||||||
return infoBytes, nil
|
return infoBytes, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// GoMod implements the (./pkg/storage).Getter interface
|
// GoMod implements the (./pkg/storage).Getter interface.
|
||||||
func (s *Storage) GoMod(ctx context.Context, module string, version string) ([]byte, error) {
|
func (s *Storage) GoMod(ctx context.Context, module, version string) ([]byte, error) {
|
||||||
const op errors.Op = "azureblob.GoMod"
|
const op errors.Op = "azureblob.GoMod"
|
||||||
ctx, span := observ.StartSpan(ctx, op.String())
|
ctx, span := observ.StartSpan(ctx, op.String())
|
||||||
defer span.End()
|
defer span.End()
|
||||||
@@ -64,7 +64,7 @@ func (s *Storage) GoMod(ctx context.Context, module string, version string) ([]b
|
|||||||
|
|
||||||
modBytes, err := io.ReadAll(modReader)
|
modBytes, err := io.ReadAll(modReader)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.E(op, fmt.Errorf("could not get new reader for mod file: %s", err), errors.M(module), errors.V(version))
|
return nil, errors.E(op, fmt.Errorf("could not get new reader for mod file: %w", err), errors.M(module), errors.V(version))
|
||||||
}
|
}
|
||||||
|
|
||||||
err = modReader.Close()
|
err = modReader.Close()
|
||||||
@@ -75,8 +75,8 @@ func (s *Storage) GoMod(ctx context.Context, module string, version string) ([]b
|
|||||||
return modBytes, nil
|
return modBytes, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Zip implements the (./pkg/storage).Getter interface
|
// Zip implements the (./pkg/storage).Getter interface.
|
||||||
func (s *Storage) Zip(ctx context.Context, module string, version string) (storage.SizeReadCloser, error) {
|
func (s *Storage) Zip(ctx context.Context, module, version string) (storage.SizeReadCloser, error) {
|
||||||
const op errors.Op = "azureblob.Zip"
|
const op errors.Op = "azureblob.Zip"
|
||||||
ctx, span := observ.StartSpan(ctx, op.String())
|
ctx, span := observ.StartSpan(ctx, op.String())
|
||||||
defer span.End()
|
defer span.End()
|
||||||
|
|||||||
@@ -8,8 +8,8 @@ import (
|
|||||||
"github.com/gomods/athens/pkg/observ"
|
"github.com/gomods/athens/pkg/observ"
|
||||||
)
|
)
|
||||||
|
|
||||||
// List implements the (./pkg/storage).Lister interface
|
// List implements the (./pkg/storage).Lister interface.
|
||||||
// It returns a list of versions, if any, for a given module
|
// It returns a list of versions, if any, for a given module.
|
||||||
func (s *Storage) List(ctx context.Context, module string) ([]string, error) {
|
func (s *Storage) List(ctx context.Context, module string) ([]string, error) {
|
||||||
const op errors.Op = "azureblob.List"
|
const op errors.Op = "azureblob.List"
|
||||||
ctx, span := observ.StartSpan(ctx, op.String())
|
ctx, span := observ.StartSpan(ctx, op.String())
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
package storage
|
package storage
|
||||||
|
|
||||||
// Backend is a complete storage backend (i.e. file system, database) implementation - a lister, reader and saver
|
// Backend is a complete storage backend (i.e. file system, database) implementation - a lister, reader and saver.
|
||||||
type Backend interface {
|
type Backend interface {
|
||||||
Lister
|
Lister
|
||||||
Getter
|
Getter
|
||||||
|
|||||||
@@ -6,7 +6,7 @@ import (
|
|||||||
"github.com/gomods/athens/pkg/paths"
|
"github.com/gomods/athens/pkg/paths"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Cataloger is the interface that lists all the modules and version contained in the storage
|
// Cataloger is the interface that lists all the modules and version contained in the storage.
|
||||||
type Cataloger interface {
|
type Cataloger interface {
|
||||||
// Catalog gets all the modules / versions.
|
// Catalog gets all the modules / versions.
|
||||||
Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error)
|
Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error)
|
||||||
|
|||||||
@@ -6,14 +6,14 @@ import (
|
|||||||
"github.com/gomods/athens/pkg/errors"
|
"github.com/gomods/athens/pkg/errors"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Checker is the interface that checks if the version of the module exists
|
// Checker is the interface that checks if the version of the module exists.
|
||||||
type Checker interface {
|
type Checker interface {
|
||||||
// Exists checks whether or not module in specified version is present
|
// Exists checks whether or not module in specified version is present
|
||||||
// in the backing storage
|
// in the backing storage.
|
||||||
Exists(ctx context.Context, module, version string) (bool, error)
|
Exists(ctx context.Context, module, version string) (bool, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
// WithChecker wraps the backend with a Checker implementaiton
|
// WithChecker wraps the backend with a Checker implementation.
|
||||||
func WithChecker(strg Backend) Checker {
|
func WithChecker(strg Backend) Checker {
|
||||||
if checker, ok := strg.(Checker); ok {
|
if checker, ok := strg.(Checker); ok {
|
||||||
return checker
|
return checker
|
||||||
|
|||||||
@@ -6,7 +6,6 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"math/rand"
|
"math/rand"
|
||||||
"sort"
|
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/gomods/athens/pkg/errors"
|
"github.com/gomods/athens/pkg/errors"
|
||||||
@@ -26,7 +25,6 @@ func RunTests(t *testing.T, b storage.Backend, clearBackend func() error) {
|
|||||||
testGet(t, b)
|
testGet(t, b)
|
||||||
testExists(t, b)
|
testExists(t, b)
|
||||||
testShouldNotExist(t, b)
|
testShouldNotExist(t, b)
|
||||||
// testCatalog(t, b)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// testNotFound ensures that a storage Backend
|
// testNotFound ensures that a storage Backend
|
||||||
@@ -209,49 +207,6 @@ func testDelete(t *testing.T, b storage.Backend) {
|
|||||||
require.Equal(t, false, exists)
|
require.Equal(t, false, exists)
|
||||||
}
|
}
|
||||||
|
|
||||||
func testCatalog(t *testing.T, b storage.Backend) {
|
|
||||||
cs, ok := b.(storage.Cataloger)
|
|
||||||
if !ok {
|
|
||||||
t.Skip()
|
|
||||||
}
|
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
mock := getMockModule()
|
|
||||||
zipBts, _ := io.ReadAll(mock.Zip)
|
|
||||||
modname := "github.com/gomods/testCatalogModule"
|
|
||||||
for i := 0; i < 6; i++ {
|
|
||||||
ver := fmt.Sprintf("v1.2.%04d", i)
|
|
||||||
b.Save(ctx, modname, ver, mock.Mod, bytes.NewReader(zipBts), mock.Info)
|
|
||||||
|
|
||||||
defer b.Delete(ctx, modname, ver)
|
|
||||||
}
|
|
||||||
|
|
||||||
allres, next, err := cs.Catalog(ctx, "", 5)
|
|
||||||
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Equal(t, 5, len(allres))
|
|
||||||
|
|
||||||
res, next, err := cs.Catalog(ctx, next, 50)
|
|
||||||
allres = append(allres, res...)
|
|
||||||
require.NoError(t, err)
|
|
||||||
require.Equal(t, 1, len(res))
|
|
||||||
require.Equal(t, "", next)
|
|
||||||
|
|
||||||
sort.Slice(allres, func(i, j int) bool {
|
|
||||||
if allres[i].Module == allres[j].Module {
|
|
||||||
return allres[i].Version < allres[j].Version
|
|
||||||
}
|
|
||||||
return allres[i].Module < allres[j].Module
|
|
||||||
})
|
|
||||||
require.Equal(t, modname, allres[0].Module)
|
|
||||||
require.Equal(t, "v1.2.0000", allres[0].Version)
|
|
||||||
require.Equal(t, "v1.2.0004", allres[4].Version)
|
|
||||||
|
|
||||||
for i := 1; i < len(allres); i++ {
|
|
||||||
require.NotEqual(t, allres[i].Version, allres[i-1].Version)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func getMockModule() *storage.Version {
|
func getMockModule() *storage.Version {
|
||||||
return &storage.Version{
|
return &storage.Version{
|
||||||
Info: []byte("123"),
|
Info: []byte("123"),
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ package storage
|
|||||||
|
|
||||||
import "context"
|
import "context"
|
||||||
|
|
||||||
// Deleter deletes module metadata and its source from underlying storage
|
// Deleter deletes module metadata and its source from underlying storage.
|
||||||
type Deleter interface {
|
type Deleter interface {
|
||||||
// Delete must return ErrNotFound if the module/version are not
|
// Delete must return ErrNotFound if the module/version are not
|
||||||
// found.
|
// found.
|
||||||
|
|||||||
Vendored
+14
-14
@@ -20,7 +20,7 @@ type service struct {
|
|||||||
c *http.Client
|
c *http.Client
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewClient returns an external storage client
|
// NewClient returns an external storage client.
|
||||||
func NewClient(url string, c *http.Client) storage.Backend {
|
func NewClient(url string, c *http.Client) storage.Backend {
|
||||||
if c == nil {
|
if c == nil {
|
||||||
c = &http.Client{}
|
c = &http.Client{}
|
||||||
@@ -93,7 +93,7 @@ func (s *service) Save(ctx context.Context, mod, ver string, modFile []byte, zip
|
|||||||
mw := multipart.NewWriter(pw)
|
mw := multipart.NewWriter(pw)
|
||||||
go func() {
|
go func() {
|
||||||
err := upload(mw, modFile, info, zip)
|
err := upload(mw, modFile, info, zip)
|
||||||
pw.CloseWithError(err)
|
_ = pw.CloseWithError(err)
|
||||||
}()
|
}()
|
||||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, pr)
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, pr)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -104,8 +104,8 @@ func (s *service) Save(ctx context.Context, mod, ver string, modFile []byte, zip
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.E(op, err)
|
return errors.E(op, err)
|
||||||
}
|
}
|
||||||
defer resp.Body.Close()
|
defer func() { _ = resp.Body.Close() }()
|
||||||
if resp.StatusCode != 200 {
|
if resp.StatusCode != http.StatusOK {
|
||||||
bts, _ := io.ReadAll(resp.Body)
|
bts, _ := io.ReadAll(resp.Body)
|
||||||
return errors.E(op, fmt.Errorf("unexpected status code: %v - body: %s", resp.StatusCode, bts), resp.StatusCode)
|
return errors.E(op, fmt.Errorf("unexpected status code: %v - body: %s", resp.StatusCode, bts), resp.StatusCode)
|
||||||
}
|
}
|
||||||
@@ -118,35 +118,35 @@ func (s *service) Delete(ctx context.Context, mod, ver string) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.E(op, err)
|
return errors.E(op, err)
|
||||||
}
|
}
|
||||||
defer body.Close()
|
defer func() { _ = body.Close() }()
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func upload(mw *multipart.Writer, mod, info []byte, zip io.Reader) error {
|
func upload(mw *multipart.Writer, mod, info []byte, zip io.Reader) error {
|
||||||
defer mw.Close()
|
defer func() { _ = mw.Close() }()
|
||||||
infoW, err := mw.CreateFormFile("mod.info", "mod.info")
|
infoW, err := mw.CreateFormFile("mod.info", "mod.info")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("error creating info file: %v", err)
|
return fmt.Errorf("error creating info file: %w", err)
|
||||||
}
|
}
|
||||||
_, err = infoW.Write(info)
|
_, err = infoW.Write(info)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("error writing info file: %v", err)
|
return fmt.Errorf("error writing info file: %w", err)
|
||||||
}
|
}
|
||||||
modW, err := mw.CreateFormFile("mod.mod", "mod.mod")
|
modW, err := mw.CreateFormFile("mod.mod", "mod.mod")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("error creating mod file: %v", err)
|
return fmt.Errorf("error creating mod file: %w", err)
|
||||||
}
|
}
|
||||||
_, err = modW.Write(mod)
|
_, err = modW.Write(mod)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("error writing mod file: %v", err)
|
return fmt.Errorf("error writing mod file: %w", err)
|
||||||
}
|
}
|
||||||
zipW, err := mw.CreateFormFile("mod.zip", "mod.zip")
|
zipW, err := mw.CreateFormFile("mod.zip", "mod.zip")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("error creating zip file: %v", err)
|
return fmt.Errorf("error creating zip file: %w", err)
|
||||||
}
|
}
|
||||||
_, err = io.Copy(zipW, zip)
|
_, err = io.Copy(zipW, zip)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("error writing zip file: %v", err)
|
return fmt.Errorf("error writing zip file: %w", err)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -174,9 +174,9 @@ func (s *service) doRequest(ctx context.Context, method, mod, ver, ext string) (
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, 0, errors.E(op, err)
|
return nil, 0, errors.E(op, err)
|
||||||
}
|
}
|
||||||
if resp.StatusCode != 200 {
|
if resp.StatusCode != http.StatusOK {
|
||||||
body, _ := io.ReadAll(resp.Body)
|
body, _ := io.ReadAll(resp.Body)
|
||||||
resp.Body.Close()
|
_ = resp.Body.Close()
|
||||||
return nil, 0, errors.E(op, fmt.Errorf("none 200 status code: %v - body: %s", resp.StatusCode, body), resp.StatusCode)
|
return nil, 0, errors.E(op, fmt.Errorf("none 200 status code: %v - body: %s", resp.StatusCode, body), resp.StatusCode)
|
||||||
}
|
}
|
||||||
var size int64
|
var size int64
|
||||||
|
|||||||
Vendored
+21
-21
@@ -17,7 +17,7 @@ import (
|
|||||||
|
|
||||||
// NewServer takes a storage.Backend implementation of your
|
// NewServer takes a storage.Backend implementation of your
|
||||||
// choice, and returns a new http.Handler that Athens can
|
// choice, and returns a new http.Handler that Athens can
|
||||||
// reach out to for storage operations
|
// reach out to for storage operations.
|
||||||
func NewServer(strg storage.Backend) http.Handler {
|
func NewServer(strg storage.Backend) http.Handler {
|
||||||
r := mux.NewRouter()
|
r := mux.NewRouter()
|
||||||
r.HandleFunc(download.PathList, func(w http.ResponseWriter, r *http.Request) {
|
r.HandleFunc(download.PathList, func(w http.ResponseWriter, r *http.Request) {
|
||||||
@@ -27,12 +27,12 @@ func NewServer(strg storage.Backend) http.Handler {
|
|||||||
http.Error(w, err.Error(), errors.Kind(err))
|
http.Error(w, err.Error(), errors.Kind(err))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
fmt.Fprintf(w, "%s", strings.Join(list, "\n"))
|
_, _ = fmt.Fprintf(w, "%s", strings.Join(list, "\n"))
|
||||||
}).Methods(http.MethodGet)
|
}).Methods(http.MethodGet)
|
||||||
r.HandleFunc(download.PathVersionInfo, func(w http.ResponseWriter, r *http.Request) {
|
r.HandleFunc(download.PathVersionInfo, func(w http.ResponseWriter, r *http.Request) {
|
||||||
params, err := paths.GetAllParams(r)
|
params, err := paths.GetAllParams(r)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
http.Error(w, err.Error(), 400)
|
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
info, err := strg.Info(r.Context(), params.Module, params.Version)
|
info, err := strg.Info(r.Context(), params.Module, params.Version)
|
||||||
@@ -40,12 +40,12 @@ func NewServer(strg storage.Backend) http.Handler {
|
|||||||
http.Error(w, err.Error(), errors.Kind(err))
|
http.Error(w, err.Error(), errors.Kind(err))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
w.Write(info)
|
_, _ = w.Write(info)
|
||||||
}).Methods(http.MethodGet)
|
}).Methods(http.MethodGet)
|
||||||
r.HandleFunc(download.PathVersionModule, func(w http.ResponseWriter, r *http.Request) {
|
r.HandleFunc(download.PathVersionModule, func(w http.ResponseWriter, r *http.Request) {
|
||||||
params, err := paths.GetAllParams(r)
|
params, err := paths.GetAllParams(r)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
http.Error(w, err.Error(), 400)
|
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
mod, err := strg.GoMod(r.Context(), params.Module, params.Version)
|
mod, err := strg.GoMod(r.Context(), params.Module, params.Version)
|
||||||
@@ -53,12 +53,12 @@ func NewServer(strg storage.Backend) http.Handler {
|
|||||||
http.Error(w, err.Error(), errors.Kind(err))
|
http.Error(w, err.Error(), errors.Kind(err))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
w.Write(mod)
|
_, _ = w.Write(mod)
|
||||||
}).Methods(http.MethodGet)
|
}).Methods(http.MethodGet)
|
||||||
r.HandleFunc(download.PathVersionZip, func(w http.ResponseWriter, r *http.Request) {
|
r.HandleFunc(download.PathVersionZip, func(w http.ResponseWriter, r *http.Request) {
|
||||||
params, err := paths.GetAllParams(r)
|
params, err := paths.GetAllParams(r)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
http.Error(w, err.Error(), 400)
|
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
zip, err := strg.Zip(r.Context(), params.Module, params.Version)
|
zip, err := strg.Zip(r.Context(), params.Module, params.Version)
|
||||||
@@ -66,52 +66,52 @@ func NewServer(strg storage.Backend) http.Handler {
|
|||||||
http.Error(w, err.Error(), errors.Kind(err))
|
http.Error(w, err.Error(), errors.Kind(err))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
defer zip.Close()
|
defer func() { _ = zip.Close() }()
|
||||||
w.Header().Set("Content-Length", strconv.FormatInt(zip.Size(), 10))
|
w.Header().Set("Content-Length", strconv.FormatInt(zip.Size(), 10))
|
||||||
io.Copy(w, zip)
|
_, _ = io.Copy(w, zip)
|
||||||
}).Methods(http.MethodGet)
|
}).Methods(http.MethodGet)
|
||||||
r.HandleFunc("/{module:.+}/@v/{version}.save", func(w http.ResponseWriter, r *http.Request) {
|
r.HandleFunc("/{module:.+}/@v/{version}.save", func(w http.ResponseWriter, r *http.Request) {
|
||||||
params, err := paths.GetAllParams(r)
|
params, err := paths.GetAllParams(r)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
http.Error(w, err.Error(), 400)
|
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
err = r.ParseMultipartForm(zip.MaxZipFile + zip.MaxGoMod)
|
err = r.ParseMultipartForm(zip.MaxZipFile + zip.MaxGoMod)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
http.Error(w, err.Error(), 400)
|
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
infoFile, _, err := r.FormFile("mod.info")
|
infoFile, _, err := r.FormFile("mod.info")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
http.Error(w, err.Error(), 400)
|
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
defer infoFile.Close()
|
defer func() { _ = infoFile.Close() }()
|
||||||
info, err := io.ReadAll(infoFile)
|
info, err := io.ReadAll(infoFile)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
http.Error(w, err.Error(), 400)
|
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
modReader, _, err := r.FormFile("mod.mod")
|
modReader, _, err := r.FormFile("mod.mod")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
http.Error(w, err.Error(), 400)
|
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
defer modReader.Close()
|
defer func() { _ = modReader.Close() }()
|
||||||
modFile, err := io.ReadAll(modReader)
|
modFile, err := io.ReadAll(modReader)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
http.Error(w, err.Error(), 400)
|
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
modZ, _, err := r.FormFile("mod.zip")
|
modZ, _, err := r.FormFile("mod.zip")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
http.Error(w, err.Error(), 400)
|
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
defer modZ.Close()
|
defer func() { _ = modZ.Close() }()
|
||||||
err = strg.Save(r.Context(), params.Module, params.Version, modFile, modZ, info)
|
err = strg.Save(r.Context(), params.Module, params.Version, modFile, modZ, info)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
http.Error(w, err.Error(), 400)
|
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
}).Methods(http.MethodPost)
|
}).Methods(http.MethodPost)
|
||||||
@@ -119,7 +119,7 @@ func NewServer(strg storage.Backend) http.Handler {
|
|||||||
r.HandleFunc("/{module:.+}/@v/{version}.delete", func(w http.ResponseWriter, r *http.Request) {
|
r.HandleFunc("/{module:.+}/@v/{version}.delete", func(w http.ResponseWriter, r *http.Request) {
|
||||||
params, err := paths.GetAllParams(r)
|
params, err := paths.GetAllParams(r)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
http.Error(w, err.Error(), 400)
|
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
err = strg.Delete(r.Context(), params.Module, params.Version)
|
err = strg.Delete(r.Context(), params.Module, params.Version)
|
||||||
|
|||||||
@@ -15,11 +15,11 @@ import (
|
|||||||
|
|
||||||
const tokenSeparator = "|"
|
const tokenSeparator = "|"
|
||||||
|
|
||||||
// Catalog implements the (./pkg/storage).Cataloger interface
|
// Catalog implements the (./pkg/storage).Cataloger interface.
|
||||||
// It returns a list of modules and versions contained in the storage
|
// It returns a list of modules and versions contained in the storage.
|
||||||
func (s *storageImpl) Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error) {
|
func (s *storageImpl) Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error) {
|
||||||
const op errors.Op = "fs.Catalog"
|
const op errors.Op = "fs.Catalog"
|
||||||
ctx, span := observ.StartSpan(ctx, op.String())
|
_, span := observ.StartSpan(ctx, op.String())
|
||||||
defer span.End()
|
defer span.End()
|
||||||
|
|
||||||
fromModule, fromVersion, err := modVerFromToken(token)
|
fromModule, fromVersion, err := modVerFromToken(token)
|
||||||
@@ -41,7 +41,7 @@ func (s *storageImpl) Catalog(ctx context.Context, token string, pageSize int) (
|
|||||||
|
|
||||||
m, version := filepath.Split(modVer)
|
m, version := filepath.Split(modVer)
|
||||||
module := filepath.Clean(m)
|
module := filepath.Clean(m)
|
||||||
module = strings.Replace(module, string(os.PathSeparator), "/", -1)
|
module = strings.ReplaceAll(module, string(os.PathSeparator), "/")
|
||||||
|
|
||||||
if fromModule != "" && module < fromModule { // it is ok to land on the same module
|
if fromModule != "" && module < fromModule { // it is ok to land on the same module
|
||||||
return nil
|
return nil
|
||||||
@@ -60,7 +60,7 @@ func (s *storageImpl) Catalog(ctx context.Context, token string, pageSize int) (
|
|||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
if err != nil && err != io.EOF {
|
if err != nil && !errors.IsErr(err, io.EOF) {
|
||||||
return nil, "", errors.E(op, err, errors.KindUnexpected)
|
return nil, "", errors.E(op, err, errors.KindUnexpected)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -9,14 +9,13 @@ import (
|
|||||||
"github.com/spf13/afero"
|
"github.com/spf13/afero"
|
||||||
)
|
)
|
||||||
|
|
||||||
func (v *storageImpl) Exists(ctx context.Context, module, version string) (bool, error) {
|
func (s *storageImpl) Exists(ctx context.Context, module, version string) (bool, error) {
|
||||||
const op errors.Op = "fs.Exists"
|
const op errors.Op = "fs.Exists"
|
||||||
ctx, span := observ.StartSpan(ctx, op.String())
|
_, span := observ.StartSpan(ctx, op.String())
|
||||||
defer span.End()
|
defer span.End()
|
||||||
versionedPath := v.versionLocation(module, version)
|
versionedPath := s.versionLocation(module, version)
|
||||||
|
|
||||||
files, err := afero.ReadDir(v.filesystem, versionedPath)
|
|
||||||
|
|
||||||
|
files, err := afero.ReadDir(s.filesystem, versionedPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if os.IsNotExist(err) {
|
if os.IsNotExist(err) {
|
||||||
return false, nil
|
return false, nil
|
||||||
|
|||||||
@@ -8,17 +8,17 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// Delete removes a specific version of a module.
|
// Delete removes a specific version of a module.
|
||||||
func (v *storageImpl) Delete(ctx context.Context, module, version string) error {
|
func (s *storageImpl) Delete(ctx context.Context, module, version string) error {
|
||||||
const op errors.Op = "fs.Delete"
|
const op errors.Op = "fs.Delete"
|
||||||
ctx, span := observ.StartSpan(ctx, op.String())
|
ctx, span := observ.StartSpan(ctx, op.String())
|
||||||
defer span.End()
|
defer span.End()
|
||||||
versionedPath := v.versionLocation(module, version)
|
versionedPath := s.versionLocation(module, version)
|
||||||
exists, err := v.Exists(ctx, module, version)
|
exists, err := s.Exists(ctx, module, version)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.E(op, err, errors.M(module), errors.V(version))
|
return errors.E(op, err, errors.M(module), errors.V(version))
|
||||||
}
|
}
|
||||||
if !exists {
|
if !exists {
|
||||||
return errors.E(op, errors.M(module), errors.V(version), errors.KindNotFound)
|
return errors.E(op, errors.M(module), errors.V(version), errors.KindNotFound)
|
||||||
}
|
}
|
||||||
return v.filesystem.RemoveAll(versionedPath)
|
return s.filesystem.RemoveAll(versionedPath)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -21,17 +21,16 @@ func (s *storageImpl) moduleLocation(module string) string {
|
|||||||
|
|
||||||
func (s *storageImpl) versionLocation(module, version string) string {
|
func (s *storageImpl) versionLocation(module, version string) string {
|
||||||
return filepath.Join(s.moduleLocation(module), version)
|
return filepath.Join(s.moduleLocation(module), version)
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewStorage returns a new ListerSaver implementation that stores
|
// NewStorage returns a new ListerSaver implementation that stores
|
||||||
// everything under rootDir
|
// everything under rootDir.
|
||||||
// If the root directory does not exist an error is returned
|
// If the root directory does not exist an error is returned.
|
||||||
func NewStorage(rootDir string, filesystem afero.Fs) (storage.Backend, error) {
|
func NewStorage(rootDir string, filesystem afero.Fs) (storage.Backend, error) {
|
||||||
const op errors.Op = "fs.NewStorage"
|
const op errors.Op = "fs.NewStorage"
|
||||||
exists, err := afero.Exists(filesystem, rootDir)
|
exists, err := afero.Exists(filesystem, rootDir)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.E(op, fmt.Errorf("could not check if root directory `%s` exists: %s", rootDir, err))
|
return nil, errors.E(op, fmt.Errorf("could not check if root directory `%s` exists: %w", rootDir, err))
|
||||||
}
|
}
|
||||||
if !exists {
|
if !exists {
|
||||||
return nil, errors.E(op, fmt.Errorf("root directory `%s` does not exist", rootDir))
|
return nil, errors.E(op, fmt.Errorf("root directory `%s` does not exist", rootDir))
|
||||||
|
|||||||
+12
-24
@@ -11,12 +11,12 @@ import (
|
|||||||
"github.com/spf13/afero"
|
"github.com/spf13/afero"
|
||||||
)
|
)
|
||||||
|
|
||||||
func (v *storageImpl) Info(ctx context.Context, module, version string) ([]byte, error) {
|
func (s *storageImpl) Info(ctx context.Context, module, version string) ([]byte, error) {
|
||||||
const op errors.Op = "fs.Info"
|
const op errors.Op = "fs.Info"
|
||||||
ctx, span := observ.StartSpan(ctx, op.String())
|
_, span := observ.StartSpan(ctx, op.String())
|
||||||
defer span.End()
|
defer span.End()
|
||||||
versionedPath := v.versionLocation(module, version)
|
versionedPath := s.versionLocation(module, version)
|
||||||
info, err := afero.ReadFile(v.filesystem, filepath.Join(versionedPath, version+".info"))
|
info, err := afero.ReadFile(s.filesystem, filepath.Join(versionedPath, version+".info"))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.E(op, errors.M(module), errors.V(version), errors.KindNotFound)
|
return nil, errors.E(op, errors.M(module), errors.V(version), errors.KindNotFound)
|
||||||
}
|
}
|
||||||
@@ -24,12 +24,12 @@ func (v *storageImpl) Info(ctx context.Context, module, version string) ([]byte,
|
|||||||
return info, nil
|
return info, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (v *storageImpl) GoMod(ctx context.Context, module, version string) ([]byte, error) {
|
func (s *storageImpl) GoMod(ctx context.Context, module, version string) ([]byte, error) {
|
||||||
const op errors.Op = "fs.GoMod"
|
const op errors.Op = "fs.GoMod"
|
||||||
ctx, span := observ.StartSpan(ctx, op.String())
|
_, span := observ.StartSpan(ctx, op.String())
|
||||||
defer span.End()
|
defer span.End()
|
||||||
versionedPath := v.versionLocation(module, version)
|
versionedPath := s.versionLocation(module, version)
|
||||||
mod, err := afero.ReadFile(v.filesystem, filepath.Join(versionedPath, "go.mod"))
|
mod, err := afero.ReadFile(s.filesystem, filepath.Join(versionedPath, "go.mod"))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.E(op, errors.M(module), errors.V(version), errors.KindNotFound)
|
return nil, errors.E(op, errors.M(module), errors.V(version), errors.KindNotFound)
|
||||||
}
|
}
|
||||||
@@ -37,13 +37,13 @@ func (v *storageImpl) GoMod(ctx context.Context, module, version string) ([]byte
|
|||||||
return mod, nil
|
return mod, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (v *storageImpl) Zip(ctx context.Context, module, version string) (storage.SizeReadCloser, error) {
|
func (s *storageImpl) Zip(ctx context.Context, module, version string) (storage.SizeReadCloser, error) {
|
||||||
const op errors.Op = "fs.Zip"
|
const op errors.Op = "fs.Zip"
|
||||||
ctx, span := observ.StartSpan(ctx, op.String())
|
_, span := observ.StartSpan(ctx, op.String())
|
||||||
defer span.End()
|
defer span.End()
|
||||||
versionedPath := v.versionLocation(module, version)
|
versionedPath := s.versionLocation(module, version)
|
||||||
|
|
||||||
src, err := v.filesystem.OpenFile(filepath.Join(versionedPath, "source.zip"), os.O_RDONLY, 0666)
|
src, err := s.filesystem.OpenFile(filepath.Join(versionedPath, "source.zip"), os.O_RDONLY, 0o666)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.E(op, errors.M(module), errors.V(version), errors.KindNotFound)
|
return nil, errors.E(op, errors.M(module), errors.V(version), errors.KindNotFound)
|
||||||
}
|
}
|
||||||
@@ -53,15 +53,3 @@ func (v *storageImpl) Zip(ctx context.Context, module, version string) (storage.
|
|||||||
}
|
}
|
||||||
return storage.NewSizer(src, fi.Size()), nil
|
return storage.NewSizer(src, fi.Size()), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (v *storageImpl) ZipSize(ctx context.Context, module, version string) (int64, error) {
|
|
||||||
const op errors.Op = "fs.ZipFileSize"
|
|
||||||
ctx, span := observ.StartSpan(ctx, op.String())
|
|
||||||
defer span.End()
|
|
||||||
versionedPath := v.versionLocation(module, version)
|
|
||||||
fi, err := v.filesystem.Stat(filepath.Join(versionedPath))
|
|
||||||
if err != nil {
|
|
||||||
return 0, errors.E(op, err, errors.M(module), errors.V(version), errors.KindNotFound)
|
|
||||||
}
|
|
||||||
return fi.Size(), nil
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -11,12 +11,12 @@ import (
|
|||||||
"golang.org/x/mod/semver"
|
"golang.org/x/mod/semver"
|
||||||
)
|
)
|
||||||
|
|
||||||
func (l *storageImpl) List(ctx context.Context, module string) ([]string, error) {
|
func (s *storageImpl) List(ctx context.Context, module string) ([]string, error) {
|
||||||
const op errors.Op = "fs.List"
|
const op errors.Op = "fs.List"
|
||||||
ctx, span := observ.StartSpan(ctx, op.String())
|
_, span := observ.StartSpan(ctx, op.String())
|
||||||
defer span.End()
|
defer span.End()
|
||||||
loc := l.moduleLocation(module)
|
loc := s.moduleLocation(module)
|
||||||
fileInfos, err := afero.ReadDir(l.filesystem, loc)
|
fileInfos, err := afero.ReadDir(s.filesystem, loc)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if os.IsNotExist(err) {
|
if os.IsNotExist(err) {
|
||||||
return []string{}, nil
|
return []string{}, nil
|
||||||
|
|||||||
+12
-12
@@ -13,37 +13,37 @@ import (
|
|||||||
|
|
||||||
func (s *storageImpl) Save(ctx context.Context, module, version string, mod []byte, zip io.Reader, info []byte) error {
|
func (s *storageImpl) Save(ctx context.Context, module, version string, mod []byte, zip io.Reader, info []byte) error {
|
||||||
const op errors.Op = "fs.Save"
|
const op errors.Op = "fs.Save"
|
||||||
ctx, span := observ.StartSpan(ctx, op.String())
|
_, span := observ.StartSpan(ctx, op.String())
|
||||||
defer span.End()
|
defer span.End()
|
||||||
dir := s.versionLocation(module, version)
|
dir := s.versionLocation(module, version)
|
||||||
|
|
||||||
// NB: the process's umask is subtracted from the permissions below,
|
// NB: The process's umask is subtracted from the permissions below,
|
||||||
// so a umask of for example 0077 allows directories and files to be
|
// so an umask of for example 0077 allows directories and files to be
|
||||||
// created with mode 0700 / 0600, i.e. not world- or group-readable
|
// created with mode 0700 / 0600, i.e. not world- or group-readable.
|
||||||
|
|
||||||
// make the versioned directory to hold the go.mod and the zipfile
|
// Make the versioned directory to hold the go.mod and the zipfile.
|
||||||
if err := s.filesystem.MkdirAll(dir, 0777); err != nil {
|
if err := s.filesystem.MkdirAll(dir, 0o777); err != nil {
|
||||||
return errors.E(op, err, errors.M(module), errors.V(version))
|
return errors.E(op, err, errors.M(module), errors.V(version))
|
||||||
}
|
}
|
||||||
|
|
||||||
// write the go.mod file
|
// Write the go.mod file.
|
||||||
if err := afero.WriteFile(s.filesystem, filepath.Join(dir, "go.mod"), mod, 0666); err != nil {
|
if err := afero.WriteFile(s.filesystem, filepath.Join(dir, "go.mod"), mod, 0o666); err != nil {
|
||||||
return errors.E(op, err, errors.M(module), errors.V(version))
|
return errors.E(op, err, errors.M(module), errors.V(version))
|
||||||
}
|
}
|
||||||
|
|
||||||
// write the zipfile
|
// Write the zipfile.
|
||||||
f, err := s.filesystem.OpenFile(filepath.Join(dir, "source.zip"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
|
f, err := s.filesystem.OpenFile(filepath.Join(dir, "source.zip"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o666)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.E(op, err, errors.M(module), errors.V(version))
|
return errors.E(op, err, errors.M(module), errors.V(version))
|
||||||
}
|
}
|
||||||
defer f.Close()
|
defer func() { _ = f.Close() }()
|
||||||
_, err = io.Copy(f, zip)
|
_, err = io.Copy(f, zip)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.E(op, err, errors.M(module), errors.V(version))
|
return errors.E(op, err, errors.M(module), errors.V(version))
|
||||||
}
|
}
|
||||||
|
|
||||||
// write the info file
|
// write the info file
|
||||||
err = afero.WriteFile(s.filesystem, filepath.Join(dir, version+".info"), info, 0666)
|
err = afero.WriteFile(s.filesystem, filepath.Join(dir, version+".info"), info, 0o666)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.E(op, err)
|
return errors.E(op, err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// Catalog implements the (./pkg/storage).Catalog interface
|
// Catalog implements the (./pkg/storage).Catalog interface
|
||||||
// It returns a list of versions, if any, for a given module
|
// It returns a list of versions, if any, for a given module.
|
||||||
func (s *Storage) Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error) {
|
func (s *Storage) Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error) {
|
||||||
const op errors.Op = "gcp.Catalog"
|
const op errors.Op = "gcp.Catalog"
|
||||||
ctx, span := observ.StartSpan(ctx, op.String())
|
ctx, span := observ.StartSpan(ctx, op.String())
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// Exists implements the (./pkg/storage).Checker interface
|
// Exists implements the (./pkg/storage).Checker interface
|
||||||
// returning true if the module at version exists in storage
|
// returning true if the module at version exists in storage.
|
||||||
func (s *Storage) Exists(ctx context.Context, module, version string) (bool, error) {
|
func (s *Storage) Exists(ctx context.Context, module, version string) (bool, error) {
|
||||||
const op errors.Op = "gcp.Exists"
|
const op errors.Op = "gcp.Exists"
|
||||||
ctx, span := observ.StartSpan(ctx, op.String())
|
ctx, span := observ.StartSpan(ctx, op.String())
|
||||||
@@ -21,7 +21,7 @@ func (s *Storage) Exists(ctx context.Context, module, version string) (bool, err
|
|||||||
var count int
|
var count int
|
||||||
for {
|
for {
|
||||||
attrs, err := it.Next()
|
attrs, err := it.Next()
|
||||||
if err == iterator.Done {
|
if errors.IsErr(err, iterator.Done) {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ import (
|
|||||||
"google.golang.org/api/option"
|
"google.golang.org/api/option"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Storage implements the (./pkg/storage).Backend interface
|
// Storage implements the (./pkg/storage).Backend interface.
|
||||||
type Storage struct {
|
type Storage struct {
|
||||||
bucket *storage.BucketHandle
|
bucket *storage.BucketHandle
|
||||||
timeout time.Duration
|
timeout time.Duration
|
||||||
@@ -34,8 +34,8 @@ func New(ctx context.Context, gcpConf *config.GCPConfig, timeout time.Duration)
|
|||||||
return nil, errors.E(op, err)
|
return nil, errors.E(op, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if _, err := s.bucket.Attrs(ctx); err != nil {
|
if _, err = s.bucket.Attrs(ctx); err != nil {
|
||||||
if err == storage.ErrBucketNotExist {
|
if errors.IsErr(err, storage.ErrBucketNotExist) {
|
||||||
return nil, errors.E(op, "You must manually create a storage bucket for Athens, see https://cloud.google.com/storage/docs/creating-buckets#storage-create-bucket-console")
|
return nil, errors.E(op, "You must manually create a storage bucket for Athens, see https://cloud.google.com/storage/docs/creating-buckets#storage-create-bucket-console")
|
||||||
}
|
}
|
||||||
return nil, errors.E(op, err)
|
return nil, errors.E(op, err)
|
||||||
@@ -48,21 +48,21 @@ func New(ctx context.Context, gcpConf *config.GCPConfig, timeout time.Duration)
|
|||||||
// this is so that the unit tests can use this to create their own short-lived buckets.
|
// this is so that the unit tests can use this to create their own short-lived buckets.
|
||||||
func newClient(ctx context.Context, gcpConf *config.GCPConfig, timeout time.Duration) (*Storage, error) {
|
func newClient(ctx context.Context, gcpConf *config.GCPConfig, timeout time.Duration) (*Storage, error) {
|
||||||
const op errors.Op = "gcp.newClient"
|
const op errors.Op = "gcp.newClient"
|
||||||
opts := []option.ClientOption{}
|
var opts []option.ClientOption
|
||||||
if gcpConf.JSONKey != "" {
|
if gcpConf.JSONKey != "" {
|
||||||
key, err := base64.StdEncoding.DecodeString(gcpConf.JSONKey)
|
key, err := base64.StdEncoding.DecodeString(gcpConf.JSONKey)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.E(op, fmt.Errorf("could not decode base64 json key: %v", err))
|
return nil, errors.E(op, fmt.Errorf("could not decode base64 json key: %w", err))
|
||||||
}
|
}
|
||||||
creds, err := google.CredentialsFromJSON(ctx, key, storage.ScopeReadWrite)
|
creds, err := google.CredentialsFromJSON(ctx, key, storage.ScopeReadWrite)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.E(op, fmt.Errorf("could not get GCS credentials: %v", err))
|
return nil, errors.E(op, fmt.Errorf("could not get GCS credentials: %w", err))
|
||||||
}
|
}
|
||||||
opts = append(opts, option.WithCredentials(creds))
|
opts = append(opts, option.WithCredentials(creds))
|
||||||
}
|
}
|
||||||
s, err := storage.NewClient(ctx, opts...)
|
s, err := storage.NewClient(ctx, opts...)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.E(op, fmt.Errorf("could not create new storage client: %s", err))
|
return nil, errors.E(op, fmt.Errorf("could not create new storage client: %w", err))
|
||||||
}
|
}
|
||||||
|
|
||||||
return &Storage{
|
return &Storage{
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ import (
|
|||||||
pkgstorage "github.com/gomods/athens/pkg/storage"
|
pkgstorage "github.com/gomods/athens/pkg/storage"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Info implements Getter
|
// Info implements Getter.
|
||||||
func (s *Storage) Info(ctx context.Context, module, version string) ([]byte, error) {
|
func (s *Storage) Info(ctx context.Context, module, version string) ([]byte, error) {
|
||||||
const op errors.Op = "gcp.Info"
|
const op errors.Op = "gcp.Info"
|
||||||
ctx, span := observ.StartSpan(ctx, op.String())
|
ctx, span := observ.StartSpan(ctx, op.String())
|
||||||
@@ -22,14 +22,14 @@ func (s *Storage) Info(ctx context.Context, module, version string) ([]byte, err
|
|||||||
return nil, errors.E(op, err, getErrorKind(err), errors.M(module), errors.V(version))
|
return nil, errors.E(op, err, getErrorKind(err), errors.M(module), errors.V(version))
|
||||||
}
|
}
|
||||||
infoBytes, err := io.ReadAll(infoReader)
|
infoBytes, err := io.ReadAll(infoReader)
|
||||||
infoReader.Close()
|
_ = infoReader.Close()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.E(op, err, errors.M(module), errors.V(version))
|
return nil, errors.E(op, err, errors.M(module), errors.V(version))
|
||||||
}
|
}
|
||||||
return infoBytes, nil
|
return infoBytes, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// GoMod implements Getter
|
// GoMod implements Getter.
|
||||||
func (s *Storage) GoMod(ctx context.Context, module, version string) ([]byte, error) {
|
func (s *Storage) GoMod(ctx context.Context, module, version string) ([]byte, error) {
|
||||||
const op errors.Op = "gcp.GoMod"
|
const op errors.Op = "gcp.GoMod"
|
||||||
ctx, span := observ.StartSpan(ctx, op.String())
|
ctx, span := observ.StartSpan(ctx, op.String())
|
||||||
@@ -39,15 +39,15 @@ func (s *Storage) GoMod(ctx context.Context, module, version string) ([]byte, er
|
|||||||
return nil, errors.E(op, err, getErrorKind(err), errors.M(module), errors.V(version))
|
return nil, errors.E(op, err, getErrorKind(err), errors.M(module), errors.V(version))
|
||||||
}
|
}
|
||||||
modBytes, err := io.ReadAll(modReader)
|
modBytes, err := io.ReadAll(modReader)
|
||||||
modReader.Close()
|
_ = modReader.Close()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.E(op, fmt.Errorf("could not get new reader for mod file: %s", err), errors.M(module), errors.V(version))
|
return nil, errors.E(op, fmt.Errorf("could not get new reader for mod file: %w", err), errors.M(module), errors.V(version))
|
||||||
}
|
}
|
||||||
|
|
||||||
return modBytes, nil
|
return modBytes, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Zip implements Getter
|
// Zip implements Getter.
|
||||||
func (s *Storage) Zip(ctx context.Context, module, version string) (pkgstorage.SizeReadCloser, error) {
|
func (s *Storage) Zip(ctx context.Context, module, version string) (pkgstorage.SizeReadCloser, error) {
|
||||||
const op errors.Op = "gcp.Zip"
|
const op errors.Op = "gcp.Zip"
|
||||||
ctx, span := observ.StartSpan(ctx, op.String())
|
ctx, span := observ.StartSpan(ctx, op.String())
|
||||||
@@ -56,11 +56,11 @@ func (s *Storage) Zip(ctx context.Context, module, version string) (pkgstorage.S
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.E(op, err, getErrorKind(err), errors.M(module), errors.V(version))
|
return nil, errors.E(op, err, getErrorKind(err), errors.M(module), errors.V(version))
|
||||||
}
|
}
|
||||||
return pkgstorage.NewSizer(zipReader, zipReader.Size()), nil
|
return pkgstorage.NewSizer(zipReader, zipReader.Attrs.Size), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func getErrorKind(err error) int {
|
func getErrorKind(err error) int {
|
||||||
if err == storage.ErrObjectNotExist {
|
if errors.IsErr(err, storage.ErrObjectNotExist) {
|
||||||
return errors.KindNotFound
|
return errors.KindNotFound
|
||||||
}
|
}
|
||||||
return errors.KindUnexpected
|
return errors.KindUnexpected
|
||||||
|
|||||||
@@ -10,8 +10,8 @@ import (
|
|||||||
"google.golang.org/api/iterator"
|
"google.golang.org/api/iterator"
|
||||||
)
|
)
|
||||||
|
|
||||||
// List implements the (./pkg/storage).Lister interface
|
// List implements the (./pkg/storage).Lister interface.
|
||||||
// It returns a list of versions, if any, for a given module
|
// It returns a list of versions, if any, for a given module.
|
||||||
func (s *Storage) List(ctx context.Context, module string) ([]string, error) {
|
func (s *Storage) List(ctx context.Context, module string) ([]string, error) {
|
||||||
const op errors.Op = "gcp.List"
|
const op errors.Op = "gcp.List"
|
||||||
ctx, span := observ.StartSpan(ctx, op.String())
|
ctx, span := observ.StartSpan(ctx, op.String())
|
||||||
@@ -22,7 +22,7 @@ func (s *Storage) List(ctx context.Context, module string) ([]string, error) {
|
|||||||
paths := []string{}
|
paths := []string{}
|
||||||
for {
|
for {
|
||||||
attrs, err := it.Next()
|
attrs, err := it.Next()
|
||||||
if err == iterator.Done {
|
if errors.IsErr(err, iterator.Done) {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user