mirror of
https://github.com/gomods/athens
synced 2026-02-03 07:30:32 +00:00
chore: lint code with golangci-lint (#1828)
* feat: add golangci-lint linting * chore: fix linter issues * feat: add linting into the workflow * docs: update lint docs * fix: cr suggestions * fix: remove old formatting and vetting scripts * fix: add docker make target * fix: action go caching * fix: depreciated actions checkout version * fix: cr suggestion * fix: cr suggestions --------- Co-authored-by: Manu Gupta <manugupt1@gmail.com>
This commit is contained in:
@@ -3,6 +3,18 @@ on:
|
||||
- push
|
||||
- pull_request
|
||||
jobs:
|
||||
lint:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: true
|
||||
- name: Lint code
|
||||
run: make lint-docker
|
||||
|
||||
build:
|
||||
env:
|
||||
ATHENS_MONGO_STORAGE_URL: mongodb://localhost:27017
|
||||
@@ -53,7 +65,7 @@ jobs:
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
cache: true
|
||||
- name: Run linter
|
||||
- name: Verify changes
|
||||
run: make verify
|
||||
- name: Unit tests
|
||||
run: go test -v -race ./...
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
run:
|
||||
tests: false
|
||||
timeout: 5m
|
||||
|
||||
linters-settings:
|
||||
cyclop:
|
||||
max-complexity: 12
|
||||
skip-tests: true
|
||||
gofumpt:
|
||||
extra-rules: true
|
||||
|
||||
linters:
|
||||
enable-all: true
|
||||
disable:
|
||||
- interfacer # deprecated
|
||||
- scopelint # deprecated
|
||||
- maligned # deprecated
|
||||
- golint # deprecated
|
||||
- structcheck # deprecated
|
||||
- deadcode # deprecated
|
||||
- varcheck # deprecated
|
||||
- nosnakecase # deprecated
|
||||
- ifshort # deprecated
|
||||
- errchkjson
|
||||
- exhaustive
|
||||
- exhaustivestruct
|
||||
- exhaustruct
|
||||
- forcetypeassert
|
||||
- funlen
|
||||
- gochecknoglobals
|
||||
- gochecknoinits
|
||||
- goconst
|
||||
- godox
|
||||
- goerr113
|
||||
- gomnd
|
||||
- ireturn
|
||||
- lll
|
||||
- musttag
|
||||
- nilnil
|
||||
- nlreturn
|
||||
- nonamedreturns
|
||||
- tagliatelle
|
||||
- varnamelen
|
||||
- wrapcheck
|
||||
- wsl
|
||||
- cyclop # TODO: turn this back on later
|
||||
- gocognit # TODO: turn this back on later
|
||||
- forbidigo # TODO: turn this back on later
|
||||
|
||||
issues:
|
||||
exclude-use-default: false
|
||||
exclude:
|
||||
- 'package-comments: should have a package comment'
|
||||
- 'ST1000: at least one file in a package should have a package comment'
|
||||
- 'G204: Subprocess launched with a potential tainted input or cmd arguments'
|
||||
- 'G204: Subprocess launched with variable'
|
||||
- 'G402: TLS MinVersion too low.'
|
||||
- 'const `op` is unused'
|
||||
exclude-rules:
|
||||
- path: cmd/proxy/main.go
|
||||
text: 'G108: Profiling endpoint is automatically exposed on /debug/pprof'
|
||||
- path: pkg/stash/stasher.go
|
||||
linters:
|
||||
- contextcheck
|
||||
- path: pkg/stash/with_azureblob.go # False positive
|
||||
linters:
|
||||
- bodyclose
|
||||
- path: pkg/storage/azureblob/azureblob.go # False positive
|
||||
linters:
|
||||
- bodyclose
|
||||
- path: pkg/storage/compliance/*
|
||||
linters:
|
||||
- thelper
|
||||
- gosec
|
||||
- errcheck
|
||||
- path: pkg/index/compliance/*
|
||||
linters:
|
||||
- thelper
|
||||
- gosec
|
||||
- errcheck
|
||||
+2
-2
@@ -234,10 +234,10 @@ Then open [http://localhost:1313](http://localhost:1313/).
|
||||
|
||||
# Linting
|
||||
|
||||
In our CI/CD pass, we use govet, so feel free to run it locally beforehand:
|
||||
In our CI/CD pass, we use golangci-lint, so feel free to run it locally beforehand:
|
||||
|
||||
```
|
||||
go vet ./...
|
||||
make lint
|
||||
```
|
||||
|
||||
# For People Doing a Release
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
VERSION = "unset"
|
||||
DATE=$(shell date -u +%Y-%m-%d-%H:%M:%S-%Z)
|
||||
|
||||
GOLANGCI_LINT_VERSION=v1.51.2
|
||||
|
||||
ifndef GOLANG_VERSION
|
||||
override GOLANG_VERSION = 1.19
|
||||
endif
|
||||
@@ -43,10 +45,16 @@ docs: ## build the docs docker image
|
||||
setup-dev-env:
|
||||
$(MAKE) dev
|
||||
|
||||
.PHONY: lint
|
||||
lint:
|
||||
@golangci-lint run ./...
|
||||
|
||||
.PHONY: lint-docker
|
||||
lint-docker:
|
||||
@docker run -t --rm -v $(CURDIR):/app -w /app golangci/golangci-lint:$(GOLANGCI_LINT_VERSION) golangci-lint run ./...
|
||||
|
||||
.PHONY: verify
|
||||
verify: ## verify athens codebase
|
||||
./scripts/check_gofmt.sh
|
||||
./scripts/check_govet.sh
|
||||
./scripts/check_deps.sh
|
||||
./scripts/check_conflicts.sh
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ import (
|
||||
"go.opencensus.io/plugin/ochttp"
|
||||
)
|
||||
|
||||
// Service is the name of the service that we want to tag our processes with
|
||||
// Service is the name of the service that we want to tag our processes with.
|
||||
const Service = "proxy"
|
||||
|
||||
// App is where all routes and middleware for the proxy
|
||||
@@ -126,7 +126,7 @@ func App(conf *config.Config) (http.Handler, error) {
|
||||
|
||||
store, err := GetStorage(conf.StorageType, conf.Storage, conf.TimeoutDuration(), client)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("error getting storage configuration (%s)", err)
|
||||
err = fmt.Errorf("error getting storage configuration: %w", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -140,7 +140,7 @@ func App(conf *config.Config) (http.Handler, error) {
|
||||
lggr,
|
||||
conf,
|
||||
); err != nil {
|
||||
err = fmt.Errorf("error adding proxy routes (%s)", err)
|
||||
err = fmt.Errorf("error adding proxy routes: %w", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ package actions
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
@@ -54,7 +55,7 @@ func addProxyRoutes(
|
||||
}
|
||||
supportPath := path.Join("/sumdb", sumdbURL.Host, "/supported")
|
||||
r.HandleFunc(supportPath, func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(200)
|
||||
w.WriteHeader(http.StatusOK)
|
||||
})
|
||||
sumHandler := sumdbProxy(sumdbURL, c.NoSumPatterns)
|
||||
pathPrefix := "/sumdb/" + sumdbURL.Host
|
||||
@@ -63,7 +64,7 @@ func addProxyRoutes(
|
||||
)
|
||||
}
|
||||
|
||||
// Download Protocol
|
||||
// Download Protocol:
|
||||
// the download.Protocol and the stash.Stasher interfaces are composable
|
||||
// in a middleware fashion. Therefore you can separate concerns
|
||||
// by the functionality: a download.Protocol that just takes care
|
||||
@@ -142,13 +143,13 @@ func getSingleFlight(l *log.Logger, c *config.Config, checker storage.Checker) (
|
||||
return stash.WithSingleflight, nil
|
||||
case "etcd":
|
||||
if c.SingleFlight == nil || c.SingleFlight.Etcd == nil {
|
||||
return nil, fmt.Errorf("Etcd config must be present")
|
||||
return nil, errors.New("etcd config must be present")
|
||||
}
|
||||
endpoints := strings.Split(c.SingleFlight.Etcd.Endpoints, ",")
|
||||
return stash.WithEtcd(endpoints, checker)
|
||||
case "redis":
|
||||
if c.SingleFlight == nil || c.SingleFlight.Redis == nil {
|
||||
return nil, fmt.Errorf("Redis config must be present")
|
||||
return nil, errors.New("redis config must be present")
|
||||
}
|
||||
return stash.WithRedisLock(
|
||||
&athensLoggerForRedis{logger: l},
|
||||
@@ -158,7 +159,7 @@ func getSingleFlight(l *log.Logger, c *config.Config, checker storage.Checker) (
|
||||
c.SingleFlight.Redis.LockConfig)
|
||||
case "redis-sentinel":
|
||||
if c.SingleFlight == nil || c.SingleFlight.RedisSentinel == nil {
|
||||
return nil, fmt.Errorf("Redis config must be present")
|
||||
return nil, errors.New("redis config must be present")
|
||||
}
|
||||
return stash.WithRedisSentinelLock(
|
||||
&athensLoggerForRedis{logger: l},
|
||||
|
||||
@@ -19,7 +19,7 @@ func initializeAuthFile(path string) {
|
||||
return
|
||||
}
|
||||
|
||||
fileBts, err := os.ReadFile(path)
|
||||
fileBts, err := os.ReadFile(filepath.Clean(path))
|
||||
if err != nil {
|
||||
log.Fatalf("could not read %s: %v", path, err)
|
||||
}
|
||||
@@ -31,7 +31,7 @@ func initializeAuthFile(path string) {
|
||||
|
||||
fileName := transformAuthFileName(filepath.Base(path))
|
||||
rcp := filepath.Join(hdir, fileName)
|
||||
if err := os.WriteFile(rcp, fileBts, 0600); err != nil {
|
||||
if err := os.WriteFile(rcp, fileBts, 0o600); err != nil {
|
||||
log.Fatalf("could not write to file: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -45,7 +45,7 @@ func netrcFromToken(tok string) {
|
||||
log.Fatalf("netrcFromToken: could not get homedir: %v", err)
|
||||
}
|
||||
rcp := filepath.Join(hdir, getNETRCFilename())
|
||||
if err := os.WriteFile(rcp, []byte(fileContent), 0600); err != nil {
|
||||
if err := os.WriteFile(rcp, []byte(fileContent), 0o600); err != nil {
|
||||
log.Fatalf("netrcFromToken: could not write to file: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,10 +8,8 @@ import (
|
||||
"github.com/gorilla/mux"
|
||||
)
|
||||
|
||||
var (
|
||||
// basicAuthExcludedPaths is a regular expression that matches paths that should not be protected by HTTP basic authentication.
|
||||
basicAuthExcludedPaths = regexp.MustCompile("^/(health|ready)z$")
|
||||
)
|
||||
var basicAuthExcludedPaths = regexp.MustCompile("^/(health|ready)z$")
|
||||
|
||||
func basicAuth(user, pass string) mux.MiddlewareFunc {
|
||||
return func(h http.Handler) http.Handler {
|
||||
@@ -40,9 +38,5 @@ func checkAuth(r *http.Request, user, pass string) bool {
|
||||
}
|
||||
|
||||
isPass := subtle.ConstantTimeCompare([]byte(pass), []byte(givenPass))
|
||||
if isPass != 1 {
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
return isPass == 1
|
||||
}
|
||||
|
||||
@@ -18,7 +18,7 @@ type catalogRes struct {
|
||||
NextPageToken string `json:"next,omitempty"`
|
||||
}
|
||||
|
||||
// catalogHandler implements GET baseURL/catalog
|
||||
// catalogHandler implements GET baseURL/catalog.
|
||||
func catalogHandler(s storage.Backend) http.HandlerFunc {
|
||||
const op errors.Op = "actions.CatalogHandler"
|
||||
cs, isCataloger := s.(storage.Cataloger)
|
||||
@@ -54,7 +54,7 @@ func catalogHandler(s storage.Backend) http.HandlerFunc {
|
||||
}
|
||||
|
||||
// getLimitFromParam converts a URL query parameter into an int
|
||||
// otherwise converts defaultPageSize constant
|
||||
// otherwise converts defaultPageSize constant.
|
||||
func getLimitFromParam(param string) (int, error) {
|
||||
if param == "" {
|
||||
return defaultPageSize, nil
|
||||
|
||||
@@ -5,5 +5,5 @@ import (
|
||||
)
|
||||
|
||||
func proxyHomeHandler(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte(`"Welcome to The Athens Proxy"`))
|
||||
_, _ = w.Write([]byte(`"Welcome to The Athens Proxy"`))
|
||||
}
|
||||
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// indexHandler implements GET baseURL/index
|
||||
// indexHandler implements GET baseURL/index.
|
||||
func indexHandler(index index.Indexer) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
@@ -6,7 +6,7 @@ import (
|
||||
"github.com/gomods/athens/pkg/config"
|
||||
)
|
||||
|
||||
// robotsHandler implements GET baseURL/robots.txt
|
||||
// robotsHandler implements GET baseURL/robots.txt.
|
||||
func robotsHandler(c *config.Config) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
http.ServeFile(w, r, c.RobotsFile)
|
||||
|
||||
@@ -21,7 +21,7 @@ import (
|
||||
"github.com/spf13/afero"
|
||||
)
|
||||
|
||||
// GetStorage returns storage backend based on env configuration
|
||||
// GetStorage returns storage backend based on env configuration.
|
||||
func GetStorage(storageType string, storageConfig *config.Storage, timeout time.Duration, client *http.Client) (storage.Backend, error) {
|
||||
const op errors.Op = "actions.GetStorage"
|
||||
switch storageType {
|
||||
|
||||
@@ -17,12 +17,12 @@ func sumdbProxy(url *url.URL, nosumPatterns []string) http.Handler {
|
||||
req.URL.Host = url.Host
|
||||
}
|
||||
if len(nosumPatterns) > 0 {
|
||||
return noSumWrapper(rp, url.Host, nosumPatterns)
|
||||
return noSumWrapper(rp, nosumPatterns)
|
||||
}
|
||||
return rp
|
||||
}
|
||||
|
||||
func noSumWrapper(h http.Handler, host string, patterns []string) http.Handler {
|
||||
func noSumWrapper(h http.Handler, patterns []string) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if strings.HasPrefix(r.URL.Path, "/lookup/") {
|
||||
for _, p := range patterns {
|
||||
|
||||
@@ -97,7 +97,7 @@ func TestNoSumPatterns(t *testing.T) {
|
||||
for _, tc := range noSumTestCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
w := httptest.NewRecorder()
|
||||
skipHandler := noSumWrapper(http.HandlerFunc(emptyHandler), "sum.golang.org", tc.patterns)
|
||||
skipHandler := noSumWrapper(http.HandlerFunc(emptyHandler), tc.patterns)
|
||||
req := httptest.NewRequest("GET", "/lookup/"+tc.given, nil)
|
||||
skipHandler.ServeHTTP(w, req)
|
||||
if tc.status != w.Code {
|
||||
|
||||
@@ -8,5 +8,5 @@ import (
|
||||
)
|
||||
|
||||
func versionHandler(w http.ResponseWriter, r *http.Request) {
|
||||
json.NewEncoder(w).Encode(build.Data())
|
||||
_ = json.NewEncoder(w).Encode(build.Data())
|
||||
}
|
||||
|
||||
+6
-5
@@ -2,16 +2,16 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
_ "net/http/pprof"
|
||||
"os"
|
||||
"os/signal"
|
||||
"time"
|
||||
|
||||
_ "net/http/pprof"
|
||||
|
||||
"github.com/gomods/athens/cmd/proxy/actions"
|
||||
"github.com/gomods/athens/internal/shutdown"
|
||||
"github.com/gomods/athens/pkg/build"
|
||||
@@ -47,6 +47,7 @@ func main() {
|
||||
srv := &http.Server{
|
||||
Addr: conf.Port,
|
||||
Handler: handler,
|
||||
ReadHeaderTimeout: 2 * time.Second,
|
||||
}
|
||||
idleConnsClosed := make(chan struct{})
|
||||
|
||||
@@ -54,7 +55,7 @@ func main() {
|
||||
sigint := make(chan os.Signal, 1)
|
||||
signal.Notify(sigint, shutdown.GetSignals()...)
|
||||
s := <-sigint
|
||||
log.Printf("Recived signal (%s): Shutting down server", s)
|
||||
log.Printf("Received signal (%s): Shutting down server", s)
|
||||
|
||||
// We received an interrupt signal, shut down.
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*time.Duration(conf.ShutdownTimeout))
|
||||
@@ -70,7 +71,7 @@ func main() {
|
||||
// pprof to be exposed on a different port than the application for security matters, not to expose profiling data and avoid DoS attacks (profiling slows down the service)
|
||||
// https://www.farsightsecurity.com/txt-record/2016/10/28/cmikk-go-remote-profiling/
|
||||
log.Printf("Starting `pprof` at port %v", conf.PprofPort)
|
||||
log.Fatal(http.ListenAndServe(conf.PprofPort, nil))
|
||||
log.Fatal(http.ListenAndServe(conf.PprofPort, nil)) //nolint:gosec // This should not be exposed to the world.
|
||||
}()
|
||||
}
|
||||
|
||||
@@ -81,7 +82,7 @@ func main() {
|
||||
err = srv.ListenAndServe()
|
||||
}
|
||||
|
||||
if err != http.ErrServerClosed {
|
||||
if !errors.Is(err, http.ErrServerClosed) {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
|
||||
@@ -79,8 +79,6 @@ if ($docs.IsPresent) {
|
||||
}
|
||||
|
||||
if ($verify.IsPresent) {
|
||||
execScript "check_gofmt.ps1"
|
||||
execScript "check_govet.ps1"
|
||||
execScript "check_deps.ps1"
|
||||
}
|
||||
|
||||
|
||||
+2
-2
@@ -9,7 +9,7 @@ import (
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// Details represents known data for a given build
|
||||
// Details represents known data for a given build.
|
||||
type Details struct {
|
||||
Version string `json:"version,omitempty"`
|
||||
Date string `json:"date,omitempty"`
|
||||
@@ -29,7 +29,7 @@ func String() string {
|
||||
return fmt.Sprintf("Build Details:\n\tVersion:\t%s\n\tDate:\t\t%s", version, buildDate)
|
||||
}
|
||||
|
||||
// Data returns build details as a struct
|
||||
// Data returns build details as a struct.
|
||||
func Data() Details {
|
||||
return Details{
|
||||
Version: version,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
package config
|
||||
|
||||
// AzureBlobConfig specifies the properties required to use Azure as the storage backend
|
||||
// AzureBlobConfig specifies the properties required to use Azure as the storage backend.
|
||||
type AzureBlobConfig struct {
|
||||
AccountName string `validate:"required" envconfig:"ATHENS_AZURE_ACCOUNT_NAME"`
|
||||
AccountKey string `validate:"required" envconfig:"ATHENS_AZURE_ACCOUNT_KEY"`
|
||||
|
||||
+18
-20
@@ -18,7 +18,7 @@ import (
|
||||
|
||||
const defaultConfigFile = "athens.toml"
|
||||
|
||||
// Config provides configuration values for all components
|
||||
// Config provides configuration values for all components.
|
||||
type Config struct {
|
||||
TimeoutConf
|
||||
GoEnv string `validate:"required" envconfig:"GO_ENV"`
|
||||
@@ -63,12 +63,12 @@ type Config struct {
|
||||
}
|
||||
|
||||
// EnvList is a list of key-value environment
|
||||
// variables that are passed to the Go command
|
||||
// variables that are passed to the Go command.
|
||||
type EnvList []string
|
||||
|
||||
// HasKey returns whether a key-value entry
|
||||
// is present by only checking the left of
|
||||
// key=value
|
||||
// key=value.
|
||||
func (el EnvList) HasKey(key string) bool {
|
||||
for _, env := range el {
|
||||
if strings.HasPrefix(env, key+"=") {
|
||||
@@ -79,7 +79,7 @@ func (el EnvList) HasKey(key string) bool {
|
||||
}
|
||||
|
||||
// Add adds a key=value entry to the environment
|
||||
// list
|
||||
// list.
|
||||
func (el *EnvList) Add(key, value string) {
|
||||
*el = append(*el, key+"="+value)
|
||||
}
|
||||
@@ -110,7 +110,7 @@ func (el *EnvList) Decode(value string) error {
|
||||
}
|
||||
|
||||
// Validate validates that all strings inside the
|
||||
// list are of the key=value format
|
||||
// list are of the key=value format.
|
||||
func (el EnvList) Validate() error {
|
||||
const op errors.Op = "EnvList.Validate"
|
||||
for _, env := range el {
|
||||
@@ -123,7 +123,7 @@ func (el EnvList) Validate() error {
|
||||
}
|
||||
|
||||
// Load loads the config from a file.
|
||||
// If file is not present returns default config
|
||||
// If file is not present returns default config.
|
||||
func Load(configFile string) (*Config, error) {
|
||||
// User explicitly specified a config file
|
||||
if configFile != "" {
|
||||
@@ -206,7 +206,7 @@ func defaultConfig() *Config {
|
||||
}
|
||||
|
||||
// BasicAuth returns BasicAuthUser and BasicAuthPassword
|
||||
// and ok if neither of them are empty
|
||||
// and ok if neither of them are empty.
|
||||
func (c *Config) BasicAuth() (user, pass string, ok bool) {
|
||||
user = c.BasicAuthUser
|
||||
pass = c.BasicAuthPass
|
||||
@@ -215,7 +215,7 @@ func (c *Config) BasicAuth() (user, pass string, ok bool) {
|
||||
}
|
||||
|
||||
// TLSCertFiles returns certificate and key files and an error if
|
||||
// both files doesn't exist and have approperiate file permissions
|
||||
// both files doesn't exist and have approperiate file permissions.
|
||||
func (c *Config) TLSCertFiles() (cert, key string, err error) {
|
||||
if c.TLSCertFile == "" && c.TLSKeyFile == "" {
|
||||
return "", "", nil
|
||||
@@ -223,29 +223,28 @@ func (c *Config) TLSCertFiles() (cert, key string, err error) {
|
||||
|
||||
certFile, err := os.Stat(c.TLSCertFile)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("Could not access TLSCertFile: %v", err)
|
||||
return "", "", fmt.Errorf("could not access TLSCertFile: %w", err)
|
||||
}
|
||||
|
||||
keyFile, err := os.Stat(c.TLSKeyFile)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("Could not access TLSKeyFile: %v", err)
|
||||
return "", "", fmt.Errorf("could not access TLSKeyFile: %w", err)
|
||||
}
|
||||
|
||||
if keyFile.Mode()&077 != 0 && runtime.GOOS != "windows" {
|
||||
if keyFile.Mode()&0o077 != 0 && runtime.GOOS != "windows" {
|
||||
return "", "", fmt.Errorf("TLSKeyFile should not be accessible by others")
|
||||
}
|
||||
|
||||
return certFile.Name(), keyFile.Name(), nil
|
||||
}
|
||||
|
||||
// FilterOff returns true if the FilterFile is empty
|
||||
// FilterOff returns true if the FilterFile is empty.
|
||||
func (c *Config) FilterOff() bool {
|
||||
return c.FilterFile == ""
|
||||
}
|
||||
|
||||
// ParseConfigFile parses the given file into an athens config struct
|
||||
// ParseConfigFile parses the given file into an athens config struct.
|
||||
func ParseConfigFile(configFile string) (*Config, error) {
|
||||
|
||||
var config Config
|
||||
// attempt to read the given config file
|
||||
if _, err := toml.DecodeFile(configFile, &config); err != nil {
|
||||
@@ -271,7 +270,7 @@ func ParseConfigFile(configFile string) (*Config, error) {
|
||||
return &config, nil
|
||||
}
|
||||
|
||||
// envOverride uses Environment variables to override unspecified properties
|
||||
// envOverride uses Environment variables to override unspecified properties.
|
||||
func envOverride(config *Config) error {
|
||||
const defaultPort = ":3000"
|
||||
err := envconfig.Process("athens", config)
|
||||
@@ -355,16 +354,16 @@ func validateIndex(validate *validator.Validate, indexType string, config *Index
|
||||
func GetConf(path string) (*Config, error) {
|
||||
absPath, err := filepath.Abs(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Unable to construct absolute path to test config file")
|
||||
return nil, fmt.Errorf("unable to construct absolute path to test config file")
|
||||
}
|
||||
conf, err := ParseConfigFile(absPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Unable to parse test config file: %s", err.Error())
|
||||
return nil, fmt.Errorf("unable to parse test config file: %w", err)
|
||||
}
|
||||
return conf, nil
|
||||
}
|
||||
|
||||
// checkFilePerms given a list of files
|
||||
// checkFilePerms given a list of files.
|
||||
func checkFilePerms(files ...string) error {
|
||||
const op = "config.checkFilePerms"
|
||||
|
||||
@@ -384,10 +383,9 @@ func checkFilePerms(files ...string) error {
|
||||
// Assume unix based system (MacOS and Linux)
|
||||
// the bit mask is calculated using the umask command which tells which permissions
|
||||
// should not be allowed for a particular user, group or world
|
||||
if fInfo.Mode()&0077 != 0 && runtime.GOOS != "windows" {
|
||||
if fInfo.Mode()&0o077 != 0 && runtime.GOOS != "windows" {
|
||||
return errors.E(op, f+" should have at most rwx,-, - (bit mask 077) as permission")
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
package config
|
||||
|
||||
// DiskConfig specifies the properties required to use Disk as the storage backend
|
||||
// DiskConfig specifies the properties required to use Disk as the storage backend.
|
||||
type DiskConfig struct {
|
||||
RootPath string `validate:"required" envconfig:"ATHENS_DISK_STORAGE_ROOT"`
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
package config
|
||||
|
||||
// External specifies configuration for an external http storage
|
||||
// External specifies configuration for an external http storage.
|
||||
type External struct {
|
||||
URL string `validate:"required" envconfig:"ATHENS_EXTERNAL_STORAGE_URL"`
|
||||
}
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
package config
|
||||
|
||||
// GCPConfig specifies the properties required to use GCP as the storage backend
|
||||
// GCPConfig specifies the properties required to use GCP as the storage backend.
|
||||
type GCPConfig struct {
|
||||
ProjectID string `envconfig:"GOOGLE_CLOUD_PROJECT"`
|
||||
Bucket string `validate:"required" envconfig:"ATHENS_STORAGE_GCP_BUCKET"`
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
package config
|
||||
|
||||
// Index is the config for various index storage backends
|
||||
// Index is the config for various index storage backends.
|
||||
type Index struct {
|
||||
MySQL *MySQL
|
||||
Postgres *Postgres
|
||||
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
package config
|
||||
|
||||
// MinioConfig specifies the properties required to use Minio or DigitalOcean Spaces
|
||||
// as the storage backend
|
||||
// as the storage backend.
|
||||
type MinioConfig struct {
|
||||
Endpoint string `validate:"required" envconfig:"ATHENS_MINIO_ENDPOINT"`
|
||||
Key string `validate:"required" envconfig:"ATHENS_MINIO_ACCESS_KEY_ID"`
|
||||
|
||||
@@ -7,20 +7,20 @@ import (
|
||||
)
|
||||
|
||||
// PackageVersionedName return package full name used in storage.
|
||||
// E.g athens/@v/v1.0.mod
|
||||
// E.g athens/@v/v1.0.mod.
|
||||
func PackageVersionedName(module, version, ext string) string {
|
||||
return fmt.Sprintf("%s/@v/%s.%s", module, version, ext)
|
||||
}
|
||||
|
||||
// FmtModVer is a helper function that can take
|
||||
// pkg/a/b and v2.3.1 and returns pkg/a/b@v2.3.1
|
||||
// pkg/a/b and v2.3.1 and returns pkg/a/b@v2.3.1.
|
||||
func FmtModVer(mod, ver string) string {
|
||||
return fmt.Sprintf("%s@%s", mod, ver)
|
||||
}
|
||||
|
||||
// ModuleVersionFromPath returns module and version from a
|
||||
// storage path
|
||||
// E.g athens/@v/v1.0.info -> athens and v.1.0
|
||||
// storage path.
|
||||
// E.g athens/@v/v1.0.info -> athens and v.1.0.
|
||||
func ModuleVersionFromPath(path string) (string, string) {
|
||||
segments := strings.Split(path, "/@v/")
|
||||
if len(segments) != 2 {
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
package config
|
||||
|
||||
// MongoConfig specifies the properties required to use MongoDB as the storage backend
|
||||
// MongoConfig specifies the properties required to use MongoDB as the storage backend.
|
||||
type MongoConfig struct {
|
||||
URL string `validate:"required" envconfig:"ATHENS_MONGO_STORAGE_URL"`
|
||||
DefaultDBName string `envconfig:"ATHENS_MONGO_DEFAULT_DATABASE" default:"athens"`
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
package config
|
||||
|
||||
// MySQL config
|
||||
// MySQL config.
|
||||
type MySQL struct {
|
||||
Protocol string `validate:"required" envconfig:"ATHENS_INDEX_MYSQL_PROTOCOL"`
|
||||
Host string `validate:"required" envconfig:"ATHENS_INDEX_MYSQL_HOST"`
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
package config
|
||||
|
||||
// Postgres config
|
||||
// Postgres config.
|
||||
type Postgres struct {
|
||||
Host string `validate:"required" envconfig:"ATHENS_INDEX_POSTGRES_HOST"`
|
||||
Port int `validate:"required" envconfig:"ATHENS_INDEX_POSTGRES_PORT"`
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
package config
|
||||
|
||||
// S3Config specifies the properties required to use S3 as the storage backend
|
||||
// S3Config specifies the properties required to use S3 as the storage backend.
|
||||
type S3Config struct {
|
||||
Region string `validate:"required" envconfig:"AWS_REGION"`
|
||||
Key string `envconfig:"AWS_ACCESS_KEY_ID"`
|
||||
|
||||
@@ -25,7 +25,7 @@ type Redis struct {
|
||||
}
|
||||
|
||||
// RedisSentinel is the configuration for using redis with sentinel
|
||||
// for SingleFlight
|
||||
// for SingleFlight.
|
||||
type RedisSentinel struct {
|
||||
Endpoints []string `envconfig:"ATHENS_REDIS_SENTINEL_ENDPOINTS"`
|
||||
MasterName string `envconfig:"ATHENS_REDIS_SENTINEL_MASTER_NAME"`
|
||||
@@ -33,12 +33,14 @@ type RedisSentinel struct {
|
||||
LockConfig *RedisLockConfig
|
||||
}
|
||||
|
||||
// RedisLockConfig is the configuration for redis locking.
|
||||
type RedisLockConfig struct {
|
||||
Timeout int `envconfig:"ATHENS_REDIS_LOCK_TIMEOUT"`
|
||||
TTL int `envconfig:"ATHENS_REDIS_LOCK_TTL"`
|
||||
MaxRetries int `envconfig:"ATHENS_REDIS_LOCK_MAX_RETRIES"`
|
||||
}
|
||||
|
||||
// DefaultRedisLockConfig returns the default redis locking configuration.
|
||||
func DefaultRedisLockConfig() *RedisLockConfig {
|
||||
return &RedisLockConfig{
|
||||
TTL: 900,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
package config
|
||||
|
||||
// Storage provides configs for various storage backends
|
||||
// Storage provides configs for various storage backends.
|
||||
type Storage struct {
|
||||
Disk *DiskConfig
|
||||
GCP *GCPConfig
|
||||
|
||||
@@ -2,17 +2,17 @@ package config
|
||||
|
||||
import "time"
|
||||
|
||||
// TimeoutConf is a common struct for anything with a timeout
|
||||
// TimeoutConf is a common struct for anything with a timeout.
|
||||
type TimeoutConf struct {
|
||||
Timeout int `validate:"required"`
|
||||
}
|
||||
|
||||
// TimeoutDuration returns the timeout as time.duration
|
||||
// TimeoutDuration returns the timeout as time.duration.
|
||||
func (t *TimeoutConf) TimeoutDuration() time.Duration {
|
||||
return GetTimeoutDuration(t.Timeout)
|
||||
}
|
||||
|
||||
// GetTimeoutDuration returns the timeout as time.duration
|
||||
// GetTimeoutDuration returns the timeout as time.duration.
|
||||
func GetTimeoutDuration(timeout int) time.Duration {
|
||||
return time.Second * time.Duration(timeout)
|
||||
}
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"github.com/gomods/athens/pkg/paths"
|
||||
)
|
||||
|
||||
func getModuleParams(r *http.Request, op errors.Op) (mod string, ver string, err error) {
|
||||
func getModuleParams(r *http.Request, op errors.Op) (mod, ver string, err error) {
|
||||
params, err := paths.GetAllParams(r)
|
||||
if err != nil {
|
||||
return "", "", errors.E(op, err, errors.KindBadRequest)
|
||||
|
||||
@@ -15,8 +15,7 @@ import (
|
||||
// a ready-to-go http handler that serves up cmd/go's download protocol.
|
||||
type ProtocolHandler func(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler
|
||||
|
||||
// HandlerOpts are the generic options
|
||||
// for a ProtocolHandler
|
||||
// HandlerOpts are the generic options for a ProtocolHandler.
|
||||
type HandlerOpts struct {
|
||||
Protocol Protocol
|
||||
Logger *log.Logger
|
||||
@@ -26,7 +25,7 @@ type HandlerOpts struct {
|
||||
// LogEntryHandler pulls a log entry from the request context. Thanks to the
|
||||
// LogEntryMiddleware, we should have a log entry stored in the context for each
|
||||
// request with request-specific fields. This will grab the entry and pass it to
|
||||
// the protocol handlers
|
||||
// the protocol handlers.
|
||||
func LogEntryHandler(ph ProtocolHandler, opts *HandlerOpts) http.Handler {
|
||||
f := func(w http.ResponseWriter, r *http.Request) {
|
||||
ent := log.EntryFromContext(r.Context())
|
||||
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
// PathLatest URL.
|
||||
const PathLatest = "/{module:.+}/@latest"
|
||||
|
||||
// LatestHandler implements GET baseURL/module/@latest
|
||||
// LatestHandler implements GET baseURL/module/@latest.
|
||||
func LatestHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler {
|
||||
const op errors.Op = "download.LatestHandler"
|
||||
f := func(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
@@ -14,7 +14,7 @@ import (
|
||||
// PathList URL.
|
||||
const PathList = "/{module:.+}/@v/list"
|
||||
|
||||
// ListHandler implements GET baseURL/module/@v/list
|
||||
// ListHandler implements GET baseURL/module/@v/list.
|
||||
func ListHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler {
|
||||
const op errors.Op = "download.ListHandler"
|
||||
f := func(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/gomods/athens/pkg/errors"
|
||||
@@ -16,7 +17,7 @@ import (
|
||||
// when a module is not found in storage.
|
||||
type Mode string
|
||||
|
||||
// DownloadMode constants. For more information see config.dev.toml
|
||||
// DownloadMode constants. For more information see config.dev.toml.
|
||||
const (
|
||||
Sync Mode = "sync"
|
||||
Async Mode = "async"
|
||||
@@ -59,7 +60,7 @@ func NewFile(m Mode, downloadURL string) (*DownloadFile, error) {
|
||||
|
||||
if strings.HasPrefix(string(m), "file:") {
|
||||
filePath := string(m[5:])
|
||||
bts, err := os.ReadFile(filePath)
|
||||
bts, err := os.ReadFile(filepath.Clean(filePath))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -48,7 +48,7 @@ type Opts struct {
|
||||
NetworkMode string
|
||||
}
|
||||
|
||||
// NetworkMode constants
|
||||
// NetworkMode constants.
|
||||
const (
|
||||
Strict = "strict"
|
||||
Offline = "offline"
|
||||
@@ -263,12 +263,12 @@ func (p *protocol) processDownload(ctx context.Context, mod, ver string, f func(
|
||||
}
|
||||
return f(newVer)
|
||||
case mode.Async:
|
||||
go p.stasher.Stash(ctx, mod, ver)
|
||||
go func() { _, _ = p.stasher.Stash(ctx, mod, ver) }()
|
||||
return errors.E(op, "async: module not found", errors.KindNotFound)
|
||||
case mode.Redirect:
|
||||
return errors.E(op, "redirect", errors.KindRedirect)
|
||||
case mode.AsyncRedirect:
|
||||
go p.stasher.Stash(ctx, mod, ver)
|
||||
go func() { _, _ = p.stasher.Stash(ctx, mod, ver) }()
|
||||
return errors.E(op, "async_redirect: module not found", errors.KindRedirect)
|
||||
case mode.None:
|
||||
return errors.E(op, "none", errors.KindNotFound)
|
||||
@@ -276,7 +276,7 @@ func (p *protocol) processDownload(ctx context.Context, mod, ver string, f func(
|
||||
return nil
|
||||
}
|
||||
|
||||
// union concatenates two version lists and removes duplicates
|
||||
// union concatenates two version lists and removes duplicates.
|
||||
func union(list1, list2 []string) []string {
|
||||
if list1 == nil {
|
||||
list1 = []string{}
|
||||
@@ -284,10 +284,10 @@ func union(list1, list2 []string) []string {
|
||||
if list2 == nil {
|
||||
list2 = []string{}
|
||||
}
|
||||
list := append(list1, list2...)
|
||||
list1 = append(list1, list2...)
|
||||
unique := []string{}
|
||||
m := make(map[string]struct{})
|
||||
for _, v := range list {
|
||||
for _, v := range list1 {
|
||||
if _, ok := m[v]; !ok {
|
||||
unique = append(unique, v)
|
||||
m[v] = struct{}{}
|
||||
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
// PathVersionInfo URL.
|
||||
const PathVersionInfo = "/{module:.+}/@v/{version}.info"
|
||||
|
||||
// InfoHandler implements GET baseURL/module/@v/version.info
|
||||
// InfoHandler implements GET baseURL/module/@v/version.info.
|
||||
func InfoHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler {
|
||||
const op errors.Op = "download.InfoHandler"
|
||||
f := func(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -38,7 +38,7 @@ func InfoHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handle
|
||||
w.WriteHeader(errors.Kind(err))
|
||||
}
|
||||
|
||||
w.Write(info)
|
||||
_, _ = w.Write(info)
|
||||
}
|
||||
return http.HandlerFunc(f)
|
||||
}
|
||||
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
// PathVersionModule URL.
|
||||
const PathVersionModule = "/{module:.+}/@v/{version}.mod"
|
||||
|
||||
// ModuleHandler implements GET baseURL/module/@v/version.mod
|
||||
// ModuleHandler implements GET baseURL/module/@v/version.mod.
|
||||
func ModuleHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler {
|
||||
const op errors.Op = "download.VersionModuleHandler"
|
||||
f := func(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -42,7 +42,7 @@ func ModuleHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Hand
|
||||
return
|
||||
}
|
||||
|
||||
w.Write(modBts)
|
||||
_, _ = w.Write(modBts)
|
||||
}
|
||||
return http.HandlerFunc(f)
|
||||
}
|
||||
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
// PathVersionZip URL.
|
||||
const PathVersionZip = "/{module:.+}/@v/{version}.zip"
|
||||
|
||||
// ZipHandler implements GET baseURL/module/@v/version.zip
|
||||
// ZipHandler implements GET baseURL/module/@v/version.zip.
|
||||
func ZipHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler {
|
||||
const op errors.Op = "download.ZipHandler"
|
||||
f := func(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -41,7 +41,7 @@ func ZipHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler
|
||||
w.WriteHeader(errors.Kind(err))
|
||||
return
|
||||
}
|
||||
defer zip.Close()
|
||||
defer func() { _ = zip.Close() }()
|
||||
|
||||
w.Header().Set("Content-Type", "application/zip")
|
||||
size := zip.Size()
|
||||
|
||||
+17
-6
@@ -9,7 +9,7 @@ import (
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// Kind enums
|
||||
// Kind enums.
|
||||
const (
|
||||
KindNotFound = http.StatusNotFound
|
||||
KindBadRequest = http.StatusBadRequest
|
||||
@@ -55,6 +55,16 @@ func Is(err error, kind int) bool {
|
||||
return Kind(err) == kind
|
||||
}
|
||||
|
||||
// IsErr is a convenience wrapper around the std library errors.Is.
|
||||
func IsErr(err, target error) bool {
|
||||
return errors.Is(err, target)
|
||||
}
|
||||
|
||||
// AsErr is a convenience wrapper around the std library errors.As.
|
||||
func AsErr(err error, target any) bool {
|
||||
return errors.As(err, target)
|
||||
}
|
||||
|
||||
// Op describes any independent function or
|
||||
// method in Athens. A series of operations
|
||||
// forms a more readable stack trace.
|
||||
@@ -69,7 +79,7 @@ func (o Op) String() string {
|
||||
// a module from a regular error string or version.
|
||||
type M string
|
||||
|
||||
// V represents a module version in an error
|
||||
// V represents a module version in an error.
|
||||
type V string
|
||||
|
||||
// E is a helper function to construct an Error type
|
||||
@@ -114,8 +124,8 @@ func E(op Op, args ...interface{}) error {
|
||||
// if none exists, then the level is Error because
|
||||
// it is an unexpected.
|
||||
func Severity(err error) logrus.Level {
|
||||
e, ok := err.(Error)
|
||||
if !ok {
|
||||
var e Error
|
||||
if !errors.As(err, &e) {
|
||||
return logrus.ErrorLevel
|
||||
}
|
||||
|
||||
@@ -145,8 +155,8 @@ func Expect(err error, kinds ...int) logrus.Level {
|
||||
// Kind recursively searches for the
|
||||
// first error kind it finds.
|
||||
func Kind(err error) int {
|
||||
e, ok := err.(Error)
|
||||
if !ok {
|
||||
var e Error
|
||||
if !errors.As(err, &e) {
|
||||
return KindUnexpected
|
||||
}
|
||||
|
||||
@@ -173,6 +183,7 @@ func KindText(err error) string {
|
||||
func Ops(err Error) []Op {
|
||||
ops := []Op{err.Op}
|
||||
for {
|
||||
//nolint:errorlint // We iterate the errors anyway.
|
||||
embeddedErr, ok := err.Err.(Error)
|
||||
if !ok {
|
||||
break
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
package errors
|
||||
|
||||
// IsNotFoundErr helper function for KindNotFound
|
||||
// IsNotFoundErr helper function for KindNotFound.
|
||||
func IsNotFoundErr(err error) bool {
|
||||
return Kind(err) == KindNotFound
|
||||
}
|
||||
|
||||
@@ -21,7 +21,7 @@ func RunTests(t *testing.T, indexer index.Indexer, clearIndex func() error) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var tests = []struct {
|
||||
tests := []struct {
|
||||
name string
|
||||
desc string
|
||||
limit int
|
||||
|
||||
@@ -13,7 +13,7 @@ type Line struct {
|
||||
}
|
||||
|
||||
// Indexer is an interface that can process new module@versions
|
||||
// and also retrieve 'limit' module@versions that were indexed after 'since'
|
||||
// and also retrieve 'limit' module@versions that were indexed after 'since'.
|
||||
type Indexer interface {
|
||||
// Index stores the module@version into the index backend.
|
||||
// Implementer must create the Timestamp at the time and set it
|
||||
|
||||
@@ -10,7 +10,7 @@ import (
|
||||
"github.com/gomods/athens/pkg/index"
|
||||
)
|
||||
|
||||
// New returns a new in-memory indexer
|
||||
// New returns a new in-memory indexer.
|
||||
func New() index.Indexer {
|
||||
return &indexer{}
|
||||
}
|
||||
|
||||
@@ -83,8 +83,8 @@ func (i *indexer) Lines(ctx context.Context, since time.Time, limit int) ([]*ind
|
||||
if err != nil {
|
||||
return nil, errors.E(op, err)
|
||||
}
|
||||
defer rows.Close()
|
||||
lines := []*index.Line{}
|
||||
defer func() { _ = rows.Close() }()
|
||||
var lines []*index.Line
|
||||
for rows.Next() {
|
||||
var line index.Line
|
||||
err = rows.Scan(&line.Path, &line.Version, &line.Timestamp)
|
||||
@@ -108,13 +108,14 @@ func getMySQLSource(cfg *config.MySQL) string {
|
||||
}
|
||||
|
||||
func getKind(err error) int {
|
||||
mysqlErr, ok := err.(*mysql.MySQLError)
|
||||
if !ok {
|
||||
mysqlErr := &mysql.MySQLError{}
|
||||
if !errors.AsErr(err, &mysqlErr) {
|
||||
return errors.KindUnexpected
|
||||
}
|
||||
switch mysqlErr.Number {
|
||||
case 1062:
|
||||
return errors.KindAlreadyExists
|
||||
}
|
||||
default:
|
||||
return errors.KindUnexpected
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"github.com/gomods/athens/pkg/index"
|
||||
)
|
||||
|
||||
// New returns a no-op Indexer
|
||||
// New returns a no-op Indexer.
|
||||
func New() index.Indexer {
|
||||
return indexer{}
|
||||
}
|
||||
@@ -17,6 +17,7 @@ type indexer struct{}
|
||||
func (indexer) Index(ctx context.Context, mod, ver string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (indexer) Lines(ctx context.Context, since time.Time, limit int) ([]*index.Line, error) {
|
||||
return []*index.Line{}, nil
|
||||
}
|
||||
|
||||
@@ -7,12 +7,10 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
// register the driver with database/sql
|
||||
"github.com/lib/pq"
|
||||
|
||||
"github.com/gomods/athens/pkg/config"
|
||||
"github.com/gomods/athens/pkg/errors"
|
||||
"github.com/gomods/athens/pkg/index"
|
||||
"github.com/lib/pq"
|
||||
)
|
||||
|
||||
// New returns a new Indexer with a PostgreSQL implementation.
|
||||
@@ -42,7 +40,7 @@ var schema = [...]string{
|
||||
id SERIAL PRIMARY KEY,
|
||||
path VARCHAR(255) NOT NULL,
|
||||
version VARCHAR(255) NOT NULL,
|
||||
timestamp timestamp NOT NULL
|
||||
timestamp TIMESTAMP NOT NULL
|
||||
)
|
||||
`,
|
||||
`
|
||||
@@ -82,8 +80,8 @@ func (i *indexer) Lines(ctx context.Context, since time.Time, limit int) ([]*ind
|
||||
if err != nil {
|
||||
return nil, errors.E(op, err)
|
||||
}
|
||||
defer rows.Close()
|
||||
lines := []*index.Line{}
|
||||
defer func() { _ = rows.Close() }()
|
||||
var lines []*index.Line
|
||||
for rows.Next() {
|
||||
var line index.Line
|
||||
err = rows.Scan(&line.Path, &line.Version, &line.Timestamp)
|
||||
@@ -96,7 +94,7 @@ func (i *indexer) Lines(ctx context.Context, since time.Time, limit int) ([]*ind
|
||||
}
|
||||
|
||||
func getPostgresSource(cfg *config.Postgres) string {
|
||||
args := []string{}
|
||||
args := make([]string, 0, 5+len(cfg.Params))
|
||||
args = append(args, "host="+cfg.Host)
|
||||
args = append(args, "port=", strconv.Itoa(cfg.Port))
|
||||
args = append(args, "user=", cfg.User)
|
||||
@@ -109,13 +107,14 @@ func getPostgresSource(cfg *config.Postgres) string {
|
||||
}
|
||||
|
||||
func getKind(err error) int {
|
||||
pqerr, ok := err.(*pq.Error)
|
||||
if !ok {
|
||||
pqerr := &pq.Error{}
|
||||
if !errors.AsErr(err, &pqerr) {
|
||||
return errors.KindUnexpected
|
||||
}
|
||||
switch pqerr.Code {
|
||||
case "23505":
|
||||
return errors.KindAlreadyExists
|
||||
}
|
||||
default:
|
||||
return errors.KindUnexpected
|
||||
}
|
||||
}
|
||||
|
||||
+2
-2
@@ -35,8 +35,8 @@ func (e *entry) WithFields(fields map[string]interface{}) Entry {
|
||||
}
|
||||
|
||||
func (e *entry) SystemErr(err error) {
|
||||
athensErr, ok := err.(errors.Error)
|
||||
if !ok {
|
||||
var athensErr errors.Error
|
||||
if !errors.IsErr(err, athensErr) {
|
||||
e.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
+1
-1
@@ -41,7 +41,7 @@ func (l *Logger) WithFields(fields map[string]interface{}) Entry {
|
||||
return &entry{e}
|
||||
}
|
||||
|
||||
// NoOpLogger provides a Logger that does nothing
|
||||
// NoOpLogger provides a Logger that does nothing.
|
||||
func NoOpLogger() *Logger {
|
||||
return &Logger{
|
||||
Logger: &logrus.Logger{},
|
||||
|
||||
@@ -8,14 +8,14 @@ type ctxKey string
|
||||
|
||||
const logEntryKey ctxKey = "log-entry-context-key"
|
||||
|
||||
// SetEntryInContext stores an Entry in the request context
|
||||
// SetEntryInContext stores an Entry in the request context.
|
||||
func SetEntryInContext(ctx context.Context, e Entry) context.Context {
|
||||
return context.WithValue(ctx, logEntryKey, e)
|
||||
}
|
||||
|
||||
// EntryFromContext returns an Entry that has been stored in the request context.
|
||||
// If there is no value for the key or the type assertion fails, it returns a new
|
||||
// entry from the provided logger
|
||||
// entry from the provided logger.
|
||||
func EntryFromContext(ctx context.Context) Entry {
|
||||
e, ok := ctx.Value(logEntryKey).(Entry)
|
||||
if !ok || e == nil {
|
||||
|
||||
@@ -4,7 +4,7 @@ import (
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// ContentType writes writes an application/json
|
||||
// ContentType writes an application/json
|
||||
// Content-Type header.
|
||||
func ContentType(h http.Handler) http.Handler {
|
||||
f := func(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
@@ -10,7 +10,7 @@ import (
|
||||
)
|
||||
|
||||
// LogEntryMiddleware builds a log.Entry, setting the request fields
|
||||
// and storing it in the context to be used throughout the stack
|
||||
// and storing it in the context to be used throughout the stack.
|
||||
func LogEntryMiddleware(lggr *log.Logger) mux.MiddlewareFunc {
|
||||
return func(h http.Handler) http.Handler {
|
||||
f := func(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
@@ -9,7 +9,7 @@ import (
|
||||
|
||||
// WithRequestID ensures a request id is in the
|
||||
// request context by either the incoming header
|
||||
// or creating a new one
|
||||
// or creating a new one.
|
||||
func WithRequestID(h http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
requestID := r.Header.Get(requestid.HeaderKey)
|
||||
|
||||
@@ -14,7 +14,7 @@ import (
|
||||
)
|
||||
|
||||
// NewValidationMiddleware builds a middleware function that performs validation checks by calling
|
||||
// an external webhook
|
||||
// an external webhook.
|
||||
func NewValidationMiddleware(client *http.Client, validatorHook string) mux.MiddlewareFunc {
|
||||
const op errors.Op = "actions.NewValidationMiddleware"
|
||||
return func(h http.Handler) http.Handler {
|
||||
@@ -50,7 +50,7 @@ func NewValidationMiddleware(client *http.Client, validatorHook string) mux.Midd
|
||||
}
|
||||
}
|
||||
|
||||
func maybeLogValidationReason(context context.Context, message string, mod string, version string) {
|
||||
func maybeLogValidationReason(context context.Context, message, mod, version string) {
|
||||
if len(message) > 0 {
|
||||
entry := log.EntryFromContext(context)
|
||||
entry.Warnf("error validating %s@%s %s", mod, version, message)
|
||||
@@ -85,6 +85,10 @@ func validate(ctx context.Context, client *http.Client, hook, mod, ver string) (
|
||||
if err != nil {
|
||||
return validationResponse{Valid: false}, errors.E(op, err)
|
||||
}
|
||||
defer func() {
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
}()
|
||||
|
||||
switch resp.StatusCode {
|
||||
case http.StatusOK:
|
||||
@@ -97,8 +101,6 @@ func validate(ctx context.Context, client *http.Client, hook, mod, ver string) (
|
||||
}
|
||||
|
||||
func validationResponseFromRequest(resp *http.Response) validationResponse {
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return validationResponse{Valid: resp.StatusCode == http.StatusOK, Message: body}
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@ import (
|
||||
"github.com/gomods/athens/pkg/storage"
|
||||
)
|
||||
|
||||
// Fetcher fetches module from an upstream source
|
||||
// Fetcher fetches module from an upstream source.
|
||||
type Fetcher interface {
|
||||
// Fetch downloads the sources from an upstream and returns the corresponding
|
||||
// .info, .mod, and .zip files.
|
||||
|
||||
+15
-16
@@ -3,6 +3,7 @@ package module
|
||||
import (
|
||||
"bufio"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
@@ -14,15 +15,15 @@ var (
|
||||
versionSeparator = "."
|
||||
)
|
||||
|
||||
// Filter is a filter of modules
|
||||
// Filter is a filter of modules.
|
||||
type Filter struct {
|
||||
root ruleNode
|
||||
filePath string
|
||||
}
|
||||
|
||||
// NewFilter creates new filter based on rules defined in a configuration file
|
||||
// WARNING: this is not concurrently safe
|
||||
// Configuration consists of two operations: + for include and - for exclude
|
||||
// NewFilter creates new filter based on rules defined in a configuration file.
|
||||
// WARNING: this is not concurrently safe.
|
||||
// Configuration consists of two operations: + for include and - for exclude:
|
||||
// e.g.
|
||||
// - github.com/a
|
||||
// - github.com/a/b
|
||||
@@ -33,7 +34,7 @@ type Filter struct {
|
||||
// -
|
||||
// + github.com/a
|
||||
//
|
||||
// will exclude all items from communication except github.com/a
|
||||
// will exclude all items from communication except github.com/a.
|
||||
func NewFilter(filterFilePath string) (*Filter, error) {
|
||||
// Do not return an error if the file path is empty
|
||||
// Do not attempt to parse it as well.
|
||||
@@ -42,10 +43,9 @@ func NewFilter(filterFilePath string) (*Filter, error) {
|
||||
}
|
||||
|
||||
return initFromConfig(filterFilePath)
|
||||
|
||||
}
|
||||
|
||||
// AddRule adds rule for specified path
|
||||
// AddRule adds rule for specified path.
|
||||
func (f *Filter) AddRule(path string, qualifiers []string, rule FilterRule) {
|
||||
f.ensurePath(path)
|
||||
|
||||
@@ -70,7 +70,7 @@ func (f *Filter) AddRule(path string, qualifiers []string, rule FilterRule) {
|
||||
latest.next[last] = rn
|
||||
}
|
||||
|
||||
// Rule returns the filter rule to be applied to the given path
|
||||
// Rule returns the filter rule to be applied to the given path.
|
||||
func (f *Filter) Rule(path, version string) FilterRule {
|
||||
segs := getPathSegments(path)
|
||||
rule := f.getAssociatedRule(version, segs...)
|
||||
@@ -145,7 +145,6 @@ func initFromConfig(filePath string) (*Filter, error) {
|
||||
f.root = rn
|
||||
|
||||
for idx, line := range lines {
|
||||
|
||||
// Ignore newline
|
||||
if len(line) == 0 {
|
||||
continue
|
||||
@@ -160,7 +159,7 @@ func initFromConfig(filePath string) (*Filter, error) {
|
||||
}
|
||||
|
||||
ruleSign := strings.TrimSpace(split[0])
|
||||
rule := Default
|
||||
var rule FilterRule
|
||||
switch ruleSign {
|
||||
case "+":
|
||||
rule = Include
|
||||
@@ -195,11 +194,11 @@ func initFromConfig(filePath string) (*Filter, error) {
|
||||
|
||||
// matches checks if the given version matches the given qualifier.
|
||||
// Qualifiers can be:
|
||||
// - plain versions
|
||||
// - v1.2.3 enables v1.2.3
|
||||
// - ~1.2.3: enables 1.2.x which are at least 1.2.3
|
||||
// - ^1.2.3: enables 1.x.x which are at least 1.2.3
|
||||
// - <1.2.3: enables everything lower than 1.2.3 includes 1.2.2 and 0.58.9 as well
|
||||
// - plain versions.
|
||||
// - v1.2.3 enables v1.2.3.
|
||||
// - ~1.2.3: enables 1.2.x which are at least 1.2.3.
|
||||
// - ^1.2.3: enables 1.x.x which are at least 1.2.3.
|
||||
// - <1.2.3: enables everything lower than 1.2.3 includes 1.2.2 and 0.58.9 as well.
|
||||
func matches(version, qualifier string) bool {
|
||||
if len(qualifier) < 2 || len(version) < 1 {
|
||||
return false
|
||||
@@ -297,7 +296,7 @@ func newRule(r FilterRule) ruleNode {
|
||||
func getConfigLines(filterFile string) ([]string, error) {
|
||||
const op errors.Op = "module.getConfigLines"
|
||||
|
||||
f, err := os.Open(filterFile)
|
||||
f, err := os.Open(filepath.Clean(filterFile))
|
||||
if err != nil {
|
||||
return nil, errors.E(op, err)
|
||||
}
|
||||
|
||||
@@ -1,16 +1,16 @@
|
||||
package module
|
||||
|
||||
// FilterRule defines behavior of module communication
|
||||
// FilterRule defines behavior of module communication.
|
||||
type FilterRule int
|
||||
|
||||
const (
|
||||
// Default filter rule does not alter default/parent behavior
|
||||
// Default filter rule does not alter default/parent behavior.
|
||||
Default FilterRule = iota
|
||||
// Include treats modules the usual way
|
||||
// Used for reverting Exclude of parent path
|
||||
// Include treats modules the usual way.
|
||||
// Used for reverting Exclude of parent path.
|
||||
Include
|
||||
// Exclude filter rule excludes package and its children from communication
|
||||
// Exclude filter rule excludes package and its children from communication.
|
||||
Exclude
|
||||
// Direct filter rule forces the package to be fetched directly from upstream proxy
|
||||
// Direct filter rule forces the package to be fetched directly from upstream proxy.
|
||||
Direct
|
||||
)
|
||||
|
||||
@@ -35,7 +35,7 @@ type goModule struct {
|
||||
GoModSum string `json:"goModSum"` // checksum for go.mod (as in go.sum)
|
||||
}
|
||||
|
||||
// NewGoGetFetcher creates fetcher which uses go get tool to fetch modules
|
||||
// NewGoGetFetcher creates fetcher which uses go get tool to fetch modules.
|
||||
func NewGoGetFetcher(goBinaryName, gogetDir string, envVars []string, fs afero.Fs) (Fetcher, error) {
|
||||
const op errors.Op = "module.NewGoGetFetcher"
|
||||
if err := validGoBinary(goBinaryName); err != nil {
|
||||
@@ -64,7 +64,7 @@ func (g *goGetFetcher) Fetch(ctx context.Context, mod, ver string) (*storage.Ver
|
||||
sourcePath := filepath.Join(goPathRoot, "src")
|
||||
modPath := filepath.Join(sourcePath, getRepoDirName(mod, ver))
|
||||
if err := g.fs.MkdirAll(modPath, os.ModeDir|os.ModePerm); err != nil {
|
||||
clearFiles(g.fs, goPathRoot)
|
||||
_ = clearFiles(g.fs, goPathRoot)
|
||||
return nil, errors.E(op, err)
|
||||
}
|
||||
|
||||
@@ -72,14 +72,13 @@ func (g *goGetFetcher) Fetch(ctx context.Context, mod, ver string) (*storage.Ver
|
||||
ctx,
|
||||
g.goBinaryName,
|
||||
g.envVars,
|
||||
g.fs,
|
||||
goPathRoot,
|
||||
modPath,
|
||||
mod,
|
||||
ver,
|
||||
)
|
||||
if err != nil {
|
||||
clearFiles(g.fs, goPathRoot)
|
||||
_ = clearFiles(g.fs, goPathRoot)
|
||||
return nil, errors.E(op, err)
|
||||
}
|
||||
|
||||
@@ -116,7 +115,6 @@ func downloadModule(
|
||||
ctx context.Context,
|
||||
goBinaryName string,
|
||||
envVars []string,
|
||||
fs afero.Fs,
|
||||
gopath,
|
||||
repoRoot,
|
||||
module,
|
||||
@@ -137,7 +135,7 @@ func downloadModule(
|
||||
|
||||
err := cmd.Run()
|
||||
if err != nil {
|
||||
err = fmt.Errorf("%v: %s", err, stderr)
|
||||
err = fmt.Errorf("%w: %s", err, stderr)
|
||||
var m goModule
|
||||
if jsonErr := json.NewDecoder(stdout).Decode(&m); jsonErr != nil {
|
||||
return goModule{}, errors.E(op, err)
|
||||
@@ -165,17 +163,17 @@ func isLimitHit(o string) bool {
|
||||
}
|
||||
|
||||
// getRepoDirName takes a raw repository URI and a version and creates a directory name that the
|
||||
// repository contents can be put into
|
||||
// repository contents can be put into.
|
||||
func getRepoDirName(repoURI, version string) string {
|
||||
escapedURI := strings.Replace(repoURI, "/", "-", -1)
|
||||
escapedURI := strings.ReplaceAll(repoURI, "/", "-")
|
||||
return fmt.Sprintf("%s-%s", escapedURI, version)
|
||||
}
|
||||
|
||||
func validGoBinary(name string) error {
|
||||
const op errors.Op = "module.validGoBinary"
|
||||
err := exec.Command(name).Run()
|
||||
_, ok := err.(*exec.ExitError)
|
||||
if err != nil && !ok {
|
||||
eErr := &exec.ExitError{}
|
||||
if err != nil && !errors.AsErr(err, &eErr) {
|
||||
return errors.E(op, err)
|
||||
}
|
||||
return nil
|
||||
|
||||
@@ -28,7 +28,7 @@ type vcsLister struct {
|
||||
fs afero.Fs
|
||||
}
|
||||
|
||||
// NewVCSLister creates an UpstreamLister which uses VCS to fetch a list of available versions
|
||||
// NewVCSLister creates an UpstreamLister which uses VCS to fetch a list of available versions.
|
||||
func NewVCSLister(goBinPath string, env []string, fs afero.Fs) UpstreamLister {
|
||||
return &vcsLister{
|
||||
goBinPath: goBinPath,
|
||||
@@ -39,13 +39,13 @@ func NewVCSLister(goBinPath string, env []string, fs afero.Fs) UpstreamLister {
|
||||
|
||||
func (l *vcsLister) List(ctx context.Context, module string) (*storage.RevInfo, []string, error) {
|
||||
const op errors.Op = "vcsLister.List"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
_, span := observ.StartSpan(ctx, op.String())
|
||||
defer span.End()
|
||||
tmpDir, err := afero.TempDir(l.fs, "", "go-list")
|
||||
if err != nil {
|
||||
return nil, nil, errors.E(op, err)
|
||||
}
|
||||
defer l.fs.RemoveAll(tmpDir)
|
||||
defer func() { _ = l.fs.RemoveAll(tmpDir) }()
|
||||
|
||||
cmd := exec.Command(
|
||||
l.goBinPath,
|
||||
@@ -62,12 +62,12 @@ func (l *vcsLister) List(ctx context.Context, module string) (*storage.RevInfo,
|
||||
if err != nil {
|
||||
return nil, nil, errors.E(op, err)
|
||||
}
|
||||
defer clearFiles(l.fs, gopath)
|
||||
defer func() { _ = clearFiles(l.fs, gopath) }()
|
||||
cmd.Env = prepareEnv(gopath, l.env)
|
||||
|
||||
err = cmd.Run()
|
||||
if err != nil {
|
||||
err = fmt.Errorf("%v: %s", err, stderr)
|
||||
err = fmt.Errorf("%w: %s", err, stderr)
|
||||
// as of now, we can't recognize between a true NotFound
|
||||
// and an unexpected error, so we choose the more
|
||||
// hopeful path of NotFound. This way the Go command
|
||||
|
||||
@@ -9,7 +9,7 @@ import (
|
||||
|
||||
// prepareEnv will return all the appropriate
|
||||
// environment variables for a Go Command to run
|
||||
// successfully (such as GOPATH, GOCACHE, PATH etc)
|
||||
// successfully (such as GOPATH, GOCACHE, PATH etc).
|
||||
func prepareEnv(gopath string, envVars []string) []string {
|
||||
gopathEnv := fmt.Sprintf("GOPATH=%s", gopath)
|
||||
cacheEnv := fmt.Sprintf("GOCACHE=%s", filepath.Join(gopath, "cache"))
|
||||
|
||||
@@ -14,10 +14,10 @@ type zipReadCloser struct {
|
||||
goPath string
|
||||
}
|
||||
|
||||
// Close closes the zip file handle and clears up disk space used by the underlying disk ref
|
||||
// It is the caller's responsibility to call this method to free up utilized disk space
|
||||
// Close closes the zip file handle and clears up disk space used by the underlying disk ref.
|
||||
// It is the caller's responsibility to call this method to free up utilized disk space.
|
||||
func (rc *zipReadCloser) Close() error {
|
||||
rc.zip.Close()
|
||||
_ = rc.zip.Close()
|
||||
return clearFiles(rc.fs, rc.goPath)
|
||||
}
|
||||
|
||||
@@ -25,8 +25,8 @@ func (rc *zipReadCloser) Read(p []byte) (n int, err error) {
|
||||
return rc.zip.Read(p)
|
||||
}
|
||||
|
||||
// clearFiles deletes all data from the given fs at path root
|
||||
// This function must be called when zip is closed to cleanup the entire GOPATH created by the diskref
|
||||
// clearFiles deletes all data from the given fs at path root.
|
||||
// This function must be called when zip is closed to cleanup the entire GOPATH created by the diskref.
|
||||
func clearFiles(fs afero.Fs, root string) error {
|
||||
const op errors.Op = "module.ClearFiles"
|
||||
// This is required because vgo ensures dependencies are read-only
|
||||
@@ -36,7 +36,7 @@ func clearFiles(fs afero.Fs, root string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return fs.Chmod(path, 0770)
|
||||
return fs.Chmod(path, 0o770)
|
||||
}
|
||||
err := afero.Walk(fs, root, walkFn)
|
||||
if err != nil {
|
||||
|
||||
+18
-18
@@ -13,16 +13,16 @@ import (
|
||||
|
||||
// RegisterExporter determines the type of TraceExporter service for exporting traces from opencensus
|
||||
// User can choose from multiple tracing services (datadog, jaegar)
|
||||
// RegisterExporter returns the 'Flush' function for that particular tracing service
|
||||
func RegisterExporter(traceExporter, URL, service, ENV string) (func(), error) {
|
||||
// RegisterExporter returns the 'Flush' function for that particular tracing service.
|
||||
func RegisterExporter(traceExporter, url, service, env string) (func(), error) {
|
||||
const op errors.Op = "observ.RegisterExporter"
|
||||
switch traceExporter {
|
||||
case "jaeger":
|
||||
return registerJaegerExporter(URL, service, ENV)
|
||||
return registerJaegerExporter(url, service, env)
|
||||
case "datadog":
|
||||
return registerDatadogExporter(URL, service, ENV)
|
||||
return registerDatadogExporter(url, service, env)
|
||||
case "stackdriver":
|
||||
return registerStackdriverExporter(URL, ENV)
|
||||
return registerStackdriverExporter(url, env)
|
||||
case "":
|
||||
return nil, errors.E(op, "Exporter not specified. Traces won't be exported")
|
||||
default:
|
||||
@@ -32,14 +32,14 @@ func RegisterExporter(traceExporter, URL, service, ENV string) (func(), error) {
|
||||
|
||||
// registerJaegerExporter creates a jaeger exporter for exporting traces to opencensus.
|
||||
// Currently uses the 'TraceExporter' variable in the config file.
|
||||
// It should in the future have a nice sampling rate defined
|
||||
func registerJaegerExporter(URL, service, ENV string) (func(), error) {
|
||||
// It should in the future have a nice sampling rate defined.
|
||||
func registerJaegerExporter(url, service, env string) (func(), error) {
|
||||
const op errors.Op = "observ.registerJaegarExporter"
|
||||
if URL == "" {
|
||||
if url == "" {
|
||||
return nil, errors.E(op, "Exporter URL is empty. Traces won't be exported")
|
||||
}
|
||||
ex, err := jaeger.NewExporter(jaeger.Options{
|
||||
Endpoint: URL,
|
||||
Endpoint: url,
|
||||
Process: jaeger.Process{
|
||||
ServiceName: service,
|
||||
Tags: []jaeger.Tag{
|
||||
@@ -53,41 +53,41 @@ func registerJaegerExporter(URL, service, ENV string) (func(), error) {
|
||||
if err != nil {
|
||||
return nil, errors.E(op, err)
|
||||
}
|
||||
traceRegisterExporter(ex, ENV)
|
||||
traceRegisterExporter(ex, env)
|
||||
return ex.Flush, nil
|
||||
}
|
||||
|
||||
func traceRegisterExporter(exporter trace.Exporter, ENV string) {
|
||||
func traceRegisterExporter(exporter trace.Exporter, env string) {
|
||||
trace.RegisterExporter(exporter)
|
||||
if ENV == "development" {
|
||||
if env == "development" {
|
||||
trace.ApplyConfig(trace.Config{DefaultSampler: trace.AlwaysSample()})
|
||||
}
|
||||
}
|
||||
|
||||
// registerDatadogTracerExporter creates a datadog exporter.
|
||||
// Currently uses the 'TraceExporter' variable in the config file.
|
||||
func registerDatadogExporter(URL, service, ENV string) (func(), error) {
|
||||
func registerDatadogExporter(url, service, env string) (func(), error) {
|
||||
ex := datadog.NewExporter(
|
||||
datadog.Options{
|
||||
TraceAddr: URL,
|
||||
TraceAddr: url,
|
||||
Service: service,
|
||||
})
|
||||
traceRegisterExporter(ex, ENV)
|
||||
traceRegisterExporter(ex, env)
|
||||
return ex.Stop, nil
|
||||
}
|
||||
|
||||
func registerStackdriverExporter(projectID, ENV string) (func(), error) {
|
||||
func registerStackdriverExporter(projectID, env string) (func(), error) {
|
||||
const op errors.Op = "observ.registerStackdriverExporter"
|
||||
ex, err := stackdriver.NewExporter(stackdriver.Options{ProjectID: projectID})
|
||||
if err != nil {
|
||||
return nil, errors.E(op, err)
|
||||
}
|
||||
traceRegisterExporter(ex, ENV)
|
||||
traceRegisterExporter(ex, env)
|
||||
return ex.Flush, nil
|
||||
}
|
||||
|
||||
// StartSpan takes in a Context Interface and opName and starts a span. It returns the new attached ObserverContext
|
||||
// and span
|
||||
// and span.
|
||||
func StartSpan(ctx context.Context, op string) (context.Context, *trace.Span) {
|
||||
return trace.StartSpan(ctx, op)
|
||||
}
|
||||
|
||||
+4
-4
@@ -15,10 +15,10 @@ import (
|
||||
)
|
||||
|
||||
// RegisterStatsExporter determines the type of StatsExporter service for exporting stats from Opencensus
|
||||
// Currently it supports: prometheus
|
||||
// Currently it supports: prometheus.
|
||||
func RegisterStatsExporter(r *mux.Router, statsExporter, service string) (func(), error) {
|
||||
const op errors.Op = "observ.RegisterStatsExporter"
|
||||
var stop = func() {}
|
||||
stop := func() {}
|
||||
var err error
|
||||
switch statsExporter {
|
||||
case "prometheus":
|
||||
@@ -38,7 +38,7 @@ func RegisterStatsExporter(r *mux.Router, statsExporter, service string) (func()
|
||||
default:
|
||||
return nil, errors.E(op, fmt.Sprintf("StatsExporter %s not supported. Please open PR or an issue at github.com/gomods/athens", statsExporter))
|
||||
}
|
||||
if err := registerViews(); err != nil {
|
||||
if err = registerViews(); err != nil {
|
||||
return nil, errors.E(op, err)
|
||||
}
|
||||
|
||||
@@ -89,7 +89,7 @@ func registerStatsStackDriverExporter(projectID string) (func(), error) {
|
||||
return sd.Flush, nil
|
||||
}
|
||||
|
||||
// registerViews register stats which should be collected in Athens
|
||||
// registerViews register stats which should be collected in Athens.
|
||||
func registerViews() error {
|
||||
const op errors.Op = "observ.registerViews"
|
||||
if err := view.Register(
|
||||
|
||||
+2
-1
@@ -19,8 +19,9 @@ func DecodePath(encoding string) (path string, err error) {
|
||||
return path, nil
|
||||
}
|
||||
|
||||
// Ripped from cmd/go
|
||||
// Ripped from cmd/go.
|
||||
func decodeString(encoding string) (string, bool) {
|
||||
//nolint:prealloc
|
||||
var buf []byte
|
||||
|
||||
bang := false
|
||||
|
||||
+5
-5
@@ -9,7 +9,7 @@ import (
|
||||
"github.com/gorilla/mux"
|
||||
)
|
||||
|
||||
// GetModule gets the module from the path of a ?go-get=1 request
|
||||
// GetModule gets the module from the path of a ?go-get=1 request.
|
||||
func GetModule(r *http.Request) (string, error) {
|
||||
const op errors.Op = "paths.GetModule"
|
||||
module := mux.Vars(r)["module"]
|
||||
@@ -19,7 +19,7 @@ func GetModule(r *http.Request) (string, error) {
|
||||
return DecodePath(module)
|
||||
}
|
||||
|
||||
// GetVersion gets the version from the path of a ?go-get=1 request
|
||||
// GetVersion gets the version from the path of a ?go-get=1 request.
|
||||
func GetVersion(r *http.Request) (string, error) {
|
||||
const op errors.Op = "paths.GetVersion"
|
||||
|
||||
@@ -31,13 +31,13 @@ func GetVersion(r *http.Request) (string, error) {
|
||||
}
|
||||
|
||||
// AllPathParams holds the module and version in the path of a ?go-get=1
|
||||
// request
|
||||
// request.
|
||||
type AllPathParams struct {
|
||||
Module string `json:"module"`
|
||||
Version string `json:"version"`
|
||||
}
|
||||
|
||||
// GetAllParams fetches the path params from r and returns them
|
||||
// GetAllParams fetches the path params from r and returns them.
|
||||
func GetAllParams(r *http.Request) (*AllPathParams, error) {
|
||||
const op errors.Op = "paths.GetAllParams"
|
||||
mod, err := GetModule(r)
|
||||
@@ -54,7 +54,7 @@ func GetAllParams(r *http.Request) (*AllPathParams, error) {
|
||||
}
|
||||
|
||||
// MatchesPattern reports whether the path prefix of target matches
|
||||
// pattern (as defined by path.Match)
|
||||
// pattern (as defined by path.Match).
|
||||
//
|
||||
// This tries to keep the same behavior as GOPRIVATE/GONOPROXY/GONOSUMDB,
|
||||
// and is adopted from:
|
||||
|
||||
@@ -3,18 +3,18 @@ package requestid
|
||||
import "context"
|
||||
|
||||
// HeaderKey is the header key that athens uses
|
||||
// to pass request ids into logs and outbound requests
|
||||
// to pass request ids into logs and outbound requests.
|
||||
const HeaderKey = "Athens-Request-ID"
|
||||
|
||||
type key struct{}
|
||||
|
||||
// SetInContext sets the given requestID into the context
|
||||
// SetInContext sets the given requestID into the context.
|
||||
func SetInContext(ctx context.Context, id string) context.Context {
|
||||
return context.WithValue(ctx, key{}, id)
|
||||
}
|
||||
|
||||
// FromContext returns a requestID from the context or an empty
|
||||
// string if not found
|
||||
// string if not found.
|
||||
func FromContext(ctx context.Context) string {
|
||||
id, _ := ctx.Value(key{}).(string)
|
||||
return id
|
||||
|
||||
@@ -19,7 +19,7 @@ import (
|
||||
// what was requested, this is helpful if what was requested
|
||||
// was a descriptive version such as a branch name or a full commit sha.
|
||||
type Stasher interface {
|
||||
Stash(ctx context.Context, mod string, ver string) (string, error)
|
||||
Stash(ctx context.Context, mod, ver string) (string, error)
|
||||
}
|
||||
|
||||
// Wrapper helps extend the main stasher's functionality with addons.
|
||||
@@ -46,7 +46,7 @@ type stasher struct {
|
||||
|
||||
func (s *stasher) Stash(ctx context.Context, mod, ver string) (string, error) {
|
||||
const op errors.Op = "stasher.Stash"
|
||||
_, span := observ.StartSpan(ctx, op.String())
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
defer span.End()
|
||||
log.EntryFromContext(ctx).Debugf("saving %s@%s to storage...", mod, ver)
|
||||
|
||||
@@ -58,7 +58,7 @@ func (s *stasher) Stash(ctx context.Context, mod, ver string) (string, error) {
|
||||
if err != nil {
|
||||
return "", errors.E(op, err)
|
||||
}
|
||||
defer v.Zip.Close()
|
||||
defer func() { _ = v.Zip.Close() }()
|
||||
if v.Semver != ver {
|
||||
exists, err := s.checker.Exists(ctx, mod, v.Semver)
|
||||
if err != nil {
|
||||
|
||||
@@ -132,8 +132,8 @@ func (s *azblobLock) acquireLease(ctx context.Context, blobURL azblob.BlockBlobU
|
||||
_, err := blobURL.Upload(tctx, bytes.NewReader([]byte{1}), azblob.BlobHTTPHeaders{}, nil, azblob.BlobAccessConditions{})
|
||||
if err != nil {
|
||||
// if the blob is already leased we will get http.StatusPreconditionFailed while writing to that blob
|
||||
stgErr, ok := err.(azblob.StorageError)
|
||||
if !ok || stgErr.Response().StatusCode != http.StatusPreconditionFailed {
|
||||
var stgErr azblob.StorageError
|
||||
if !errors.AsErr(err, &stgErr) || stgErr.Response().StatusCode != http.StatusPreconditionFailed {
|
||||
return "", errors.E(op, err)
|
||||
}
|
||||
}
|
||||
@@ -147,7 +147,8 @@ func (s *azblobLock) acquireLease(ctx context.Context, blobURL azblob.BlockBlobU
|
||||
res, err := blobURL.AcquireLease(tctx, leaseID.String(), 15, azblob.ModifiedAccessConditions{})
|
||||
if err != nil {
|
||||
// if the blob is already leased we will get http.StatusConflict - wait and try again
|
||||
if stgErr, ok := err.(azblob.StorageError); ok && stgErr.Response().StatusCode == http.StatusConflict {
|
||||
var stgErr azblob.StorageError
|
||||
if ok := errors.AsErr(err, &stgErr); ok && stgErr.Response().StatusCode == http.StatusConflict {
|
||||
select {
|
||||
case <-time.After(1 * time.Second):
|
||||
continue
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
"github.com/gomods/athens/pkg/errors"
|
||||
"github.com/gomods/athens/pkg/observ"
|
||||
"github.com/gomods/athens/pkg/storage"
|
||||
"go.etcd.io/etcd/client/v3"
|
||||
clientv3 "go.etcd.io/etcd/client/v3"
|
||||
"go.etcd.io/etcd/client/v3/concurrency"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
@@ -13,14 +13,14 @@ import (
|
||||
"github.com/gomods/athens/pkg/storage"
|
||||
)
|
||||
|
||||
// RedisLogger mirrors github.com/go-redis/redis/v8/internal.Logging
|
||||
// RedisLogger mirrors github.com/go-redis/redis/v8/internal.Logging.
|
||||
type RedisLogger interface {
|
||||
Printf(ctx context.Context, format string, v ...interface{})
|
||||
}
|
||||
|
||||
// WithRedisLock returns a distributed singleflight
|
||||
// using a redis cluster. If it cannot connect, it will return an error.
|
||||
func WithRedisLock(l RedisLogger, endpoint string, password string, checker storage.Checker, lockConfig *config.RedisLockConfig) (Wrapper, error) {
|
||||
func WithRedisLock(l RedisLogger, endpoint, password string, checker storage.Checker, lockConfig *config.RedisLockConfig) (Wrapper, error) {
|
||||
redis.SetLogger(l)
|
||||
|
||||
const op errors.Op = "stash.WithRedisLock"
|
||||
|
||||
@@ -10,7 +10,7 @@ import (
|
||||
)
|
||||
|
||||
// WithRedisSentinelLock returns a distributed singleflight
|
||||
// with a redis cluster that utilizes sentinel for quorum and failover
|
||||
// with a redis cluster that utilizes sentinel for quorum and failover.
|
||||
func WithRedisSentinelLock(l RedisLogger, endpoints []string, master, password string, checker storage.Checker, lockConfig *config.RedisLockConfig) (Wrapper, error) {
|
||||
redis.SetLogger(l)
|
||||
|
||||
|
||||
@@ -15,14 +15,6 @@ import (
|
||||
"github.com/gomods/athens/pkg/storage"
|
||||
)
|
||||
|
||||
type client interface {
|
||||
UploadWithContext(ctx context.Context, path, contentType string, content io.Reader) error
|
||||
BlobExists(ctx context.Context, path string) (bool, error)
|
||||
ReadBlob(ctx context.Context, path string) (io.ReadCloser, error)
|
||||
ListBlobs(ctx context.Context, prefix string) ([]string, error)
|
||||
DeleteBlob(ctx context.Context, path string) error
|
||||
}
|
||||
|
||||
type azureBlobStoreClient struct {
|
||||
containerURL *azblob.ContainerURL
|
||||
}
|
||||
@@ -45,13 +37,13 @@ func newBlobStoreClient(accountURL *url.URL, accountName, accountKey, containerN
|
||||
}
|
||||
|
||||
// Storage implements (github.com/gomods/athens/pkg/storage).Saver and
|
||||
// also provides a function to fetch the location of a module
|
||||
// also provides a function to fetch the location of a module.
|
||||
type Storage struct {
|
||||
client *azureBlobStoreClient
|
||||
timeout time.Duration
|
||||
}
|
||||
|
||||
// New creates a new azure blobs storage
|
||||
// New creates a new azure blobs storage.
|
||||
func New(conf *config.AzureBlobConfig, timeout time.Duration) (*Storage, error) {
|
||||
const op errors.Op = "azureblob.New"
|
||||
u, err := url.Parse(fmt.Sprintf("https://%s.blob.core.windows.net", conf.AccountName))
|
||||
@@ -65,7 +57,7 @@ func New(conf *config.AzureBlobConfig, timeout time.Duration) (*Storage, error)
|
||||
return &Storage{client: cl, timeout: timeout}, nil
|
||||
}
|
||||
|
||||
// BlobExists checks if a particular blob exists in the container
|
||||
// BlobExists checks if a particular blob exists in the container.
|
||||
func (c *azureBlobStoreClient) BlobExists(ctx context.Context, path string) (bool, error) {
|
||||
const op errors.Op = "azureblob.BlobExists"
|
||||
// TODO: Any better way of doing this ?
|
||||
@@ -73,10 +65,8 @@ func (c *azureBlobStoreClient) BlobExists(ctx context.Context, path string) (boo
|
||||
_, err := blobURL.GetProperties(ctx, azblob.BlobAccessConditions{})
|
||||
if err != nil {
|
||||
var serr azblob.StorageError
|
||||
var ok bool
|
||||
|
||||
if serr, ok = err.(azblob.StorageError); !ok {
|
||||
return false, errors.E(op, fmt.Errorf("Error in casting to azure error type %v", err))
|
||||
if !errors.AsErr(err, &serr) {
|
||||
return false, errors.E(op, fmt.Errorf("error in casting to azure error type %w", err))
|
||||
}
|
||||
if serr.Response().StatusCode == http.StatusNotFound {
|
||||
return false, nil
|
||||
@@ -85,10 +75,9 @@ func (c *azureBlobStoreClient) BlobExists(ctx context.Context, path string) (boo
|
||||
return false, errors.E(op, err)
|
||||
}
|
||||
return true, nil
|
||||
|
||||
}
|
||||
|
||||
// ReadBlob returns a storage.SizeReadCloser for the contents of a blob
|
||||
// ReadBlob returns a storage.SizeReadCloser for the contents of a blob.
|
||||
func (c *azureBlobStoreClient) ReadBlob(ctx context.Context, path string) (storage.SizeReadCloser, error) {
|
||||
const op errors.Op = "azureblob.ReadBlob"
|
||||
blobURL := c.containerURL.NewBlockBlobURL(path)
|
||||
@@ -101,7 +90,7 @@ func (c *azureBlobStoreClient) ReadBlob(ctx context.Context, path string) (stora
|
||||
return storage.NewSizer(rc, size), nil
|
||||
}
|
||||
|
||||
// ListBlobs will list all blobs which has the given prefix
|
||||
// ListBlobs will list all blobs which has the given prefix.
|
||||
func (c *azureBlobStoreClient) ListBlobs(ctx context.Context, prefix string) ([]string, error) {
|
||||
const op errors.Op = "azureblob.ListBlobs"
|
||||
var blobs []string
|
||||
@@ -122,7 +111,7 @@ func (c *azureBlobStoreClient) ListBlobs(ctx context.Context, prefix string) ([]
|
||||
return blobs, nil
|
||||
}
|
||||
|
||||
// DeleteBlob deletes the blob with the given path
|
||||
// DeleteBlob deletes the blob with the given path.
|
||||
func (c *azureBlobStoreClient) DeleteBlob(ctx context.Context, path string) error {
|
||||
const op errors.Op = "azureblob.DeleteBlob"
|
||||
blobURL := c.containerURL.NewBlockBlobURL(path)
|
||||
@@ -133,7 +122,7 @@ func (c *azureBlobStoreClient) DeleteBlob(ctx context.Context, path string) erro
|
||||
return nil
|
||||
}
|
||||
|
||||
// UploadWithContext uploads a blob to the container
|
||||
// UploadWithContext uploads a blob to the container.
|
||||
func (c *azureBlobStoreClient) UploadWithContext(ctx context.Context, path, contentType string, content io.Reader) error {
|
||||
const op errors.Op = "azureblob.UploadWithContext"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
|
||||
@@ -12,8 +12,8 @@ import (
|
||||
"github.com/gomods/athens/pkg/paths"
|
||||
)
|
||||
|
||||
// Catalog implements the (./pkg/storage).Catalog interface
|
||||
// It returns a list of versions, if any, for a given module
|
||||
// Catalog implements the (./pkg/storage).Catalog interface.
|
||||
// It returns a list of versions, if any, for a given module.
|
||||
func (s *Storage) Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error) {
|
||||
const op errors.Op = "azblob.Catalog"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
|
||||
@@ -9,8 +9,8 @@ import (
|
||||
)
|
||||
|
||||
// Exists implements the (./pkg/storage).Checker interface
|
||||
// returning true if the module at version exists in storage
|
||||
func (s *Storage) Exists(ctx context.Context, module string, version string) (bool, error) {
|
||||
// returning true if the module at version exists in storage.
|
||||
func (s *Storage) Exists(ctx context.Context, module, version string) (bool, error) {
|
||||
const op errors.Op = "azureblob.Exists"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
defer span.End()
|
||||
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
// Delete implements the (./pkg/storage).Deleter interface and
|
||||
// removes a version of a module from storage. Returning ErrNotFound
|
||||
// if the version does not exist.
|
||||
func (s *Storage) Delete(ctx context.Context, module string, version string) error {
|
||||
func (s *Storage) Delete(ctx context.Context, module, version string) error {
|
||||
const op errors.Op = "azureblob.Delete"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
defer span.End()
|
||||
|
||||
@@ -11,8 +11,8 @@ import (
|
||||
"github.com/gomods/athens/pkg/storage"
|
||||
)
|
||||
|
||||
// Info implements the (./pkg/storage).Getter interface
|
||||
func (s *Storage) Info(ctx context.Context, module string, version string) ([]byte, error) {
|
||||
// Info implements the (./pkg/storage).Getter interface.
|
||||
func (s *Storage) Info(ctx context.Context, module, version string) ([]byte, error) {
|
||||
const op errors.Op = "azureblob.Info"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
defer span.End()
|
||||
@@ -43,8 +43,8 @@ func (s *Storage) Info(ctx context.Context, module string, version string) ([]by
|
||||
return infoBytes, nil
|
||||
}
|
||||
|
||||
// GoMod implements the (./pkg/storage).Getter interface
|
||||
func (s *Storage) GoMod(ctx context.Context, module string, version string) ([]byte, error) {
|
||||
// GoMod implements the (./pkg/storage).Getter interface.
|
||||
func (s *Storage) GoMod(ctx context.Context, module, version string) ([]byte, error) {
|
||||
const op errors.Op = "azureblob.GoMod"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
defer span.End()
|
||||
@@ -64,7 +64,7 @@ func (s *Storage) GoMod(ctx context.Context, module string, version string) ([]b
|
||||
|
||||
modBytes, err := io.ReadAll(modReader)
|
||||
if err != nil {
|
||||
return nil, errors.E(op, fmt.Errorf("could not get new reader for mod file: %s", err), errors.M(module), errors.V(version))
|
||||
return nil, errors.E(op, fmt.Errorf("could not get new reader for mod file: %w", err), errors.M(module), errors.V(version))
|
||||
}
|
||||
|
||||
err = modReader.Close()
|
||||
@@ -75,8 +75,8 @@ func (s *Storage) GoMod(ctx context.Context, module string, version string) ([]b
|
||||
return modBytes, nil
|
||||
}
|
||||
|
||||
// Zip implements the (./pkg/storage).Getter interface
|
||||
func (s *Storage) Zip(ctx context.Context, module string, version string) (storage.SizeReadCloser, error) {
|
||||
// Zip implements the (./pkg/storage).Getter interface.
|
||||
func (s *Storage) Zip(ctx context.Context, module, version string) (storage.SizeReadCloser, error) {
|
||||
const op errors.Op = "azureblob.Zip"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
defer span.End()
|
||||
|
||||
@@ -8,8 +8,8 @@ import (
|
||||
"github.com/gomods/athens/pkg/observ"
|
||||
)
|
||||
|
||||
// List implements the (./pkg/storage).Lister interface
|
||||
// It returns a list of versions, if any, for a given module
|
||||
// List implements the (./pkg/storage).Lister interface.
|
||||
// It returns a list of versions, if any, for a given module.
|
||||
func (s *Storage) List(ctx context.Context, module string) ([]string, error) {
|
||||
const op errors.Op = "azureblob.List"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
package storage
|
||||
|
||||
// Backend is a complete storage backend (i.e. file system, database) implementation - a lister, reader and saver
|
||||
// Backend is a complete storage backend (i.e. file system, database) implementation - a lister, reader and saver.
|
||||
type Backend interface {
|
||||
Lister
|
||||
Getter
|
||||
|
||||
@@ -6,7 +6,7 @@ import (
|
||||
"github.com/gomods/athens/pkg/paths"
|
||||
)
|
||||
|
||||
// Cataloger is the interface that lists all the modules and version contained in the storage
|
||||
// Cataloger is the interface that lists all the modules and version contained in the storage.
|
||||
type Cataloger interface {
|
||||
// Catalog gets all the modules / versions.
|
||||
Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error)
|
||||
|
||||
@@ -6,14 +6,14 @@ import (
|
||||
"github.com/gomods/athens/pkg/errors"
|
||||
)
|
||||
|
||||
// Checker is the interface that checks if the version of the module exists
|
||||
// Checker is the interface that checks if the version of the module exists.
|
||||
type Checker interface {
|
||||
// Exists checks whether or not module in specified version is present
|
||||
// in the backing storage
|
||||
// in the backing storage.
|
||||
Exists(ctx context.Context, module, version string) (bool, error)
|
||||
}
|
||||
|
||||
// WithChecker wraps the backend with a Checker implementaiton
|
||||
// WithChecker wraps the backend with a Checker implementation.
|
||||
func WithChecker(strg Backend) Checker {
|
||||
if checker, ok := strg.(Checker); ok {
|
||||
return checker
|
||||
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"math/rand"
|
||||
"sort"
|
||||
"testing"
|
||||
|
||||
"github.com/gomods/athens/pkg/errors"
|
||||
@@ -26,7 +25,6 @@ func RunTests(t *testing.T, b storage.Backend, clearBackend func() error) {
|
||||
testGet(t, b)
|
||||
testExists(t, b)
|
||||
testShouldNotExist(t, b)
|
||||
// testCatalog(t, b)
|
||||
}
|
||||
|
||||
// testNotFound ensures that a storage Backend
|
||||
@@ -209,49 +207,6 @@ func testDelete(t *testing.T, b storage.Backend) {
|
||||
require.Equal(t, false, exists)
|
||||
}
|
||||
|
||||
func testCatalog(t *testing.T, b storage.Backend) {
|
||||
cs, ok := b.(storage.Cataloger)
|
||||
if !ok {
|
||||
t.Skip()
|
||||
}
|
||||
ctx := context.Background()
|
||||
|
||||
mock := getMockModule()
|
||||
zipBts, _ := io.ReadAll(mock.Zip)
|
||||
modname := "github.com/gomods/testCatalogModule"
|
||||
for i := 0; i < 6; i++ {
|
||||
ver := fmt.Sprintf("v1.2.%04d", i)
|
||||
b.Save(ctx, modname, ver, mock.Mod, bytes.NewReader(zipBts), mock.Info)
|
||||
|
||||
defer b.Delete(ctx, modname, ver)
|
||||
}
|
||||
|
||||
allres, next, err := cs.Catalog(ctx, "", 5)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 5, len(allres))
|
||||
|
||||
res, next, err := cs.Catalog(ctx, next, 50)
|
||||
allres = append(allres, res...)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, len(res))
|
||||
require.Equal(t, "", next)
|
||||
|
||||
sort.Slice(allres, func(i, j int) bool {
|
||||
if allres[i].Module == allres[j].Module {
|
||||
return allres[i].Version < allres[j].Version
|
||||
}
|
||||
return allres[i].Module < allres[j].Module
|
||||
})
|
||||
require.Equal(t, modname, allres[0].Module)
|
||||
require.Equal(t, "v1.2.0000", allres[0].Version)
|
||||
require.Equal(t, "v1.2.0004", allres[4].Version)
|
||||
|
||||
for i := 1; i < len(allres); i++ {
|
||||
require.NotEqual(t, allres[i].Version, allres[i-1].Version)
|
||||
}
|
||||
}
|
||||
|
||||
func getMockModule() *storage.Version {
|
||||
return &storage.Version{
|
||||
Info: []byte("123"),
|
||||
|
||||
@@ -2,7 +2,7 @@ package storage
|
||||
|
||||
import "context"
|
||||
|
||||
// Deleter deletes module metadata and its source from underlying storage
|
||||
// Deleter deletes module metadata and its source from underlying storage.
|
||||
type Deleter interface {
|
||||
// Delete must return ErrNotFound if the module/version are not
|
||||
// found.
|
||||
|
||||
Vendored
+14
-14
@@ -20,7 +20,7 @@ type service struct {
|
||||
c *http.Client
|
||||
}
|
||||
|
||||
// NewClient returns an external storage client
|
||||
// NewClient returns an external storage client.
|
||||
func NewClient(url string, c *http.Client) storage.Backend {
|
||||
if c == nil {
|
||||
c = &http.Client{}
|
||||
@@ -93,7 +93,7 @@ func (s *service) Save(ctx context.Context, mod, ver string, modFile []byte, zip
|
||||
mw := multipart.NewWriter(pw)
|
||||
go func() {
|
||||
err := upload(mw, modFile, info, zip)
|
||||
pw.CloseWithError(err)
|
||||
_ = pw.CloseWithError(err)
|
||||
}()
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, pr)
|
||||
if err != nil {
|
||||
@@ -104,8 +104,8 @@ func (s *service) Save(ctx context.Context, mod, ver string, modFile []byte, zip
|
||||
if err != nil {
|
||||
return errors.E(op, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != 200 {
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
bts, _ := io.ReadAll(resp.Body)
|
||||
return errors.E(op, fmt.Errorf("unexpected status code: %v - body: %s", resp.StatusCode, bts), resp.StatusCode)
|
||||
}
|
||||
@@ -118,35 +118,35 @@ func (s *service) Delete(ctx context.Context, mod, ver string) error {
|
||||
if err != nil {
|
||||
return errors.E(op, err)
|
||||
}
|
||||
defer body.Close()
|
||||
defer func() { _ = body.Close() }()
|
||||
return nil
|
||||
}
|
||||
|
||||
func upload(mw *multipart.Writer, mod, info []byte, zip io.Reader) error {
|
||||
defer mw.Close()
|
||||
defer func() { _ = mw.Close() }()
|
||||
infoW, err := mw.CreateFormFile("mod.info", "mod.info")
|
||||
if err != nil {
|
||||
return fmt.Errorf("error creating info file: %v", err)
|
||||
return fmt.Errorf("error creating info file: %w", err)
|
||||
}
|
||||
_, err = infoW.Write(info)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error writing info file: %v", err)
|
||||
return fmt.Errorf("error writing info file: %w", err)
|
||||
}
|
||||
modW, err := mw.CreateFormFile("mod.mod", "mod.mod")
|
||||
if err != nil {
|
||||
return fmt.Errorf("error creating mod file: %v", err)
|
||||
return fmt.Errorf("error creating mod file: %w", err)
|
||||
}
|
||||
_, err = modW.Write(mod)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error writing mod file: %v", err)
|
||||
return fmt.Errorf("error writing mod file: %w", err)
|
||||
}
|
||||
zipW, err := mw.CreateFormFile("mod.zip", "mod.zip")
|
||||
if err != nil {
|
||||
return fmt.Errorf("error creating zip file: %v", err)
|
||||
return fmt.Errorf("error creating zip file: %w", err)
|
||||
}
|
||||
_, err = io.Copy(zipW, zip)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error writing zip file: %v", err)
|
||||
return fmt.Errorf("error writing zip file: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -174,9 +174,9 @@ func (s *service) doRequest(ctx context.Context, method, mod, ver, ext string) (
|
||||
if err != nil {
|
||||
return nil, 0, errors.E(op, err)
|
||||
}
|
||||
if resp.StatusCode != 200 {
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
_ = resp.Body.Close()
|
||||
return nil, 0, errors.E(op, fmt.Errorf("none 200 status code: %v - body: %s", resp.StatusCode, body), resp.StatusCode)
|
||||
}
|
||||
var size int64
|
||||
|
||||
Vendored
+21
-21
@@ -17,7 +17,7 @@ import (
|
||||
|
||||
// NewServer takes a storage.Backend implementation of your
|
||||
// choice, and returns a new http.Handler that Athens can
|
||||
// reach out to for storage operations
|
||||
// reach out to for storage operations.
|
||||
func NewServer(strg storage.Backend) http.Handler {
|
||||
r := mux.NewRouter()
|
||||
r.HandleFunc(download.PathList, func(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -27,12 +27,12 @@ func NewServer(strg storage.Backend) http.Handler {
|
||||
http.Error(w, err.Error(), errors.Kind(err))
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(w, "%s", strings.Join(list, "\n"))
|
||||
_, _ = fmt.Fprintf(w, "%s", strings.Join(list, "\n"))
|
||||
}).Methods(http.MethodGet)
|
||||
r.HandleFunc(download.PathVersionInfo, func(w http.ResponseWriter, r *http.Request) {
|
||||
params, err := paths.GetAllParams(r)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), 400)
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
info, err := strg.Info(r.Context(), params.Module, params.Version)
|
||||
@@ -40,12 +40,12 @@ func NewServer(strg storage.Backend) http.Handler {
|
||||
http.Error(w, err.Error(), errors.Kind(err))
|
||||
return
|
||||
}
|
||||
w.Write(info)
|
||||
_, _ = w.Write(info)
|
||||
}).Methods(http.MethodGet)
|
||||
r.HandleFunc(download.PathVersionModule, func(w http.ResponseWriter, r *http.Request) {
|
||||
params, err := paths.GetAllParams(r)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), 400)
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
mod, err := strg.GoMod(r.Context(), params.Module, params.Version)
|
||||
@@ -53,12 +53,12 @@ func NewServer(strg storage.Backend) http.Handler {
|
||||
http.Error(w, err.Error(), errors.Kind(err))
|
||||
return
|
||||
}
|
||||
w.Write(mod)
|
||||
_, _ = w.Write(mod)
|
||||
}).Methods(http.MethodGet)
|
||||
r.HandleFunc(download.PathVersionZip, func(w http.ResponseWriter, r *http.Request) {
|
||||
params, err := paths.GetAllParams(r)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), 400)
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
zip, err := strg.Zip(r.Context(), params.Module, params.Version)
|
||||
@@ -66,52 +66,52 @@ func NewServer(strg storage.Backend) http.Handler {
|
||||
http.Error(w, err.Error(), errors.Kind(err))
|
||||
return
|
||||
}
|
||||
defer zip.Close()
|
||||
defer func() { _ = zip.Close() }()
|
||||
w.Header().Set("Content-Length", strconv.FormatInt(zip.Size(), 10))
|
||||
io.Copy(w, zip)
|
||||
_, _ = io.Copy(w, zip)
|
||||
}).Methods(http.MethodGet)
|
||||
r.HandleFunc("/{module:.+}/@v/{version}.save", func(w http.ResponseWriter, r *http.Request) {
|
||||
params, err := paths.GetAllParams(r)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), 400)
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
err = r.ParseMultipartForm(zip.MaxZipFile + zip.MaxGoMod)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), 400)
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
infoFile, _, err := r.FormFile("mod.info")
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), 400)
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
defer infoFile.Close()
|
||||
defer func() { _ = infoFile.Close() }()
|
||||
info, err := io.ReadAll(infoFile)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), 400)
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
modReader, _, err := r.FormFile("mod.mod")
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), 400)
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
defer modReader.Close()
|
||||
defer func() { _ = modReader.Close() }()
|
||||
modFile, err := io.ReadAll(modReader)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), 400)
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
modZ, _, err := r.FormFile("mod.zip")
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), 400)
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
defer modZ.Close()
|
||||
defer func() { _ = modZ.Close() }()
|
||||
err = strg.Save(r.Context(), params.Module, params.Version, modFile, modZ, info)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), 400)
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
}).Methods(http.MethodPost)
|
||||
@@ -119,7 +119,7 @@ func NewServer(strg storage.Backend) http.Handler {
|
||||
r.HandleFunc("/{module:.+}/@v/{version}.delete", func(w http.ResponseWriter, r *http.Request) {
|
||||
params, err := paths.GetAllParams(r)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), 400)
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
err = strg.Delete(r.Context(), params.Module, params.Version)
|
||||
|
||||
@@ -15,11 +15,11 @@ import (
|
||||
|
||||
const tokenSeparator = "|"
|
||||
|
||||
// Catalog implements the (./pkg/storage).Cataloger interface
|
||||
// It returns a list of modules and versions contained in the storage
|
||||
// Catalog implements the (./pkg/storage).Cataloger interface.
|
||||
// It returns a list of modules and versions contained in the storage.
|
||||
func (s *storageImpl) Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error) {
|
||||
const op errors.Op = "fs.Catalog"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
_, span := observ.StartSpan(ctx, op.String())
|
||||
defer span.End()
|
||||
|
||||
fromModule, fromVersion, err := modVerFromToken(token)
|
||||
@@ -41,7 +41,7 @@ func (s *storageImpl) Catalog(ctx context.Context, token string, pageSize int) (
|
||||
|
||||
m, version := filepath.Split(modVer)
|
||||
module := filepath.Clean(m)
|
||||
module = strings.Replace(module, string(os.PathSeparator), "/", -1)
|
||||
module = strings.ReplaceAll(module, string(os.PathSeparator), "/")
|
||||
|
||||
if fromModule != "" && module < fromModule { // it is ok to land on the same module
|
||||
return nil
|
||||
@@ -60,7 +60,7 @@ func (s *storageImpl) Catalog(ctx context.Context, token string, pageSize int) (
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil && err != io.EOF {
|
||||
if err != nil && !errors.IsErr(err, io.EOF) {
|
||||
return nil, "", errors.E(op, err, errors.KindUnexpected)
|
||||
}
|
||||
|
||||
|
||||
@@ -9,14 +9,13 @@ import (
|
||||
"github.com/spf13/afero"
|
||||
)
|
||||
|
||||
func (v *storageImpl) Exists(ctx context.Context, module, version string) (bool, error) {
|
||||
func (s *storageImpl) Exists(ctx context.Context, module, version string) (bool, error) {
|
||||
const op errors.Op = "fs.Exists"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
_, span := observ.StartSpan(ctx, op.String())
|
||||
defer span.End()
|
||||
versionedPath := v.versionLocation(module, version)
|
||||
|
||||
files, err := afero.ReadDir(v.filesystem, versionedPath)
|
||||
versionedPath := s.versionLocation(module, version)
|
||||
|
||||
files, err := afero.ReadDir(s.filesystem, versionedPath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return false, nil
|
||||
|
||||
@@ -8,17 +8,17 @@ import (
|
||||
)
|
||||
|
||||
// Delete removes a specific version of a module.
|
||||
func (v *storageImpl) Delete(ctx context.Context, module, version string) error {
|
||||
func (s *storageImpl) Delete(ctx context.Context, module, version string) error {
|
||||
const op errors.Op = "fs.Delete"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
defer span.End()
|
||||
versionedPath := v.versionLocation(module, version)
|
||||
exists, err := v.Exists(ctx, module, version)
|
||||
versionedPath := s.versionLocation(module, version)
|
||||
exists, err := s.Exists(ctx, module, version)
|
||||
if err != nil {
|
||||
return errors.E(op, err, errors.M(module), errors.V(version))
|
||||
}
|
||||
if !exists {
|
||||
return errors.E(op, errors.M(module), errors.V(version), errors.KindNotFound)
|
||||
}
|
||||
return v.filesystem.RemoveAll(versionedPath)
|
||||
return s.filesystem.RemoveAll(versionedPath)
|
||||
}
|
||||
|
||||
@@ -21,17 +21,16 @@ func (s *storageImpl) moduleLocation(module string) string {
|
||||
|
||||
func (s *storageImpl) versionLocation(module, version string) string {
|
||||
return filepath.Join(s.moduleLocation(module), version)
|
||||
|
||||
}
|
||||
|
||||
// NewStorage returns a new ListerSaver implementation that stores
|
||||
// everything under rootDir
|
||||
// If the root directory does not exist an error is returned
|
||||
// everything under rootDir.
|
||||
// If the root directory does not exist an error is returned.
|
||||
func NewStorage(rootDir string, filesystem afero.Fs) (storage.Backend, error) {
|
||||
const op errors.Op = "fs.NewStorage"
|
||||
exists, err := afero.Exists(filesystem, rootDir)
|
||||
if err != nil {
|
||||
return nil, errors.E(op, fmt.Errorf("could not check if root directory `%s` exists: %s", rootDir, err))
|
||||
return nil, errors.E(op, fmt.Errorf("could not check if root directory `%s` exists: %w", rootDir, err))
|
||||
}
|
||||
if !exists {
|
||||
return nil, errors.E(op, fmt.Errorf("root directory `%s` does not exist", rootDir))
|
||||
|
||||
+12
-24
@@ -11,12 +11,12 @@ import (
|
||||
"github.com/spf13/afero"
|
||||
)
|
||||
|
||||
func (v *storageImpl) Info(ctx context.Context, module, version string) ([]byte, error) {
|
||||
func (s *storageImpl) Info(ctx context.Context, module, version string) ([]byte, error) {
|
||||
const op errors.Op = "fs.Info"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
_, span := observ.StartSpan(ctx, op.String())
|
||||
defer span.End()
|
||||
versionedPath := v.versionLocation(module, version)
|
||||
info, err := afero.ReadFile(v.filesystem, filepath.Join(versionedPath, version+".info"))
|
||||
versionedPath := s.versionLocation(module, version)
|
||||
info, err := afero.ReadFile(s.filesystem, filepath.Join(versionedPath, version+".info"))
|
||||
if err != nil {
|
||||
return nil, errors.E(op, errors.M(module), errors.V(version), errors.KindNotFound)
|
||||
}
|
||||
@@ -24,12 +24,12 @@ func (v *storageImpl) Info(ctx context.Context, module, version string) ([]byte,
|
||||
return info, nil
|
||||
}
|
||||
|
||||
func (v *storageImpl) GoMod(ctx context.Context, module, version string) ([]byte, error) {
|
||||
func (s *storageImpl) GoMod(ctx context.Context, module, version string) ([]byte, error) {
|
||||
const op errors.Op = "fs.GoMod"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
_, span := observ.StartSpan(ctx, op.String())
|
||||
defer span.End()
|
||||
versionedPath := v.versionLocation(module, version)
|
||||
mod, err := afero.ReadFile(v.filesystem, filepath.Join(versionedPath, "go.mod"))
|
||||
versionedPath := s.versionLocation(module, version)
|
||||
mod, err := afero.ReadFile(s.filesystem, filepath.Join(versionedPath, "go.mod"))
|
||||
if err != nil {
|
||||
return nil, errors.E(op, errors.M(module), errors.V(version), errors.KindNotFound)
|
||||
}
|
||||
@@ -37,13 +37,13 @@ func (v *storageImpl) GoMod(ctx context.Context, module, version string) ([]byte
|
||||
return mod, nil
|
||||
}
|
||||
|
||||
func (v *storageImpl) Zip(ctx context.Context, module, version string) (storage.SizeReadCloser, error) {
|
||||
func (s *storageImpl) Zip(ctx context.Context, module, version string) (storage.SizeReadCloser, error) {
|
||||
const op errors.Op = "fs.Zip"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
_, span := observ.StartSpan(ctx, op.String())
|
||||
defer span.End()
|
||||
versionedPath := v.versionLocation(module, version)
|
||||
versionedPath := s.versionLocation(module, version)
|
||||
|
||||
src, err := v.filesystem.OpenFile(filepath.Join(versionedPath, "source.zip"), os.O_RDONLY, 0666)
|
||||
src, err := s.filesystem.OpenFile(filepath.Join(versionedPath, "source.zip"), os.O_RDONLY, 0o666)
|
||||
if err != nil {
|
||||
return nil, errors.E(op, errors.M(module), errors.V(version), errors.KindNotFound)
|
||||
}
|
||||
@@ -53,15 +53,3 @@ func (v *storageImpl) Zip(ctx context.Context, module, version string) (storage.
|
||||
}
|
||||
return storage.NewSizer(src, fi.Size()), nil
|
||||
}
|
||||
|
||||
func (v *storageImpl) ZipSize(ctx context.Context, module, version string) (int64, error) {
|
||||
const op errors.Op = "fs.ZipFileSize"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
defer span.End()
|
||||
versionedPath := v.versionLocation(module, version)
|
||||
fi, err := v.filesystem.Stat(filepath.Join(versionedPath))
|
||||
if err != nil {
|
||||
return 0, errors.E(op, err, errors.M(module), errors.V(version), errors.KindNotFound)
|
||||
}
|
||||
return fi.Size(), nil
|
||||
}
|
||||
|
||||
@@ -11,12 +11,12 @@ import (
|
||||
"golang.org/x/mod/semver"
|
||||
)
|
||||
|
||||
func (l *storageImpl) List(ctx context.Context, module string) ([]string, error) {
|
||||
func (s *storageImpl) List(ctx context.Context, module string) ([]string, error) {
|
||||
const op errors.Op = "fs.List"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
_, span := observ.StartSpan(ctx, op.String())
|
||||
defer span.End()
|
||||
loc := l.moduleLocation(module)
|
||||
fileInfos, err := afero.ReadDir(l.filesystem, loc)
|
||||
loc := s.moduleLocation(module)
|
||||
fileInfos, err := afero.ReadDir(s.filesystem, loc)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return []string{}, nil
|
||||
|
||||
+12
-12
@@ -13,37 +13,37 @@ import (
|
||||
|
||||
func (s *storageImpl) Save(ctx context.Context, module, version string, mod []byte, zip io.Reader, info []byte) error {
|
||||
const op errors.Op = "fs.Save"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
_, span := observ.StartSpan(ctx, op.String())
|
||||
defer span.End()
|
||||
dir := s.versionLocation(module, version)
|
||||
|
||||
// NB: the process's umask is subtracted from the permissions below,
|
||||
// so a umask of for example 0077 allows directories and files to be
|
||||
// created with mode 0700 / 0600, i.e. not world- or group-readable
|
||||
// NB: The process's umask is subtracted from the permissions below,
|
||||
// so an umask of for example 0077 allows directories and files to be
|
||||
// created with mode 0700 / 0600, i.e. not world- or group-readable.
|
||||
|
||||
// make the versioned directory to hold the go.mod and the zipfile
|
||||
if err := s.filesystem.MkdirAll(dir, 0777); err != nil {
|
||||
// Make the versioned directory to hold the go.mod and the zipfile.
|
||||
if err := s.filesystem.MkdirAll(dir, 0o777); err != nil {
|
||||
return errors.E(op, err, errors.M(module), errors.V(version))
|
||||
}
|
||||
|
||||
// write the go.mod file
|
||||
if err := afero.WriteFile(s.filesystem, filepath.Join(dir, "go.mod"), mod, 0666); err != nil {
|
||||
// Write the go.mod file.
|
||||
if err := afero.WriteFile(s.filesystem, filepath.Join(dir, "go.mod"), mod, 0o666); err != nil {
|
||||
return errors.E(op, err, errors.M(module), errors.V(version))
|
||||
}
|
||||
|
||||
// write the zipfile
|
||||
f, err := s.filesystem.OpenFile(filepath.Join(dir, "source.zip"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
|
||||
// Write the zipfile.
|
||||
f, err := s.filesystem.OpenFile(filepath.Join(dir, "source.zip"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o666)
|
||||
if err != nil {
|
||||
return errors.E(op, err, errors.M(module), errors.V(version))
|
||||
}
|
||||
defer f.Close()
|
||||
defer func() { _ = f.Close() }()
|
||||
_, err = io.Copy(f, zip)
|
||||
if err != nil {
|
||||
return errors.E(op, err, errors.M(module), errors.V(version))
|
||||
}
|
||||
|
||||
// write the info file
|
||||
err = afero.WriteFile(s.filesystem, filepath.Join(dir, version+".info"), info, 0666)
|
||||
err = afero.WriteFile(s.filesystem, filepath.Join(dir, version+".info"), info, 0o666)
|
||||
if err != nil {
|
||||
return errors.E(op, err)
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@ import (
|
||||
)
|
||||
|
||||
// Catalog implements the (./pkg/storage).Catalog interface
|
||||
// It returns a list of versions, if any, for a given module
|
||||
// It returns a list of versions, if any, for a given module.
|
||||
func (s *Storage) Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error) {
|
||||
const op errors.Op = "gcp.Catalog"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
)
|
||||
|
||||
// Exists implements the (./pkg/storage).Checker interface
|
||||
// returning true if the module at version exists in storage
|
||||
// returning true if the module at version exists in storage.
|
||||
func (s *Storage) Exists(ctx context.Context, module, version string) (bool, error) {
|
||||
const op errors.Op = "gcp.Exists"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
@@ -21,7 +21,7 @@ func (s *Storage) Exists(ctx context.Context, module, version string) (bool, err
|
||||
var count int
|
||||
for {
|
||||
attrs, err := it.Next()
|
||||
if err == iterator.Done {
|
||||
if errors.IsErr(err, iterator.Done) {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
"google.golang.org/api/option"
|
||||
)
|
||||
|
||||
// Storage implements the (./pkg/storage).Backend interface
|
||||
// Storage implements the (./pkg/storage).Backend interface.
|
||||
type Storage struct {
|
||||
bucket *storage.BucketHandle
|
||||
timeout time.Duration
|
||||
@@ -34,8 +34,8 @@ func New(ctx context.Context, gcpConf *config.GCPConfig, timeout time.Duration)
|
||||
return nil, errors.E(op, err)
|
||||
}
|
||||
|
||||
if _, err := s.bucket.Attrs(ctx); err != nil {
|
||||
if err == storage.ErrBucketNotExist {
|
||||
if _, err = s.bucket.Attrs(ctx); err != nil {
|
||||
if errors.IsErr(err, storage.ErrBucketNotExist) {
|
||||
return nil, errors.E(op, "You must manually create a storage bucket for Athens, see https://cloud.google.com/storage/docs/creating-buckets#storage-create-bucket-console")
|
||||
}
|
||||
return nil, errors.E(op, err)
|
||||
@@ -48,21 +48,21 @@ func New(ctx context.Context, gcpConf *config.GCPConfig, timeout time.Duration)
|
||||
// this is so that the unit tests can use this to create their own short-lived buckets.
|
||||
func newClient(ctx context.Context, gcpConf *config.GCPConfig, timeout time.Duration) (*Storage, error) {
|
||||
const op errors.Op = "gcp.newClient"
|
||||
opts := []option.ClientOption{}
|
||||
var opts []option.ClientOption
|
||||
if gcpConf.JSONKey != "" {
|
||||
key, err := base64.StdEncoding.DecodeString(gcpConf.JSONKey)
|
||||
if err != nil {
|
||||
return nil, errors.E(op, fmt.Errorf("could not decode base64 json key: %v", err))
|
||||
return nil, errors.E(op, fmt.Errorf("could not decode base64 json key: %w", err))
|
||||
}
|
||||
creds, err := google.CredentialsFromJSON(ctx, key, storage.ScopeReadWrite)
|
||||
if err != nil {
|
||||
return nil, errors.E(op, fmt.Errorf("could not get GCS credentials: %v", err))
|
||||
return nil, errors.E(op, fmt.Errorf("could not get GCS credentials: %w", err))
|
||||
}
|
||||
opts = append(opts, option.WithCredentials(creds))
|
||||
}
|
||||
s, err := storage.NewClient(ctx, opts...)
|
||||
if err != nil {
|
||||
return nil, errors.E(op, fmt.Errorf("could not create new storage client: %s", err))
|
||||
return nil, errors.E(op, fmt.Errorf("could not create new storage client: %w", err))
|
||||
}
|
||||
|
||||
return &Storage{
|
||||
|
||||
@@ -12,7 +12,7 @@ import (
|
||||
pkgstorage "github.com/gomods/athens/pkg/storage"
|
||||
)
|
||||
|
||||
// Info implements Getter
|
||||
// Info implements Getter.
|
||||
func (s *Storage) Info(ctx context.Context, module, version string) ([]byte, error) {
|
||||
const op errors.Op = "gcp.Info"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
@@ -22,14 +22,14 @@ func (s *Storage) Info(ctx context.Context, module, version string) ([]byte, err
|
||||
return nil, errors.E(op, err, getErrorKind(err), errors.M(module), errors.V(version))
|
||||
}
|
||||
infoBytes, err := io.ReadAll(infoReader)
|
||||
infoReader.Close()
|
||||
_ = infoReader.Close()
|
||||
if err != nil {
|
||||
return nil, errors.E(op, err, errors.M(module), errors.V(version))
|
||||
}
|
||||
return infoBytes, nil
|
||||
}
|
||||
|
||||
// GoMod implements Getter
|
||||
// GoMod implements Getter.
|
||||
func (s *Storage) GoMod(ctx context.Context, module, version string) ([]byte, error) {
|
||||
const op errors.Op = "gcp.GoMod"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
@@ -39,15 +39,15 @@ func (s *Storage) GoMod(ctx context.Context, module, version string) ([]byte, er
|
||||
return nil, errors.E(op, err, getErrorKind(err), errors.M(module), errors.V(version))
|
||||
}
|
||||
modBytes, err := io.ReadAll(modReader)
|
||||
modReader.Close()
|
||||
_ = modReader.Close()
|
||||
if err != nil {
|
||||
return nil, errors.E(op, fmt.Errorf("could not get new reader for mod file: %s", err), errors.M(module), errors.V(version))
|
||||
return nil, errors.E(op, fmt.Errorf("could not get new reader for mod file: %w", err), errors.M(module), errors.V(version))
|
||||
}
|
||||
|
||||
return modBytes, nil
|
||||
}
|
||||
|
||||
// Zip implements Getter
|
||||
// Zip implements Getter.
|
||||
func (s *Storage) Zip(ctx context.Context, module, version string) (pkgstorage.SizeReadCloser, error) {
|
||||
const op errors.Op = "gcp.Zip"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
@@ -56,11 +56,11 @@ func (s *Storage) Zip(ctx context.Context, module, version string) (pkgstorage.S
|
||||
if err != nil {
|
||||
return nil, errors.E(op, err, getErrorKind(err), errors.M(module), errors.V(version))
|
||||
}
|
||||
return pkgstorage.NewSizer(zipReader, zipReader.Size()), nil
|
||||
return pkgstorage.NewSizer(zipReader, zipReader.Attrs.Size), nil
|
||||
}
|
||||
|
||||
func getErrorKind(err error) int {
|
||||
if err == storage.ErrObjectNotExist {
|
||||
if errors.IsErr(err, storage.ErrObjectNotExist) {
|
||||
return errors.KindNotFound
|
||||
}
|
||||
return errors.KindUnexpected
|
||||
|
||||
@@ -10,8 +10,8 @@ import (
|
||||
"google.golang.org/api/iterator"
|
||||
)
|
||||
|
||||
// List implements the (./pkg/storage).Lister interface
|
||||
// It returns a list of versions, if any, for a given module
|
||||
// List implements the (./pkg/storage).Lister interface.
|
||||
// It returns a list of versions, if any, for a given module.
|
||||
func (s *Storage) List(ctx context.Context, module string) ([]string, error) {
|
||||
const op errors.Op = "gcp.List"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
@@ -22,7 +22,7 @@ func (s *Storage) List(ctx context.Context, module string) ([]string, error) {
|
||||
paths := []string{}
|
||||
for {
|
||||
attrs, err := it.Next()
|
||||
if err == iterator.Done {
|
||||
if errors.IsErr(err, iterator.Done) {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user