chore: lint code with golangci-lint (#1828)

* feat: add golangci-lint linting

* chore: fix linter issues

* feat: add linting into the workflow

* docs: update lint docs

* fix: cr suggestions

* fix: remove old formatting and vetting scripts

* fix: add docker make target

* fix: action go caching

* fix: depreciated actions checkout version

* fix: cr suggestion

* fix: cr suggestions

---------

Co-authored-by: Manu Gupta <manugupt1@gmail.com>
This commit is contained in:
Nicholas Wiersma
2023-02-25 06:39:17 +02:00
committed by GitHub
parent 66582eebfe
commit d932d50232
134 changed files with 596 additions and 600 deletions
+13 -1
View File
@@ -3,6 +3,18 @@ on:
- push - push
- pull_request - pull_request
jobs: jobs:
lint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Set up Go
uses: actions/setup-go@v3
with:
go-version-file: 'go.mod'
cache: true
- name: Lint code
run: make lint-docker
build: build:
env: env:
ATHENS_MONGO_STORAGE_URL: mongodb://localhost:27017 ATHENS_MONGO_STORAGE_URL: mongodb://localhost:27017
@@ -53,7 +65,7 @@ jobs:
with: with:
go-version-file: 'go.mod' go-version-file: 'go.mod'
cache: true cache: true
- name: Run linter - name: Verify changes
run: make verify run: make verify
- name: Unit tests - name: Unit tests
run: go test -v -race ./... run: go test -v -race ./...
+80
View File
@@ -0,0 +1,80 @@
run:
tests: false
timeout: 5m
linters-settings:
cyclop:
max-complexity: 12
skip-tests: true
gofumpt:
extra-rules: true
linters:
enable-all: true
disable:
- interfacer # deprecated
- scopelint # deprecated
- maligned # deprecated
- golint # deprecated
- structcheck # deprecated
- deadcode # deprecated
- varcheck # deprecated
- nosnakecase # deprecated
- ifshort # deprecated
- errchkjson
- exhaustive
- exhaustivestruct
- exhaustruct
- forcetypeassert
- funlen
- gochecknoglobals
- gochecknoinits
- goconst
- godox
- goerr113
- gomnd
- ireturn
- lll
- musttag
- nilnil
- nlreturn
- nonamedreturns
- tagliatelle
- varnamelen
- wrapcheck
- wsl
- cyclop # TODO: turn this back on later
- gocognit # TODO: turn this back on later
- forbidigo # TODO: turn this back on later
issues:
exclude-use-default: false
exclude:
- 'package-comments: should have a package comment'
- 'ST1000: at least one file in a package should have a package comment'
- 'G204: Subprocess launched with a potential tainted input or cmd arguments'
- 'G204: Subprocess launched with variable'
- 'G402: TLS MinVersion too low.'
- 'const `op` is unused'
exclude-rules:
- path: cmd/proxy/main.go
text: 'G108: Profiling endpoint is automatically exposed on /debug/pprof'
- path: pkg/stash/stasher.go
linters:
- contextcheck
- path: pkg/stash/with_azureblob.go # False positive
linters:
- bodyclose
- path: pkg/storage/azureblob/azureblob.go # False positive
linters:
- bodyclose
- path: pkg/storage/compliance/*
linters:
- thelper
- gosec
- errcheck
- path: pkg/index/compliance/*
linters:
- thelper
- gosec
- errcheck
+2 -2
View File
@@ -234,10 +234,10 @@ Then open [http://localhost:1313](http://localhost:1313/).
# Linting # Linting
In our CI/CD pass, we use govet, so feel free to run it locally beforehand: In our CI/CD pass, we use golangci-lint, so feel free to run it locally beforehand:
``` ```
go vet ./... make lint
``` ```
# For People Doing a Release # For People Doing a Release
+10 -2
View File
@@ -1,6 +1,8 @@
VERSION = "unset" VERSION = "unset"
DATE=$(shell date -u +%Y-%m-%d-%H:%M:%S-%Z) DATE=$(shell date -u +%Y-%m-%d-%H:%M:%S-%Z)
GOLANGCI_LINT_VERSION=v1.51.2
ifndef GOLANG_VERSION ifndef GOLANG_VERSION
override GOLANG_VERSION = 1.19 override GOLANG_VERSION = 1.19
endif endif
@@ -43,10 +45,16 @@ docs: ## build the docs docker image
setup-dev-env: setup-dev-env:
$(MAKE) dev $(MAKE) dev
.PHONY: lint
lint:
@golangci-lint run ./...
.PHONY: lint-docker
lint-docker:
@docker run -t --rm -v $(CURDIR):/app -w /app golangci/golangci-lint:$(GOLANGCI_LINT_VERSION) golangci-lint run ./...
.PHONY: verify .PHONY: verify
verify: ## verify athens codebase verify: ## verify athens codebase
./scripts/check_gofmt.sh
./scripts/check_govet.sh
./scripts/check_deps.sh ./scripts/check_deps.sh
./scripts/check_conflicts.sh ./scripts/check_conflicts.sh
+3 -3
View File
@@ -16,7 +16,7 @@ import (
"go.opencensus.io/plugin/ochttp" "go.opencensus.io/plugin/ochttp"
) )
// Service is the name of the service that we want to tag our processes with // Service is the name of the service that we want to tag our processes with.
const Service = "proxy" const Service = "proxy"
// App is where all routes and middleware for the proxy // App is where all routes and middleware for the proxy
@@ -126,7 +126,7 @@ func App(conf *config.Config) (http.Handler, error) {
store, err := GetStorage(conf.StorageType, conf.Storage, conf.TimeoutDuration(), client) store, err := GetStorage(conf.StorageType, conf.Storage, conf.TimeoutDuration(), client)
if err != nil { if err != nil {
err = fmt.Errorf("error getting storage configuration (%s)", err) err = fmt.Errorf("error getting storage configuration: %w", err)
return nil, err return nil, err
} }
@@ -140,7 +140,7 @@ func App(conf *config.Config) (http.Handler, error) {
lggr, lggr,
conf, conf,
); err != nil { ); err != nil {
err = fmt.Errorf("error adding proxy routes (%s)", err) err = fmt.Errorf("error adding proxy routes: %w", err)
return nil, err return nil, err
} }
+6 -5
View File
@@ -2,6 +2,7 @@ package actions
import ( import (
"context" "context"
"errors"
"fmt" "fmt"
"net/http" "net/http"
"net/url" "net/url"
@@ -54,7 +55,7 @@ func addProxyRoutes(
} }
supportPath := path.Join("/sumdb", sumdbURL.Host, "/supported") supportPath := path.Join("/sumdb", sumdbURL.Host, "/supported")
r.HandleFunc(supportPath, func(w http.ResponseWriter, r *http.Request) { r.HandleFunc(supportPath, func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(200) w.WriteHeader(http.StatusOK)
}) })
sumHandler := sumdbProxy(sumdbURL, c.NoSumPatterns) sumHandler := sumdbProxy(sumdbURL, c.NoSumPatterns)
pathPrefix := "/sumdb/" + sumdbURL.Host pathPrefix := "/sumdb/" + sumdbURL.Host
@@ -63,7 +64,7 @@ func addProxyRoutes(
) )
} }
// Download Protocol // Download Protocol:
// the download.Protocol and the stash.Stasher interfaces are composable // the download.Protocol and the stash.Stasher interfaces are composable
// in a middleware fashion. Therefore you can separate concerns // in a middleware fashion. Therefore you can separate concerns
// by the functionality: a download.Protocol that just takes care // by the functionality: a download.Protocol that just takes care
@@ -142,13 +143,13 @@ func getSingleFlight(l *log.Logger, c *config.Config, checker storage.Checker) (
return stash.WithSingleflight, nil return stash.WithSingleflight, nil
case "etcd": case "etcd":
if c.SingleFlight == nil || c.SingleFlight.Etcd == nil { if c.SingleFlight == nil || c.SingleFlight.Etcd == nil {
return nil, fmt.Errorf("Etcd config must be present") return nil, errors.New("etcd config must be present")
} }
endpoints := strings.Split(c.SingleFlight.Etcd.Endpoints, ",") endpoints := strings.Split(c.SingleFlight.Etcd.Endpoints, ",")
return stash.WithEtcd(endpoints, checker) return stash.WithEtcd(endpoints, checker)
case "redis": case "redis":
if c.SingleFlight == nil || c.SingleFlight.Redis == nil { if c.SingleFlight == nil || c.SingleFlight.Redis == nil {
return nil, fmt.Errorf("Redis config must be present") return nil, errors.New("redis config must be present")
} }
return stash.WithRedisLock( return stash.WithRedisLock(
&athensLoggerForRedis{logger: l}, &athensLoggerForRedis{logger: l},
@@ -158,7 +159,7 @@ func getSingleFlight(l *log.Logger, c *config.Config, checker storage.Checker) (
c.SingleFlight.Redis.LockConfig) c.SingleFlight.Redis.LockConfig)
case "redis-sentinel": case "redis-sentinel":
if c.SingleFlight == nil || c.SingleFlight.RedisSentinel == nil { if c.SingleFlight == nil || c.SingleFlight.RedisSentinel == nil {
return nil, fmt.Errorf("Redis config must be present") return nil, errors.New("redis config must be present")
} }
return stash.WithRedisSentinelLock( return stash.WithRedisSentinelLock(
&athensLoggerForRedis{logger: l}, &athensLoggerForRedis{logger: l},
+3 -3
View File
@@ -19,7 +19,7 @@ func initializeAuthFile(path string) {
return return
} }
fileBts, err := os.ReadFile(path) fileBts, err := os.ReadFile(filepath.Clean(path))
if err != nil { if err != nil {
log.Fatalf("could not read %s: %v", path, err) log.Fatalf("could not read %s: %v", path, err)
} }
@@ -31,7 +31,7 @@ func initializeAuthFile(path string) {
fileName := transformAuthFileName(filepath.Base(path)) fileName := transformAuthFileName(filepath.Base(path))
rcp := filepath.Join(hdir, fileName) rcp := filepath.Join(hdir, fileName)
if err := os.WriteFile(rcp, fileBts, 0600); err != nil { if err := os.WriteFile(rcp, fileBts, 0o600); err != nil {
log.Fatalf("could not write to file: %v", err) log.Fatalf("could not write to file: %v", err)
} }
} }
@@ -45,7 +45,7 @@ func netrcFromToken(tok string) {
log.Fatalf("netrcFromToken: could not get homedir: %v", err) log.Fatalf("netrcFromToken: could not get homedir: %v", err)
} }
rcp := filepath.Join(hdir, getNETRCFilename()) rcp := filepath.Join(hdir, getNETRCFilename())
if err := os.WriteFile(rcp, []byte(fileContent), 0600); err != nil { if err := os.WriteFile(rcp, []byte(fileContent), 0o600); err != nil {
log.Fatalf("netrcFromToken: could not write to file: %v", err) log.Fatalf("netrcFromToken: could not write to file: %v", err)
} }
} }
+3 -9
View File
@@ -8,10 +8,8 @@ import (
"github.com/gorilla/mux" "github.com/gorilla/mux"
) )
var ( // basicAuthExcludedPaths is a regular expression that matches paths that should not be protected by HTTP basic authentication.
// basicAuthExcludedPaths is a regular expression that matches paths that should not be protected by HTTP basic authentication. var basicAuthExcludedPaths = regexp.MustCompile("^/(health|ready)z$")
basicAuthExcludedPaths = regexp.MustCompile("^/(health|ready)z$")
)
func basicAuth(user, pass string) mux.MiddlewareFunc { func basicAuth(user, pass string) mux.MiddlewareFunc {
return func(h http.Handler) http.Handler { return func(h http.Handler) http.Handler {
@@ -40,9 +38,5 @@ func checkAuth(r *http.Request, user, pass string) bool {
} }
isPass := subtle.ConstantTimeCompare([]byte(pass), []byte(givenPass)) isPass := subtle.ConstantTimeCompare([]byte(pass), []byte(givenPass))
if isPass != 1 { return isPass == 1
return false
}
return true
} }
+2 -2
View File
@@ -18,7 +18,7 @@ type catalogRes struct {
NextPageToken string `json:"next,omitempty"` NextPageToken string `json:"next,omitempty"`
} }
// catalogHandler implements GET baseURL/catalog // catalogHandler implements GET baseURL/catalog.
func catalogHandler(s storage.Backend) http.HandlerFunc { func catalogHandler(s storage.Backend) http.HandlerFunc {
const op errors.Op = "actions.CatalogHandler" const op errors.Op = "actions.CatalogHandler"
cs, isCataloger := s.(storage.Cataloger) cs, isCataloger := s.(storage.Cataloger)
@@ -54,7 +54,7 @@ func catalogHandler(s storage.Backend) http.HandlerFunc {
} }
// getLimitFromParam converts a URL query parameter into an int // getLimitFromParam converts a URL query parameter into an int
// otherwise converts defaultPageSize constant // otherwise converts defaultPageSize constant.
func getLimitFromParam(param string) (int, error) { func getLimitFromParam(param string) (int, error) {
if param == "" { if param == "" {
return defaultPageSize, nil return defaultPageSize, nil
+1 -1
View File
@@ -5,5 +5,5 @@ import (
) )
func proxyHomeHandler(w http.ResponseWriter, r *http.Request) { func proxyHomeHandler(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(`"Welcome to The Athens Proxy"`)) _, _ = w.Write([]byte(`"Welcome to The Athens Proxy"`))
} }
+1 -1
View File
@@ -13,7 +13,7 @@ import (
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
) )
// indexHandler implements GET baseURL/index // indexHandler implements GET baseURL/index.
func indexHandler(index index.Indexer) http.HandlerFunc { func indexHandler(index index.Indexer) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) { return func(w http.ResponseWriter, r *http.Request) {
ctx := r.Context() ctx := r.Context()
+1 -1
View File
@@ -6,7 +6,7 @@ import (
"github.com/gomods/athens/pkg/config" "github.com/gomods/athens/pkg/config"
) )
// robotsHandler implements GET baseURL/robots.txt // robotsHandler implements GET baseURL/robots.txt.
func robotsHandler(c *config.Config) http.HandlerFunc { func robotsHandler(c *config.Config) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) { return func(w http.ResponseWriter, r *http.Request) {
http.ServeFile(w, r, c.RobotsFile) http.ServeFile(w, r, c.RobotsFile)
+1 -1
View File
@@ -21,7 +21,7 @@ import (
"github.com/spf13/afero" "github.com/spf13/afero"
) )
// GetStorage returns storage backend based on env configuration // GetStorage returns storage backend based on env configuration.
func GetStorage(storageType string, storageConfig *config.Storage, timeout time.Duration, client *http.Client) (storage.Backend, error) { func GetStorage(storageType string, storageConfig *config.Storage, timeout time.Duration, client *http.Client) (storage.Backend, error) {
const op errors.Op = "actions.GetStorage" const op errors.Op = "actions.GetStorage"
switch storageType { switch storageType {
+2 -2
View File
@@ -17,12 +17,12 @@ func sumdbProxy(url *url.URL, nosumPatterns []string) http.Handler {
req.URL.Host = url.Host req.URL.Host = url.Host
} }
if len(nosumPatterns) > 0 { if len(nosumPatterns) > 0 {
return noSumWrapper(rp, url.Host, nosumPatterns) return noSumWrapper(rp, nosumPatterns)
} }
return rp return rp
} }
func noSumWrapper(h http.Handler, host string, patterns []string) http.Handler { func noSumWrapper(h http.Handler, patterns []string) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if strings.HasPrefix(r.URL.Path, "/lookup/") { if strings.HasPrefix(r.URL.Path, "/lookup/") {
for _, p := range patterns { for _, p := range patterns {
+1 -1
View File
@@ -97,7 +97,7 @@ func TestNoSumPatterns(t *testing.T) {
for _, tc := range noSumTestCases { for _, tc := range noSumTestCases {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
w := httptest.NewRecorder() w := httptest.NewRecorder()
skipHandler := noSumWrapper(http.HandlerFunc(emptyHandler), "sum.golang.org", tc.patterns) skipHandler := noSumWrapper(http.HandlerFunc(emptyHandler), tc.patterns)
req := httptest.NewRequest("GET", "/lookup/"+tc.given, nil) req := httptest.NewRequest("GET", "/lookup/"+tc.given, nil)
skipHandler.ServeHTTP(w, req) skipHandler.ServeHTTP(w, req)
if tc.status != w.Code { if tc.status != w.Code {
+1 -1
View File
@@ -8,5 +8,5 @@ import (
) )
func versionHandler(w http.ResponseWriter, r *http.Request) { func versionHandler(w http.ResponseWriter, r *http.Request) {
json.NewEncoder(w).Encode(build.Data()) _ = json.NewEncoder(w).Encode(build.Data())
} }
+8 -7
View File
@@ -2,16 +2,16 @@ package main
import ( import (
"context" "context"
"errors"
"flag" "flag"
"fmt" "fmt"
"log" "log"
"net/http" "net/http"
_ "net/http/pprof"
"os" "os"
"os/signal" "os/signal"
"time" "time"
_ "net/http/pprof"
"github.com/gomods/athens/cmd/proxy/actions" "github.com/gomods/athens/cmd/proxy/actions"
"github.com/gomods/athens/internal/shutdown" "github.com/gomods/athens/internal/shutdown"
"github.com/gomods/athens/pkg/build" "github.com/gomods/athens/pkg/build"
@@ -45,8 +45,9 @@ func main() {
} }
srv := &http.Server{ srv := &http.Server{
Addr: conf.Port, Addr: conf.Port,
Handler: handler, Handler: handler,
ReadHeaderTimeout: 2 * time.Second,
} }
idleConnsClosed := make(chan struct{}) idleConnsClosed := make(chan struct{})
@@ -54,7 +55,7 @@ func main() {
sigint := make(chan os.Signal, 1) sigint := make(chan os.Signal, 1)
signal.Notify(sigint, shutdown.GetSignals()...) signal.Notify(sigint, shutdown.GetSignals()...)
s := <-sigint s := <-sigint
log.Printf("Recived signal (%s): Shutting down server", s) log.Printf("Received signal (%s): Shutting down server", s)
// We received an interrupt signal, shut down. // We received an interrupt signal, shut down.
ctx, cancel := context.WithTimeout(context.Background(), time.Second*time.Duration(conf.ShutdownTimeout)) ctx, cancel := context.WithTimeout(context.Background(), time.Second*time.Duration(conf.ShutdownTimeout))
@@ -70,7 +71,7 @@ func main() {
// pprof to be exposed on a different port than the application for security matters, not to expose profiling data and avoid DoS attacks (profiling slows down the service) // pprof to be exposed on a different port than the application for security matters, not to expose profiling data and avoid DoS attacks (profiling slows down the service)
// https://www.farsightsecurity.com/txt-record/2016/10/28/cmikk-go-remote-profiling/ // https://www.farsightsecurity.com/txt-record/2016/10/28/cmikk-go-remote-profiling/
log.Printf("Starting `pprof` at port %v", conf.PprofPort) log.Printf("Starting `pprof` at port %v", conf.PprofPort)
log.Fatal(http.ListenAndServe(conf.PprofPort, nil)) log.Fatal(http.ListenAndServe(conf.PprofPort, nil)) //nolint:gosec // This should not be exposed to the world.
}() }()
} }
@@ -81,7 +82,7 @@ func main() {
err = srv.ListenAndServe() err = srv.ListenAndServe()
} }
if err != http.ErrServerClosed { if !errors.Is(err, http.ErrServerClosed) {
log.Fatal(err) log.Fatal(err)
} }
-2
View File
@@ -79,8 +79,6 @@ if ($docs.IsPresent) {
} }
if ($verify.IsPresent) { if ($verify.IsPresent) {
execScript "check_gofmt.ps1"
execScript "check_govet.ps1"
execScript "check_deps.ps1" execScript "check_deps.ps1"
} }
+2 -2
View File
@@ -9,7 +9,7 @@ import (
"fmt" "fmt"
) )
// Details represents known data for a given build // Details represents known data for a given build.
type Details struct { type Details struct {
Version string `json:"version,omitempty"` Version string `json:"version,omitempty"`
Date string `json:"date,omitempty"` Date string `json:"date,omitempty"`
@@ -29,7 +29,7 @@ func String() string {
return fmt.Sprintf("Build Details:\n\tVersion:\t%s\n\tDate:\t\t%s", version, buildDate) return fmt.Sprintf("Build Details:\n\tVersion:\t%s\n\tDate:\t\t%s", version, buildDate)
} }
// Data returns build details as a struct // Data returns build details as a struct.
func Data() Details { func Data() Details {
return Details{ return Details{
Version: version, Version: version,
+1 -1
View File
@@ -1,6 +1,6 @@
package config package config
// AzureBlobConfig specifies the properties required to use Azure as the storage backend // AzureBlobConfig specifies the properties required to use Azure as the storage backend.
type AzureBlobConfig struct { type AzureBlobConfig struct {
AccountName string `validate:"required" envconfig:"ATHENS_AZURE_ACCOUNT_NAME"` AccountName string `validate:"required" envconfig:"ATHENS_AZURE_ACCOUNT_NAME"`
AccountKey string `validate:"required" envconfig:"ATHENS_AZURE_ACCOUNT_KEY"` AccountKey string `validate:"required" envconfig:"ATHENS_AZURE_ACCOUNT_KEY"`
+18 -20
View File
@@ -18,7 +18,7 @@ import (
const defaultConfigFile = "athens.toml" const defaultConfigFile = "athens.toml"
// Config provides configuration values for all components // Config provides configuration values for all components.
type Config struct { type Config struct {
TimeoutConf TimeoutConf
GoEnv string `validate:"required" envconfig:"GO_ENV"` GoEnv string `validate:"required" envconfig:"GO_ENV"`
@@ -63,12 +63,12 @@ type Config struct {
} }
// EnvList is a list of key-value environment // EnvList is a list of key-value environment
// variables that are passed to the Go command // variables that are passed to the Go command.
type EnvList []string type EnvList []string
// HasKey returns whether a key-value entry // HasKey returns whether a key-value entry
// is present by only checking the left of // is present by only checking the left of
// key=value // key=value.
func (el EnvList) HasKey(key string) bool { func (el EnvList) HasKey(key string) bool {
for _, env := range el { for _, env := range el {
if strings.HasPrefix(env, key+"=") { if strings.HasPrefix(env, key+"=") {
@@ -79,7 +79,7 @@ func (el EnvList) HasKey(key string) bool {
} }
// Add adds a key=value entry to the environment // Add adds a key=value entry to the environment
// list // list.
func (el *EnvList) Add(key, value string) { func (el *EnvList) Add(key, value string) {
*el = append(*el, key+"="+value) *el = append(*el, key+"="+value)
} }
@@ -110,7 +110,7 @@ func (el *EnvList) Decode(value string) error {
} }
// Validate validates that all strings inside the // Validate validates that all strings inside the
// list are of the key=value format // list are of the key=value format.
func (el EnvList) Validate() error { func (el EnvList) Validate() error {
const op errors.Op = "EnvList.Validate" const op errors.Op = "EnvList.Validate"
for _, env := range el { for _, env := range el {
@@ -123,7 +123,7 @@ func (el EnvList) Validate() error {
} }
// Load loads the config from a file. // Load loads the config from a file.
// If file is not present returns default config // If file is not present returns default config.
func Load(configFile string) (*Config, error) { func Load(configFile string) (*Config, error) {
// User explicitly specified a config file // User explicitly specified a config file
if configFile != "" { if configFile != "" {
@@ -206,7 +206,7 @@ func defaultConfig() *Config {
} }
// BasicAuth returns BasicAuthUser and BasicAuthPassword // BasicAuth returns BasicAuthUser and BasicAuthPassword
// and ok if neither of them are empty // and ok if neither of them are empty.
func (c *Config) BasicAuth() (user, pass string, ok bool) { func (c *Config) BasicAuth() (user, pass string, ok bool) {
user = c.BasicAuthUser user = c.BasicAuthUser
pass = c.BasicAuthPass pass = c.BasicAuthPass
@@ -215,7 +215,7 @@ func (c *Config) BasicAuth() (user, pass string, ok bool) {
} }
// TLSCertFiles returns certificate and key files and an error if // TLSCertFiles returns certificate and key files and an error if
// both files doesn't exist and have approperiate file permissions // both files doesn't exist and have approperiate file permissions.
func (c *Config) TLSCertFiles() (cert, key string, err error) { func (c *Config) TLSCertFiles() (cert, key string, err error) {
if c.TLSCertFile == "" && c.TLSKeyFile == "" { if c.TLSCertFile == "" && c.TLSKeyFile == "" {
return "", "", nil return "", "", nil
@@ -223,29 +223,28 @@ func (c *Config) TLSCertFiles() (cert, key string, err error) {
certFile, err := os.Stat(c.TLSCertFile) certFile, err := os.Stat(c.TLSCertFile)
if err != nil { if err != nil {
return "", "", fmt.Errorf("Could not access TLSCertFile: %v", err) return "", "", fmt.Errorf("could not access TLSCertFile: %w", err)
} }
keyFile, err := os.Stat(c.TLSKeyFile) keyFile, err := os.Stat(c.TLSKeyFile)
if err != nil { if err != nil {
return "", "", fmt.Errorf("Could not access TLSKeyFile: %v", err) return "", "", fmt.Errorf("could not access TLSKeyFile: %w", err)
} }
if keyFile.Mode()&077 != 0 && runtime.GOOS != "windows" { if keyFile.Mode()&0o077 != 0 && runtime.GOOS != "windows" {
return "", "", fmt.Errorf("TLSKeyFile should not be accessible by others") return "", "", fmt.Errorf("TLSKeyFile should not be accessible by others")
} }
return certFile.Name(), keyFile.Name(), nil return certFile.Name(), keyFile.Name(), nil
} }
// FilterOff returns true if the FilterFile is empty // FilterOff returns true if the FilterFile is empty.
func (c *Config) FilterOff() bool { func (c *Config) FilterOff() bool {
return c.FilterFile == "" return c.FilterFile == ""
} }
// ParseConfigFile parses the given file into an athens config struct // ParseConfigFile parses the given file into an athens config struct.
func ParseConfigFile(configFile string) (*Config, error) { func ParseConfigFile(configFile string) (*Config, error) {
var config Config var config Config
// attempt to read the given config file // attempt to read the given config file
if _, err := toml.DecodeFile(configFile, &config); err != nil { if _, err := toml.DecodeFile(configFile, &config); err != nil {
@@ -271,7 +270,7 @@ func ParseConfigFile(configFile string) (*Config, error) {
return &config, nil return &config, nil
} }
// envOverride uses Environment variables to override unspecified properties // envOverride uses Environment variables to override unspecified properties.
func envOverride(config *Config) error { func envOverride(config *Config) error {
const defaultPort = ":3000" const defaultPort = ":3000"
err := envconfig.Process("athens", config) err := envconfig.Process("athens", config)
@@ -355,16 +354,16 @@ func validateIndex(validate *validator.Validate, indexType string, config *Index
func GetConf(path string) (*Config, error) { func GetConf(path string) (*Config, error) {
absPath, err := filepath.Abs(path) absPath, err := filepath.Abs(path)
if err != nil { if err != nil {
return nil, fmt.Errorf("Unable to construct absolute path to test config file") return nil, fmt.Errorf("unable to construct absolute path to test config file")
} }
conf, err := ParseConfigFile(absPath) conf, err := ParseConfigFile(absPath)
if err != nil { if err != nil {
return nil, fmt.Errorf("Unable to parse test config file: %s", err.Error()) return nil, fmt.Errorf("unable to parse test config file: %w", err)
} }
return conf, nil return conf, nil
} }
// checkFilePerms given a list of files // checkFilePerms given a list of files.
func checkFilePerms(files ...string) error { func checkFilePerms(files ...string) error {
const op = "config.checkFilePerms" const op = "config.checkFilePerms"
@@ -384,10 +383,9 @@ func checkFilePerms(files ...string) error {
// Assume unix based system (MacOS and Linux) // Assume unix based system (MacOS and Linux)
// the bit mask is calculated using the umask command which tells which permissions // the bit mask is calculated using the umask command which tells which permissions
// should not be allowed for a particular user, group or world // should not be allowed for a particular user, group or world
if fInfo.Mode()&0077 != 0 && runtime.GOOS != "windows" { if fInfo.Mode()&0o077 != 0 && runtime.GOOS != "windows" {
return errors.E(op, f+" should have at most rwx,-, - (bit mask 077) as permission") return errors.E(op, f+" should have at most rwx,-, - (bit mask 077) as permission")
} }
} }
return nil return nil
+1 -1
View File
@@ -1,6 +1,6 @@
package config package config
// DiskConfig specifies the properties required to use Disk as the storage backend // DiskConfig specifies the properties required to use Disk as the storage backend.
type DiskConfig struct { type DiskConfig struct {
RootPath string `validate:"required" envconfig:"ATHENS_DISK_STORAGE_ROOT"` RootPath string `validate:"required" envconfig:"ATHENS_DISK_STORAGE_ROOT"`
} }
+1 -1
View File
@@ -1,6 +1,6 @@
package config package config
// External specifies configuration for an external http storage // External specifies configuration for an external http storage.
type External struct { type External struct {
URL string `validate:"required" envconfig:"ATHENS_EXTERNAL_STORAGE_URL"` URL string `validate:"required" envconfig:"ATHENS_EXTERNAL_STORAGE_URL"`
} }
+1 -1
View File
@@ -1,6 +1,6 @@
package config package config
// GCPConfig specifies the properties required to use GCP as the storage backend // GCPConfig specifies the properties required to use GCP as the storage backend.
type GCPConfig struct { type GCPConfig struct {
ProjectID string `envconfig:"GOOGLE_CLOUD_PROJECT"` ProjectID string `envconfig:"GOOGLE_CLOUD_PROJECT"`
Bucket string `validate:"required" envconfig:"ATHENS_STORAGE_GCP_BUCKET"` Bucket string `validate:"required" envconfig:"ATHENS_STORAGE_GCP_BUCKET"`
+1 -1
View File
@@ -1,6 +1,6 @@
package config package config
// Index is the config for various index storage backends // Index is the config for various index storage backends.
type Index struct { type Index struct {
MySQL *MySQL MySQL *MySQL
Postgres *Postgres Postgres *Postgres
+1 -1
View File
@@ -1,7 +1,7 @@
package config package config
// MinioConfig specifies the properties required to use Minio or DigitalOcean Spaces // MinioConfig specifies the properties required to use Minio or DigitalOcean Spaces
// as the storage backend // as the storage backend.
type MinioConfig struct { type MinioConfig struct {
Endpoint string `validate:"required" envconfig:"ATHENS_MINIO_ENDPOINT"` Endpoint string `validate:"required" envconfig:"ATHENS_MINIO_ENDPOINT"`
Key string `validate:"required" envconfig:"ATHENS_MINIO_ACCESS_KEY_ID"` Key string `validate:"required" envconfig:"ATHENS_MINIO_ACCESS_KEY_ID"`
+4 -4
View File
@@ -7,20 +7,20 @@ import (
) )
// PackageVersionedName return package full name used in storage. // PackageVersionedName return package full name used in storage.
// E.g athens/@v/v1.0.mod // E.g athens/@v/v1.0.mod.
func PackageVersionedName(module, version, ext string) string { func PackageVersionedName(module, version, ext string) string {
return fmt.Sprintf("%s/@v/%s.%s", module, version, ext) return fmt.Sprintf("%s/@v/%s.%s", module, version, ext)
} }
// FmtModVer is a helper function that can take // FmtModVer is a helper function that can take
// pkg/a/b and v2.3.1 and returns pkg/a/b@v2.3.1 // pkg/a/b and v2.3.1 and returns pkg/a/b@v2.3.1.
func FmtModVer(mod, ver string) string { func FmtModVer(mod, ver string) string {
return fmt.Sprintf("%s@%s", mod, ver) return fmt.Sprintf("%s@%s", mod, ver)
} }
// ModuleVersionFromPath returns module and version from a // ModuleVersionFromPath returns module and version from a
// storage path // storage path.
// E.g athens/@v/v1.0.info -> athens and v.1.0 // E.g athens/@v/v1.0.info -> athens and v.1.0.
func ModuleVersionFromPath(path string) (string, string) { func ModuleVersionFromPath(path string) (string, string) {
segments := strings.Split(path, "/@v/") segments := strings.Split(path, "/@v/")
if len(segments) != 2 { if len(segments) != 2 {
+1 -1
View File
@@ -1,6 +1,6 @@
package config package config
// MongoConfig specifies the properties required to use MongoDB as the storage backend // MongoConfig specifies the properties required to use MongoDB as the storage backend.
type MongoConfig struct { type MongoConfig struct {
URL string `validate:"required" envconfig:"ATHENS_MONGO_STORAGE_URL"` URL string `validate:"required" envconfig:"ATHENS_MONGO_STORAGE_URL"`
DefaultDBName string `envconfig:"ATHENS_MONGO_DEFAULT_DATABASE" default:"athens"` DefaultDBName string `envconfig:"ATHENS_MONGO_DEFAULT_DATABASE" default:"athens"`
+1 -1
View File
@@ -1,6 +1,6 @@
package config package config
// MySQL config // MySQL config.
type MySQL struct { type MySQL struct {
Protocol string `validate:"required" envconfig:"ATHENS_INDEX_MYSQL_PROTOCOL"` Protocol string `validate:"required" envconfig:"ATHENS_INDEX_MYSQL_PROTOCOL"`
Host string `validate:"required" envconfig:"ATHENS_INDEX_MYSQL_HOST"` Host string `validate:"required" envconfig:"ATHENS_INDEX_MYSQL_HOST"`
+1 -1
View File
@@ -1,6 +1,6 @@
package config package config
// Postgres config // Postgres config.
type Postgres struct { type Postgres struct {
Host string `validate:"required" envconfig:"ATHENS_INDEX_POSTGRES_HOST"` Host string `validate:"required" envconfig:"ATHENS_INDEX_POSTGRES_HOST"`
Port int `validate:"required" envconfig:"ATHENS_INDEX_POSTGRES_PORT"` Port int `validate:"required" envconfig:"ATHENS_INDEX_POSTGRES_PORT"`
+1 -1
View File
@@ -1,6 +1,6 @@
package config package config
// S3Config specifies the properties required to use S3 as the storage backend // S3Config specifies the properties required to use S3 as the storage backend.
type S3Config struct { type S3Config struct {
Region string `validate:"required" envconfig:"AWS_REGION"` Region string `validate:"required" envconfig:"AWS_REGION"`
Key string `envconfig:"AWS_ACCESS_KEY_ID"` Key string `envconfig:"AWS_ACCESS_KEY_ID"`
+3 -1
View File
@@ -25,7 +25,7 @@ type Redis struct {
} }
// RedisSentinel is the configuration for using redis with sentinel // RedisSentinel is the configuration for using redis with sentinel
// for SingleFlight // for SingleFlight.
type RedisSentinel struct { type RedisSentinel struct {
Endpoints []string `envconfig:"ATHENS_REDIS_SENTINEL_ENDPOINTS"` Endpoints []string `envconfig:"ATHENS_REDIS_SENTINEL_ENDPOINTS"`
MasterName string `envconfig:"ATHENS_REDIS_SENTINEL_MASTER_NAME"` MasterName string `envconfig:"ATHENS_REDIS_SENTINEL_MASTER_NAME"`
@@ -33,12 +33,14 @@ type RedisSentinel struct {
LockConfig *RedisLockConfig LockConfig *RedisLockConfig
} }
// RedisLockConfig is the configuration for redis locking.
type RedisLockConfig struct { type RedisLockConfig struct {
Timeout int `envconfig:"ATHENS_REDIS_LOCK_TIMEOUT"` Timeout int `envconfig:"ATHENS_REDIS_LOCK_TIMEOUT"`
TTL int `envconfig:"ATHENS_REDIS_LOCK_TTL"` TTL int `envconfig:"ATHENS_REDIS_LOCK_TTL"`
MaxRetries int `envconfig:"ATHENS_REDIS_LOCK_MAX_RETRIES"` MaxRetries int `envconfig:"ATHENS_REDIS_LOCK_MAX_RETRIES"`
} }
// DefaultRedisLockConfig returns the default redis locking configuration.
func DefaultRedisLockConfig() *RedisLockConfig { func DefaultRedisLockConfig() *RedisLockConfig {
return &RedisLockConfig{ return &RedisLockConfig{
TTL: 900, TTL: 900,
+1 -1
View File
@@ -1,6 +1,6 @@
package config package config
// Storage provides configs for various storage backends // Storage provides configs for various storage backends.
type Storage struct { type Storage struct {
Disk *DiskConfig Disk *DiskConfig
GCP *GCPConfig GCP *GCPConfig
+3 -3
View File
@@ -2,17 +2,17 @@ package config
import "time" import "time"
// TimeoutConf is a common struct for anything with a timeout // TimeoutConf is a common struct for anything with a timeout.
type TimeoutConf struct { type TimeoutConf struct {
Timeout int `validate:"required"` Timeout int `validate:"required"`
} }
// TimeoutDuration returns the timeout as time.duration // TimeoutDuration returns the timeout as time.duration.
func (t *TimeoutConf) TimeoutDuration() time.Duration { func (t *TimeoutConf) TimeoutDuration() time.Duration {
return GetTimeoutDuration(t.Timeout) return GetTimeoutDuration(t.Timeout)
} }
// GetTimeoutDuration returns the timeout as time.duration // GetTimeoutDuration returns the timeout as time.duration.
func GetTimeoutDuration(timeout int) time.Duration { func GetTimeoutDuration(timeout int) time.Duration {
return time.Second * time.Duration(timeout) return time.Second * time.Duration(timeout)
} }
+1 -1
View File
@@ -7,7 +7,7 @@ import (
"github.com/gomods/athens/pkg/paths" "github.com/gomods/athens/pkg/paths"
) )
func getModuleParams(r *http.Request, op errors.Op) (mod string, ver string, err error) { func getModuleParams(r *http.Request, op errors.Op) (mod, ver string, err error) {
params, err := paths.GetAllParams(r) params, err := paths.GetAllParams(r)
if err != nil { if err != nil {
return "", "", errors.E(op, err, errors.KindBadRequest) return "", "", errors.E(op, err, errors.KindBadRequest)
+2 -3
View File
@@ -15,8 +15,7 @@ import (
// a ready-to-go http handler that serves up cmd/go's download protocol. // a ready-to-go http handler that serves up cmd/go's download protocol.
type ProtocolHandler func(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler type ProtocolHandler func(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler
// HandlerOpts are the generic options // HandlerOpts are the generic options for a ProtocolHandler.
// for a ProtocolHandler
type HandlerOpts struct { type HandlerOpts struct {
Protocol Protocol Protocol Protocol
Logger *log.Logger Logger *log.Logger
@@ -26,7 +25,7 @@ type HandlerOpts struct {
// LogEntryHandler pulls a log entry from the request context. Thanks to the // LogEntryHandler pulls a log entry from the request context. Thanks to the
// LogEntryMiddleware, we should have a log entry stored in the context for each // LogEntryMiddleware, we should have a log entry stored in the context for each
// request with request-specific fields. This will grab the entry and pass it to // request with request-specific fields. This will grab the entry and pass it to
// the protocol handlers // the protocol handlers.
func LogEntryHandler(ph ProtocolHandler, opts *HandlerOpts) http.Handler { func LogEntryHandler(ph ProtocolHandler, opts *HandlerOpts) http.Handler {
f := func(w http.ResponseWriter, r *http.Request) { f := func(w http.ResponseWriter, r *http.Request) {
ent := log.EntryFromContext(r.Context()) ent := log.EntryFromContext(r.Context())
+1 -1
View File
@@ -13,7 +13,7 @@ import (
// PathLatest URL. // PathLatest URL.
const PathLatest = "/{module:.+}/@latest" const PathLatest = "/{module:.+}/@latest"
// LatestHandler implements GET baseURL/module/@latest // LatestHandler implements GET baseURL/module/@latest.
func LatestHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler { func LatestHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler {
const op errors.Op = "download.LatestHandler" const op errors.Op = "download.LatestHandler"
f := func(w http.ResponseWriter, r *http.Request) { f := func(w http.ResponseWriter, r *http.Request) {
+1 -1
View File
@@ -14,7 +14,7 @@ import (
// PathList URL. // PathList URL.
const PathList = "/{module:.+}/@v/list" const PathList = "/{module:.+}/@v/list"
// ListHandler implements GET baseURL/module/@v/list // ListHandler implements GET baseURL/module/@v/list.
func ListHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler { func ListHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler {
const op errors.Op = "download.ListHandler" const op errors.Op = "download.ListHandler"
f := func(w http.ResponseWriter, r *http.Request) { f := func(w http.ResponseWriter, r *http.Request) {
+3 -2
View File
@@ -4,6 +4,7 @@ import (
"encoding/base64" "encoding/base64"
"fmt" "fmt"
"os" "os"
"path/filepath"
"strings" "strings"
"github.com/gomods/athens/pkg/errors" "github.com/gomods/athens/pkg/errors"
@@ -16,7 +17,7 @@ import (
// when a module is not found in storage. // when a module is not found in storage.
type Mode string type Mode string
// DownloadMode constants. For more information see config.dev.toml // DownloadMode constants. For more information see config.dev.toml.
const ( const (
Sync Mode = "sync" Sync Mode = "sync"
Async Mode = "async" Async Mode = "async"
@@ -59,7 +60,7 @@ func NewFile(m Mode, downloadURL string) (*DownloadFile, error) {
if strings.HasPrefix(string(m), "file:") { if strings.HasPrefix(string(m), "file:") {
filePath := string(m[5:]) filePath := string(m[5:])
bts, err := os.ReadFile(filePath) bts, err := os.ReadFile(filepath.Clean(filePath))
if err != nil { if err != nil {
return nil, err return nil, err
} }
+6 -6
View File
@@ -48,7 +48,7 @@ type Opts struct {
NetworkMode string NetworkMode string
} }
// NetworkMode constants // NetworkMode constants.
const ( const (
Strict = "strict" Strict = "strict"
Offline = "offline" Offline = "offline"
@@ -263,12 +263,12 @@ func (p *protocol) processDownload(ctx context.Context, mod, ver string, f func(
} }
return f(newVer) return f(newVer)
case mode.Async: case mode.Async:
go p.stasher.Stash(ctx, mod, ver) go func() { _, _ = p.stasher.Stash(ctx, mod, ver) }()
return errors.E(op, "async: module not found", errors.KindNotFound) return errors.E(op, "async: module not found", errors.KindNotFound)
case mode.Redirect: case mode.Redirect:
return errors.E(op, "redirect", errors.KindRedirect) return errors.E(op, "redirect", errors.KindRedirect)
case mode.AsyncRedirect: case mode.AsyncRedirect:
go p.stasher.Stash(ctx, mod, ver) go func() { _, _ = p.stasher.Stash(ctx, mod, ver) }()
return errors.E(op, "async_redirect: module not found", errors.KindRedirect) return errors.E(op, "async_redirect: module not found", errors.KindRedirect)
case mode.None: case mode.None:
return errors.E(op, "none", errors.KindNotFound) return errors.E(op, "none", errors.KindNotFound)
@@ -276,7 +276,7 @@ func (p *protocol) processDownload(ctx context.Context, mod, ver string, f func(
return nil return nil
} }
// union concatenates two version lists and removes duplicates // union concatenates two version lists and removes duplicates.
func union(list1, list2 []string) []string { func union(list1, list2 []string) []string {
if list1 == nil { if list1 == nil {
list1 = []string{} list1 = []string{}
@@ -284,10 +284,10 @@ func union(list1, list2 []string) []string {
if list2 == nil { if list2 == nil {
list2 = []string{} list2 = []string{}
} }
list := append(list1, list2...) list1 = append(list1, list2...)
unique := []string{} unique := []string{}
m := make(map[string]struct{}) m := make(map[string]struct{})
for _, v := range list { for _, v := range list1 {
if _, ok := m[v]; !ok { if _, ok := m[v]; !ok {
unique = append(unique, v) unique = append(unique, v)
m[v] = struct{}{} m[v] = struct{}{}
+2 -2
View File
@@ -11,7 +11,7 @@ import (
// PathVersionInfo URL. // PathVersionInfo URL.
const PathVersionInfo = "/{module:.+}/@v/{version}.info" const PathVersionInfo = "/{module:.+}/@v/{version}.info"
// InfoHandler implements GET baseURL/module/@v/version.info // InfoHandler implements GET baseURL/module/@v/version.info.
func InfoHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler { func InfoHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler {
const op errors.Op = "download.InfoHandler" const op errors.Op = "download.InfoHandler"
f := func(w http.ResponseWriter, r *http.Request) { f := func(w http.ResponseWriter, r *http.Request) {
@@ -38,7 +38,7 @@ func InfoHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handle
w.WriteHeader(errors.Kind(err)) w.WriteHeader(errors.Kind(err))
} }
w.Write(info) _, _ = w.Write(info)
} }
return http.HandlerFunc(f) return http.HandlerFunc(f)
} }
+2 -2
View File
@@ -11,7 +11,7 @@ import (
// PathVersionModule URL. // PathVersionModule URL.
const PathVersionModule = "/{module:.+}/@v/{version}.mod" const PathVersionModule = "/{module:.+}/@v/{version}.mod"
// ModuleHandler implements GET baseURL/module/@v/version.mod // ModuleHandler implements GET baseURL/module/@v/version.mod.
func ModuleHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler { func ModuleHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler {
const op errors.Op = "download.VersionModuleHandler" const op errors.Op = "download.VersionModuleHandler"
f := func(w http.ResponseWriter, r *http.Request) { f := func(w http.ResponseWriter, r *http.Request) {
@@ -42,7 +42,7 @@ func ModuleHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Hand
return return
} }
w.Write(modBts) _, _ = w.Write(modBts)
} }
return http.HandlerFunc(f) return http.HandlerFunc(f)
} }
+2 -2
View File
@@ -13,7 +13,7 @@ import (
// PathVersionZip URL. // PathVersionZip URL.
const PathVersionZip = "/{module:.+}/@v/{version}.zip" const PathVersionZip = "/{module:.+}/@v/{version}.zip"
// ZipHandler implements GET baseURL/module/@v/version.zip // ZipHandler implements GET baseURL/module/@v/version.zip.
func ZipHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler { func ZipHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler {
const op errors.Op = "download.ZipHandler" const op errors.Op = "download.ZipHandler"
f := func(w http.ResponseWriter, r *http.Request) { f := func(w http.ResponseWriter, r *http.Request) {
@@ -41,7 +41,7 @@ func ZipHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler
w.WriteHeader(errors.Kind(err)) w.WriteHeader(errors.Kind(err))
return return
} }
defer zip.Close() defer func() { _ = zip.Close() }()
w.Header().Set("Content-Type", "application/zip") w.Header().Set("Content-Type", "application/zip")
size := zip.Size() size := zip.Size()
+17 -6
View File
@@ -9,7 +9,7 @@ import (
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
) )
// Kind enums // Kind enums.
const ( const (
KindNotFound = http.StatusNotFound KindNotFound = http.StatusNotFound
KindBadRequest = http.StatusBadRequest KindBadRequest = http.StatusBadRequest
@@ -55,6 +55,16 @@ func Is(err error, kind int) bool {
return Kind(err) == kind return Kind(err) == kind
} }
// IsErr is a convenience wrapper around the std library errors.Is.
func IsErr(err, target error) bool {
return errors.Is(err, target)
}
// AsErr is a convenience wrapper around the std library errors.As.
func AsErr(err error, target any) bool {
return errors.As(err, target)
}
// Op describes any independent function or // Op describes any independent function or
// method in Athens. A series of operations // method in Athens. A series of operations
// forms a more readable stack trace. // forms a more readable stack trace.
@@ -69,7 +79,7 @@ func (o Op) String() string {
// a module from a regular error string or version. // a module from a regular error string or version.
type M string type M string
// V represents a module version in an error // V represents a module version in an error.
type V string type V string
// E is a helper function to construct an Error type // E is a helper function to construct an Error type
@@ -114,8 +124,8 @@ func E(op Op, args ...interface{}) error {
// if none exists, then the level is Error because // if none exists, then the level is Error because
// it is an unexpected. // it is an unexpected.
func Severity(err error) logrus.Level { func Severity(err error) logrus.Level {
e, ok := err.(Error) var e Error
if !ok { if !errors.As(err, &e) {
return logrus.ErrorLevel return logrus.ErrorLevel
} }
@@ -145,8 +155,8 @@ func Expect(err error, kinds ...int) logrus.Level {
// Kind recursively searches for the // Kind recursively searches for the
// first error kind it finds. // first error kind it finds.
func Kind(err error) int { func Kind(err error) int {
e, ok := err.(Error) var e Error
if !ok { if !errors.As(err, &e) {
return KindUnexpected return KindUnexpected
} }
@@ -173,6 +183,7 @@ func KindText(err error) string {
func Ops(err Error) []Op { func Ops(err Error) []Op {
ops := []Op{err.Op} ops := []Op{err.Op}
for { for {
//nolint:errorlint // We iterate the errors anyway.
embeddedErr, ok := err.Err.(Error) embeddedErr, ok := err.Err.(Error)
if !ok { if !ok {
break break
+1 -1
View File
@@ -1,6 +1,6 @@
package errors package errors
// IsNotFoundErr helper function for KindNotFound // IsNotFoundErr helper function for KindNotFound.
func IsNotFoundErr(err error) bool { func IsNotFoundErr(err error) bool {
return Kind(err) == KindNotFound return Kind(err) == KindNotFound
} }
+1 -1
View File
@@ -21,7 +21,7 @@ func RunTests(t *testing.T, indexer index.Indexer, clearIndex func() error) {
t.Fatal(err) t.Fatal(err)
} }
var tests = []struct { tests := []struct {
name string name string
desc string desc string
limit int limit int
+1 -1
View File
@@ -13,7 +13,7 @@ type Line struct {
} }
// Indexer is an interface that can process new module@versions // Indexer is an interface that can process new module@versions
// and also retrieve 'limit' module@versions that were indexed after 'since' // and also retrieve 'limit' module@versions that were indexed after 'since'.
type Indexer interface { type Indexer interface {
// Index stores the module@version into the index backend. // Index stores the module@version into the index backend.
// Implementer must create the Timestamp at the time and set it // Implementer must create the Timestamp at the time and set it
+1 -1
View File
@@ -10,7 +10,7 @@ import (
"github.com/gomods/athens/pkg/index" "github.com/gomods/athens/pkg/index"
) )
// New returns a new in-memory indexer // New returns a new in-memory indexer.
func New() index.Indexer { func New() index.Indexer {
return &indexer{} return &indexer{}
} }
+6 -5
View File
@@ -83,8 +83,8 @@ func (i *indexer) Lines(ctx context.Context, since time.Time, limit int) ([]*ind
if err != nil { if err != nil {
return nil, errors.E(op, err) return nil, errors.E(op, err)
} }
defer rows.Close() defer func() { _ = rows.Close() }()
lines := []*index.Line{} var lines []*index.Line
for rows.Next() { for rows.Next() {
var line index.Line var line index.Line
err = rows.Scan(&line.Path, &line.Version, &line.Timestamp) err = rows.Scan(&line.Path, &line.Version, &line.Timestamp)
@@ -108,13 +108,14 @@ func getMySQLSource(cfg *config.MySQL) string {
} }
func getKind(err error) int { func getKind(err error) int {
mysqlErr, ok := err.(*mysql.MySQLError) mysqlErr := &mysql.MySQLError{}
if !ok { if !errors.AsErr(err, &mysqlErr) {
return errors.KindUnexpected return errors.KindUnexpected
} }
switch mysqlErr.Number { switch mysqlErr.Number {
case 1062: case 1062:
return errors.KindAlreadyExists return errors.KindAlreadyExists
default:
return errors.KindUnexpected
} }
return errors.KindUnexpected
} }
+2 -1
View File
@@ -7,7 +7,7 @@ import (
"github.com/gomods/athens/pkg/index" "github.com/gomods/athens/pkg/index"
) )
// New returns a no-op Indexer // New returns a no-op Indexer.
func New() index.Indexer { func New() index.Indexer {
return indexer{} return indexer{}
} }
@@ -17,6 +17,7 @@ type indexer struct{}
func (indexer) Index(ctx context.Context, mod, ver string) error { func (indexer) Index(ctx context.Context, mod, ver string) error {
return nil return nil
} }
func (indexer) Lines(ctx context.Context, since time.Time, limit int) ([]*index.Line, error) { func (indexer) Lines(ctx context.Context, since time.Time, limit int) ([]*index.Line, error) {
return []*index.Line{}, nil return []*index.Line{}, nil
} }
+10 -11
View File
@@ -7,17 +7,15 @@ import (
"strings" "strings"
"time" "time"
// register the driver with database/sql
"github.com/lib/pq"
"github.com/gomods/athens/pkg/config" "github.com/gomods/athens/pkg/config"
"github.com/gomods/athens/pkg/errors" "github.com/gomods/athens/pkg/errors"
"github.com/gomods/athens/pkg/index" "github.com/gomods/athens/pkg/index"
"github.com/lib/pq"
) )
// New returns a new Indexer with a PostgreSQL implementation. // New returns a new Indexer with a PostgreSQL implementation.
// It attempts to connect to the DB and create the index table // It attempts to connect to the DB and create the index table
// if it doesn ot already exist. // if it does not already exist.
func New(cfg *config.Postgres) (index.Indexer, error) { func New(cfg *config.Postgres) (index.Indexer, error) {
dataSource := getPostgresSource(cfg) dataSource := getPostgresSource(cfg)
db, err := sql.Open("postgres", dataSource) db, err := sql.Open("postgres", dataSource)
@@ -42,7 +40,7 @@ var schema = [...]string{
id SERIAL PRIMARY KEY, id SERIAL PRIMARY KEY,
path VARCHAR(255) NOT NULL, path VARCHAR(255) NOT NULL,
version VARCHAR(255) NOT NULL, version VARCHAR(255) NOT NULL,
timestamp timestamp NOT NULL timestamp TIMESTAMP NOT NULL
) )
`, `,
` `
@@ -82,8 +80,8 @@ func (i *indexer) Lines(ctx context.Context, since time.Time, limit int) ([]*ind
if err != nil { if err != nil {
return nil, errors.E(op, err) return nil, errors.E(op, err)
} }
defer rows.Close() defer func() { _ = rows.Close() }()
lines := []*index.Line{} var lines []*index.Line
for rows.Next() { for rows.Next() {
var line index.Line var line index.Line
err = rows.Scan(&line.Path, &line.Version, &line.Timestamp) err = rows.Scan(&line.Path, &line.Version, &line.Timestamp)
@@ -96,7 +94,7 @@ func (i *indexer) Lines(ctx context.Context, since time.Time, limit int) ([]*ind
} }
func getPostgresSource(cfg *config.Postgres) string { func getPostgresSource(cfg *config.Postgres) string {
args := []string{} args := make([]string, 0, 5+len(cfg.Params))
args = append(args, "host="+cfg.Host) args = append(args, "host="+cfg.Host)
args = append(args, "port=", strconv.Itoa(cfg.Port)) args = append(args, "port=", strconv.Itoa(cfg.Port))
args = append(args, "user=", cfg.User) args = append(args, "user=", cfg.User)
@@ -109,13 +107,14 @@ func getPostgresSource(cfg *config.Postgres) string {
} }
func getKind(err error) int { func getKind(err error) int {
pqerr, ok := err.(*pq.Error) pqerr := &pq.Error{}
if !ok { if !errors.AsErr(err, &pqerr) {
return errors.KindUnexpected return errors.KindUnexpected
} }
switch pqerr.Code { switch pqerr.Code {
case "23505": case "23505":
return errors.KindAlreadyExists return errors.KindAlreadyExists
default:
return errors.KindUnexpected
} }
return errors.KindUnexpected
} }
+2 -2
View File
@@ -35,8 +35,8 @@ func (e *entry) WithFields(fields map[string]interface{}) Entry {
} }
func (e *entry) SystemErr(err error) { func (e *entry) SystemErr(err error) {
athensErr, ok := err.(errors.Error) var athensErr errors.Error
if !ok { if !errors.IsErr(err, athensErr) {
e.Error(err) e.Error(err)
return return
} }
+1 -1
View File
@@ -41,7 +41,7 @@ func (l *Logger) WithFields(fields map[string]interface{}) Entry {
return &entry{e} return &entry{e}
} }
// NoOpLogger provides a Logger that does nothing // NoOpLogger provides a Logger that does nothing.
func NoOpLogger() *Logger { func NoOpLogger() *Logger {
return &Logger{ return &Logger{
Logger: &logrus.Logger{}, Logger: &logrus.Logger{},
+2 -2
View File
@@ -8,14 +8,14 @@ type ctxKey string
const logEntryKey ctxKey = "log-entry-context-key" const logEntryKey ctxKey = "log-entry-context-key"
// SetEntryInContext stores an Entry in the request context // SetEntryInContext stores an Entry in the request context.
func SetEntryInContext(ctx context.Context, e Entry) context.Context { func SetEntryInContext(ctx context.Context, e Entry) context.Context {
return context.WithValue(ctx, logEntryKey, e) return context.WithValue(ctx, logEntryKey, e)
} }
// EntryFromContext returns an Entry that has been stored in the request context. // EntryFromContext returns an Entry that has been stored in the request context.
// If there is no value for the key or the type assertion fails, it returns a new // If there is no value for the key or the type assertion fails, it returns a new
// entry from the provided logger // entry from the provided logger.
func EntryFromContext(ctx context.Context) Entry { func EntryFromContext(ctx context.Context) Entry {
e, ok := ctx.Value(logEntryKey).(Entry) e, ok := ctx.Value(logEntryKey).(Entry)
if !ok || e == nil { if !ok || e == nil {
+1 -1
View File
@@ -4,7 +4,7 @@ import (
"net/http" "net/http"
) )
// ContentType writes writes an application/json // ContentType writes an application/json
// Content-Type header. // Content-Type header.
func ContentType(h http.Handler) http.Handler { func ContentType(h http.Handler) http.Handler {
f := func(w http.ResponseWriter, r *http.Request) { f := func(w http.ResponseWriter, r *http.Request) {
+1 -1
View File
@@ -10,7 +10,7 @@ import (
) )
// LogEntryMiddleware builds a log.Entry, setting the request fields // LogEntryMiddleware builds a log.Entry, setting the request fields
// and storing it in the context to be used throughout the stack // and storing it in the context to be used throughout the stack.
func LogEntryMiddleware(lggr *log.Logger) mux.MiddlewareFunc { func LogEntryMiddleware(lggr *log.Logger) mux.MiddlewareFunc {
return func(h http.Handler) http.Handler { return func(h http.Handler) http.Handler {
f := func(w http.ResponseWriter, r *http.Request) { f := func(w http.ResponseWriter, r *http.Request) {
+1 -1
View File
@@ -9,7 +9,7 @@ import (
// WithRequestID ensures a request id is in the // WithRequestID ensures a request id is in the
// request context by either the incoming header // request context by either the incoming header
// or creating a new one // or creating a new one.
func WithRequestID(h http.Handler) http.Handler { func WithRequestID(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
requestID := r.Header.Get(requestid.HeaderKey) requestID := r.Header.Get(requestid.HeaderKey)
+6 -4
View File
@@ -14,7 +14,7 @@ import (
) )
// NewValidationMiddleware builds a middleware function that performs validation checks by calling // NewValidationMiddleware builds a middleware function that performs validation checks by calling
// an external webhook // an external webhook.
func NewValidationMiddleware(client *http.Client, validatorHook string) mux.MiddlewareFunc { func NewValidationMiddleware(client *http.Client, validatorHook string) mux.MiddlewareFunc {
const op errors.Op = "actions.NewValidationMiddleware" const op errors.Op = "actions.NewValidationMiddleware"
return func(h http.Handler) http.Handler { return func(h http.Handler) http.Handler {
@@ -50,7 +50,7 @@ func NewValidationMiddleware(client *http.Client, validatorHook string) mux.Midd
} }
} }
func maybeLogValidationReason(context context.Context, message string, mod string, version string) { func maybeLogValidationReason(context context.Context, message, mod, version string) {
if len(message) > 0 { if len(message) > 0 {
entry := log.EntryFromContext(context) entry := log.EntryFromContext(context)
entry.Warnf("error validating %s@%s %s", mod, version, message) entry.Warnf("error validating %s@%s %s", mod, version, message)
@@ -85,6 +85,10 @@ func validate(ctx context.Context, client *http.Client, hook, mod, ver string) (
if err != nil { if err != nil {
return validationResponse{Valid: false}, errors.E(op, err) return validationResponse{Valid: false}, errors.E(op, err)
} }
defer func() {
_, _ = io.Copy(io.Discard, resp.Body)
_ = resp.Body.Close()
}()
switch resp.StatusCode { switch resp.StatusCode {
case http.StatusOK: case http.StatusOK:
@@ -97,8 +101,6 @@ func validate(ctx context.Context, client *http.Client, hook, mod, ver string) (
} }
func validationResponseFromRequest(resp *http.Response) validationResponse { func validationResponseFromRequest(resp *http.Response) validationResponse {
defer resp.Body.Close()
body, _ := io.ReadAll(resp.Body) body, _ := io.ReadAll(resp.Body)
return validationResponse{Valid: resp.StatusCode == http.StatusOK, Message: body} return validationResponse{Valid: resp.StatusCode == http.StatusOK, Message: body}
} }
+1 -1
View File
@@ -6,7 +6,7 @@ import (
"github.com/gomods/athens/pkg/storage" "github.com/gomods/athens/pkg/storage"
) )
// Fetcher fetches module from an upstream source // Fetcher fetches module from an upstream source.
type Fetcher interface { type Fetcher interface {
// Fetch downloads the sources from an upstream and returns the corresponding // Fetch downloads the sources from an upstream and returns the corresponding
// .info, .mod, and .zip files. // .info, .mod, and .zip files.
+15 -16
View File
@@ -3,6 +3,7 @@ package module
import ( import (
"bufio" "bufio"
"os" "os"
"path/filepath"
"strconv" "strconv"
"strings" "strings"
@@ -14,15 +15,15 @@ var (
versionSeparator = "." versionSeparator = "."
) )
// Filter is a filter of modules // Filter is a filter of modules.
type Filter struct { type Filter struct {
root ruleNode root ruleNode
filePath string filePath string
} }
// NewFilter creates new filter based on rules defined in a configuration file // NewFilter creates new filter based on rules defined in a configuration file.
// WARNING: this is not concurrently safe // WARNING: this is not concurrently safe.
// Configuration consists of two operations: + for include and - for exclude // Configuration consists of two operations: + for include and - for exclude:
// e.g. // e.g.
// - github.com/a // - github.com/a
// - github.com/a/b // - github.com/a/b
@@ -33,7 +34,7 @@ type Filter struct {
// - // -
// + github.com/a // + github.com/a
// //
// will exclude all items from communication except github.com/a // will exclude all items from communication except github.com/a.
func NewFilter(filterFilePath string) (*Filter, error) { func NewFilter(filterFilePath string) (*Filter, error) {
// Do not return an error if the file path is empty // Do not return an error if the file path is empty
// Do not attempt to parse it as well. // Do not attempt to parse it as well.
@@ -42,10 +43,9 @@ func NewFilter(filterFilePath string) (*Filter, error) {
} }
return initFromConfig(filterFilePath) return initFromConfig(filterFilePath)
} }
// AddRule adds rule for specified path // AddRule adds rule for specified path.
func (f *Filter) AddRule(path string, qualifiers []string, rule FilterRule) { func (f *Filter) AddRule(path string, qualifiers []string, rule FilterRule) {
f.ensurePath(path) f.ensurePath(path)
@@ -70,7 +70,7 @@ func (f *Filter) AddRule(path string, qualifiers []string, rule FilterRule) {
latest.next[last] = rn latest.next[last] = rn
} }
// Rule returns the filter rule to be applied to the given path // Rule returns the filter rule to be applied to the given path.
func (f *Filter) Rule(path, version string) FilterRule { func (f *Filter) Rule(path, version string) FilterRule {
segs := getPathSegments(path) segs := getPathSegments(path)
rule := f.getAssociatedRule(version, segs...) rule := f.getAssociatedRule(version, segs...)
@@ -145,7 +145,6 @@ func initFromConfig(filePath string) (*Filter, error) {
f.root = rn f.root = rn
for idx, line := range lines { for idx, line := range lines {
// Ignore newline // Ignore newline
if len(line) == 0 { if len(line) == 0 {
continue continue
@@ -160,7 +159,7 @@ func initFromConfig(filePath string) (*Filter, error) {
} }
ruleSign := strings.TrimSpace(split[0]) ruleSign := strings.TrimSpace(split[0])
rule := Default var rule FilterRule
switch ruleSign { switch ruleSign {
case "+": case "+":
rule = Include rule = Include
@@ -195,11 +194,11 @@ func initFromConfig(filePath string) (*Filter, error) {
// matches checks if the given version matches the given qualifier. // matches checks if the given version matches the given qualifier.
// Qualifiers can be: // Qualifiers can be:
// - plain versions // - plain versions.
// - v1.2.3 enables v1.2.3 // - v1.2.3 enables v1.2.3.
// - ~1.2.3: enables 1.2.x which are at least 1.2.3 // - ~1.2.3: enables 1.2.x which are at least 1.2.3.
// - ^1.2.3: enables 1.x.x which are at least 1.2.3 // - ^1.2.3: enables 1.x.x which are at least 1.2.3.
// - <1.2.3: enables everything lower than 1.2.3 includes 1.2.2 and 0.58.9 as well // - <1.2.3: enables everything lower than 1.2.3 includes 1.2.2 and 0.58.9 as well.
func matches(version, qualifier string) bool { func matches(version, qualifier string) bool {
if len(qualifier) < 2 || len(version) < 1 { if len(qualifier) < 2 || len(version) < 1 {
return false return false
@@ -297,7 +296,7 @@ func newRule(r FilterRule) ruleNode {
func getConfigLines(filterFile string) ([]string, error) { func getConfigLines(filterFile string) ([]string, error) {
const op errors.Op = "module.getConfigLines" const op errors.Op = "module.getConfigLines"
f, err := os.Open(filterFile) f, err := os.Open(filepath.Clean(filterFile))
if err != nil { if err != nil {
return nil, errors.E(op, err) return nil, errors.E(op, err)
} }
+6 -6
View File
@@ -1,16 +1,16 @@
package module package module
// FilterRule defines behavior of module communication // FilterRule defines behavior of module communication.
type FilterRule int type FilterRule int
const ( const (
// Default filter rule does not alter default/parent behavior // Default filter rule does not alter default/parent behavior.
Default FilterRule = iota Default FilterRule = iota
// Include treats modules the usual way // Include treats modules the usual way.
// Used for reverting Exclude of parent path // Used for reverting Exclude of parent path.
Include Include
// Exclude filter rule excludes package and its children from communication // Exclude filter rule excludes package and its children from communication.
Exclude Exclude
// Direct filter rule forces the package to be fetched directly from upstream proxy // Direct filter rule forces the package to be fetched directly from upstream proxy.
Direct Direct
) )
+8 -10
View File
@@ -35,7 +35,7 @@ type goModule struct {
GoModSum string `json:"goModSum"` // checksum for go.mod (as in go.sum) GoModSum string `json:"goModSum"` // checksum for go.mod (as in go.sum)
} }
// NewGoGetFetcher creates fetcher which uses go get tool to fetch modules // NewGoGetFetcher creates fetcher which uses go get tool to fetch modules.
func NewGoGetFetcher(goBinaryName, gogetDir string, envVars []string, fs afero.Fs) (Fetcher, error) { func NewGoGetFetcher(goBinaryName, gogetDir string, envVars []string, fs afero.Fs) (Fetcher, error) {
const op errors.Op = "module.NewGoGetFetcher" const op errors.Op = "module.NewGoGetFetcher"
if err := validGoBinary(goBinaryName); err != nil { if err := validGoBinary(goBinaryName); err != nil {
@@ -64,7 +64,7 @@ func (g *goGetFetcher) Fetch(ctx context.Context, mod, ver string) (*storage.Ver
sourcePath := filepath.Join(goPathRoot, "src") sourcePath := filepath.Join(goPathRoot, "src")
modPath := filepath.Join(sourcePath, getRepoDirName(mod, ver)) modPath := filepath.Join(sourcePath, getRepoDirName(mod, ver))
if err := g.fs.MkdirAll(modPath, os.ModeDir|os.ModePerm); err != nil { if err := g.fs.MkdirAll(modPath, os.ModeDir|os.ModePerm); err != nil {
clearFiles(g.fs, goPathRoot) _ = clearFiles(g.fs, goPathRoot)
return nil, errors.E(op, err) return nil, errors.E(op, err)
} }
@@ -72,14 +72,13 @@ func (g *goGetFetcher) Fetch(ctx context.Context, mod, ver string) (*storage.Ver
ctx, ctx,
g.goBinaryName, g.goBinaryName,
g.envVars, g.envVars,
g.fs,
goPathRoot, goPathRoot,
modPath, modPath,
mod, mod,
ver, ver,
) )
if err != nil { if err != nil {
clearFiles(g.fs, goPathRoot) _ = clearFiles(g.fs, goPathRoot)
return nil, errors.E(op, err) return nil, errors.E(op, err)
} }
@@ -116,7 +115,6 @@ func downloadModule(
ctx context.Context, ctx context.Context,
goBinaryName string, goBinaryName string,
envVars []string, envVars []string,
fs afero.Fs,
gopath, gopath,
repoRoot, repoRoot,
module, module,
@@ -137,7 +135,7 @@ func downloadModule(
err := cmd.Run() err := cmd.Run()
if err != nil { if err != nil {
err = fmt.Errorf("%v: %s", err, stderr) err = fmt.Errorf("%w: %s", err, stderr)
var m goModule var m goModule
if jsonErr := json.NewDecoder(stdout).Decode(&m); jsonErr != nil { if jsonErr := json.NewDecoder(stdout).Decode(&m); jsonErr != nil {
return goModule{}, errors.E(op, err) return goModule{}, errors.E(op, err)
@@ -165,17 +163,17 @@ func isLimitHit(o string) bool {
} }
// getRepoDirName takes a raw repository URI and a version and creates a directory name that the // getRepoDirName takes a raw repository URI and a version and creates a directory name that the
// repository contents can be put into // repository contents can be put into.
func getRepoDirName(repoURI, version string) string { func getRepoDirName(repoURI, version string) string {
escapedURI := strings.Replace(repoURI, "/", "-", -1) escapedURI := strings.ReplaceAll(repoURI, "/", "-")
return fmt.Sprintf("%s-%s", escapedURI, version) return fmt.Sprintf("%s-%s", escapedURI, version)
} }
func validGoBinary(name string) error { func validGoBinary(name string) error {
const op errors.Op = "module.validGoBinary" const op errors.Op = "module.validGoBinary"
err := exec.Command(name).Run() err := exec.Command(name).Run()
_, ok := err.(*exec.ExitError) eErr := &exec.ExitError{}
if err != nil && !ok { if err != nil && !errors.AsErr(err, &eErr) {
return errors.E(op, err) return errors.E(op, err)
} }
return nil return nil
+5 -5
View File
@@ -28,7 +28,7 @@ type vcsLister struct {
fs afero.Fs fs afero.Fs
} }
// NewVCSLister creates an UpstreamLister which uses VCS to fetch a list of available versions // NewVCSLister creates an UpstreamLister which uses VCS to fetch a list of available versions.
func NewVCSLister(goBinPath string, env []string, fs afero.Fs) UpstreamLister { func NewVCSLister(goBinPath string, env []string, fs afero.Fs) UpstreamLister {
return &vcsLister{ return &vcsLister{
goBinPath: goBinPath, goBinPath: goBinPath,
@@ -39,13 +39,13 @@ func NewVCSLister(goBinPath string, env []string, fs afero.Fs) UpstreamLister {
func (l *vcsLister) List(ctx context.Context, module string) (*storage.RevInfo, []string, error) { func (l *vcsLister) List(ctx context.Context, module string) (*storage.RevInfo, []string, error) {
const op errors.Op = "vcsLister.List" const op errors.Op = "vcsLister.List"
ctx, span := observ.StartSpan(ctx, op.String()) _, span := observ.StartSpan(ctx, op.String())
defer span.End() defer span.End()
tmpDir, err := afero.TempDir(l.fs, "", "go-list") tmpDir, err := afero.TempDir(l.fs, "", "go-list")
if err != nil { if err != nil {
return nil, nil, errors.E(op, err) return nil, nil, errors.E(op, err)
} }
defer l.fs.RemoveAll(tmpDir) defer func() { _ = l.fs.RemoveAll(tmpDir) }()
cmd := exec.Command( cmd := exec.Command(
l.goBinPath, l.goBinPath,
@@ -62,12 +62,12 @@ func (l *vcsLister) List(ctx context.Context, module string) (*storage.RevInfo,
if err != nil { if err != nil {
return nil, nil, errors.E(op, err) return nil, nil, errors.E(op, err)
} }
defer clearFiles(l.fs, gopath) defer func() { _ = clearFiles(l.fs, gopath) }()
cmd.Env = prepareEnv(gopath, l.env) cmd.Env = prepareEnv(gopath, l.env)
err = cmd.Run() err = cmd.Run()
if err != nil { if err != nil {
err = fmt.Errorf("%v: %s", err, stderr) err = fmt.Errorf("%w: %s", err, stderr)
// as of now, we can't recognize between a true NotFound // as of now, we can't recognize between a true NotFound
// and an unexpected error, so we choose the more // and an unexpected error, so we choose the more
// hopeful path of NotFound. This way the Go command // hopeful path of NotFound. This way the Go command
+1 -1
View File
@@ -9,7 +9,7 @@ import (
// prepareEnv will return all the appropriate // prepareEnv will return all the appropriate
// environment variables for a Go Command to run // environment variables for a Go Command to run
// successfully (such as GOPATH, GOCACHE, PATH etc) // successfully (such as GOPATH, GOCACHE, PATH etc).
func prepareEnv(gopath string, envVars []string) []string { func prepareEnv(gopath string, envVars []string) []string {
gopathEnv := fmt.Sprintf("GOPATH=%s", gopath) gopathEnv := fmt.Sprintf("GOPATH=%s", gopath)
cacheEnv := fmt.Sprintf("GOCACHE=%s", filepath.Join(gopath, "cache")) cacheEnv := fmt.Sprintf("GOCACHE=%s", filepath.Join(gopath, "cache"))
+6 -6
View File
@@ -14,10 +14,10 @@ type zipReadCloser struct {
goPath string goPath string
} }
// Close closes the zip file handle and clears up disk space used by the underlying disk ref // Close closes the zip file handle and clears up disk space used by the underlying disk ref.
// It is the caller's responsibility to call this method to free up utilized disk space // It is the caller's responsibility to call this method to free up utilized disk space.
func (rc *zipReadCloser) Close() error { func (rc *zipReadCloser) Close() error {
rc.zip.Close() _ = rc.zip.Close()
return clearFiles(rc.fs, rc.goPath) return clearFiles(rc.fs, rc.goPath)
} }
@@ -25,8 +25,8 @@ func (rc *zipReadCloser) Read(p []byte) (n int, err error) {
return rc.zip.Read(p) return rc.zip.Read(p)
} }
// clearFiles deletes all data from the given fs at path root // clearFiles deletes all data from the given fs at path root.
// This function must be called when zip is closed to cleanup the entire GOPATH created by the diskref // This function must be called when zip is closed to cleanup the entire GOPATH created by the diskref.
func clearFiles(fs afero.Fs, root string) error { func clearFiles(fs afero.Fs, root string) error {
const op errors.Op = "module.ClearFiles" const op errors.Op = "module.ClearFiles"
// This is required because vgo ensures dependencies are read-only // This is required because vgo ensures dependencies are read-only
@@ -36,7 +36,7 @@ func clearFiles(fs afero.Fs, root string) error {
if err != nil { if err != nil {
return err return err
} }
return fs.Chmod(path, 0770) return fs.Chmod(path, 0o770)
} }
err := afero.Walk(fs, root, walkFn) err := afero.Walk(fs, root, walkFn)
if err != nil { if err != nil {
+18 -18
View File
@@ -13,16 +13,16 @@ import (
// RegisterExporter determines the type of TraceExporter service for exporting traces from opencensus // RegisterExporter determines the type of TraceExporter service for exporting traces from opencensus
// User can choose from multiple tracing services (datadog, jaegar) // User can choose from multiple tracing services (datadog, jaegar)
// RegisterExporter returns the 'Flush' function for that particular tracing service // RegisterExporter returns the 'Flush' function for that particular tracing service.
func RegisterExporter(traceExporter, URL, service, ENV string) (func(), error) { func RegisterExporter(traceExporter, url, service, env string) (func(), error) {
const op errors.Op = "observ.RegisterExporter" const op errors.Op = "observ.RegisterExporter"
switch traceExporter { switch traceExporter {
case "jaeger": case "jaeger":
return registerJaegerExporter(URL, service, ENV) return registerJaegerExporter(url, service, env)
case "datadog": case "datadog":
return registerDatadogExporter(URL, service, ENV) return registerDatadogExporter(url, service, env)
case "stackdriver": case "stackdriver":
return registerStackdriverExporter(URL, ENV) return registerStackdriverExporter(url, env)
case "": case "":
return nil, errors.E(op, "Exporter not specified. Traces won't be exported") return nil, errors.E(op, "Exporter not specified. Traces won't be exported")
default: default:
@@ -32,14 +32,14 @@ func RegisterExporter(traceExporter, URL, service, ENV string) (func(), error) {
// registerJaegerExporter creates a jaeger exporter for exporting traces to opencensus. // registerJaegerExporter creates a jaeger exporter for exporting traces to opencensus.
// Currently uses the 'TraceExporter' variable in the config file. // Currently uses the 'TraceExporter' variable in the config file.
// It should in the future have a nice sampling rate defined // It should in the future have a nice sampling rate defined.
func registerJaegerExporter(URL, service, ENV string) (func(), error) { func registerJaegerExporter(url, service, env string) (func(), error) {
const op errors.Op = "observ.registerJaegarExporter" const op errors.Op = "observ.registerJaegarExporter"
if URL == "" { if url == "" {
return nil, errors.E(op, "Exporter URL is empty. Traces won't be exported") return nil, errors.E(op, "Exporter URL is empty. Traces won't be exported")
} }
ex, err := jaeger.NewExporter(jaeger.Options{ ex, err := jaeger.NewExporter(jaeger.Options{
Endpoint: URL, Endpoint: url,
Process: jaeger.Process{ Process: jaeger.Process{
ServiceName: service, ServiceName: service,
Tags: []jaeger.Tag{ Tags: []jaeger.Tag{
@@ -53,41 +53,41 @@ func registerJaegerExporter(URL, service, ENV string) (func(), error) {
if err != nil { if err != nil {
return nil, errors.E(op, err) return nil, errors.E(op, err)
} }
traceRegisterExporter(ex, ENV) traceRegisterExporter(ex, env)
return ex.Flush, nil return ex.Flush, nil
} }
func traceRegisterExporter(exporter trace.Exporter, ENV string) { func traceRegisterExporter(exporter trace.Exporter, env string) {
trace.RegisterExporter(exporter) trace.RegisterExporter(exporter)
if ENV == "development" { if env == "development" {
trace.ApplyConfig(trace.Config{DefaultSampler: trace.AlwaysSample()}) trace.ApplyConfig(trace.Config{DefaultSampler: trace.AlwaysSample()})
} }
} }
// registerDatadogTracerExporter creates a datadog exporter. // registerDatadogTracerExporter creates a datadog exporter.
// Currently uses the 'TraceExporter' variable in the config file. // Currently uses the 'TraceExporter' variable in the config file.
func registerDatadogExporter(URL, service, ENV string) (func(), error) { func registerDatadogExporter(url, service, env string) (func(), error) {
ex := datadog.NewExporter( ex := datadog.NewExporter(
datadog.Options{ datadog.Options{
TraceAddr: URL, TraceAddr: url,
Service: service, Service: service,
}) })
traceRegisterExporter(ex, ENV) traceRegisterExporter(ex, env)
return ex.Stop, nil return ex.Stop, nil
} }
func registerStackdriverExporter(projectID, ENV string) (func(), error) { func registerStackdriverExporter(projectID, env string) (func(), error) {
const op errors.Op = "observ.registerStackdriverExporter" const op errors.Op = "observ.registerStackdriverExporter"
ex, err := stackdriver.NewExporter(stackdriver.Options{ProjectID: projectID}) ex, err := stackdriver.NewExporter(stackdriver.Options{ProjectID: projectID})
if err != nil { if err != nil {
return nil, errors.E(op, err) return nil, errors.E(op, err)
} }
traceRegisterExporter(ex, ENV) traceRegisterExporter(ex, env)
return ex.Flush, nil return ex.Flush, nil
} }
// StartSpan takes in a Context Interface and opName and starts a span. It returns the new attached ObserverContext // StartSpan takes in a Context Interface and opName and starts a span. It returns the new attached ObserverContext
// and span // and span.
func StartSpan(ctx context.Context, op string) (context.Context, *trace.Span) { func StartSpan(ctx context.Context, op string) (context.Context, *trace.Span) {
return trace.StartSpan(ctx, op) return trace.StartSpan(ctx, op)
} }
+4 -4
View File
@@ -15,10 +15,10 @@ import (
) )
// RegisterStatsExporter determines the type of StatsExporter service for exporting stats from Opencensus // RegisterStatsExporter determines the type of StatsExporter service for exporting stats from Opencensus
// Currently it supports: prometheus // Currently it supports: prometheus.
func RegisterStatsExporter(r *mux.Router, statsExporter, service string) (func(), error) { func RegisterStatsExporter(r *mux.Router, statsExporter, service string) (func(), error) {
const op errors.Op = "observ.RegisterStatsExporter" const op errors.Op = "observ.RegisterStatsExporter"
var stop = func() {} stop := func() {}
var err error var err error
switch statsExporter { switch statsExporter {
case "prometheus": case "prometheus":
@@ -38,7 +38,7 @@ func RegisterStatsExporter(r *mux.Router, statsExporter, service string) (func()
default: default:
return nil, errors.E(op, fmt.Sprintf("StatsExporter %s not supported. Please open PR or an issue at github.com/gomods/athens", statsExporter)) return nil, errors.E(op, fmt.Sprintf("StatsExporter %s not supported. Please open PR or an issue at github.com/gomods/athens", statsExporter))
} }
if err := registerViews(); err != nil { if err = registerViews(); err != nil {
return nil, errors.E(op, err) return nil, errors.E(op, err)
} }
@@ -89,7 +89,7 @@ func registerStatsStackDriverExporter(projectID string) (func(), error) {
return sd.Flush, nil return sd.Flush, nil
} }
// registerViews register stats which should be collected in Athens // registerViews register stats which should be collected in Athens.
func registerViews() error { func registerViews() error {
const op errors.Op = "observ.registerViews" const op errors.Op = "observ.registerViews"
if err := view.Register( if err := view.Register(
+2 -1
View File
@@ -19,8 +19,9 @@ func DecodePath(encoding string) (path string, err error) {
return path, nil return path, nil
} }
// Ripped from cmd/go // Ripped from cmd/go.
func decodeString(encoding string) (string, bool) { func decodeString(encoding string) (string, bool) {
//nolint:prealloc
var buf []byte var buf []byte
bang := false bang := false
+5 -5
View File
@@ -9,7 +9,7 @@ import (
"github.com/gorilla/mux" "github.com/gorilla/mux"
) )
// GetModule gets the module from the path of a ?go-get=1 request // GetModule gets the module from the path of a ?go-get=1 request.
func GetModule(r *http.Request) (string, error) { func GetModule(r *http.Request) (string, error) {
const op errors.Op = "paths.GetModule" const op errors.Op = "paths.GetModule"
module := mux.Vars(r)["module"] module := mux.Vars(r)["module"]
@@ -19,7 +19,7 @@ func GetModule(r *http.Request) (string, error) {
return DecodePath(module) return DecodePath(module)
} }
// GetVersion gets the version from the path of a ?go-get=1 request // GetVersion gets the version from the path of a ?go-get=1 request.
func GetVersion(r *http.Request) (string, error) { func GetVersion(r *http.Request) (string, error) {
const op errors.Op = "paths.GetVersion" const op errors.Op = "paths.GetVersion"
@@ -31,13 +31,13 @@ func GetVersion(r *http.Request) (string, error) {
} }
// AllPathParams holds the module and version in the path of a ?go-get=1 // AllPathParams holds the module and version in the path of a ?go-get=1
// request // request.
type AllPathParams struct { type AllPathParams struct {
Module string `json:"module"` Module string `json:"module"`
Version string `json:"version"` Version string `json:"version"`
} }
// GetAllParams fetches the path params from r and returns them // GetAllParams fetches the path params from r and returns them.
func GetAllParams(r *http.Request) (*AllPathParams, error) { func GetAllParams(r *http.Request) (*AllPathParams, error) {
const op errors.Op = "paths.GetAllParams" const op errors.Op = "paths.GetAllParams"
mod, err := GetModule(r) mod, err := GetModule(r)
@@ -54,7 +54,7 @@ func GetAllParams(r *http.Request) (*AllPathParams, error) {
} }
// MatchesPattern reports whether the path prefix of target matches // MatchesPattern reports whether the path prefix of target matches
// pattern (as defined by path.Match) // pattern (as defined by path.Match).
// //
// This tries to keep the same behavior as GOPRIVATE/GONOPROXY/GONOSUMDB, // This tries to keep the same behavior as GOPRIVATE/GONOPROXY/GONOSUMDB,
// and is adopted from: // and is adopted from:
+3 -3
View File
@@ -3,18 +3,18 @@ package requestid
import "context" import "context"
// HeaderKey is the header key that athens uses // HeaderKey is the header key that athens uses
// to pass request ids into logs and outbound requests // to pass request ids into logs and outbound requests.
const HeaderKey = "Athens-Request-ID" const HeaderKey = "Athens-Request-ID"
type key struct{} type key struct{}
// SetInContext sets the given requestID into the context // SetInContext sets the given requestID into the context.
func SetInContext(ctx context.Context, id string) context.Context { func SetInContext(ctx context.Context, id string) context.Context {
return context.WithValue(ctx, key{}, id) return context.WithValue(ctx, key{}, id)
} }
// FromContext returns a requestID from the context or an empty // FromContext returns a requestID from the context or an empty
// string if not found // string if not found.
func FromContext(ctx context.Context) string { func FromContext(ctx context.Context) string {
id, _ := ctx.Value(key{}).(string) id, _ := ctx.Value(key{}).(string)
return id return id
+3 -3
View File
@@ -19,7 +19,7 @@ import (
// what was requested, this is helpful if what was requested // what was requested, this is helpful if what was requested
// was a descriptive version such as a branch name or a full commit sha. // was a descriptive version such as a branch name or a full commit sha.
type Stasher interface { type Stasher interface {
Stash(ctx context.Context, mod string, ver string) (string, error) Stash(ctx context.Context, mod, ver string) (string, error)
} }
// Wrapper helps extend the main stasher's functionality with addons. // Wrapper helps extend the main stasher's functionality with addons.
@@ -46,7 +46,7 @@ type stasher struct {
func (s *stasher) Stash(ctx context.Context, mod, ver string) (string, error) { func (s *stasher) Stash(ctx context.Context, mod, ver string) (string, error) {
const op errors.Op = "stasher.Stash" const op errors.Op = "stasher.Stash"
_, span := observ.StartSpan(ctx, op.String()) ctx, span := observ.StartSpan(ctx, op.String())
defer span.End() defer span.End()
log.EntryFromContext(ctx).Debugf("saving %s@%s to storage...", mod, ver) log.EntryFromContext(ctx).Debugf("saving %s@%s to storage...", mod, ver)
@@ -58,7 +58,7 @@ func (s *stasher) Stash(ctx context.Context, mod, ver string) (string, error) {
if err != nil { if err != nil {
return "", errors.E(op, err) return "", errors.E(op, err)
} }
defer v.Zip.Close() defer func() { _ = v.Zip.Close() }()
if v.Semver != ver { if v.Semver != ver {
exists, err := s.checker.Exists(ctx, mod, v.Semver) exists, err := s.checker.Exists(ctx, mod, v.Semver)
if err != nil { if err != nil {
+4 -3
View File
@@ -132,8 +132,8 @@ func (s *azblobLock) acquireLease(ctx context.Context, blobURL azblob.BlockBlobU
_, err := blobURL.Upload(tctx, bytes.NewReader([]byte{1}), azblob.BlobHTTPHeaders{}, nil, azblob.BlobAccessConditions{}) _, err := blobURL.Upload(tctx, bytes.NewReader([]byte{1}), azblob.BlobHTTPHeaders{}, nil, azblob.BlobAccessConditions{})
if err != nil { if err != nil {
// if the blob is already leased we will get http.StatusPreconditionFailed while writing to that blob // if the blob is already leased we will get http.StatusPreconditionFailed while writing to that blob
stgErr, ok := err.(azblob.StorageError) var stgErr azblob.StorageError
if !ok || stgErr.Response().StatusCode != http.StatusPreconditionFailed { if !errors.AsErr(err, &stgErr) || stgErr.Response().StatusCode != http.StatusPreconditionFailed {
return "", errors.E(op, err) return "", errors.E(op, err)
} }
} }
@@ -147,7 +147,8 @@ func (s *azblobLock) acquireLease(ctx context.Context, blobURL azblob.BlockBlobU
res, err := blobURL.AcquireLease(tctx, leaseID.String(), 15, azblob.ModifiedAccessConditions{}) res, err := blobURL.AcquireLease(tctx, leaseID.String(), 15, azblob.ModifiedAccessConditions{})
if err != nil { if err != nil {
// if the blob is already leased we will get http.StatusConflict - wait and try again // if the blob is already leased we will get http.StatusConflict - wait and try again
if stgErr, ok := err.(azblob.StorageError); ok && stgErr.Response().StatusCode == http.StatusConflict { var stgErr azblob.StorageError
if ok := errors.AsErr(err, &stgErr); ok && stgErr.Response().StatusCode == http.StatusConflict {
select { select {
case <-time.After(1 * time.Second): case <-time.After(1 * time.Second):
continue continue
+1 -1
View File
@@ -8,7 +8,7 @@ import (
"github.com/gomods/athens/pkg/errors" "github.com/gomods/athens/pkg/errors"
"github.com/gomods/athens/pkg/observ" "github.com/gomods/athens/pkg/observ"
"github.com/gomods/athens/pkg/storage" "github.com/gomods/athens/pkg/storage"
"go.etcd.io/etcd/client/v3" clientv3 "go.etcd.io/etcd/client/v3"
"go.etcd.io/etcd/client/v3/concurrency" "go.etcd.io/etcd/client/v3/concurrency"
"golang.org/x/sync/errgroup" "golang.org/x/sync/errgroup"
) )
+2 -2
View File
@@ -13,14 +13,14 @@ import (
"github.com/gomods/athens/pkg/storage" "github.com/gomods/athens/pkg/storage"
) )
// RedisLogger mirrors github.com/go-redis/redis/v8/internal.Logging // RedisLogger mirrors github.com/go-redis/redis/v8/internal.Logging.
type RedisLogger interface { type RedisLogger interface {
Printf(ctx context.Context, format string, v ...interface{}) Printf(ctx context.Context, format string, v ...interface{})
} }
// WithRedisLock returns a distributed singleflight // WithRedisLock returns a distributed singleflight
// using a redis cluster. If it cannot connect, it will return an error. // using a redis cluster. If it cannot connect, it will return an error.
func WithRedisLock(l RedisLogger, endpoint string, password string, checker storage.Checker, lockConfig *config.RedisLockConfig) (Wrapper, error) { func WithRedisLock(l RedisLogger, endpoint, password string, checker storage.Checker, lockConfig *config.RedisLockConfig) (Wrapper, error) {
redis.SetLogger(l) redis.SetLogger(l)
const op errors.Op = "stash.WithRedisLock" const op errors.Op = "stash.WithRedisLock"
+1 -1
View File
@@ -10,7 +10,7 @@ import (
) )
// WithRedisSentinelLock returns a distributed singleflight // WithRedisSentinelLock returns a distributed singleflight
// with a redis cluster that utilizes sentinel for quorum and failover // with a redis cluster that utilizes sentinel for quorum and failover.
func WithRedisSentinelLock(l RedisLogger, endpoints []string, master, password string, checker storage.Checker, lockConfig *config.RedisLockConfig) (Wrapper, error) { func WithRedisSentinelLock(l RedisLogger, endpoints []string, master, password string, checker storage.Checker, lockConfig *config.RedisLockConfig) (Wrapper, error) {
redis.SetLogger(l) redis.SetLogger(l)
+9 -20
View File
@@ -15,14 +15,6 @@ import (
"github.com/gomods/athens/pkg/storage" "github.com/gomods/athens/pkg/storage"
) )
type client interface {
UploadWithContext(ctx context.Context, path, contentType string, content io.Reader) error
BlobExists(ctx context.Context, path string) (bool, error)
ReadBlob(ctx context.Context, path string) (io.ReadCloser, error)
ListBlobs(ctx context.Context, prefix string) ([]string, error)
DeleteBlob(ctx context.Context, path string) error
}
type azureBlobStoreClient struct { type azureBlobStoreClient struct {
containerURL *azblob.ContainerURL containerURL *azblob.ContainerURL
} }
@@ -45,13 +37,13 @@ func newBlobStoreClient(accountURL *url.URL, accountName, accountKey, containerN
} }
// Storage implements (github.com/gomods/athens/pkg/storage).Saver and // Storage implements (github.com/gomods/athens/pkg/storage).Saver and
// also provides a function to fetch the location of a module // also provides a function to fetch the location of a module.
type Storage struct { type Storage struct {
client *azureBlobStoreClient client *azureBlobStoreClient
timeout time.Duration timeout time.Duration
} }
// New creates a new azure blobs storage // New creates a new azure blobs storage.
func New(conf *config.AzureBlobConfig, timeout time.Duration) (*Storage, error) { func New(conf *config.AzureBlobConfig, timeout time.Duration) (*Storage, error) {
const op errors.Op = "azureblob.New" const op errors.Op = "azureblob.New"
u, err := url.Parse(fmt.Sprintf("https://%s.blob.core.windows.net", conf.AccountName)) u, err := url.Parse(fmt.Sprintf("https://%s.blob.core.windows.net", conf.AccountName))
@@ -65,7 +57,7 @@ func New(conf *config.AzureBlobConfig, timeout time.Duration) (*Storage, error)
return &Storage{client: cl, timeout: timeout}, nil return &Storage{client: cl, timeout: timeout}, nil
} }
// BlobExists checks if a particular blob exists in the container // BlobExists checks if a particular blob exists in the container.
func (c *azureBlobStoreClient) BlobExists(ctx context.Context, path string) (bool, error) { func (c *azureBlobStoreClient) BlobExists(ctx context.Context, path string) (bool, error) {
const op errors.Op = "azureblob.BlobExists" const op errors.Op = "azureblob.BlobExists"
// TODO: Any better way of doing this ? // TODO: Any better way of doing this ?
@@ -73,10 +65,8 @@ func (c *azureBlobStoreClient) BlobExists(ctx context.Context, path string) (boo
_, err := blobURL.GetProperties(ctx, azblob.BlobAccessConditions{}) _, err := blobURL.GetProperties(ctx, azblob.BlobAccessConditions{})
if err != nil { if err != nil {
var serr azblob.StorageError var serr azblob.StorageError
var ok bool if !errors.AsErr(err, &serr) {
return false, errors.E(op, fmt.Errorf("error in casting to azure error type %w", err))
if serr, ok = err.(azblob.StorageError); !ok {
return false, errors.E(op, fmt.Errorf("Error in casting to azure error type %v", err))
} }
if serr.Response().StatusCode == http.StatusNotFound { if serr.Response().StatusCode == http.StatusNotFound {
return false, nil return false, nil
@@ -85,10 +75,9 @@ func (c *azureBlobStoreClient) BlobExists(ctx context.Context, path string) (boo
return false, errors.E(op, err) return false, errors.E(op, err)
} }
return true, nil return true, nil
} }
// ReadBlob returns a storage.SizeReadCloser for the contents of a blob // ReadBlob returns a storage.SizeReadCloser for the contents of a blob.
func (c *azureBlobStoreClient) ReadBlob(ctx context.Context, path string) (storage.SizeReadCloser, error) { func (c *azureBlobStoreClient) ReadBlob(ctx context.Context, path string) (storage.SizeReadCloser, error) {
const op errors.Op = "azureblob.ReadBlob" const op errors.Op = "azureblob.ReadBlob"
blobURL := c.containerURL.NewBlockBlobURL(path) blobURL := c.containerURL.NewBlockBlobURL(path)
@@ -101,7 +90,7 @@ func (c *azureBlobStoreClient) ReadBlob(ctx context.Context, path string) (stora
return storage.NewSizer(rc, size), nil return storage.NewSizer(rc, size), nil
} }
// ListBlobs will list all blobs which has the given prefix // ListBlobs will list all blobs which has the given prefix.
func (c *azureBlobStoreClient) ListBlobs(ctx context.Context, prefix string) ([]string, error) { func (c *azureBlobStoreClient) ListBlobs(ctx context.Context, prefix string) ([]string, error) {
const op errors.Op = "azureblob.ListBlobs" const op errors.Op = "azureblob.ListBlobs"
var blobs []string var blobs []string
@@ -122,7 +111,7 @@ func (c *azureBlobStoreClient) ListBlobs(ctx context.Context, prefix string) ([]
return blobs, nil return blobs, nil
} }
// DeleteBlob deletes the blob with the given path // DeleteBlob deletes the blob with the given path.
func (c *azureBlobStoreClient) DeleteBlob(ctx context.Context, path string) error { func (c *azureBlobStoreClient) DeleteBlob(ctx context.Context, path string) error {
const op errors.Op = "azureblob.DeleteBlob" const op errors.Op = "azureblob.DeleteBlob"
blobURL := c.containerURL.NewBlockBlobURL(path) blobURL := c.containerURL.NewBlockBlobURL(path)
@@ -133,7 +122,7 @@ func (c *azureBlobStoreClient) DeleteBlob(ctx context.Context, path string) erro
return nil return nil
} }
// UploadWithContext uploads a blob to the container // UploadWithContext uploads a blob to the container.
func (c *azureBlobStoreClient) UploadWithContext(ctx context.Context, path, contentType string, content io.Reader) error { func (c *azureBlobStoreClient) UploadWithContext(ctx context.Context, path, contentType string, content io.Reader) error {
const op errors.Op = "azureblob.UploadWithContext" const op errors.Op = "azureblob.UploadWithContext"
ctx, span := observ.StartSpan(ctx, op.String()) ctx, span := observ.StartSpan(ctx, op.String())
+2 -2
View File
@@ -12,8 +12,8 @@ import (
"github.com/gomods/athens/pkg/paths" "github.com/gomods/athens/pkg/paths"
) )
// Catalog implements the (./pkg/storage).Catalog interface // Catalog implements the (./pkg/storage).Catalog interface.
// It returns a list of versions, if any, for a given module // It returns a list of versions, if any, for a given module.
func (s *Storage) Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error) { func (s *Storage) Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error) {
const op errors.Op = "azblob.Catalog" const op errors.Op = "azblob.Catalog"
ctx, span := observ.StartSpan(ctx, op.String()) ctx, span := observ.StartSpan(ctx, op.String())
+2 -2
View File
@@ -9,8 +9,8 @@ import (
) )
// Exists implements the (./pkg/storage).Checker interface // Exists implements the (./pkg/storage).Checker interface
// returning true if the module at version exists in storage // returning true if the module at version exists in storage.
func (s *Storage) Exists(ctx context.Context, module string, version string) (bool, error) { func (s *Storage) Exists(ctx context.Context, module, version string) (bool, error) {
const op errors.Op = "azureblob.Exists" const op errors.Op = "azureblob.Exists"
ctx, span := observ.StartSpan(ctx, op.String()) ctx, span := observ.StartSpan(ctx, op.String())
defer span.End() defer span.End()
+1 -1
View File
@@ -11,7 +11,7 @@ import (
// Delete implements the (./pkg/storage).Deleter interface and // Delete implements the (./pkg/storage).Deleter interface and
// removes a version of a module from storage. Returning ErrNotFound // removes a version of a module from storage. Returning ErrNotFound
// if the version does not exist. // if the version does not exist.
func (s *Storage) Delete(ctx context.Context, module string, version string) error { func (s *Storage) Delete(ctx context.Context, module, version string) error {
const op errors.Op = "azureblob.Delete" const op errors.Op = "azureblob.Delete"
ctx, span := observ.StartSpan(ctx, op.String()) ctx, span := observ.StartSpan(ctx, op.String())
defer span.End() defer span.End()
+7 -7
View File
@@ -11,8 +11,8 @@ import (
"github.com/gomods/athens/pkg/storage" "github.com/gomods/athens/pkg/storage"
) )
// Info implements the (./pkg/storage).Getter interface // Info implements the (./pkg/storage).Getter interface.
func (s *Storage) Info(ctx context.Context, module string, version string) ([]byte, error) { func (s *Storage) Info(ctx context.Context, module, version string) ([]byte, error) {
const op errors.Op = "azureblob.Info" const op errors.Op = "azureblob.Info"
ctx, span := observ.StartSpan(ctx, op.String()) ctx, span := observ.StartSpan(ctx, op.String())
defer span.End() defer span.End()
@@ -43,8 +43,8 @@ func (s *Storage) Info(ctx context.Context, module string, version string) ([]by
return infoBytes, nil return infoBytes, nil
} }
// GoMod implements the (./pkg/storage).Getter interface // GoMod implements the (./pkg/storage).Getter interface.
func (s *Storage) GoMod(ctx context.Context, module string, version string) ([]byte, error) { func (s *Storage) GoMod(ctx context.Context, module, version string) ([]byte, error) {
const op errors.Op = "azureblob.GoMod" const op errors.Op = "azureblob.GoMod"
ctx, span := observ.StartSpan(ctx, op.String()) ctx, span := observ.StartSpan(ctx, op.String())
defer span.End() defer span.End()
@@ -64,7 +64,7 @@ func (s *Storage) GoMod(ctx context.Context, module string, version string) ([]b
modBytes, err := io.ReadAll(modReader) modBytes, err := io.ReadAll(modReader)
if err != nil { if err != nil {
return nil, errors.E(op, fmt.Errorf("could not get new reader for mod file: %s", err), errors.M(module), errors.V(version)) return nil, errors.E(op, fmt.Errorf("could not get new reader for mod file: %w", err), errors.M(module), errors.V(version))
} }
err = modReader.Close() err = modReader.Close()
@@ -75,8 +75,8 @@ func (s *Storage) GoMod(ctx context.Context, module string, version string) ([]b
return modBytes, nil return modBytes, nil
} }
// Zip implements the (./pkg/storage).Getter interface // Zip implements the (./pkg/storage).Getter interface.
func (s *Storage) Zip(ctx context.Context, module string, version string) (storage.SizeReadCloser, error) { func (s *Storage) Zip(ctx context.Context, module, version string) (storage.SizeReadCloser, error) {
const op errors.Op = "azureblob.Zip" const op errors.Op = "azureblob.Zip"
ctx, span := observ.StartSpan(ctx, op.String()) ctx, span := observ.StartSpan(ctx, op.String())
defer span.End() defer span.End()
+2 -2
View File
@@ -8,8 +8,8 @@ import (
"github.com/gomods/athens/pkg/observ" "github.com/gomods/athens/pkg/observ"
) )
// List implements the (./pkg/storage).Lister interface // List implements the (./pkg/storage).Lister interface.
// It returns a list of versions, if any, for a given module // It returns a list of versions, if any, for a given module.
func (s *Storage) List(ctx context.Context, module string) ([]string, error) { func (s *Storage) List(ctx context.Context, module string) ([]string, error) {
const op errors.Op = "azureblob.List" const op errors.Op = "azureblob.List"
ctx, span := observ.StartSpan(ctx, op.String()) ctx, span := observ.StartSpan(ctx, op.String())
+1 -1
View File
@@ -1,6 +1,6 @@
package storage package storage
// Backend is a complete storage backend (i.e. file system, database) implementation - a lister, reader and saver // Backend is a complete storage backend (i.e. file system, database) implementation - a lister, reader and saver.
type Backend interface { type Backend interface {
Lister Lister
Getter Getter
+1 -1
View File
@@ -6,7 +6,7 @@ import (
"github.com/gomods/athens/pkg/paths" "github.com/gomods/athens/pkg/paths"
) )
// Cataloger is the interface that lists all the modules and version contained in the storage // Cataloger is the interface that lists all the modules and version contained in the storage.
type Cataloger interface { type Cataloger interface {
// Catalog gets all the modules / versions. // Catalog gets all the modules / versions.
Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error) Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error)
+3 -3
View File
@@ -6,14 +6,14 @@ import (
"github.com/gomods/athens/pkg/errors" "github.com/gomods/athens/pkg/errors"
) )
// Checker is the interface that checks if the version of the module exists // Checker is the interface that checks if the version of the module exists.
type Checker interface { type Checker interface {
// Exists checks whether or not module in specified version is present // Exists checks whether or not module in specified version is present
// in the backing storage // in the backing storage.
Exists(ctx context.Context, module, version string) (bool, error) Exists(ctx context.Context, module, version string) (bool, error)
} }
// WithChecker wraps the backend with a Checker implementaiton // WithChecker wraps the backend with a Checker implementation.
func WithChecker(strg Backend) Checker { func WithChecker(strg Backend) Checker {
if checker, ok := strg.(Checker); ok { if checker, ok := strg.(Checker); ok {
return checker return checker
-45
View File
@@ -6,7 +6,6 @@ import (
"fmt" "fmt"
"io" "io"
"math/rand" "math/rand"
"sort"
"testing" "testing"
"github.com/gomods/athens/pkg/errors" "github.com/gomods/athens/pkg/errors"
@@ -26,7 +25,6 @@ func RunTests(t *testing.T, b storage.Backend, clearBackend func() error) {
testGet(t, b) testGet(t, b)
testExists(t, b) testExists(t, b)
testShouldNotExist(t, b) testShouldNotExist(t, b)
// testCatalog(t, b)
} }
// testNotFound ensures that a storage Backend // testNotFound ensures that a storage Backend
@@ -209,49 +207,6 @@ func testDelete(t *testing.T, b storage.Backend) {
require.Equal(t, false, exists) require.Equal(t, false, exists)
} }
func testCatalog(t *testing.T, b storage.Backend) {
cs, ok := b.(storage.Cataloger)
if !ok {
t.Skip()
}
ctx := context.Background()
mock := getMockModule()
zipBts, _ := io.ReadAll(mock.Zip)
modname := "github.com/gomods/testCatalogModule"
for i := 0; i < 6; i++ {
ver := fmt.Sprintf("v1.2.%04d", i)
b.Save(ctx, modname, ver, mock.Mod, bytes.NewReader(zipBts), mock.Info)
defer b.Delete(ctx, modname, ver)
}
allres, next, err := cs.Catalog(ctx, "", 5)
require.NoError(t, err)
require.Equal(t, 5, len(allres))
res, next, err := cs.Catalog(ctx, next, 50)
allres = append(allres, res...)
require.NoError(t, err)
require.Equal(t, 1, len(res))
require.Equal(t, "", next)
sort.Slice(allres, func(i, j int) bool {
if allres[i].Module == allres[j].Module {
return allres[i].Version < allres[j].Version
}
return allres[i].Module < allres[j].Module
})
require.Equal(t, modname, allres[0].Module)
require.Equal(t, "v1.2.0000", allres[0].Version)
require.Equal(t, "v1.2.0004", allres[4].Version)
for i := 1; i < len(allres); i++ {
require.NotEqual(t, allres[i].Version, allres[i-1].Version)
}
}
func getMockModule() *storage.Version { func getMockModule() *storage.Version {
return &storage.Version{ return &storage.Version{
Info: []byte("123"), Info: []byte("123"),
+1 -1
View File
@@ -2,7 +2,7 @@ package storage
import "context" import "context"
// Deleter deletes module metadata and its source from underlying storage // Deleter deletes module metadata and its source from underlying storage.
type Deleter interface { type Deleter interface {
// Delete must return ErrNotFound if the module/version are not // Delete must return ErrNotFound if the module/version are not
// found. // found.
+14 -14
View File
@@ -20,7 +20,7 @@ type service struct {
c *http.Client c *http.Client
} }
// NewClient returns an external storage client // NewClient returns an external storage client.
func NewClient(url string, c *http.Client) storage.Backend { func NewClient(url string, c *http.Client) storage.Backend {
if c == nil { if c == nil {
c = &http.Client{} c = &http.Client{}
@@ -93,7 +93,7 @@ func (s *service) Save(ctx context.Context, mod, ver string, modFile []byte, zip
mw := multipart.NewWriter(pw) mw := multipart.NewWriter(pw)
go func() { go func() {
err := upload(mw, modFile, info, zip) err := upload(mw, modFile, info, zip)
pw.CloseWithError(err) _ = pw.CloseWithError(err)
}() }()
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, pr) req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, pr)
if err != nil { if err != nil {
@@ -104,8 +104,8 @@ func (s *service) Save(ctx context.Context, mod, ver string, modFile []byte, zip
if err != nil { if err != nil {
return errors.E(op, err) return errors.E(op, err)
} }
defer resp.Body.Close() defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != 200 { if resp.StatusCode != http.StatusOK {
bts, _ := io.ReadAll(resp.Body) bts, _ := io.ReadAll(resp.Body)
return errors.E(op, fmt.Errorf("unexpected status code: %v - body: %s", resp.StatusCode, bts), resp.StatusCode) return errors.E(op, fmt.Errorf("unexpected status code: %v - body: %s", resp.StatusCode, bts), resp.StatusCode)
} }
@@ -118,35 +118,35 @@ func (s *service) Delete(ctx context.Context, mod, ver string) error {
if err != nil { if err != nil {
return errors.E(op, err) return errors.E(op, err)
} }
defer body.Close() defer func() { _ = body.Close() }()
return nil return nil
} }
func upload(mw *multipart.Writer, mod, info []byte, zip io.Reader) error { func upload(mw *multipart.Writer, mod, info []byte, zip io.Reader) error {
defer mw.Close() defer func() { _ = mw.Close() }()
infoW, err := mw.CreateFormFile("mod.info", "mod.info") infoW, err := mw.CreateFormFile("mod.info", "mod.info")
if err != nil { if err != nil {
return fmt.Errorf("error creating info file: %v", err) return fmt.Errorf("error creating info file: %w", err)
} }
_, err = infoW.Write(info) _, err = infoW.Write(info)
if err != nil { if err != nil {
return fmt.Errorf("error writing info file: %v", err) return fmt.Errorf("error writing info file: %w", err)
} }
modW, err := mw.CreateFormFile("mod.mod", "mod.mod") modW, err := mw.CreateFormFile("mod.mod", "mod.mod")
if err != nil { if err != nil {
return fmt.Errorf("error creating mod file: %v", err) return fmt.Errorf("error creating mod file: %w", err)
} }
_, err = modW.Write(mod) _, err = modW.Write(mod)
if err != nil { if err != nil {
return fmt.Errorf("error writing mod file: %v", err) return fmt.Errorf("error writing mod file: %w", err)
} }
zipW, err := mw.CreateFormFile("mod.zip", "mod.zip") zipW, err := mw.CreateFormFile("mod.zip", "mod.zip")
if err != nil { if err != nil {
return fmt.Errorf("error creating zip file: %v", err) return fmt.Errorf("error creating zip file: %w", err)
} }
_, err = io.Copy(zipW, zip) _, err = io.Copy(zipW, zip)
if err != nil { if err != nil {
return fmt.Errorf("error writing zip file: %v", err) return fmt.Errorf("error writing zip file: %w", err)
} }
return nil return nil
} }
@@ -174,9 +174,9 @@ func (s *service) doRequest(ctx context.Context, method, mod, ver, ext string) (
if err != nil { if err != nil {
return nil, 0, errors.E(op, err) return nil, 0, errors.E(op, err)
} }
if resp.StatusCode != 200 { if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body) body, _ := io.ReadAll(resp.Body)
resp.Body.Close() _ = resp.Body.Close()
return nil, 0, errors.E(op, fmt.Errorf("none 200 status code: %v - body: %s", resp.StatusCode, body), resp.StatusCode) return nil, 0, errors.E(op, fmt.Errorf("none 200 status code: %v - body: %s", resp.StatusCode, body), resp.StatusCode)
} }
var size int64 var size int64
+21 -21
View File
@@ -17,7 +17,7 @@ import (
// NewServer takes a storage.Backend implementation of your // NewServer takes a storage.Backend implementation of your
// choice, and returns a new http.Handler that Athens can // choice, and returns a new http.Handler that Athens can
// reach out to for storage operations // reach out to for storage operations.
func NewServer(strg storage.Backend) http.Handler { func NewServer(strg storage.Backend) http.Handler {
r := mux.NewRouter() r := mux.NewRouter()
r.HandleFunc(download.PathList, func(w http.ResponseWriter, r *http.Request) { r.HandleFunc(download.PathList, func(w http.ResponseWriter, r *http.Request) {
@@ -27,12 +27,12 @@ func NewServer(strg storage.Backend) http.Handler {
http.Error(w, err.Error(), errors.Kind(err)) http.Error(w, err.Error(), errors.Kind(err))
return return
} }
fmt.Fprintf(w, "%s", strings.Join(list, "\n")) _, _ = fmt.Fprintf(w, "%s", strings.Join(list, "\n"))
}).Methods(http.MethodGet) }).Methods(http.MethodGet)
r.HandleFunc(download.PathVersionInfo, func(w http.ResponseWriter, r *http.Request) { r.HandleFunc(download.PathVersionInfo, func(w http.ResponseWriter, r *http.Request) {
params, err := paths.GetAllParams(r) params, err := paths.GetAllParams(r)
if err != nil { if err != nil {
http.Error(w, err.Error(), 400) http.Error(w, err.Error(), http.StatusBadRequest)
return return
} }
info, err := strg.Info(r.Context(), params.Module, params.Version) info, err := strg.Info(r.Context(), params.Module, params.Version)
@@ -40,12 +40,12 @@ func NewServer(strg storage.Backend) http.Handler {
http.Error(w, err.Error(), errors.Kind(err)) http.Error(w, err.Error(), errors.Kind(err))
return return
} }
w.Write(info) _, _ = w.Write(info)
}).Methods(http.MethodGet) }).Methods(http.MethodGet)
r.HandleFunc(download.PathVersionModule, func(w http.ResponseWriter, r *http.Request) { r.HandleFunc(download.PathVersionModule, func(w http.ResponseWriter, r *http.Request) {
params, err := paths.GetAllParams(r) params, err := paths.GetAllParams(r)
if err != nil { if err != nil {
http.Error(w, err.Error(), 400) http.Error(w, err.Error(), http.StatusBadRequest)
return return
} }
mod, err := strg.GoMod(r.Context(), params.Module, params.Version) mod, err := strg.GoMod(r.Context(), params.Module, params.Version)
@@ -53,12 +53,12 @@ func NewServer(strg storage.Backend) http.Handler {
http.Error(w, err.Error(), errors.Kind(err)) http.Error(w, err.Error(), errors.Kind(err))
return return
} }
w.Write(mod) _, _ = w.Write(mod)
}).Methods(http.MethodGet) }).Methods(http.MethodGet)
r.HandleFunc(download.PathVersionZip, func(w http.ResponseWriter, r *http.Request) { r.HandleFunc(download.PathVersionZip, func(w http.ResponseWriter, r *http.Request) {
params, err := paths.GetAllParams(r) params, err := paths.GetAllParams(r)
if err != nil { if err != nil {
http.Error(w, err.Error(), 400) http.Error(w, err.Error(), http.StatusBadRequest)
return return
} }
zip, err := strg.Zip(r.Context(), params.Module, params.Version) zip, err := strg.Zip(r.Context(), params.Module, params.Version)
@@ -66,52 +66,52 @@ func NewServer(strg storage.Backend) http.Handler {
http.Error(w, err.Error(), errors.Kind(err)) http.Error(w, err.Error(), errors.Kind(err))
return return
} }
defer zip.Close() defer func() { _ = zip.Close() }()
w.Header().Set("Content-Length", strconv.FormatInt(zip.Size(), 10)) w.Header().Set("Content-Length", strconv.FormatInt(zip.Size(), 10))
io.Copy(w, zip) _, _ = io.Copy(w, zip)
}).Methods(http.MethodGet) }).Methods(http.MethodGet)
r.HandleFunc("/{module:.+}/@v/{version}.save", func(w http.ResponseWriter, r *http.Request) { r.HandleFunc("/{module:.+}/@v/{version}.save", func(w http.ResponseWriter, r *http.Request) {
params, err := paths.GetAllParams(r) params, err := paths.GetAllParams(r)
if err != nil { if err != nil {
http.Error(w, err.Error(), 400) http.Error(w, err.Error(), http.StatusBadRequest)
return return
} }
err = r.ParseMultipartForm(zip.MaxZipFile + zip.MaxGoMod) err = r.ParseMultipartForm(zip.MaxZipFile + zip.MaxGoMod)
if err != nil { if err != nil {
http.Error(w, err.Error(), 400) http.Error(w, err.Error(), http.StatusBadRequest)
return return
} }
infoFile, _, err := r.FormFile("mod.info") infoFile, _, err := r.FormFile("mod.info")
if err != nil { if err != nil {
http.Error(w, err.Error(), 400) http.Error(w, err.Error(), http.StatusBadRequest)
return return
} }
defer infoFile.Close() defer func() { _ = infoFile.Close() }()
info, err := io.ReadAll(infoFile) info, err := io.ReadAll(infoFile)
if err != nil { if err != nil {
http.Error(w, err.Error(), 400) http.Error(w, err.Error(), http.StatusBadRequest)
return return
} }
modReader, _, err := r.FormFile("mod.mod") modReader, _, err := r.FormFile("mod.mod")
if err != nil { if err != nil {
http.Error(w, err.Error(), 400) http.Error(w, err.Error(), http.StatusBadRequest)
return return
} }
defer modReader.Close() defer func() { _ = modReader.Close() }()
modFile, err := io.ReadAll(modReader) modFile, err := io.ReadAll(modReader)
if err != nil { if err != nil {
http.Error(w, err.Error(), 400) http.Error(w, err.Error(), http.StatusBadRequest)
return return
} }
modZ, _, err := r.FormFile("mod.zip") modZ, _, err := r.FormFile("mod.zip")
if err != nil { if err != nil {
http.Error(w, err.Error(), 400) http.Error(w, err.Error(), http.StatusBadRequest)
return return
} }
defer modZ.Close() defer func() { _ = modZ.Close() }()
err = strg.Save(r.Context(), params.Module, params.Version, modFile, modZ, info) err = strg.Save(r.Context(), params.Module, params.Version, modFile, modZ, info)
if err != nil { if err != nil {
http.Error(w, err.Error(), 400) http.Error(w, err.Error(), http.StatusBadRequest)
return return
} }
}).Methods(http.MethodPost) }).Methods(http.MethodPost)
@@ -119,7 +119,7 @@ func NewServer(strg storage.Backend) http.Handler {
r.HandleFunc("/{module:.+}/@v/{version}.delete", func(w http.ResponseWriter, r *http.Request) { r.HandleFunc("/{module:.+}/@v/{version}.delete", func(w http.ResponseWriter, r *http.Request) {
params, err := paths.GetAllParams(r) params, err := paths.GetAllParams(r)
if err != nil { if err != nil {
http.Error(w, err.Error(), 400) http.Error(w, err.Error(), http.StatusBadRequest)
return return
} }
err = strg.Delete(r.Context(), params.Module, params.Version) err = strg.Delete(r.Context(), params.Module, params.Version)
+5 -5
View File
@@ -15,11 +15,11 @@ import (
const tokenSeparator = "|" const tokenSeparator = "|"
// Catalog implements the (./pkg/storage).Cataloger interface // Catalog implements the (./pkg/storage).Cataloger interface.
// It returns a list of modules and versions contained in the storage // It returns a list of modules and versions contained in the storage.
func (s *storageImpl) Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error) { func (s *storageImpl) Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error) {
const op errors.Op = "fs.Catalog" const op errors.Op = "fs.Catalog"
ctx, span := observ.StartSpan(ctx, op.String()) _, span := observ.StartSpan(ctx, op.String())
defer span.End() defer span.End()
fromModule, fromVersion, err := modVerFromToken(token) fromModule, fromVersion, err := modVerFromToken(token)
@@ -41,7 +41,7 @@ func (s *storageImpl) Catalog(ctx context.Context, token string, pageSize int) (
m, version := filepath.Split(modVer) m, version := filepath.Split(modVer)
module := filepath.Clean(m) module := filepath.Clean(m)
module = strings.Replace(module, string(os.PathSeparator), "/", -1) module = strings.ReplaceAll(module, string(os.PathSeparator), "/")
if fromModule != "" && module < fromModule { // it is ok to land on the same module if fromModule != "" && module < fromModule { // it is ok to land on the same module
return nil return nil
@@ -60,7 +60,7 @@ func (s *storageImpl) Catalog(ctx context.Context, token string, pageSize int) (
} }
return nil return nil
}) })
if err != nil && err != io.EOF { if err != nil && !errors.IsErr(err, io.EOF) {
return nil, "", errors.E(op, err, errors.KindUnexpected) return nil, "", errors.E(op, err, errors.KindUnexpected)
} }
+4 -5
View File
@@ -9,14 +9,13 @@ import (
"github.com/spf13/afero" "github.com/spf13/afero"
) )
func (v *storageImpl) Exists(ctx context.Context, module, version string) (bool, error) { func (s *storageImpl) Exists(ctx context.Context, module, version string) (bool, error) {
const op errors.Op = "fs.Exists" const op errors.Op = "fs.Exists"
ctx, span := observ.StartSpan(ctx, op.String()) _, span := observ.StartSpan(ctx, op.String())
defer span.End() defer span.End()
versionedPath := v.versionLocation(module, version) versionedPath := s.versionLocation(module, version)
files, err := afero.ReadDir(v.filesystem, versionedPath)
files, err := afero.ReadDir(s.filesystem, versionedPath)
if err != nil { if err != nil {
if os.IsNotExist(err) { if os.IsNotExist(err) {
return false, nil return false, nil
+4 -4
View File
@@ -8,17 +8,17 @@ import (
) )
// Delete removes a specific version of a module. // Delete removes a specific version of a module.
func (v *storageImpl) Delete(ctx context.Context, module, version string) error { func (s *storageImpl) Delete(ctx context.Context, module, version string) error {
const op errors.Op = "fs.Delete" const op errors.Op = "fs.Delete"
ctx, span := observ.StartSpan(ctx, op.String()) ctx, span := observ.StartSpan(ctx, op.String())
defer span.End() defer span.End()
versionedPath := v.versionLocation(module, version) versionedPath := s.versionLocation(module, version)
exists, err := v.Exists(ctx, module, version) exists, err := s.Exists(ctx, module, version)
if err != nil { if err != nil {
return errors.E(op, err, errors.M(module), errors.V(version)) return errors.E(op, err, errors.M(module), errors.V(version))
} }
if !exists { if !exists {
return errors.E(op, errors.M(module), errors.V(version), errors.KindNotFound) return errors.E(op, errors.M(module), errors.V(version), errors.KindNotFound)
} }
return v.filesystem.RemoveAll(versionedPath) return s.filesystem.RemoveAll(versionedPath)
} }
+3 -4
View File
@@ -21,17 +21,16 @@ func (s *storageImpl) moduleLocation(module string) string {
func (s *storageImpl) versionLocation(module, version string) string { func (s *storageImpl) versionLocation(module, version string) string {
return filepath.Join(s.moduleLocation(module), version) return filepath.Join(s.moduleLocation(module), version)
} }
// NewStorage returns a new ListerSaver implementation that stores // NewStorage returns a new ListerSaver implementation that stores
// everything under rootDir // everything under rootDir.
// If the root directory does not exist an error is returned // If the root directory does not exist an error is returned.
func NewStorage(rootDir string, filesystem afero.Fs) (storage.Backend, error) { func NewStorage(rootDir string, filesystem afero.Fs) (storage.Backend, error) {
const op errors.Op = "fs.NewStorage" const op errors.Op = "fs.NewStorage"
exists, err := afero.Exists(filesystem, rootDir) exists, err := afero.Exists(filesystem, rootDir)
if err != nil { if err != nil {
return nil, errors.E(op, fmt.Errorf("could not check if root directory `%s` exists: %s", rootDir, err)) return nil, errors.E(op, fmt.Errorf("could not check if root directory `%s` exists: %w", rootDir, err))
} }
if !exists { if !exists {
return nil, errors.E(op, fmt.Errorf("root directory `%s` does not exist", rootDir)) return nil, errors.E(op, fmt.Errorf("root directory `%s` does not exist", rootDir))
+12 -24
View File
@@ -11,12 +11,12 @@ import (
"github.com/spf13/afero" "github.com/spf13/afero"
) )
func (v *storageImpl) Info(ctx context.Context, module, version string) ([]byte, error) { func (s *storageImpl) Info(ctx context.Context, module, version string) ([]byte, error) {
const op errors.Op = "fs.Info" const op errors.Op = "fs.Info"
ctx, span := observ.StartSpan(ctx, op.String()) _, span := observ.StartSpan(ctx, op.String())
defer span.End() defer span.End()
versionedPath := v.versionLocation(module, version) versionedPath := s.versionLocation(module, version)
info, err := afero.ReadFile(v.filesystem, filepath.Join(versionedPath, version+".info")) info, err := afero.ReadFile(s.filesystem, filepath.Join(versionedPath, version+".info"))
if err != nil { if err != nil {
return nil, errors.E(op, errors.M(module), errors.V(version), errors.KindNotFound) return nil, errors.E(op, errors.M(module), errors.V(version), errors.KindNotFound)
} }
@@ -24,12 +24,12 @@ func (v *storageImpl) Info(ctx context.Context, module, version string) ([]byte,
return info, nil return info, nil
} }
func (v *storageImpl) GoMod(ctx context.Context, module, version string) ([]byte, error) { func (s *storageImpl) GoMod(ctx context.Context, module, version string) ([]byte, error) {
const op errors.Op = "fs.GoMod" const op errors.Op = "fs.GoMod"
ctx, span := observ.StartSpan(ctx, op.String()) _, span := observ.StartSpan(ctx, op.String())
defer span.End() defer span.End()
versionedPath := v.versionLocation(module, version) versionedPath := s.versionLocation(module, version)
mod, err := afero.ReadFile(v.filesystem, filepath.Join(versionedPath, "go.mod")) mod, err := afero.ReadFile(s.filesystem, filepath.Join(versionedPath, "go.mod"))
if err != nil { if err != nil {
return nil, errors.E(op, errors.M(module), errors.V(version), errors.KindNotFound) return nil, errors.E(op, errors.M(module), errors.V(version), errors.KindNotFound)
} }
@@ -37,13 +37,13 @@ func (v *storageImpl) GoMod(ctx context.Context, module, version string) ([]byte
return mod, nil return mod, nil
} }
func (v *storageImpl) Zip(ctx context.Context, module, version string) (storage.SizeReadCloser, error) { func (s *storageImpl) Zip(ctx context.Context, module, version string) (storage.SizeReadCloser, error) {
const op errors.Op = "fs.Zip" const op errors.Op = "fs.Zip"
ctx, span := observ.StartSpan(ctx, op.String()) _, span := observ.StartSpan(ctx, op.String())
defer span.End() defer span.End()
versionedPath := v.versionLocation(module, version) versionedPath := s.versionLocation(module, version)
src, err := v.filesystem.OpenFile(filepath.Join(versionedPath, "source.zip"), os.O_RDONLY, 0666) src, err := s.filesystem.OpenFile(filepath.Join(versionedPath, "source.zip"), os.O_RDONLY, 0o666)
if err != nil { if err != nil {
return nil, errors.E(op, errors.M(module), errors.V(version), errors.KindNotFound) return nil, errors.E(op, errors.M(module), errors.V(version), errors.KindNotFound)
} }
@@ -53,15 +53,3 @@ func (v *storageImpl) Zip(ctx context.Context, module, version string) (storage.
} }
return storage.NewSizer(src, fi.Size()), nil return storage.NewSizer(src, fi.Size()), nil
} }
func (v *storageImpl) ZipSize(ctx context.Context, module, version string) (int64, error) {
const op errors.Op = "fs.ZipFileSize"
ctx, span := observ.StartSpan(ctx, op.String())
defer span.End()
versionedPath := v.versionLocation(module, version)
fi, err := v.filesystem.Stat(filepath.Join(versionedPath))
if err != nil {
return 0, errors.E(op, err, errors.M(module), errors.V(version), errors.KindNotFound)
}
return fi.Size(), nil
}
+4 -4
View File
@@ -11,12 +11,12 @@ import (
"golang.org/x/mod/semver" "golang.org/x/mod/semver"
) )
func (l *storageImpl) List(ctx context.Context, module string) ([]string, error) { func (s *storageImpl) List(ctx context.Context, module string) ([]string, error) {
const op errors.Op = "fs.List" const op errors.Op = "fs.List"
ctx, span := observ.StartSpan(ctx, op.String()) _, span := observ.StartSpan(ctx, op.String())
defer span.End() defer span.End()
loc := l.moduleLocation(module) loc := s.moduleLocation(module)
fileInfos, err := afero.ReadDir(l.filesystem, loc) fileInfos, err := afero.ReadDir(s.filesystem, loc)
if err != nil { if err != nil {
if os.IsNotExist(err) { if os.IsNotExist(err) {
return []string{}, nil return []string{}, nil
+12 -12
View File
@@ -13,37 +13,37 @@ import (
func (s *storageImpl) Save(ctx context.Context, module, version string, mod []byte, zip io.Reader, info []byte) error { func (s *storageImpl) Save(ctx context.Context, module, version string, mod []byte, zip io.Reader, info []byte) error {
const op errors.Op = "fs.Save" const op errors.Op = "fs.Save"
ctx, span := observ.StartSpan(ctx, op.String()) _, span := observ.StartSpan(ctx, op.String())
defer span.End() defer span.End()
dir := s.versionLocation(module, version) dir := s.versionLocation(module, version)
// NB: the process's umask is subtracted from the permissions below, // NB: The process's umask is subtracted from the permissions below,
// so a umask of for example 0077 allows directories and files to be // so an umask of for example 0077 allows directories and files to be
// created with mode 0700 / 0600, i.e. not world- or group-readable // created with mode 0700 / 0600, i.e. not world- or group-readable.
// make the versioned directory to hold the go.mod and the zipfile // Make the versioned directory to hold the go.mod and the zipfile.
if err := s.filesystem.MkdirAll(dir, 0777); err != nil { if err := s.filesystem.MkdirAll(dir, 0o777); err != nil {
return errors.E(op, err, errors.M(module), errors.V(version)) return errors.E(op, err, errors.M(module), errors.V(version))
} }
// write the go.mod file // Write the go.mod file.
if err := afero.WriteFile(s.filesystem, filepath.Join(dir, "go.mod"), mod, 0666); err != nil { if err := afero.WriteFile(s.filesystem, filepath.Join(dir, "go.mod"), mod, 0o666); err != nil {
return errors.E(op, err, errors.M(module), errors.V(version)) return errors.E(op, err, errors.M(module), errors.V(version))
} }
// write the zipfile // Write the zipfile.
f, err := s.filesystem.OpenFile(filepath.Join(dir, "source.zip"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666) f, err := s.filesystem.OpenFile(filepath.Join(dir, "source.zip"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o666)
if err != nil { if err != nil {
return errors.E(op, err, errors.M(module), errors.V(version)) return errors.E(op, err, errors.M(module), errors.V(version))
} }
defer f.Close() defer func() { _ = f.Close() }()
_, err = io.Copy(f, zip) _, err = io.Copy(f, zip)
if err != nil { if err != nil {
return errors.E(op, err, errors.M(module), errors.V(version)) return errors.E(op, err, errors.M(module), errors.V(version))
} }
// write the info file // write the info file
err = afero.WriteFile(s.filesystem, filepath.Join(dir, version+".info"), info, 0666) err = afero.WriteFile(s.filesystem, filepath.Join(dir, version+".info"), info, 0o666)
if err != nil { if err != nil {
return errors.E(op, err) return errors.E(op, err)
} }
+1 -1
View File
@@ -14,7 +14,7 @@ import (
) )
// Catalog implements the (./pkg/storage).Catalog interface // Catalog implements the (./pkg/storage).Catalog interface
// It returns a list of versions, if any, for a given module // It returns a list of versions, if any, for a given module.
func (s *Storage) Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error) { func (s *Storage) Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error) {
const op errors.Op = "gcp.Catalog" const op errors.Op = "gcp.Catalog"
ctx, span := observ.StartSpan(ctx, op.String()) ctx, span := observ.StartSpan(ctx, op.String())
+2 -2
View File
@@ -11,7 +11,7 @@ import (
) )
// Exists implements the (./pkg/storage).Checker interface // Exists implements the (./pkg/storage).Checker interface
// returning true if the module at version exists in storage // returning true if the module at version exists in storage.
func (s *Storage) Exists(ctx context.Context, module, version string) (bool, error) { func (s *Storage) Exists(ctx context.Context, module, version string) (bool, error) {
const op errors.Op = "gcp.Exists" const op errors.Op = "gcp.Exists"
ctx, span := observ.StartSpan(ctx, op.String()) ctx, span := observ.StartSpan(ctx, op.String())
@@ -21,7 +21,7 @@ func (s *Storage) Exists(ctx context.Context, module, version string) (bool, err
var count int var count int
for { for {
attrs, err := it.Next() attrs, err := it.Next()
if err == iterator.Done { if errors.IsErr(err, iterator.Done) {
break break
} }
if err != nil { if err != nil {
+7 -7
View File
@@ -13,7 +13,7 @@ import (
"google.golang.org/api/option" "google.golang.org/api/option"
) )
// Storage implements the (./pkg/storage).Backend interface // Storage implements the (./pkg/storage).Backend interface.
type Storage struct { type Storage struct {
bucket *storage.BucketHandle bucket *storage.BucketHandle
timeout time.Duration timeout time.Duration
@@ -34,8 +34,8 @@ func New(ctx context.Context, gcpConf *config.GCPConfig, timeout time.Duration)
return nil, errors.E(op, err) return nil, errors.E(op, err)
} }
if _, err := s.bucket.Attrs(ctx); err != nil { if _, err = s.bucket.Attrs(ctx); err != nil {
if err == storage.ErrBucketNotExist { if errors.IsErr(err, storage.ErrBucketNotExist) {
return nil, errors.E(op, "You must manually create a storage bucket for Athens, see https://cloud.google.com/storage/docs/creating-buckets#storage-create-bucket-console") return nil, errors.E(op, "You must manually create a storage bucket for Athens, see https://cloud.google.com/storage/docs/creating-buckets#storage-create-bucket-console")
} }
return nil, errors.E(op, err) return nil, errors.E(op, err)
@@ -48,21 +48,21 @@ func New(ctx context.Context, gcpConf *config.GCPConfig, timeout time.Duration)
// this is so that the unit tests can use this to create their own short-lived buckets. // this is so that the unit tests can use this to create their own short-lived buckets.
func newClient(ctx context.Context, gcpConf *config.GCPConfig, timeout time.Duration) (*Storage, error) { func newClient(ctx context.Context, gcpConf *config.GCPConfig, timeout time.Duration) (*Storage, error) {
const op errors.Op = "gcp.newClient" const op errors.Op = "gcp.newClient"
opts := []option.ClientOption{} var opts []option.ClientOption
if gcpConf.JSONKey != "" { if gcpConf.JSONKey != "" {
key, err := base64.StdEncoding.DecodeString(gcpConf.JSONKey) key, err := base64.StdEncoding.DecodeString(gcpConf.JSONKey)
if err != nil { if err != nil {
return nil, errors.E(op, fmt.Errorf("could not decode base64 json key: %v", err)) return nil, errors.E(op, fmt.Errorf("could not decode base64 json key: %w", err))
} }
creds, err := google.CredentialsFromJSON(ctx, key, storage.ScopeReadWrite) creds, err := google.CredentialsFromJSON(ctx, key, storage.ScopeReadWrite)
if err != nil { if err != nil {
return nil, errors.E(op, fmt.Errorf("could not get GCS credentials: %v", err)) return nil, errors.E(op, fmt.Errorf("could not get GCS credentials: %w", err))
} }
opts = append(opts, option.WithCredentials(creds)) opts = append(opts, option.WithCredentials(creds))
} }
s, err := storage.NewClient(ctx, opts...) s, err := storage.NewClient(ctx, opts...)
if err != nil { if err != nil {
return nil, errors.E(op, fmt.Errorf("could not create new storage client: %s", err)) return nil, errors.E(op, fmt.Errorf("could not create new storage client: %w", err))
} }
return &Storage{ return &Storage{
+8 -8
View File
@@ -12,7 +12,7 @@ import (
pkgstorage "github.com/gomods/athens/pkg/storage" pkgstorage "github.com/gomods/athens/pkg/storage"
) )
// Info implements Getter // Info implements Getter.
func (s *Storage) Info(ctx context.Context, module, version string) ([]byte, error) { func (s *Storage) Info(ctx context.Context, module, version string) ([]byte, error) {
const op errors.Op = "gcp.Info" const op errors.Op = "gcp.Info"
ctx, span := observ.StartSpan(ctx, op.String()) ctx, span := observ.StartSpan(ctx, op.String())
@@ -22,14 +22,14 @@ func (s *Storage) Info(ctx context.Context, module, version string) ([]byte, err
return nil, errors.E(op, err, getErrorKind(err), errors.M(module), errors.V(version)) return nil, errors.E(op, err, getErrorKind(err), errors.M(module), errors.V(version))
} }
infoBytes, err := io.ReadAll(infoReader) infoBytes, err := io.ReadAll(infoReader)
infoReader.Close() _ = infoReader.Close()
if err != nil { if err != nil {
return nil, errors.E(op, err, errors.M(module), errors.V(version)) return nil, errors.E(op, err, errors.M(module), errors.V(version))
} }
return infoBytes, nil return infoBytes, nil
} }
// GoMod implements Getter // GoMod implements Getter.
func (s *Storage) GoMod(ctx context.Context, module, version string) ([]byte, error) { func (s *Storage) GoMod(ctx context.Context, module, version string) ([]byte, error) {
const op errors.Op = "gcp.GoMod" const op errors.Op = "gcp.GoMod"
ctx, span := observ.StartSpan(ctx, op.String()) ctx, span := observ.StartSpan(ctx, op.String())
@@ -39,15 +39,15 @@ func (s *Storage) GoMod(ctx context.Context, module, version string) ([]byte, er
return nil, errors.E(op, err, getErrorKind(err), errors.M(module), errors.V(version)) return nil, errors.E(op, err, getErrorKind(err), errors.M(module), errors.V(version))
} }
modBytes, err := io.ReadAll(modReader) modBytes, err := io.ReadAll(modReader)
modReader.Close() _ = modReader.Close()
if err != nil { if err != nil {
return nil, errors.E(op, fmt.Errorf("could not get new reader for mod file: %s", err), errors.M(module), errors.V(version)) return nil, errors.E(op, fmt.Errorf("could not get new reader for mod file: %w", err), errors.M(module), errors.V(version))
} }
return modBytes, nil return modBytes, nil
} }
// Zip implements Getter // Zip implements Getter.
func (s *Storage) Zip(ctx context.Context, module, version string) (pkgstorage.SizeReadCloser, error) { func (s *Storage) Zip(ctx context.Context, module, version string) (pkgstorage.SizeReadCloser, error) {
const op errors.Op = "gcp.Zip" const op errors.Op = "gcp.Zip"
ctx, span := observ.StartSpan(ctx, op.String()) ctx, span := observ.StartSpan(ctx, op.String())
@@ -56,11 +56,11 @@ func (s *Storage) Zip(ctx context.Context, module, version string) (pkgstorage.S
if err != nil { if err != nil {
return nil, errors.E(op, err, getErrorKind(err), errors.M(module), errors.V(version)) return nil, errors.E(op, err, getErrorKind(err), errors.M(module), errors.V(version))
} }
return pkgstorage.NewSizer(zipReader, zipReader.Size()), nil return pkgstorage.NewSizer(zipReader, zipReader.Attrs.Size), nil
} }
func getErrorKind(err error) int { func getErrorKind(err error) int {
if err == storage.ErrObjectNotExist { if errors.IsErr(err, storage.ErrObjectNotExist) {
return errors.KindNotFound return errors.KindNotFound
} }
return errors.KindUnexpected return errors.KindUnexpected
+3 -3
View File
@@ -10,8 +10,8 @@ import (
"google.golang.org/api/iterator" "google.golang.org/api/iterator"
) )
// List implements the (./pkg/storage).Lister interface // List implements the (./pkg/storage).Lister interface.
// It returns a list of versions, if any, for a given module // It returns a list of versions, if any, for a given module.
func (s *Storage) List(ctx context.Context, module string) ([]string, error) { func (s *Storage) List(ctx context.Context, module string) ([]string, error) {
const op errors.Op = "gcp.List" const op errors.Op = "gcp.List"
ctx, span := observ.StartSpan(ctx, op.String()) ctx, span := observ.StartSpan(ctx, op.String())
@@ -22,7 +22,7 @@ func (s *Storage) List(ctx context.Context, module string) ([]string, error) {
paths := []string{} paths := []string{}
for { for {
attrs, err := it.Next() attrs, err := it.Next()
if err == iterator.Done { if errors.IsErr(err, iterator.Done) {
break break
} }
if err != nil { if err != nil {

Some files were not shown because too many files have changed in this diff Show More