mirror of
https://github.com/gomods/athens
synced 2026-02-03 11:00:32 +00:00
chore: lint code with golangci-lint (#1828)
* feat: add golangci-lint linting * chore: fix linter issues * feat: add linting into the workflow * docs: update lint docs * fix: cr suggestions * fix: remove old formatting and vetting scripts * fix: add docker make target * fix: action go caching * fix: depreciated actions checkout version * fix: cr suggestion * fix: cr suggestions --------- Co-authored-by: Manu Gupta <manugupt1@gmail.com>
This commit is contained in:
+2
-2
@@ -9,7 +9,7 @@ import (
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// Details represents known data for a given build
|
||||
// Details represents known data for a given build.
|
||||
type Details struct {
|
||||
Version string `json:"version,omitempty"`
|
||||
Date string `json:"date,omitempty"`
|
||||
@@ -29,7 +29,7 @@ func String() string {
|
||||
return fmt.Sprintf("Build Details:\n\tVersion:\t%s\n\tDate:\t\t%s", version, buildDate)
|
||||
}
|
||||
|
||||
// Data returns build details as a struct
|
||||
// Data returns build details as a struct.
|
||||
func Data() Details {
|
||||
return Details{
|
||||
Version: version,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
package config
|
||||
|
||||
// AzureBlobConfig specifies the properties required to use Azure as the storage backend
|
||||
// AzureBlobConfig specifies the properties required to use Azure as the storage backend.
|
||||
type AzureBlobConfig struct {
|
||||
AccountName string `validate:"required" envconfig:"ATHENS_AZURE_ACCOUNT_NAME"`
|
||||
AccountKey string `validate:"required" envconfig:"ATHENS_AZURE_ACCOUNT_KEY"`
|
||||
|
||||
+18
-20
@@ -18,7 +18,7 @@ import (
|
||||
|
||||
const defaultConfigFile = "athens.toml"
|
||||
|
||||
// Config provides configuration values for all components
|
||||
// Config provides configuration values for all components.
|
||||
type Config struct {
|
||||
TimeoutConf
|
||||
GoEnv string `validate:"required" envconfig:"GO_ENV"`
|
||||
@@ -63,12 +63,12 @@ type Config struct {
|
||||
}
|
||||
|
||||
// EnvList is a list of key-value environment
|
||||
// variables that are passed to the Go command
|
||||
// variables that are passed to the Go command.
|
||||
type EnvList []string
|
||||
|
||||
// HasKey returns whether a key-value entry
|
||||
// is present by only checking the left of
|
||||
// key=value
|
||||
// key=value.
|
||||
func (el EnvList) HasKey(key string) bool {
|
||||
for _, env := range el {
|
||||
if strings.HasPrefix(env, key+"=") {
|
||||
@@ -79,7 +79,7 @@ func (el EnvList) HasKey(key string) bool {
|
||||
}
|
||||
|
||||
// Add adds a key=value entry to the environment
|
||||
// list
|
||||
// list.
|
||||
func (el *EnvList) Add(key, value string) {
|
||||
*el = append(*el, key+"="+value)
|
||||
}
|
||||
@@ -110,7 +110,7 @@ func (el *EnvList) Decode(value string) error {
|
||||
}
|
||||
|
||||
// Validate validates that all strings inside the
|
||||
// list are of the key=value format
|
||||
// list are of the key=value format.
|
||||
func (el EnvList) Validate() error {
|
||||
const op errors.Op = "EnvList.Validate"
|
||||
for _, env := range el {
|
||||
@@ -123,7 +123,7 @@ func (el EnvList) Validate() error {
|
||||
}
|
||||
|
||||
// Load loads the config from a file.
|
||||
// If file is not present returns default config
|
||||
// If file is not present returns default config.
|
||||
func Load(configFile string) (*Config, error) {
|
||||
// User explicitly specified a config file
|
||||
if configFile != "" {
|
||||
@@ -206,7 +206,7 @@ func defaultConfig() *Config {
|
||||
}
|
||||
|
||||
// BasicAuth returns BasicAuthUser and BasicAuthPassword
|
||||
// and ok if neither of them are empty
|
||||
// and ok if neither of them are empty.
|
||||
func (c *Config) BasicAuth() (user, pass string, ok bool) {
|
||||
user = c.BasicAuthUser
|
||||
pass = c.BasicAuthPass
|
||||
@@ -215,7 +215,7 @@ func (c *Config) BasicAuth() (user, pass string, ok bool) {
|
||||
}
|
||||
|
||||
// TLSCertFiles returns certificate and key files and an error if
|
||||
// both files doesn't exist and have approperiate file permissions
|
||||
// both files doesn't exist and have approperiate file permissions.
|
||||
func (c *Config) TLSCertFiles() (cert, key string, err error) {
|
||||
if c.TLSCertFile == "" && c.TLSKeyFile == "" {
|
||||
return "", "", nil
|
||||
@@ -223,29 +223,28 @@ func (c *Config) TLSCertFiles() (cert, key string, err error) {
|
||||
|
||||
certFile, err := os.Stat(c.TLSCertFile)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("Could not access TLSCertFile: %v", err)
|
||||
return "", "", fmt.Errorf("could not access TLSCertFile: %w", err)
|
||||
}
|
||||
|
||||
keyFile, err := os.Stat(c.TLSKeyFile)
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("Could not access TLSKeyFile: %v", err)
|
||||
return "", "", fmt.Errorf("could not access TLSKeyFile: %w", err)
|
||||
}
|
||||
|
||||
if keyFile.Mode()&077 != 0 && runtime.GOOS != "windows" {
|
||||
if keyFile.Mode()&0o077 != 0 && runtime.GOOS != "windows" {
|
||||
return "", "", fmt.Errorf("TLSKeyFile should not be accessible by others")
|
||||
}
|
||||
|
||||
return certFile.Name(), keyFile.Name(), nil
|
||||
}
|
||||
|
||||
// FilterOff returns true if the FilterFile is empty
|
||||
// FilterOff returns true if the FilterFile is empty.
|
||||
func (c *Config) FilterOff() bool {
|
||||
return c.FilterFile == ""
|
||||
}
|
||||
|
||||
// ParseConfigFile parses the given file into an athens config struct
|
||||
// ParseConfigFile parses the given file into an athens config struct.
|
||||
func ParseConfigFile(configFile string) (*Config, error) {
|
||||
|
||||
var config Config
|
||||
// attempt to read the given config file
|
||||
if _, err := toml.DecodeFile(configFile, &config); err != nil {
|
||||
@@ -271,7 +270,7 @@ func ParseConfigFile(configFile string) (*Config, error) {
|
||||
return &config, nil
|
||||
}
|
||||
|
||||
// envOverride uses Environment variables to override unspecified properties
|
||||
// envOverride uses Environment variables to override unspecified properties.
|
||||
func envOverride(config *Config) error {
|
||||
const defaultPort = ":3000"
|
||||
err := envconfig.Process("athens", config)
|
||||
@@ -355,16 +354,16 @@ func validateIndex(validate *validator.Validate, indexType string, config *Index
|
||||
func GetConf(path string) (*Config, error) {
|
||||
absPath, err := filepath.Abs(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Unable to construct absolute path to test config file")
|
||||
return nil, fmt.Errorf("unable to construct absolute path to test config file")
|
||||
}
|
||||
conf, err := ParseConfigFile(absPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Unable to parse test config file: %s", err.Error())
|
||||
return nil, fmt.Errorf("unable to parse test config file: %w", err)
|
||||
}
|
||||
return conf, nil
|
||||
}
|
||||
|
||||
// checkFilePerms given a list of files
|
||||
// checkFilePerms given a list of files.
|
||||
func checkFilePerms(files ...string) error {
|
||||
const op = "config.checkFilePerms"
|
||||
|
||||
@@ -384,10 +383,9 @@ func checkFilePerms(files ...string) error {
|
||||
// Assume unix based system (MacOS and Linux)
|
||||
// the bit mask is calculated using the umask command which tells which permissions
|
||||
// should not be allowed for a particular user, group or world
|
||||
if fInfo.Mode()&0077 != 0 && runtime.GOOS != "windows" {
|
||||
if fInfo.Mode()&0o077 != 0 && runtime.GOOS != "windows" {
|
||||
return errors.E(op, f+" should have at most rwx,-, - (bit mask 077) as permission")
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
package config
|
||||
|
||||
// DiskConfig specifies the properties required to use Disk as the storage backend
|
||||
// DiskConfig specifies the properties required to use Disk as the storage backend.
|
||||
type DiskConfig struct {
|
||||
RootPath string `validate:"required" envconfig:"ATHENS_DISK_STORAGE_ROOT"`
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
package config
|
||||
|
||||
// External specifies configuration for an external http storage
|
||||
// External specifies configuration for an external http storage.
|
||||
type External struct {
|
||||
URL string `validate:"required" envconfig:"ATHENS_EXTERNAL_STORAGE_URL"`
|
||||
}
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
package config
|
||||
|
||||
// GCPConfig specifies the properties required to use GCP as the storage backend
|
||||
// GCPConfig specifies the properties required to use GCP as the storage backend.
|
||||
type GCPConfig struct {
|
||||
ProjectID string `envconfig:"GOOGLE_CLOUD_PROJECT"`
|
||||
Bucket string `validate:"required" envconfig:"ATHENS_STORAGE_GCP_BUCKET"`
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
package config
|
||||
|
||||
// Index is the config for various index storage backends
|
||||
// Index is the config for various index storage backends.
|
||||
type Index struct {
|
||||
MySQL *MySQL
|
||||
Postgres *Postgres
|
||||
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
package config
|
||||
|
||||
// MinioConfig specifies the properties required to use Minio or DigitalOcean Spaces
|
||||
// as the storage backend
|
||||
// as the storage backend.
|
||||
type MinioConfig struct {
|
||||
Endpoint string `validate:"required" envconfig:"ATHENS_MINIO_ENDPOINT"`
|
||||
Key string `validate:"required" envconfig:"ATHENS_MINIO_ACCESS_KEY_ID"`
|
||||
|
||||
@@ -7,20 +7,20 @@ import (
|
||||
)
|
||||
|
||||
// PackageVersionedName return package full name used in storage.
|
||||
// E.g athens/@v/v1.0.mod
|
||||
// E.g athens/@v/v1.0.mod.
|
||||
func PackageVersionedName(module, version, ext string) string {
|
||||
return fmt.Sprintf("%s/@v/%s.%s", module, version, ext)
|
||||
}
|
||||
|
||||
// FmtModVer is a helper function that can take
|
||||
// pkg/a/b and v2.3.1 and returns pkg/a/b@v2.3.1
|
||||
// pkg/a/b and v2.3.1 and returns pkg/a/b@v2.3.1.
|
||||
func FmtModVer(mod, ver string) string {
|
||||
return fmt.Sprintf("%s@%s", mod, ver)
|
||||
}
|
||||
|
||||
// ModuleVersionFromPath returns module and version from a
|
||||
// storage path
|
||||
// E.g athens/@v/v1.0.info -> athens and v.1.0
|
||||
// storage path.
|
||||
// E.g athens/@v/v1.0.info -> athens and v.1.0.
|
||||
func ModuleVersionFromPath(path string) (string, string) {
|
||||
segments := strings.Split(path, "/@v/")
|
||||
if len(segments) != 2 {
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
package config
|
||||
|
||||
// MongoConfig specifies the properties required to use MongoDB as the storage backend
|
||||
// MongoConfig specifies the properties required to use MongoDB as the storage backend.
|
||||
type MongoConfig struct {
|
||||
URL string `validate:"required" envconfig:"ATHENS_MONGO_STORAGE_URL"`
|
||||
DefaultDBName string `envconfig:"ATHENS_MONGO_DEFAULT_DATABASE" default:"athens"`
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
package config
|
||||
|
||||
// MySQL config
|
||||
// MySQL config.
|
||||
type MySQL struct {
|
||||
Protocol string `validate:"required" envconfig:"ATHENS_INDEX_MYSQL_PROTOCOL"`
|
||||
Host string `validate:"required" envconfig:"ATHENS_INDEX_MYSQL_HOST"`
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
package config
|
||||
|
||||
// Postgres config
|
||||
// Postgres config.
|
||||
type Postgres struct {
|
||||
Host string `validate:"required" envconfig:"ATHENS_INDEX_POSTGRES_HOST"`
|
||||
Port int `validate:"required" envconfig:"ATHENS_INDEX_POSTGRES_PORT"`
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
package config
|
||||
|
||||
// S3Config specifies the properties required to use S3 as the storage backend
|
||||
// S3Config specifies the properties required to use S3 as the storage backend.
|
||||
type S3Config struct {
|
||||
Region string `validate:"required" envconfig:"AWS_REGION"`
|
||||
Key string `envconfig:"AWS_ACCESS_KEY_ID"`
|
||||
|
||||
@@ -25,7 +25,7 @@ type Redis struct {
|
||||
}
|
||||
|
||||
// RedisSentinel is the configuration for using redis with sentinel
|
||||
// for SingleFlight
|
||||
// for SingleFlight.
|
||||
type RedisSentinel struct {
|
||||
Endpoints []string `envconfig:"ATHENS_REDIS_SENTINEL_ENDPOINTS"`
|
||||
MasterName string `envconfig:"ATHENS_REDIS_SENTINEL_MASTER_NAME"`
|
||||
@@ -33,12 +33,14 @@ type RedisSentinel struct {
|
||||
LockConfig *RedisLockConfig
|
||||
}
|
||||
|
||||
// RedisLockConfig is the configuration for redis locking.
|
||||
type RedisLockConfig struct {
|
||||
Timeout int `envconfig:"ATHENS_REDIS_LOCK_TIMEOUT"`
|
||||
TTL int `envconfig:"ATHENS_REDIS_LOCK_TTL"`
|
||||
MaxRetries int `envconfig:"ATHENS_REDIS_LOCK_MAX_RETRIES"`
|
||||
}
|
||||
|
||||
// DefaultRedisLockConfig returns the default redis locking configuration.
|
||||
func DefaultRedisLockConfig() *RedisLockConfig {
|
||||
return &RedisLockConfig{
|
||||
TTL: 900,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
package config
|
||||
|
||||
// Storage provides configs for various storage backends
|
||||
// Storage provides configs for various storage backends.
|
||||
type Storage struct {
|
||||
Disk *DiskConfig
|
||||
GCP *GCPConfig
|
||||
|
||||
@@ -2,17 +2,17 @@ package config
|
||||
|
||||
import "time"
|
||||
|
||||
// TimeoutConf is a common struct for anything with a timeout
|
||||
// TimeoutConf is a common struct for anything with a timeout.
|
||||
type TimeoutConf struct {
|
||||
Timeout int `validate:"required"`
|
||||
}
|
||||
|
||||
// TimeoutDuration returns the timeout as time.duration
|
||||
// TimeoutDuration returns the timeout as time.duration.
|
||||
func (t *TimeoutConf) TimeoutDuration() time.Duration {
|
||||
return GetTimeoutDuration(t.Timeout)
|
||||
}
|
||||
|
||||
// GetTimeoutDuration returns the timeout as time.duration
|
||||
// GetTimeoutDuration returns the timeout as time.duration.
|
||||
func GetTimeoutDuration(timeout int) time.Duration {
|
||||
return time.Second * time.Duration(timeout)
|
||||
}
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"github.com/gomods/athens/pkg/paths"
|
||||
)
|
||||
|
||||
func getModuleParams(r *http.Request, op errors.Op) (mod string, ver string, err error) {
|
||||
func getModuleParams(r *http.Request, op errors.Op) (mod, ver string, err error) {
|
||||
params, err := paths.GetAllParams(r)
|
||||
if err != nil {
|
||||
return "", "", errors.E(op, err, errors.KindBadRequest)
|
||||
|
||||
@@ -15,8 +15,7 @@ import (
|
||||
// a ready-to-go http handler that serves up cmd/go's download protocol.
|
||||
type ProtocolHandler func(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler
|
||||
|
||||
// HandlerOpts are the generic options
|
||||
// for a ProtocolHandler
|
||||
// HandlerOpts are the generic options for a ProtocolHandler.
|
||||
type HandlerOpts struct {
|
||||
Protocol Protocol
|
||||
Logger *log.Logger
|
||||
@@ -26,7 +25,7 @@ type HandlerOpts struct {
|
||||
// LogEntryHandler pulls a log entry from the request context. Thanks to the
|
||||
// LogEntryMiddleware, we should have a log entry stored in the context for each
|
||||
// request with request-specific fields. This will grab the entry and pass it to
|
||||
// the protocol handlers
|
||||
// the protocol handlers.
|
||||
func LogEntryHandler(ph ProtocolHandler, opts *HandlerOpts) http.Handler {
|
||||
f := func(w http.ResponseWriter, r *http.Request) {
|
||||
ent := log.EntryFromContext(r.Context())
|
||||
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
// PathLatest URL.
|
||||
const PathLatest = "/{module:.+}/@latest"
|
||||
|
||||
// LatestHandler implements GET baseURL/module/@latest
|
||||
// LatestHandler implements GET baseURL/module/@latest.
|
||||
func LatestHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler {
|
||||
const op errors.Op = "download.LatestHandler"
|
||||
f := func(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
@@ -14,7 +14,7 @@ import (
|
||||
// PathList URL.
|
||||
const PathList = "/{module:.+}/@v/list"
|
||||
|
||||
// ListHandler implements GET baseURL/module/@v/list
|
||||
// ListHandler implements GET baseURL/module/@v/list.
|
||||
func ListHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler {
|
||||
const op errors.Op = "download.ListHandler"
|
||||
f := func(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/gomods/athens/pkg/errors"
|
||||
@@ -16,7 +17,7 @@ import (
|
||||
// when a module is not found in storage.
|
||||
type Mode string
|
||||
|
||||
// DownloadMode constants. For more information see config.dev.toml
|
||||
// DownloadMode constants. For more information see config.dev.toml.
|
||||
const (
|
||||
Sync Mode = "sync"
|
||||
Async Mode = "async"
|
||||
@@ -59,7 +60,7 @@ func NewFile(m Mode, downloadURL string) (*DownloadFile, error) {
|
||||
|
||||
if strings.HasPrefix(string(m), "file:") {
|
||||
filePath := string(m[5:])
|
||||
bts, err := os.ReadFile(filePath)
|
||||
bts, err := os.ReadFile(filepath.Clean(filePath))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -48,7 +48,7 @@ type Opts struct {
|
||||
NetworkMode string
|
||||
}
|
||||
|
||||
// NetworkMode constants
|
||||
// NetworkMode constants.
|
||||
const (
|
||||
Strict = "strict"
|
||||
Offline = "offline"
|
||||
@@ -263,12 +263,12 @@ func (p *protocol) processDownload(ctx context.Context, mod, ver string, f func(
|
||||
}
|
||||
return f(newVer)
|
||||
case mode.Async:
|
||||
go p.stasher.Stash(ctx, mod, ver)
|
||||
go func() { _, _ = p.stasher.Stash(ctx, mod, ver) }()
|
||||
return errors.E(op, "async: module not found", errors.KindNotFound)
|
||||
case mode.Redirect:
|
||||
return errors.E(op, "redirect", errors.KindRedirect)
|
||||
case mode.AsyncRedirect:
|
||||
go p.stasher.Stash(ctx, mod, ver)
|
||||
go func() { _, _ = p.stasher.Stash(ctx, mod, ver) }()
|
||||
return errors.E(op, "async_redirect: module not found", errors.KindRedirect)
|
||||
case mode.None:
|
||||
return errors.E(op, "none", errors.KindNotFound)
|
||||
@@ -276,7 +276,7 @@ func (p *protocol) processDownload(ctx context.Context, mod, ver string, f func(
|
||||
return nil
|
||||
}
|
||||
|
||||
// union concatenates two version lists and removes duplicates
|
||||
// union concatenates two version lists and removes duplicates.
|
||||
func union(list1, list2 []string) []string {
|
||||
if list1 == nil {
|
||||
list1 = []string{}
|
||||
@@ -284,10 +284,10 @@ func union(list1, list2 []string) []string {
|
||||
if list2 == nil {
|
||||
list2 = []string{}
|
||||
}
|
||||
list := append(list1, list2...)
|
||||
list1 = append(list1, list2...)
|
||||
unique := []string{}
|
||||
m := make(map[string]struct{})
|
||||
for _, v := range list {
|
||||
for _, v := range list1 {
|
||||
if _, ok := m[v]; !ok {
|
||||
unique = append(unique, v)
|
||||
m[v] = struct{}{}
|
||||
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
// PathVersionInfo URL.
|
||||
const PathVersionInfo = "/{module:.+}/@v/{version}.info"
|
||||
|
||||
// InfoHandler implements GET baseURL/module/@v/version.info
|
||||
// InfoHandler implements GET baseURL/module/@v/version.info.
|
||||
func InfoHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler {
|
||||
const op errors.Op = "download.InfoHandler"
|
||||
f := func(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -38,7 +38,7 @@ func InfoHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handle
|
||||
w.WriteHeader(errors.Kind(err))
|
||||
}
|
||||
|
||||
w.Write(info)
|
||||
_, _ = w.Write(info)
|
||||
}
|
||||
return http.HandlerFunc(f)
|
||||
}
|
||||
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
// PathVersionModule URL.
|
||||
const PathVersionModule = "/{module:.+}/@v/{version}.mod"
|
||||
|
||||
// ModuleHandler implements GET baseURL/module/@v/version.mod
|
||||
// ModuleHandler implements GET baseURL/module/@v/version.mod.
|
||||
func ModuleHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler {
|
||||
const op errors.Op = "download.VersionModuleHandler"
|
||||
f := func(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -42,7 +42,7 @@ func ModuleHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Hand
|
||||
return
|
||||
}
|
||||
|
||||
w.Write(modBts)
|
||||
_, _ = w.Write(modBts)
|
||||
}
|
||||
return http.HandlerFunc(f)
|
||||
}
|
||||
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
// PathVersionZip URL.
|
||||
const PathVersionZip = "/{module:.+}/@v/{version}.zip"
|
||||
|
||||
// ZipHandler implements GET baseURL/module/@v/version.zip
|
||||
// ZipHandler implements GET baseURL/module/@v/version.zip.
|
||||
func ZipHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler {
|
||||
const op errors.Op = "download.ZipHandler"
|
||||
f := func(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -41,7 +41,7 @@ func ZipHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler
|
||||
w.WriteHeader(errors.Kind(err))
|
||||
return
|
||||
}
|
||||
defer zip.Close()
|
||||
defer func() { _ = zip.Close() }()
|
||||
|
||||
w.Header().Set("Content-Type", "application/zip")
|
||||
size := zip.Size()
|
||||
|
||||
+17
-6
@@ -9,7 +9,7 @@ import (
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// Kind enums
|
||||
// Kind enums.
|
||||
const (
|
||||
KindNotFound = http.StatusNotFound
|
||||
KindBadRequest = http.StatusBadRequest
|
||||
@@ -55,6 +55,16 @@ func Is(err error, kind int) bool {
|
||||
return Kind(err) == kind
|
||||
}
|
||||
|
||||
// IsErr is a convenience wrapper around the std library errors.Is.
|
||||
func IsErr(err, target error) bool {
|
||||
return errors.Is(err, target)
|
||||
}
|
||||
|
||||
// AsErr is a convenience wrapper around the std library errors.As.
|
||||
func AsErr(err error, target any) bool {
|
||||
return errors.As(err, target)
|
||||
}
|
||||
|
||||
// Op describes any independent function or
|
||||
// method in Athens. A series of operations
|
||||
// forms a more readable stack trace.
|
||||
@@ -69,7 +79,7 @@ func (o Op) String() string {
|
||||
// a module from a regular error string or version.
|
||||
type M string
|
||||
|
||||
// V represents a module version in an error
|
||||
// V represents a module version in an error.
|
||||
type V string
|
||||
|
||||
// E is a helper function to construct an Error type
|
||||
@@ -114,8 +124,8 @@ func E(op Op, args ...interface{}) error {
|
||||
// if none exists, then the level is Error because
|
||||
// it is an unexpected.
|
||||
func Severity(err error) logrus.Level {
|
||||
e, ok := err.(Error)
|
||||
if !ok {
|
||||
var e Error
|
||||
if !errors.As(err, &e) {
|
||||
return logrus.ErrorLevel
|
||||
}
|
||||
|
||||
@@ -145,8 +155,8 @@ func Expect(err error, kinds ...int) logrus.Level {
|
||||
// Kind recursively searches for the
|
||||
// first error kind it finds.
|
||||
func Kind(err error) int {
|
||||
e, ok := err.(Error)
|
||||
if !ok {
|
||||
var e Error
|
||||
if !errors.As(err, &e) {
|
||||
return KindUnexpected
|
||||
}
|
||||
|
||||
@@ -173,6 +183,7 @@ func KindText(err error) string {
|
||||
func Ops(err Error) []Op {
|
||||
ops := []Op{err.Op}
|
||||
for {
|
||||
//nolint:errorlint // We iterate the errors anyway.
|
||||
embeddedErr, ok := err.Err.(Error)
|
||||
if !ok {
|
||||
break
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
package errors
|
||||
|
||||
// IsNotFoundErr helper function for KindNotFound
|
||||
// IsNotFoundErr helper function for KindNotFound.
|
||||
func IsNotFoundErr(err error) bool {
|
||||
return Kind(err) == KindNotFound
|
||||
}
|
||||
|
||||
@@ -21,7 +21,7 @@ func RunTests(t *testing.T, indexer index.Indexer, clearIndex func() error) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
var tests = []struct {
|
||||
tests := []struct {
|
||||
name string
|
||||
desc string
|
||||
limit int
|
||||
|
||||
@@ -13,7 +13,7 @@ type Line struct {
|
||||
}
|
||||
|
||||
// Indexer is an interface that can process new module@versions
|
||||
// and also retrieve 'limit' module@versions that were indexed after 'since'
|
||||
// and also retrieve 'limit' module@versions that were indexed after 'since'.
|
||||
type Indexer interface {
|
||||
// Index stores the module@version into the index backend.
|
||||
// Implementer must create the Timestamp at the time and set it
|
||||
|
||||
@@ -10,7 +10,7 @@ import (
|
||||
"github.com/gomods/athens/pkg/index"
|
||||
)
|
||||
|
||||
// New returns a new in-memory indexer
|
||||
// New returns a new in-memory indexer.
|
||||
func New() index.Indexer {
|
||||
return &indexer{}
|
||||
}
|
||||
|
||||
@@ -83,8 +83,8 @@ func (i *indexer) Lines(ctx context.Context, since time.Time, limit int) ([]*ind
|
||||
if err != nil {
|
||||
return nil, errors.E(op, err)
|
||||
}
|
||||
defer rows.Close()
|
||||
lines := []*index.Line{}
|
||||
defer func() { _ = rows.Close() }()
|
||||
var lines []*index.Line
|
||||
for rows.Next() {
|
||||
var line index.Line
|
||||
err = rows.Scan(&line.Path, &line.Version, &line.Timestamp)
|
||||
@@ -108,13 +108,14 @@ func getMySQLSource(cfg *config.MySQL) string {
|
||||
}
|
||||
|
||||
func getKind(err error) int {
|
||||
mysqlErr, ok := err.(*mysql.MySQLError)
|
||||
if !ok {
|
||||
mysqlErr := &mysql.MySQLError{}
|
||||
if !errors.AsErr(err, &mysqlErr) {
|
||||
return errors.KindUnexpected
|
||||
}
|
||||
switch mysqlErr.Number {
|
||||
case 1062:
|
||||
return errors.KindAlreadyExists
|
||||
default:
|
||||
return errors.KindUnexpected
|
||||
}
|
||||
return errors.KindUnexpected
|
||||
}
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"github.com/gomods/athens/pkg/index"
|
||||
)
|
||||
|
||||
// New returns a no-op Indexer
|
||||
// New returns a no-op Indexer.
|
||||
func New() index.Indexer {
|
||||
return indexer{}
|
||||
}
|
||||
@@ -17,6 +17,7 @@ type indexer struct{}
|
||||
func (indexer) Index(ctx context.Context, mod, ver string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (indexer) Lines(ctx context.Context, since time.Time, limit int) ([]*index.Line, error) {
|
||||
return []*index.Line{}, nil
|
||||
}
|
||||
|
||||
@@ -7,17 +7,15 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
// register the driver with database/sql
|
||||
"github.com/lib/pq"
|
||||
|
||||
"github.com/gomods/athens/pkg/config"
|
||||
"github.com/gomods/athens/pkg/errors"
|
||||
"github.com/gomods/athens/pkg/index"
|
||||
"github.com/lib/pq"
|
||||
)
|
||||
|
||||
// New returns a new Indexer with a PostgreSQL implementation.
|
||||
// It attempts to connect to the DB and create the index table
|
||||
// if it doesn ot already exist.
|
||||
// if it does not already exist.
|
||||
func New(cfg *config.Postgres) (index.Indexer, error) {
|
||||
dataSource := getPostgresSource(cfg)
|
||||
db, err := sql.Open("postgres", dataSource)
|
||||
@@ -42,7 +40,7 @@ var schema = [...]string{
|
||||
id SERIAL PRIMARY KEY,
|
||||
path VARCHAR(255) NOT NULL,
|
||||
version VARCHAR(255) NOT NULL,
|
||||
timestamp timestamp NOT NULL
|
||||
timestamp TIMESTAMP NOT NULL
|
||||
)
|
||||
`,
|
||||
`
|
||||
@@ -82,8 +80,8 @@ func (i *indexer) Lines(ctx context.Context, since time.Time, limit int) ([]*ind
|
||||
if err != nil {
|
||||
return nil, errors.E(op, err)
|
||||
}
|
||||
defer rows.Close()
|
||||
lines := []*index.Line{}
|
||||
defer func() { _ = rows.Close() }()
|
||||
var lines []*index.Line
|
||||
for rows.Next() {
|
||||
var line index.Line
|
||||
err = rows.Scan(&line.Path, &line.Version, &line.Timestamp)
|
||||
@@ -96,7 +94,7 @@ func (i *indexer) Lines(ctx context.Context, since time.Time, limit int) ([]*ind
|
||||
}
|
||||
|
||||
func getPostgresSource(cfg *config.Postgres) string {
|
||||
args := []string{}
|
||||
args := make([]string, 0, 5+len(cfg.Params))
|
||||
args = append(args, "host="+cfg.Host)
|
||||
args = append(args, "port=", strconv.Itoa(cfg.Port))
|
||||
args = append(args, "user=", cfg.User)
|
||||
@@ -109,13 +107,14 @@ func getPostgresSource(cfg *config.Postgres) string {
|
||||
}
|
||||
|
||||
func getKind(err error) int {
|
||||
pqerr, ok := err.(*pq.Error)
|
||||
if !ok {
|
||||
pqerr := &pq.Error{}
|
||||
if !errors.AsErr(err, &pqerr) {
|
||||
return errors.KindUnexpected
|
||||
}
|
||||
switch pqerr.Code {
|
||||
case "23505":
|
||||
return errors.KindAlreadyExists
|
||||
default:
|
||||
return errors.KindUnexpected
|
||||
}
|
||||
return errors.KindUnexpected
|
||||
}
|
||||
|
||||
+2
-2
@@ -35,8 +35,8 @@ func (e *entry) WithFields(fields map[string]interface{}) Entry {
|
||||
}
|
||||
|
||||
func (e *entry) SystemErr(err error) {
|
||||
athensErr, ok := err.(errors.Error)
|
||||
if !ok {
|
||||
var athensErr errors.Error
|
||||
if !errors.IsErr(err, athensErr) {
|
||||
e.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
+1
-1
@@ -41,7 +41,7 @@ func (l *Logger) WithFields(fields map[string]interface{}) Entry {
|
||||
return &entry{e}
|
||||
}
|
||||
|
||||
// NoOpLogger provides a Logger that does nothing
|
||||
// NoOpLogger provides a Logger that does nothing.
|
||||
func NoOpLogger() *Logger {
|
||||
return &Logger{
|
||||
Logger: &logrus.Logger{},
|
||||
|
||||
@@ -8,14 +8,14 @@ type ctxKey string
|
||||
|
||||
const logEntryKey ctxKey = "log-entry-context-key"
|
||||
|
||||
// SetEntryInContext stores an Entry in the request context
|
||||
// SetEntryInContext stores an Entry in the request context.
|
||||
func SetEntryInContext(ctx context.Context, e Entry) context.Context {
|
||||
return context.WithValue(ctx, logEntryKey, e)
|
||||
}
|
||||
|
||||
// EntryFromContext returns an Entry that has been stored in the request context.
|
||||
// If there is no value for the key or the type assertion fails, it returns a new
|
||||
// entry from the provided logger
|
||||
// entry from the provided logger.
|
||||
func EntryFromContext(ctx context.Context) Entry {
|
||||
e, ok := ctx.Value(logEntryKey).(Entry)
|
||||
if !ok || e == nil {
|
||||
|
||||
@@ -4,7 +4,7 @@ import (
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// ContentType writes writes an application/json
|
||||
// ContentType writes an application/json
|
||||
// Content-Type header.
|
||||
func ContentType(h http.Handler) http.Handler {
|
||||
f := func(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
@@ -10,7 +10,7 @@ import (
|
||||
)
|
||||
|
||||
// LogEntryMiddleware builds a log.Entry, setting the request fields
|
||||
// and storing it in the context to be used throughout the stack
|
||||
// and storing it in the context to be used throughout the stack.
|
||||
func LogEntryMiddleware(lggr *log.Logger) mux.MiddlewareFunc {
|
||||
return func(h http.Handler) http.Handler {
|
||||
f := func(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
@@ -9,7 +9,7 @@ import (
|
||||
|
||||
// WithRequestID ensures a request id is in the
|
||||
// request context by either the incoming header
|
||||
// or creating a new one
|
||||
// or creating a new one.
|
||||
func WithRequestID(h http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
requestID := r.Header.Get(requestid.HeaderKey)
|
||||
|
||||
@@ -14,7 +14,7 @@ import (
|
||||
)
|
||||
|
||||
// NewValidationMiddleware builds a middleware function that performs validation checks by calling
|
||||
// an external webhook
|
||||
// an external webhook.
|
||||
func NewValidationMiddleware(client *http.Client, validatorHook string) mux.MiddlewareFunc {
|
||||
const op errors.Op = "actions.NewValidationMiddleware"
|
||||
return func(h http.Handler) http.Handler {
|
||||
@@ -50,7 +50,7 @@ func NewValidationMiddleware(client *http.Client, validatorHook string) mux.Midd
|
||||
}
|
||||
}
|
||||
|
||||
func maybeLogValidationReason(context context.Context, message string, mod string, version string) {
|
||||
func maybeLogValidationReason(context context.Context, message, mod, version string) {
|
||||
if len(message) > 0 {
|
||||
entry := log.EntryFromContext(context)
|
||||
entry.Warnf("error validating %s@%s %s", mod, version, message)
|
||||
@@ -85,6 +85,10 @@ func validate(ctx context.Context, client *http.Client, hook, mod, ver string) (
|
||||
if err != nil {
|
||||
return validationResponse{Valid: false}, errors.E(op, err)
|
||||
}
|
||||
defer func() {
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
}()
|
||||
|
||||
switch resp.StatusCode {
|
||||
case http.StatusOK:
|
||||
@@ -97,8 +101,6 @@ func validate(ctx context.Context, client *http.Client, hook, mod, ver string) (
|
||||
}
|
||||
|
||||
func validationResponseFromRequest(resp *http.Response) validationResponse {
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
return validationResponse{Valid: resp.StatusCode == http.StatusOK, Message: body}
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@ import (
|
||||
"github.com/gomods/athens/pkg/storage"
|
||||
)
|
||||
|
||||
// Fetcher fetches module from an upstream source
|
||||
// Fetcher fetches module from an upstream source.
|
||||
type Fetcher interface {
|
||||
// Fetch downloads the sources from an upstream and returns the corresponding
|
||||
// .info, .mod, and .zip files.
|
||||
|
||||
+15
-16
@@ -3,6 +3,7 @@ package module
|
||||
import (
|
||||
"bufio"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
@@ -14,15 +15,15 @@ var (
|
||||
versionSeparator = "."
|
||||
)
|
||||
|
||||
// Filter is a filter of modules
|
||||
// Filter is a filter of modules.
|
||||
type Filter struct {
|
||||
root ruleNode
|
||||
filePath string
|
||||
}
|
||||
|
||||
// NewFilter creates new filter based on rules defined in a configuration file
|
||||
// WARNING: this is not concurrently safe
|
||||
// Configuration consists of two operations: + for include and - for exclude
|
||||
// NewFilter creates new filter based on rules defined in a configuration file.
|
||||
// WARNING: this is not concurrently safe.
|
||||
// Configuration consists of two operations: + for include and - for exclude:
|
||||
// e.g.
|
||||
// - github.com/a
|
||||
// - github.com/a/b
|
||||
@@ -33,7 +34,7 @@ type Filter struct {
|
||||
// -
|
||||
// + github.com/a
|
||||
//
|
||||
// will exclude all items from communication except github.com/a
|
||||
// will exclude all items from communication except github.com/a.
|
||||
func NewFilter(filterFilePath string) (*Filter, error) {
|
||||
// Do not return an error if the file path is empty
|
||||
// Do not attempt to parse it as well.
|
||||
@@ -42,10 +43,9 @@ func NewFilter(filterFilePath string) (*Filter, error) {
|
||||
}
|
||||
|
||||
return initFromConfig(filterFilePath)
|
||||
|
||||
}
|
||||
|
||||
// AddRule adds rule for specified path
|
||||
// AddRule adds rule for specified path.
|
||||
func (f *Filter) AddRule(path string, qualifiers []string, rule FilterRule) {
|
||||
f.ensurePath(path)
|
||||
|
||||
@@ -70,7 +70,7 @@ func (f *Filter) AddRule(path string, qualifiers []string, rule FilterRule) {
|
||||
latest.next[last] = rn
|
||||
}
|
||||
|
||||
// Rule returns the filter rule to be applied to the given path
|
||||
// Rule returns the filter rule to be applied to the given path.
|
||||
func (f *Filter) Rule(path, version string) FilterRule {
|
||||
segs := getPathSegments(path)
|
||||
rule := f.getAssociatedRule(version, segs...)
|
||||
@@ -145,7 +145,6 @@ func initFromConfig(filePath string) (*Filter, error) {
|
||||
f.root = rn
|
||||
|
||||
for idx, line := range lines {
|
||||
|
||||
// Ignore newline
|
||||
if len(line) == 0 {
|
||||
continue
|
||||
@@ -160,7 +159,7 @@ func initFromConfig(filePath string) (*Filter, error) {
|
||||
}
|
||||
|
||||
ruleSign := strings.TrimSpace(split[0])
|
||||
rule := Default
|
||||
var rule FilterRule
|
||||
switch ruleSign {
|
||||
case "+":
|
||||
rule = Include
|
||||
@@ -195,11 +194,11 @@ func initFromConfig(filePath string) (*Filter, error) {
|
||||
|
||||
// matches checks if the given version matches the given qualifier.
|
||||
// Qualifiers can be:
|
||||
// - plain versions
|
||||
// - v1.2.3 enables v1.2.3
|
||||
// - ~1.2.3: enables 1.2.x which are at least 1.2.3
|
||||
// - ^1.2.3: enables 1.x.x which are at least 1.2.3
|
||||
// - <1.2.3: enables everything lower than 1.2.3 includes 1.2.2 and 0.58.9 as well
|
||||
// - plain versions.
|
||||
// - v1.2.3 enables v1.2.3.
|
||||
// - ~1.2.3: enables 1.2.x which are at least 1.2.3.
|
||||
// - ^1.2.3: enables 1.x.x which are at least 1.2.3.
|
||||
// - <1.2.3: enables everything lower than 1.2.3 includes 1.2.2 and 0.58.9 as well.
|
||||
func matches(version, qualifier string) bool {
|
||||
if len(qualifier) < 2 || len(version) < 1 {
|
||||
return false
|
||||
@@ -297,7 +296,7 @@ func newRule(r FilterRule) ruleNode {
|
||||
func getConfigLines(filterFile string) ([]string, error) {
|
||||
const op errors.Op = "module.getConfigLines"
|
||||
|
||||
f, err := os.Open(filterFile)
|
||||
f, err := os.Open(filepath.Clean(filterFile))
|
||||
if err != nil {
|
||||
return nil, errors.E(op, err)
|
||||
}
|
||||
|
||||
@@ -1,16 +1,16 @@
|
||||
package module
|
||||
|
||||
// FilterRule defines behavior of module communication
|
||||
// FilterRule defines behavior of module communication.
|
||||
type FilterRule int
|
||||
|
||||
const (
|
||||
// Default filter rule does not alter default/parent behavior
|
||||
// Default filter rule does not alter default/parent behavior.
|
||||
Default FilterRule = iota
|
||||
// Include treats modules the usual way
|
||||
// Used for reverting Exclude of parent path
|
||||
// Include treats modules the usual way.
|
||||
// Used for reverting Exclude of parent path.
|
||||
Include
|
||||
// Exclude filter rule excludes package and its children from communication
|
||||
// Exclude filter rule excludes package and its children from communication.
|
||||
Exclude
|
||||
// Direct filter rule forces the package to be fetched directly from upstream proxy
|
||||
// Direct filter rule forces the package to be fetched directly from upstream proxy.
|
||||
Direct
|
||||
)
|
||||
|
||||
@@ -35,7 +35,7 @@ type goModule struct {
|
||||
GoModSum string `json:"goModSum"` // checksum for go.mod (as in go.sum)
|
||||
}
|
||||
|
||||
// NewGoGetFetcher creates fetcher which uses go get tool to fetch modules
|
||||
// NewGoGetFetcher creates fetcher which uses go get tool to fetch modules.
|
||||
func NewGoGetFetcher(goBinaryName, gogetDir string, envVars []string, fs afero.Fs) (Fetcher, error) {
|
||||
const op errors.Op = "module.NewGoGetFetcher"
|
||||
if err := validGoBinary(goBinaryName); err != nil {
|
||||
@@ -64,7 +64,7 @@ func (g *goGetFetcher) Fetch(ctx context.Context, mod, ver string) (*storage.Ver
|
||||
sourcePath := filepath.Join(goPathRoot, "src")
|
||||
modPath := filepath.Join(sourcePath, getRepoDirName(mod, ver))
|
||||
if err := g.fs.MkdirAll(modPath, os.ModeDir|os.ModePerm); err != nil {
|
||||
clearFiles(g.fs, goPathRoot)
|
||||
_ = clearFiles(g.fs, goPathRoot)
|
||||
return nil, errors.E(op, err)
|
||||
}
|
||||
|
||||
@@ -72,14 +72,13 @@ func (g *goGetFetcher) Fetch(ctx context.Context, mod, ver string) (*storage.Ver
|
||||
ctx,
|
||||
g.goBinaryName,
|
||||
g.envVars,
|
||||
g.fs,
|
||||
goPathRoot,
|
||||
modPath,
|
||||
mod,
|
||||
ver,
|
||||
)
|
||||
if err != nil {
|
||||
clearFiles(g.fs, goPathRoot)
|
||||
_ = clearFiles(g.fs, goPathRoot)
|
||||
return nil, errors.E(op, err)
|
||||
}
|
||||
|
||||
@@ -116,7 +115,6 @@ func downloadModule(
|
||||
ctx context.Context,
|
||||
goBinaryName string,
|
||||
envVars []string,
|
||||
fs afero.Fs,
|
||||
gopath,
|
||||
repoRoot,
|
||||
module,
|
||||
@@ -137,7 +135,7 @@ func downloadModule(
|
||||
|
||||
err := cmd.Run()
|
||||
if err != nil {
|
||||
err = fmt.Errorf("%v: %s", err, stderr)
|
||||
err = fmt.Errorf("%w: %s", err, stderr)
|
||||
var m goModule
|
||||
if jsonErr := json.NewDecoder(stdout).Decode(&m); jsonErr != nil {
|
||||
return goModule{}, errors.E(op, err)
|
||||
@@ -165,17 +163,17 @@ func isLimitHit(o string) bool {
|
||||
}
|
||||
|
||||
// getRepoDirName takes a raw repository URI and a version and creates a directory name that the
|
||||
// repository contents can be put into
|
||||
// repository contents can be put into.
|
||||
func getRepoDirName(repoURI, version string) string {
|
||||
escapedURI := strings.Replace(repoURI, "/", "-", -1)
|
||||
escapedURI := strings.ReplaceAll(repoURI, "/", "-")
|
||||
return fmt.Sprintf("%s-%s", escapedURI, version)
|
||||
}
|
||||
|
||||
func validGoBinary(name string) error {
|
||||
const op errors.Op = "module.validGoBinary"
|
||||
err := exec.Command(name).Run()
|
||||
_, ok := err.(*exec.ExitError)
|
||||
if err != nil && !ok {
|
||||
eErr := &exec.ExitError{}
|
||||
if err != nil && !errors.AsErr(err, &eErr) {
|
||||
return errors.E(op, err)
|
||||
}
|
||||
return nil
|
||||
|
||||
@@ -28,7 +28,7 @@ type vcsLister struct {
|
||||
fs afero.Fs
|
||||
}
|
||||
|
||||
// NewVCSLister creates an UpstreamLister which uses VCS to fetch a list of available versions
|
||||
// NewVCSLister creates an UpstreamLister which uses VCS to fetch a list of available versions.
|
||||
func NewVCSLister(goBinPath string, env []string, fs afero.Fs) UpstreamLister {
|
||||
return &vcsLister{
|
||||
goBinPath: goBinPath,
|
||||
@@ -39,13 +39,13 @@ func NewVCSLister(goBinPath string, env []string, fs afero.Fs) UpstreamLister {
|
||||
|
||||
func (l *vcsLister) List(ctx context.Context, module string) (*storage.RevInfo, []string, error) {
|
||||
const op errors.Op = "vcsLister.List"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
_, span := observ.StartSpan(ctx, op.String())
|
||||
defer span.End()
|
||||
tmpDir, err := afero.TempDir(l.fs, "", "go-list")
|
||||
if err != nil {
|
||||
return nil, nil, errors.E(op, err)
|
||||
}
|
||||
defer l.fs.RemoveAll(tmpDir)
|
||||
defer func() { _ = l.fs.RemoveAll(tmpDir) }()
|
||||
|
||||
cmd := exec.Command(
|
||||
l.goBinPath,
|
||||
@@ -62,12 +62,12 @@ func (l *vcsLister) List(ctx context.Context, module string) (*storage.RevInfo,
|
||||
if err != nil {
|
||||
return nil, nil, errors.E(op, err)
|
||||
}
|
||||
defer clearFiles(l.fs, gopath)
|
||||
defer func() { _ = clearFiles(l.fs, gopath) }()
|
||||
cmd.Env = prepareEnv(gopath, l.env)
|
||||
|
||||
err = cmd.Run()
|
||||
if err != nil {
|
||||
err = fmt.Errorf("%v: %s", err, stderr)
|
||||
err = fmt.Errorf("%w: %s", err, stderr)
|
||||
// as of now, we can't recognize between a true NotFound
|
||||
// and an unexpected error, so we choose the more
|
||||
// hopeful path of NotFound. This way the Go command
|
||||
|
||||
@@ -9,7 +9,7 @@ import (
|
||||
|
||||
// prepareEnv will return all the appropriate
|
||||
// environment variables for a Go Command to run
|
||||
// successfully (such as GOPATH, GOCACHE, PATH etc)
|
||||
// successfully (such as GOPATH, GOCACHE, PATH etc).
|
||||
func prepareEnv(gopath string, envVars []string) []string {
|
||||
gopathEnv := fmt.Sprintf("GOPATH=%s", gopath)
|
||||
cacheEnv := fmt.Sprintf("GOCACHE=%s", filepath.Join(gopath, "cache"))
|
||||
|
||||
@@ -14,10 +14,10 @@ type zipReadCloser struct {
|
||||
goPath string
|
||||
}
|
||||
|
||||
// Close closes the zip file handle and clears up disk space used by the underlying disk ref
|
||||
// It is the caller's responsibility to call this method to free up utilized disk space
|
||||
// Close closes the zip file handle and clears up disk space used by the underlying disk ref.
|
||||
// It is the caller's responsibility to call this method to free up utilized disk space.
|
||||
func (rc *zipReadCloser) Close() error {
|
||||
rc.zip.Close()
|
||||
_ = rc.zip.Close()
|
||||
return clearFiles(rc.fs, rc.goPath)
|
||||
}
|
||||
|
||||
@@ -25,8 +25,8 @@ func (rc *zipReadCloser) Read(p []byte) (n int, err error) {
|
||||
return rc.zip.Read(p)
|
||||
}
|
||||
|
||||
// clearFiles deletes all data from the given fs at path root
|
||||
// This function must be called when zip is closed to cleanup the entire GOPATH created by the diskref
|
||||
// clearFiles deletes all data from the given fs at path root.
|
||||
// This function must be called when zip is closed to cleanup the entire GOPATH created by the diskref.
|
||||
func clearFiles(fs afero.Fs, root string) error {
|
||||
const op errors.Op = "module.ClearFiles"
|
||||
// This is required because vgo ensures dependencies are read-only
|
||||
@@ -36,7 +36,7 @@ func clearFiles(fs afero.Fs, root string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return fs.Chmod(path, 0770)
|
||||
return fs.Chmod(path, 0o770)
|
||||
}
|
||||
err := afero.Walk(fs, root, walkFn)
|
||||
if err != nil {
|
||||
|
||||
+18
-18
@@ -13,16 +13,16 @@ import (
|
||||
|
||||
// RegisterExporter determines the type of TraceExporter service for exporting traces from opencensus
|
||||
// User can choose from multiple tracing services (datadog, jaegar)
|
||||
// RegisterExporter returns the 'Flush' function for that particular tracing service
|
||||
func RegisterExporter(traceExporter, URL, service, ENV string) (func(), error) {
|
||||
// RegisterExporter returns the 'Flush' function for that particular tracing service.
|
||||
func RegisterExporter(traceExporter, url, service, env string) (func(), error) {
|
||||
const op errors.Op = "observ.RegisterExporter"
|
||||
switch traceExporter {
|
||||
case "jaeger":
|
||||
return registerJaegerExporter(URL, service, ENV)
|
||||
return registerJaegerExporter(url, service, env)
|
||||
case "datadog":
|
||||
return registerDatadogExporter(URL, service, ENV)
|
||||
return registerDatadogExporter(url, service, env)
|
||||
case "stackdriver":
|
||||
return registerStackdriverExporter(URL, ENV)
|
||||
return registerStackdriverExporter(url, env)
|
||||
case "":
|
||||
return nil, errors.E(op, "Exporter not specified. Traces won't be exported")
|
||||
default:
|
||||
@@ -32,14 +32,14 @@ func RegisterExporter(traceExporter, URL, service, ENV string) (func(), error) {
|
||||
|
||||
// registerJaegerExporter creates a jaeger exporter for exporting traces to opencensus.
|
||||
// Currently uses the 'TraceExporter' variable in the config file.
|
||||
// It should in the future have a nice sampling rate defined
|
||||
func registerJaegerExporter(URL, service, ENV string) (func(), error) {
|
||||
// It should in the future have a nice sampling rate defined.
|
||||
func registerJaegerExporter(url, service, env string) (func(), error) {
|
||||
const op errors.Op = "observ.registerJaegarExporter"
|
||||
if URL == "" {
|
||||
if url == "" {
|
||||
return nil, errors.E(op, "Exporter URL is empty. Traces won't be exported")
|
||||
}
|
||||
ex, err := jaeger.NewExporter(jaeger.Options{
|
||||
Endpoint: URL,
|
||||
Endpoint: url,
|
||||
Process: jaeger.Process{
|
||||
ServiceName: service,
|
||||
Tags: []jaeger.Tag{
|
||||
@@ -53,41 +53,41 @@ func registerJaegerExporter(URL, service, ENV string) (func(), error) {
|
||||
if err != nil {
|
||||
return nil, errors.E(op, err)
|
||||
}
|
||||
traceRegisterExporter(ex, ENV)
|
||||
traceRegisterExporter(ex, env)
|
||||
return ex.Flush, nil
|
||||
}
|
||||
|
||||
func traceRegisterExporter(exporter trace.Exporter, ENV string) {
|
||||
func traceRegisterExporter(exporter trace.Exporter, env string) {
|
||||
trace.RegisterExporter(exporter)
|
||||
if ENV == "development" {
|
||||
if env == "development" {
|
||||
trace.ApplyConfig(trace.Config{DefaultSampler: trace.AlwaysSample()})
|
||||
}
|
||||
}
|
||||
|
||||
// registerDatadogTracerExporter creates a datadog exporter.
|
||||
// Currently uses the 'TraceExporter' variable in the config file.
|
||||
func registerDatadogExporter(URL, service, ENV string) (func(), error) {
|
||||
func registerDatadogExporter(url, service, env string) (func(), error) {
|
||||
ex := datadog.NewExporter(
|
||||
datadog.Options{
|
||||
TraceAddr: URL,
|
||||
TraceAddr: url,
|
||||
Service: service,
|
||||
})
|
||||
traceRegisterExporter(ex, ENV)
|
||||
traceRegisterExporter(ex, env)
|
||||
return ex.Stop, nil
|
||||
}
|
||||
|
||||
func registerStackdriverExporter(projectID, ENV string) (func(), error) {
|
||||
func registerStackdriverExporter(projectID, env string) (func(), error) {
|
||||
const op errors.Op = "observ.registerStackdriverExporter"
|
||||
ex, err := stackdriver.NewExporter(stackdriver.Options{ProjectID: projectID})
|
||||
if err != nil {
|
||||
return nil, errors.E(op, err)
|
||||
}
|
||||
traceRegisterExporter(ex, ENV)
|
||||
traceRegisterExporter(ex, env)
|
||||
return ex.Flush, nil
|
||||
}
|
||||
|
||||
// StartSpan takes in a Context Interface and opName and starts a span. It returns the new attached ObserverContext
|
||||
// and span
|
||||
// and span.
|
||||
func StartSpan(ctx context.Context, op string) (context.Context, *trace.Span) {
|
||||
return trace.StartSpan(ctx, op)
|
||||
}
|
||||
|
||||
+4
-4
@@ -15,10 +15,10 @@ import (
|
||||
)
|
||||
|
||||
// RegisterStatsExporter determines the type of StatsExporter service for exporting stats from Opencensus
|
||||
// Currently it supports: prometheus
|
||||
// Currently it supports: prometheus.
|
||||
func RegisterStatsExporter(r *mux.Router, statsExporter, service string) (func(), error) {
|
||||
const op errors.Op = "observ.RegisterStatsExporter"
|
||||
var stop = func() {}
|
||||
stop := func() {}
|
||||
var err error
|
||||
switch statsExporter {
|
||||
case "prometheus":
|
||||
@@ -38,7 +38,7 @@ func RegisterStatsExporter(r *mux.Router, statsExporter, service string) (func()
|
||||
default:
|
||||
return nil, errors.E(op, fmt.Sprintf("StatsExporter %s not supported. Please open PR or an issue at github.com/gomods/athens", statsExporter))
|
||||
}
|
||||
if err := registerViews(); err != nil {
|
||||
if err = registerViews(); err != nil {
|
||||
return nil, errors.E(op, err)
|
||||
}
|
||||
|
||||
@@ -89,7 +89,7 @@ func registerStatsStackDriverExporter(projectID string) (func(), error) {
|
||||
return sd.Flush, nil
|
||||
}
|
||||
|
||||
// registerViews register stats which should be collected in Athens
|
||||
// registerViews register stats which should be collected in Athens.
|
||||
func registerViews() error {
|
||||
const op errors.Op = "observ.registerViews"
|
||||
if err := view.Register(
|
||||
|
||||
+2
-1
@@ -19,8 +19,9 @@ func DecodePath(encoding string) (path string, err error) {
|
||||
return path, nil
|
||||
}
|
||||
|
||||
// Ripped from cmd/go
|
||||
// Ripped from cmd/go.
|
||||
func decodeString(encoding string) (string, bool) {
|
||||
//nolint:prealloc
|
||||
var buf []byte
|
||||
|
||||
bang := false
|
||||
|
||||
+5
-5
@@ -9,7 +9,7 @@ import (
|
||||
"github.com/gorilla/mux"
|
||||
)
|
||||
|
||||
// GetModule gets the module from the path of a ?go-get=1 request
|
||||
// GetModule gets the module from the path of a ?go-get=1 request.
|
||||
func GetModule(r *http.Request) (string, error) {
|
||||
const op errors.Op = "paths.GetModule"
|
||||
module := mux.Vars(r)["module"]
|
||||
@@ -19,7 +19,7 @@ func GetModule(r *http.Request) (string, error) {
|
||||
return DecodePath(module)
|
||||
}
|
||||
|
||||
// GetVersion gets the version from the path of a ?go-get=1 request
|
||||
// GetVersion gets the version from the path of a ?go-get=1 request.
|
||||
func GetVersion(r *http.Request) (string, error) {
|
||||
const op errors.Op = "paths.GetVersion"
|
||||
|
||||
@@ -31,13 +31,13 @@ func GetVersion(r *http.Request) (string, error) {
|
||||
}
|
||||
|
||||
// AllPathParams holds the module and version in the path of a ?go-get=1
|
||||
// request
|
||||
// request.
|
||||
type AllPathParams struct {
|
||||
Module string `json:"module"`
|
||||
Version string `json:"version"`
|
||||
}
|
||||
|
||||
// GetAllParams fetches the path params from r and returns them
|
||||
// GetAllParams fetches the path params from r and returns them.
|
||||
func GetAllParams(r *http.Request) (*AllPathParams, error) {
|
||||
const op errors.Op = "paths.GetAllParams"
|
||||
mod, err := GetModule(r)
|
||||
@@ -54,7 +54,7 @@ func GetAllParams(r *http.Request) (*AllPathParams, error) {
|
||||
}
|
||||
|
||||
// MatchesPattern reports whether the path prefix of target matches
|
||||
// pattern (as defined by path.Match)
|
||||
// pattern (as defined by path.Match).
|
||||
//
|
||||
// This tries to keep the same behavior as GOPRIVATE/GONOPROXY/GONOSUMDB,
|
||||
// and is adopted from:
|
||||
|
||||
@@ -3,18 +3,18 @@ package requestid
|
||||
import "context"
|
||||
|
||||
// HeaderKey is the header key that athens uses
|
||||
// to pass request ids into logs and outbound requests
|
||||
// to pass request ids into logs and outbound requests.
|
||||
const HeaderKey = "Athens-Request-ID"
|
||||
|
||||
type key struct{}
|
||||
|
||||
// SetInContext sets the given requestID into the context
|
||||
// SetInContext sets the given requestID into the context.
|
||||
func SetInContext(ctx context.Context, id string) context.Context {
|
||||
return context.WithValue(ctx, key{}, id)
|
||||
}
|
||||
|
||||
// FromContext returns a requestID from the context or an empty
|
||||
// string if not found
|
||||
// string if not found.
|
||||
func FromContext(ctx context.Context) string {
|
||||
id, _ := ctx.Value(key{}).(string)
|
||||
return id
|
||||
|
||||
@@ -19,7 +19,7 @@ import (
|
||||
// what was requested, this is helpful if what was requested
|
||||
// was a descriptive version such as a branch name or a full commit sha.
|
||||
type Stasher interface {
|
||||
Stash(ctx context.Context, mod string, ver string) (string, error)
|
||||
Stash(ctx context.Context, mod, ver string) (string, error)
|
||||
}
|
||||
|
||||
// Wrapper helps extend the main stasher's functionality with addons.
|
||||
@@ -46,7 +46,7 @@ type stasher struct {
|
||||
|
||||
func (s *stasher) Stash(ctx context.Context, mod, ver string) (string, error) {
|
||||
const op errors.Op = "stasher.Stash"
|
||||
_, span := observ.StartSpan(ctx, op.String())
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
defer span.End()
|
||||
log.EntryFromContext(ctx).Debugf("saving %s@%s to storage...", mod, ver)
|
||||
|
||||
@@ -58,7 +58,7 @@ func (s *stasher) Stash(ctx context.Context, mod, ver string) (string, error) {
|
||||
if err != nil {
|
||||
return "", errors.E(op, err)
|
||||
}
|
||||
defer v.Zip.Close()
|
||||
defer func() { _ = v.Zip.Close() }()
|
||||
if v.Semver != ver {
|
||||
exists, err := s.checker.Exists(ctx, mod, v.Semver)
|
||||
if err != nil {
|
||||
|
||||
@@ -132,8 +132,8 @@ func (s *azblobLock) acquireLease(ctx context.Context, blobURL azblob.BlockBlobU
|
||||
_, err := blobURL.Upload(tctx, bytes.NewReader([]byte{1}), azblob.BlobHTTPHeaders{}, nil, azblob.BlobAccessConditions{})
|
||||
if err != nil {
|
||||
// if the blob is already leased we will get http.StatusPreconditionFailed while writing to that blob
|
||||
stgErr, ok := err.(azblob.StorageError)
|
||||
if !ok || stgErr.Response().StatusCode != http.StatusPreconditionFailed {
|
||||
var stgErr azblob.StorageError
|
||||
if !errors.AsErr(err, &stgErr) || stgErr.Response().StatusCode != http.StatusPreconditionFailed {
|
||||
return "", errors.E(op, err)
|
||||
}
|
||||
}
|
||||
@@ -147,7 +147,8 @@ func (s *azblobLock) acquireLease(ctx context.Context, blobURL azblob.BlockBlobU
|
||||
res, err := blobURL.AcquireLease(tctx, leaseID.String(), 15, azblob.ModifiedAccessConditions{})
|
||||
if err != nil {
|
||||
// if the blob is already leased we will get http.StatusConflict - wait and try again
|
||||
if stgErr, ok := err.(azblob.StorageError); ok && stgErr.Response().StatusCode == http.StatusConflict {
|
||||
var stgErr azblob.StorageError
|
||||
if ok := errors.AsErr(err, &stgErr); ok && stgErr.Response().StatusCode == http.StatusConflict {
|
||||
select {
|
||||
case <-time.After(1 * time.Second):
|
||||
continue
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
"github.com/gomods/athens/pkg/errors"
|
||||
"github.com/gomods/athens/pkg/observ"
|
||||
"github.com/gomods/athens/pkg/storage"
|
||||
"go.etcd.io/etcd/client/v3"
|
||||
clientv3 "go.etcd.io/etcd/client/v3"
|
||||
"go.etcd.io/etcd/client/v3/concurrency"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
@@ -13,14 +13,14 @@ import (
|
||||
"github.com/gomods/athens/pkg/storage"
|
||||
)
|
||||
|
||||
// RedisLogger mirrors github.com/go-redis/redis/v8/internal.Logging
|
||||
// RedisLogger mirrors github.com/go-redis/redis/v8/internal.Logging.
|
||||
type RedisLogger interface {
|
||||
Printf(ctx context.Context, format string, v ...interface{})
|
||||
}
|
||||
|
||||
// WithRedisLock returns a distributed singleflight
|
||||
// using a redis cluster. If it cannot connect, it will return an error.
|
||||
func WithRedisLock(l RedisLogger, endpoint string, password string, checker storage.Checker, lockConfig *config.RedisLockConfig) (Wrapper, error) {
|
||||
func WithRedisLock(l RedisLogger, endpoint, password string, checker storage.Checker, lockConfig *config.RedisLockConfig) (Wrapper, error) {
|
||||
redis.SetLogger(l)
|
||||
|
||||
const op errors.Op = "stash.WithRedisLock"
|
||||
|
||||
@@ -10,7 +10,7 @@ import (
|
||||
)
|
||||
|
||||
// WithRedisSentinelLock returns a distributed singleflight
|
||||
// with a redis cluster that utilizes sentinel for quorum and failover
|
||||
// with a redis cluster that utilizes sentinel for quorum and failover.
|
||||
func WithRedisSentinelLock(l RedisLogger, endpoints []string, master, password string, checker storage.Checker, lockConfig *config.RedisLockConfig) (Wrapper, error) {
|
||||
redis.SetLogger(l)
|
||||
|
||||
|
||||
@@ -15,14 +15,6 @@ import (
|
||||
"github.com/gomods/athens/pkg/storage"
|
||||
)
|
||||
|
||||
type client interface {
|
||||
UploadWithContext(ctx context.Context, path, contentType string, content io.Reader) error
|
||||
BlobExists(ctx context.Context, path string) (bool, error)
|
||||
ReadBlob(ctx context.Context, path string) (io.ReadCloser, error)
|
||||
ListBlobs(ctx context.Context, prefix string) ([]string, error)
|
||||
DeleteBlob(ctx context.Context, path string) error
|
||||
}
|
||||
|
||||
type azureBlobStoreClient struct {
|
||||
containerURL *azblob.ContainerURL
|
||||
}
|
||||
@@ -45,13 +37,13 @@ func newBlobStoreClient(accountURL *url.URL, accountName, accountKey, containerN
|
||||
}
|
||||
|
||||
// Storage implements (github.com/gomods/athens/pkg/storage).Saver and
|
||||
// also provides a function to fetch the location of a module
|
||||
// also provides a function to fetch the location of a module.
|
||||
type Storage struct {
|
||||
client *azureBlobStoreClient
|
||||
timeout time.Duration
|
||||
}
|
||||
|
||||
// New creates a new azure blobs storage
|
||||
// New creates a new azure blobs storage.
|
||||
func New(conf *config.AzureBlobConfig, timeout time.Duration) (*Storage, error) {
|
||||
const op errors.Op = "azureblob.New"
|
||||
u, err := url.Parse(fmt.Sprintf("https://%s.blob.core.windows.net", conf.AccountName))
|
||||
@@ -65,7 +57,7 @@ func New(conf *config.AzureBlobConfig, timeout time.Duration) (*Storage, error)
|
||||
return &Storage{client: cl, timeout: timeout}, nil
|
||||
}
|
||||
|
||||
// BlobExists checks if a particular blob exists in the container
|
||||
// BlobExists checks if a particular blob exists in the container.
|
||||
func (c *azureBlobStoreClient) BlobExists(ctx context.Context, path string) (bool, error) {
|
||||
const op errors.Op = "azureblob.BlobExists"
|
||||
// TODO: Any better way of doing this ?
|
||||
@@ -73,10 +65,8 @@ func (c *azureBlobStoreClient) BlobExists(ctx context.Context, path string) (boo
|
||||
_, err := blobURL.GetProperties(ctx, azblob.BlobAccessConditions{})
|
||||
if err != nil {
|
||||
var serr azblob.StorageError
|
||||
var ok bool
|
||||
|
||||
if serr, ok = err.(azblob.StorageError); !ok {
|
||||
return false, errors.E(op, fmt.Errorf("Error in casting to azure error type %v", err))
|
||||
if !errors.AsErr(err, &serr) {
|
||||
return false, errors.E(op, fmt.Errorf("error in casting to azure error type %w", err))
|
||||
}
|
||||
if serr.Response().StatusCode == http.StatusNotFound {
|
||||
return false, nil
|
||||
@@ -85,10 +75,9 @@ func (c *azureBlobStoreClient) BlobExists(ctx context.Context, path string) (boo
|
||||
return false, errors.E(op, err)
|
||||
}
|
||||
return true, nil
|
||||
|
||||
}
|
||||
|
||||
// ReadBlob returns a storage.SizeReadCloser for the contents of a blob
|
||||
// ReadBlob returns a storage.SizeReadCloser for the contents of a blob.
|
||||
func (c *azureBlobStoreClient) ReadBlob(ctx context.Context, path string) (storage.SizeReadCloser, error) {
|
||||
const op errors.Op = "azureblob.ReadBlob"
|
||||
blobURL := c.containerURL.NewBlockBlobURL(path)
|
||||
@@ -101,7 +90,7 @@ func (c *azureBlobStoreClient) ReadBlob(ctx context.Context, path string) (stora
|
||||
return storage.NewSizer(rc, size), nil
|
||||
}
|
||||
|
||||
// ListBlobs will list all blobs which has the given prefix
|
||||
// ListBlobs will list all blobs which has the given prefix.
|
||||
func (c *azureBlobStoreClient) ListBlobs(ctx context.Context, prefix string) ([]string, error) {
|
||||
const op errors.Op = "azureblob.ListBlobs"
|
||||
var blobs []string
|
||||
@@ -122,7 +111,7 @@ func (c *azureBlobStoreClient) ListBlobs(ctx context.Context, prefix string) ([]
|
||||
return blobs, nil
|
||||
}
|
||||
|
||||
// DeleteBlob deletes the blob with the given path
|
||||
// DeleteBlob deletes the blob with the given path.
|
||||
func (c *azureBlobStoreClient) DeleteBlob(ctx context.Context, path string) error {
|
||||
const op errors.Op = "azureblob.DeleteBlob"
|
||||
blobURL := c.containerURL.NewBlockBlobURL(path)
|
||||
@@ -133,7 +122,7 @@ func (c *azureBlobStoreClient) DeleteBlob(ctx context.Context, path string) erro
|
||||
return nil
|
||||
}
|
||||
|
||||
// UploadWithContext uploads a blob to the container
|
||||
// UploadWithContext uploads a blob to the container.
|
||||
func (c *azureBlobStoreClient) UploadWithContext(ctx context.Context, path, contentType string, content io.Reader) error {
|
||||
const op errors.Op = "azureblob.UploadWithContext"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
|
||||
@@ -12,8 +12,8 @@ import (
|
||||
"github.com/gomods/athens/pkg/paths"
|
||||
)
|
||||
|
||||
// Catalog implements the (./pkg/storage).Catalog interface
|
||||
// It returns a list of versions, if any, for a given module
|
||||
// Catalog implements the (./pkg/storage).Catalog interface.
|
||||
// It returns a list of versions, if any, for a given module.
|
||||
func (s *Storage) Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error) {
|
||||
const op errors.Op = "azblob.Catalog"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
|
||||
@@ -9,8 +9,8 @@ import (
|
||||
)
|
||||
|
||||
// Exists implements the (./pkg/storage).Checker interface
|
||||
// returning true if the module at version exists in storage
|
||||
func (s *Storage) Exists(ctx context.Context, module string, version string) (bool, error) {
|
||||
// returning true if the module at version exists in storage.
|
||||
func (s *Storage) Exists(ctx context.Context, module, version string) (bool, error) {
|
||||
const op errors.Op = "azureblob.Exists"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
defer span.End()
|
||||
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
// Delete implements the (./pkg/storage).Deleter interface and
|
||||
// removes a version of a module from storage. Returning ErrNotFound
|
||||
// if the version does not exist.
|
||||
func (s *Storage) Delete(ctx context.Context, module string, version string) error {
|
||||
func (s *Storage) Delete(ctx context.Context, module, version string) error {
|
||||
const op errors.Op = "azureblob.Delete"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
defer span.End()
|
||||
|
||||
@@ -11,8 +11,8 @@ import (
|
||||
"github.com/gomods/athens/pkg/storage"
|
||||
)
|
||||
|
||||
// Info implements the (./pkg/storage).Getter interface
|
||||
func (s *Storage) Info(ctx context.Context, module string, version string) ([]byte, error) {
|
||||
// Info implements the (./pkg/storage).Getter interface.
|
||||
func (s *Storage) Info(ctx context.Context, module, version string) ([]byte, error) {
|
||||
const op errors.Op = "azureblob.Info"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
defer span.End()
|
||||
@@ -43,8 +43,8 @@ func (s *Storage) Info(ctx context.Context, module string, version string) ([]by
|
||||
return infoBytes, nil
|
||||
}
|
||||
|
||||
// GoMod implements the (./pkg/storage).Getter interface
|
||||
func (s *Storage) GoMod(ctx context.Context, module string, version string) ([]byte, error) {
|
||||
// GoMod implements the (./pkg/storage).Getter interface.
|
||||
func (s *Storage) GoMod(ctx context.Context, module, version string) ([]byte, error) {
|
||||
const op errors.Op = "azureblob.GoMod"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
defer span.End()
|
||||
@@ -64,7 +64,7 @@ func (s *Storage) GoMod(ctx context.Context, module string, version string) ([]b
|
||||
|
||||
modBytes, err := io.ReadAll(modReader)
|
||||
if err != nil {
|
||||
return nil, errors.E(op, fmt.Errorf("could not get new reader for mod file: %s", err), errors.M(module), errors.V(version))
|
||||
return nil, errors.E(op, fmt.Errorf("could not get new reader for mod file: %w", err), errors.M(module), errors.V(version))
|
||||
}
|
||||
|
||||
err = modReader.Close()
|
||||
@@ -75,8 +75,8 @@ func (s *Storage) GoMod(ctx context.Context, module string, version string) ([]b
|
||||
return modBytes, nil
|
||||
}
|
||||
|
||||
// Zip implements the (./pkg/storage).Getter interface
|
||||
func (s *Storage) Zip(ctx context.Context, module string, version string) (storage.SizeReadCloser, error) {
|
||||
// Zip implements the (./pkg/storage).Getter interface.
|
||||
func (s *Storage) Zip(ctx context.Context, module, version string) (storage.SizeReadCloser, error) {
|
||||
const op errors.Op = "azureblob.Zip"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
defer span.End()
|
||||
|
||||
@@ -8,8 +8,8 @@ import (
|
||||
"github.com/gomods/athens/pkg/observ"
|
||||
)
|
||||
|
||||
// List implements the (./pkg/storage).Lister interface
|
||||
// It returns a list of versions, if any, for a given module
|
||||
// List implements the (./pkg/storage).Lister interface.
|
||||
// It returns a list of versions, if any, for a given module.
|
||||
func (s *Storage) List(ctx context.Context, module string) ([]string, error) {
|
||||
const op errors.Op = "azureblob.List"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
package storage
|
||||
|
||||
// Backend is a complete storage backend (i.e. file system, database) implementation - a lister, reader and saver
|
||||
// Backend is a complete storage backend (i.e. file system, database) implementation - a lister, reader and saver.
|
||||
type Backend interface {
|
||||
Lister
|
||||
Getter
|
||||
|
||||
@@ -6,7 +6,7 @@ import (
|
||||
"github.com/gomods/athens/pkg/paths"
|
||||
)
|
||||
|
||||
// Cataloger is the interface that lists all the modules and version contained in the storage
|
||||
// Cataloger is the interface that lists all the modules and version contained in the storage.
|
||||
type Cataloger interface {
|
||||
// Catalog gets all the modules / versions.
|
||||
Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error)
|
||||
|
||||
@@ -6,14 +6,14 @@ import (
|
||||
"github.com/gomods/athens/pkg/errors"
|
||||
)
|
||||
|
||||
// Checker is the interface that checks if the version of the module exists
|
||||
// Checker is the interface that checks if the version of the module exists.
|
||||
type Checker interface {
|
||||
// Exists checks whether or not module in specified version is present
|
||||
// in the backing storage
|
||||
// in the backing storage.
|
||||
Exists(ctx context.Context, module, version string) (bool, error)
|
||||
}
|
||||
|
||||
// WithChecker wraps the backend with a Checker implementaiton
|
||||
// WithChecker wraps the backend with a Checker implementation.
|
||||
func WithChecker(strg Backend) Checker {
|
||||
if checker, ok := strg.(Checker); ok {
|
||||
return checker
|
||||
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"math/rand"
|
||||
"sort"
|
||||
"testing"
|
||||
|
||||
"github.com/gomods/athens/pkg/errors"
|
||||
@@ -26,7 +25,6 @@ func RunTests(t *testing.T, b storage.Backend, clearBackend func() error) {
|
||||
testGet(t, b)
|
||||
testExists(t, b)
|
||||
testShouldNotExist(t, b)
|
||||
// testCatalog(t, b)
|
||||
}
|
||||
|
||||
// testNotFound ensures that a storage Backend
|
||||
@@ -209,49 +207,6 @@ func testDelete(t *testing.T, b storage.Backend) {
|
||||
require.Equal(t, false, exists)
|
||||
}
|
||||
|
||||
func testCatalog(t *testing.T, b storage.Backend) {
|
||||
cs, ok := b.(storage.Cataloger)
|
||||
if !ok {
|
||||
t.Skip()
|
||||
}
|
||||
ctx := context.Background()
|
||||
|
||||
mock := getMockModule()
|
||||
zipBts, _ := io.ReadAll(mock.Zip)
|
||||
modname := "github.com/gomods/testCatalogModule"
|
||||
for i := 0; i < 6; i++ {
|
||||
ver := fmt.Sprintf("v1.2.%04d", i)
|
||||
b.Save(ctx, modname, ver, mock.Mod, bytes.NewReader(zipBts), mock.Info)
|
||||
|
||||
defer b.Delete(ctx, modname, ver)
|
||||
}
|
||||
|
||||
allres, next, err := cs.Catalog(ctx, "", 5)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 5, len(allres))
|
||||
|
||||
res, next, err := cs.Catalog(ctx, next, 50)
|
||||
allres = append(allres, res...)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, len(res))
|
||||
require.Equal(t, "", next)
|
||||
|
||||
sort.Slice(allres, func(i, j int) bool {
|
||||
if allres[i].Module == allres[j].Module {
|
||||
return allres[i].Version < allres[j].Version
|
||||
}
|
||||
return allres[i].Module < allres[j].Module
|
||||
})
|
||||
require.Equal(t, modname, allres[0].Module)
|
||||
require.Equal(t, "v1.2.0000", allres[0].Version)
|
||||
require.Equal(t, "v1.2.0004", allres[4].Version)
|
||||
|
||||
for i := 1; i < len(allres); i++ {
|
||||
require.NotEqual(t, allres[i].Version, allres[i-1].Version)
|
||||
}
|
||||
}
|
||||
|
||||
func getMockModule() *storage.Version {
|
||||
return &storage.Version{
|
||||
Info: []byte("123"),
|
||||
|
||||
@@ -2,7 +2,7 @@ package storage
|
||||
|
||||
import "context"
|
||||
|
||||
// Deleter deletes module metadata and its source from underlying storage
|
||||
// Deleter deletes module metadata and its source from underlying storage.
|
||||
type Deleter interface {
|
||||
// Delete must return ErrNotFound if the module/version are not
|
||||
// found.
|
||||
|
||||
Vendored
+14
-14
@@ -20,7 +20,7 @@ type service struct {
|
||||
c *http.Client
|
||||
}
|
||||
|
||||
// NewClient returns an external storage client
|
||||
// NewClient returns an external storage client.
|
||||
func NewClient(url string, c *http.Client) storage.Backend {
|
||||
if c == nil {
|
||||
c = &http.Client{}
|
||||
@@ -93,7 +93,7 @@ func (s *service) Save(ctx context.Context, mod, ver string, modFile []byte, zip
|
||||
mw := multipart.NewWriter(pw)
|
||||
go func() {
|
||||
err := upload(mw, modFile, info, zip)
|
||||
pw.CloseWithError(err)
|
||||
_ = pw.CloseWithError(err)
|
||||
}()
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, pr)
|
||||
if err != nil {
|
||||
@@ -104,8 +104,8 @@ func (s *service) Save(ctx context.Context, mod, ver string, modFile []byte, zip
|
||||
if err != nil {
|
||||
return errors.E(op, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != 200 {
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
bts, _ := io.ReadAll(resp.Body)
|
||||
return errors.E(op, fmt.Errorf("unexpected status code: %v - body: %s", resp.StatusCode, bts), resp.StatusCode)
|
||||
}
|
||||
@@ -118,35 +118,35 @@ func (s *service) Delete(ctx context.Context, mod, ver string) error {
|
||||
if err != nil {
|
||||
return errors.E(op, err)
|
||||
}
|
||||
defer body.Close()
|
||||
defer func() { _ = body.Close() }()
|
||||
return nil
|
||||
}
|
||||
|
||||
func upload(mw *multipart.Writer, mod, info []byte, zip io.Reader) error {
|
||||
defer mw.Close()
|
||||
defer func() { _ = mw.Close() }()
|
||||
infoW, err := mw.CreateFormFile("mod.info", "mod.info")
|
||||
if err != nil {
|
||||
return fmt.Errorf("error creating info file: %v", err)
|
||||
return fmt.Errorf("error creating info file: %w", err)
|
||||
}
|
||||
_, err = infoW.Write(info)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error writing info file: %v", err)
|
||||
return fmt.Errorf("error writing info file: %w", err)
|
||||
}
|
||||
modW, err := mw.CreateFormFile("mod.mod", "mod.mod")
|
||||
if err != nil {
|
||||
return fmt.Errorf("error creating mod file: %v", err)
|
||||
return fmt.Errorf("error creating mod file: %w", err)
|
||||
}
|
||||
_, err = modW.Write(mod)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error writing mod file: %v", err)
|
||||
return fmt.Errorf("error writing mod file: %w", err)
|
||||
}
|
||||
zipW, err := mw.CreateFormFile("mod.zip", "mod.zip")
|
||||
if err != nil {
|
||||
return fmt.Errorf("error creating zip file: %v", err)
|
||||
return fmt.Errorf("error creating zip file: %w", err)
|
||||
}
|
||||
_, err = io.Copy(zipW, zip)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error writing zip file: %v", err)
|
||||
return fmt.Errorf("error writing zip file: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -174,9 +174,9 @@ func (s *service) doRequest(ctx context.Context, method, mod, ver, ext string) (
|
||||
if err != nil {
|
||||
return nil, 0, errors.E(op, err)
|
||||
}
|
||||
if resp.StatusCode != 200 {
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
_ = resp.Body.Close()
|
||||
return nil, 0, errors.E(op, fmt.Errorf("none 200 status code: %v - body: %s", resp.StatusCode, body), resp.StatusCode)
|
||||
}
|
||||
var size int64
|
||||
|
||||
Vendored
+21
-21
@@ -17,7 +17,7 @@ import (
|
||||
|
||||
// NewServer takes a storage.Backend implementation of your
|
||||
// choice, and returns a new http.Handler that Athens can
|
||||
// reach out to for storage operations
|
||||
// reach out to for storage operations.
|
||||
func NewServer(strg storage.Backend) http.Handler {
|
||||
r := mux.NewRouter()
|
||||
r.HandleFunc(download.PathList, func(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -27,12 +27,12 @@ func NewServer(strg storage.Backend) http.Handler {
|
||||
http.Error(w, err.Error(), errors.Kind(err))
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(w, "%s", strings.Join(list, "\n"))
|
||||
_, _ = fmt.Fprintf(w, "%s", strings.Join(list, "\n"))
|
||||
}).Methods(http.MethodGet)
|
||||
r.HandleFunc(download.PathVersionInfo, func(w http.ResponseWriter, r *http.Request) {
|
||||
params, err := paths.GetAllParams(r)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), 400)
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
info, err := strg.Info(r.Context(), params.Module, params.Version)
|
||||
@@ -40,12 +40,12 @@ func NewServer(strg storage.Backend) http.Handler {
|
||||
http.Error(w, err.Error(), errors.Kind(err))
|
||||
return
|
||||
}
|
||||
w.Write(info)
|
||||
_, _ = w.Write(info)
|
||||
}).Methods(http.MethodGet)
|
||||
r.HandleFunc(download.PathVersionModule, func(w http.ResponseWriter, r *http.Request) {
|
||||
params, err := paths.GetAllParams(r)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), 400)
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
mod, err := strg.GoMod(r.Context(), params.Module, params.Version)
|
||||
@@ -53,12 +53,12 @@ func NewServer(strg storage.Backend) http.Handler {
|
||||
http.Error(w, err.Error(), errors.Kind(err))
|
||||
return
|
||||
}
|
||||
w.Write(mod)
|
||||
_, _ = w.Write(mod)
|
||||
}).Methods(http.MethodGet)
|
||||
r.HandleFunc(download.PathVersionZip, func(w http.ResponseWriter, r *http.Request) {
|
||||
params, err := paths.GetAllParams(r)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), 400)
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
zip, err := strg.Zip(r.Context(), params.Module, params.Version)
|
||||
@@ -66,52 +66,52 @@ func NewServer(strg storage.Backend) http.Handler {
|
||||
http.Error(w, err.Error(), errors.Kind(err))
|
||||
return
|
||||
}
|
||||
defer zip.Close()
|
||||
defer func() { _ = zip.Close() }()
|
||||
w.Header().Set("Content-Length", strconv.FormatInt(zip.Size(), 10))
|
||||
io.Copy(w, zip)
|
||||
_, _ = io.Copy(w, zip)
|
||||
}).Methods(http.MethodGet)
|
||||
r.HandleFunc("/{module:.+}/@v/{version}.save", func(w http.ResponseWriter, r *http.Request) {
|
||||
params, err := paths.GetAllParams(r)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), 400)
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
err = r.ParseMultipartForm(zip.MaxZipFile + zip.MaxGoMod)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), 400)
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
infoFile, _, err := r.FormFile("mod.info")
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), 400)
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
defer infoFile.Close()
|
||||
defer func() { _ = infoFile.Close() }()
|
||||
info, err := io.ReadAll(infoFile)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), 400)
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
modReader, _, err := r.FormFile("mod.mod")
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), 400)
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
defer modReader.Close()
|
||||
defer func() { _ = modReader.Close() }()
|
||||
modFile, err := io.ReadAll(modReader)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), 400)
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
modZ, _, err := r.FormFile("mod.zip")
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), 400)
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
defer modZ.Close()
|
||||
defer func() { _ = modZ.Close() }()
|
||||
err = strg.Save(r.Context(), params.Module, params.Version, modFile, modZ, info)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), 400)
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
}).Methods(http.MethodPost)
|
||||
@@ -119,7 +119,7 @@ func NewServer(strg storage.Backend) http.Handler {
|
||||
r.HandleFunc("/{module:.+}/@v/{version}.delete", func(w http.ResponseWriter, r *http.Request) {
|
||||
params, err := paths.GetAllParams(r)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), 400)
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
err = strg.Delete(r.Context(), params.Module, params.Version)
|
||||
|
||||
@@ -15,11 +15,11 @@ import (
|
||||
|
||||
const tokenSeparator = "|"
|
||||
|
||||
// Catalog implements the (./pkg/storage).Cataloger interface
|
||||
// It returns a list of modules and versions contained in the storage
|
||||
// Catalog implements the (./pkg/storage).Cataloger interface.
|
||||
// It returns a list of modules and versions contained in the storage.
|
||||
func (s *storageImpl) Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error) {
|
||||
const op errors.Op = "fs.Catalog"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
_, span := observ.StartSpan(ctx, op.String())
|
||||
defer span.End()
|
||||
|
||||
fromModule, fromVersion, err := modVerFromToken(token)
|
||||
@@ -41,7 +41,7 @@ func (s *storageImpl) Catalog(ctx context.Context, token string, pageSize int) (
|
||||
|
||||
m, version := filepath.Split(modVer)
|
||||
module := filepath.Clean(m)
|
||||
module = strings.Replace(module, string(os.PathSeparator), "/", -1)
|
||||
module = strings.ReplaceAll(module, string(os.PathSeparator), "/")
|
||||
|
||||
if fromModule != "" && module < fromModule { // it is ok to land on the same module
|
||||
return nil
|
||||
@@ -60,7 +60,7 @@ func (s *storageImpl) Catalog(ctx context.Context, token string, pageSize int) (
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil && err != io.EOF {
|
||||
if err != nil && !errors.IsErr(err, io.EOF) {
|
||||
return nil, "", errors.E(op, err, errors.KindUnexpected)
|
||||
}
|
||||
|
||||
|
||||
@@ -9,14 +9,13 @@ import (
|
||||
"github.com/spf13/afero"
|
||||
)
|
||||
|
||||
func (v *storageImpl) Exists(ctx context.Context, module, version string) (bool, error) {
|
||||
func (s *storageImpl) Exists(ctx context.Context, module, version string) (bool, error) {
|
||||
const op errors.Op = "fs.Exists"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
_, span := observ.StartSpan(ctx, op.String())
|
||||
defer span.End()
|
||||
versionedPath := v.versionLocation(module, version)
|
||||
|
||||
files, err := afero.ReadDir(v.filesystem, versionedPath)
|
||||
versionedPath := s.versionLocation(module, version)
|
||||
|
||||
files, err := afero.ReadDir(s.filesystem, versionedPath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return false, nil
|
||||
|
||||
@@ -8,17 +8,17 @@ import (
|
||||
)
|
||||
|
||||
// Delete removes a specific version of a module.
|
||||
func (v *storageImpl) Delete(ctx context.Context, module, version string) error {
|
||||
func (s *storageImpl) Delete(ctx context.Context, module, version string) error {
|
||||
const op errors.Op = "fs.Delete"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
defer span.End()
|
||||
versionedPath := v.versionLocation(module, version)
|
||||
exists, err := v.Exists(ctx, module, version)
|
||||
versionedPath := s.versionLocation(module, version)
|
||||
exists, err := s.Exists(ctx, module, version)
|
||||
if err != nil {
|
||||
return errors.E(op, err, errors.M(module), errors.V(version))
|
||||
}
|
||||
if !exists {
|
||||
return errors.E(op, errors.M(module), errors.V(version), errors.KindNotFound)
|
||||
}
|
||||
return v.filesystem.RemoveAll(versionedPath)
|
||||
return s.filesystem.RemoveAll(versionedPath)
|
||||
}
|
||||
|
||||
@@ -21,17 +21,16 @@ func (s *storageImpl) moduleLocation(module string) string {
|
||||
|
||||
func (s *storageImpl) versionLocation(module, version string) string {
|
||||
return filepath.Join(s.moduleLocation(module), version)
|
||||
|
||||
}
|
||||
|
||||
// NewStorage returns a new ListerSaver implementation that stores
|
||||
// everything under rootDir
|
||||
// If the root directory does not exist an error is returned
|
||||
// everything under rootDir.
|
||||
// If the root directory does not exist an error is returned.
|
||||
func NewStorage(rootDir string, filesystem afero.Fs) (storage.Backend, error) {
|
||||
const op errors.Op = "fs.NewStorage"
|
||||
exists, err := afero.Exists(filesystem, rootDir)
|
||||
if err != nil {
|
||||
return nil, errors.E(op, fmt.Errorf("could not check if root directory `%s` exists: %s", rootDir, err))
|
||||
return nil, errors.E(op, fmt.Errorf("could not check if root directory `%s` exists: %w", rootDir, err))
|
||||
}
|
||||
if !exists {
|
||||
return nil, errors.E(op, fmt.Errorf("root directory `%s` does not exist", rootDir))
|
||||
|
||||
+12
-24
@@ -11,12 +11,12 @@ import (
|
||||
"github.com/spf13/afero"
|
||||
)
|
||||
|
||||
func (v *storageImpl) Info(ctx context.Context, module, version string) ([]byte, error) {
|
||||
func (s *storageImpl) Info(ctx context.Context, module, version string) ([]byte, error) {
|
||||
const op errors.Op = "fs.Info"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
_, span := observ.StartSpan(ctx, op.String())
|
||||
defer span.End()
|
||||
versionedPath := v.versionLocation(module, version)
|
||||
info, err := afero.ReadFile(v.filesystem, filepath.Join(versionedPath, version+".info"))
|
||||
versionedPath := s.versionLocation(module, version)
|
||||
info, err := afero.ReadFile(s.filesystem, filepath.Join(versionedPath, version+".info"))
|
||||
if err != nil {
|
||||
return nil, errors.E(op, errors.M(module), errors.V(version), errors.KindNotFound)
|
||||
}
|
||||
@@ -24,12 +24,12 @@ func (v *storageImpl) Info(ctx context.Context, module, version string) ([]byte,
|
||||
return info, nil
|
||||
}
|
||||
|
||||
func (v *storageImpl) GoMod(ctx context.Context, module, version string) ([]byte, error) {
|
||||
func (s *storageImpl) GoMod(ctx context.Context, module, version string) ([]byte, error) {
|
||||
const op errors.Op = "fs.GoMod"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
_, span := observ.StartSpan(ctx, op.String())
|
||||
defer span.End()
|
||||
versionedPath := v.versionLocation(module, version)
|
||||
mod, err := afero.ReadFile(v.filesystem, filepath.Join(versionedPath, "go.mod"))
|
||||
versionedPath := s.versionLocation(module, version)
|
||||
mod, err := afero.ReadFile(s.filesystem, filepath.Join(versionedPath, "go.mod"))
|
||||
if err != nil {
|
||||
return nil, errors.E(op, errors.M(module), errors.V(version), errors.KindNotFound)
|
||||
}
|
||||
@@ -37,13 +37,13 @@ func (v *storageImpl) GoMod(ctx context.Context, module, version string) ([]byte
|
||||
return mod, nil
|
||||
}
|
||||
|
||||
func (v *storageImpl) Zip(ctx context.Context, module, version string) (storage.SizeReadCloser, error) {
|
||||
func (s *storageImpl) Zip(ctx context.Context, module, version string) (storage.SizeReadCloser, error) {
|
||||
const op errors.Op = "fs.Zip"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
_, span := observ.StartSpan(ctx, op.String())
|
||||
defer span.End()
|
||||
versionedPath := v.versionLocation(module, version)
|
||||
versionedPath := s.versionLocation(module, version)
|
||||
|
||||
src, err := v.filesystem.OpenFile(filepath.Join(versionedPath, "source.zip"), os.O_RDONLY, 0666)
|
||||
src, err := s.filesystem.OpenFile(filepath.Join(versionedPath, "source.zip"), os.O_RDONLY, 0o666)
|
||||
if err != nil {
|
||||
return nil, errors.E(op, errors.M(module), errors.V(version), errors.KindNotFound)
|
||||
}
|
||||
@@ -53,15 +53,3 @@ func (v *storageImpl) Zip(ctx context.Context, module, version string) (storage.
|
||||
}
|
||||
return storage.NewSizer(src, fi.Size()), nil
|
||||
}
|
||||
|
||||
func (v *storageImpl) ZipSize(ctx context.Context, module, version string) (int64, error) {
|
||||
const op errors.Op = "fs.ZipFileSize"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
defer span.End()
|
||||
versionedPath := v.versionLocation(module, version)
|
||||
fi, err := v.filesystem.Stat(filepath.Join(versionedPath))
|
||||
if err != nil {
|
||||
return 0, errors.E(op, err, errors.M(module), errors.V(version), errors.KindNotFound)
|
||||
}
|
||||
return fi.Size(), nil
|
||||
}
|
||||
|
||||
@@ -11,12 +11,12 @@ import (
|
||||
"golang.org/x/mod/semver"
|
||||
)
|
||||
|
||||
func (l *storageImpl) List(ctx context.Context, module string) ([]string, error) {
|
||||
func (s *storageImpl) List(ctx context.Context, module string) ([]string, error) {
|
||||
const op errors.Op = "fs.List"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
_, span := observ.StartSpan(ctx, op.String())
|
||||
defer span.End()
|
||||
loc := l.moduleLocation(module)
|
||||
fileInfos, err := afero.ReadDir(l.filesystem, loc)
|
||||
loc := s.moduleLocation(module)
|
||||
fileInfos, err := afero.ReadDir(s.filesystem, loc)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return []string{}, nil
|
||||
|
||||
+12
-12
@@ -13,37 +13,37 @@ import (
|
||||
|
||||
func (s *storageImpl) Save(ctx context.Context, module, version string, mod []byte, zip io.Reader, info []byte) error {
|
||||
const op errors.Op = "fs.Save"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
_, span := observ.StartSpan(ctx, op.String())
|
||||
defer span.End()
|
||||
dir := s.versionLocation(module, version)
|
||||
|
||||
// NB: the process's umask is subtracted from the permissions below,
|
||||
// so a umask of for example 0077 allows directories and files to be
|
||||
// created with mode 0700 / 0600, i.e. not world- or group-readable
|
||||
// NB: The process's umask is subtracted from the permissions below,
|
||||
// so an umask of for example 0077 allows directories and files to be
|
||||
// created with mode 0700 / 0600, i.e. not world- or group-readable.
|
||||
|
||||
// make the versioned directory to hold the go.mod and the zipfile
|
||||
if err := s.filesystem.MkdirAll(dir, 0777); err != nil {
|
||||
// Make the versioned directory to hold the go.mod and the zipfile.
|
||||
if err := s.filesystem.MkdirAll(dir, 0o777); err != nil {
|
||||
return errors.E(op, err, errors.M(module), errors.V(version))
|
||||
}
|
||||
|
||||
// write the go.mod file
|
||||
if err := afero.WriteFile(s.filesystem, filepath.Join(dir, "go.mod"), mod, 0666); err != nil {
|
||||
// Write the go.mod file.
|
||||
if err := afero.WriteFile(s.filesystem, filepath.Join(dir, "go.mod"), mod, 0o666); err != nil {
|
||||
return errors.E(op, err, errors.M(module), errors.V(version))
|
||||
}
|
||||
|
||||
// write the zipfile
|
||||
f, err := s.filesystem.OpenFile(filepath.Join(dir, "source.zip"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
|
||||
// Write the zipfile.
|
||||
f, err := s.filesystem.OpenFile(filepath.Join(dir, "source.zip"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o666)
|
||||
if err != nil {
|
||||
return errors.E(op, err, errors.M(module), errors.V(version))
|
||||
}
|
||||
defer f.Close()
|
||||
defer func() { _ = f.Close() }()
|
||||
_, err = io.Copy(f, zip)
|
||||
if err != nil {
|
||||
return errors.E(op, err, errors.M(module), errors.V(version))
|
||||
}
|
||||
|
||||
// write the info file
|
||||
err = afero.WriteFile(s.filesystem, filepath.Join(dir, version+".info"), info, 0666)
|
||||
err = afero.WriteFile(s.filesystem, filepath.Join(dir, version+".info"), info, 0o666)
|
||||
if err != nil {
|
||||
return errors.E(op, err)
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@ import (
|
||||
)
|
||||
|
||||
// Catalog implements the (./pkg/storage).Catalog interface
|
||||
// It returns a list of versions, if any, for a given module
|
||||
// It returns a list of versions, if any, for a given module.
|
||||
func (s *Storage) Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error) {
|
||||
const op errors.Op = "gcp.Catalog"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
)
|
||||
|
||||
// Exists implements the (./pkg/storage).Checker interface
|
||||
// returning true if the module at version exists in storage
|
||||
// returning true if the module at version exists in storage.
|
||||
func (s *Storage) Exists(ctx context.Context, module, version string) (bool, error) {
|
||||
const op errors.Op = "gcp.Exists"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
@@ -21,7 +21,7 @@ func (s *Storage) Exists(ctx context.Context, module, version string) (bool, err
|
||||
var count int
|
||||
for {
|
||||
attrs, err := it.Next()
|
||||
if err == iterator.Done {
|
||||
if errors.IsErr(err, iterator.Done) {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
"google.golang.org/api/option"
|
||||
)
|
||||
|
||||
// Storage implements the (./pkg/storage).Backend interface
|
||||
// Storage implements the (./pkg/storage).Backend interface.
|
||||
type Storage struct {
|
||||
bucket *storage.BucketHandle
|
||||
timeout time.Duration
|
||||
@@ -34,8 +34,8 @@ func New(ctx context.Context, gcpConf *config.GCPConfig, timeout time.Duration)
|
||||
return nil, errors.E(op, err)
|
||||
}
|
||||
|
||||
if _, err := s.bucket.Attrs(ctx); err != nil {
|
||||
if err == storage.ErrBucketNotExist {
|
||||
if _, err = s.bucket.Attrs(ctx); err != nil {
|
||||
if errors.IsErr(err, storage.ErrBucketNotExist) {
|
||||
return nil, errors.E(op, "You must manually create a storage bucket for Athens, see https://cloud.google.com/storage/docs/creating-buckets#storage-create-bucket-console")
|
||||
}
|
||||
return nil, errors.E(op, err)
|
||||
@@ -48,21 +48,21 @@ func New(ctx context.Context, gcpConf *config.GCPConfig, timeout time.Duration)
|
||||
// this is so that the unit tests can use this to create their own short-lived buckets.
|
||||
func newClient(ctx context.Context, gcpConf *config.GCPConfig, timeout time.Duration) (*Storage, error) {
|
||||
const op errors.Op = "gcp.newClient"
|
||||
opts := []option.ClientOption{}
|
||||
var opts []option.ClientOption
|
||||
if gcpConf.JSONKey != "" {
|
||||
key, err := base64.StdEncoding.DecodeString(gcpConf.JSONKey)
|
||||
if err != nil {
|
||||
return nil, errors.E(op, fmt.Errorf("could not decode base64 json key: %v", err))
|
||||
return nil, errors.E(op, fmt.Errorf("could not decode base64 json key: %w", err))
|
||||
}
|
||||
creds, err := google.CredentialsFromJSON(ctx, key, storage.ScopeReadWrite)
|
||||
if err != nil {
|
||||
return nil, errors.E(op, fmt.Errorf("could not get GCS credentials: %v", err))
|
||||
return nil, errors.E(op, fmt.Errorf("could not get GCS credentials: %w", err))
|
||||
}
|
||||
opts = append(opts, option.WithCredentials(creds))
|
||||
}
|
||||
s, err := storage.NewClient(ctx, opts...)
|
||||
if err != nil {
|
||||
return nil, errors.E(op, fmt.Errorf("could not create new storage client: %s", err))
|
||||
return nil, errors.E(op, fmt.Errorf("could not create new storage client: %w", err))
|
||||
}
|
||||
|
||||
return &Storage{
|
||||
|
||||
@@ -12,7 +12,7 @@ import (
|
||||
pkgstorage "github.com/gomods/athens/pkg/storage"
|
||||
)
|
||||
|
||||
// Info implements Getter
|
||||
// Info implements Getter.
|
||||
func (s *Storage) Info(ctx context.Context, module, version string) ([]byte, error) {
|
||||
const op errors.Op = "gcp.Info"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
@@ -22,14 +22,14 @@ func (s *Storage) Info(ctx context.Context, module, version string) ([]byte, err
|
||||
return nil, errors.E(op, err, getErrorKind(err), errors.M(module), errors.V(version))
|
||||
}
|
||||
infoBytes, err := io.ReadAll(infoReader)
|
||||
infoReader.Close()
|
||||
_ = infoReader.Close()
|
||||
if err != nil {
|
||||
return nil, errors.E(op, err, errors.M(module), errors.V(version))
|
||||
}
|
||||
return infoBytes, nil
|
||||
}
|
||||
|
||||
// GoMod implements Getter
|
||||
// GoMod implements Getter.
|
||||
func (s *Storage) GoMod(ctx context.Context, module, version string) ([]byte, error) {
|
||||
const op errors.Op = "gcp.GoMod"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
@@ -39,15 +39,15 @@ func (s *Storage) GoMod(ctx context.Context, module, version string) ([]byte, er
|
||||
return nil, errors.E(op, err, getErrorKind(err), errors.M(module), errors.V(version))
|
||||
}
|
||||
modBytes, err := io.ReadAll(modReader)
|
||||
modReader.Close()
|
||||
_ = modReader.Close()
|
||||
if err != nil {
|
||||
return nil, errors.E(op, fmt.Errorf("could not get new reader for mod file: %s", err), errors.M(module), errors.V(version))
|
||||
return nil, errors.E(op, fmt.Errorf("could not get new reader for mod file: %w", err), errors.M(module), errors.V(version))
|
||||
}
|
||||
|
||||
return modBytes, nil
|
||||
}
|
||||
|
||||
// Zip implements Getter
|
||||
// Zip implements Getter.
|
||||
func (s *Storage) Zip(ctx context.Context, module, version string) (pkgstorage.SizeReadCloser, error) {
|
||||
const op errors.Op = "gcp.Zip"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
@@ -56,11 +56,11 @@ func (s *Storage) Zip(ctx context.Context, module, version string) (pkgstorage.S
|
||||
if err != nil {
|
||||
return nil, errors.E(op, err, getErrorKind(err), errors.M(module), errors.V(version))
|
||||
}
|
||||
return pkgstorage.NewSizer(zipReader, zipReader.Size()), nil
|
||||
return pkgstorage.NewSizer(zipReader, zipReader.Attrs.Size), nil
|
||||
}
|
||||
|
||||
func getErrorKind(err error) int {
|
||||
if err == storage.ErrObjectNotExist {
|
||||
if errors.IsErr(err, storage.ErrObjectNotExist) {
|
||||
return errors.KindNotFound
|
||||
}
|
||||
return errors.KindUnexpected
|
||||
|
||||
@@ -10,8 +10,8 @@ import (
|
||||
"google.golang.org/api/iterator"
|
||||
)
|
||||
|
||||
// List implements the (./pkg/storage).Lister interface
|
||||
// It returns a list of versions, if any, for a given module
|
||||
// List implements the (./pkg/storage).Lister interface.
|
||||
// It returns a list of versions, if any, for a given module.
|
||||
func (s *Storage) List(ctx context.Context, module string) ([]string, error) {
|
||||
const op errors.Op = "gcp.List"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
@@ -22,7 +22,7 @@ func (s *Storage) List(ctx context.Context, module string) ([]string, error) {
|
||||
paths := []string{}
|
||||
for {
|
||||
attrs, err := it.Next()
|
||||
if err == iterator.Done {
|
||||
if errors.IsErr(err, iterator.Done) {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
|
||||
@@ -53,15 +53,15 @@ func (s *Storage) upload(ctx context.Context, path string, stream io.Reader) err
|
||||
// Once we support private storage buckets this may need refactoring
|
||||
// unless there is a way to set the default perms in the project.
|
||||
if _, err := io.Copy(wc, stream); err != nil {
|
||||
wc.Close()
|
||||
_ = wc.Close()
|
||||
return err
|
||||
}
|
||||
|
||||
err := wc.Close()
|
||||
if err != nil {
|
||||
kind := errors.KindBadRequest
|
||||
apiErr, ok := err.(*googleapi.Error)
|
||||
if ok && apiErr.Code == 412 {
|
||||
apiErr := &googleapi.Error{}
|
||||
if errors.AsErr(err, &apiErr) && apiErr.Code == 412 {
|
||||
kind = errors.KindAlreadyExists
|
||||
}
|
||||
return errors.E(op, err, kind)
|
||||
|
||||
@@ -5,7 +5,7 @@ import (
|
||||
"io"
|
||||
)
|
||||
|
||||
// Getter gets module metadata and its source from underlying storage
|
||||
// Getter gets module metadata and its source from underlying storage.
|
||||
type Getter interface {
|
||||
Info(ctx context.Context, module, vsn string) ([]byte, error)
|
||||
GoMod(ctx context.Context, module, vsn string) ([]byte, error)
|
||||
@@ -14,14 +14,14 @@ type Getter interface {
|
||||
|
||||
// SizeReadCloser extends io.ReadCloser
|
||||
// with a Size() method that tells you the
|
||||
// length of the io.ReadCloser if read in full
|
||||
// length of the io.ReadCloser if read in full.
|
||||
type SizeReadCloser interface {
|
||||
io.ReadCloser
|
||||
Size() int64
|
||||
}
|
||||
|
||||
// NewSizer is a helper wrapper to return an implementation
|
||||
// of ReadCloserSizer
|
||||
// of ReadCloserSizer.
|
||||
func NewSizer(rc io.ReadCloser, size int64) SizeReadCloser {
|
||||
return &sizeReadCloser{rc, size}
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@ package storage
|
||||
|
||||
import "context"
|
||||
|
||||
// Lister is the interface that lists versions of a specific baseURL & module
|
||||
// Lister is the interface that lists versions of a specific baseURL & module.
|
||||
type Lister interface {
|
||||
// List gets all the versions for the given baseURL & module.
|
||||
// It returns ErrNotFound if the module isn't found
|
||||
|
||||
@@ -9,19 +9,19 @@ import (
|
||||
"github.com/spf13/afero"
|
||||
)
|
||||
|
||||
// NewStorage creates new in-memory storage using the afero.NewMemMapFs() in memory file system
|
||||
// NewStorage creates new in-memory storage using the afero.NewMemMapFs() in memory file system.
|
||||
func NewStorage() (storage.Backend, error) {
|
||||
const op errors.Op = "mem.NewStorage"
|
||||
|
||||
memFs := afero.NewMemMapFs()
|
||||
tmpDir, err := afero.TempDir(memFs, "", "")
|
||||
if err != nil {
|
||||
return nil, errors.E(op, fmt.Errorf("could not create temp dir for 'In Memory' storage (%s)", err))
|
||||
return nil, errors.E(op, fmt.Errorf("could not create temp dir for 'In Memory' storage: %w", err))
|
||||
}
|
||||
|
||||
memStorage, err := fs.NewStorage(tmpDir, memFs)
|
||||
if err != nil {
|
||||
return nil, errors.E(op, fmt.Errorf("could not create storage from memory fs (%s)", err))
|
||||
return nil, errors.E(op, fmt.Errorf("could not create storage from memory fs: %w", err))
|
||||
}
|
||||
return memStorage, nil
|
||||
}
|
||||
|
||||
@@ -11,11 +11,11 @@ import (
|
||||
"github.com/minio/minio-go/v6"
|
||||
)
|
||||
|
||||
// Catalog implements the (./pkg/storage).Cataloger interface
|
||||
// It returns a list of modules and versions contained in the storage
|
||||
// Catalog implements the (./pkg/storage).Cataloger interface.
|
||||
// It returns a list of modules and versions contained in the storage.
|
||||
func (s *storageImpl) Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error) {
|
||||
const op errors.Op = "minio.Catalog"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
_, span := observ.StartSpan(ctx, op.String())
|
||||
defer span.End()
|
||||
res := make([]paths.AllPathParams, 0)
|
||||
count := pageSize
|
||||
@@ -50,7 +50,7 @@ func fetchModsAndVersions(objects []minio.ObjectInfo, elementsNum int) ([]paths.
|
||||
continue
|
||||
}
|
||||
|
||||
p, err := parseMinioKey(&o)
|
||||
p, err := parseMinioKey(o)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
@@ -66,7 +66,7 @@ func fetchModsAndVersions(objects []minio.ObjectInfo, elementsNum int) ([]paths.
|
||||
return res, lastKey
|
||||
}
|
||||
|
||||
func parseMinioKey(o *minio.ObjectInfo) (paths.AllPathParams, error) {
|
||||
func parseMinioKey(o minio.ObjectInfo) (paths.AllPathParams, error) {
|
||||
const op errors.Op = "minio.parseMinioKey"
|
||||
|
||||
_, m, v := extractKey(o.Key)
|
||||
|
||||
@@ -8,21 +8,17 @@ import (
|
||||
"github.com/gomods/athens/pkg/observ"
|
||||
)
|
||||
|
||||
const (
|
||||
minioErrorCodeNoSuchKey = "NoSuchKey"
|
||||
)
|
||||
|
||||
func (v *storageImpl) Exists(ctx context.Context, module, version string) (bool, error) {
|
||||
func (s *storageImpl) Exists(ctx context.Context, module, version string) (bool, error) {
|
||||
const op errors.Op = "minio.Exists"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
_, span := observ.StartSpan(ctx, op.String())
|
||||
defer span.End()
|
||||
versionedPath := v.versionLocation(module, version)
|
||||
versionedPath := s.versionLocation(module, version)
|
||||
modPath := fmt.Sprintf("%s/go.mod", versionedPath)
|
||||
infoPath := fmt.Sprintf("%s/%s.info", versionedPath, version)
|
||||
zipPath := fmt.Sprintf("%s/source.zip", versionedPath)
|
||||
|
||||
var count int
|
||||
objectCh, err := v.minioCore.ListObjectsV2(v.bucketName, versionedPath, "", false, "", 0, "")
|
||||
objectCh, err := s.minioCore.ListObjectsV2(s.bucketName, versionedPath, "", false, "", 0, "")
|
||||
if err != nil {
|
||||
return false, errors.E(op, err, errors.M(module), errors.V(version))
|
||||
}
|
||||
|
||||
@@ -8,11 +8,11 @@ import (
|
||||
"github.com/gomods/athens/pkg/observ"
|
||||
)
|
||||
|
||||
func (v *storageImpl) Delete(ctx context.Context, module, version string) error {
|
||||
func (s *storageImpl) Delete(ctx context.Context, module, version string) error {
|
||||
const op errors.Op = "minio.Delete"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
defer span.End()
|
||||
exists, err := v.Exists(ctx, module, version)
|
||||
exists, err := s.Exists(ctx, module, version)
|
||||
if err != nil {
|
||||
return errors.E(op, err, errors.M(module), errors.V(version))
|
||||
}
|
||||
@@ -21,20 +21,20 @@ func (v *storageImpl) Delete(ctx context.Context, module, version string) error
|
||||
return errors.E(op, errors.M(module), errors.V(version), errors.KindNotFound)
|
||||
}
|
||||
|
||||
versionedPath := v.versionLocation(module, version)
|
||||
versionedPath := s.versionLocation(module, version)
|
||||
|
||||
modPath := fmt.Sprintf("%s/go.mod", versionedPath)
|
||||
if err := v.minioClient.RemoveObject(v.bucketName, modPath); err != nil {
|
||||
if err := s.minioClient.RemoveObject(s.bucketName, modPath); err != nil {
|
||||
return errors.E(op, err, errors.M(module), errors.V(version))
|
||||
}
|
||||
|
||||
zipPath := fmt.Sprintf("%s/source.zip", versionedPath)
|
||||
if err := v.minioClient.RemoveObject(v.bucketName, zipPath); err != nil {
|
||||
if err := s.minioClient.RemoveObject(s.bucketName, zipPath); err != nil {
|
||||
return errors.E(op, err, errors.M(module), errors.V(version))
|
||||
}
|
||||
|
||||
infoPath := fmt.Sprintf("%s/%s.info", versionedPath, version)
|
||||
err = v.minioClient.RemoveObject(v.bucketName, infoPath)
|
||||
err = s.minioClient.RemoveObject(s.bucketName, infoPath)
|
||||
if err != nil {
|
||||
return errors.E(op, err, errors.M(module), errors.V(version))
|
||||
}
|
||||
|
||||
+18
-17
@@ -12,16 +12,16 @@ import (
|
||||
minio "github.com/minio/minio-go/v6"
|
||||
)
|
||||
|
||||
func (v *storageImpl) Info(ctx context.Context, module, vsn string) ([]byte, error) {
|
||||
func (s *storageImpl) Info(ctx context.Context, module, vsn string) ([]byte, error) {
|
||||
const op errors.Op = "minio.Info"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
_, span := observ.StartSpan(ctx, op.String())
|
||||
defer span.End()
|
||||
infoPath := fmt.Sprintf("%s/%s.info", v.versionLocation(module, vsn), vsn)
|
||||
infoReader, err := v.minioClient.GetObject(v.bucketName, infoPath, minio.GetObjectOptions{})
|
||||
infoPath := fmt.Sprintf("%s/%s.info", s.versionLocation(module, vsn), vsn)
|
||||
infoReader, err := s.minioClient.GetObject(s.bucketName, infoPath, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, errors.E(op, err)
|
||||
}
|
||||
defer infoReader.Close()
|
||||
defer func() { _ = infoReader.Close() }()
|
||||
info, err := io.ReadAll(infoReader)
|
||||
if err != nil {
|
||||
return nil, transformNotFoundErr(op, module, vsn, err)
|
||||
@@ -30,16 +30,16 @@ func (v *storageImpl) Info(ctx context.Context, module, vsn string) ([]byte, err
|
||||
return info, nil
|
||||
}
|
||||
|
||||
func (v *storageImpl) GoMod(ctx context.Context, module, vsn string) ([]byte, error) {
|
||||
func (s *storageImpl) GoMod(ctx context.Context, module, vsn string) ([]byte, error) {
|
||||
const op errors.Op = "minio.GoMod"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
_, span := observ.StartSpan(ctx, op.String())
|
||||
defer span.End()
|
||||
modPath := fmt.Sprintf("%s/go.mod", v.versionLocation(module, vsn))
|
||||
modReader, err := v.minioClient.GetObject(v.bucketName, modPath, minio.GetObjectOptions{})
|
||||
modPath := fmt.Sprintf("%s/go.mod", s.versionLocation(module, vsn))
|
||||
modReader, err := s.minioClient.GetObject(s.bucketName, modPath, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, errors.E(op, err)
|
||||
}
|
||||
defer modReader.Close()
|
||||
defer func() { _ = modReader.Close() }()
|
||||
mod, err := io.ReadAll(modReader)
|
||||
if err != nil {
|
||||
return nil, transformNotFoundErr(op, module, vsn, err)
|
||||
@@ -48,31 +48,32 @@ func (v *storageImpl) GoMod(ctx context.Context, module, vsn string) ([]byte, er
|
||||
return mod, nil
|
||||
}
|
||||
|
||||
func (v *storageImpl) Zip(ctx context.Context, module, vsn string) (storage.SizeReadCloser, error) {
|
||||
func (s *storageImpl) Zip(ctx context.Context, module, vsn string) (storage.SizeReadCloser, error) {
|
||||
const op errors.Op = "minio.Zip"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
_, span := observ.StartSpan(ctx, op.String())
|
||||
defer span.End()
|
||||
|
||||
zipPath := fmt.Sprintf("%s/source.zip", v.versionLocation(module, vsn))
|
||||
_, err := v.minioClient.StatObject(v.bucketName, zipPath, minio.StatObjectOptions{})
|
||||
zipPath := fmt.Sprintf("%s/source.zip", s.versionLocation(module, vsn))
|
||||
_, err := s.minioClient.StatObject(s.bucketName, zipPath, minio.StatObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, errors.E(op, err, errors.KindNotFound, errors.M(module), errors.V(vsn))
|
||||
}
|
||||
|
||||
zipReader, err := v.minioClient.GetObject(v.bucketName, zipPath, minio.GetObjectOptions{})
|
||||
zipReader, err := s.minioClient.GetObject(s.bucketName, zipPath, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, errors.E(op, err)
|
||||
}
|
||||
oi, err := zipReader.Stat()
|
||||
if err != nil {
|
||||
zipReader.Close()
|
||||
_ = zipReader.Close()
|
||||
return nil, errors.E(op, err)
|
||||
}
|
||||
return storage.NewSizer(zipReader, oi.Size), nil
|
||||
}
|
||||
|
||||
func transformNotFoundErr(op errors.Op, module, version string, err error) error {
|
||||
if eresp, ok := err.(minio.ErrorResponse); ok {
|
||||
var eresp minio.ErrorResponse
|
||||
if errors.AsErr(err, &eresp) {
|
||||
if eresp.StatusCode == http.StatusNotFound {
|
||||
return errors.E(op, errors.M(module), errors.V(version), errors.KindNotFound)
|
||||
}
|
||||
|
||||
@@ -5,7 +5,7 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
func extractKey(objectKey string) (key string, module string, version string) {
|
||||
func extractKey(objectKey string) (key, module, version string) {
|
||||
var err error
|
||||
key, err = url.PathUnescape(objectKey)
|
||||
if err != nil {
|
||||
@@ -14,8 +14,8 @@ func extractKey(objectKey string) (key string, module string, version string) {
|
||||
|
||||
parts := strings.Split(key, "/")
|
||||
version = parts[len(parts)-2]
|
||||
module = strings.Replace(key, version, "", -2)
|
||||
module = strings.Replace(module, "//.info", "", -1)
|
||||
module = strings.ReplaceAll(key, version, "")
|
||||
module = strings.ReplaceAll(module, "//.info", "")
|
||||
|
||||
return key, module, version
|
||||
}
|
||||
|
||||
@@ -9,26 +9,25 @@ import (
|
||||
"github.com/gomods/athens/pkg/observ"
|
||||
)
|
||||
|
||||
func (l *storageImpl) List(ctx context.Context, module string) ([]string, error) {
|
||||
func (s *storageImpl) List(ctx context.Context, module string) ([]string, error) {
|
||||
const op errors.Op = "minio.List"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
_, span := observ.StartSpan(ctx, op.String())
|
||||
defer span.End()
|
||||
doneCh := make(chan struct{})
|
||||
defer close(doneCh)
|
||||
searchPrefix := module + "/"
|
||||
objectCh, err := l.minioCore.ListObjectsV2(l.bucketName, searchPrefix, "", false, "", 0, "")
|
||||
|
||||
objectCh, err := s.minioCore.ListObjectsV2(s.bucketName, searchPrefix, "", false, "", 0, "")
|
||||
if err != nil {
|
||||
return nil, errors.E(op, err, errors.M(module))
|
||||
}
|
||||
ret := []string{}
|
||||
var ret []string
|
||||
for _, object := range objectCh.Contents {
|
||||
if object.Err != nil {
|
||||
return nil, errors.E(op, object.Err, errors.M(module))
|
||||
}
|
||||
|
||||
key, _, ver := extractKey(object.Key)
|
||||
goModKey := fmt.Sprintf("%s/go.mod", l.versionLocation(module, ver))
|
||||
goModKey := fmt.Sprintf("%s/go.mod", s.versionLocation(module, ver))
|
||||
if goModKey == key {
|
||||
ret = append(ret, ver)
|
||||
}
|
||||
|
||||
@@ -21,7 +21,7 @@ func (s *storageImpl) versionLocation(module, version string) string {
|
||||
}
|
||||
|
||||
// NewStorage returns a connected Minio or DigitalOcean Spaces storage
|
||||
// that implements storage.Backend
|
||||
// that implements storage.Backend.
|
||||
func NewStorage(conf *config.MinioConfig, timeout time.Duration) (storage.Backend, error) {
|
||||
const op errors.Op = "minio.NewStorage"
|
||||
endpoint := conf.Endpoint
|
||||
|
||||
@@ -15,7 +15,7 @@ import (
|
||||
|
||||
func (s *storageImpl) Save(ctx context.Context, module, vsn string, mod []byte, zip io.Reader, info []byte) error {
|
||||
const op errors.Op = "storage.minio.Save"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
_, span := observ.StartSpan(ctx, op.String())
|
||||
defer span.End()
|
||||
dir := s.versionLocation(module, vsn)
|
||||
modFileName := dir + "/" + "go.mod"
|
||||
|
||||
@@ -7,26 +7,26 @@ import (
|
||||
|
||||
"github.com/gomods/athens/pkg/config"
|
||||
"github.com/gomods/athens/pkg/errors"
|
||||
multierror "github.com/hashicorp/go-multierror"
|
||||
"github.com/hashicorp/go-multierror"
|
||||
)
|
||||
|
||||
// Deleter takes a path to a file and deletes it from the blob store
|
||||
// Deleter takes a path to a file and deletes it from the blob store.
|
||||
type Deleter func(ctx context.Context, path string) error
|
||||
|
||||
// Delete deletes .info, .mod and .zip files from the blob store in parallel.
|
||||
// Returns multierror containing errors from all deletes and timeouts
|
||||
func Delete(ctx context.Context, module, version string, delete Deleter, timeout time.Duration) error {
|
||||
// Returns multierror containing errors from all deletes and timeouts.
|
||||
func Delete(ctx context.Context, module, version string, del Deleter, timeout time.Duration) error {
|
||||
const op errors.Op = "module.Delete"
|
||||
tctx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
|
||||
del := func(ext string) <-chan error {
|
||||
delFn := func(ext string) <-chan error {
|
||||
ec := make(chan error)
|
||||
|
||||
go func() {
|
||||
defer close(ec)
|
||||
p := config.PackageVersionedName(module, version, ext)
|
||||
ec <- delete(tctx, p)
|
||||
ec <- del(tctx, p)
|
||||
}()
|
||||
return ec
|
||||
}
|
||||
@@ -34,10 +34,10 @@ func Delete(ctx context.Context, module, version string, delete Deleter, timeout
|
||||
errChan := make(chan error, numFiles)
|
||||
delOrAbort := func(ext string) {
|
||||
select {
|
||||
case err := <-del(ext):
|
||||
case err := <-delFn(ext):
|
||||
errChan <- err
|
||||
case <-tctx.Done():
|
||||
errChan <- fmt.Errorf("deleting %s.%s.%s failed: %s", module, version, ext, tctx.Err())
|
||||
errChan <- fmt.Errorf("deleting %s.%s.%s failed: %w", module, version, ext, tctx.Err())
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -13,11 +13,11 @@ import (
|
||||
|
||||
const numFiles = 3
|
||||
|
||||
// Uploader takes a stream and saves it to the blob store under a given path
|
||||
// Uploader takes a stream and saves it to the blob store under a given path.
|
||||
type Uploader func(ctx context.Context, path, contentType string, stream io.Reader) error
|
||||
|
||||
// Upload saves .info, .mod and .zip files to the blob store in parallel.
|
||||
// Returns multierror containing errors from all uploads and timeouts
|
||||
// Returns multierror containing errors from all uploads and timeouts.
|
||||
func Upload(ctx context.Context, module, version string, info, mod, zip io.Reader, uploader Uploader, timeout time.Duration) error {
|
||||
const op errors.Op = "module.Upload"
|
||||
tctx, cancel := context.WithTimeout(ctx, timeout)
|
||||
@@ -40,7 +40,7 @@ func Upload(ctx context.Context, module, version string, info, mod, zip io.Reade
|
||||
case err := <-save(ext, contentType, stream):
|
||||
errChan <- err
|
||||
case <-tctx.Done():
|
||||
errChan <- fmt.Errorf("uploading %s.%s.%s failed: %s", module, version, ext, tctx.Err())
|
||||
errChan <- fmt.Errorf("uploading %s.%s.%s failed: %w", module, version, ext, tctx.Err())
|
||||
}
|
||||
}
|
||||
go saveOrAbort("info", "application/json", info)
|
||||
|
||||
@@ -2,6 +2,7 @@ package mongo
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/gomods/athens/pkg/errors"
|
||||
"github.com/gomods/athens/pkg/paths"
|
||||
"github.com/gomods/athens/pkg/storage"
|
||||
@@ -11,8 +12,8 @@ import (
|
||||
"go.mongodb.org/mongo-driver/mongo/options"
|
||||
)
|
||||
|
||||
// Catalog implements the (./pkg/storage).Cataloger interface
|
||||
// It returns a list of modules and versions contained in the storage
|
||||
// Catalog implements the (./pkg/storage).Cataloger interface.
|
||||
// It returns a list of modules and versions contained in the storage.
|
||||
func (s *ModuleStore) Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error) {
|
||||
const op errors.Op = "mongo.Catalog"
|
||||
q := bson.M{}
|
||||
@@ -33,7 +34,6 @@ func (s *ModuleStore) Catalog(ctx context.Context, token string, pageSize int) (
|
||||
modules := make([]storage.Module, 0)
|
||||
findOptions := options.Find().SetProjection(projection).SetSort(sort).SetLimit(int64(pageSize))
|
||||
cursor, err := c.Find(tctx, q, findOptions)
|
||||
|
||||
if err != nil {
|
||||
return nil, "", errors.E(op, err)
|
||||
}
|
||||
@@ -53,13 +53,13 @@ func (s *ModuleStore) Catalog(ctx context.Context, token string, pageSize int) (
|
||||
return nil, "", nil
|
||||
}
|
||||
|
||||
var versions = make([]paths.AllPathParams, len(modules))
|
||||
versions := make([]paths.AllPathParams, len(modules))
|
||||
for i := range modules {
|
||||
versions[i].Module = modules[i].Module
|
||||
versions[i].Version = modules[i].Version
|
||||
}
|
||||
|
||||
var next = modules[len(modules)-1].ID.Hex()
|
||||
next := modules[len(modules)-1].ID.Hex()
|
||||
if len(modules) < pageSize {
|
||||
return versions, "", nil
|
||||
}
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
)
|
||||
|
||||
// Exists checks for a specific version of a module
|
||||
// Exists checks for a specific version of a module.
|
||||
func (s *ModuleStore) Exists(ctx context.Context, module, vsn string) (bool, error) {
|
||||
var op errors.Op = "mongo.Exists"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
"go.mongodb.org/mongo-driver/x/bsonx"
|
||||
)
|
||||
|
||||
// Delete removes a specific version of a module
|
||||
// Delete removes a specific version of a module.
|
||||
func (s *ModuleStore) Delete(ctx context.Context, module, version string) error {
|
||||
const op errors.Op = "mongo.Delete"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
@@ -40,11 +40,11 @@ func (s *ModuleStore) Delete(ctx context.Context, module, version string) error
|
||||
|
||||
var x bsonx.Doc
|
||||
for cursor.Next(ctx) {
|
||||
cursor.Decode(&x)
|
||||
_ = cursor.Decode(&x)
|
||||
}
|
||||
if err = bucket.Delete(x.Lookup("_id").ObjectID()); err != nil {
|
||||
kind := errors.KindUnexpected
|
||||
if err == gridfs.ErrFileNotFound {
|
||||
if errors.IsErr(err, gridfs.ErrFileNotFound) {
|
||||
kind = errors.KindNotFound
|
||||
}
|
||||
return errors.E(op, err, kind, errors.M(module), errors.V(version))
|
||||
|
||||
@@ -13,14 +13,13 @@ import (
|
||||
"go.mongodb.org/mongo-driver/x/bsonx"
|
||||
)
|
||||
|
||||
// Info implements storage.Getter
|
||||
// Info implements storage.Getter.
|
||||
func (s *ModuleStore) Info(ctx context.Context, module, vsn string) ([]byte, error) {
|
||||
const op errors.Op = "mongo.Info"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
defer span.End()
|
||||
|
||||
result, err := query(ctx, s, module, vsn)
|
||||
|
||||
if err != nil {
|
||||
return nil, errors.E(op, err)
|
||||
}
|
||||
@@ -28,14 +27,13 @@ func (s *ModuleStore) Info(ctx context.Context, module, vsn string) ([]byte, err
|
||||
return result.Info, nil
|
||||
}
|
||||
|
||||
// GoMod implements storage.Getter
|
||||
// GoMod implements storage.Getter.
|
||||
func (s *ModuleStore) GoMod(ctx context.Context, module, vsn string) ([]byte, error) {
|
||||
const op errors.Op = "mongo.GoMod"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
defer span.End()
|
||||
|
||||
result, err := query(ctx, s, module, vsn)
|
||||
|
||||
if err != nil {
|
||||
return nil, errors.E(op, err)
|
||||
}
|
||||
@@ -43,7 +41,7 @@ func (s *ModuleStore) GoMod(ctx context.Context, module, vsn string) ([]byte, er
|
||||
return result.Mod, nil
|
||||
}
|
||||
|
||||
// Zip implements storage.Getter
|
||||
// Zip implements storage.Getter.
|
||||
func (s *ModuleStore) Zip(ctx context.Context, module, vsn string) (storage.SizeReadCloser, error) {
|
||||
const op errors.Op = "mongo.Zip"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
@@ -59,7 +57,7 @@ func (s *ModuleStore) Zip(ctx context.Context, module, vsn string) (storage.Size
|
||||
dStream, err := bucket.OpenDownloadStreamByName(zipName, options.GridFSName())
|
||||
if err != nil {
|
||||
kind := errors.KindUnexpected
|
||||
if err == gridfs.ErrFileNotFound {
|
||||
if errors.IsErr(err, gridfs.ErrFileNotFound) {
|
||||
kind = errors.KindNotFound
|
||||
}
|
||||
return nil, errors.E(op, err, kind, errors.M(module), errors.V(vsn))
|
||||
@@ -79,7 +77,7 @@ func (s *ModuleStore) Zip(ctx context.Context, module, vsn string) (storage.Size
|
||||
return storage.NewSizer(dStream, size), nil
|
||||
}
|
||||
|
||||
// Query connects to and queries storage module
|
||||
// Query connects to and queries storage module.
|
||||
func query(ctx context.Context, s *ModuleStore, module, vsn string) (*storage.Module, error) {
|
||||
const op errors.Op = "mongo.query"
|
||||
ctx, span := observ.StartSpan(ctx, op.String())
|
||||
@@ -95,7 +93,7 @@ func query(ctx context.Context, s *ModuleStore, module, vsn string) (*storage.Mo
|
||||
queryResult := c.FindOne(tctx, bson.M{"module": module, "version": vsn})
|
||||
if queryErr := queryResult.Err(); queryErr != nil {
|
||||
kind := errors.KindUnexpected
|
||||
if queryErr == mongo.ErrNoDocuments {
|
||||
if errors.IsErr(queryErr, mongo.ErrNoDocuments) {
|
||||
kind = errors.KindNotFound
|
||||
}
|
||||
return nil, errors.E(op, queryErr, kind, errors.M(module), errors.V(vsn))
|
||||
@@ -103,7 +101,7 @@ func query(ctx context.Context, s *ModuleStore, module, vsn string) (*storage.Mo
|
||||
|
||||
if err := queryResult.Decode(result); err != nil {
|
||||
kind := errors.KindUnexpected
|
||||
if err == mongo.ErrNoDocuments {
|
||||
if errors.IsErr(err, mongo.ErrNoDocuments) {
|
||||
kind = errors.KindNotFound
|
||||
}
|
||||
return nil, errors.E(op, err, kind, errors.M(module), errors.V(vsn))
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user