chore: lint code with golangci-lint (#1828)

* feat: add golangci-lint linting

* chore: fix linter issues

* feat: add linting into the workflow

* docs: update lint docs

* fix: cr suggestions

* fix: remove old formatting and vetting scripts

* fix: add docker make target

* fix: action go caching

* fix: depreciated actions checkout version

* fix: cr suggestion

* fix: cr suggestions

---------

Co-authored-by: Manu Gupta <manugupt1@gmail.com>
This commit is contained in:
Nicholas Wiersma
2023-02-25 06:39:17 +02:00
committed by GitHub
parent 66582eebfe
commit d932d50232
134 changed files with 596 additions and 600 deletions
+2 -2
View File
@@ -9,7 +9,7 @@ import (
"fmt"
)
// Details represents known data for a given build
// Details represents known data for a given build.
type Details struct {
Version string `json:"version,omitempty"`
Date string `json:"date,omitempty"`
@@ -29,7 +29,7 @@ func String() string {
return fmt.Sprintf("Build Details:\n\tVersion:\t%s\n\tDate:\t\t%s", version, buildDate)
}
// Data returns build details as a struct
// Data returns build details as a struct.
func Data() Details {
return Details{
Version: version,
+1 -1
View File
@@ -1,6 +1,6 @@
package config
// AzureBlobConfig specifies the properties required to use Azure as the storage backend
// AzureBlobConfig specifies the properties required to use Azure as the storage backend.
type AzureBlobConfig struct {
AccountName string `validate:"required" envconfig:"ATHENS_AZURE_ACCOUNT_NAME"`
AccountKey string `validate:"required" envconfig:"ATHENS_AZURE_ACCOUNT_KEY"`
+18 -20
View File
@@ -18,7 +18,7 @@ import (
const defaultConfigFile = "athens.toml"
// Config provides configuration values for all components
// Config provides configuration values for all components.
type Config struct {
TimeoutConf
GoEnv string `validate:"required" envconfig:"GO_ENV"`
@@ -63,12 +63,12 @@ type Config struct {
}
// EnvList is a list of key-value environment
// variables that are passed to the Go command
// variables that are passed to the Go command.
type EnvList []string
// HasKey returns whether a key-value entry
// is present by only checking the left of
// key=value
// key=value.
func (el EnvList) HasKey(key string) bool {
for _, env := range el {
if strings.HasPrefix(env, key+"=") {
@@ -79,7 +79,7 @@ func (el EnvList) HasKey(key string) bool {
}
// Add adds a key=value entry to the environment
// list
// list.
func (el *EnvList) Add(key, value string) {
*el = append(*el, key+"="+value)
}
@@ -110,7 +110,7 @@ func (el *EnvList) Decode(value string) error {
}
// Validate validates that all strings inside the
// list are of the key=value format
// list are of the key=value format.
func (el EnvList) Validate() error {
const op errors.Op = "EnvList.Validate"
for _, env := range el {
@@ -123,7 +123,7 @@ func (el EnvList) Validate() error {
}
// Load loads the config from a file.
// If file is not present returns default config
// If file is not present returns default config.
func Load(configFile string) (*Config, error) {
// User explicitly specified a config file
if configFile != "" {
@@ -206,7 +206,7 @@ func defaultConfig() *Config {
}
// BasicAuth returns BasicAuthUser and BasicAuthPassword
// and ok if neither of them are empty
// and ok if neither of them are empty.
func (c *Config) BasicAuth() (user, pass string, ok bool) {
user = c.BasicAuthUser
pass = c.BasicAuthPass
@@ -215,7 +215,7 @@ func (c *Config) BasicAuth() (user, pass string, ok bool) {
}
// TLSCertFiles returns certificate and key files and an error if
// both files doesn't exist and have approperiate file permissions
// both files doesn't exist and have approperiate file permissions.
func (c *Config) TLSCertFiles() (cert, key string, err error) {
if c.TLSCertFile == "" && c.TLSKeyFile == "" {
return "", "", nil
@@ -223,29 +223,28 @@ func (c *Config) TLSCertFiles() (cert, key string, err error) {
certFile, err := os.Stat(c.TLSCertFile)
if err != nil {
return "", "", fmt.Errorf("Could not access TLSCertFile: %v", err)
return "", "", fmt.Errorf("could not access TLSCertFile: %w", err)
}
keyFile, err := os.Stat(c.TLSKeyFile)
if err != nil {
return "", "", fmt.Errorf("Could not access TLSKeyFile: %v", err)
return "", "", fmt.Errorf("could not access TLSKeyFile: %w", err)
}
if keyFile.Mode()&077 != 0 && runtime.GOOS != "windows" {
if keyFile.Mode()&0o077 != 0 && runtime.GOOS != "windows" {
return "", "", fmt.Errorf("TLSKeyFile should not be accessible by others")
}
return certFile.Name(), keyFile.Name(), nil
}
// FilterOff returns true if the FilterFile is empty
// FilterOff returns true if the FilterFile is empty.
func (c *Config) FilterOff() bool {
return c.FilterFile == ""
}
// ParseConfigFile parses the given file into an athens config struct
// ParseConfigFile parses the given file into an athens config struct.
func ParseConfigFile(configFile string) (*Config, error) {
var config Config
// attempt to read the given config file
if _, err := toml.DecodeFile(configFile, &config); err != nil {
@@ -271,7 +270,7 @@ func ParseConfigFile(configFile string) (*Config, error) {
return &config, nil
}
// envOverride uses Environment variables to override unspecified properties
// envOverride uses Environment variables to override unspecified properties.
func envOverride(config *Config) error {
const defaultPort = ":3000"
err := envconfig.Process("athens", config)
@@ -355,16 +354,16 @@ func validateIndex(validate *validator.Validate, indexType string, config *Index
func GetConf(path string) (*Config, error) {
absPath, err := filepath.Abs(path)
if err != nil {
return nil, fmt.Errorf("Unable to construct absolute path to test config file")
return nil, fmt.Errorf("unable to construct absolute path to test config file")
}
conf, err := ParseConfigFile(absPath)
if err != nil {
return nil, fmt.Errorf("Unable to parse test config file: %s", err.Error())
return nil, fmt.Errorf("unable to parse test config file: %w", err)
}
return conf, nil
}
// checkFilePerms given a list of files
// checkFilePerms given a list of files.
func checkFilePerms(files ...string) error {
const op = "config.checkFilePerms"
@@ -384,10 +383,9 @@ func checkFilePerms(files ...string) error {
// Assume unix based system (MacOS and Linux)
// the bit mask is calculated using the umask command which tells which permissions
// should not be allowed for a particular user, group or world
if fInfo.Mode()&0077 != 0 && runtime.GOOS != "windows" {
if fInfo.Mode()&0o077 != 0 && runtime.GOOS != "windows" {
return errors.E(op, f+" should have at most rwx,-, - (bit mask 077) as permission")
}
}
return nil
+1 -1
View File
@@ -1,6 +1,6 @@
package config
// DiskConfig specifies the properties required to use Disk as the storage backend
// DiskConfig specifies the properties required to use Disk as the storage backend.
type DiskConfig struct {
RootPath string `validate:"required" envconfig:"ATHENS_DISK_STORAGE_ROOT"`
}
+1 -1
View File
@@ -1,6 +1,6 @@
package config
// External specifies configuration for an external http storage
// External specifies configuration for an external http storage.
type External struct {
URL string `validate:"required" envconfig:"ATHENS_EXTERNAL_STORAGE_URL"`
}
+1 -1
View File
@@ -1,6 +1,6 @@
package config
// GCPConfig specifies the properties required to use GCP as the storage backend
// GCPConfig specifies the properties required to use GCP as the storage backend.
type GCPConfig struct {
ProjectID string `envconfig:"GOOGLE_CLOUD_PROJECT"`
Bucket string `validate:"required" envconfig:"ATHENS_STORAGE_GCP_BUCKET"`
+1 -1
View File
@@ -1,6 +1,6 @@
package config
// Index is the config for various index storage backends
// Index is the config for various index storage backends.
type Index struct {
MySQL *MySQL
Postgres *Postgres
+1 -1
View File
@@ -1,7 +1,7 @@
package config
// MinioConfig specifies the properties required to use Minio or DigitalOcean Spaces
// as the storage backend
// as the storage backend.
type MinioConfig struct {
Endpoint string `validate:"required" envconfig:"ATHENS_MINIO_ENDPOINT"`
Key string `validate:"required" envconfig:"ATHENS_MINIO_ACCESS_KEY_ID"`
+4 -4
View File
@@ -7,20 +7,20 @@ import (
)
// PackageVersionedName return package full name used in storage.
// E.g athens/@v/v1.0.mod
// E.g athens/@v/v1.0.mod.
func PackageVersionedName(module, version, ext string) string {
return fmt.Sprintf("%s/@v/%s.%s", module, version, ext)
}
// FmtModVer is a helper function that can take
// pkg/a/b and v2.3.1 and returns pkg/a/b@v2.3.1
// pkg/a/b and v2.3.1 and returns pkg/a/b@v2.3.1.
func FmtModVer(mod, ver string) string {
return fmt.Sprintf("%s@%s", mod, ver)
}
// ModuleVersionFromPath returns module and version from a
// storage path
// E.g athens/@v/v1.0.info -> athens and v.1.0
// storage path.
// E.g athens/@v/v1.0.info -> athens and v.1.0.
func ModuleVersionFromPath(path string) (string, string) {
segments := strings.Split(path, "/@v/")
if len(segments) != 2 {
+1 -1
View File
@@ -1,6 +1,6 @@
package config
// MongoConfig specifies the properties required to use MongoDB as the storage backend
// MongoConfig specifies the properties required to use MongoDB as the storage backend.
type MongoConfig struct {
URL string `validate:"required" envconfig:"ATHENS_MONGO_STORAGE_URL"`
DefaultDBName string `envconfig:"ATHENS_MONGO_DEFAULT_DATABASE" default:"athens"`
+1 -1
View File
@@ -1,6 +1,6 @@
package config
// MySQL config
// MySQL config.
type MySQL struct {
Protocol string `validate:"required" envconfig:"ATHENS_INDEX_MYSQL_PROTOCOL"`
Host string `validate:"required" envconfig:"ATHENS_INDEX_MYSQL_HOST"`
+1 -1
View File
@@ -1,6 +1,6 @@
package config
// Postgres config
// Postgres config.
type Postgres struct {
Host string `validate:"required" envconfig:"ATHENS_INDEX_POSTGRES_HOST"`
Port int `validate:"required" envconfig:"ATHENS_INDEX_POSTGRES_PORT"`
+1 -1
View File
@@ -1,6 +1,6 @@
package config
// S3Config specifies the properties required to use S3 as the storage backend
// S3Config specifies the properties required to use S3 as the storage backend.
type S3Config struct {
Region string `validate:"required" envconfig:"AWS_REGION"`
Key string `envconfig:"AWS_ACCESS_KEY_ID"`
+3 -1
View File
@@ -25,7 +25,7 @@ type Redis struct {
}
// RedisSentinel is the configuration for using redis with sentinel
// for SingleFlight
// for SingleFlight.
type RedisSentinel struct {
Endpoints []string `envconfig:"ATHENS_REDIS_SENTINEL_ENDPOINTS"`
MasterName string `envconfig:"ATHENS_REDIS_SENTINEL_MASTER_NAME"`
@@ -33,12 +33,14 @@ type RedisSentinel struct {
LockConfig *RedisLockConfig
}
// RedisLockConfig is the configuration for redis locking.
type RedisLockConfig struct {
Timeout int `envconfig:"ATHENS_REDIS_LOCK_TIMEOUT"`
TTL int `envconfig:"ATHENS_REDIS_LOCK_TTL"`
MaxRetries int `envconfig:"ATHENS_REDIS_LOCK_MAX_RETRIES"`
}
// DefaultRedisLockConfig returns the default redis locking configuration.
func DefaultRedisLockConfig() *RedisLockConfig {
return &RedisLockConfig{
TTL: 900,
+1 -1
View File
@@ -1,6 +1,6 @@
package config
// Storage provides configs for various storage backends
// Storage provides configs for various storage backends.
type Storage struct {
Disk *DiskConfig
GCP *GCPConfig
+3 -3
View File
@@ -2,17 +2,17 @@ package config
import "time"
// TimeoutConf is a common struct for anything with a timeout
// TimeoutConf is a common struct for anything with a timeout.
type TimeoutConf struct {
Timeout int `validate:"required"`
}
// TimeoutDuration returns the timeout as time.duration
// TimeoutDuration returns the timeout as time.duration.
func (t *TimeoutConf) TimeoutDuration() time.Duration {
return GetTimeoutDuration(t.Timeout)
}
// GetTimeoutDuration returns the timeout as time.duration
// GetTimeoutDuration returns the timeout as time.duration.
func GetTimeoutDuration(timeout int) time.Duration {
return time.Second * time.Duration(timeout)
}
+1 -1
View File
@@ -7,7 +7,7 @@ import (
"github.com/gomods/athens/pkg/paths"
)
func getModuleParams(r *http.Request, op errors.Op) (mod string, ver string, err error) {
func getModuleParams(r *http.Request, op errors.Op) (mod, ver string, err error) {
params, err := paths.GetAllParams(r)
if err != nil {
return "", "", errors.E(op, err, errors.KindBadRequest)
+2 -3
View File
@@ -15,8 +15,7 @@ import (
// a ready-to-go http handler that serves up cmd/go's download protocol.
type ProtocolHandler func(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler
// HandlerOpts are the generic options
// for a ProtocolHandler
// HandlerOpts are the generic options for a ProtocolHandler.
type HandlerOpts struct {
Protocol Protocol
Logger *log.Logger
@@ -26,7 +25,7 @@ type HandlerOpts struct {
// LogEntryHandler pulls a log entry from the request context. Thanks to the
// LogEntryMiddleware, we should have a log entry stored in the context for each
// request with request-specific fields. This will grab the entry and pass it to
// the protocol handlers
// the protocol handlers.
func LogEntryHandler(ph ProtocolHandler, opts *HandlerOpts) http.Handler {
f := func(w http.ResponseWriter, r *http.Request) {
ent := log.EntryFromContext(r.Context())
+1 -1
View File
@@ -13,7 +13,7 @@ import (
// PathLatest URL.
const PathLatest = "/{module:.+}/@latest"
// LatestHandler implements GET baseURL/module/@latest
// LatestHandler implements GET baseURL/module/@latest.
func LatestHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler {
const op errors.Op = "download.LatestHandler"
f := func(w http.ResponseWriter, r *http.Request) {
+1 -1
View File
@@ -14,7 +14,7 @@ import (
// PathList URL.
const PathList = "/{module:.+}/@v/list"
// ListHandler implements GET baseURL/module/@v/list
// ListHandler implements GET baseURL/module/@v/list.
func ListHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler {
const op errors.Op = "download.ListHandler"
f := func(w http.ResponseWriter, r *http.Request) {
+3 -2
View File
@@ -4,6 +4,7 @@ import (
"encoding/base64"
"fmt"
"os"
"path/filepath"
"strings"
"github.com/gomods/athens/pkg/errors"
@@ -16,7 +17,7 @@ import (
// when a module is not found in storage.
type Mode string
// DownloadMode constants. For more information see config.dev.toml
// DownloadMode constants. For more information see config.dev.toml.
const (
Sync Mode = "sync"
Async Mode = "async"
@@ -59,7 +60,7 @@ func NewFile(m Mode, downloadURL string) (*DownloadFile, error) {
if strings.HasPrefix(string(m), "file:") {
filePath := string(m[5:])
bts, err := os.ReadFile(filePath)
bts, err := os.ReadFile(filepath.Clean(filePath))
if err != nil {
return nil, err
}
+6 -6
View File
@@ -48,7 +48,7 @@ type Opts struct {
NetworkMode string
}
// NetworkMode constants
// NetworkMode constants.
const (
Strict = "strict"
Offline = "offline"
@@ -263,12 +263,12 @@ func (p *protocol) processDownload(ctx context.Context, mod, ver string, f func(
}
return f(newVer)
case mode.Async:
go p.stasher.Stash(ctx, mod, ver)
go func() { _, _ = p.stasher.Stash(ctx, mod, ver) }()
return errors.E(op, "async: module not found", errors.KindNotFound)
case mode.Redirect:
return errors.E(op, "redirect", errors.KindRedirect)
case mode.AsyncRedirect:
go p.stasher.Stash(ctx, mod, ver)
go func() { _, _ = p.stasher.Stash(ctx, mod, ver) }()
return errors.E(op, "async_redirect: module not found", errors.KindRedirect)
case mode.None:
return errors.E(op, "none", errors.KindNotFound)
@@ -276,7 +276,7 @@ func (p *protocol) processDownload(ctx context.Context, mod, ver string, f func(
return nil
}
// union concatenates two version lists and removes duplicates
// union concatenates two version lists and removes duplicates.
func union(list1, list2 []string) []string {
if list1 == nil {
list1 = []string{}
@@ -284,10 +284,10 @@ func union(list1, list2 []string) []string {
if list2 == nil {
list2 = []string{}
}
list := append(list1, list2...)
list1 = append(list1, list2...)
unique := []string{}
m := make(map[string]struct{})
for _, v := range list {
for _, v := range list1 {
if _, ok := m[v]; !ok {
unique = append(unique, v)
m[v] = struct{}{}
+2 -2
View File
@@ -11,7 +11,7 @@ import (
// PathVersionInfo URL.
const PathVersionInfo = "/{module:.+}/@v/{version}.info"
// InfoHandler implements GET baseURL/module/@v/version.info
// InfoHandler implements GET baseURL/module/@v/version.info.
func InfoHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler {
const op errors.Op = "download.InfoHandler"
f := func(w http.ResponseWriter, r *http.Request) {
@@ -38,7 +38,7 @@ func InfoHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handle
w.WriteHeader(errors.Kind(err))
}
w.Write(info)
_, _ = w.Write(info)
}
return http.HandlerFunc(f)
}
+2 -2
View File
@@ -11,7 +11,7 @@ import (
// PathVersionModule URL.
const PathVersionModule = "/{module:.+}/@v/{version}.mod"
// ModuleHandler implements GET baseURL/module/@v/version.mod
// ModuleHandler implements GET baseURL/module/@v/version.mod.
func ModuleHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler {
const op errors.Op = "download.VersionModuleHandler"
f := func(w http.ResponseWriter, r *http.Request) {
@@ -42,7 +42,7 @@ func ModuleHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Hand
return
}
w.Write(modBts)
_, _ = w.Write(modBts)
}
return http.HandlerFunc(f)
}
+2 -2
View File
@@ -13,7 +13,7 @@ import (
// PathVersionZip URL.
const PathVersionZip = "/{module:.+}/@v/{version}.zip"
// ZipHandler implements GET baseURL/module/@v/version.zip
// ZipHandler implements GET baseURL/module/@v/version.zip.
func ZipHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler {
const op errors.Op = "download.ZipHandler"
f := func(w http.ResponseWriter, r *http.Request) {
@@ -41,7 +41,7 @@ func ZipHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler
w.WriteHeader(errors.Kind(err))
return
}
defer zip.Close()
defer func() { _ = zip.Close() }()
w.Header().Set("Content-Type", "application/zip")
size := zip.Size()
+17 -6
View File
@@ -9,7 +9,7 @@ import (
"github.com/sirupsen/logrus"
)
// Kind enums
// Kind enums.
const (
KindNotFound = http.StatusNotFound
KindBadRequest = http.StatusBadRequest
@@ -55,6 +55,16 @@ func Is(err error, kind int) bool {
return Kind(err) == kind
}
// IsErr is a convenience wrapper around the std library errors.Is.
func IsErr(err, target error) bool {
return errors.Is(err, target)
}
// AsErr is a convenience wrapper around the std library errors.As.
func AsErr(err error, target any) bool {
return errors.As(err, target)
}
// Op describes any independent function or
// method in Athens. A series of operations
// forms a more readable stack trace.
@@ -69,7 +79,7 @@ func (o Op) String() string {
// a module from a regular error string or version.
type M string
// V represents a module version in an error
// V represents a module version in an error.
type V string
// E is a helper function to construct an Error type
@@ -114,8 +124,8 @@ func E(op Op, args ...interface{}) error {
// if none exists, then the level is Error because
// it is an unexpected.
func Severity(err error) logrus.Level {
e, ok := err.(Error)
if !ok {
var e Error
if !errors.As(err, &e) {
return logrus.ErrorLevel
}
@@ -145,8 +155,8 @@ func Expect(err error, kinds ...int) logrus.Level {
// Kind recursively searches for the
// first error kind it finds.
func Kind(err error) int {
e, ok := err.(Error)
if !ok {
var e Error
if !errors.As(err, &e) {
return KindUnexpected
}
@@ -173,6 +183,7 @@ func KindText(err error) string {
func Ops(err Error) []Op {
ops := []Op{err.Op}
for {
//nolint:errorlint // We iterate the errors anyway.
embeddedErr, ok := err.Err.(Error)
if !ok {
break
+1 -1
View File
@@ -1,6 +1,6 @@
package errors
// IsNotFoundErr helper function for KindNotFound
// IsNotFoundErr helper function for KindNotFound.
func IsNotFoundErr(err error) bool {
return Kind(err) == KindNotFound
}
+1 -1
View File
@@ -21,7 +21,7 @@ func RunTests(t *testing.T, indexer index.Indexer, clearIndex func() error) {
t.Fatal(err)
}
var tests = []struct {
tests := []struct {
name string
desc string
limit int
+1 -1
View File
@@ -13,7 +13,7 @@ type Line struct {
}
// Indexer is an interface that can process new module@versions
// and also retrieve 'limit' module@versions that were indexed after 'since'
// and also retrieve 'limit' module@versions that were indexed after 'since'.
type Indexer interface {
// Index stores the module@version into the index backend.
// Implementer must create the Timestamp at the time and set it
+1 -1
View File
@@ -10,7 +10,7 @@ import (
"github.com/gomods/athens/pkg/index"
)
// New returns a new in-memory indexer
// New returns a new in-memory indexer.
func New() index.Indexer {
return &indexer{}
}
+6 -5
View File
@@ -83,8 +83,8 @@ func (i *indexer) Lines(ctx context.Context, since time.Time, limit int) ([]*ind
if err != nil {
return nil, errors.E(op, err)
}
defer rows.Close()
lines := []*index.Line{}
defer func() { _ = rows.Close() }()
var lines []*index.Line
for rows.Next() {
var line index.Line
err = rows.Scan(&line.Path, &line.Version, &line.Timestamp)
@@ -108,13 +108,14 @@ func getMySQLSource(cfg *config.MySQL) string {
}
func getKind(err error) int {
mysqlErr, ok := err.(*mysql.MySQLError)
if !ok {
mysqlErr := &mysql.MySQLError{}
if !errors.AsErr(err, &mysqlErr) {
return errors.KindUnexpected
}
switch mysqlErr.Number {
case 1062:
return errors.KindAlreadyExists
default:
return errors.KindUnexpected
}
return errors.KindUnexpected
}
+2 -1
View File
@@ -7,7 +7,7 @@ import (
"github.com/gomods/athens/pkg/index"
)
// New returns a no-op Indexer
// New returns a no-op Indexer.
func New() index.Indexer {
return indexer{}
}
@@ -17,6 +17,7 @@ type indexer struct{}
func (indexer) Index(ctx context.Context, mod, ver string) error {
return nil
}
func (indexer) Lines(ctx context.Context, since time.Time, limit int) ([]*index.Line, error) {
return []*index.Line{}, nil
}
+10 -11
View File
@@ -7,17 +7,15 @@ import (
"strings"
"time"
// register the driver with database/sql
"github.com/lib/pq"
"github.com/gomods/athens/pkg/config"
"github.com/gomods/athens/pkg/errors"
"github.com/gomods/athens/pkg/index"
"github.com/lib/pq"
)
// New returns a new Indexer with a PostgreSQL implementation.
// It attempts to connect to the DB and create the index table
// if it doesn ot already exist.
// if it does not already exist.
func New(cfg *config.Postgres) (index.Indexer, error) {
dataSource := getPostgresSource(cfg)
db, err := sql.Open("postgres", dataSource)
@@ -42,7 +40,7 @@ var schema = [...]string{
id SERIAL PRIMARY KEY,
path VARCHAR(255) NOT NULL,
version VARCHAR(255) NOT NULL,
timestamp timestamp NOT NULL
timestamp TIMESTAMP NOT NULL
)
`,
`
@@ -82,8 +80,8 @@ func (i *indexer) Lines(ctx context.Context, since time.Time, limit int) ([]*ind
if err != nil {
return nil, errors.E(op, err)
}
defer rows.Close()
lines := []*index.Line{}
defer func() { _ = rows.Close() }()
var lines []*index.Line
for rows.Next() {
var line index.Line
err = rows.Scan(&line.Path, &line.Version, &line.Timestamp)
@@ -96,7 +94,7 @@ func (i *indexer) Lines(ctx context.Context, since time.Time, limit int) ([]*ind
}
func getPostgresSource(cfg *config.Postgres) string {
args := []string{}
args := make([]string, 0, 5+len(cfg.Params))
args = append(args, "host="+cfg.Host)
args = append(args, "port=", strconv.Itoa(cfg.Port))
args = append(args, "user=", cfg.User)
@@ -109,13 +107,14 @@ func getPostgresSource(cfg *config.Postgres) string {
}
func getKind(err error) int {
pqerr, ok := err.(*pq.Error)
if !ok {
pqerr := &pq.Error{}
if !errors.AsErr(err, &pqerr) {
return errors.KindUnexpected
}
switch pqerr.Code {
case "23505":
return errors.KindAlreadyExists
default:
return errors.KindUnexpected
}
return errors.KindUnexpected
}
+2 -2
View File
@@ -35,8 +35,8 @@ func (e *entry) WithFields(fields map[string]interface{}) Entry {
}
func (e *entry) SystemErr(err error) {
athensErr, ok := err.(errors.Error)
if !ok {
var athensErr errors.Error
if !errors.IsErr(err, athensErr) {
e.Error(err)
return
}
+1 -1
View File
@@ -41,7 +41,7 @@ func (l *Logger) WithFields(fields map[string]interface{}) Entry {
return &entry{e}
}
// NoOpLogger provides a Logger that does nothing
// NoOpLogger provides a Logger that does nothing.
func NoOpLogger() *Logger {
return &Logger{
Logger: &logrus.Logger{},
+2 -2
View File
@@ -8,14 +8,14 @@ type ctxKey string
const logEntryKey ctxKey = "log-entry-context-key"
// SetEntryInContext stores an Entry in the request context
// SetEntryInContext stores an Entry in the request context.
func SetEntryInContext(ctx context.Context, e Entry) context.Context {
return context.WithValue(ctx, logEntryKey, e)
}
// EntryFromContext returns an Entry that has been stored in the request context.
// If there is no value for the key or the type assertion fails, it returns a new
// entry from the provided logger
// entry from the provided logger.
func EntryFromContext(ctx context.Context) Entry {
e, ok := ctx.Value(logEntryKey).(Entry)
if !ok || e == nil {
+1 -1
View File
@@ -4,7 +4,7 @@ import (
"net/http"
)
// ContentType writes writes an application/json
// ContentType writes an application/json
// Content-Type header.
func ContentType(h http.Handler) http.Handler {
f := func(w http.ResponseWriter, r *http.Request) {
+1 -1
View File
@@ -10,7 +10,7 @@ import (
)
// LogEntryMiddleware builds a log.Entry, setting the request fields
// and storing it in the context to be used throughout the stack
// and storing it in the context to be used throughout the stack.
func LogEntryMiddleware(lggr *log.Logger) mux.MiddlewareFunc {
return func(h http.Handler) http.Handler {
f := func(w http.ResponseWriter, r *http.Request) {
+1 -1
View File
@@ -9,7 +9,7 @@ import (
// WithRequestID ensures a request id is in the
// request context by either the incoming header
// or creating a new one
// or creating a new one.
func WithRequestID(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
requestID := r.Header.Get(requestid.HeaderKey)
+6 -4
View File
@@ -14,7 +14,7 @@ import (
)
// NewValidationMiddleware builds a middleware function that performs validation checks by calling
// an external webhook
// an external webhook.
func NewValidationMiddleware(client *http.Client, validatorHook string) mux.MiddlewareFunc {
const op errors.Op = "actions.NewValidationMiddleware"
return func(h http.Handler) http.Handler {
@@ -50,7 +50,7 @@ func NewValidationMiddleware(client *http.Client, validatorHook string) mux.Midd
}
}
func maybeLogValidationReason(context context.Context, message string, mod string, version string) {
func maybeLogValidationReason(context context.Context, message, mod, version string) {
if len(message) > 0 {
entry := log.EntryFromContext(context)
entry.Warnf("error validating %s@%s %s", mod, version, message)
@@ -85,6 +85,10 @@ func validate(ctx context.Context, client *http.Client, hook, mod, ver string) (
if err != nil {
return validationResponse{Valid: false}, errors.E(op, err)
}
defer func() {
_, _ = io.Copy(io.Discard, resp.Body)
_ = resp.Body.Close()
}()
switch resp.StatusCode {
case http.StatusOK:
@@ -97,8 +101,6 @@ func validate(ctx context.Context, client *http.Client, hook, mod, ver string) (
}
func validationResponseFromRequest(resp *http.Response) validationResponse {
defer resp.Body.Close()
body, _ := io.ReadAll(resp.Body)
return validationResponse{Valid: resp.StatusCode == http.StatusOK, Message: body}
}
+1 -1
View File
@@ -6,7 +6,7 @@ import (
"github.com/gomods/athens/pkg/storage"
)
// Fetcher fetches module from an upstream source
// Fetcher fetches module from an upstream source.
type Fetcher interface {
// Fetch downloads the sources from an upstream and returns the corresponding
// .info, .mod, and .zip files.
+15 -16
View File
@@ -3,6 +3,7 @@ package module
import (
"bufio"
"os"
"path/filepath"
"strconv"
"strings"
@@ -14,15 +15,15 @@ var (
versionSeparator = "."
)
// Filter is a filter of modules
// Filter is a filter of modules.
type Filter struct {
root ruleNode
filePath string
}
// NewFilter creates new filter based on rules defined in a configuration file
// WARNING: this is not concurrently safe
// Configuration consists of two operations: + for include and - for exclude
// NewFilter creates new filter based on rules defined in a configuration file.
// WARNING: this is not concurrently safe.
// Configuration consists of two operations: + for include and - for exclude:
// e.g.
// - github.com/a
// - github.com/a/b
@@ -33,7 +34,7 @@ type Filter struct {
// -
// + github.com/a
//
// will exclude all items from communication except github.com/a
// will exclude all items from communication except github.com/a.
func NewFilter(filterFilePath string) (*Filter, error) {
// Do not return an error if the file path is empty
// Do not attempt to parse it as well.
@@ -42,10 +43,9 @@ func NewFilter(filterFilePath string) (*Filter, error) {
}
return initFromConfig(filterFilePath)
}
// AddRule adds rule for specified path
// AddRule adds rule for specified path.
func (f *Filter) AddRule(path string, qualifiers []string, rule FilterRule) {
f.ensurePath(path)
@@ -70,7 +70,7 @@ func (f *Filter) AddRule(path string, qualifiers []string, rule FilterRule) {
latest.next[last] = rn
}
// Rule returns the filter rule to be applied to the given path
// Rule returns the filter rule to be applied to the given path.
func (f *Filter) Rule(path, version string) FilterRule {
segs := getPathSegments(path)
rule := f.getAssociatedRule(version, segs...)
@@ -145,7 +145,6 @@ func initFromConfig(filePath string) (*Filter, error) {
f.root = rn
for idx, line := range lines {
// Ignore newline
if len(line) == 0 {
continue
@@ -160,7 +159,7 @@ func initFromConfig(filePath string) (*Filter, error) {
}
ruleSign := strings.TrimSpace(split[0])
rule := Default
var rule FilterRule
switch ruleSign {
case "+":
rule = Include
@@ -195,11 +194,11 @@ func initFromConfig(filePath string) (*Filter, error) {
// matches checks if the given version matches the given qualifier.
// Qualifiers can be:
// - plain versions
// - v1.2.3 enables v1.2.3
// - ~1.2.3: enables 1.2.x which are at least 1.2.3
// - ^1.2.3: enables 1.x.x which are at least 1.2.3
// - <1.2.3: enables everything lower than 1.2.3 includes 1.2.2 and 0.58.9 as well
// - plain versions.
// - v1.2.3 enables v1.2.3.
// - ~1.2.3: enables 1.2.x which are at least 1.2.3.
// - ^1.2.3: enables 1.x.x which are at least 1.2.3.
// - <1.2.3: enables everything lower than 1.2.3 includes 1.2.2 and 0.58.9 as well.
func matches(version, qualifier string) bool {
if len(qualifier) < 2 || len(version) < 1 {
return false
@@ -297,7 +296,7 @@ func newRule(r FilterRule) ruleNode {
func getConfigLines(filterFile string) ([]string, error) {
const op errors.Op = "module.getConfigLines"
f, err := os.Open(filterFile)
f, err := os.Open(filepath.Clean(filterFile))
if err != nil {
return nil, errors.E(op, err)
}
+6 -6
View File
@@ -1,16 +1,16 @@
package module
// FilterRule defines behavior of module communication
// FilterRule defines behavior of module communication.
type FilterRule int
const (
// Default filter rule does not alter default/parent behavior
// Default filter rule does not alter default/parent behavior.
Default FilterRule = iota
// Include treats modules the usual way
// Used for reverting Exclude of parent path
// Include treats modules the usual way.
// Used for reverting Exclude of parent path.
Include
// Exclude filter rule excludes package and its children from communication
// Exclude filter rule excludes package and its children from communication.
Exclude
// Direct filter rule forces the package to be fetched directly from upstream proxy
// Direct filter rule forces the package to be fetched directly from upstream proxy.
Direct
)
+8 -10
View File
@@ -35,7 +35,7 @@ type goModule struct {
GoModSum string `json:"goModSum"` // checksum for go.mod (as in go.sum)
}
// NewGoGetFetcher creates fetcher which uses go get tool to fetch modules
// NewGoGetFetcher creates fetcher which uses go get tool to fetch modules.
func NewGoGetFetcher(goBinaryName, gogetDir string, envVars []string, fs afero.Fs) (Fetcher, error) {
const op errors.Op = "module.NewGoGetFetcher"
if err := validGoBinary(goBinaryName); err != nil {
@@ -64,7 +64,7 @@ func (g *goGetFetcher) Fetch(ctx context.Context, mod, ver string) (*storage.Ver
sourcePath := filepath.Join(goPathRoot, "src")
modPath := filepath.Join(sourcePath, getRepoDirName(mod, ver))
if err := g.fs.MkdirAll(modPath, os.ModeDir|os.ModePerm); err != nil {
clearFiles(g.fs, goPathRoot)
_ = clearFiles(g.fs, goPathRoot)
return nil, errors.E(op, err)
}
@@ -72,14 +72,13 @@ func (g *goGetFetcher) Fetch(ctx context.Context, mod, ver string) (*storage.Ver
ctx,
g.goBinaryName,
g.envVars,
g.fs,
goPathRoot,
modPath,
mod,
ver,
)
if err != nil {
clearFiles(g.fs, goPathRoot)
_ = clearFiles(g.fs, goPathRoot)
return nil, errors.E(op, err)
}
@@ -116,7 +115,6 @@ func downloadModule(
ctx context.Context,
goBinaryName string,
envVars []string,
fs afero.Fs,
gopath,
repoRoot,
module,
@@ -137,7 +135,7 @@ func downloadModule(
err := cmd.Run()
if err != nil {
err = fmt.Errorf("%v: %s", err, stderr)
err = fmt.Errorf("%w: %s", err, stderr)
var m goModule
if jsonErr := json.NewDecoder(stdout).Decode(&m); jsonErr != nil {
return goModule{}, errors.E(op, err)
@@ -165,17 +163,17 @@ func isLimitHit(o string) bool {
}
// getRepoDirName takes a raw repository URI and a version and creates a directory name that the
// repository contents can be put into
// repository contents can be put into.
func getRepoDirName(repoURI, version string) string {
escapedURI := strings.Replace(repoURI, "/", "-", -1)
escapedURI := strings.ReplaceAll(repoURI, "/", "-")
return fmt.Sprintf("%s-%s", escapedURI, version)
}
func validGoBinary(name string) error {
const op errors.Op = "module.validGoBinary"
err := exec.Command(name).Run()
_, ok := err.(*exec.ExitError)
if err != nil && !ok {
eErr := &exec.ExitError{}
if err != nil && !errors.AsErr(err, &eErr) {
return errors.E(op, err)
}
return nil
+5 -5
View File
@@ -28,7 +28,7 @@ type vcsLister struct {
fs afero.Fs
}
// NewVCSLister creates an UpstreamLister which uses VCS to fetch a list of available versions
// NewVCSLister creates an UpstreamLister which uses VCS to fetch a list of available versions.
func NewVCSLister(goBinPath string, env []string, fs afero.Fs) UpstreamLister {
return &vcsLister{
goBinPath: goBinPath,
@@ -39,13 +39,13 @@ func NewVCSLister(goBinPath string, env []string, fs afero.Fs) UpstreamLister {
func (l *vcsLister) List(ctx context.Context, module string) (*storage.RevInfo, []string, error) {
const op errors.Op = "vcsLister.List"
ctx, span := observ.StartSpan(ctx, op.String())
_, span := observ.StartSpan(ctx, op.String())
defer span.End()
tmpDir, err := afero.TempDir(l.fs, "", "go-list")
if err != nil {
return nil, nil, errors.E(op, err)
}
defer l.fs.RemoveAll(tmpDir)
defer func() { _ = l.fs.RemoveAll(tmpDir) }()
cmd := exec.Command(
l.goBinPath,
@@ -62,12 +62,12 @@ func (l *vcsLister) List(ctx context.Context, module string) (*storage.RevInfo,
if err != nil {
return nil, nil, errors.E(op, err)
}
defer clearFiles(l.fs, gopath)
defer func() { _ = clearFiles(l.fs, gopath) }()
cmd.Env = prepareEnv(gopath, l.env)
err = cmd.Run()
if err != nil {
err = fmt.Errorf("%v: %s", err, stderr)
err = fmt.Errorf("%w: %s", err, stderr)
// as of now, we can't recognize between a true NotFound
// and an unexpected error, so we choose the more
// hopeful path of NotFound. This way the Go command
+1 -1
View File
@@ -9,7 +9,7 @@ import (
// prepareEnv will return all the appropriate
// environment variables for a Go Command to run
// successfully (such as GOPATH, GOCACHE, PATH etc)
// successfully (such as GOPATH, GOCACHE, PATH etc).
func prepareEnv(gopath string, envVars []string) []string {
gopathEnv := fmt.Sprintf("GOPATH=%s", gopath)
cacheEnv := fmt.Sprintf("GOCACHE=%s", filepath.Join(gopath, "cache"))
+6 -6
View File
@@ -14,10 +14,10 @@ type zipReadCloser struct {
goPath string
}
// Close closes the zip file handle and clears up disk space used by the underlying disk ref
// It is the caller's responsibility to call this method to free up utilized disk space
// Close closes the zip file handle and clears up disk space used by the underlying disk ref.
// It is the caller's responsibility to call this method to free up utilized disk space.
func (rc *zipReadCloser) Close() error {
rc.zip.Close()
_ = rc.zip.Close()
return clearFiles(rc.fs, rc.goPath)
}
@@ -25,8 +25,8 @@ func (rc *zipReadCloser) Read(p []byte) (n int, err error) {
return rc.zip.Read(p)
}
// clearFiles deletes all data from the given fs at path root
// This function must be called when zip is closed to cleanup the entire GOPATH created by the diskref
// clearFiles deletes all data from the given fs at path root.
// This function must be called when zip is closed to cleanup the entire GOPATH created by the diskref.
func clearFiles(fs afero.Fs, root string) error {
const op errors.Op = "module.ClearFiles"
// This is required because vgo ensures dependencies are read-only
@@ -36,7 +36,7 @@ func clearFiles(fs afero.Fs, root string) error {
if err != nil {
return err
}
return fs.Chmod(path, 0770)
return fs.Chmod(path, 0o770)
}
err := afero.Walk(fs, root, walkFn)
if err != nil {
+18 -18
View File
@@ -13,16 +13,16 @@ import (
// RegisterExporter determines the type of TraceExporter service for exporting traces from opencensus
// User can choose from multiple tracing services (datadog, jaegar)
// RegisterExporter returns the 'Flush' function for that particular tracing service
func RegisterExporter(traceExporter, URL, service, ENV string) (func(), error) {
// RegisterExporter returns the 'Flush' function for that particular tracing service.
func RegisterExporter(traceExporter, url, service, env string) (func(), error) {
const op errors.Op = "observ.RegisterExporter"
switch traceExporter {
case "jaeger":
return registerJaegerExporter(URL, service, ENV)
return registerJaegerExporter(url, service, env)
case "datadog":
return registerDatadogExporter(URL, service, ENV)
return registerDatadogExporter(url, service, env)
case "stackdriver":
return registerStackdriverExporter(URL, ENV)
return registerStackdriverExporter(url, env)
case "":
return nil, errors.E(op, "Exporter not specified. Traces won't be exported")
default:
@@ -32,14 +32,14 @@ func RegisterExporter(traceExporter, URL, service, ENV string) (func(), error) {
// registerJaegerExporter creates a jaeger exporter for exporting traces to opencensus.
// Currently uses the 'TraceExporter' variable in the config file.
// It should in the future have a nice sampling rate defined
func registerJaegerExporter(URL, service, ENV string) (func(), error) {
// It should in the future have a nice sampling rate defined.
func registerJaegerExporter(url, service, env string) (func(), error) {
const op errors.Op = "observ.registerJaegarExporter"
if URL == "" {
if url == "" {
return nil, errors.E(op, "Exporter URL is empty. Traces won't be exported")
}
ex, err := jaeger.NewExporter(jaeger.Options{
Endpoint: URL,
Endpoint: url,
Process: jaeger.Process{
ServiceName: service,
Tags: []jaeger.Tag{
@@ -53,41 +53,41 @@ func registerJaegerExporter(URL, service, ENV string) (func(), error) {
if err != nil {
return nil, errors.E(op, err)
}
traceRegisterExporter(ex, ENV)
traceRegisterExporter(ex, env)
return ex.Flush, nil
}
func traceRegisterExporter(exporter trace.Exporter, ENV string) {
func traceRegisterExporter(exporter trace.Exporter, env string) {
trace.RegisterExporter(exporter)
if ENV == "development" {
if env == "development" {
trace.ApplyConfig(trace.Config{DefaultSampler: trace.AlwaysSample()})
}
}
// registerDatadogTracerExporter creates a datadog exporter.
// Currently uses the 'TraceExporter' variable in the config file.
func registerDatadogExporter(URL, service, ENV string) (func(), error) {
func registerDatadogExporter(url, service, env string) (func(), error) {
ex := datadog.NewExporter(
datadog.Options{
TraceAddr: URL,
TraceAddr: url,
Service: service,
})
traceRegisterExporter(ex, ENV)
traceRegisterExporter(ex, env)
return ex.Stop, nil
}
func registerStackdriverExporter(projectID, ENV string) (func(), error) {
func registerStackdriverExporter(projectID, env string) (func(), error) {
const op errors.Op = "observ.registerStackdriverExporter"
ex, err := stackdriver.NewExporter(stackdriver.Options{ProjectID: projectID})
if err != nil {
return nil, errors.E(op, err)
}
traceRegisterExporter(ex, ENV)
traceRegisterExporter(ex, env)
return ex.Flush, nil
}
// StartSpan takes in a Context Interface and opName and starts a span. It returns the new attached ObserverContext
// and span
// and span.
func StartSpan(ctx context.Context, op string) (context.Context, *trace.Span) {
return trace.StartSpan(ctx, op)
}
+4 -4
View File
@@ -15,10 +15,10 @@ import (
)
// RegisterStatsExporter determines the type of StatsExporter service for exporting stats from Opencensus
// Currently it supports: prometheus
// Currently it supports: prometheus.
func RegisterStatsExporter(r *mux.Router, statsExporter, service string) (func(), error) {
const op errors.Op = "observ.RegisterStatsExporter"
var stop = func() {}
stop := func() {}
var err error
switch statsExporter {
case "prometheus":
@@ -38,7 +38,7 @@ func RegisterStatsExporter(r *mux.Router, statsExporter, service string) (func()
default:
return nil, errors.E(op, fmt.Sprintf("StatsExporter %s not supported. Please open PR or an issue at github.com/gomods/athens", statsExporter))
}
if err := registerViews(); err != nil {
if err = registerViews(); err != nil {
return nil, errors.E(op, err)
}
@@ -89,7 +89,7 @@ func registerStatsStackDriverExporter(projectID string) (func(), error) {
return sd.Flush, nil
}
// registerViews register stats which should be collected in Athens
// registerViews register stats which should be collected in Athens.
func registerViews() error {
const op errors.Op = "observ.registerViews"
if err := view.Register(
+2 -1
View File
@@ -19,8 +19,9 @@ func DecodePath(encoding string) (path string, err error) {
return path, nil
}
// Ripped from cmd/go
// Ripped from cmd/go.
func decodeString(encoding string) (string, bool) {
//nolint:prealloc
var buf []byte
bang := false
+5 -5
View File
@@ -9,7 +9,7 @@ import (
"github.com/gorilla/mux"
)
// GetModule gets the module from the path of a ?go-get=1 request
// GetModule gets the module from the path of a ?go-get=1 request.
func GetModule(r *http.Request) (string, error) {
const op errors.Op = "paths.GetModule"
module := mux.Vars(r)["module"]
@@ -19,7 +19,7 @@ func GetModule(r *http.Request) (string, error) {
return DecodePath(module)
}
// GetVersion gets the version from the path of a ?go-get=1 request
// GetVersion gets the version from the path of a ?go-get=1 request.
func GetVersion(r *http.Request) (string, error) {
const op errors.Op = "paths.GetVersion"
@@ -31,13 +31,13 @@ func GetVersion(r *http.Request) (string, error) {
}
// AllPathParams holds the module and version in the path of a ?go-get=1
// request
// request.
type AllPathParams struct {
Module string `json:"module"`
Version string `json:"version"`
}
// GetAllParams fetches the path params from r and returns them
// GetAllParams fetches the path params from r and returns them.
func GetAllParams(r *http.Request) (*AllPathParams, error) {
const op errors.Op = "paths.GetAllParams"
mod, err := GetModule(r)
@@ -54,7 +54,7 @@ func GetAllParams(r *http.Request) (*AllPathParams, error) {
}
// MatchesPattern reports whether the path prefix of target matches
// pattern (as defined by path.Match)
// pattern (as defined by path.Match).
//
// This tries to keep the same behavior as GOPRIVATE/GONOPROXY/GONOSUMDB,
// and is adopted from:
+3 -3
View File
@@ -3,18 +3,18 @@ package requestid
import "context"
// HeaderKey is the header key that athens uses
// to pass request ids into logs and outbound requests
// to pass request ids into logs and outbound requests.
const HeaderKey = "Athens-Request-ID"
type key struct{}
// SetInContext sets the given requestID into the context
// SetInContext sets the given requestID into the context.
func SetInContext(ctx context.Context, id string) context.Context {
return context.WithValue(ctx, key{}, id)
}
// FromContext returns a requestID from the context or an empty
// string if not found
// string if not found.
func FromContext(ctx context.Context) string {
id, _ := ctx.Value(key{}).(string)
return id
+3 -3
View File
@@ -19,7 +19,7 @@ import (
// what was requested, this is helpful if what was requested
// was a descriptive version such as a branch name or a full commit sha.
type Stasher interface {
Stash(ctx context.Context, mod string, ver string) (string, error)
Stash(ctx context.Context, mod, ver string) (string, error)
}
// Wrapper helps extend the main stasher's functionality with addons.
@@ -46,7 +46,7 @@ type stasher struct {
func (s *stasher) Stash(ctx context.Context, mod, ver string) (string, error) {
const op errors.Op = "stasher.Stash"
_, span := observ.StartSpan(ctx, op.String())
ctx, span := observ.StartSpan(ctx, op.String())
defer span.End()
log.EntryFromContext(ctx).Debugf("saving %s@%s to storage...", mod, ver)
@@ -58,7 +58,7 @@ func (s *stasher) Stash(ctx context.Context, mod, ver string) (string, error) {
if err != nil {
return "", errors.E(op, err)
}
defer v.Zip.Close()
defer func() { _ = v.Zip.Close() }()
if v.Semver != ver {
exists, err := s.checker.Exists(ctx, mod, v.Semver)
if err != nil {
+4 -3
View File
@@ -132,8 +132,8 @@ func (s *azblobLock) acquireLease(ctx context.Context, blobURL azblob.BlockBlobU
_, err := blobURL.Upload(tctx, bytes.NewReader([]byte{1}), azblob.BlobHTTPHeaders{}, nil, azblob.BlobAccessConditions{})
if err != nil {
// if the blob is already leased we will get http.StatusPreconditionFailed while writing to that blob
stgErr, ok := err.(azblob.StorageError)
if !ok || stgErr.Response().StatusCode != http.StatusPreconditionFailed {
var stgErr azblob.StorageError
if !errors.AsErr(err, &stgErr) || stgErr.Response().StatusCode != http.StatusPreconditionFailed {
return "", errors.E(op, err)
}
}
@@ -147,7 +147,8 @@ func (s *azblobLock) acquireLease(ctx context.Context, blobURL azblob.BlockBlobU
res, err := blobURL.AcquireLease(tctx, leaseID.String(), 15, azblob.ModifiedAccessConditions{})
if err != nil {
// if the blob is already leased we will get http.StatusConflict - wait and try again
if stgErr, ok := err.(azblob.StorageError); ok && stgErr.Response().StatusCode == http.StatusConflict {
var stgErr azblob.StorageError
if ok := errors.AsErr(err, &stgErr); ok && stgErr.Response().StatusCode == http.StatusConflict {
select {
case <-time.After(1 * time.Second):
continue
+1 -1
View File
@@ -8,7 +8,7 @@ import (
"github.com/gomods/athens/pkg/errors"
"github.com/gomods/athens/pkg/observ"
"github.com/gomods/athens/pkg/storage"
"go.etcd.io/etcd/client/v3"
clientv3 "go.etcd.io/etcd/client/v3"
"go.etcd.io/etcd/client/v3/concurrency"
"golang.org/x/sync/errgroup"
)
+2 -2
View File
@@ -13,14 +13,14 @@ import (
"github.com/gomods/athens/pkg/storage"
)
// RedisLogger mirrors github.com/go-redis/redis/v8/internal.Logging
// RedisLogger mirrors github.com/go-redis/redis/v8/internal.Logging.
type RedisLogger interface {
Printf(ctx context.Context, format string, v ...interface{})
}
// WithRedisLock returns a distributed singleflight
// using a redis cluster. If it cannot connect, it will return an error.
func WithRedisLock(l RedisLogger, endpoint string, password string, checker storage.Checker, lockConfig *config.RedisLockConfig) (Wrapper, error) {
func WithRedisLock(l RedisLogger, endpoint, password string, checker storage.Checker, lockConfig *config.RedisLockConfig) (Wrapper, error) {
redis.SetLogger(l)
const op errors.Op = "stash.WithRedisLock"
+1 -1
View File
@@ -10,7 +10,7 @@ import (
)
// WithRedisSentinelLock returns a distributed singleflight
// with a redis cluster that utilizes sentinel for quorum and failover
// with a redis cluster that utilizes sentinel for quorum and failover.
func WithRedisSentinelLock(l RedisLogger, endpoints []string, master, password string, checker storage.Checker, lockConfig *config.RedisLockConfig) (Wrapper, error) {
redis.SetLogger(l)
+9 -20
View File
@@ -15,14 +15,6 @@ import (
"github.com/gomods/athens/pkg/storage"
)
type client interface {
UploadWithContext(ctx context.Context, path, contentType string, content io.Reader) error
BlobExists(ctx context.Context, path string) (bool, error)
ReadBlob(ctx context.Context, path string) (io.ReadCloser, error)
ListBlobs(ctx context.Context, prefix string) ([]string, error)
DeleteBlob(ctx context.Context, path string) error
}
type azureBlobStoreClient struct {
containerURL *azblob.ContainerURL
}
@@ -45,13 +37,13 @@ func newBlobStoreClient(accountURL *url.URL, accountName, accountKey, containerN
}
// Storage implements (github.com/gomods/athens/pkg/storage).Saver and
// also provides a function to fetch the location of a module
// also provides a function to fetch the location of a module.
type Storage struct {
client *azureBlobStoreClient
timeout time.Duration
}
// New creates a new azure blobs storage
// New creates a new azure blobs storage.
func New(conf *config.AzureBlobConfig, timeout time.Duration) (*Storage, error) {
const op errors.Op = "azureblob.New"
u, err := url.Parse(fmt.Sprintf("https://%s.blob.core.windows.net", conf.AccountName))
@@ -65,7 +57,7 @@ func New(conf *config.AzureBlobConfig, timeout time.Duration) (*Storage, error)
return &Storage{client: cl, timeout: timeout}, nil
}
// BlobExists checks if a particular blob exists in the container
// BlobExists checks if a particular blob exists in the container.
func (c *azureBlobStoreClient) BlobExists(ctx context.Context, path string) (bool, error) {
const op errors.Op = "azureblob.BlobExists"
// TODO: Any better way of doing this ?
@@ -73,10 +65,8 @@ func (c *azureBlobStoreClient) BlobExists(ctx context.Context, path string) (boo
_, err := blobURL.GetProperties(ctx, azblob.BlobAccessConditions{})
if err != nil {
var serr azblob.StorageError
var ok bool
if serr, ok = err.(azblob.StorageError); !ok {
return false, errors.E(op, fmt.Errorf("Error in casting to azure error type %v", err))
if !errors.AsErr(err, &serr) {
return false, errors.E(op, fmt.Errorf("error in casting to azure error type %w", err))
}
if serr.Response().StatusCode == http.StatusNotFound {
return false, nil
@@ -85,10 +75,9 @@ func (c *azureBlobStoreClient) BlobExists(ctx context.Context, path string) (boo
return false, errors.E(op, err)
}
return true, nil
}
// ReadBlob returns a storage.SizeReadCloser for the contents of a blob
// ReadBlob returns a storage.SizeReadCloser for the contents of a blob.
func (c *azureBlobStoreClient) ReadBlob(ctx context.Context, path string) (storage.SizeReadCloser, error) {
const op errors.Op = "azureblob.ReadBlob"
blobURL := c.containerURL.NewBlockBlobURL(path)
@@ -101,7 +90,7 @@ func (c *azureBlobStoreClient) ReadBlob(ctx context.Context, path string) (stora
return storage.NewSizer(rc, size), nil
}
// ListBlobs will list all blobs which has the given prefix
// ListBlobs will list all blobs which has the given prefix.
func (c *azureBlobStoreClient) ListBlobs(ctx context.Context, prefix string) ([]string, error) {
const op errors.Op = "azureblob.ListBlobs"
var blobs []string
@@ -122,7 +111,7 @@ func (c *azureBlobStoreClient) ListBlobs(ctx context.Context, prefix string) ([]
return blobs, nil
}
// DeleteBlob deletes the blob with the given path
// DeleteBlob deletes the blob with the given path.
func (c *azureBlobStoreClient) DeleteBlob(ctx context.Context, path string) error {
const op errors.Op = "azureblob.DeleteBlob"
blobURL := c.containerURL.NewBlockBlobURL(path)
@@ -133,7 +122,7 @@ func (c *azureBlobStoreClient) DeleteBlob(ctx context.Context, path string) erro
return nil
}
// UploadWithContext uploads a blob to the container
// UploadWithContext uploads a blob to the container.
func (c *azureBlobStoreClient) UploadWithContext(ctx context.Context, path, contentType string, content io.Reader) error {
const op errors.Op = "azureblob.UploadWithContext"
ctx, span := observ.StartSpan(ctx, op.String())
+2 -2
View File
@@ -12,8 +12,8 @@ import (
"github.com/gomods/athens/pkg/paths"
)
// Catalog implements the (./pkg/storage).Catalog interface
// It returns a list of versions, if any, for a given module
// Catalog implements the (./pkg/storage).Catalog interface.
// It returns a list of versions, if any, for a given module.
func (s *Storage) Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error) {
const op errors.Op = "azblob.Catalog"
ctx, span := observ.StartSpan(ctx, op.String())
+2 -2
View File
@@ -9,8 +9,8 @@ import (
)
// Exists implements the (./pkg/storage).Checker interface
// returning true if the module at version exists in storage
func (s *Storage) Exists(ctx context.Context, module string, version string) (bool, error) {
// returning true if the module at version exists in storage.
func (s *Storage) Exists(ctx context.Context, module, version string) (bool, error) {
const op errors.Op = "azureblob.Exists"
ctx, span := observ.StartSpan(ctx, op.String())
defer span.End()
+1 -1
View File
@@ -11,7 +11,7 @@ import (
// Delete implements the (./pkg/storage).Deleter interface and
// removes a version of a module from storage. Returning ErrNotFound
// if the version does not exist.
func (s *Storage) Delete(ctx context.Context, module string, version string) error {
func (s *Storage) Delete(ctx context.Context, module, version string) error {
const op errors.Op = "azureblob.Delete"
ctx, span := observ.StartSpan(ctx, op.String())
defer span.End()
+7 -7
View File
@@ -11,8 +11,8 @@ import (
"github.com/gomods/athens/pkg/storage"
)
// Info implements the (./pkg/storage).Getter interface
func (s *Storage) Info(ctx context.Context, module string, version string) ([]byte, error) {
// Info implements the (./pkg/storage).Getter interface.
func (s *Storage) Info(ctx context.Context, module, version string) ([]byte, error) {
const op errors.Op = "azureblob.Info"
ctx, span := observ.StartSpan(ctx, op.String())
defer span.End()
@@ -43,8 +43,8 @@ func (s *Storage) Info(ctx context.Context, module string, version string) ([]by
return infoBytes, nil
}
// GoMod implements the (./pkg/storage).Getter interface
func (s *Storage) GoMod(ctx context.Context, module string, version string) ([]byte, error) {
// GoMod implements the (./pkg/storage).Getter interface.
func (s *Storage) GoMod(ctx context.Context, module, version string) ([]byte, error) {
const op errors.Op = "azureblob.GoMod"
ctx, span := observ.StartSpan(ctx, op.String())
defer span.End()
@@ -64,7 +64,7 @@ func (s *Storage) GoMod(ctx context.Context, module string, version string) ([]b
modBytes, err := io.ReadAll(modReader)
if err != nil {
return nil, errors.E(op, fmt.Errorf("could not get new reader for mod file: %s", err), errors.M(module), errors.V(version))
return nil, errors.E(op, fmt.Errorf("could not get new reader for mod file: %w", err), errors.M(module), errors.V(version))
}
err = modReader.Close()
@@ -75,8 +75,8 @@ func (s *Storage) GoMod(ctx context.Context, module string, version string) ([]b
return modBytes, nil
}
// Zip implements the (./pkg/storage).Getter interface
func (s *Storage) Zip(ctx context.Context, module string, version string) (storage.SizeReadCloser, error) {
// Zip implements the (./pkg/storage).Getter interface.
func (s *Storage) Zip(ctx context.Context, module, version string) (storage.SizeReadCloser, error) {
const op errors.Op = "azureblob.Zip"
ctx, span := observ.StartSpan(ctx, op.String())
defer span.End()
+2 -2
View File
@@ -8,8 +8,8 @@ import (
"github.com/gomods/athens/pkg/observ"
)
// List implements the (./pkg/storage).Lister interface
// It returns a list of versions, if any, for a given module
// List implements the (./pkg/storage).Lister interface.
// It returns a list of versions, if any, for a given module.
func (s *Storage) List(ctx context.Context, module string) ([]string, error) {
const op errors.Op = "azureblob.List"
ctx, span := observ.StartSpan(ctx, op.String())
+1 -1
View File
@@ -1,6 +1,6 @@
package storage
// Backend is a complete storage backend (i.e. file system, database) implementation - a lister, reader and saver
// Backend is a complete storage backend (i.e. file system, database) implementation - a lister, reader and saver.
type Backend interface {
Lister
Getter
+1 -1
View File
@@ -6,7 +6,7 @@ import (
"github.com/gomods/athens/pkg/paths"
)
// Cataloger is the interface that lists all the modules and version contained in the storage
// Cataloger is the interface that lists all the modules and version contained in the storage.
type Cataloger interface {
// Catalog gets all the modules / versions.
Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error)
+3 -3
View File
@@ -6,14 +6,14 @@ import (
"github.com/gomods/athens/pkg/errors"
)
// Checker is the interface that checks if the version of the module exists
// Checker is the interface that checks if the version of the module exists.
type Checker interface {
// Exists checks whether or not module in specified version is present
// in the backing storage
// in the backing storage.
Exists(ctx context.Context, module, version string) (bool, error)
}
// WithChecker wraps the backend with a Checker implementaiton
// WithChecker wraps the backend with a Checker implementation.
func WithChecker(strg Backend) Checker {
if checker, ok := strg.(Checker); ok {
return checker
-45
View File
@@ -6,7 +6,6 @@ import (
"fmt"
"io"
"math/rand"
"sort"
"testing"
"github.com/gomods/athens/pkg/errors"
@@ -26,7 +25,6 @@ func RunTests(t *testing.T, b storage.Backend, clearBackend func() error) {
testGet(t, b)
testExists(t, b)
testShouldNotExist(t, b)
// testCatalog(t, b)
}
// testNotFound ensures that a storage Backend
@@ -209,49 +207,6 @@ func testDelete(t *testing.T, b storage.Backend) {
require.Equal(t, false, exists)
}
func testCatalog(t *testing.T, b storage.Backend) {
cs, ok := b.(storage.Cataloger)
if !ok {
t.Skip()
}
ctx := context.Background()
mock := getMockModule()
zipBts, _ := io.ReadAll(mock.Zip)
modname := "github.com/gomods/testCatalogModule"
for i := 0; i < 6; i++ {
ver := fmt.Sprintf("v1.2.%04d", i)
b.Save(ctx, modname, ver, mock.Mod, bytes.NewReader(zipBts), mock.Info)
defer b.Delete(ctx, modname, ver)
}
allres, next, err := cs.Catalog(ctx, "", 5)
require.NoError(t, err)
require.Equal(t, 5, len(allres))
res, next, err := cs.Catalog(ctx, next, 50)
allres = append(allres, res...)
require.NoError(t, err)
require.Equal(t, 1, len(res))
require.Equal(t, "", next)
sort.Slice(allres, func(i, j int) bool {
if allres[i].Module == allres[j].Module {
return allres[i].Version < allres[j].Version
}
return allres[i].Module < allres[j].Module
})
require.Equal(t, modname, allres[0].Module)
require.Equal(t, "v1.2.0000", allres[0].Version)
require.Equal(t, "v1.2.0004", allres[4].Version)
for i := 1; i < len(allres); i++ {
require.NotEqual(t, allres[i].Version, allres[i-1].Version)
}
}
func getMockModule() *storage.Version {
return &storage.Version{
Info: []byte("123"),
+1 -1
View File
@@ -2,7 +2,7 @@ package storage
import "context"
// Deleter deletes module metadata and its source from underlying storage
// Deleter deletes module metadata and its source from underlying storage.
type Deleter interface {
// Delete must return ErrNotFound if the module/version are not
// found.
+14 -14
View File
@@ -20,7 +20,7 @@ type service struct {
c *http.Client
}
// NewClient returns an external storage client
// NewClient returns an external storage client.
func NewClient(url string, c *http.Client) storage.Backend {
if c == nil {
c = &http.Client{}
@@ -93,7 +93,7 @@ func (s *service) Save(ctx context.Context, mod, ver string, modFile []byte, zip
mw := multipart.NewWriter(pw)
go func() {
err := upload(mw, modFile, info, zip)
pw.CloseWithError(err)
_ = pw.CloseWithError(err)
}()
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, pr)
if err != nil {
@@ -104,8 +104,8 @@ func (s *service) Save(ctx context.Context, mod, ver string, modFile []byte, zip
if err != nil {
return errors.E(op, err)
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusOK {
bts, _ := io.ReadAll(resp.Body)
return errors.E(op, fmt.Errorf("unexpected status code: %v - body: %s", resp.StatusCode, bts), resp.StatusCode)
}
@@ -118,35 +118,35 @@ func (s *service) Delete(ctx context.Context, mod, ver string) error {
if err != nil {
return errors.E(op, err)
}
defer body.Close()
defer func() { _ = body.Close() }()
return nil
}
func upload(mw *multipart.Writer, mod, info []byte, zip io.Reader) error {
defer mw.Close()
defer func() { _ = mw.Close() }()
infoW, err := mw.CreateFormFile("mod.info", "mod.info")
if err != nil {
return fmt.Errorf("error creating info file: %v", err)
return fmt.Errorf("error creating info file: %w", err)
}
_, err = infoW.Write(info)
if err != nil {
return fmt.Errorf("error writing info file: %v", err)
return fmt.Errorf("error writing info file: %w", err)
}
modW, err := mw.CreateFormFile("mod.mod", "mod.mod")
if err != nil {
return fmt.Errorf("error creating mod file: %v", err)
return fmt.Errorf("error creating mod file: %w", err)
}
_, err = modW.Write(mod)
if err != nil {
return fmt.Errorf("error writing mod file: %v", err)
return fmt.Errorf("error writing mod file: %w", err)
}
zipW, err := mw.CreateFormFile("mod.zip", "mod.zip")
if err != nil {
return fmt.Errorf("error creating zip file: %v", err)
return fmt.Errorf("error creating zip file: %w", err)
}
_, err = io.Copy(zipW, zip)
if err != nil {
return fmt.Errorf("error writing zip file: %v", err)
return fmt.Errorf("error writing zip file: %w", err)
}
return nil
}
@@ -174,9 +174,9 @@ func (s *service) doRequest(ctx context.Context, method, mod, ver, ext string) (
if err != nil {
return nil, 0, errors.E(op, err)
}
if resp.StatusCode != 200 {
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
resp.Body.Close()
_ = resp.Body.Close()
return nil, 0, errors.E(op, fmt.Errorf("none 200 status code: %v - body: %s", resp.StatusCode, body), resp.StatusCode)
}
var size int64
+21 -21
View File
@@ -17,7 +17,7 @@ import (
// NewServer takes a storage.Backend implementation of your
// choice, and returns a new http.Handler that Athens can
// reach out to for storage operations
// reach out to for storage operations.
func NewServer(strg storage.Backend) http.Handler {
r := mux.NewRouter()
r.HandleFunc(download.PathList, func(w http.ResponseWriter, r *http.Request) {
@@ -27,12 +27,12 @@ func NewServer(strg storage.Backend) http.Handler {
http.Error(w, err.Error(), errors.Kind(err))
return
}
fmt.Fprintf(w, "%s", strings.Join(list, "\n"))
_, _ = fmt.Fprintf(w, "%s", strings.Join(list, "\n"))
}).Methods(http.MethodGet)
r.HandleFunc(download.PathVersionInfo, func(w http.ResponseWriter, r *http.Request) {
params, err := paths.GetAllParams(r)
if err != nil {
http.Error(w, err.Error(), 400)
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
info, err := strg.Info(r.Context(), params.Module, params.Version)
@@ -40,12 +40,12 @@ func NewServer(strg storage.Backend) http.Handler {
http.Error(w, err.Error(), errors.Kind(err))
return
}
w.Write(info)
_, _ = w.Write(info)
}).Methods(http.MethodGet)
r.HandleFunc(download.PathVersionModule, func(w http.ResponseWriter, r *http.Request) {
params, err := paths.GetAllParams(r)
if err != nil {
http.Error(w, err.Error(), 400)
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
mod, err := strg.GoMod(r.Context(), params.Module, params.Version)
@@ -53,12 +53,12 @@ func NewServer(strg storage.Backend) http.Handler {
http.Error(w, err.Error(), errors.Kind(err))
return
}
w.Write(mod)
_, _ = w.Write(mod)
}).Methods(http.MethodGet)
r.HandleFunc(download.PathVersionZip, func(w http.ResponseWriter, r *http.Request) {
params, err := paths.GetAllParams(r)
if err != nil {
http.Error(w, err.Error(), 400)
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
zip, err := strg.Zip(r.Context(), params.Module, params.Version)
@@ -66,52 +66,52 @@ func NewServer(strg storage.Backend) http.Handler {
http.Error(w, err.Error(), errors.Kind(err))
return
}
defer zip.Close()
defer func() { _ = zip.Close() }()
w.Header().Set("Content-Length", strconv.FormatInt(zip.Size(), 10))
io.Copy(w, zip)
_, _ = io.Copy(w, zip)
}).Methods(http.MethodGet)
r.HandleFunc("/{module:.+}/@v/{version}.save", func(w http.ResponseWriter, r *http.Request) {
params, err := paths.GetAllParams(r)
if err != nil {
http.Error(w, err.Error(), 400)
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
err = r.ParseMultipartForm(zip.MaxZipFile + zip.MaxGoMod)
if err != nil {
http.Error(w, err.Error(), 400)
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
infoFile, _, err := r.FormFile("mod.info")
if err != nil {
http.Error(w, err.Error(), 400)
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
defer infoFile.Close()
defer func() { _ = infoFile.Close() }()
info, err := io.ReadAll(infoFile)
if err != nil {
http.Error(w, err.Error(), 400)
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
modReader, _, err := r.FormFile("mod.mod")
if err != nil {
http.Error(w, err.Error(), 400)
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
defer modReader.Close()
defer func() { _ = modReader.Close() }()
modFile, err := io.ReadAll(modReader)
if err != nil {
http.Error(w, err.Error(), 400)
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
modZ, _, err := r.FormFile("mod.zip")
if err != nil {
http.Error(w, err.Error(), 400)
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
defer modZ.Close()
defer func() { _ = modZ.Close() }()
err = strg.Save(r.Context(), params.Module, params.Version, modFile, modZ, info)
if err != nil {
http.Error(w, err.Error(), 400)
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
}).Methods(http.MethodPost)
@@ -119,7 +119,7 @@ func NewServer(strg storage.Backend) http.Handler {
r.HandleFunc("/{module:.+}/@v/{version}.delete", func(w http.ResponseWriter, r *http.Request) {
params, err := paths.GetAllParams(r)
if err != nil {
http.Error(w, err.Error(), 400)
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
err = strg.Delete(r.Context(), params.Module, params.Version)
+5 -5
View File
@@ -15,11 +15,11 @@ import (
const tokenSeparator = "|"
// Catalog implements the (./pkg/storage).Cataloger interface
// It returns a list of modules and versions contained in the storage
// Catalog implements the (./pkg/storage).Cataloger interface.
// It returns a list of modules and versions contained in the storage.
func (s *storageImpl) Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error) {
const op errors.Op = "fs.Catalog"
ctx, span := observ.StartSpan(ctx, op.String())
_, span := observ.StartSpan(ctx, op.String())
defer span.End()
fromModule, fromVersion, err := modVerFromToken(token)
@@ -41,7 +41,7 @@ func (s *storageImpl) Catalog(ctx context.Context, token string, pageSize int) (
m, version := filepath.Split(modVer)
module := filepath.Clean(m)
module = strings.Replace(module, string(os.PathSeparator), "/", -1)
module = strings.ReplaceAll(module, string(os.PathSeparator), "/")
if fromModule != "" && module < fromModule { // it is ok to land on the same module
return nil
@@ -60,7 +60,7 @@ func (s *storageImpl) Catalog(ctx context.Context, token string, pageSize int) (
}
return nil
})
if err != nil && err != io.EOF {
if err != nil && !errors.IsErr(err, io.EOF) {
return nil, "", errors.E(op, err, errors.KindUnexpected)
}
+4 -5
View File
@@ -9,14 +9,13 @@ import (
"github.com/spf13/afero"
)
func (v *storageImpl) Exists(ctx context.Context, module, version string) (bool, error) {
func (s *storageImpl) Exists(ctx context.Context, module, version string) (bool, error) {
const op errors.Op = "fs.Exists"
ctx, span := observ.StartSpan(ctx, op.String())
_, span := observ.StartSpan(ctx, op.String())
defer span.End()
versionedPath := v.versionLocation(module, version)
files, err := afero.ReadDir(v.filesystem, versionedPath)
versionedPath := s.versionLocation(module, version)
files, err := afero.ReadDir(s.filesystem, versionedPath)
if err != nil {
if os.IsNotExist(err) {
return false, nil
+4 -4
View File
@@ -8,17 +8,17 @@ import (
)
// Delete removes a specific version of a module.
func (v *storageImpl) Delete(ctx context.Context, module, version string) error {
func (s *storageImpl) Delete(ctx context.Context, module, version string) error {
const op errors.Op = "fs.Delete"
ctx, span := observ.StartSpan(ctx, op.String())
defer span.End()
versionedPath := v.versionLocation(module, version)
exists, err := v.Exists(ctx, module, version)
versionedPath := s.versionLocation(module, version)
exists, err := s.Exists(ctx, module, version)
if err != nil {
return errors.E(op, err, errors.M(module), errors.V(version))
}
if !exists {
return errors.E(op, errors.M(module), errors.V(version), errors.KindNotFound)
}
return v.filesystem.RemoveAll(versionedPath)
return s.filesystem.RemoveAll(versionedPath)
}
+3 -4
View File
@@ -21,17 +21,16 @@ func (s *storageImpl) moduleLocation(module string) string {
func (s *storageImpl) versionLocation(module, version string) string {
return filepath.Join(s.moduleLocation(module), version)
}
// NewStorage returns a new ListerSaver implementation that stores
// everything under rootDir
// If the root directory does not exist an error is returned
// everything under rootDir.
// If the root directory does not exist an error is returned.
func NewStorage(rootDir string, filesystem afero.Fs) (storage.Backend, error) {
const op errors.Op = "fs.NewStorage"
exists, err := afero.Exists(filesystem, rootDir)
if err != nil {
return nil, errors.E(op, fmt.Errorf("could not check if root directory `%s` exists: %s", rootDir, err))
return nil, errors.E(op, fmt.Errorf("could not check if root directory `%s` exists: %w", rootDir, err))
}
if !exists {
return nil, errors.E(op, fmt.Errorf("root directory `%s` does not exist", rootDir))
+12 -24
View File
@@ -11,12 +11,12 @@ import (
"github.com/spf13/afero"
)
func (v *storageImpl) Info(ctx context.Context, module, version string) ([]byte, error) {
func (s *storageImpl) Info(ctx context.Context, module, version string) ([]byte, error) {
const op errors.Op = "fs.Info"
ctx, span := observ.StartSpan(ctx, op.String())
_, span := observ.StartSpan(ctx, op.String())
defer span.End()
versionedPath := v.versionLocation(module, version)
info, err := afero.ReadFile(v.filesystem, filepath.Join(versionedPath, version+".info"))
versionedPath := s.versionLocation(module, version)
info, err := afero.ReadFile(s.filesystem, filepath.Join(versionedPath, version+".info"))
if err != nil {
return nil, errors.E(op, errors.M(module), errors.V(version), errors.KindNotFound)
}
@@ -24,12 +24,12 @@ func (v *storageImpl) Info(ctx context.Context, module, version string) ([]byte,
return info, nil
}
func (v *storageImpl) GoMod(ctx context.Context, module, version string) ([]byte, error) {
func (s *storageImpl) GoMod(ctx context.Context, module, version string) ([]byte, error) {
const op errors.Op = "fs.GoMod"
ctx, span := observ.StartSpan(ctx, op.String())
_, span := observ.StartSpan(ctx, op.String())
defer span.End()
versionedPath := v.versionLocation(module, version)
mod, err := afero.ReadFile(v.filesystem, filepath.Join(versionedPath, "go.mod"))
versionedPath := s.versionLocation(module, version)
mod, err := afero.ReadFile(s.filesystem, filepath.Join(versionedPath, "go.mod"))
if err != nil {
return nil, errors.E(op, errors.M(module), errors.V(version), errors.KindNotFound)
}
@@ -37,13 +37,13 @@ func (v *storageImpl) GoMod(ctx context.Context, module, version string) ([]byte
return mod, nil
}
func (v *storageImpl) Zip(ctx context.Context, module, version string) (storage.SizeReadCloser, error) {
func (s *storageImpl) Zip(ctx context.Context, module, version string) (storage.SizeReadCloser, error) {
const op errors.Op = "fs.Zip"
ctx, span := observ.StartSpan(ctx, op.String())
_, span := observ.StartSpan(ctx, op.String())
defer span.End()
versionedPath := v.versionLocation(module, version)
versionedPath := s.versionLocation(module, version)
src, err := v.filesystem.OpenFile(filepath.Join(versionedPath, "source.zip"), os.O_RDONLY, 0666)
src, err := s.filesystem.OpenFile(filepath.Join(versionedPath, "source.zip"), os.O_RDONLY, 0o666)
if err != nil {
return nil, errors.E(op, errors.M(module), errors.V(version), errors.KindNotFound)
}
@@ -53,15 +53,3 @@ func (v *storageImpl) Zip(ctx context.Context, module, version string) (storage.
}
return storage.NewSizer(src, fi.Size()), nil
}
func (v *storageImpl) ZipSize(ctx context.Context, module, version string) (int64, error) {
const op errors.Op = "fs.ZipFileSize"
ctx, span := observ.StartSpan(ctx, op.String())
defer span.End()
versionedPath := v.versionLocation(module, version)
fi, err := v.filesystem.Stat(filepath.Join(versionedPath))
if err != nil {
return 0, errors.E(op, err, errors.M(module), errors.V(version), errors.KindNotFound)
}
return fi.Size(), nil
}
+4 -4
View File
@@ -11,12 +11,12 @@ import (
"golang.org/x/mod/semver"
)
func (l *storageImpl) List(ctx context.Context, module string) ([]string, error) {
func (s *storageImpl) List(ctx context.Context, module string) ([]string, error) {
const op errors.Op = "fs.List"
ctx, span := observ.StartSpan(ctx, op.String())
_, span := observ.StartSpan(ctx, op.String())
defer span.End()
loc := l.moduleLocation(module)
fileInfos, err := afero.ReadDir(l.filesystem, loc)
loc := s.moduleLocation(module)
fileInfos, err := afero.ReadDir(s.filesystem, loc)
if err != nil {
if os.IsNotExist(err) {
return []string{}, nil
+12 -12
View File
@@ -13,37 +13,37 @@ import (
func (s *storageImpl) Save(ctx context.Context, module, version string, mod []byte, zip io.Reader, info []byte) error {
const op errors.Op = "fs.Save"
ctx, span := observ.StartSpan(ctx, op.String())
_, span := observ.StartSpan(ctx, op.String())
defer span.End()
dir := s.versionLocation(module, version)
// NB: the process's umask is subtracted from the permissions below,
// so a umask of for example 0077 allows directories and files to be
// created with mode 0700 / 0600, i.e. not world- or group-readable
// NB: The process's umask is subtracted from the permissions below,
// so an umask of for example 0077 allows directories and files to be
// created with mode 0700 / 0600, i.e. not world- or group-readable.
// make the versioned directory to hold the go.mod and the zipfile
if err := s.filesystem.MkdirAll(dir, 0777); err != nil {
// Make the versioned directory to hold the go.mod and the zipfile.
if err := s.filesystem.MkdirAll(dir, 0o777); err != nil {
return errors.E(op, err, errors.M(module), errors.V(version))
}
// write the go.mod file
if err := afero.WriteFile(s.filesystem, filepath.Join(dir, "go.mod"), mod, 0666); err != nil {
// Write the go.mod file.
if err := afero.WriteFile(s.filesystem, filepath.Join(dir, "go.mod"), mod, 0o666); err != nil {
return errors.E(op, err, errors.M(module), errors.V(version))
}
// write the zipfile
f, err := s.filesystem.OpenFile(filepath.Join(dir, "source.zip"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
// Write the zipfile.
f, err := s.filesystem.OpenFile(filepath.Join(dir, "source.zip"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o666)
if err != nil {
return errors.E(op, err, errors.M(module), errors.V(version))
}
defer f.Close()
defer func() { _ = f.Close() }()
_, err = io.Copy(f, zip)
if err != nil {
return errors.E(op, err, errors.M(module), errors.V(version))
}
// write the info file
err = afero.WriteFile(s.filesystem, filepath.Join(dir, version+".info"), info, 0666)
err = afero.WriteFile(s.filesystem, filepath.Join(dir, version+".info"), info, 0o666)
if err != nil {
return errors.E(op, err)
}
+1 -1
View File
@@ -14,7 +14,7 @@ import (
)
// Catalog implements the (./pkg/storage).Catalog interface
// It returns a list of versions, if any, for a given module
// It returns a list of versions, if any, for a given module.
func (s *Storage) Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error) {
const op errors.Op = "gcp.Catalog"
ctx, span := observ.StartSpan(ctx, op.String())
+2 -2
View File
@@ -11,7 +11,7 @@ import (
)
// Exists implements the (./pkg/storage).Checker interface
// returning true if the module at version exists in storage
// returning true if the module at version exists in storage.
func (s *Storage) Exists(ctx context.Context, module, version string) (bool, error) {
const op errors.Op = "gcp.Exists"
ctx, span := observ.StartSpan(ctx, op.String())
@@ -21,7 +21,7 @@ func (s *Storage) Exists(ctx context.Context, module, version string) (bool, err
var count int
for {
attrs, err := it.Next()
if err == iterator.Done {
if errors.IsErr(err, iterator.Done) {
break
}
if err != nil {
+7 -7
View File
@@ -13,7 +13,7 @@ import (
"google.golang.org/api/option"
)
// Storage implements the (./pkg/storage).Backend interface
// Storage implements the (./pkg/storage).Backend interface.
type Storage struct {
bucket *storage.BucketHandle
timeout time.Duration
@@ -34,8 +34,8 @@ func New(ctx context.Context, gcpConf *config.GCPConfig, timeout time.Duration)
return nil, errors.E(op, err)
}
if _, err := s.bucket.Attrs(ctx); err != nil {
if err == storage.ErrBucketNotExist {
if _, err = s.bucket.Attrs(ctx); err != nil {
if errors.IsErr(err, storage.ErrBucketNotExist) {
return nil, errors.E(op, "You must manually create a storage bucket for Athens, see https://cloud.google.com/storage/docs/creating-buckets#storage-create-bucket-console")
}
return nil, errors.E(op, err)
@@ -48,21 +48,21 @@ func New(ctx context.Context, gcpConf *config.GCPConfig, timeout time.Duration)
// this is so that the unit tests can use this to create their own short-lived buckets.
func newClient(ctx context.Context, gcpConf *config.GCPConfig, timeout time.Duration) (*Storage, error) {
const op errors.Op = "gcp.newClient"
opts := []option.ClientOption{}
var opts []option.ClientOption
if gcpConf.JSONKey != "" {
key, err := base64.StdEncoding.DecodeString(gcpConf.JSONKey)
if err != nil {
return nil, errors.E(op, fmt.Errorf("could not decode base64 json key: %v", err))
return nil, errors.E(op, fmt.Errorf("could not decode base64 json key: %w", err))
}
creds, err := google.CredentialsFromJSON(ctx, key, storage.ScopeReadWrite)
if err != nil {
return nil, errors.E(op, fmt.Errorf("could not get GCS credentials: %v", err))
return nil, errors.E(op, fmt.Errorf("could not get GCS credentials: %w", err))
}
opts = append(opts, option.WithCredentials(creds))
}
s, err := storage.NewClient(ctx, opts...)
if err != nil {
return nil, errors.E(op, fmt.Errorf("could not create new storage client: %s", err))
return nil, errors.E(op, fmt.Errorf("could not create new storage client: %w", err))
}
return &Storage{
+8 -8
View File
@@ -12,7 +12,7 @@ import (
pkgstorage "github.com/gomods/athens/pkg/storage"
)
// Info implements Getter
// Info implements Getter.
func (s *Storage) Info(ctx context.Context, module, version string) ([]byte, error) {
const op errors.Op = "gcp.Info"
ctx, span := observ.StartSpan(ctx, op.String())
@@ -22,14 +22,14 @@ func (s *Storage) Info(ctx context.Context, module, version string) ([]byte, err
return nil, errors.E(op, err, getErrorKind(err), errors.M(module), errors.V(version))
}
infoBytes, err := io.ReadAll(infoReader)
infoReader.Close()
_ = infoReader.Close()
if err != nil {
return nil, errors.E(op, err, errors.M(module), errors.V(version))
}
return infoBytes, nil
}
// GoMod implements Getter
// GoMod implements Getter.
func (s *Storage) GoMod(ctx context.Context, module, version string) ([]byte, error) {
const op errors.Op = "gcp.GoMod"
ctx, span := observ.StartSpan(ctx, op.String())
@@ -39,15 +39,15 @@ func (s *Storage) GoMod(ctx context.Context, module, version string) ([]byte, er
return nil, errors.E(op, err, getErrorKind(err), errors.M(module), errors.V(version))
}
modBytes, err := io.ReadAll(modReader)
modReader.Close()
_ = modReader.Close()
if err != nil {
return nil, errors.E(op, fmt.Errorf("could not get new reader for mod file: %s", err), errors.M(module), errors.V(version))
return nil, errors.E(op, fmt.Errorf("could not get new reader for mod file: %w", err), errors.M(module), errors.V(version))
}
return modBytes, nil
}
// Zip implements Getter
// Zip implements Getter.
func (s *Storage) Zip(ctx context.Context, module, version string) (pkgstorage.SizeReadCloser, error) {
const op errors.Op = "gcp.Zip"
ctx, span := observ.StartSpan(ctx, op.String())
@@ -56,11 +56,11 @@ func (s *Storage) Zip(ctx context.Context, module, version string) (pkgstorage.S
if err != nil {
return nil, errors.E(op, err, getErrorKind(err), errors.M(module), errors.V(version))
}
return pkgstorage.NewSizer(zipReader, zipReader.Size()), nil
return pkgstorage.NewSizer(zipReader, zipReader.Attrs.Size), nil
}
func getErrorKind(err error) int {
if err == storage.ErrObjectNotExist {
if errors.IsErr(err, storage.ErrObjectNotExist) {
return errors.KindNotFound
}
return errors.KindUnexpected
+3 -3
View File
@@ -10,8 +10,8 @@ import (
"google.golang.org/api/iterator"
)
// List implements the (./pkg/storage).Lister interface
// It returns a list of versions, if any, for a given module
// List implements the (./pkg/storage).Lister interface.
// It returns a list of versions, if any, for a given module.
func (s *Storage) List(ctx context.Context, module string) ([]string, error) {
const op errors.Op = "gcp.List"
ctx, span := observ.StartSpan(ctx, op.String())
@@ -22,7 +22,7 @@ func (s *Storage) List(ctx context.Context, module string) ([]string, error) {
paths := []string{}
for {
attrs, err := it.Next()
if err == iterator.Done {
if errors.IsErr(err, iterator.Done) {
break
}
if err != nil {
+3 -3
View File
@@ -53,15 +53,15 @@ func (s *Storage) upload(ctx context.Context, path string, stream io.Reader) err
// Once we support private storage buckets this may need refactoring
// unless there is a way to set the default perms in the project.
if _, err := io.Copy(wc, stream); err != nil {
wc.Close()
_ = wc.Close()
return err
}
err := wc.Close()
if err != nil {
kind := errors.KindBadRequest
apiErr, ok := err.(*googleapi.Error)
if ok && apiErr.Code == 412 {
apiErr := &googleapi.Error{}
if errors.AsErr(err, &apiErr) && apiErr.Code == 412 {
kind = errors.KindAlreadyExists
}
return errors.E(op, err, kind)
+3 -3
View File
@@ -5,7 +5,7 @@ import (
"io"
)
// Getter gets module metadata and its source from underlying storage
// Getter gets module metadata and its source from underlying storage.
type Getter interface {
Info(ctx context.Context, module, vsn string) ([]byte, error)
GoMod(ctx context.Context, module, vsn string) ([]byte, error)
@@ -14,14 +14,14 @@ type Getter interface {
// SizeReadCloser extends io.ReadCloser
// with a Size() method that tells you the
// length of the io.ReadCloser if read in full
// length of the io.ReadCloser if read in full.
type SizeReadCloser interface {
io.ReadCloser
Size() int64
}
// NewSizer is a helper wrapper to return an implementation
// of ReadCloserSizer
// of ReadCloserSizer.
func NewSizer(rc io.ReadCloser, size int64) SizeReadCloser {
return &sizeReadCloser{rc, size}
}
+1 -1
View File
@@ -2,7 +2,7 @@ package storage
import "context"
// Lister is the interface that lists versions of a specific baseURL & module
// Lister is the interface that lists versions of a specific baseURL & module.
type Lister interface {
// List gets all the versions for the given baseURL & module.
// It returns ErrNotFound if the module isn't found
+3 -3
View File
@@ -9,19 +9,19 @@ import (
"github.com/spf13/afero"
)
// NewStorage creates new in-memory storage using the afero.NewMemMapFs() in memory file system
// NewStorage creates new in-memory storage using the afero.NewMemMapFs() in memory file system.
func NewStorage() (storage.Backend, error) {
const op errors.Op = "mem.NewStorage"
memFs := afero.NewMemMapFs()
tmpDir, err := afero.TempDir(memFs, "", "")
if err != nil {
return nil, errors.E(op, fmt.Errorf("could not create temp dir for 'In Memory' storage (%s)", err))
return nil, errors.E(op, fmt.Errorf("could not create temp dir for 'In Memory' storage: %w", err))
}
memStorage, err := fs.NewStorage(tmpDir, memFs)
if err != nil {
return nil, errors.E(op, fmt.Errorf("could not create storage from memory fs (%s)", err))
return nil, errors.E(op, fmt.Errorf("could not create storage from memory fs: %w", err))
}
return memStorage, nil
}
+5 -5
View File
@@ -11,11 +11,11 @@ import (
"github.com/minio/minio-go/v6"
)
// Catalog implements the (./pkg/storage).Cataloger interface
// It returns a list of modules and versions contained in the storage
// Catalog implements the (./pkg/storage).Cataloger interface.
// It returns a list of modules and versions contained in the storage.
func (s *storageImpl) Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error) {
const op errors.Op = "minio.Catalog"
ctx, span := observ.StartSpan(ctx, op.String())
_, span := observ.StartSpan(ctx, op.String())
defer span.End()
res := make([]paths.AllPathParams, 0)
count := pageSize
@@ -50,7 +50,7 @@ func fetchModsAndVersions(objects []minio.ObjectInfo, elementsNum int) ([]paths.
continue
}
p, err := parseMinioKey(&o)
p, err := parseMinioKey(o)
if err != nil {
continue
}
@@ -66,7 +66,7 @@ func fetchModsAndVersions(objects []minio.ObjectInfo, elementsNum int) ([]paths.
return res, lastKey
}
func parseMinioKey(o *minio.ObjectInfo) (paths.AllPathParams, error) {
func parseMinioKey(o minio.ObjectInfo) (paths.AllPathParams, error) {
const op errors.Op = "minio.parseMinioKey"
_, m, v := extractKey(o.Key)
+4 -8
View File
@@ -8,21 +8,17 @@ import (
"github.com/gomods/athens/pkg/observ"
)
const (
minioErrorCodeNoSuchKey = "NoSuchKey"
)
func (v *storageImpl) Exists(ctx context.Context, module, version string) (bool, error) {
func (s *storageImpl) Exists(ctx context.Context, module, version string) (bool, error) {
const op errors.Op = "minio.Exists"
ctx, span := observ.StartSpan(ctx, op.String())
_, span := observ.StartSpan(ctx, op.String())
defer span.End()
versionedPath := v.versionLocation(module, version)
versionedPath := s.versionLocation(module, version)
modPath := fmt.Sprintf("%s/go.mod", versionedPath)
infoPath := fmt.Sprintf("%s/%s.info", versionedPath, version)
zipPath := fmt.Sprintf("%s/source.zip", versionedPath)
var count int
objectCh, err := v.minioCore.ListObjectsV2(v.bucketName, versionedPath, "", false, "", 0, "")
objectCh, err := s.minioCore.ListObjectsV2(s.bucketName, versionedPath, "", false, "", 0, "")
if err != nil {
return false, errors.E(op, err, errors.M(module), errors.V(version))
}
+6 -6
View File
@@ -8,11 +8,11 @@ import (
"github.com/gomods/athens/pkg/observ"
)
func (v *storageImpl) Delete(ctx context.Context, module, version string) error {
func (s *storageImpl) Delete(ctx context.Context, module, version string) error {
const op errors.Op = "minio.Delete"
ctx, span := observ.StartSpan(ctx, op.String())
defer span.End()
exists, err := v.Exists(ctx, module, version)
exists, err := s.Exists(ctx, module, version)
if err != nil {
return errors.E(op, err, errors.M(module), errors.V(version))
}
@@ -21,20 +21,20 @@ func (v *storageImpl) Delete(ctx context.Context, module, version string) error
return errors.E(op, errors.M(module), errors.V(version), errors.KindNotFound)
}
versionedPath := v.versionLocation(module, version)
versionedPath := s.versionLocation(module, version)
modPath := fmt.Sprintf("%s/go.mod", versionedPath)
if err := v.minioClient.RemoveObject(v.bucketName, modPath); err != nil {
if err := s.minioClient.RemoveObject(s.bucketName, modPath); err != nil {
return errors.E(op, err, errors.M(module), errors.V(version))
}
zipPath := fmt.Sprintf("%s/source.zip", versionedPath)
if err := v.minioClient.RemoveObject(v.bucketName, zipPath); err != nil {
if err := s.minioClient.RemoveObject(s.bucketName, zipPath); err != nil {
return errors.E(op, err, errors.M(module), errors.V(version))
}
infoPath := fmt.Sprintf("%s/%s.info", versionedPath, version)
err = v.minioClient.RemoveObject(v.bucketName, infoPath)
err = s.minioClient.RemoveObject(s.bucketName, infoPath)
if err != nil {
return errors.E(op, err, errors.M(module), errors.V(version))
}
+18 -17
View File
@@ -12,16 +12,16 @@ import (
minio "github.com/minio/minio-go/v6"
)
func (v *storageImpl) Info(ctx context.Context, module, vsn string) ([]byte, error) {
func (s *storageImpl) Info(ctx context.Context, module, vsn string) ([]byte, error) {
const op errors.Op = "minio.Info"
ctx, span := observ.StartSpan(ctx, op.String())
_, span := observ.StartSpan(ctx, op.String())
defer span.End()
infoPath := fmt.Sprintf("%s/%s.info", v.versionLocation(module, vsn), vsn)
infoReader, err := v.minioClient.GetObject(v.bucketName, infoPath, minio.GetObjectOptions{})
infoPath := fmt.Sprintf("%s/%s.info", s.versionLocation(module, vsn), vsn)
infoReader, err := s.minioClient.GetObject(s.bucketName, infoPath, minio.GetObjectOptions{})
if err != nil {
return nil, errors.E(op, err)
}
defer infoReader.Close()
defer func() { _ = infoReader.Close() }()
info, err := io.ReadAll(infoReader)
if err != nil {
return nil, transformNotFoundErr(op, module, vsn, err)
@@ -30,16 +30,16 @@ func (v *storageImpl) Info(ctx context.Context, module, vsn string) ([]byte, err
return info, nil
}
func (v *storageImpl) GoMod(ctx context.Context, module, vsn string) ([]byte, error) {
func (s *storageImpl) GoMod(ctx context.Context, module, vsn string) ([]byte, error) {
const op errors.Op = "minio.GoMod"
ctx, span := observ.StartSpan(ctx, op.String())
_, span := observ.StartSpan(ctx, op.String())
defer span.End()
modPath := fmt.Sprintf("%s/go.mod", v.versionLocation(module, vsn))
modReader, err := v.minioClient.GetObject(v.bucketName, modPath, minio.GetObjectOptions{})
modPath := fmt.Sprintf("%s/go.mod", s.versionLocation(module, vsn))
modReader, err := s.minioClient.GetObject(s.bucketName, modPath, minio.GetObjectOptions{})
if err != nil {
return nil, errors.E(op, err)
}
defer modReader.Close()
defer func() { _ = modReader.Close() }()
mod, err := io.ReadAll(modReader)
if err != nil {
return nil, transformNotFoundErr(op, module, vsn, err)
@@ -48,31 +48,32 @@ func (v *storageImpl) GoMod(ctx context.Context, module, vsn string) ([]byte, er
return mod, nil
}
func (v *storageImpl) Zip(ctx context.Context, module, vsn string) (storage.SizeReadCloser, error) {
func (s *storageImpl) Zip(ctx context.Context, module, vsn string) (storage.SizeReadCloser, error) {
const op errors.Op = "minio.Zip"
ctx, span := observ.StartSpan(ctx, op.String())
_, span := observ.StartSpan(ctx, op.String())
defer span.End()
zipPath := fmt.Sprintf("%s/source.zip", v.versionLocation(module, vsn))
_, err := v.minioClient.StatObject(v.bucketName, zipPath, minio.StatObjectOptions{})
zipPath := fmt.Sprintf("%s/source.zip", s.versionLocation(module, vsn))
_, err := s.minioClient.StatObject(s.bucketName, zipPath, minio.StatObjectOptions{})
if err != nil {
return nil, errors.E(op, err, errors.KindNotFound, errors.M(module), errors.V(vsn))
}
zipReader, err := v.minioClient.GetObject(v.bucketName, zipPath, minio.GetObjectOptions{})
zipReader, err := s.minioClient.GetObject(s.bucketName, zipPath, minio.GetObjectOptions{})
if err != nil {
return nil, errors.E(op, err)
}
oi, err := zipReader.Stat()
if err != nil {
zipReader.Close()
_ = zipReader.Close()
return nil, errors.E(op, err)
}
return storage.NewSizer(zipReader, oi.Size), nil
}
func transformNotFoundErr(op errors.Op, module, version string, err error) error {
if eresp, ok := err.(minio.ErrorResponse); ok {
var eresp minio.ErrorResponse
if errors.AsErr(err, &eresp) {
if eresp.StatusCode == http.StatusNotFound {
return errors.E(op, errors.M(module), errors.V(version), errors.KindNotFound)
}
+3 -3
View File
@@ -5,7 +5,7 @@ import (
"strings"
)
func extractKey(objectKey string) (key string, module string, version string) {
func extractKey(objectKey string) (key, module, version string) {
var err error
key, err = url.PathUnescape(objectKey)
if err != nil {
@@ -14,8 +14,8 @@ func extractKey(objectKey string) (key string, module string, version string) {
parts := strings.Split(key, "/")
version = parts[len(parts)-2]
module = strings.Replace(key, version, "", -2)
module = strings.Replace(module, "//.info", "", -1)
module = strings.ReplaceAll(key, version, "")
module = strings.ReplaceAll(module, "//.info", "")
return key, module, version
}
+5 -6
View File
@@ -9,26 +9,25 @@ import (
"github.com/gomods/athens/pkg/observ"
)
func (l *storageImpl) List(ctx context.Context, module string) ([]string, error) {
func (s *storageImpl) List(ctx context.Context, module string) ([]string, error) {
const op errors.Op = "minio.List"
ctx, span := observ.StartSpan(ctx, op.String())
_, span := observ.StartSpan(ctx, op.String())
defer span.End()
doneCh := make(chan struct{})
defer close(doneCh)
searchPrefix := module + "/"
objectCh, err := l.minioCore.ListObjectsV2(l.bucketName, searchPrefix, "", false, "", 0, "")
objectCh, err := s.minioCore.ListObjectsV2(s.bucketName, searchPrefix, "", false, "", 0, "")
if err != nil {
return nil, errors.E(op, err, errors.M(module))
}
ret := []string{}
var ret []string
for _, object := range objectCh.Contents {
if object.Err != nil {
return nil, errors.E(op, object.Err, errors.M(module))
}
key, _, ver := extractKey(object.Key)
goModKey := fmt.Sprintf("%s/go.mod", l.versionLocation(module, ver))
goModKey := fmt.Sprintf("%s/go.mod", s.versionLocation(module, ver))
if goModKey == key {
ret = append(ret, ver)
}
+1 -1
View File
@@ -21,7 +21,7 @@ func (s *storageImpl) versionLocation(module, version string) string {
}
// NewStorage returns a connected Minio or DigitalOcean Spaces storage
// that implements storage.Backend
// that implements storage.Backend.
func NewStorage(conf *config.MinioConfig, timeout time.Duration) (storage.Backend, error) {
const op errors.Op = "minio.NewStorage"
endpoint := conf.Endpoint
+1 -1
View File
@@ -15,7 +15,7 @@ import (
func (s *storageImpl) Save(ctx context.Context, module, vsn string, mod []byte, zip io.Reader, info []byte) error {
const op errors.Op = "storage.minio.Save"
ctx, span := observ.StartSpan(ctx, op.String())
_, span := observ.StartSpan(ctx, op.String())
defer span.End()
dir := s.versionLocation(module, vsn)
modFileName := dir + "/" + "go.mod"
+8 -8
View File
@@ -7,26 +7,26 @@ import (
"github.com/gomods/athens/pkg/config"
"github.com/gomods/athens/pkg/errors"
multierror "github.com/hashicorp/go-multierror"
"github.com/hashicorp/go-multierror"
)
// Deleter takes a path to a file and deletes it from the blob store
// Deleter takes a path to a file and deletes it from the blob store.
type Deleter func(ctx context.Context, path string) error
// Delete deletes .info, .mod and .zip files from the blob store in parallel.
// Returns multierror containing errors from all deletes and timeouts
func Delete(ctx context.Context, module, version string, delete Deleter, timeout time.Duration) error {
// Returns multierror containing errors from all deletes and timeouts.
func Delete(ctx context.Context, module, version string, del Deleter, timeout time.Duration) error {
const op errors.Op = "module.Delete"
tctx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
del := func(ext string) <-chan error {
delFn := func(ext string) <-chan error {
ec := make(chan error)
go func() {
defer close(ec)
p := config.PackageVersionedName(module, version, ext)
ec <- delete(tctx, p)
ec <- del(tctx, p)
}()
return ec
}
@@ -34,10 +34,10 @@ func Delete(ctx context.Context, module, version string, delete Deleter, timeout
errChan := make(chan error, numFiles)
delOrAbort := func(ext string) {
select {
case err := <-del(ext):
case err := <-delFn(ext):
errChan <- err
case <-tctx.Done():
errChan <- fmt.Errorf("deleting %s.%s.%s failed: %s", module, version, ext, tctx.Err())
errChan <- fmt.Errorf("deleting %s.%s.%s failed: %w", module, version, ext, tctx.Err())
}
}
+3 -3
View File
@@ -13,11 +13,11 @@ import (
const numFiles = 3
// Uploader takes a stream and saves it to the blob store under a given path
// Uploader takes a stream and saves it to the blob store under a given path.
type Uploader func(ctx context.Context, path, contentType string, stream io.Reader) error
// Upload saves .info, .mod and .zip files to the blob store in parallel.
// Returns multierror containing errors from all uploads and timeouts
// Returns multierror containing errors from all uploads and timeouts.
func Upload(ctx context.Context, module, version string, info, mod, zip io.Reader, uploader Uploader, timeout time.Duration) error {
const op errors.Op = "module.Upload"
tctx, cancel := context.WithTimeout(ctx, timeout)
@@ -40,7 +40,7 @@ func Upload(ctx context.Context, module, version string, info, mod, zip io.Reade
case err := <-save(ext, contentType, stream):
errChan <- err
case <-tctx.Done():
errChan <- fmt.Errorf("uploading %s.%s.%s failed: %s", module, version, ext, tctx.Err())
errChan <- fmt.Errorf("uploading %s.%s.%s failed: %w", module, version, ext, tctx.Err())
}
}
go saveOrAbort("info", "application/json", info)
+5 -5
View File
@@ -2,6 +2,7 @@ package mongo
import (
"context"
"github.com/gomods/athens/pkg/errors"
"github.com/gomods/athens/pkg/paths"
"github.com/gomods/athens/pkg/storage"
@@ -11,8 +12,8 @@ import (
"go.mongodb.org/mongo-driver/mongo/options"
)
// Catalog implements the (./pkg/storage).Cataloger interface
// It returns a list of modules and versions contained in the storage
// Catalog implements the (./pkg/storage).Cataloger interface.
// It returns a list of modules and versions contained in the storage.
func (s *ModuleStore) Catalog(ctx context.Context, token string, pageSize int) ([]paths.AllPathParams, string, error) {
const op errors.Op = "mongo.Catalog"
q := bson.M{}
@@ -33,7 +34,6 @@ func (s *ModuleStore) Catalog(ctx context.Context, token string, pageSize int) (
modules := make([]storage.Module, 0)
findOptions := options.Find().SetProjection(projection).SetSort(sort).SetLimit(int64(pageSize))
cursor, err := c.Find(tctx, q, findOptions)
if err != nil {
return nil, "", errors.E(op, err)
}
@@ -53,13 +53,13 @@ func (s *ModuleStore) Catalog(ctx context.Context, token string, pageSize int) (
return nil, "", nil
}
var versions = make([]paths.AllPathParams, len(modules))
versions := make([]paths.AllPathParams, len(modules))
for i := range modules {
versions[i].Module = modules[i].Module
versions[i].Version = modules[i].Version
}
var next = modules[len(modules)-1].ID.Hex()
next := modules[len(modules)-1].ID.Hex()
if len(modules) < pageSize {
return versions, "", nil
}
+1 -1
View File
@@ -8,7 +8,7 @@ import (
"go.mongodb.org/mongo-driver/bson"
)
// Exists checks for a specific version of a module
// Exists checks for a specific version of a module.
func (s *ModuleStore) Exists(ctx context.Context, module, vsn string) (bool, error) {
var op errors.Op = "mongo.Exists"
ctx, span := observ.StartSpan(ctx, op.String())
+3 -3
View File
@@ -11,7 +11,7 @@ import (
"go.mongodb.org/mongo-driver/x/bsonx"
)
// Delete removes a specific version of a module
// Delete removes a specific version of a module.
func (s *ModuleStore) Delete(ctx context.Context, module, version string) error {
const op errors.Op = "mongo.Delete"
ctx, span := observ.StartSpan(ctx, op.String())
@@ -40,11 +40,11 @@ func (s *ModuleStore) Delete(ctx context.Context, module, version string) error
var x bsonx.Doc
for cursor.Next(ctx) {
cursor.Decode(&x)
_ = cursor.Decode(&x)
}
if err = bucket.Delete(x.Lookup("_id").ObjectID()); err != nil {
kind := errors.KindUnexpected
if err == gridfs.ErrFileNotFound {
if errors.IsErr(err, gridfs.ErrFileNotFound) {
kind = errors.KindNotFound
}
return errors.E(op, err, kind, errors.M(module), errors.V(version))
+7 -9
View File
@@ -13,14 +13,13 @@ import (
"go.mongodb.org/mongo-driver/x/bsonx"
)
// Info implements storage.Getter
// Info implements storage.Getter.
func (s *ModuleStore) Info(ctx context.Context, module, vsn string) ([]byte, error) {
const op errors.Op = "mongo.Info"
ctx, span := observ.StartSpan(ctx, op.String())
defer span.End()
result, err := query(ctx, s, module, vsn)
if err != nil {
return nil, errors.E(op, err)
}
@@ -28,14 +27,13 @@ func (s *ModuleStore) Info(ctx context.Context, module, vsn string) ([]byte, err
return result.Info, nil
}
// GoMod implements storage.Getter
// GoMod implements storage.Getter.
func (s *ModuleStore) GoMod(ctx context.Context, module, vsn string) ([]byte, error) {
const op errors.Op = "mongo.GoMod"
ctx, span := observ.StartSpan(ctx, op.String())
defer span.End()
result, err := query(ctx, s, module, vsn)
if err != nil {
return nil, errors.E(op, err)
}
@@ -43,7 +41,7 @@ func (s *ModuleStore) GoMod(ctx context.Context, module, vsn string) ([]byte, er
return result.Mod, nil
}
// Zip implements storage.Getter
// Zip implements storage.Getter.
func (s *ModuleStore) Zip(ctx context.Context, module, vsn string) (storage.SizeReadCloser, error) {
const op errors.Op = "mongo.Zip"
ctx, span := observ.StartSpan(ctx, op.String())
@@ -59,7 +57,7 @@ func (s *ModuleStore) Zip(ctx context.Context, module, vsn string) (storage.Size
dStream, err := bucket.OpenDownloadStreamByName(zipName, options.GridFSName())
if err != nil {
kind := errors.KindUnexpected
if err == gridfs.ErrFileNotFound {
if errors.IsErr(err, gridfs.ErrFileNotFound) {
kind = errors.KindNotFound
}
return nil, errors.E(op, err, kind, errors.M(module), errors.V(vsn))
@@ -79,7 +77,7 @@ func (s *ModuleStore) Zip(ctx context.Context, module, vsn string) (storage.Size
return storage.NewSizer(dStream, size), nil
}
// Query connects to and queries storage module
// Query connects to and queries storage module.
func query(ctx context.Context, s *ModuleStore, module, vsn string) (*storage.Module, error) {
const op errors.Op = "mongo.query"
ctx, span := observ.StartSpan(ctx, op.String())
@@ -95,7 +93,7 @@ func query(ctx context.Context, s *ModuleStore, module, vsn string) (*storage.Mo
queryResult := c.FindOne(tctx, bson.M{"module": module, "version": vsn})
if queryErr := queryResult.Err(); queryErr != nil {
kind := errors.KindUnexpected
if queryErr == mongo.ErrNoDocuments {
if errors.IsErr(queryErr, mongo.ErrNoDocuments) {
kind = errors.KindNotFound
}
return nil, errors.E(op, queryErr, kind, errors.M(module), errors.V(vsn))
@@ -103,7 +101,7 @@ func query(ctx context.Context, s *ModuleStore, module, vsn string) (*storage.Mo
if err := queryResult.Decode(result); err != nil {
kind := errors.KindUnexpected
if err == mongo.ErrNoDocuments {
if errors.IsErr(err, mongo.ErrNoDocuments) {
kind = errors.KindNotFound
}
return nil, errors.E(op, err, kind, errors.M(module), errors.V(vsn))

Some files were not shown because too many files have changed in this diff Show More