diff --git a/appveyor.yml b/appveyor.yml index 2da8d497..58917515 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -2,6 +2,8 @@ build: off clone_folder: c:\gopath\src\github.com\gomods\athens +image: Previous Visual Studio 2019 + environment: GOPATH: c:\gopath GO111MODULE: on @@ -11,5 +13,6 @@ environment: stack: go 1.14 test_script: + - go version - go test ./... diff --git a/cmd/proxy/actions/app_proxy.go b/cmd/proxy/actions/app_proxy.go index 7a2e2685..3d1e3b7f 100644 --- a/cmd/proxy/actions/app_proxy.go +++ b/cmd/proxy/actions/app_proxy.go @@ -11,6 +11,11 @@ import ( "github.com/gomods/athens/pkg/download" "github.com/gomods/athens/pkg/download/addons" "github.com/gomods/athens/pkg/download/mode" + "github.com/gomods/athens/pkg/index" + "github.com/gomods/athens/pkg/index/mem" + "github.com/gomods/athens/pkg/index/mysql" + "github.com/gomods/athens/pkg/index/nop" + "github.com/gomods/athens/pkg/index/postgres" "github.com/gomods/athens/pkg/log" "github.com/gomods/athens/pkg/module" "github.com/gomods/athens/pkg/stash" @@ -32,6 +37,12 @@ func addProxyRoutes( r.HandleFunc("/catalog", catalogHandler(s)) r.HandleFunc("/robots.txt", robotsHandler(c)) + indexer, err := getIndex(c) + if err != nil { + return err + } + r.HandleFunc("/index", indexHandler(indexer)) + for _, sumdb := range c.SumDBs { sumdbURL, err := url.Parse(sumdb) if err != nil { @@ -96,7 +107,7 @@ func addProxyRoutes( if err != nil { return err } - st := stash.New(mf, s, stash.WithPool(c.GoGetWorkers), withSingleFlight) + st := stash.New(mf, s, indexer, stash.WithPool(c.GoGetWorkers), withSingleFlight) df, err := mode.NewFile(c.DownloadMode, c.DownloadURL) if err != nil { @@ -157,3 +168,17 @@ func getSingleFlight(c *config.Config, checker storage.Checker) (stash.Wrapper, return nil, fmt.Errorf("unrecognized single flight type: %v", c.SingleFlightType) } } + +func getIndex(c *config.Config) (index.Indexer, error) { + switch c.IndexType { + case "", "none": + return nop.New(), nil + case "memory": + return mem.New(), nil + case "mysql": + return mysql.New(c.Index.MySQL) + case "postgres": + return postgres.New(c.Index.Postgres) + } + return nil, fmt.Errorf("unknown index type: %q", c.IndexType) +} diff --git a/cmd/proxy/actions/index.go b/cmd/proxy/actions/index.go new file mode 100644 index 00000000..3be942c0 --- /dev/null +++ b/cmd/proxy/actions/index.go @@ -0,0 +1,52 @@ +package actions + +import ( + "encoding/json" + "net/http" + "strconv" + "time" + + "github.com/gomods/athens/pkg/errors" + "github.com/gomods/athens/pkg/index" +) + +// indexHandler implements GET baseURL/index +func indexHandler(index index.Indexer) http.HandlerFunc { + const op errors.Op = "actions.IndexHandler" + return func(w http.ResponseWriter, r *http.Request) { + var ( + err error + limit int + since time.Time + ) + if limitStr := r.FormValue("limit"); limitStr != "" { + limit, err = strconv.Atoi(limitStr) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + } + if sinceStr := r.FormValue("since"); sinceStr != "" { + since, err = time.Parse(time.RFC3339, sinceStr) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + } + if limit <= 0 { + limit = 2000 + } + list, err := index.Lines(r.Context(), since, limit) + if err != nil { + http.Error(w, err.Error(), 500) + return + } + enc := json.NewEncoder(w) + for _, meta := range list { + if err = enc.Encode(meta); err != nil { + http.Error(w, err.Error(), 500) + return + } + } + } +} diff --git a/cmd/proxy/actions/storage.go b/cmd/proxy/actions/storage.go index 6dceb1a0..b1e51b5a 100644 --- a/cmd/proxy/actions/storage.go +++ b/cmd/proxy/actions/storage.go @@ -21,7 +21,7 @@ import ( ) // GetStorage returns storage backend based on env configuration -func GetStorage(storageType string, storageConfig *config.StorageConfig, timeout time.Duration, client *http.Client) (storage.Backend, error) { +func GetStorage(storageType string, storageConfig *config.Storage, timeout time.Duration, client *http.Client) (storage.Backend, error) { const op errors.Op = "actions.GetStorage" switch storageType { case "memory": diff --git a/config.dev.toml b/config.dev.toml index b27e1291..cf579ecb 100755 --- a/config.dev.toml +++ b/config.dev.toml @@ -282,6 +282,12 @@ DownloadURL = "" # Env override: ATHENS_SINGLE_FLIGHT_TYPE SingleFlightType = "memory" +# IndexType sets the type of an index backend Athens will use. +# Possible values are none, memory, mysql, postgres +# Defaults to none +# Env override: ATHENS_INDEX_TYPE +IndexType = "none" + [SingleFlight] [SingleFlight.Etcd] # Endpoints are comma separated URLs that determine all distributed etcd servers. @@ -475,3 +481,65 @@ SingleFlightType = "memory" # details. # Env override: ATHENS_EXTERNAL_STORAGE_URL URL = "" + +[Index] + [Index.MySQL] + # MySQL protocol + # Env override: ATHENS_INDEX_MYSQL_PROTOCOL + Protocol = "tcp" + + # MySQL user name + # Env override: ATHENS_INDEX_MYSQL_HOST + Host = "localhost" + + # MySQL user name + # Env override: ATHENS_INDEX_MYSQL_PORT + Port = 3306 + + # MySQL user name + # Env override: ATHENS_INDEX_MYSQL_USER + User = "root" + + # MySQL user name + # Env override: ATHENS_INDEX_MYSQL_PASSWORD + Password = "" + + # MySQL database + # Env override: ATHENS_INDEX_MYSQL_DATABASE + Database = "athens" + + # MySQL query parameters + # Environment overrides must be in the following format: + # ATHENS_INDEX_MYSQL_PARAMS="parseTime:true,timeout=90s" + # Env override: ATHENS_INDEX_MYSQL_PARAMS + [Index.MySQL.Params] + parseTime = "true" + timeout = "30s" + [Index.Postgres] + # Postgres user name + # Env override: ATHENS_INDEX_POSTGRES_HOST + Host = "localhost" + + # Postgres user name + # Env override: ATHENS_INDEX_POSTGRES_PORT + Port = 5432 + + # Postgres user name + # Env override: ATHENS_INDEX_POSTGRES_USER + User = "postgres" + + # Postgres user name + # Env override: ATHENS_INDEX_POSTGRES_PASSWORD + Password = "" + + # Postgres database + # Env override: ATHENS_INDEX_POSTGRES_DATABASE + Database = "athens" + + # Postgres query parameters + # Environment overrides must be in the following format: + # ATHENS_INDEX_POSTGRES_PARAMS="connect_timeout:30s,sslmode=disable" + # Env override: ATHENS_INDEX_POSTGRES_PARAMS + [Index.Postgres.Params] + connect_timeout = "30s" + sslmode = "disable" diff --git a/go.mod b/go.mod index 35017d0c..3cf48d1b 100644 --- a/go.mod +++ b/go.mod @@ -17,10 +17,10 @@ require ( github.com/bsm/redislock v0.4.2 github.com/codegangsta/negroni v1.0.0 // indirect github.com/fatih/color v1.7.0 - github.com/go-ini/ini v1.25.4 // indirect github.com/go-playground/locales v0.12.1 // indirect github.com/go-playground/universal-translator v0.16.0 // indirect github.com/go-redis/redis/v7 v7.2.0 + github.com/go-sql-driver/mysql v1.5.0 github.com/gobuffalo/envy v1.6.7 github.com/gobuffalo/httptest v1.0.4 github.com/golang/protobuf v1.3.3 // indirect @@ -32,6 +32,7 @@ require ( github.com/hashicorp/hcl2 v0.0.0-20190503213020-640445e16309 github.com/kelseyhightower/envconfig v1.3.0 github.com/kylelemons/godebug v1.1.0 // indirect + github.com/lib/pq v1.7.0 github.com/minio/minio-go/v6 v6.0.43 github.com/mitchellh/go-homedir v1.1.0 github.com/philhofer/fwd v1.0.0 // indirect diff --git a/go.sum b/go.sum index 06ab9913..d8cc4cac 100644 --- a/go.sum +++ b/go.sum @@ -68,8 +68,6 @@ github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb github.com/apparentlymart/go-dump v0.0.0-20180507223929-23540a00eaa3/go.mod h1:oL81AME2rN47vu18xqj1S1jPIPuN7afo62yKTNn3XMM= github.com/apparentlymart/go-textseg v1.0.0 h1:rRmlIsPEEhUTIKQb7T++Nz/A5Q6C9IuX2wFoYVvnCs0= github.com/apparentlymart/go-textseg v1.0.0/go.mod h1:z96Txxhf3xSFMPmb5X/1W05FF/Nj9VFpLOpjS5yuumk= -github.com/aws/aws-sdk-go v1.15.24 h1:xLAdTA/ore6xdPAljzZRed7IGqQgC+nY+ERS5vaj4Ro= -github.com/aws/aws-sdk-go v1.15.24/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= github.com/aws/aws-sdk-go v1.32.7 h1:H4VgdCSF1cHw0VD8zGc98T1bGdACoLkh/vK2L6wgOUU= github.com/aws/aws-sdk-go v1.32.7/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= @@ -113,8 +111,6 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-ini/ini v1.25.4 h1:Mujh4R/dH6YL8bxuISne3xX2+qcQ9p0IxKAP6ExWoUo= -github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-playground/locales v0.12.1 h1:2FITxuFt/xuCNP1Acdhv62OzaCiviiE4kotfhkmOqEc= @@ -124,6 +120,7 @@ github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEK github.com/go-redis/redis/v7 v7.0.0-beta.4/go.mod h1:xhhSbUMTsleRPur+Vgx9sUHtyN33bdjxY+9/0n9Ig8s= github.com/go-redis/redis/v7 v7.2.0 h1:CrCexy/jYWZjW0AyVoHlcJUeZN19VWlbepTh1Vq6dJs= github.com/go-redis/redis/v7 v7.2.0/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg= +github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -216,8 +213,6 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8 h1:12VvqtR6Aowv3l/EQUlocDHW2Cp4G9WJVH7uyH8QFJE= -github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= @@ -246,6 +241,8 @@ github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3v github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/lib/pq v1.7.0 h1:h93mCPfUSkaul3Ka/VG8uZdmW1uMHDGxzu0NWHuJmHY= +github.com/lib/pq v1.7.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/markbates/hmax v1.0.0 h1:yo2N0gBoCnUMKhV/VRLHomT6Y9wUm+oQQENuWJqCdlM= github.com/markbates/hmax v1.0.0/go.mod h1:cOkR9dktiESxIMu+65oc/r/bdY4bE8zZw3OLhLx0X2c= github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= @@ -286,8 +283,7 @@ github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -338,6 +334,7 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/technosophos/moniker v0.0.0-20180509230615-a5dbd03a2245 h1:DNVk+NIkGS0RbLkjQOLCJb/759yfCysThkMbl7EXxyY= github.com/technosophos/moniker v0.0.0-20180509230615-a5dbd03a2245/go.mod h1:O1c8HleITsZqzNZDjSNzirUGsMT0oGu9LhHKoJrqO+A= diff --git a/pkg/config/config.go b/pkg/config/config.go index 2975e942..108c0a6a 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -54,8 +54,10 @@ type Config struct { DownloadURL string `envconfig:"ATHENS_DOWNLOAD_URL"` SingleFlightType string `envconfig:"ATHENS_SINGLE_FLIGHT_TYPE"` RobotsFile string `envconfig:"ATHENS_ROBOTS_FILE"` + IndexType string `envconfig:"ATHENS_INDEX_TYPE"` SingleFlight *SingleFlight - Storage *StorageConfig + Storage *Storage + Index *Index } // EnvList is a list of key-value environment @@ -161,6 +163,7 @@ func defaultConfig() *Config { DownloadMode: "sync", DownloadURL: "", RobotsFile: "robots.txt", + IndexType: "none", SingleFlight: &SingleFlight{ Etcd: &Etcd{"localhost:2379,localhost:22379,localhost:32379"}, Redis: &Redis{"127.0.0.1:6379", ""}, @@ -170,6 +173,31 @@ func defaultConfig() *Config { SentinelPassword: "sekret", }, }, + Index: &Index{ + MySQL: &MySQL{ + Protocol: "tcp", + Host: "localhost", + Port: 3306, + User: "root", + Password: "", + Database: "athens", + Params: map[string]string{ + "parseTime": "true", + "timeout": "30s", + }, + }, + Postgres: &Postgres{ + Host: "localhost", + Port: 5432, + User: "postgres", + Password: "", + Database: "athens", + Params: map[string]string{ + "connect_timeout": "30", + "sslmode": "disable", + }, + }, + }, } } @@ -267,29 +295,54 @@ func ensurePortFormat(s string) string { func validateConfig(config Config) error { validate := validator.New() - err := validate.StructExcept(config, "Storage") + err := validate.StructExcept(config, "Storage", "Index") if err != nil { return err } - switch config.StorageType { + err = validateStorage(validate, config.StorageType, config.Storage) + if err != nil { + return err + } + err = validateIndex(validate, config.IndexType, config.Index) + if err != nil { + return err + } + return nil +} + +func validateStorage(validate *validator.Validate, storageType string, config *Storage) error { + switch storageType { case "memory": return nil case "mongo": - return validate.Struct(config.Storage.Mongo) + return validate.Struct(config.Mongo) case "disk": - return validate.Struct(config.Storage.Disk) + return validate.Struct(config.Disk) case "minio": - return validate.Struct(config.Storage.Minio) + return validate.Struct(config.Minio) case "gcp": - return validate.Struct(config.Storage.GCP) + return validate.Struct(config.GCP) case "s3": - return validate.Struct(config.Storage.S3) + return validate.Struct(config.S3) case "azureblob": - return validate.Struct(config.Storage.AzureBlob) + return validate.Struct(config.AzureBlob) case "external": - return validate.Struct(config.Storage.External) + return validate.Struct(config.External) default: - return fmt.Errorf("storage type %s is unknown", config.StorageType) + return fmt.Errorf("storage type %q is unknown", storageType) + } +} + +func validateIndex(validate *validator.Validate, indexType string, config *Index) error { + switch indexType { + case "", "none", "memory": + return nil + case "mysql": + return validate.Struct(config.MySQL) + case "postgres": + return validate.Struct(config.Postgres) + default: + return fmt.Errorf("index type %q is unknown", indexType) } } diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 3c06a9da..3592a2cb 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -24,14 +24,14 @@ func testConfigFile(t *testing.T) (testConfigFile string) { } func compareConfigs(parsedConf *Config, expConf *Config, t *testing.T) { - opts := cmpopts.IgnoreTypes(StorageConfig{}, SingleFlight{}) + opts := cmpopts.IgnoreTypes(Storage{}, SingleFlight{}, Index{}) eq := cmp.Equal(parsedConf, expConf, opts) if !eq { t.Errorf("Parsed Example configuration did not match expected values. Expected: %+v. Actual: %+v", expConf, parsedConf) } } -func compareStorageConfigs(parsedStorage *StorageConfig, expStorage *StorageConfig, t *testing.T) { +func compareStorageConfigs(parsedStorage *Storage, expStorage *Storage, t *testing.T) { eq := cmp.Equal(parsedStorage.Mongo, expStorage.Mongo) if !eq { t.Errorf("Parsed Example Storage configuration did not match expected values. Expected: %+v. Actual: %+v", expStorage.Mongo, parsedStorage.Mongo) @@ -91,10 +91,11 @@ func TestEnvOverrides(t *testing.T) { PathPrefix: "prefix", NETRCPath: "/test/path/.netrc", HGRCPath: "/test/path/.hgrc", - Storage: &StorageConfig{}, + Storage: &Storage{}, GoBinaryEnvVars: []string{"GOPROXY=direct"}, SingleFlight: &SingleFlight{}, RobotsFile: "robots.txt", + Index: &Index{}, } envVars := getEnvMap(expConf) @@ -157,7 +158,7 @@ func TestEnsurePortFormat(t *testing.T) { } func TestStorageEnvOverrides(t *testing.T) { - expStorage := &StorageConfig{ + expStorage := &Storage{ Disk: &DiskConfig{ RootPath: "/my/root/path", }, @@ -209,7 +210,7 @@ func TestParseExampleConfig(t *testing.T) { // initialize all struct pointers so we get all applicable env variables emptyConf := &Config{ - Storage: &StorageConfig{ + Storage: &Storage{ Disk: &DiskConfig{}, GCP: &GCPConfig{}, Minio: &MinioConfig{ @@ -219,6 +220,7 @@ func TestParseExampleConfig(t *testing.T) { S3: &S3Config{}, }, SingleFlight: &SingleFlight{}, + Index: &Index{}, } // unset all environment variables envVars := getEnvMap(emptyConf) @@ -229,7 +231,7 @@ func TestParseExampleConfig(t *testing.T) { os.Unsetenv(k) } - expStorage := &StorageConfig{ + expStorage := &Storage{ Disk: &DiskConfig{ RootPath: "/path/on/disk", }, @@ -289,6 +291,8 @@ func TestParseExampleConfig(t *testing.T) { NoSumPatterns: []string{}, DownloadMode: "sync", RobotsFile: "robots.txt", + IndexType: "none", + Index: &Index{}, } absPath, err := filepath.Abs(testConfigFile(t)) @@ -477,7 +481,7 @@ func TestDefaultConfigMatchesConfigFile(t *testing.T) { defConf := defaultConfig() - ignoreStorageOpts := cmpopts.IgnoreTypes(&StorageConfig{}) + ignoreStorageOpts := cmpopts.IgnoreTypes(&Storage{}, &Index{}) ignoreGoEnvOpts := cmpopts.IgnoreFields(Config{}, "GoEnv") eq := cmp.Equal(defConf, parsedConf, ignoreStorageOpts, ignoreGoEnvOpts) if !eq { diff --git a/pkg/config/index.go b/pkg/config/index.go new file mode 100644 index 00000000..ed050ea9 --- /dev/null +++ b/pkg/config/index.go @@ -0,0 +1,7 @@ +package config + +// Index is the config for various index storage backends +type Index struct { + MySQL *MySQL + Postgres *Postgres +} diff --git a/pkg/config/mysql.go b/pkg/config/mysql.go new file mode 100644 index 00000000..3234bc10 --- /dev/null +++ b/pkg/config/mysql.go @@ -0,0 +1,12 @@ +package config + +// MySQL config +type MySQL struct { + Protocol string `validate:"required" envconfig:"ATHENS_INDEX_MYSQL_PROTOCOL"` + Host string `validate:"required" envconfig:"ATHENS_INDEX_MYSQL_HOST"` + Port int `validate:"" envconfig:"ATHENS_INDEX_MYSQL_PORT"` + User string `validate:"required" envconfig:"ATHENS_INDEX_MYSQL_USER"` + Password string `validate:"" envconfig:"ATHENS_INDEX_MYSQL_PASSWORD"` + Database string `validate:"required" envconfig:"ATHENS_INDEX_MYSQL_DATABASE"` + Params map[string]string `validate:"required" envconfig:"ATHENS_INDEX_MYSQL_PARAMS"` +} diff --git a/pkg/config/postgres.go b/pkg/config/postgres.go new file mode 100644 index 00000000..4a4c07fc --- /dev/null +++ b/pkg/config/postgres.go @@ -0,0 +1,11 @@ +package config + +// Postgres config +type Postgres struct { + Host string `validate:"required" envconfig:"ATHENS_INDEX_POSTGRES_HOST"` + Port int `validate:"required" envconfig:"ATHENS_INDEX_POSTGRES_PORT"` + User string `validate:"required" envconfig:"ATHENS_INDEX_POSTGRES_USER"` + Password string `validate:"" envconfig:"ATHENS_INDEX_POSTGRES_PASSWORD"` + Database string `validate:"required" envconfig:"ATHENS_INDEX_POSTGRES_DATABASE"` + Params map[string]string `validate:"required" envconfig:"ATHENS_INDEX_POSTGRES_PARAMS"` +} diff --git a/pkg/config/storage.go b/pkg/config/storage.go index 749977bd..9dd8ffa9 100644 --- a/pkg/config/storage.go +++ b/pkg/config/storage.go @@ -1,7 +1,7 @@ package config -// StorageConfig provides configs for various storage backends -type StorageConfig struct { +// Storage provides configs for various storage backends +type Storage struct { Disk *DiskConfig GCP *GCPConfig Minio *MinioConfig diff --git a/pkg/download/protocol_test.go b/pkg/download/protocol_test.go index 068e2887..3c307190 100644 --- a/pkg/download/protocol_test.go +++ b/pkg/download/protocol_test.go @@ -15,6 +15,7 @@ import ( "github.com/gomods/athens/pkg/config" "github.com/gomods/athens/pkg/download/mode" "github.com/gomods/athens/pkg/errors" + "github.com/gomods/athens/pkg/index/nop" "github.com/gomods/athens/pkg/module" "github.com/gomods/athens/pkg/stash" "github.com/gomods/athens/pkg/storage" @@ -44,7 +45,7 @@ func getDP(t *testing.T) Protocol { if err != nil { t.Fatal(err) } - st := stash.New(mf, s) + st := stash.New(mf, s, nop.New()) return New(&Opts{s, st, module.NewVCSLister(goBin, conf.GoBinaryEnvVars, fs), nil}) } @@ -280,7 +281,7 @@ func TestDownloadProtocol(t *testing.T) { t.Fatal(err) } mp := &mockFetcher{} - st := stash.New(mp, s) + st := stash.New(mp, s, nop.New()) dp := New(&Opts{s, st, nil, nil}) ctx := context.Background() @@ -332,7 +333,7 @@ func TestDownloadProtocolWhenFetchFails(t *testing.T) { t.Fatal(err) } mp := ¬FoundFetcher{} - st := stash.New(mp, s) + st := stash.New(mp, s, nop.New()) dp := New(&Opts{s, st, nil, nil}) ctx := context.Background() _, err = dp.GoMod(ctx, fakeMod.mod, fakeMod.ver) diff --git a/pkg/index/compliance/compliance.go b/pkg/index/compliance/compliance.go new file mode 100644 index 00000000..6c1fdf1b --- /dev/null +++ b/pkg/index/compliance/compliance.go @@ -0,0 +1,118 @@ +package compliance + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/gomods/athens/pkg/index" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/technosophos/moniker" +) + +func RunTests(t *testing.T, indexer index.Indexer, clearIndex func() error) { + if err := clearIndex(); err != nil { + t.Fatal(err) + } + + var tests = []struct { + name string + desc string + limit int + preTest func(t *testing.T) ([]*index.Line, time.Time) + }{ + { + name: "empty", + desc: "an empty index should return an empty slice", + preTest: func(t *testing.T) ([]*index.Line, time.Time) { return []*index.Line{}, time.Time{} }, + limit: 2000, + }, + { + name: "happy path", + desc: "given 10 modules, return all of them in correct order", + preTest: func(t *testing.T) ([]*index.Line, time.Time) { + return seed(t, indexer, 10), time.Time{} + }, + limit: 2000, + }, + { + name: "respect the limit", + desc: "givn 10 modules and a 'limit' of 5, only return the first five lines", + preTest: func(t *testing.T) ([]*index.Line, time.Time) { + lines := seed(t, indexer, 10) + return lines[0:5], time.Time{} + }, + limit: 5, + }, + { + name: "respect the time", + desc: "given 10 modules, 'since' should filter out the ones that came before it", + preTest: func(t *testing.T) ([]*index.Line, time.Time) { + err := indexer.Index(context.Background(), "tobeignored", "v1.2.3") + if err != nil { + t.Fatal(err) + } + time.Sleep(50 * time.Millisecond) + now := time.Now() + lines := seed(t, indexer, 5) + return lines, now + }, + limit: 2000, + }, + { + name: "ignore the past", + desc: "no line should be returned if 'since' is after all of the indexed modules", + preTest: func(t *testing.T) ([]*index.Line, time.Time) { + seed(t, indexer, 5) + time.Sleep(50 * time.Millisecond) + return []*index.Line{}, time.Now() + }, + limit: 2000, + }, + { + name: "no limit no line", + desc: "if limit is set to zero, then nothing should be returned", + preTest: func(t *testing.T) ([]*index.Line, time.Time) { + seed(t, indexer, 5) + return []*index.Line{}, time.Time{} + }, + limit: 0, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Log(tc.desc) + t.Cleanup(func() { + if err := clearIndex(); err != nil { + t.Fatal(err) + } + }) + expected, since := tc.preTest(t) + given, err := indexer.Lines(context.Background(), since, tc.limit) + if err != nil { + t.Fatal(err) + } + opts := cmpopts.IgnoreFields(index.Line{}, "Timestamp") + if !cmp.Equal(given, expected, opts) { + t.Fatal(cmp.Diff(expected, given, opts)) + } + }) + } +} + +func seed(t *testing.T, indexer index.Indexer, num int) []*index.Line { + lines := []*index.Line{} + t.Helper() + for i := 0; i < num; i++ { + mod := moniker.New().NameSep("_") + ver := fmt.Sprintf("%d.0.0", i) + err := indexer.Index(context.Background(), mod, ver) + if err != nil { + t.Fatal(err) + } + lines = append(lines, &index.Line{Path: mod, Version: ver}) + } + return lines +} diff --git a/pkg/index/indexer.go b/pkg/index/indexer.go new file mode 100644 index 00000000..395aa0c5 --- /dev/null +++ b/pkg/index/indexer.go @@ -0,0 +1,26 @@ +package index + +import ( + "context" + "time" +) + +// Line represents a module@version line +// with its metadata such as creation time. +type Line struct { + Path, Version string + Timestamp time.Time +} + +// Indexer is an interface that can process new module@versions +// and also retrieve 'limit' module@versions that were indexed after 'since' +type Indexer interface { + // Index stores the module@version into the index backend. + // Implementer must create the Timestamp at the time and set it + // to the time this method is call. + Index(ctx context.Context, mod, ver string) error + + // Lines returns the module@version lines given the time and limit + // constraints + Lines(ctx context.Context, since time.Time, limit int) ([]*Line, error) +} diff --git a/pkg/index/mem/mem.go b/pkg/index/mem/mem.go new file mode 100644 index 00000000..4ffb5ffc --- /dev/null +++ b/pkg/index/mem/mem.go @@ -0,0 +1,51 @@ +package mem + +import ( + "context" + "sync" + "time" + + "github.com/gomods/athens/pkg/errors" + "github.com/gomods/athens/pkg/index" +) + +// New returns a new in-memory indexer +func New() index.Indexer { + return &indexer{} +} + +type indexer struct { + mu sync.RWMutex + lines []*index.Line +} + +func (i *indexer) Index(ctx context.Context, mod, ver string) error { + const op errors.Op = "mem.Index" + i.mu.Lock() + i.lines = append(i.lines, &index.Line{ + Path: mod, + Version: ver, + Timestamp: time.Now(), + }) + i.mu.Unlock() + return nil +} + +func (i *indexer) Lines(ctx context.Context, since time.Time, limit int) ([]*index.Line, error) { + const op errors.Op = "mem.Lines" + lines := []*index.Line{} + var count int + i.mu.RLock() + defer i.mu.RUnlock() + for _, line := range i.lines { + if count >= limit { + break + } + if since.After(line.Timestamp) { + continue + } + lines = append(lines, line) + count++ + } + return lines, nil +} diff --git a/pkg/index/mem/mem_test.go b/pkg/index/mem/mem_test.go new file mode 100644 index 00000000..657effdf --- /dev/null +++ b/pkg/index/mem/mem_test.go @@ -0,0 +1,20 @@ +package mem + +import ( + "testing" + + "github.com/gomods/athens/pkg/index" + "github.com/gomods/athens/pkg/index/compliance" +) + +func TestMem(t *testing.T) { + indexer := &indexer{} + compliance.RunTests(t, indexer, indexer.clear) +} + +func (i *indexer) clear() error { + i.mu.Lock() + i.lines = []*index.Line{} + i.mu.Unlock() + return nil +} diff --git a/pkg/index/mysql/mysql.go b/pkg/index/mysql/mysql.go new file mode 100644 index 00000000..1ca85956 --- /dev/null +++ b/pkg/index/mysql/mysql.go @@ -0,0 +1,105 @@ +package mysql + +import ( + "context" + "database/sql" + "fmt" + "time" + + "github.com/go-sql-driver/mysql" + "github.com/gomods/athens/pkg/config" + "github.com/gomods/athens/pkg/errors" + "github.com/gomods/athens/pkg/index" +) + +func New(cfg *config.MySQL) (index.Indexer, error) { + dataSource := getMySQLSource(cfg) + db, err := sql.Open("mysql", dataSource) + if err != nil { + return nil, err + } + if err = db.Ping(); err != nil { + return nil, err + } + _, err = db.Exec(schema) + if err != nil { + return nil, err + } + return &indexer{db}, nil +} + +const schema = ` + CREATE TABLE IF NOT EXISTS indexes( + id INT + AUTO_INCREMENT + PRIMARY KEY + COMMENT 'Unique identifier for a module line', + + path VARCHAR(255) + NOT NULL + COMMENT 'Import path of the module', + + version VARCHAR(255) + NOT NULL + COMMENT 'Module version', + + timestamp TIMESTAMP(6) + COMMENT 'Date and time when the module was first created', + + INDEX (timestamp), + UNIQUE INDEX idx_module_version (path, version) + ) CHARACTER SET utf8; +` + +type indexer struct { + db *sql.DB +} + +func (i *indexer) Index(ctx context.Context, mod, ver string) error { + const op errors.Op = "mysql.Index" + _, err := i.db.ExecContext( + ctx, + `INSERT INTO indexes (path, version, timestamp) VALUES (?, ?, ?)`, + mod, + ver, + time.Now().Format(time.RFC3339Nano), + ) + if err != nil { + return errors.E(op, err) + } + return nil +} + +func (i *indexer) Lines(ctx context.Context, since time.Time, limit int) ([]*index.Line, error) { + const op errors.Op = "mysql.Lines" + if since.IsZero() { + since = time.Unix(0, 0) + } + sinceStr := since.Format(time.RFC3339Nano) + rows, err := i.db.QueryContext(ctx, `SELECT path, version, timestamp FROM indexes WHERE timestamp >= ? LIMIT ?`, sinceStr, limit) + if err != nil { + return nil, errors.E(op, err) + } + defer rows.Close() + lines := []*index.Line{} + for rows.Next() { + var line index.Line + err = rows.Scan(&line.Path, &line.Version, &line.Timestamp) + if err != nil { + return nil, errors.E(op, err) + } + lines = append(lines, &line) + } + return lines, nil +} + +func getMySQLSource(cfg *config.MySQL) string { + c := mysql.NewConfig() + c.Net = cfg.Protocol + c.Addr = fmt.Sprintf("%s:%d", cfg.Host, cfg.Port) + c.User = cfg.User + c.Passwd = cfg.Password + c.DBName = cfg.Database + c.Params = cfg.Params + return c.FormatDSN() +} diff --git a/pkg/index/mysql/mysql_test.go b/pkg/index/mysql/mysql_test.go new file mode 100644 index 00000000..93fcd9cf --- /dev/null +++ b/pkg/index/mysql/mysql_test.go @@ -0,0 +1,35 @@ +package mysql + +import ( + "os" + "testing" + + "github.com/gomods/athens/pkg/config" + "github.com/gomods/athens/pkg/index/compliance" +) + +func TestMySQL(t *testing.T) { + if os.Getenv("TEST_INDEX_MYSQL") != "true" { + t.SkipNow() + } + cfg := getTestConfig(t) + i, err := New(cfg) + if err != nil { + t.Fatal(err) + } + compliance.RunTests(t, i, i.(*indexer).clear) +} + +func (i *indexer) clear() error { + _, err := i.db.Exec(`DELETE FROM indexes`) + return err +} + +func getTestConfig(t *testing.T) *config.MySQL { + t.Helper() + cfg, err := config.Load("") + if err != nil { + t.Fatal(err) + } + return cfg.Index.MySQL +} diff --git a/pkg/index/nop/nop.go b/pkg/index/nop/nop.go new file mode 100644 index 00000000..5140d249 --- /dev/null +++ b/pkg/index/nop/nop.go @@ -0,0 +1,22 @@ +package nop + +import ( + "context" + "time" + + "github.com/gomods/athens/pkg/index" +) + +// New returns a no-op Indexer +func New() index.Indexer { + return indexer{} +} + +type indexer struct{} + +func (indexer) Index(ctx context.Context, mod, ver string) error { + return nil +} +func (indexer) Lines(ctx context.Context, since time.Time, limit int) ([]*index.Line, error) { + return []*index.Line{}, nil +} diff --git a/pkg/index/postgres/postgres.go b/pkg/index/postgres/postgres.go new file mode 100644 index 00000000..2f3c0b5a --- /dev/null +++ b/pkg/index/postgres/postgres.go @@ -0,0 +1,106 @@ +package postgres + +import ( + "context" + "database/sql" + "strconv" + "strings" + "time" + + // register the driver with database/sql + _ "github.com/lib/pq" + + "github.com/gomods/athens/pkg/config" + "github.com/gomods/athens/pkg/errors" + "github.com/gomods/athens/pkg/index" +) + +func New(cfg *config.Postgres) (index.Indexer, error) { + dataSource := getPostgresSource(cfg) + db, err := sql.Open("postgres", dataSource) + if err != nil { + return nil, err + } + if err = db.Ping(); err != nil { + return nil, err + } + for _, statement := range schema { + _, err = db.Exec(statement) + if err != nil { + return nil, err + } + } + return &indexer{db}, nil +} + +var schema = [...]string{ + ` + CREATE TABLE IF NOT EXISTS indexes( + id SERIAL PRIMARY KEY, + path VARCHAR(255) NOT NULL, + version VARCHAR(255) NOT NULL, + timestamp timestamp NOT NULL + ) + `, + ` + CREATE INDEX IF NOT EXISTS idx_timestamp ON indexes (timestamp) + `, + ` + CREATE UNIQUE INDEX IF NOT EXISTS idx_module_version ON indexes (path, version) + `, +} + +type indexer struct { + db *sql.DB +} + +func (i *indexer) Index(ctx context.Context, mod, ver string) error { + const op errors.Op = "postgres.Index" + _, err := i.db.ExecContext( + ctx, + `INSERT INTO indexes (path, version, timestamp) VALUES ($1, $2, $3)`, + mod, + ver, + time.Now().Format(time.RFC3339Nano), + ) + if err != nil { + return errors.E(op, err) + } + return nil +} + +func (i *indexer) Lines(ctx context.Context, since time.Time, limit int) ([]*index.Line, error) { + const op errors.Op = "postgres.Lines" + if since.IsZero() { + since = time.Unix(0, 0) + } + sinceStr := since.Format(time.RFC3339Nano) + rows, err := i.db.QueryContext(ctx, `SELECT path, version, timestamp FROM indexes WHERE timestamp >= $1 LIMIT $2`, sinceStr, limit) + if err != nil { + return nil, errors.E(op, err) + } + defer rows.Close() + lines := []*index.Line{} + for rows.Next() { + var line index.Line + err = rows.Scan(&line.Path, &line.Version, &line.Timestamp) + if err != nil { + return nil, errors.E(op, err) + } + lines = append(lines, &line) + } + return lines, nil +} + +func getPostgresSource(cfg *config.Postgres) string { + args := []string{} + args = append(args, "host="+cfg.Host) + args = append(args, "port=", strconv.Itoa(cfg.Port)) + args = append(args, "user=", cfg.User) + args = append(args, "dbname=", cfg.Database) + args = append(args, "password="+cfg.Password) + for k, v := range cfg.Params { + args = append(args, k+"="+v) + } + return strings.Join(args, " ") +} diff --git a/pkg/index/postgres/postgres_test.go b/pkg/index/postgres/postgres_test.go new file mode 100644 index 00000000..287877d6 --- /dev/null +++ b/pkg/index/postgres/postgres_test.go @@ -0,0 +1,35 @@ +package postgres + +import ( + "os" + "testing" + + "github.com/gomods/athens/pkg/config" + "github.com/gomods/athens/pkg/index/compliance" +) + +func TestPostgres(t *testing.T) { + if os.Getenv("TEST_INDEX_POSTGRES") != "true" { + t.SkipNow() + } + cfg := getTestConfig(t) + i, err := New(cfg) + if err != nil { + t.Fatal(err) + } + compliance.RunTests(t, i, i.(*indexer).clear) +} + +func (i *indexer) clear() error { + _, err := i.db.Exec(`DELETE FROM indexes`) + return err +} + +func getTestConfig(t *testing.T) *config.Postgres { + t.Helper() + cfg, err := config.Load("") + if err != nil { + t.Fatal(err) + } + return cfg.Index.Postgres +} diff --git a/pkg/stash/stasher.go b/pkg/stash/stasher.go index 4a35fc85..8d567cb5 100644 --- a/pkg/stash/stasher.go +++ b/pkg/stash/stasher.go @@ -5,6 +5,7 @@ import ( "time" "github.com/gomods/athens/pkg/errors" + "github.com/gomods/athens/pkg/index" "github.com/gomods/athens/pkg/log" "github.com/gomods/athens/pkg/module" "github.com/gomods/athens/pkg/observ" @@ -13,7 +14,7 @@ import ( ) // Stasher has the job of taking a module -// from an upstream entity and stashing it to a Storage Backend. +// from an upstream entity and stashing it to a Storage Backend and Index. // It also returns a string that represents a semver version of // what was requested, this is helpful if what was requested // was a descriptive version such as a branch name or a full commit sha. @@ -27,8 +28,8 @@ type Wrapper func(Stasher) Stasher // New returns a plain stasher that takes // a module from a download.Protocol and // stashes it into a backend.Storage. -func New(f module.Fetcher, s storage.Backend, wrappers ...Wrapper) Stasher { - var st Stasher = &stasher{f, s, storage.WithChecker(s)} +func New(f module.Fetcher, s storage.Backend, indexer index.Indexer, wrappers ...Wrapper) Stasher { + var st Stasher = &stasher{f, s, storage.WithChecker(s), indexer} for _, w := range wrappers { st = w(st) } @@ -40,6 +41,7 @@ type stasher struct { fetcher module.Fetcher storage storage.Backend checker storage.Checker + indexer index.Indexer } func (s *stasher) Stash(ctx context.Context, mod, ver string) (string, error) { @@ -71,6 +73,10 @@ func (s *stasher) Stash(ctx context.Context, mod, ver string) (string, error) { if err != nil { return "", errors.E(op, err) } + err = s.indexer.Index(ctx, mod, v.Semver) + if err != nil { + return "", errors.E(op, err) + } return v.Semver, nil } @@ -80,6 +86,5 @@ func (s *stasher) fetchModule(ctx context.Context, mod, ver string) (*storage.Ve if err != nil { return nil, errors.E(op, err) } - return v, nil } diff --git a/pkg/stash/stasher_test.go b/pkg/stash/stasher_test.go index ae406e83..0f53b788 100644 --- a/pkg/stash/stasher_test.go +++ b/pkg/stash/stasher_test.go @@ -7,6 +7,7 @@ import ( "strings" "testing" + "github.com/gomods/athens/pkg/index/nop" "github.com/gomods/athens/pkg/storage" ) @@ -54,7 +55,7 @@ func TestStash(t *testing.T) { var mf mockFetcher mf.ver = testCase.modVer - s := New(&mf, &ms) + s := New(&mf, &ms, nop.New()) newVersion, err := s.Stash(context.Background(), "module", testCase.ver) if err != nil { t.Fatal(err)