diff --git a/.travis.yml b/.travis.yml index afdf7c8a..73313c90 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,16 +1,20 @@ services: - mongodb + - postgresql language: go install: false go: - "1.10.x" -services: - - mongodb -before_script: - - GO_FILES=$(find . -iname '*.go' -type f | grep -v /vendor/) # All the .go files, excluding vendor/ - - go get github.com/golang/lint/golint # Linter script: - test -z $(gofmt -s -l $GO_FILES) # Fail if a .go file hasn't been formatted with gofmt - go vet ./... # Go static analyzer - - golint -set_exit_status $(go list ./...) # Linter + - golint -set_exit_status $(go list ./...) # Linter - go test -race ./... # Run all the tests with the race detector enabled +before_script: + - GO_FILES=$(find . -iname '*.go' -type f | grep -v /vendor/) # All the .go files, excluding vendor/ + - go get github.com/golang/lint/golint + - export POP_PATH=$PWD/cmd/proxy + - export GO_ENV=test_postgres + - go get -u -v github.com/gobuffalo/buffalo/buffalo + - buffalo db create + - buffalo db migrate up diff --git a/cmd/olympus/actions/storage.go b/cmd/olympus/actions/storage.go index dfd2c33d..bb85e91f 100644 --- a/cmd/olympus/actions/storage.go +++ b/cmd/olympus/actions/storage.go @@ -7,6 +7,7 @@ import ( "github.com/gomods/athens/pkg/storage" "github.com/gomods/athens/pkg/storage/fs" "github.com/gomods/athens/pkg/storage/mongo" + "github.com/gomods/athens/pkg/storage/rdbms" "github.com/spf13/afero" ) @@ -32,6 +33,12 @@ func newStorage() (storage.Backend, error) { return nil, fmt.Errorf("missing mongo URL (%s)", err) } return mongo.NewStorage(mongoURI), nil + case "postgres", "sqlite", "cockroach", "mysql": + connectionName, err := envy.MustGet("ATHENS_RDBMS_STORAGE_NAME") + if err != nil { + return nil, fmt.Errorf("missing rdbms connectionName (%s)", err) + } + return rdbms.NewRDBMSStorage(connectionName), nil default: return nil, fmt.Errorf("storage type %s is unknown", storageType) } diff --git a/cmd/olympus/database.yml b/cmd/olympus/database.yml index 00e0dd02..81113e82 100644 --- a/cmd/olympus/database.yml +++ b/cmd/olympus/database.yml @@ -6,6 +6,14 @@ development: user: vgp password: vgp +test_postgres: + dialect: "postgres" + database: athens_development + user: postgres + password: '' + host: 127.0.0.1 + pool: 5 + test: dialect: "mysql" database: olympusdb diff --git a/cmd/proxy/actions/app.go b/cmd/proxy/actions/app.go index 86556005..559f6625 100644 --- a/cmd/proxy/actions/app.go +++ b/cmd/proxy/actions/app.go @@ -130,6 +130,11 @@ func getStorage() (storage.Backend, error) { if err != nil { return nil, fmt.Errorf("missing disk storage root (%s)", err) } + case "postgres", "sqlite", "cockroach", "mysql": + storageRoot, err = envy.MustGet("ATHENS_RDBMS_STORAGE_NAME") + if err != nil { + return nil, fmt.Errorf("missing rdbms connectionName (%s)", err) + } } return newStorage(storageType, storageRoot) diff --git a/cmd/proxy/actions/storage.go b/cmd/proxy/actions/storage.go index 773d5f42..a96997ee 100644 --- a/cmd/proxy/actions/storage.go +++ b/cmd/proxy/actions/storage.go @@ -6,6 +6,7 @@ import ( "github.com/gomods/athens/pkg/storage" "github.com/gomods/athens/pkg/storage/fs" "github.com/gomods/athens/pkg/storage/mongo" + "github.com/gomods/athens/pkg/storage/rdbms" "github.com/spf13/afero" ) @@ -22,6 +23,8 @@ func newStorage(storageType, storageLocation string) (storage.Backend, error) { return fs.NewStorage(storageLocation, afero.NewOsFs()), nil case "mongo": return mongo.NewStorage(storageLocation), nil + case "postgres", "sqlite", "cockroach", "mysql": + return rdbms.NewRDBMSStorage(storageLocation), nil default: return nil, fmt.Errorf("storage type %s is unknown", storageType) } diff --git a/cmd/proxy/database.yml b/cmd/proxy/database.yml index 00e0dd02..041166ac 100644 --- a/cmd/proxy/database.yml +++ b/cmd/proxy/database.yml @@ -13,7 +13,15 @@ test: port: 3306 user: vgp password: vgp - + +test_postgres: + dialect: "postgres" + database: athens_development + user: postgres + password: '' + host: 127.0.0.1 + pool: 5 + production: dialect: "mysql" database: olympusdb diff --git a/migrations/20180326213718_create_modules.down.fizz b/migrations/20180326213718_create_modules.down.fizz new file mode 100644 index 00000000..ce9dc011 --- /dev/null +++ b/migrations/20180326213718_create_modules.down.fizz @@ -0,0 +1 @@ +drop_table("modules") \ No newline at end of file diff --git a/migrations/20180326213718_create_modules.up.fizz b/migrations/20180326213718_create_modules.up.fizz new file mode 100644 index 00000000..e94d2966 --- /dev/null +++ b/migrations/20180326213718_create_modules.up.fizz @@ -0,0 +1,7 @@ +create_table("modules", func(t) { + t.Column("id", "uuid", {"primary": true}) + t.Column("module", "text", {}) + t.Column("version", "text", {}) + t.Column("mod", "blob", {}) + t.Column("zip", "blob", {}) +}) \ No newline at end of file diff --git a/pkg/storage/rdbms/all_test.go b/pkg/storage/rdbms/all_test.go new file mode 100644 index 00000000..30d2e5b2 --- /dev/null +++ b/pkg/storage/rdbms/all_test.go @@ -0,0 +1,39 @@ +package rdbms + +import ( + "testing" + + "github.com/gobuffalo/suite" + "github.com/gomods/athens/pkg/storage" +) + +const ( + module = "testmodule" + version = "v1.0.0" +) + +var ( + // TODO: put these values inside of the suite, and generate longer values. + // This should help catch edge cases, like https://github.com/gomods/athens/issues/38 + // + // Also, consider doing something similar to what testing/quick does + // with the Generator interface (https://godoc.org/testing/quick#Generator). + // The rough, simplified idea would be to run a single test case multiple + // times over different (increasing) values. + mod = []byte("123") + zip = []byte("456") +) + +type RDBMSTestSuite struct { + *suite.Model + storage storage.BackendConnector +} + +func (rd *RDBMSTestSuite) SetupTest() { + rd.storage = &ModuleStore{conn: rd.DB} + rd.Model.SetupTest() +} + +func Test_ActionSuite(t *testing.T) { + suite.Run(t, &RDBMSTestSuite{Model: suite.NewModel()}) +} diff --git a/pkg/storage/rdbms/getter.go b/pkg/storage/rdbms/getter.go new file mode 100644 index 00000000..62f05e96 --- /dev/null +++ b/pkg/storage/rdbms/getter.go @@ -0,0 +1,29 @@ +package rdbms + +import ( + "bytes" + "io/ioutil" + "time" + + "github.com/gomods/athens/pkg/storage" + "github.com/gomods/athens/pkg/storage/rdbms/models" +) + +// Get a specific version of a module +func (r *ModuleStore) Get(module, vsn string) (*storage.Version, error) { + result := models.Module{} + query := r.conn.Where("module = ?", module).Where("version = ?", vsn) + if err := query.First(&result); err != nil { + return nil, err + } + return &storage.Version{ + RevInfo: storage.RevInfo{ + Version: result.Version, + Name: result.Version, + Short: result.Version, + Time: time.Now(), + }, + Mod: result.Mod, + Zip: ioutil.NopCloser(bytes.NewReader(result.Zip)), + }, nil +} diff --git a/pkg/storage/rdbms/getter_test.go b/pkg/storage/rdbms/getter_test.go new file mode 100644 index 00000000..157be367 --- /dev/null +++ b/pkg/storage/rdbms/getter_test.go @@ -0,0 +1,3 @@ +package rdbms + +// see rdbms_test.go for a round-trip test that subsumes tests for the saver diff --git a/pkg/storage/rdbms/lister.go b/pkg/storage/rdbms/lister.go new file mode 100644 index 00000000..1e0fc39a --- /dev/null +++ b/pkg/storage/rdbms/lister.go @@ -0,0 +1,21 @@ +package rdbms + +import ( + "github.com/gomods/athens/pkg/storage/rdbms/models" +) + +// List lists all versions of a module +func (r *ModuleStore) List(module string) ([]string, error) { + result := make([]models.Module, 0) + err := r.conn.Where("module = ?", module).All(&result) + if err != nil { + return nil, err + } + + versions := make([]string, len(result)) + for i := range result { + versions[i] = result[i].Version + } + + return versions, nil +} diff --git a/pkg/storage/rdbms/lister_test.go b/pkg/storage/rdbms/lister_test.go new file mode 100644 index 00000000..99e3ed68 --- /dev/null +++ b/pkg/storage/rdbms/lister_test.go @@ -0,0 +1,12 @@ +package rdbms + +func (rd *RDBMSTestSuite) TestList() { + r := rd.Require() + versions := []string{"v1.0.0", "v1.1.0", "v1.2.0"} + for _, version := range versions { + rd.storage.Save(module, version, mod, zip) + } + retVersions, err := rd.storage.List(module) + r.NoError(err) + r.Equal(versions, retVersions) +} diff --git a/pkg/storage/rdbms/models/modules.go b/pkg/storage/rdbms/models/modules.go new file mode 100644 index 00000000..922ea267 --- /dev/null +++ b/pkg/storage/rdbms/models/modules.go @@ -0,0 +1,60 @@ +package models + +import ( + "encoding/json" + "time" + + "github.com/gobuffalo/pop" + "github.com/gobuffalo/uuid" + "github.com/gobuffalo/validate" + "github.com/gobuffalo/validate/validators" +) + +// Module is a model where data is stored. +type Module struct { + ID uuid.UUID `json:"id" db:"id"` + CreatedAt time.Time `json:"created_at" db:"created_at"` + UpdatedAt time.Time `json:"updated_at" db:"updated_at"` + Module string `json:"module" db:"module"` + Version string `json:"version" db:"version"` + Mod []byte `json:"mod" db:"mod"` + Zip []byte `json:"zip" db:"zip"` +} + +// String is not required by pop and may be deleted +func (m Module) String() string { + jm, _ := json.Marshal(m) + return string(jm) +} + +// Modules is not required by pop and may be deleted +type Modules []Module + +// String is not required by pop and may be deleted +func (m Modules) String() string { + jm, _ := json.Marshal(m) + return string(jm) +} + +// Validate gets run every time you call a "pop.Validate*" (pop.ValidateAndSave, pop.ValidateAndCreate, pop.ValidateAndUpdate) method. +// This method is not required and may be deleted. +func (m *Module) Validate(tx *pop.Connection) (*validate.Errors, error) { + return validate.Validate( + &validators.StringIsPresent{Field: m.Module, Name: "Module"}, + &validators.StringIsPresent{Field: m.Version, Name: "Version"}, + &validators.BytesArePresent{Field: m.Mod, Name: "Mod"}, + &validators.BytesArePresent{Field: m.Zip, Name: "Zip"}, + ), nil +} + +// ValidateCreate gets run every time you call "pop.ValidateAndCreate" method. +// This method is not required and may be deleted. +func (m *Module) ValidateCreate(tx *pop.Connection) (*validate.Errors, error) { + return validate.NewErrors(), nil +} + +// ValidateUpdate gets run every time you call "pop.ValidateAndUpdate" method. +// This method is not required and may be deleted. +func (m *Module) ValidateUpdate(tx *pop.Connection) (*validate.Errors, error) { + return validate.NewErrors(), nil +} diff --git a/pkg/storage/rdbms/rdbms.go b/pkg/storage/rdbms/rdbms.go new file mode 100644 index 00000000..46417392 --- /dev/null +++ b/pkg/storage/rdbms/rdbms.go @@ -0,0 +1,31 @@ +package rdbms + +import ( + "github.com/gobuffalo/pop" +) + +// ModuleStore represents a rdbms(postgres, mysql, sqlite, cockroachdb) backed storage backend. +type ModuleStore struct { + conn *pop.Connection + connectionName string // settings name from database.yml +} + +// NewRDBMSStorage returns an unconnected RDBMS Module Storage +// that satisfies the Storage interface. You must call +// Connect() on the returned store before using it. +// connectionName +func NewRDBMSStorage(connectionName string) *ModuleStore { + return &ModuleStore{ + connectionName: connectionName, + } +} + +// Connect creates connection to rdmbs backend. +func (r *ModuleStore) Connect() error { + c, err := pop.Connect(r.connectionName) + if err != nil { + return err + } + r.conn = c + return nil +} diff --git a/pkg/storage/rdbms/rdbms_test.go b/pkg/storage/rdbms/rdbms_test.go new file mode 100644 index 00000000..b7ca494e --- /dev/null +++ b/pkg/storage/rdbms/rdbms_test.go @@ -0,0 +1,37 @@ +package rdbms + +import ( + "io/ioutil" +) + +func (rd *RDBMSTestSuite) TestGetSaveListRoundTrip() { + r := rd.Require() + err := rd.storage.Save(module, version, mod, zip) + r.NoError(err) + listedVersions, err := rd.storage.List(module) + r.NoError(err) + r.Equal(1, len(listedVersions)) + retVersion := listedVersions[0] + r.Equal(version, retVersion) + gotten, err := rd.storage.Get(module, version) + r.NoError(err) + defer gotten.Zip.Close() + r.Equal(version, gotten.RevInfo.Version) + r.Equal(version, gotten.RevInfo.Name) + r.Equal(version, gotten.RevInfo.Short) + // TODO: test the time + r.Equal(gotten.Mod, mod) + zipContent, err := ioutil.ReadAll(gotten.Zip) + r.NoError(err) + r.Equal(zipContent, zip) +} + +func (rd *RDBMSTestSuite) TestNewRDBMSStorage() { + r := rd.Require() + e := "development" + getterSaver := NewRDBMSStorage(e) + getterSaver.Connect() + + r.NotNil(getterSaver.conn) + r.Equal(getterSaver.connectionName, e) +} diff --git a/pkg/storage/rdbms/saver.go b/pkg/storage/rdbms/saver.go new file mode 100644 index 00000000..9918cc31 --- /dev/null +++ b/pkg/storage/rdbms/saver.go @@ -0,0 +1,17 @@ +package rdbms + +import ( + "github.com/gomods/athens/pkg/storage/rdbms/models" +) + +// Save stores a module in rdbms storage. +func (r *ModuleStore) Save(module, version string, mod, zip []byte) error { + m := &models.Module{ + Module: module, + Version: version, + Mod: mod, + Zip: zip, + } + + return r.conn.Create(m) +} diff --git a/pkg/storage/rdbms/saver_test.go b/pkg/storage/rdbms/saver_test.go new file mode 100644 index 00000000..157be367 --- /dev/null +++ b/pkg/storage/rdbms/saver_test.go @@ -0,0 +1,3 @@ +package rdbms + +// see rdbms_test.go for a round-trip test that subsumes tests for the saver