Add rdbms backend (#74)

* Add rdbms backend

* Remove unused imports

* Fix ci

* Fix problems

* Now test works

* Fix lint errors

* Change name to more descriptive

* Add logic to support rdbms storage

* Update to new newStorage logic

.travis.yml

@@ -1,16 +1,20 @@
 services:
   - mongodb
+  - postgresql
 language: go
 install: false
 go:
   - "1.10.x"
-services:
-  - mongodb
-before_script:
-  - GO_FILES=$(find . -iname '*.go' -type f | grep -v /vendor/) # All the .go files, excluding vendor/
-  - go get github.com/golang/lint/golint                        # Linter
 script:
   - test -z $(gofmt -s -l $GO_FILES)         # Fail if a .go file hasn't been formatted with gofmt
   - go vet ./...                             # Go static analyzer
-  - golint -set_exit_status $(go list ./...) # Linter             
+  - golint -set_exit_status $(go list ./...) # Linter
   - go test -race ./...                      # Run all the tests with the race detector enabled
+before_script:
+  - GO_FILES=$(find . -iname '*.go' -type f | grep -v /vendor/) # All the .go files, excluding vendor/
+  - go get github.com/golang/lint/golint
+  - export POP_PATH=$PWD/cmd/proxy
+  - export GO_ENV=test_postgres
+  - go get -u -v github.com/gobuffalo/buffalo/buffalo
+  - buffalo db create
+  - buffalo db migrate up

cmd/olympus/actions/storage.go

@@ -7,6 +7,7 @@ import (
 	"github.com/gomods/athens/pkg/storage"
 	"github.com/gomods/athens/pkg/storage/fs"
 	"github.com/gomods/athens/pkg/storage/mongo"
+	"github.com/gomods/athens/pkg/storage/rdbms"
 	"github.com/spf13/afero"
 )
 
@@ -32,6 +33,12 @@ func newStorage() (storage.Backend, error) {
 			return nil, fmt.Errorf("missing mongo URL (%s)", err)
 		}
 		return mongo.NewStorage(mongoURI), nil
+	case "postgres", "sqlite", "cockroach", "mysql":
+		connectionName, err := envy.MustGet("ATHENS_RDBMS_STORAGE_NAME")
+		if err != nil {
+			return nil, fmt.Errorf("missing rdbms connectionName (%s)", err)
+		}
+		return rdbms.NewRDBMSStorage(connectionName), nil
 	default:
 		return nil, fmt.Errorf("storage type %s is unknown", storageType)
 	}

cmd/olympus/database.yml

@@ -6,6 +6,14 @@ development:
   user: vgp
   password: vgp
 
+test_postgres:
+  dialect: "postgres"
+  database: athens_development
+  user: postgres
+  password: ''
+  host: 127.0.0.1
+  pool: 5
+
 test:
   dialect: "mysql"
   database: olympusdb

cmd/proxy/actions/app.go

@@ -130,6 +130,11 @@ func getStorage() (storage.Backend, error) {
 		if err != nil {
 			return nil, fmt.Errorf("missing disk storage root (%s)", err)
 		}
+	case "postgres", "sqlite", "cockroach", "mysql":
+		storageRoot, err = envy.MustGet("ATHENS_RDBMS_STORAGE_NAME")
+		if err != nil {
+			return nil, fmt.Errorf("missing rdbms connectionName (%s)", err)
+		}
 	}
 
 	return newStorage(storageType, storageRoot)

cmd/proxy/actions/storage.go

@@ -6,6 +6,7 @@ import (
 	"github.com/gomods/athens/pkg/storage"
 	"github.com/gomods/athens/pkg/storage/fs"
 	"github.com/gomods/athens/pkg/storage/mongo"
+	"github.com/gomods/athens/pkg/storage/rdbms"
 	"github.com/spf13/afero"
 )
 
@@ -22,6 +23,8 @@ func newStorage(storageType, storageLocation string) (storage.Backend, error) {
 		return fs.NewStorage(storageLocation, afero.NewOsFs()), nil
 	case "mongo":
 		return mongo.NewStorage(storageLocation), nil
+	case "postgres", "sqlite", "cockroach", "mysql":
+		return rdbms.NewRDBMSStorage(storageLocation), nil
 	default:
 		return nil, fmt.Errorf("storage type %s is unknown", storageType)
 	}

cmd/proxy/database.yml

@@ -13,7 +13,15 @@ test:
   port: 3306
   user: vgp
   password: vgp
-  
+
+test_postgres:
+  dialect: "postgres"
+  database: athens_development
+  user: postgres
+  password: ''
+  host: 127.0.0.1
+  pool: 5
+
 production:
   dialect: "mysql"
   database: olympusdb

migrations/20180326213718_create_modules.down.fizz

@@ -0,0 +1 @@
+drop_table("modules")

migrations/20180326213718_create_modules.up.fizz

@@ -0,0 +1,7 @@
+create_table("modules", func(t) {
+	t.Column("id", "uuid", {"primary": true})
+	t.Column("module", "text", {})
+	t.Column("version", "text", {})
+	t.Column("mod", "blob", {})
+	t.Column("zip", "blob", {})
+})

pkg/storage/rdbms/all_test.go

@@ -0,0 +1,39 @@
+package rdbms
+
+import (
+	"testing"
+
+	"github.com/gobuffalo/suite"
+	"github.com/gomods/athens/pkg/storage"
+)
+
+const (
+	module  = "testmodule"
+	version = "v1.0.0"
+)
+
+var (
+	// TODO: put these values inside of the suite, and generate longer values.
+	// This should help catch edge cases, like https://github.com/gomods/athens/issues/38
+	//
+	// Also, consider doing something similar to what testing/quick does
+	// with the Generator interface (https://godoc.org/testing/quick#Generator).
+	// The rough, simplified idea would be to run a single test case multiple
+	// times over different (increasing) values.
+	mod = []byte("123")
+	zip = []byte("456")
+)
+
+type RDBMSTestSuite struct {
+	*suite.Model
+	storage storage.BackendConnector
+}
+
+func (rd *RDBMSTestSuite) SetupTest() {
+	rd.storage = &ModuleStore{conn: rd.DB}
+	rd.Model.SetupTest()
+}
+
+func Test_ActionSuite(t *testing.T) {
+	suite.Run(t, &RDBMSTestSuite{Model: suite.NewModel()})
+}

pkg/storage/rdbms/getter.go

@@ -0,0 +1,29 @@
+package rdbms
+
+import (
+	"bytes"
+	"io/ioutil"
+	"time"
+
+	"github.com/gomods/athens/pkg/storage"
+	"github.com/gomods/athens/pkg/storage/rdbms/models"
+)
+
+// Get a specific version of a module
+func (r *ModuleStore) Get(module, vsn string) (*storage.Version, error) {
+	result := models.Module{}
+	query := r.conn.Where("module = ?", module).Where("version = ?", vsn)
+	if err := query.First(&result); err != nil {
+		return nil, err
+	}
+	return &storage.Version{
+		RevInfo: storage.RevInfo{
+			Version: result.Version,
+			Name:    result.Version,
+			Short:   result.Version,
+			Time:    time.Now(),
+		},
+		Mod: result.Mod,
+		Zip: ioutil.NopCloser(bytes.NewReader(result.Zip)),
+	}, nil
+}

pkg/storage/rdbms/getter_test.go

@@ -0,0 +1,3 @@
+package rdbms
+
+// see rdbms_test.go for a round-trip test that subsumes tests for the saver

pkg/storage/rdbms/lister.go

@@ -0,0 +1,21 @@
+package rdbms
+
+import (
+	"github.com/gomods/athens/pkg/storage/rdbms/models"
+)
+
+// List lists all versions of a module
+func (r *ModuleStore) List(module string) ([]string, error) {
+	result := make([]models.Module, 0)
+	err := r.conn.Where("module = ?", module).All(&result)
+	if err != nil {
+		return nil, err
+	}
+
+	versions := make([]string, len(result))
+	for i := range result {
+		versions[i] = result[i].Version
+	}
+
+	return versions, nil
+}

pkg/storage/rdbms/lister_test.go

@@ -0,0 +1,12 @@
+package rdbms
+
+func (rd *RDBMSTestSuite) TestList() {
+	r := rd.Require()
+	versions := []string{"v1.0.0", "v1.1.0", "v1.2.0"}
+	for _, version := range versions {
+		rd.storage.Save(module, version, mod, zip)
+	}
+	retVersions, err := rd.storage.List(module)
+	r.NoError(err)
+	r.Equal(versions, retVersions)
+}

pkg/storage/rdbms/models/modules.go

@@ -0,0 +1,60 @@
+package models
+
+import (
+	"encoding/json"
+	"time"
+
+	"github.com/gobuffalo/pop"
+	"github.com/gobuffalo/uuid"
+	"github.com/gobuffalo/validate"
+	"github.com/gobuffalo/validate/validators"
+)
+
+// Module is a model where data is stored.
+type Module struct {
+	ID        uuid.UUID `json:"id" db:"id"`
+	CreatedAt time.Time `json:"created_at" db:"created_at"`
+	UpdatedAt time.Time `json:"updated_at" db:"updated_at"`
+	Module    string    `json:"module" db:"module"`
+	Version   string    `json:"version" db:"version"`
+	Mod       []byte    `json:"mod" db:"mod"`
+	Zip       []byte    `json:"zip" db:"zip"`
+}
+
+// String is not required by pop and may be deleted
+func (m Module) String() string {
+	jm, _ := json.Marshal(m)
+	return string(jm)
+}
+
+// Modules is not required by pop and may be deleted
+type Modules []Module
+
+// String is not required by pop and may be deleted
+func (m Modules) String() string {
+	jm, _ := json.Marshal(m)
+	return string(jm)
+}
+
+// Validate gets run every time you call a "pop.Validate*" (pop.ValidateAndSave, pop.ValidateAndCreate, pop.ValidateAndUpdate) method.
+// This method is not required and may be deleted.
+func (m *Module) Validate(tx *pop.Connection) (*validate.Errors, error) {
+	return validate.Validate(
+		&validators.StringIsPresent{Field: m.Module, Name: "Module"},
+		&validators.StringIsPresent{Field: m.Version, Name: "Version"},
+		&validators.BytesArePresent{Field: m.Mod, Name: "Mod"},
+		&validators.BytesArePresent{Field: m.Zip, Name: "Zip"},
+	), nil
+}
+
+// ValidateCreate gets run every time you call "pop.ValidateAndCreate" method.
+// This method is not required and may be deleted.
+func (m *Module) ValidateCreate(tx *pop.Connection) (*validate.Errors, error) {
+	return validate.NewErrors(), nil
+}
+
+// ValidateUpdate gets run every time you call "pop.ValidateAndUpdate" method.
+// This method is not required and may be deleted.
+func (m *Module) ValidateUpdate(tx *pop.Connection) (*validate.Errors, error) {
+	return validate.NewErrors(), nil
+}

pkg/storage/rdbms/rdbms.go

@@ -0,0 +1,31 @@
+package rdbms
+
+import (
+	"github.com/gobuffalo/pop"
+)
+
+// ModuleStore represents a rdbms(postgres, mysql, sqlite, cockroachdb) backed storage backend.
+type ModuleStore struct {
+	conn           *pop.Connection
+	connectionName string // settings name from database.yml
+}
+
+// NewRDBMSStorage  returns an unconnected RDBMS Module Storage
+// that satisfies the Storage interface. You must call
+// Connect() on the returned store before using it.
+// connectionName
+func NewRDBMSStorage(connectionName string) *ModuleStore {
+	return &ModuleStore{
+		connectionName: connectionName,
+	}
+}
+
+// Connect creates connection to rdmbs backend.
+func (r *ModuleStore) Connect() error {
+	c, err := pop.Connect(r.connectionName)
+	if err != nil {
+		return err
+	}
+	r.conn = c
+	return nil
+}

pkg/storage/rdbms/rdbms_test.go

@@ -0,0 +1,37 @@
+package rdbms
+
+import (
+	"io/ioutil"
+)
+
+func (rd *RDBMSTestSuite) TestGetSaveListRoundTrip() {
+	r := rd.Require()
+	err := rd.storage.Save(module, version, mod, zip)
+	r.NoError(err)
+	listedVersions, err := rd.storage.List(module)
+	r.NoError(err)
+	r.Equal(1, len(listedVersions))
+	retVersion := listedVersions[0]
+	r.Equal(version, retVersion)
+	gotten, err := rd.storage.Get(module, version)
+	r.NoError(err)
+	defer gotten.Zip.Close()
+	r.Equal(version, gotten.RevInfo.Version)
+	r.Equal(version, gotten.RevInfo.Name)
+	r.Equal(version, gotten.RevInfo.Short)
+	// TODO: test the time
+	r.Equal(gotten.Mod, mod)
+	zipContent, err := ioutil.ReadAll(gotten.Zip)
+	r.NoError(err)
+	r.Equal(zipContent, zip)
+}
+
+func (rd *RDBMSTestSuite) TestNewRDBMSStorage() {
+	r := rd.Require()
+	e := "development"
+	getterSaver := NewRDBMSStorage(e)
+	getterSaver.Connect()
+
+	r.NotNil(getterSaver.conn)
+	r.Equal(getterSaver.connectionName, e)
+}

pkg/storage/rdbms/saver.go

@@ -0,0 +1,17 @@
+package rdbms
+
+import (
+	"github.com/gomods/athens/pkg/storage/rdbms/models"
+)
+
+// Save stores a module in rdbms storage.
+func (r *ModuleStore) Save(module, version string, mod, zip []byte) error {
+	m := &models.Module{
+		Module:  module,
+		Version: version,
+		Mod:     mod,
+		Zip:     zip,
+	}
+
+	return r.conn.Create(m)
+}

pkg/storage/rdbms/saver_test.go

@@ -0,0 +1,3 @@
+package rdbms
+
+// see rdbms_test.go for a round-trip test that subsumes tests for the saver