add rdbms support for cdn metadata storage (#151)

* add rdbms support for cdn metadata storage

* move cdn metadata model to the metadata pkg

* simplify get

* add mongo saver and test

* remove 'stutters'

* change default table name

* missing cdn metadata pkg rename

cmd/olympus/actions/goget.go

@@ -6,13 +6,13 @@ import (
 	"strings"
 
 	"github.com/gobuffalo/buffalo"
-	"github.com/gomods/athens/pkg/cdn"
+	cdnmetadata "github.com/gomods/athens/pkg/cdn/metadata"
 	"github.com/gomods/athens/pkg/paths"
 )
 
 // GoGet is middleware that checks for the 'go-get=1' query string. If it exists,
 // uses getter to determine the redirect location
-func GoGet(getter cdn.Getter) buffalo.MiddlewareFunc {
+func GoGet(getter cdnmetadata.Getter) buffalo.MiddlewareFunc {
 	return func(next buffalo.Handler) buffalo.Handler {
 		return func(c buffalo.Context) error {
 			if strings.Contains(c.Request().URL.Query().Get("go-get"), "1") {
@@ -23,7 +23,7 @@ func GoGet(getter cdn.Getter) buffalo.MiddlewareFunc {
 	}
 }
 
-func goGetMeta(c buffalo.Context, getter cdn.Getter) error {
+func goGetMeta(c buffalo.Context, getter cdnmetadata.Getter) error {
 	params, err := paths.GetAllParams(c)
 	if err != nil {
 		return err

cmd/proxy/actions/cdn.go

@@ -1,10 +1,10 @@
 package actions
 
 import (
-	"github.com/gomods/athens/pkg/cdn"
-	"github.com/gomods/athens/pkg/cdn/fake"
+	cdnmetadata "github.com/gomods/athens/pkg/cdn/metadata"
+	"github.com/gomods/athens/pkg/cdn/metadata/fake"
 )
 
-func newCDNGetter() cdn.Getter {
+func newCDNGetter() cdnmetadata.Getter {
 	return &fake.Getter{URL: "https://mycdn.com"}
 }

cmd/proxy/actions/goget.go

@@ -5,13 +5,13 @@ import (
 	"net/http"
 
 	"github.com/gobuffalo/buffalo"
-	"github.com/gomods/athens/pkg/cdn"
+	cdnmetadata "github.com/gomods/athens/pkg/cdn/metadata"
 	"github.com/gomods/athens/pkg/paths"
 )
 
 // GoGet is middleware that checks for the 'go-get=1' query string. If it exists,
 // uses getter to determine the redirect location
-func GoGet(getter cdn.Getter) buffalo.MiddlewareFunc {
+func GoGet(getter cdnmetadata.Getter) buffalo.MiddlewareFunc {
 	return func(next buffalo.Handler) buffalo.Handler {
 		return func(c buffalo.Context) error {
 			if paths.IsGoGet(c.Request().URL) {
@@ -22,7 +22,7 @@ func GoGet(getter cdn.Getter) buffalo.MiddlewareFunc {
 	}
 }
 
-func goGetMeta(c buffalo.Context, getter cdn.Getter) error {
+func goGetMeta(c buffalo.Context, getter cdnmetadata.Getter) error {
 	params, err := paths.GetAllParams(c)
 	if err != nil {
 		return err

migrations/20180604072327_create_cdn_metadata_entries.down.fizz

@@ -0,0 +1 @@
+drop_table("cdn_metadata_entries")

migrations/20180604072327_create_cdn_metadata_entries.up.fizz

@@ -0,0 +1,5 @@
+create_table("cdn_metadata_entries", func(t) {
+	t.Column("id", "uuid", {"primary": true})
+	t.Column("module", "text", {})
+	t.Column("redirect_url", "text", {})
+})

pkg/cdn/metadata/cdn_metadata_entry.go

@@ -0,0 +1,55 @@
+package metadata
+
+import (
+	"encoding/json"
+	"time"
+
+	"github.com/gobuffalo/pop"
+	"github.com/gobuffalo/uuid"
+	"github.com/gobuffalo/validate"
+	"github.com/gobuffalo/validate/validators"
+)
+
+// CDNMetadataEntry stores the module name and cdn URL.
+type CDNMetadataEntry struct {
+	ID          uuid.UUID `json:"id" db:"id" bson:"id"`
+	CreatedAt   time.Time `json:"created_at" db:"created_at" bson:"created_at"`
+	UpdatedAt   time.Time `json:"updated_at" db:"updated_at" bson:"updated_at"`
+	Module      string    `json:"module" db:"module" bson:"module"`
+	RedirectURL string    `json:"redirect_url" db:"redirect_url" bson:"redirect_url"`
+}
+
+// String is not required by pop and may be deleted
+func (e CDNMetadataEntry) String() string {
+	je, _ := json.Marshal(e)
+	return string(je)
+}
+
+// CdnMetadataEntries is not required by pop and may be deleted
+type CdnMetadataEntries []CDNMetadataEntry
+
+// Validate gets run every time you call a "pop.Validate*" (pop.ValidateAndSave, pop.ValidateAndCreate, pop.ValidateAndUpdate) method.
+// This method is not required and may be deleted.
+func (e *CDNMetadataEntry) Validate(tx *pop.Connection) (*validate.Errors, error) {
+	return validate.Validate(
+		&validators.StringIsPresent{Field: e.Module, Name: "Module"},
+		&validators.StringIsPresent{Field: e.RedirectURL, Name: "RedirectURL"},
+	), nil
+}
+
+// ValidateCreate gets run every time you call "pop.ValidateAndCreate" method.
+// This method is not required and may be deleted.
+func (e *CDNMetadataEntry) ValidateCreate(tx *pop.Connection) (*validate.Errors, error) {
+	return validate.NewErrors(), nil
+}
+
+// ValidateUpdate gets run every time you call "pop.ValidateAndUpdate" method.
+// This method is not required and may be deleted.
+func (e *CDNMetadataEntry) ValidateUpdate(tx *pop.Connection) (*validate.Errors, error) {
+	return validate.NewErrors(), nil
+}
+
+// TableName changes the default name which would be c_d_n_metadata_entry
+func (e *CDNMetadataEntry) TableName() string {
+	return "cdn_metadata_entries"
+}

pkg/cdn/metadata/fake/getter.go

@@ -1,6 +1,6 @@
 package fake
 
-// Getter is a (./pkg/cdn).Getter implementation that always returns URL
+// Getter is a (./pkg/cdn/metadata).Getter implementation that always returns URL
 type Getter struct {
 	URL string
 }

pkg/cdn/metadata/getter.go

@@ -1,4 +1,4 @@
-package cdn
+package metadata
 
 // Getter gets the details about a given baseURL & module and returns the base
 // URL of the module metadata and content. For example, if

pkg/cdn/metadata/mongo/getter.go

@@ -0,0 +1,15 @@
+package mongo
+
+import (
+	"github.com/globalsign/mgo/bson"
+	"github.com/gomods/athens/pkg/cdn/metadata"
+)
+
+// Get retrieves the cdn base URL for a module
+func (s *MetadataStore) Get(module string) (string, error) {
+	coll := s.session.DB(s.db).C(s.col)
+	params := bson.M{"module": module}
+	entry := metadata.CDNMetadataEntry{}
+	err := coll.Find(params).One(&entry)
+	return entry.RedirectURL, err
+}

pkg/cdn/metadata/mongo/mongo.go

@@ -0,0 +1,39 @@
+package mongo
+
+import "github.com/globalsign/mgo"
+
+// MetadataStore represents a Mongo backed metadata store.
+type MetadataStore struct {
+	session *mgo.Session
+	db      string
+	col     string
+	url     string
+}
+
+// NewStorage returns an unconnected Mongo backed storage
+// that satisfies the Storage interface.  You must call
+// Connect() on the returned store before using it.
+func NewStorage(url, dbName string) *MetadataStore {
+	return &MetadataStore{url: url, db: dbName}
+}
+
+// Connect conntect the the newly created mongo backend.
+func (m *MetadataStore) Connect() error {
+	s, err := mgo.Dial(m.url)
+	if err != nil {
+		return err
+	}
+	m.session = s
+
+	m.col = "cdn_metadata"
+
+	index := mgo.Index{
+		Key:        []string{"base_url", "module"},
+		Unique:     true,
+		DropDups:   true,
+		Background: true,
+		Sparse:     true,
+	}
+	c := m.session.DB(m.db).C(m.col)
+	return c.EnsureIndex(index)
+}

pkg/cdn/metadata/mongo/mongo_test.go

@@ -0,0 +1,51 @@
+package mongo
+
+import (
+	"testing"
+
+	"github.com/gomods/athens/pkg/cdn/metadata"
+	"github.com/gomods/athens/pkg/fixtures"
+	"github.com/stretchr/testify/suite"
+)
+
+const (
+	module      = "testmodule"
+	redirectURL = "https://mycdn.com/gomods.io/my/testmodule"
+)
+
+type MongoTests struct {
+	*fixtures.Mongo
+	storage metadata.StorageConnector
+}
+
+func TestMongo(t *testing.T) {
+	suite.Run(t, &MongoTests{Mongo: fixtures.NewMongo(fixtures.DefaultMongoURL)})
+}
+
+func (m *MongoTests) SetupTest() {
+	m.Mongo.SetupTest()
+	store := NewStorage(fixtures.DefaultMongoURL, m.Mongo.DBName)
+	store.Connect()
+	m.storage = store
+}
+
+func (m *MongoTests) TestGetSaveListRoundTrip() {
+	r := m.Require()
+	err := m.storage.Save(module, redirectURL)
+	r.NoError(err)
+
+	gotten, err := m.storage.Get(module)
+	r.NoError(err)
+	r.Equal(gotten, redirectURL)
+}
+
+func (m *MongoTests) TestNewMongoStorage() {
+	r := m.Require()
+	getterSaver := NewStorage(fixtures.DefaultMongoURL, m.Mongo.DBName)
+	getterSaver.Connect()
+
+	r.NotNil(getterSaver.col)
+	r.NotNil(getterSaver.db)
+	r.NotNil(getterSaver.session)
+	r.Equal(getterSaver.url, fixtures.DefaultMongoURL)
+}

pkg/cdn/metadata/mongo/saver.go

@@ -0,0 +1,26 @@
+package mongo
+
+import (
+	"time"
+
+	"github.com/gobuffalo/uuid"
+	"github.com/gomods/athens/pkg/cdn/metadata"
+)
+
+// Save stores a module in mongo storage.
+func (s *MetadataStore) Save(module, redirectURL string) error {
+	id, err := uuid.NewV4()
+	if err != nil {
+		return err
+	}
+	m := &metadata.CDNMetadataEntry{
+		ID:          id,
+		Module:      module,
+		RedirectURL: redirectURL,
+		CreatedAt:   time.Now().UTC(),
+		UpdatedAt:   time.Now().UTC(),
+	}
+
+	c := s.session.DB(s.db).C(s.col)
+	return c.Insert(m)
+}

pkg/cdn/metadata/rdbms/all_test.go

@@ -0,0 +1,27 @@
+package rdbms
+
+import (
+	"testing"
+
+	"github.com/gobuffalo/suite"
+	"github.com/gomods/athens/pkg/cdn/metadata"
+)
+
+const (
+	module      = "testmodule"
+	redirectURL = "https://mycdn.com/gomods.io/my/testmodule"
+)
+
+type RDBMSTestSuite struct {
+	*suite.Model
+	storage metadata.StorageConnector
+}
+
+func (rd *RDBMSTestSuite) SetupTest() {
+	rd.storage = &MetadataStore{conn: rd.DB}
+	rd.Model.SetupTest()
+}
+
+func Test_ActionSuite(t *testing.T) {
+	suite.Run(t, &RDBMSTestSuite{Model: suite.NewModel()})
+}

pkg/cdn/metadata/rdbms/getter.go

@@ -0,0 +1,11 @@
+package rdbms
+
+import "github.com/gomods/athens/pkg/cdn/metadata"
+
+// Get retrieves the cdn base URL for a module
+func (s *MetadataStore) Get(module string) (string, error) {
+	result := metadata.CDNMetadataEntry{}
+	query := s.conn.Where("module = ?", module)
+	err := query.First(&result)
+	return result.RedirectURL, err
+}

pkg/cdn/metadata/rdbms/rdbms.go

@@ -0,0 +1,31 @@
+package rdbms
+
+import (
+	"github.com/gobuffalo/pop"
+)
+
+// MetadataStore represents a rdbms(postgres, mysql, sqlite, cockroachdb) backed metadata store.
+type MetadataStore struct {
+	conn           *pop.Connection
+	connectionName string // settings name from database.yml
+}
+
+// NewStorage  returns an unconnected RDBMS Metadata Storage
+// that satisfies the Getter and Setter interfaces. You must call
+// Connect() on the returned store before using it.
+// connectionName
+func NewStorage(connectionName string) *MetadataStore {
+	return &MetadataStore{
+		connectionName: connectionName,
+	}
+}
+
+// Connect creates connection to rdmbs backend.
+func (r *MetadataStore) Connect() error {
+	c, err := pop.Connect(r.connectionName)
+	if err != nil {
+		return err
+	}
+	r.conn = c
+	return nil
+}

pkg/cdn/metadata/rdbms/rdbms_test.go

@@ -0,0 +1,21 @@
+package rdbms
+
+func (rd *RDBMSTestSuite) TestGetSaveRoundTrip() {
+	r := rd.Require()
+	err := rd.storage.Save(module, redirectURL)
+	r.NoError(err)
+
+	gotten, err := rd.storage.Get(module)
+	r.NoError(err)
+	r.Equal(gotten, redirectURL)
+}
+
+func (rd *RDBMSTestSuite) TestNewRDBMSStorage() {
+	r := rd.Require()
+	e := "development"
+	getterSaver := NewStorage(e)
+	getterSaver.Connect()
+
+	r.NotNil(getterSaver.conn)
+	r.Equal(getterSaver.connectionName, e)
+}

pkg/cdn/metadata/rdbms/saver.go

@@ -0,0 +1,9 @@
+package rdbms
+
+import "github.com/gomods/athens/pkg/cdn/metadata"
+
+// Save saves the module and it's cdn base URL.
+func (s *MetadataStore) Save(module, redirectURL string) error {
+	r := metadata.CDNMetadataEntry{Module: module, RedirectURL: redirectURL}
+	return s.conn.Create(&r)
+}

pkg/cdn/metadata/saver.go

@@ -0,0 +1,9 @@
+package metadata
+
+// Saver saves the module's base URL
+// of the module metadata and content. For example:
+// module: gomods.io/my/module - URL: 'https://mycdn.com/gomods.io/my/module'
+type Saver interface {
+	// Save saves the module and base URL pair.
+	Save(module, redirectURL string) error
+}

pkg/cdn/metadata/storage.go

@@ -0,0 +1,7 @@
+package metadata
+
+// Storage is a complete cdn metadata storage backend (i.e. file system, database) implementation - a getter and saver
+type Storage interface {
+	Getter
+	Saver
+}

pkg/cdn/metadata/storageconnector.go

@@ -0,0 +1,27 @@
+package metadata
+
+// StorageConnector is a regular storage Storage with Connect functionality
+type StorageConnector interface {
+	Storage
+	Connect() error
+}
+
+type noOpConnectedStorage struct {
+	s Storage
+}
+
+// NoOpStorageConnector wraps storage Storage with Connect functionality
+func NoOpStorageConnector(s Storage) StorageConnector {
+	return noOpConnectedStorage{s: s}
+}
+
+func (n noOpConnectedStorage) Connect() error {
+	return nil
+}
+
+func (n noOpConnectedStorage) Get(module string) (string, error) {
+	return n.s.Get(module)
+}
+func (n noOpConnectedStorage) Save(module, redirectURL string) error {
+	return n.s.Save(module, redirectURL)
+}

pkg/cdn/mongo/entry.go

@@ -1,9 +0,0 @@
-package mongo
-
-// Entry is stored in the DB. Right now it just holds a redirect URL to
-// the CDN
-type Entry struct {
-	Module      string `bson:"module"`
-	RedirectURL string `bson:"redirect_url"`
-	// Other fields?
-}

pkg/cdn/mongo/getter.go

@@ -1,22 +0,0 @@
-package mongo
-
-import (
-	"github.com/globalsign/mgo"
-	"github.com/globalsign/mgo/bson"
-)
-
-type getter struct {
-	conn *mgo.Session
-	db   string
-	coll string
-}
-
-func (g *getter) Get(module string) (string, error) {
-	coll := g.conn.DB(g.db).C(g.coll)
-	params := bson.M{"module": module}
-	entry := Entry{}
-	if err := coll.Find(params).One(&entry); err != nil {
-		return "", nil
-	}
-	return entry.RedirectURL, nil
-}

pkg/cdn/mongo/mongo_test.go

@@ -1,19 +0,0 @@
-package mongo
-
-import (
-	"testing"
-
-	"github.com/gomods/athens/pkg/fixtures"
-	"github.com/stretchr/testify/suite"
-)
-
-type MongoTests struct {
-	*fixtures.Mongo
-}
-
-func RunMongoTests(t *testing.T) {
-	suite.Run(t, &MongoTests{Mongo: fixtures.NewMongo(fixtures.DefaultMongoURL)})
-}
-
-// TODO: add round-trip tests when a mongo saver is done
-// (https://github.com/gomods/athens/issues/50)

pkg/fixtures/mongo.go

@@ -25,7 +25,7 @@ const DefaultMongoURL = "127.0.0.1:27017"
 type Mongo struct {
 	suite.Suite
 	url    string
-	dbName string
+	DBName string
 	DB     *mgo.Database
 }
 
@@ -36,7 +36,7 @@ type Mongo struct {
 func (m *Mongo) SetupTest() {
 	sess, err := mgo.Dial(m.url)
 	m.Require().NoError(err)
-	m.DB = sess.DB(m.dbName)
+	m.DB = sess.DB(m.DBName)
 }
 
 // TearDownTest drops the database that was created in SetupTest
@@ -50,6 +50,6 @@ func (m *Mongo) TearDownTest() {
 func NewMongo(url string) *Mongo {
 	return &Mongo{
 		url:    url,
-		dbName: names.NameSep("-") + "-athens-testing",
+		DBName: names.NameSep("-") + "-athens-testing",
 	}
 }