primal/athens
1
0
Fork 0
mirror of https://github.com/gomods/athens synced 2026-02-03 11:00:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implemented backing mongo storage for modules (#64)

Browse Source 
* mongo storage initial impl

* added tests

* mongo tests passing
This commit is contained in:
Michal Pristas
2018-03-20 19:28:42 +01:00
committed by Aaron Schlesinger
parent 23f469ac18
commit 0a6e4407a1
21 changed files with 388 additions and 138 deletions
Download Patch File Download Diff File Expand all files Collapse all files
actions/storage.go
+33 -26
View File
@@ -1,26 +1,33 @@
package actions
import (
"fmt"
"github.com/gobuffalo/envy"
"github.com/gomods/athens/pkg/storage"
"github.com/gomods/athens/pkg/storage/disk"
"github.com/gomods/athens/pkg/storage/memory"
)
func newStorage() (storage.Storage, error) {
storageType := envy.Get("ATHENS_STORAGE_TYPE", "memory")
switch storageType {
case "memory":
return memory.New(), nil
case "disk":
rootLocation, err := envy.MustGet("ATHENS_DISK_STORAGE_ROOT")
if err != nil {
return nil, fmt.Errorf("missing disk storage root (%s)", err)
}
return disk.NewStorage(rootLocation), nil
default:
return nil, fmt.Errorf("storage type %s is unknown", storageType)
}
}
package actions
import (
"fmt"
"github.com/gobuffalo/envy"
"github.com/gomods/athens/pkg/storage"
"github.com/gomods/athens/pkg/storage/disk"
"github.com/gomods/athens/pkg/storage/memory"
"github.com/gomods/athens/pkg/storage/mongo"
)
func newStorage() (storage.Storage, error) {
storageType := envy.Get("ATHENS_STORAGE_TYPE", "memory")
switch storageType {
case "memory":
return memory.NewMemoryStorage(), nil
case "disk":
rootLocation, err := envy.MustGet("ATHENS_DISK_STORAGE_ROOT")
if err != nil {
return nil, fmt.Errorf("missing disk storage root (%s)", err)
}
return disk.NewStorage(rootLocation), nil
case "mongo":
mongoURI, err := envy.MustGet("ATHENS_MONGO_STORAGE_URL")
if err != nil {
return nil, fmt.Errorf("missing mongo URL (%s)", err)
}
return mongo.NewMongoStorage(mongoURI), nil
default:
return nil, fmt.Errorf("storage type %s is unknown", storageType)
}
}
pkg/storage/disk/all_test.go
+49 -48
View File
@@ -1,48 +1,49 @@
package disk
import (
"io/ioutil"
"os"
"testing"
"github.com/stretchr/testify/suite"
)
const (
baseURL = "testbaseurl"
module = "testmodule"
version = "v1.0.0"
)
var (
// TODO: put these values inside of the suite, and generate longer values.
// This should help catch edge cases, like https://github.com/gomods/athens/issues/38
//
// Also, consider doing something similar to what testing/quick does
// with the Generator interface (https://godoc.org/testing/quick#Generator).
// The rough, simplified idea would be to run a single test case multiple
// times over different (increasing) values.
mod = []byte("123")
zip = []byte("456")
)
type DiskTests struct {
suite.Suite
storage Storage
rootDir string
}
func (d *DiskTests) SetupTest() {
r, err := ioutil.TempDir("", "athens-disk-tests")
d.Require().NoError(err)
d.storage = NewStorage(r)
d.rootDir = r
}
func (d *DiskTests) TearDownTest() {
d.Require().NoError(os.RemoveAll(d.rootDir))
}
func TestDiskStorage(t *testing.T) {
suite.Run(t, new(DiskTests))
}
package disk
import (
"io/ioutil"
"os"
"testing"
"github.com/gomods/athens/pkg/storage"
"github.com/stretchr/testify/suite"
)
const (
baseURL = "testbaseurl"
module = "testmodule"
version = "v1.0.0"
)
var (
// TODO: put these values inside of the suite, and generate longer values.
// This should help catch edge cases, like https://github.com/gomods/athens/issues/38
//
// Also, consider doing something similar to what testing/quick does
// with the Generator interface (https://godoc.org/testing/quick#Generator).
// The rough, simplified idea would be to run a single test case multiple
// times over different (increasing) values.
mod = []byte("123")
zip = []byte("456")
)
type DiskTests struct {
suite.Suite
storage storage.Storage
rootDir string
}
func (d *DiskTests) SetupTest() {
r, err := ioutil.TempDir("", "athens-disk-tests")
d.Require().NoError(err)
d.storage = NewStorage(r)
d.rootDir = r
}
func (d *DiskTests) TearDownTest() {
d.Require().NoError(os.RemoveAll(d.rootDir))
}
func TestDiskStorage(t *testing.T) {
suite.Run(t, new(DiskTests))
}
pkg/storage/disk/disk.go
+1 -11
View File
@@ -6,15 +6,6 @@ import (
"github.com/gomods/athens/pkg/storage"
)
// Storage is the only interface defined by the disk storage. Use
// NewStorage to create one of these. Everything is all in one
// because it all has to share the same tree
type Storage interface {
storage.Lister
storage.Getter
storage.Saver
}
type storageImpl struct {
rootDir string
}
@@ -30,7 +21,6 @@ func (s *storageImpl) versionDiskLocation(baseURL, module, version string) strin
// NewStorage returns a new ListerSaver implementation that stores
// everything under rootDir
func NewStorage(rootDir string) Storage {
func NewStorage(rootDir string) storage.Storage {
return &storageImpl{rootDir: rootDir}
}
pkg/storage/getter.go
+1 -1
View File
@@ -1,6 +1,6 @@
package storage
type Getter interface {
// must return NotFoundErr if the coordinates are not found
// must return ErrNotFound if the coordinates are not found
Get(baseURL, module, vsn string) (*Version, error)
}
pkg/storage/lister.go
+1 -1
View File
@@ -3,6 +3,6 @@ package storage
// Lister is the interface that lists versions of a specific baseURL & module
type Lister interface {
// List gets all the versions for the given baseURL & module.
// It returns NotFoundErr if baseURL/module isn't found
// It returns ErrNotFound if baseURL/module isn't found
List(baseURL, module string) ([]string, error)
}
pkg/storage/memory/all_test.go
+31 -30
View File
@@ -1,30 +1,31 @@
package memory
import (
"testing"
"github.com/stretchr/testify/suite"
)
const (
baseURL = "base.com"
module = "my/module"
)
var (
mod = []byte("123")
zip = []byte("456")
)
type MemoryTests struct {
suite.Suite
mem GetterSaver
}
func (m *MemoryTests) SetupTest() {
m.mem = New()
}
func TestMemoryStorage(t *testing.T) {
suite.Run(t, new(MemoryTests))
}
package memory
import (
"testing"
"github.com/gomods/athens/pkg/storage"
"github.com/stretchr/testify/suite"
)
const (
baseURL = "base.com"
module = "my/module"
)
var (
mod = []byte("123")
zip = []byte("456")
)
type MemoryTests struct {
suite.Suite
mem storage.Storage
}
func (m *MemoryTests) SetupTest() {
m.mem = NewMemoryStorage()
}
func TestMemoryStorage(t *testing.T) {
suite.Run(t, new(MemoryTests))
}
pkg/storage/memory/lister.go
+1 -1
View File
@@ -10,7 +10,7 @@ func (v *getterSaverImpl) List(basePath, module string) ([]string, error) {
defer v.RUnlock()
versions, ok := v.versions[key]
if !ok {
return nil, storage.NotFoundErr{BasePath: basePath, Module: module}
return nil, storage.ErrNotFound{BasePath: basePath, Module: module}
}
ret := make([]string, len(versions))
for i, version := range versions {
pkg/storage/memory/memory.go
+3 -7
View File
@@ -7,12 +7,6 @@ import (
"github.com/gomods/athens/pkg/storage"
)
type GetterSaver interface {
storage.Lister
storage.Getter
storage.Saver
}
type getterSaverImpl struct {
*sync.RWMutex
versions map[string][]*storage.Version
@@ -22,7 +16,9 @@ func (e *getterSaverImpl) key(baseURL, module string) string {
return fmt.Sprintf("%s/%s", baseURL, module)
}
func New() GetterSaver {
// NewMemoryStorage returns a new ListerSaver implementation that stores
// everything in memory
func NewMemoryStorage() storage.Storage {
return &getterSaverImpl{
RWMutex: new(sync.RWMutex),
versions: make(map[string][]*storage.Version),
pkg/storage/memory/memory_test.go
+10 -10
View File
@@ -1,10 +1,10 @@
package memory
func (m *MemoryTests) TestNew() {
r := m.Require()
getterSaverIface := New()
getterSaver, ok := getterSaverIface.(*getterSaverImpl)
r.True(ok)
r.NotNil(getterSaver.versions)
r.NotNil(getterSaver.RWMutex)
}
package memory
func (m *MemoryTests) TestNewMemoryStorage() {
r := m.Require()
getterSaverIface := NewMemoryStorage()
getterSaver, ok := getterSaverIface.(*getterSaverImpl)
r.True(ok)
r.NotNil(getterSaver.versions)
r.NotNil(getterSaver.RWMutex)
}
pkg/storage/module.go
+9
View File
@@ -0,0 +1,9 @@
package storage
type Module struct {
BaseURL string `bson:"base_url"`
Module string `bson:"module"`
Version string `bson:"version"`
Mod []byte `bson:"mod"`
Zip []byte `bson:"zip"`
}
pkg/storage/mongo/all_test.go
+43
View File
@@ -0,0 +1,43 @@
package mongo
import (
"testing"
"github.com/gomods/athens/pkg/storage"
"github.com/stretchr/testify/suite"
)
const (
baseURL = "testbaseurl"
module = "testmodule"
version = "v1.0.0"
)
var (
// TODO: put these values inside of the suite, and generate longer values.
// This should help catch edge cases, like https://github.com/gomods/athens/issues/38
//
// Also, consider doing something similar to what testing/quick does
// with the Generator interface (https://godoc.org/testing/quick#Generator).
// The rough, simplified idea would be to run a single test case multiple
// times over different (increasing) values.
mod = []byte("123")
zip = []byte("456")
)
type MongoTests struct {
suite.Suite
storage storage.StorageConnector
}
func (d *MongoTests) SetupTest() {
store := NewMongoStorage("mongodb://127.0.0.1:27017")
store.Connect()
store.s.DB(store.d).C(store.c).RemoveAll(nil)
d.storage = store
}
func TestDiskStorage(t *testing.T) {
suite.Run(t, new(MongoTests))
}
pkg/storage/mongo/getter.go
+33
View File
@@ -0,0 +1,33 @@
package mongo
import (
"bytes"
"io/ioutil"
"strings"
"time"
"github.com/globalsign/mgo/bson"
"github.com/gomods/athens/pkg/storage"
)
func (s *MongoModuleStore) Get(baseURL, module, vsn string) (*storage.Version, error) {
c := s.s.DB(s.d).C(s.c)
result := &storage.Module{}
err := c.Find(bson.M{"base_url": baseURL, "module": module, "version": vsn}).One(result)
if err != nil {
if strings.Contains(err.Error(), "not found") {
err = storage.ErrVersionNotFound{BasePath: baseURL, Module: module, Version: vsn}
}
return nil, err
}
return &storage.Version{
RevInfo: storage.RevInfo{
Version: result.Version,
Name: result.Version,
Short: result.Version,
Time: time.Now(),
},
Mod: result.Mod,
Zip: ioutil.NopCloser(bytes.NewReader(result.Zip)),
}, nil
}
pkg/storage/mongo/getter_test.go
+3
View File
@@ -0,0 +1,3 @@
package mongo
// see mongo_test.go for a round-trip test that subsumes tests for the saver
pkg/storage/mongo/lister.go
+27
View File
@@ -0,0 +1,27 @@
package mongo
import (
"strings"
"github.com/globalsign/mgo/bson"
"github.com/gomods/athens/pkg/storage"
)
func (s *MongoModuleStore) List(baseURL, module string) ([]string, error) {
c := s.s.DB(s.d).C(s.c)
result := make([]storage.Module, 0)
err := c.Find(bson.M{"base_url": baseURL, "module": module}).All(&result)
if err != nil {
if strings.Contains(err.Error(), "not found") {
err = storage.ErrNotFound{Module: module, BasePath: baseURL}
}
return nil, err
}
versions := make([]string, len(result))
for i, r := range result {
versions[i] = r.Version
}
return versions, nil
}
pkg/storage/mongo/lister_test.go
+12
View File
@@ -0,0 +1,12 @@
package mongo
func (m *MongoTests) TestList() {
r := m.Require()
versions := []string{"v1.0.0", "v1.1.0", "v1.2.0"}
for _, version := range versions {
m.storage.Save(baseURL, module, version, mod, zip)
}
retVersions, err := m.storage.List(baseURL, module)
r.NoError(err)
r.Equal(versions, retVersions)
}
pkg/storage/mongo/mongo.go
+41
View File
@@ -0,0 +1,41 @@
package mongo
import (
"github.com/globalsign/mgo"
)
type MongoModuleStore struct {
s *mgo.Session
d string // database
c string // collection
url string
}
// NewMongoStorage returns an unconnected Mongo Module Storage
// that satisfies the Storage interface. You must call
// Connect() on the returned store before using it.
func NewMongoStorage(url string) *MongoModuleStore {
return &MongoModuleStore{url: url}
}
func (m *MongoModuleStore) Connect() error {
s, err := mgo.Dial(m.url)
if err != nil {
return err
}
m.s = s
// TODO: database and collection as env vars, or params to New()? together with user/mongo
m.d = "athens"
m.c = "modules"
index := mgo.Index{
Key: []string{"base_url", "module", "version"},
Unique: true,
DropDups: true,
Background: true,
Sparse: true,
}
c := m.s.DB(m.d).C(m.c)
return c.EnsureIndex(index)
}
pkg/storage/mongo/mongo_test.go
+38
View File
@@ -0,0 +1,38 @@
package mongo
import (
"io/ioutil"
)
func (m *MongoTests) TestGetSaveListRoundTrip() {
r := m.Require()
m.storage.Save(baseURL, module, version, mod, zip)
listedVersions, err := m.storage.List(baseURL, module)
r.NoError(err)
r.Equal(1, len(listedVersions))
retVersion := listedVersions[0]
r.Equal(version, retVersion)
gotten, err := m.storage.Get(baseURL, module, version)
r.NoError(err)
defer gotten.Zip.Close()
r.Equal(version, gotten.RevInfo.Version)
r.Equal(version, gotten.RevInfo.Name)
r.Equal(version, gotten.RevInfo.Short)
// TODO: test the time
r.Equal(gotten.Mod, mod)
zipContent, err := ioutil.ReadAll(gotten.Zip)
r.NoError(err)
r.Equal(zipContent, zip)
}
func (m *MongoTests) TestNewMongoStorage() {
r := m.Require()
url := "mongodb://127.0.0.1:27017"
getterSaver := NewMongoStorage(url)
getterSaver.Connect()
r.NotNil(getterSaver.c)
r.NotNil(getterSaver.d)
r.NotNil(getterSaver.s)
r.Equal(getterSaver.url, url)
}
pkg/storage/mongo/saver.go
+16
View File
@@ -0,0 +1,16 @@
package mongo
import "github.com/gomods/athens/pkg/storage"
func (s *MongoModuleStore) Save(baseURL, module, version string, mod, zip []byte) error {
m := &storage.Module{
BaseURL: baseURL,
Module: module,
Version: version,
Mod: mod,
Zip: zip,
}
c := s.s.DB(s.d).C(s.c)
return c.Insert(m)
}
pkg/storage/mongo/saver_test.go
+3
View File
@@ -0,0 +1,3 @@
package mongo
// see mongo_test.go for a round-trip test that subsumes tests for the saver
pkg/storage/not_found_err.go
+3 -3
View File
@@ -4,12 +4,12 @@ import (
"fmt"
)
type NotFoundErr struct {
type ErrNotFound struct {
BasePath string
Module string
}
func (n NotFoundErr) Error() string {
func (n ErrNotFound) Error() string {
return fmt.Sprintf("%s/%s not found", n.BasePath, n.Module)
}
@@ -24,7 +24,7 @@ func (e ErrVersionNotFound) Error() string {
}
func IsNotFoundError(err error) bool {
_, ok := err.(NotFoundErr)
_, ok := err.(ErrNotFound)
return ok
}
pkg/storage/storageconnector.go
+30
View File
@@ -0,0 +1,30 @@
package storage
// StorageConnector is a regular storage with Connect functionality
type StorageConnector interface {
Storage
Connect() error
}
type noOpConnectedStorage struct {
s Storage
}
// NoOpStorageConnector wraps storage with Connect functionality
func NoOpStorageConnector(s Storage) StorageConnector {
return noOpConnectedStorage{s: s}
}
func (n noOpConnectedStorage) Connect() error {
return nil
}
func (n noOpConnectedStorage) Get(baseURL, module, vsn string) (*Version, error) {
return n.s.Get(baseURL, module, vsn)
}
func (n noOpConnectedStorage) List(baseURL, module string) ([]string, error) {
return n.s.List(baseURL, module)
}
func (n noOpConnectedStorage) Save(baseURL, module, version string, mod, zip []byte) error {
return n.s.Save(baseURL, module, version, mod, zip)
}