primal/athens
1
0
Fork 0
mirror of https://github.com/gomods/athens synced 2026-02-03 11:00:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

Refactor storage tests (#727)

Browse Source 
* Refactor storage tests

* minio: check object err

* remove inaccurate test

* storage/fs: use memory
This commit is contained in:
Marwan Sulaiman
2018-10-04 19:16:24 -04:00
committed by Aaron Schlesinger
parent 337488c202
commit 97d8013876
20 changed files with 378 additions and 752 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.travis.yml
+1
View File
@@ -8,6 +8,7 @@ env:
global:
- PATH=${PATH}:./bin
- GO_ENV=test
- ATHENS_MINIO_ENDPOINT=127.0.0.1:9001
- MINIO_ACCESS_KEY=minio
- MINIO_SECRET_KEY=minio123
- CODE_COV=1
cmd/olympus/actions/actions_test.go
+1
View File
@@ -23,6 +23,7 @@ type ActionSuite struct {
}
func Test_ActionSuite(t *testing.T) {
t.SkipNow() // Olympus to be removed.
conf, err := config.GetConf(testConfigFile)
if err != nil {
t.Fatalf("Unable to parse config file: %s", err.Error())
pkg/config/parse_test.go
-53
View File
@@ -284,59 +284,6 @@ func TestParseExampleConfig(t *testing.T) {
restoreEnv(envVarBackup)
}
// TestConfigOverridesDefault validates that a value provided by the config is not overwritten during parsing
func TestConfigOverridesDefault(t *testing.T) {
// set values to anything but defaults
config := &Config{
TimeoutConf: TimeoutConf{
Timeout: 1,
},
Storage: &StorageConfig{
Minio: &MinioConfig{
Bucket: "notgomods",
EnableSSL: false,
TimeoutConf: TimeoutConf{
Timeout: 42,
},
},
},
}
// should be identical to config above
expConfig := &Config{
TimeoutConf: config.TimeoutConf,
Storage: &StorageConfig{
Minio: &MinioConfig{
Bucket: config.Storage.Minio.Bucket,
EnableSSL: config.Storage.Minio.EnableSSL,
TimeoutConf: config.Storage.Minio.TimeoutConf,
},
},
}
// unset all environment variables
envVars := getEnvMap(&Config{})
envVarBackup := map[string]string{}
for k := range envVars {
oldVal := os.Getenv(k)
envVarBackup[k] = oldVal
os.Unsetenv(k)
}
envOverride(config)
if config.Timeout != expConfig.Timeout {
t.Errorf("Default timeout is overriding specified timeout")
}
if !cmp.Equal(config.Storage.Minio, expConfig.Storage.Minio) {
t.Errorf("Default Minio config is overriding specified config")
}
restoreEnv(envVarBackup)
}
func getEnvMap(config *Config) map[string]string {
envVars := map[string]string{
pkg/fixtures/mongo.go
-55
View File
@@ -1,55 +0,0 @@
package fixtures
import (
"github.com/globalsign/mgo"
"github.com/stretchr/testify/suite"
"github.com/technosophos/moniker"
)
var names = moniker.NewAlliterator()
// DefaultMongoURL is the default Mongo URL for testing
const DefaultMongoURL = "127.0.0.1:27017"
// Mongo is a text fixture for use with github.com/stretchr/testify/suite tests
//
// use it like this:
//
// type MyTests struct {
// *fixtures.Mongo
// }
//
// func RunMyTests(t *testing.T) {
// suite.Run(t, &MyTests{Mongo: New(DefaultURL)})
// }
type Mongo struct {
suite.Suite
url string
DBName string
DB *mgo.Database
}
// SetupTest creates a new mongo connection and DB, and attaches it to a
// session before each test executes
//
// This implements the SetupTestSuite interface
func (m *Mongo) SetupTest() {
sess, err := mgo.Dial(m.url)
m.Require().NoError(err)
m.DB = sess.DB(m.DBName)
}
// TearDownTest drops the database that was created in SetupTest
//
// This implements the TearDownTestSuite interface
func (m *Mongo) TearDownTest() {
m.Require().NoError(m.DB.DropDatabase())
}
// NewMongo creates a new Mongo test fixture
func NewMongo(url string) *Mongo {
return &Mongo{
url: url,
DBName: names.NameSep("-") + "-athens-testing",
}
}
pkg/storage/compliance/benchmarks.go
+112
View File
@@ -0,0 +1,112 @@
package compliance
import (
"bytes"
"context"
"fmt"
"io/ioutil"
"testing"
"github.com/gomods/athens/pkg/storage"
"github.com/stretchr/testify/require"
)
// RunBenchmarks takes a backend and runs benchmarks against
// saving and loading modules.
func RunBenchmarks(b *testing.B, s storage.Backend, clear func() error) {
benchList(b, s, clear)
benchSave(b, s, clear)
benchDelete(b, s, clear)
benchExists(b, s, clear)
}
func benchList(b *testing.B, s storage.Backend, clear func() error) {
require.NoError(b, clear())
defer require.NoError(b, clear())
module, version := "benchListModule", "1.0.1"
mock := getMockModule()
err := s.Save(
context.Background(),
module,
version,
mock.Mod,
mock.Zip,
mock.Info,
)
require.NoError(b, err, "save for storage failed")
b.Run("list", func(b *testing.B) {
for i := 0; i < b.N; i++ {
_, err := s.List(context.Background(), module)
require.NoError(b, err, "Error in listing module")
}
})
}
func benchSave(b *testing.B, s storage.Backend, clear func() error) {
require.NoError(b, clear())
defer require.NoError(b, clear())
module, version := "benchSaveModule", "1.0.1"
mock := getMockModule()
zipBts, err := ioutil.ReadAll(mock.Zip)
require.NoError(b, err)
mi := 0
ctx := context.Background()
b.Run("save", func(b *testing.B) {
for i := 0; i < b.N; i++ {
err := s.Save(
ctx,
fmt.Sprintf("save-%s-%d", module, mi),
version,
mock.Mod,
bytes.NewReader(zipBts),
mock.Info,
)
require.NoError(b, err)
mi++
}
})
}
func benchDelete(b *testing.B, s storage.Backend, clear func() error) {
require.NoError(b, clear())
defer require.NoError(b, clear())
module, version := "benchDeleteModule", "1.0.1"
mock := getMockModule()
zipBts, err := ioutil.ReadAll(mock.Zip)
require.NoError(b, err)
ctx := context.Background()
b.Run("delete", func(b *testing.B) {
for i := 0; i < b.N; i++ {
name := fmt.Sprintf("del-%s-%d", module, i)
err := s.Save(ctx, name, version, mock.Mod, bytes.NewReader(zipBts), mock.Info)
require.NoError(b, err, "saving %s for storage failed", name)
err = s.Delete(ctx, name, version)
require.NoError(b, err, "delete failed: %s", name)
}
})
}
func benchExists(b *testing.B, s storage.Backend, clear func() error) {
require.NoError(b, clear())
defer require.NoError(b, clear())
module, version := "benchExistsModule", "1.0.1"
mock := getMockModule()
ctx := context.Background()
err := s.Save(ctx, module, version, mock.Mod, mock.Zip, mock.Info)
require.NoError(b, err)
b.Run("exists", func(b *testing.B) {
for i := 0; i < b.N; i++ {
exists, err := s.Exists(ctx, module, version)
require.NoError(b, err)
require.True(b, exists)
}
})
}
pkg/storage/compliance/tests.go
+129
View File
@@ -0,0 +1,129 @@
package compliance
import (
"bytes"
"context"
"fmt"
"io/ioutil"
"math/rand"
"testing"
"github.com/gomods/athens/pkg/errors"
"github.com/gomods/athens/pkg/storage"
"github.com/stretchr/testify/require"
)
// RunTests takes a backend implementation and runs compliance tests
// against the interface.
func RunTests(t *testing.T, b storage.Backend, clearBackend func() error) {
require.NoError(t, clearBackend(), "clearing backend failed")
defer require.NoError(t, clearBackend(), "clearning backend failed")
testNotFound(t, b)
testList(t, b)
testDelete(t, b)
testGet(t, b)
}
// testNotFound ensures that a storage Backend
// returns a KindNotFound error when asking for
// non existing modules.
func testNotFound(t *testing.T, b storage.Backend) {
mod, ver := "xxx", "yyy"
ctx := context.Background()
err := b.Delete(ctx, mod, ver)
require.Error(t, err)
require.Equal(t, errors.KindNotFound, errors.Kind(err))
_, err = b.GoMod(ctx, mod, ver)
require.Error(t, err)
require.Equal(t, errors.KindNotFound, errors.Kind(err))
_, err = b.Info(ctx, mod, ver)
require.Error(t, err)
require.Equal(t, errors.KindNotFound, errors.Kind(err))
vs, err := b.List(ctx, mod)
require.NoError(t, err)
require.Equal(t, 0, len(vs))
_, err = b.Zip(ctx, mod, ver)
require.Error(t, err)
require.Equal(t, errors.KindNotFound, errors.Kind(err))
}
// testList tests that a storage Backend returns
// the exact list of versions that are saved.
func testList(t *testing.T, b storage.Backend) {
ctx := context.Background()
modname := "listMod"
versions := []string{"v1.1.0", "v1.2.0", "v1.3.0"}
for _, version := range versions {
mock := getMockModule()
err := b.Save(
ctx,
modname,
version,
mock.Mod,
mock.Zip,
mock.Info,
)
require.NoError(t, err, "Save for storage failed")
}
retVersions, err := b.List(ctx, modname)
require.NoError(t, err)
require.Equal(t, versions, retVersions)
}
// testGet saves and retrieves a module successfully.
func testGet(t *testing.T, b storage.Backend) {
ctx := context.Background()
modname := "getTestModule"
ver := "v1.2.3"
mock := getMockModule()
zipBts, _ := ioutil.ReadAll(mock.Zip)
b.Save(ctx, modname, ver, mock.Mod, bytes.NewReader(zipBts), mock.Info)
info, err := b.Info(ctx, modname, ver)
require.NoError(t, err)
require.Equal(t, mock.Info, info)
mod, err := b.GoMod(ctx, modname, ver)
require.NoError(t, err)
require.Equal(t, mock.Mod, mod)
zip, err := b.Zip(ctx, modname, ver)
require.NoError(t, err)
givenZipBts, err := ioutil.ReadAll(zip)
require.NoError(t, err)
require.Equal(t, zipBts, givenZipBts)
}
// testDelete tests that a module can be deleted from a
// storage Backend and the the Exists method returns false
// afterwards.
func testDelete(t *testing.T, b storage.Backend) {
ctx := context.Background()
modname := "deleteModule"
version := fmt.Sprintf("%s%d", "delete", rand.Int())
mock := getMockModule()
err := b.Save(ctx, modname, version, mock.Mod, mock.Zip, mock.Info)
require.NoError(t, err)
err = b.Delete(ctx, modname, version)
require.NoError(t, err)
exists, err := b.Exists(ctx, modname, version)
require.NoError(t, err)
require.Equal(t, false, exists)
}
func getMockModule() *storage.Version {
return &storage.Version{
Info: []byte("123"),
Mod: []byte("456"),
Zip: ioutil.NopCloser(bytes.NewReader([]byte("789"))),
}
}
pkg/storage/fs/all_test.go
-39
View File
@@ -1,39 +0,0 @@
package fs
import (
"testing"
"github.com/gomods/athens/pkg/storage"
"github.com/spf13/afero"
"github.com/stretchr/testify/suite"
)
const (
module = "testmodule"
version = "v1.0.0"
)
type FsTests struct {
suite.Suite
storage storage.Backend
rootDir string
fs afero.Fs
}
func (d *FsTests) SetupTest() {
memFs := afero.NewOsFs()
r, err := afero.TempDir(memFs, "", "athens-fs-tests")
d.Require().NoError(err)
d.storage, err = NewStorage(r, memFs)
d.Require().NoError(err)
d.rootDir = r
d.fs = memFs
}
func (d *FsTests) TearDownTest() {
d.Require().NoError(d.fs.RemoveAll(d.rootDir))
}
func TestDiskStorage(t *testing.T) {
suite.Run(t, new(FsTests))
}
pkg/storage/fs/fs_test.go
+39 -8
View File
@@ -1,14 +1,45 @@
package fs
import (
"path/filepath"
"os"
"testing"
"github.com/gomods/athens/pkg/storage/compliance"
"github.com/spf13/afero"
"github.com/stretchr/testify/require"
)
func (d *FsTests) TestLocationFuncs() {
r := d.Require()
storage := d.storage.(*storageImpl)
moduleLoc := storage.moduleLocation(module)
r.Equal(filepath.Join(d.rootDir, module), moduleLoc)
versionedLoc := storage.versionLocation(module, version)
r.Equal(filepath.Join(d.rootDir, module, version), versionedLoc)
func TestBackend(t *testing.T) {
fs := afero.NewMemMapFs()
b := getStorage(t, fs)
compliance.RunTests(t, b, b.clear)
fs.RemoveAll(b.rootDir)
}
func BenchmarkBackend(b *testing.B) {
fs := afero.NewOsFs()
backend := getStorage(b, fs)
compliance.RunBenchmarks(b, backend, backend.clear)
fs.RemoveAll(backend.rootDir)
}
func BenchmarkMemory(b *testing.B) {
backend := getStorage(b, afero.NewMemMapFs())
compliance.RunBenchmarks(b, backend, backend.clear)
}
func (s *storageImpl) clear() error {
if err := s.filesystem.RemoveAll(s.rootDir); err != nil {
return err
}
return s.filesystem.Mkdir(s.rootDir, os.ModeDir|os.ModePerm)
}
func getStorage(tb testing.TB, fs afero.Fs) *storageImpl {
tb.Helper()
dir, err := afero.TempDir(fs, "", "athens-fs-test")
require.NoError(tb, err, "could not create temp dir")
backend, err := NewStorage(dir, fs)
require.NoError(tb, err)
return backend.(*storageImpl)
}
pkg/storage/fs/test_suite.go
-48
View File
@@ -1,48 +0,0 @@
package fs
import (
"github.com/gomods/athens/pkg/storage"
"github.com/spf13/afero"
)
// TestSuite implements storage.TestSuite interface
type TestSuite struct {
storage storage.Backend
fs afero.Fs
rootDir string
}
// NewTestSuite creates a common test suite
func NewTestSuite() (storage.TestSuite, error) {
osFs := afero.NewOsFs()
r, err := afero.TempDir(osFs, "", "athens-fs-storage-tests")
if err != nil {
return nil, err
}
fsStore, err := NewStorage(r, osFs)
if err != nil {
return nil, err
}
return &TestSuite{
fs: osFs,
rootDir: r,
storage: fsStore,
}, nil
}
// Storage retrieves initialized storage backend
func (ts *TestSuite) Storage() storage.Backend {
return ts.storage
}
// StorageHumanReadableName retrieves readable identifier of the storage
func (ts *TestSuite) StorageHumanReadableName() string {
return "FileSystem"
}
// Cleanup tears down test
func (ts *TestSuite) Cleanup() error {
return ts.fs.RemoveAll(ts.rootDir)
}
pkg/storage/mem/test_suite.go
-34
View File
@@ -1,34 +0,0 @@
package mem
import (
"github.com/gomods/athens/pkg/storage"
)
// TestSuite implements storage.TestSuite interface
type TestSuite struct {
storage storage.Backend
}
// NewTestSuite creates a common test suite
func NewTestSuite() (storage.TestSuite, error) {
memStore, err := NewStorage()
return &TestSuite{
storage: memStore,
}, err
}
// Storage retrieves initialized storage backend
func (ts *TestSuite) Storage() storage.Backend {
return ts.storage
}
// StorageHumanReadableName retrieves readable identifier of the storage
func (ts *TestSuite) StorageHumanReadableName() string {
return "In-memory"
}
// Cleanup tears down test
func (ts *TestSuite) Cleanup() error {
return nil
}
pkg/storage/minio/minio_test.go
+53
View File
@@ -0,0 +1,53 @@
package minio
import (
"os"
"testing"
"github.com/gomods/athens/pkg/config"
"github.com/gomods/athens/pkg/storage/compliance"
)
func TestBackend(t *testing.T) {
backend := getStorage(t)
compliance.RunTests(t, backend, backend.clear)
}
func BenchmarkBackend(b *testing.B) {
backend := getStorage(b)
compliance.RunBenchmarks(b, backend, backend.clear)
}
func (s *storageImpl) clear() error {
doneCh := make(chan struct{})
defer close(doneCh)
objectCh := s.minioClient.ListObjectsV2(s.bucketName, "", true, doneCh)
for object := range objectCh {
if object.Err != nil {
return object.Err
}
if err := s.minioClient.RemoveObject(s.bucketName, object.Key); err != nil {
return err
}
}
return nil
}
func getStorage(t testing.TB) *storageImpl {
url := os.Getenv("ATHENS_MINIO_ENDPOINT")
if url == "" {
t.SkipNow()
}
backend, err := NewStorage(&config.MinioConfig{
Endpoint: url,
Key: "minio",
Secret: "minio123",
Bucket: "gomods",
})
if err != nil {
t.Fatal(err)
}
return backend.(*storageImpl)
}
pkg/storage/minio/test_suite.go
-59
View File
@@ -1,59 +0,0 @@
package minio
import (
"fmt"
"github.com/gomods/athens/pkg/config"
"github.com/gomods/athens/pkg/storage"
minio "github.com/minio/minio-go"
)
// TestSuite implements storage.TestSuite interface
type TestSuite struct {
storage storage.Backend
conf *config.MinioConfig
}
// NewTestSuite creates a common test suite
func NewTestSuite(conf *config.MinioConfig) (storage.TestSuite, error) {
minioStorage, err := newTestStore(conf)
return &TestSuite{
storage: minioStorage,
conf: conf,
}, err
}
func newTestStore(conf *config.MinioConfig) (storage.Backend, error) {
minioStore, err := NewStorage(conf)
if err != nil {
return nil, fmt.Errorf("Not able to connect to minio storage: %s", err.Error())
}
return minioStore, nil
}
// Storage retrieves initialized storage backend
func (ts *TestSuite) Storage() storage.Backend {
return ts.storage
}
// StorageHumanReadableName retrieves readable identifier of the storage
func (ts *TestSuite) StorageHumanReadableName() string {
return "Minio"
}
// Cleanup tears down test
func (ts *TestSuite) Cleanup() error {
minioClient, _ := minio.New(ts.conf.Endpoint, ts.conf.Key, ts.conf.Secret, ts.conf.EnableSSL)
doneCh := make(chan struct{})
defer close(doneCh)
objectCh := minioClient.ListObjectsV2(ts.conf.Bucket, "", true, doneCh)
for object := range objectCh {
//TODO: could return multi error and clean other objects
if err := minioClient.RemoveObject(ts.conf.Bucket, object.Key); err != nil {
return err
}
}
return nil
}
pkg/storage/mongo/local_test.go
-46
View File
@@ -1,46 +0,0 @@
package mongo
import (
"path/filepath"
"testing"
"github.com/gomods/athens/pkg/config"
"github.com/stretchr/testify/suite"
)
var (
testConfigFile = filepath.Join("..", "..", "..", "config.dev.toml")
)
type MongoTests struct {
suite.Suite
}
func (m *MongoTests) SetupTest() {
conf, err := config.GetConf(testConfigFile)
if err != nil {
m.T().Fatalf("Unable to parse config file: %s", err.Error())
}
_, err = newTestStore(conf.Storage.Mongo)
m.Require().NoError(err)
}
func (m *MongoTests) TestNewMongoStorage() {
r := m.Require()
conf, err := config.GetConf(testConfigFile)
if err != nil {
m.T().Fatalf("Unable to parse config file: %s", err.Error())
}
getterSaver, err := NewStorage(conf.Storage.Mongo)
r.NoError(err)
r.NotNil(getterSaver.c)
r.NotNil(getterSaver.d)
r.NotNil(getterSaver.s)
r.Equal(getterSaver.url, conf.Storage.Mongo.URL)
}
func TestMongoStorage(t *testing.T) {
suite.Run(t, new(MongoTests))
}
pkg/storage/mongo/mongo.go
+5 -1
View File
@@ -37,8 +37,8 @@ func NewStorage(conf *config.MongoConfig) (*ModuleStore, error) {
if err != nil {
return nil, errors.E(op, err)
}
return ms, nil
return ms, nil
}
func (m *ModuleStore) connect() error {
@@ -50,6 +50,10 @@ func (m *ModuleStore) connect() error {
return errors.E(op, err)
}
return m.initDatabase()
}
func (m *ModuleStore) initDatabase() error {
// TODO: database and collection as env vars, or params to New()? together with user/mongo
m.d = "athens"
m.c = "modules"
pkg/storage/mongo/mongo_test.go
+38
View File
@@ -0,0 +1,38 @@
package mongo
import (
"os"
"testing"
"github.com/gomods/athens/pkg/config"
"github.com/gomods/athens/pkg/storage/compliance"
"github.com/stretchr/testify/require"
)
func TestBackend(t *testing.T) {
backend := getStorage(t)
compliance.RunTests(t, backend, backend.clear)
}
func (m *ModuleStore) clear() error {
m.s.DB(m.d).C(m.c).DropCollection()
return m.initDatabase()
}
func BenchmarkBackend(b *testing.B) {
backend := getStorage(b)
compliance.RunBenchmarks(b, backend, backend.clear)
}
func getStorage(tb testing.TB) *ModuleStore {
url := os.Getenv("ATHENS_MONGO_URL")
if url == "" {
tb.SkipNow()
}
backend, err := NewStorage(&config.MongoConfig{URL: url})
require.NoError(tb, err)
return backend
}
pkg/storage/mongo/test_suite.go
-56
View File
@@ -1,56 +0,0 @@
package mongo
import (
"fmt"
"github.com/globalsign/mgo"
"github.com/gomods/athens/pkg/config"
"github.com/gomods/athens/pkg/storage"
)
// TestSuite implements storage.TestSuite interface
type TestSuite struct {
storage *ModuleStore
conf *config.MongoConfig
}
// NewTestSuite creates a common test suite
func NewTestSuite(conf *config.MongoConfig) (storage.TestSuite, error) {
ms, err := newTestStore(conf)
if err != nil {
return nil, err
}
return &TestSuite{
storage: ms,
conf: conf,
}, err
}
func newTestStore(conf *config.MongoConfig) (*ModuleStore, error) {
mongoStore, err := NewStorage(conf)
if err != nil {
return nil, fmt.Errorf("Not able to connect to mongo storage: %s", err.Error())
}
return mongoStore, nil
}
// Storage retrieves initialized storage backend
func (ts *TestSuite) Storage() storage.Backend {
return ts.storage
}
// StorageHumanReadableName retrieves readable identifier of the storage
func (ts *TestSuite) StorageHumanReadableName() string {
return "Mongo"
}
// Cleanup tears down test
func (ts *TestSuite) Cleanup() error {
s, err := mgo.DialWithTimeout(ts.conf.URL, ts.conf.TimeoutDuration())
if err != nil {
return nil
}
defer s.Close()
return s.DB(ts.storage.d).C(ts.storage.c).DropCollection()
}
pkg/storage/storage_tests/module_storage/storage_benchmark_test.go
-148
View File
@@ -1,148 +0,0 @@
package modulestorage
import (
"bytes"
"context"
"fmt"
"testing"
"github.com/gomods/athens/pkg/config"
"github.com/gomods/athens/pkg/errors"
"github.com/gomods/athens/pkg/storage"
"github.com/gomods/athens/pkg/storage/fs"
"github.com/gomods/athens/pkg/storage/mem"
"github.com/gomods/athens/pkg/storage/minio"
"github.com/gomods/athens/pkg/storage/mongo"
"github.com/stretchr/testify/require"
)
func BenchmarkStorageList(b *testing.B) {
module, version := "athens_module", "1.0.1"
zip, info, mod := []byte("zip_data"), []byte("info"), []byte("mod_info")
for _, store := range getStores(b) {
backend := store.Storage()
require.NoError(b, backend.Save(context.Background(), module, version, mod, bytes.NewReader(zip), info), "Save for storage %s failed", backend)
b.ResetTimer()
b.Run(store.StorageHumanReadableName(), func(b *testing.B) {
for i := 0; i < b.N; i++ {
_, err := backend.List(context.Background(), module)
require.NoError(b, err, "Error in listing module")
}
})
require.NoError(b, store.Cleanup())
}
}
func BenchmarkStorageSave(b *testing.B) {
module, version := "athens_module", "1.0.1"
zip, info, mod := []byte("zip_data"), []byte("info"), []byte("mod_info")
for _, store := range getStores(b) {
backend := store.Storage()
b.ResetTimer()
mi := 0
b.Run(store.StorageHumanReadableName(), func(b *testing.B) {
for i := 0; i < b.N; i++ {
err := backend.Save(context.Background(), fmt.Sprintf("save-%s-%d", module, mi), version, mod, bytes.NewReader(zip), info)
require.NoError(b, err)
mi++
}
})
require.NoError(b, store.Cleanup())
}
}
func BenchmarkStorageSaveAndDelete(b *testing.B) {
module, version := "athens_module", "1.0.1"
zip, info, mod := []byte("zip_data"), []byte("info"), []byte("mod_info")
for _, store := range getStores(b) {
backend := store.Storage()
b.ResetTimer()
b.Run(store.StorageHumanReadableName(), func(b *testing.B) {
for i := 0; i < b.N; i++ {
name := fmt.Sprintf("del-%s-%d", module, i)
err := backend.Save(context.Background(), name, version, mod, bytes.NewReader(zip), info)
require.NoError(b, err, "save for storage %s module: %s failed", backend, name)
err = backend.Delete(context.Background(), name, version)
require.NoError(b, err, "delete failed: %s", name)
}
})
require.NoError(b, store.Cleanup())
}
}
func BenchmarkStorageDeleteNonExistingModules(b *testing.B) {
module, version := "random-module", "version"
for _, store := range getStores(b) {
backend := store.Storage()
b.ResetTimer()
b.Run(store.StorageHumanReadableName(), func(b *testing.B) {
for i := 0; i < b.N; i++ {
err := backend.Delete(context.Background(), fmt.Sprintf("del-%s-%d", module, i), version)
require.Equal(b, errors.KindNotFound, errors.Kind(err))
}
})
}
}
func BenchmarkStorageExists(b *testing.B) {
module, version := "athens_module", "1.0.1"
zip, info, mod := []byte("zip_data"), []byte("info"), []byte("mod_info")
moduleName := fmt.Sprintf("existing-%s", module)
for _, store := range getStores(b) {
backend := store.Storage()
err := backend.Save(context.Background(), moduleName, version, mod, bytes.NewReader(zip), info)
require.NoError(b, err, "exists for storage %s failed", backend)
b.ResetTimer()
b.Run(store.StorageHumanReadableName(), func(b *testing.B) {
for i := 0; i < b.N; i++ {
exists, err := backend.Exists(context.Background(), moduleName, version)
require.NoError(b, err)
require.True(b, exists)
}
})
require.NoError(b, store.Cleanup())
}
}
func getStores(b *testing.B) []storage.TestSuite {
var stores []storage.TestSuite
conf, err := config.GetConf(testConfigFile)
require.NoError(b, err)
//TODO: create the instance without model or TestSuite
fsStore, err := fs.NewTestSuite()
require.NoError(b, err, "couldn't create filesystem store")
stores = append(stores, fsStore)
mongoStore, err := mongo.NewTestSuite(conf.Storage.Mongo)
require.NoError(b, err, "couldn't create mongo store")
stores = append(stores, mongoStore)
memStore, err := mem.NewTestSuite()
require.NoError(b, err)
stores = append(stores, memStore)
minioStore, err := minio.NewTestSuite(conf.Storage.Minio)
require.NoError(b, err)
stores = append(stores, minioStore)
return stores
}
pkg/storage/storage_tests/module_storage/storage_test.go
-196
View File
@@ -1,196 +0,0 @@
package modulestorage
// The goal of this package is to provide common testing ground for all
// storage tests which can be extracted to achieve more DRY code.
// This package should contain black box tests such as Save-Get roundtrip,
// Error checking, etc.
// More detailed tests verifying internal behavior should be still implemented
// as part of specific storage packages.
import (
"bytes"
"context"
"fmt"
"io/ioutil"
"math/rand"
"path/filepath"
"testing"
"github.com/gobuffalo/suite"
"github.com/gomods/athens/pkg/config"
"github.com/gomods/athens/pkg/errors"
"github.com/gomods/athens/pkg/storage"
"github.com/gomods/athens/pkg/storage/fs"
"github.com/gomods/athens/pkg/storage/mem"
"github.com/gomods/athens/pkg/storage/minio"
"github.com/gomods/athens/pkg/storage/mongo"
)
var (
testConfigFile = filepath.Join("..", "..", "..", "..", "config.dev.toml")
)
type TestSuites struct {
*suite.Model
storages []storage.TestSuite
module string
version string
mod []byte
zip []byte
info []byte
}
func (d *TestSuites) SetupTest() {
ra := d.Require()
conf, err := config.GetConf(testConfigFile)
ra.NoError(err)
// file system
fsTests, err := fs.NewTestSuite()
ra.NoError(err)
d.storages = append(d.storages, fsTests)
// mem
memStore, err := mem.NewTestSuite()
ra.NoError(err)
d.storages = append(d.storages, memStore)
// minio
minioStorage, err := minio.NewTestSuite(conf.Storage.Minio)
ra.NoError(err)
d.storages = append(d.storages, minioStorage)
// mongo
mongoStore, err := mongo.NewTestSuite(conf.Storage.Mongo)
ra.NoError(err)
d.storages = append(d.storages, mongoStore)
d.module = "testmodule"
d.version = "v1.0.0"
d.mod = []byte("123")
d.zip = []byte("456")
d.info = []byte("789")
}
func TestModuleStorages(t *testing.T) {
suite.Run(t, &TestSuites{Model: suite.NewModel()})
}
func (d *TestSuites) TestStorages() {
for _, store := range d.storages {
// start the test with a clean store
store.Cleanup()
d.testNotFound(store)
d.testKindNotFound(store)
d.testGetSaveListRoundTrip(store)
d.testList(store)
d.testDelete(store)
// TODO: more tests to come
d.Require().NoError(store.Cleanup())
}
}
func (d *TestSuites) testNotFound(ts storage.TestSuite) {
_, err := ts.Storage().Info(context.Background(), "some", "unknown")
d.Require().Equal(true, errors.IsNotFoundErr(err), "Invalid error type for %s: %#v", ts.StorageHumanReadableName(), err)
}
func (d *TestSuites) testKindNotFound(ts storage.TestSuite) {
s := ts.Storage()
mod, ver := "xxx", "yyy"
ctx := context.Background()
r := d.Require()
hrn := ts.StorageHumanReadableName()
err := s.Delete(ctx, mod, ver)
r.Error(err, hrn)
r.Equal(errors.KindNotFound, errors.Kind(err), hrn)
_, err = s.GoMod(ctx, mod, ver)
r.Error(err, hrn)
r.Equal(errors.KindNotFound, errors.Kind(err), hrn)
_, err = s.Info(ctx, mod, ver)
r.Error(err, hrn)
r.Equal(errors.KindNotFound, errors.Kind(err), hrn)
vs, err := s.List(ctx, mod)
r.NoError(err)
r.Equal(0, len(vs))
_, err = s.Zip(ctx, mod, ver)
r.Error(err, hrn)
r.Equal(errors.KindNotFound, errors.Kind(err), hrn)
}
func (d *TestSuites) testList(ts storage.TestSuite) {
r := d.Require()
hrn := ts.StorageHumanReadableName()
versions := []string{"v1.1.0", "v1.2.0", "v1.3.0"}
for _, version := range versions {
r.NoError(ts.Storage().Save(context.Background(), d.module, version, d.mod, bytes.NewReader(d.zip), d.info), "Save for storage %s failed", hrn)
}
// append version from save-get roundtrip
versions = append([]string{d.version}, versions...)
retVersions, err := ts.Storage().List(context.Background(), d.module)
r.NoError(err, hrn)
r.Equal(versions, retVersions, hrn)
}
func (d *TestSuites) testGetSaveListRoundTrip(ts storage.TestSuite) {
r := d.Require()
hrn := ts.StorageHumanReadableName()
ts.Storage().Save(context.Background(), d.module, d.version, d.mod, bytes.NewReader(d.zip), d.info)
listedVersions, err := ts.Storage().List(context.Background(), d.module)
r.NoError(err, hrn)
r.Equal(1, len(listedVersions), hrn)
retVersion := listedVersions[0]
r.Equal(d.version, retVersion, hrn)
info, err := ts.Storage().Info(context.Background(), d.module, d.version)
r.NoError(err, hrn)
r.Equal(d.info, info, hrn)
mod, err := ts.Storage().GoMod(context.Background(), d.module, d.version)
r.NoError(err, hrn)
r.Equal(d.mod, mod, hrn)
zip, err := ts.Storage().Zip(context.Background(), d.module, d.version)
r.NoError(err, hrn)
zipContent, err := ioutil.ReadAll(zip)
r.NoError(err, hrn)
r.Equal(d.zip, zipContent, hrn)
r.NoError(zip.Close())
}
func (d *TestSuites) testDelete(ts storage.TestSuite) {
r := d.Require()
version := fmt.Sprintf("%s%d", "delete", rand.Int())
err := ts.Storage().Save(context.Background(), d.module, version, d.mod, bytes.NewReader(d.zip), d.info)
r.NoError(err)
tests := []struct {
module string
version string
want int
}{
{
module: "does/not/exist",
version: "v1.0.0",
want: errors.KindNotFound,
},
{
module: d.module,
version: version,
want: errors.KindUnexpected,
},
}
ctx := context.Background()
for _, test := range tests {
err := ts.Storage().Delete(ctx, test.module, test.version)
r.Equal(test.want, errors.Kind(err), ts.StorageHumanReadableName())
exists, err := ts.Storage().Exists(ctx, test.module, test.version)
r.NoError(err)
r.Equal(false, exists, ts.StorageHumanReadableName())
}
}
pkg/storage/test_suite.go
-8
View File
@@ -1,8 +0,0 @@
package storage
// TestSuite is common interface which each storage needs to implement
type TestSuite interface {
Storage() Backend
StorageHumanReadableName() string
Cleanup() error
}
vendor/github.com/globalsign/mgo/session.go
Generated Vendored
-1
View File
@@ -2912,7 +2912,6 @@ func (p *Pipe) SetMaxTime(d time.Duration) *Pipe {
return p
}
// Collation allows to specify language-specific rules for string comparison,
// such as rules for lettercase and accent marks.
// When specifying collation, the locale field is mandatory; all other collation