Minio conf + tests refactoring (#680)

* cleanup tests and change minio port

* fix cleanup

* cleanup

* fix config test

* add comment to travis

* revert to generic minio addr

* fix test

* switch to test config

* adapt timeouts

* use example config

* fix test... again

* add new lines

.travis.yml

@@ -20,7 +20,9 @@ before_script:
   - mkdir -p $ATHENS_DIR && rsync -azr . $ATHENS_DIR && cd $ATHENS_DIR
   - make setup-dev-env
   - wget "https://dl.minio.io/server/minio/release/linux-amd64/minio"
-  - chmod +x minio && nohup ./minio server . &
+  # moving from the default port 9000 so it doesn't conflich with other services (i.e. vscode extensions server)
+  # we also need to change it in travis because the tests are using the config.test.toml where port 9001 is used
+  - chmod +x minio && nohup ./minio server --address ":9001" . &
 
 script:
   - make verify test-unit test-e2e

config.example.toml

@@ -184,15 +184,15 @@ EnableCSRFProtection = false
     [Storage.Minio]
         # Endpoint for Minio storage
         # Env override: ATHENS_MINIO_ENDPOINT
-        Endpoint = "minio.example.com"
+        Endpoint = "127.0.0.1:9001"
 
         # Access Key for Minio storage
         # Env override: ATHENS_MINIO_ACCESS_KEY_ID
-        Key = "MY_KEY"
+        Key = "minio"
 
         # Secret Key for Minio storage
         # Env override: ATHENS_MINIO_SECRET_ACCESS_KEY
-        Secret = "MY_SECRET"
+        Secret = "minio123"
 
         # Timeout for networks calls made to Minio in seconds
         # Defaults to Global Timeout
@@ -201,8 +201,8 @@ EnableCSRFProtection = false
         # Enable SSL for Minio connections
         # Defaults to true
         # Env override: ATHENS_MINIO_USE_SSL
-        EnableSSL = true
+        EnableSSL = false
-
+        
         # Minio Bucket to use for storage
         # Defaults to gomods
         # Env override: ATHENS_MINIO_BUCKET_NAME

config.test.toml

@@ -62,7 +62,7 @@ FilterFile = "filter.conf"
 # This value is used as the default for storage backends if they don't specify timeouts
 # Defaults to 300
 # Env override: ATHENS_TIMEOUT
-Timeout = 1
+Timeout = 300
 
 # EnableCSRFProtection determines whether to enable CSRF protection.
 # Defaults to false
@@ -145,7 +145,7 @@ EnableCSRFProtection = false
         Endpoint = "cdn.example.com"
         # Timeout for networks calls made to the CDN in seconds
         # Defaults to Global Timeout
-        Timeout = 1
+        Timeout = 300
     [Storage.Disk]
         # RootPath is the Athens Disk Root folder 
         # Env override: ATHENS_DISK_STORAGE_ROOT
@@ -159,24 +159,24 @@ EnableCSRFProtection = false
         Bucket = "MY_GCP_BUCKET"
         # Timeout for networks calls made to GCP in seconds
         # Defaults to Global Timeout
-        Timeout = 1
+        Timeout = 300
     [Storage.Minio]
         # Endpoint for Minio storage
         # Env override: ATHENS_MINIO_ENDPOINT
-        Endpoint = "minio.example.com"
+        Endpoint = "127.0.0.1:9001"
         # Access Key for Minio storage
         # Env override: ATHENS_MINIO_ACCESS_KEY_ID
-        Key = "MY_KEY"
+        Key = "minio"
         # Secret Key for Minio storage
         # Env override: ATHENS_MINIO_SECRET_ACCESS_KEY
-        Secret = "MY_SECRET"
+        Secret = "minio123"
         # Timeout for networks calls made to Minio in seconds
         # Defaults to Global Timeout
-        Timeout = 1
+        Timeout = 300
         # Enable SSL for Minio connections
         # Defaults to true
         # Env override: ATHENS_MINIO_USE_SSL
-        EnableSSL = true
+        EnableSSL = false
         # Minio Bucket to use for storage
         # Defaults to gomods
         # Env override: ATHENS_MINIO_BUCKET_NAME
@@ -191,4 +191,4 @@ EnableCSRFProtection = false
         # Timeout for networks calls made to Mongo in seconds
         # Defaults to Global Timeout
         # Env override: MONGO_CONN_TIMEOUT_SEC
-        Timeout = 1
+        Timeout = 300

docker-compose.yml

@@ -13,7 +13,7 @@ services:
     image: minio/minio:latest
     command: server /data
     ports:
-      - "9000:9000"
+      - "9001:9000"
     environment:
       MINIO_ACCESS_KEY: minio
       MINIO_SECRET_KEY: minio123

pkg/config/parse_test.go

@@ -232,10 +232,10 @@ func TestParseExampleConfig(t *testing.T) {
 			},
 		},
 		Minio: &MinioConfig{
-			Endpoint:  "minio.example.com",
+			Endpoint:  "127.0.0.1:9001",
-			Key:       "MY_KEY",
+			Key:       "minio",
-			Secret:    "MY_SECRET",
+			Secret:    "minio123",
-			EnableSSL: true,
+			EnableSSL: false,
 			Bucket:    "gomods",
 			TimeoutConf: TimeoutConf{
 				Timeout: globalTimeout,

pkg/storage/fs/test_suite.go

@@ -16,20 +16,20 @@ type TestSuite struct {
 
 // NewTestSuite creates a common test suite
 func NewTestSuite(model *suite.Model) (storage.TestSuite, error) {
-	memFs := afero.NewOsFs()
+	osFs := afero.NewOsFs()
-	r, err := afero.TempDir(memFs, "", "athens-fs-storage-tests")
+	r, err := afero.TempDir(osFs, "", "athens-fs-storage-tests")
 	if err != nil {
 		return nil, err
 	}
 
-	fsStore, err := NewStorage(r, memFs)
+	fsStore, err := NewStorage(r, osFs)
 	if err != nil {
 		return nil, err
 	}
 
 	return &TestSuite{
 		Model:   model,
-		fs:      memFs,
+		fs:      osFs,
 		rootDir: r,
 		storage: fsStore,
 	}, nil

pkg/storage/minio/all_test.go

@@ -1,47 +0,0 @@
-package minio
-
-import (
-	"testing"
-
-	"github.com/gomods/athens/pkg/config"
-	"github.com/gomods/athens/pkg/storage"
-	minio "github.com/minio/minio-go"
-	"github.com/stretchr/testify/suite"
-)
-
-type MinioTests struct {
-	suite.Suite
-	storage                                            storage.Backend
-	endpoint, accessKeyID, secretAccessKey, bucketName string
-}
-
-func (d *MinioTests) SetupTest() {
-	d.endpoint = "127.0.0.1:9000"
-	d.bucketName = "gomods"
-	d.accessKeyID = "minio"
-	d.secretAccessKey = "minio123"
-	conf := &config.MinioConfig{
-		Endpoint:  d.endpoint,
-		Bucket:    d.bucketName,
-		Key:       d.accessKeyID,
-		Secret:    d.secretAccessKey,
-		EnableSSL: false,
-	}
-	storage, err := NewStorage(conf)
-	d.Require().NoError(err)
-	d.storage = storage
-}
-
-func (d *MinioTests) TearDownTest() {
-	minioClient, _ := minio.New(d.endpoint, d.accessKeyID, d.secretAccessKey, false)
-	doneCh := make(chan struct{})
-	defer close(doneCh)
-	objectCh := minioClient.ListObjectsV2(d.bucketName, "", true, doneCh)
-	for object := range objectCh {
-		d.Require().NoError(minioClient.RemoveObject(d.bucketName, object.Key))
-	}
-}
-
-func TestMinioStorage(t *testing.T) {
-	suite.Run(t, new(MinioTests))
-}

pkg/storage/minio/test_suite.go

@@ -1,6 +1,8 @@
 package minio
 
 import (
+	"fmt"
+
 	"github.com/gobuffalo/suite"
 	"github.com/gomods/athens/pkg/config"
 	"github.com/gomods/athens/pkg/storage"
@@ -10,35 +12,30 @@ import (
 // TestSuite implements storage.TestSuite interface
 type TestSuite struct {
 	*suite.Model
-	storage                                            storage.Backend
+	storage storage.Backend
-	endpoint, accessKeyID, secretAccessKey, bucketName string
+	conf    *config.MinioConfig
 }
 
 // NewTestSuite creates a common test suite
-func NewTestSuite(model *suite.Model) (storage.TestSuite, error) {
+func NewTestSuite(model *suite.Model, conf *config.MinioConfig) (storage.TestSuite, error) {
-	endpoint := "127.0.0.1:9000"
+	minioStorage, err := newTestStore(conf)
-	bucketName := "gomods"
-	accessKeyID := "minio"
-	secretAccessKey := "minio123"
-	conf := &config.MinioConfig{
-		Endpoint:  endpoint,
-		Bucket:    bucketName,
-		Key:       accessKeyID,
-		Secret:    secretAccessKey,
-		EnableSSL: false,
-	}
-	minioStorage, err := NewStorage(conf)
 
 	return &TestSuite{
-		storage:         minioStorage,
+		storage: minioStorage,
-		Model:           model,
+		Model:   model,
-		endpoint:        endpoint,
+		conf:    conf,
-		bucketName:      bucketName,
-		accessKeyID:     accessKeyID,
-		secretAccessKey: secretAccessKey,
 	}, err
 }
 
+func newTestStore(conf *config.MinioConfig) (storage.Backend, error) {
+	minioStore, err := NewStorage(conf)
+	if err != nil {
+		return nil, fmt.Errorf("Not able to connect to minio storage: %s", err.Error())
+	}
+
+	return minioStore, nil
+}
+
 // Storage retrieves initialized storage backend
 func (ts *TestSuite) Storage() storage.Backend {
 	return ts.storage
@@ -51,13 +48,13 @@ func (ts *TestSuite) StorageHumanReadableName() string {
 
 // Cleanup tears down test
 func (ts *TestSuite) Cleanup() error {
-	minioClient, _ := minio.New(ts.endpoint, ts.accessKeyID, ts.secretAccessKey, false)
+	minioClient, _ := minio.New(ts.conf.Endpoint, ts.conf.Key, ts.conf.Secret, ts.conf.EnableSSL)
 	doneCh := make(chan struct{})
 	defer close(doneCh)
-	objectCh := minioClient.ListObjectsV2(ts.bucketName, "", true, doneCh)
+	objectCh := minioClient.ListObjectsV2(ts.conf.Bucket, "", true, doneCh)
 	for object := range objectCh {
 		//TODO: could return multi error and clean other objects
-		if err := minioClient.RemoveObject(ts.bucketName, object.Key); err != nil {
+		if err := minioClient.RemoveObject(ts.conf.Bucket, object.Key); err != nil {
 			return err
 		}
 	}

pkg/storage/mongo/all_test.go

@@ -1,23 +0,0 @@
-package mongo
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/suite"
-)
-
-type MongoTests struct {
-	suite.Suite
-}
-
-func (d *MongoTests) SetupTest() {
-	ms, err := newTestStore(testConfigFile)
-
-	d.Require().NoError(err)
-
-	ms.s.DB(ms.d).C(ms.c).RemoveAll(nil)
-}
-
-func TestDiskStorage(t *testing.T) {
-	suite.Run(t, new(MongoTests))
-}

pkg/storage/mongo/local_test.go

@@ -0,0 +1,40 @@
+package mongo
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/gomods/athens/pkg/config"
+	"github.com/stretchr/testify/suite"
+)
+
+var (
+	testConfigFile = filepath.Join("..", "..", "..", "config.test.toml")
+)
+
+type MongoTests struct {
+	suite.Suite
+}
+
+func (m *MongoTests) SetupTest() {
+	conf := config.GetConfLogErr(testConfigFile, m.T())
+
+	_, err := newTestStore(conf.Storage.Mongo)
+	m.Require().NoError(err)
+}
+
+func (m *MongoTests) TestNewMongoStorage() {
+	r := m.Require()
+	conf := config.GetConfLogErr(testConfigFile, m.T())
+	getterSaver, err := NewStorage(conf.Storage.Mongo)
+
+	r.NoError(err)
+	r.NotNil(getterSaver.c)
+	r.NotNil(getterSaver.d)
+	r.NotNil(getterSaver.s)
+	r.Equal(getterSaver.url, conf.Storage.Mongo.URL)
+}
+
+func TestMongoStorage(t *testing.T) {
+	suite.Run(t, new(MongoTests))
+}

pkg/storage/mongo/mongo.go

@@ -27,7 +27,7 @@ type ModuleStore struct {
 // NewStorage returns a connected Mongo backed storage
 // that satisfies the Backend interface.
 func NewStorage(conf *config.MongoConfig) (*ModuleStore, error) {
-	const op errors.Op = "fs.NewStorage"
+	const op errors.Op = "mongo.NewStorage"
 	if conf == nil {
 		return nil, errors.E(op, "No Mongo Configuration provided")
 	}

pkg/storage/mongo/mongo_test.go

@@ -1,17 +0,0 @@
-package mongo
-
-import (
-	"github.com/gomods/athens/pkg/config"
-)
-
-func (m *MongoTests) TestNewMongoStorage() {
-	r := m.Require()
-	conf := config.GetConfLogErr(testConfigFile, m.T())
-	getterSaver, err := NewStorage(conf.Storage.Mongo)
-
-	r.NoError(err)
-	r.NotNil(getterSaver.c)
-	r.NotNil(getterSaver.d)
-	r.NotNil(getterSaver.s)
-	r.Equal(getterSaver.url, conf.Storage.Mongo.URL)
-}

pkg/storage/mongo/test_suite.go

@@ -2,7 +2,6 @@ package mongo
 
 import (
 	"fmt"
-	"path/filepath"
 
 	"github.com/globalsign/mgo"
 	"github.com/gobuffalo/suite"
@@ -10,34 +9,28 @@ import (
 	"github.com/gomods/athens/pkg/storage"
 )
 
-var (
-	testConfigFile = filepath.Join("..", "..", "..", "config.test.toml")
-)
-
 // TestSuite implements storage.TestSuite interface
 type TestSuite struct {
 	*suite.Model
 	storage *ModuleStore
+	conf    *config.MongoConfig
 }
 
 // NewTestSuite creates a common test suite
-func NewTestSuite(model *suite.Model, configFile string) (storage.TestSuite, error) {
+func NewTestSuite(model *suite.Model, conf *config.MongoConfig) (storage.TestSuite, error) {
-	ms, err := newTestStore(configFile)
+	ms, err := newTestStore(conf)
 	if err != nil {
 		return nil, err
 	}
 	return &TestSuite{
 		storage: ms,
 		Model:   model,
+		conf:    conf,
 	}, err
 }
 
-func newTestStore(configFile string) (*ModuleStore, error) {
+func newTestStore(conf *config.MongoConfig) (*ModuleStore, error) {
-	conf, err := config.GetConf(configFile)
+	mongoStore, err := NewStorage(conf)
-	if err != nil {
-		return nil, err
-	}
-	mongoStore, err := NewStorage(conf.Storage.Mongo)
 	if err != nil {
 		return nil, fmt.Errorf("Not able to connect to mongo storage: %s", err.Error())
 	}
@@ -57,17 +50,10 @@ func (ts *TestSuite) StorageHumanReadableName() string {
 
 // Cleanup tears down test
 func (ts *TestSuite) Cleanup() error {
-	conf, err := config.GetConf(testConfigFile)
+	s, err := mgo.DialWithTimeout(ts.conf.URL, ts.conf.TimeoutDuration())
 	if err != nil {
-		return err
+		return nil
 	}
-	if conf.Storage == nil || conf.Storage.Mongo == nil {
-		return fmt.Errorf("Invalid Mongo Storage Provided")
-	}
-	s, err := mgo.DialWithTimeout(conf.Storage.Mongo.URL, conf.Storage.Mongo.TimeoutDuration())
 	defer s.Close()
-	if err != nil {
+	return s.DB(ts.storage.d).C(ts.storage.c).DropCollection()
-		return err
-	}
-	return s.DB("athens").C("modules").DropCollection()
 }

pkg/storage/storage_tests/module_storage/storage_benchmark_test.go

@@ -7,6 +7,7 @@ import (
 	"testing"
 
 	"github.com/gobuffalo/suite"
+	"github.com/gomods/athens/pkg/config"
 	"github.com/gomods/athens/pkg/errors"
 	"github.com/gomods/athens/pkg/storage"
 	"github.com/gomods/athens/pkg/storage/fs"
@@ -124,13 +125,16 @@ func BenchmarkStorageExists(b *testing.B) {
 func getStores(b *testing.B) []storage.TestSuite {
 	var stores []storage.TestSuite
 
+	conf, err := config.GetConf(testConfigFile)
+	require.NoError(b, err)
+
 	//TODO: create the instance without model or TestSuite
 	model := suite.NewModel()
 	fsStore, err := fs.NewTestSuite(model)
 	require.NoError(b, err, "couldn't create filesystem store")
 	stores = append(stores, fsStore)
 
-	mongoStore, err := mongo.NewTestSuite(model, testConfigFile)
+	mongoStore, err := mongo.NewTestSuite(model, conf.Storage.Mongo)
 	require.NoError(b, err, "couldn't create mongo store")
 	stores = append(stores, mongoStore)
 
@@ -138,7 +142,7 @@ func getStores(b *testing.B) []storage.TestSuite {
 	require.NoError(b, err)
 	stores = append(stores, memStore)
 
-	minioStore, err := minio.NewTestSuite(model)
+	minioStore, err := minio.NewTestSuite(model, conf.Storage.Minio)
 	require.NoError(b, err)
 	stores = append(stores, minioStore)
 

pkg/storage/storage_tests/module_storage/storage_test.go

@@ -10,12 +10,14 @@ package modulestorage
 import (
 	"bytes"
 	"context"
+	"fmt"
 	"io/ioutil"
+	"math/rand"
 	"path/filepath"
 	"testing"
-	"time"
 
 	"github.com/gobuffalo/suite"
+	"github.com/gomods/athens/pkg/config"
 	"github.com/gomods/athens/pkg/errors"
 	"github.com/gomods/athens/pkg/storage"
 	"github.com/gomods/athens/pkg/storage/fs"
@@ -41,6 +43,9 @@ type TestSuites struct {
 func (d *TestSuites) SetupTest() {
 	ra := d.Require()
 
+	conf, err := config.GetConf(testConfigFile)
+	ra.NoError(err)
+
 	// file system
 	fsTests, err := fs.NewTestSuite(d.Model)
 	ra.NoError(err)
@@ -52,12 +57,12 @@ func (d *TestSuites) SetupTest() {
 	d.storages = append(d.storages, memStore)
 
 	// minio
-	minioStorage, err := minio.NewTestSuite(d.Model)
+	minioStorage, err := minio.NewTestSuite(d.Model, conf.Storage.Minio)
 	ra.NoError(err)
 	d.storages = append(d.storages, minioStorage)
 
 	// mongo
-	mongoStore, err := mongo.NewTestSuite(d.Model, testConfigFile)
+	mongoStore, err := mongo.NewTestSuite(d.Model, conf.Storage.Mongo)
 	ra.NoError(err)
 	d.storages = append(d.storages, mongoStore)
 
@@ -157,7 +162,7 @@ func (d *TestSuites) testGetSaveListRoundTrip(ts storage.TestSuite) {
 
 func (d *TestSuites) testDelete(ts storage.TestSuite) {
 	r := d.Require()
-	version := "delete" + time.Now().String()
+	version := fmt.Sprintf("%s%d", "delete", rand.Int())
 	err := ts.Storage().Save(context.Background(), d.module, version, d.mod, bytes.NewReader(d.zip), d.info)
 	r.NoError(err)
 

scripts/create_default_config.sh

@@ -3,5 +3,5 @@
 SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
 
 if [ ! -e "${SCRIPTS_DIR}/../config.toml" ] ; then
-    cp "${SCRIPTS_DIR}/../config.example.toml" "${SCRIPTS_DIR}/../config.toml"
+    cp "${SCRIPTS_DIR}/../config.test.toml" "${SCRIPTS_DIR}/../config.toml"
 fi

scripts/ps/create_default_config.ps1

@@ -1,6 +1,6 @@
 $repoDir = Join-Path $PSScriptRoot ".." | Join-Path -ChildPath ".."
 if (-not (Join-Path $repoDir config.toml | Test-Path)) {
-    $example = Join-Path $repoDir config.example.toml
+    $example = Join-Path $repoDir config.test.toml
     $target = Join-Path $repoDir config.toml
     Copy-Item $example $target
 }