diff --git a/.travis.yml b/.travis.yml index 3efb320b..6cdc3561 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,7 +20,9 @@ before_script: - mkdir -p $ATHENS_DIR && rsync -azr . $ATHENS_DIR && cd $ATHENS_DIR - make setup-dev-env - wget "https://dl.minio.io/server/minio/release/linux-amd64/minio" - - chmod +x minio && nohup ./minio server . & + # moving from the default port 9000 so it doesn't conflich with other services (i.e. vscode extensions server) + # we also need to change it in travis because the tests are using the config.test.toml where port 9001 is used + - chmod +x minio && nohup ./minio server --address ":9001" . & script: - make verify test-unit test-e2e diff --git a/config.example.toml b/config.example.toml index be1c5986..133592f6 100644 --- a/config.example.toml +++ b/config.example.toml @@ -184,15 +184,15 @@ EnableCSRFProtection = false [Storage.Minio] # Endpoint for Minio storage # Env override: ATHENS_MINIO_ENDPOINT - Endpoint = "minio.example.com" + Endpoint = "127.0.0.1:9001" # Access Key for Minio storage # Env override: ATHENS_MINIO_ACCESS_KEY_ID - Key = "MY_KEY" + Key = "minio" # Secret Key for Minio storage # Env override: ATHENS_MINIO_SECRET_ACCESS_KEY - Secret = "MY_SECRET" + Secret = "minio123" # Timeout for networks calls made to Minio in seconds # Defaults to Global Timeout @@ -201,8 +201,8 @@ EnableCSRFProtection = false # Enable SSL for Minio connections # Defaults to true # Env override: ATHENS_MINIO_USE_SSL - EnableSSL = true - + EnableSSL = false + # Minio Bucket to use for storage # Defaults to gomods # Env override: ATHENS_MINIO_BUCKET_NAME diff --git a/config.test.toml b/config.test.toml index 86fee2b8..663ff9f3 100644 --- a/config.test.toml +++ b/config.test.toml @@ -62,7 +62,7 @@ FilterFile = "filter.conf" # This value is used as the default for storage backends if they don't specify timeouts # Defaults to 300 # Env override: ATHENS_TIMEOUT -Timeout = 1 +Timeout = 300 # EnableCSRFProtection determines whether to enable CSRF protection. # Defaults to false @@ -145,7 +145,7 @@ EnableCSRFProtection = false Endpoint = "cdn.example.com" # Timeout for networks calls made to the CDN in seconds # Defaults to Global Timeout - Timeout = 1 + Timeout = 300 [Storage.Disk] # RootPath is the Athens Disk Root folder # Env override: ATHENS_DISK_STORAGE_ROOT @@ -159,24 +159,24 @@ EnableCSRFProtection = false Bucket = "MY_GCP_BUCKET" # Timeout for networks calls made to GCP in seconds # Defaults to Global Timeout - Timeout = 1 + Timeout = 300 [Storage.Minio] # Endpoint for Minio storage # Env override: ATHENS_MINIO_ENDPOINT - Endpoint = "minio.example.com" + Endpoint = "127.0.0.1:9001" # Access Key for Minio storage # Env override: ATHENS_MINIO_ACCESS_KEY_ID - Key = "MY_KEY" + Key = "minio" # Secret Key for Minio storage # Env override: ATHENS_MINIO_SECRET_ACCESS_KEY - Secret = "MY_SECRET" + Secret = "minio123" # Timeout for networks calls made to Minio in seconds # Defaults to Global Timeout - Timeout = 1 + Timeout = 300 # Enable SSL for Minio connections # Defaults to true # Env override: ATHENS_MINIO_USE_SSL - EnableSSL = true + EnableSSL = false # Minio Bucket to use for storage # Defaults to gomods # Env override: ATHENS_MINIO_BUCKET_NAME @@ -191,4 +191,4 @@ EnableCSRFProtection = false # Timeout for networks calls made to Mongo in seconds # Defaults to Global Timeout # Env override: MONGO_CONN_TIMEOUT_SEC - Timeout = 1 + Timeout = 300 diff --git a/docker-compose.yml b/docker-compose.yml index 74e9f480..1992e573 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,7 +13,7 @@ services: image: minio/minio:latest command: server /data ports: - - "9000:9000" + - "9001:9000" environment: MINIO_ACCESS_KEY: minio MINIO_SECRET_KEY: minio123 diff --git a/pkg/config/parse_test.go b/pkg/config/parse_test.go index 0e331a67..3da31f6d 100644 --- a/pkg/config/parse_test.go +++ b/pkg/config/parse_test.go @@ -232,10 +232,10 @@ func TestParseExampleConfig(t *testing.T) { }, }, Minio: &MinioConfig{ - Endpoint: "minio.example.com", - Key: "MY_KEY", - Secret: "MY_SECRET", - EnableSSL: true, + Endpoint: "127.0.0.1:9001", + Key: "minio", + Secret: "minio123", + EnableSSL: false, Bucket: "gomods", TimeoutConf: TimeoutConf{ Timeout: globalTimeout, diff --git a/pkg/storage/fs/test_suite.go b/pkg/storage/fs/test_suite.go index 4355c310..49f8adf7 100644 --- a/pkg/storage/fs/test_suite.go +++ b/pkg/storage/fs/test_suite.go @@ -16,20 +16,20 @@ type TestSuite struct { // NewTestSuite creates a common test suite func NewTestSuite(model *suite.Model) (storage.TestSuite, error) { - memFs := afero.NewOsFs() - r, err := afero.TempDir(memFs, "", "athens-fs-storage-tests") + osFs := afero.NewOsFs() + r, err := afero.TempDir(osFs, "", "athens-fs-storage-tests") if err != nil { return nil, err } - fsStore, err := NewStorage(r, memFs) + fsStore, err := NewStorage(r, osFs) if err != nil { return nil, err } return &TestSuite{ Model: model, - fs: memFs, + fs: osFs, rootDir: r, storage: fsStore, }, nil diff --git a/pkg/storage/minio/all_test.go b/pkg/storage/minio/all_test.go deleted file mode 100644 index 21415fa0..00000000 --- a/pkg/storage/minio/all_test.go +++ /dev/null @@ -1,47 +0,0 @@ -package minio - -import ( - "testing" - - "github.com/gomods/athens/pkg/config" - "github.com/gomods/athens/pkg/storage" - minio "github.com/minio/minio-go" - "github.com/stretchr/testify/suite" -) - -type MinioTests struct { - suite.Suite - storage storage.Backend - endpoint, accessKeyID, secretAccessKey, bucketName string -} - -func (d *MinioTests) SetupTest() { - d.endpoint = "127.0.0.1:9000" - d.bucketName = "gomods" - d.accessKeyID = "minio" - d.secretAccessKey = "minio123" - conf := &config.MinioConfig{ - Endpoint: d.endpoint, - Bucket: d.bucketName, - Key: d.accessKeyID, - Secret: d.secretAccessKey, - EnableSSL: false, - } - storage, err := NewStorage(conf) - d.Require().NoError(err) - d.storage = storage -} - -func (d *MinioTests) TearDownTest() { - minioClient, _ := minio.New(d.endpoint, d.accessKeyID, d.secretAccessKey, false) - doneCh := make(chan struct{}) - defer close(doneCh) - objectCh := minioClient.ListObjectsV2(d.bucketName, "", true, doneCh) - for object := range objectCh { - d.Require().NoError(minioClient.RemoveObject(d.bucketName, object.Key)) - } -} - -func TestMinioStorage(t *testing.T) { - suite.Run(t, new(MinioTests)) -} diff --git a/pkg/storage/minio/test_suite.go b/pkg/storage/minio/test_suite.go index 87eeeaa7..2cb4c13d 100644 --- a/pkg/storage/minio/test_suite.go +++ b/pkg/storage/minio/test_suite.go @@ -1,6 +1,8 @@ package minio import ( + "fmt" + "github.com/gobuffalo/suite" "github.com/gomods/athens/pkg/config" "github.com/gomods/athens/pkg/storage" @@ -10,35 +12,30 @@ import ( // TestSuite implements storage.TestSuite interface type TestSuite struct { *suite.Model - storage storage.Backend - endpoint, accessKeyID, secretAccessKey, bucketName string + storage storage.Backend + conf *config.MinioConfig } // NewTestSuite creates a common test suite -func NewTestSuite(model *suite.Model) (storage.TestSuite, error) { - endpoint := "127.0.0.1:9000" - bucketName := "gomods" - accessKeyID := "minio" - secretAccessKey := "minio123" - conf := &config.MinioConfig{ - Endpoint: endpoint, - Bucket: bucketName, - Key: accessKeyID, - Secret: secretAccessKey, - EnableSSL: false, - } - minioStorage, err := NewStorage(conf) +func NewTestSuite(model *suite.Model, conf *config.MinioConfig) (storage.TestSuite, error) { + minioStorage, err := newTestStore(conf) return &TestSuite{ - storage: minioStorage, - Model: model, - endpoint: endpoint, - bucketName: bucketName, - accessKeyID: accessKeyID, - secretAccessKey: secretAccessKey, + storage: minioStorage, + Model: model, + conf: conf, }, err } +func newTestStore(conf *config.MinioConfig) (storage.Backend, error) { + minioStore, err := NewStorage(conf) + if err != nil { + return nil, fmt.Errorf("Not able to connect to minio storage: %s", err.Error()) + } + + return minioStore, nil +} + // Storage retrieves initialized storage backend func (ts *TestSuite) Storage() storage.Backend { return ts.storage @@ -51,13 +48,13 @@ func (ts *TestSuite) StorageHumanReadableName() string { // Cleanup tears down test func (ts *TestSuite) Cleanup() error { - minioClient, _ := minio.New(ts.endpoint, ts.accessKeyID, ts.secretAccessKey, false) + minioClient, _ := minio.New(ts.conf.Endpoint, ts.conf.Key, ts.conf.Secret, ts.conf.EnableSSL) doneCh := make(chan struct{}) defer close(doneCh) - objectCh := minioClient.ListObjectsV2(ts.bucketName, "", true, doneCh) + objectCh := minioClient.ListObjectsV2(ts.conf.Bucket, "", true, doneCh) for object := range objectCh { //TODO: could return multi error and clean other objects - if err := minioClient.RemoveObject(ts.bucketName, object.Key); err != nil { + if err := minioClient.RemoveObject(ts.conf.Bucket, object.Key); err != nil { return err } } diff --git a/pkg/storage/mongo/all_test.go b/pkg/storage/mongo/all_test.go deleted file mode 100644 index 82988afe..00000000 --- a/pkg/storage/mongo/all_test.go +++ /dev/null @@ -1,23 +0,0 @@ -package mongo - -import ( - "testing" - - "github.com/stretchr/testify/suite" -) - -type MongoTests struct { - suite.Suite -} - -func (d *MongoTests) SetupTest() { - ms, err := newTestStore(testConfigFile) - - d.Require().NoError(err) - - ms.s.DB(ms.d).C(ms.c).RemoveAll(nil) -} - -func TestDiskStorage(t *testing.T) { - suite.Run(t, new(MongoTests)) -} diff --git a/pkg/storage/mongo/local_test.go b/pkg/storage/mongo/local_test.go new file mode 100644 index 00000000..4cc47919 --- /dev/null +++ b/pkg/storage/mongo/local_test.go @@ -0,0 +1,40 @@ +package mongo + +import ( + "path/filepath" + "testing" + + "github.com/gomods/athens/pkg/config" + "github.com/stretchr/testify/suite" +) + +var ( + testConfigFile = filepath.Join("..", "..", "..", "config.test.toml") +) + +type MongoTests struct { + suite.Suite +} + +func (m *MongoTests) SetupTest() { + conf := config.GetConfLogErr(testConfigFile, m.T()) + + _, err := newTestStore(conf.Storage.Mongo) + m.Require().NoError(err) +} + +func (m *MongoTests) TestNewMongoStorage() { + r := m.Require() + conf := config.GetConfLogErr(testConfigFile, m.T()) + getterSaver, err := NewStorage(conf.Storage.Mongo) + + r.NoError(err) + r.NotNil(getterSaver.c) + r.NotNil(getterSaver.d) + r.NotNil(getterSaver.s) + r.Equal(getterSaver.url, conf.Storage.Mongo.URL) +} + +func TestMongoStorage(t *testing.T) { + suite.Run(t, new(MongoTests)) +} diff --git a/pkg/storage/mongo/mongo.go b/pkg/storage/mongo/mongo.go index fbe487a6..3fb6e6e1 100644 --- a/pkg/storage/mongo/mongo.go +++ b/pkg/storage/mongo/mongo.go @@ -27,7 +27,7 @@ type ModuleStore struct { // NewStorage returns a connected Mongo backed storage // that satisfies the Backend interface. func NewStorage(conf *config.MongoConfig) (*ModuleStore, error) { - const op errors.Op = "fs.NewStorage" + const op errors.Op = "mongo.NewStorage" if conf == nil { return nil, errors.E(op, "No Mongo Configuration provided") } diff --git a/pkg/storage/mongo/mongo_test.go b/pkg/storage/mongo/mongo_test.go deleted file mode 100644 index 5d1ec779..00000000 --- a/pkg/storage/mongo/mongo_test.go +++ /dev/null @@ -1,17 +0,0 @@ -package mongo - -import ( - "github.com/gomods/athens/pkg/config" -) - -func (m *MongoTests) TestNewMongoStorage() { - r := m.Require() - conf := config.GetConfLogErr(testConfigFile, m.T()) - getterSaver, err := NewStorage(conf.Storage.Mongo) - - r.NoError(err) - r.NotNil(getterSaver.c) - r.NotNil(getterSaver.d) - r.NotNil(getterSaver.s) - r.Equal(getterSaver.url, conf.Storage.Mongo.URL) -} diff --git a/pkg/storage/mongo/test_suite.go b/pkg/storage/mongo/test_suite.go index eaef2cd3..7137b393 100644 --- a/pkg/storage/mongo/test_suite.go +++ b/pkg/storage/mongo/test_suite.go @@ -2,7 +2,6 @@ package mongo import ( "fmt" - "path/filepath" "github.com/globalsign/mgo" "github.com/gobuffalo/suite" @@ -10,34 +9,28 @@ import ( "github.com/gomods/athens/pkg/storage" ) -var ( - testConfigFile = filepath.Join("..", "..", "..", "config.test.toml") -) - // TestSuite implements storage.TestSuite interface type TestSuite struct { *suite.Model storage *ModuleStore + conf *config.MongoConfig } // NewTestSuite creates a common test suite -func NewTestSuite(model *suite.Model, configFile string) (storage.TestSuite, error) { - ms, err := newTestStore(configFile) +func NewTestSuite(model *suite.Model, conf *config.MongoConfig) (storage.TestSuite, error) { + ms, err := newTestStore(conf) if err != nil { return nil, err } return &TestSuite{ storage: ms, Model: model, + conf: conf, }, err } -func newTestStore(configFile string) (*ModuleStore, error) { - conf, err := config.GetConf(configFile) - if err != nil { - return nil, err - } - mongoStore, err := NewStorage(conf.Storage.Mongo) +func newTestStore(conf *config.MongoConfig) (*ModuleStore, error) { + mongoStore, err := NewStorage(conf) if err != nil { return nil, fmt.Errorf("Not able to connect to mongo storage: %s", err.Error()) } @@ -57,17 +50,10 @@ func (ts *TestSuite) StorageHumanReadableName() string { // Cleanup tears down test func (ts *TestSuite) Cleanup() error { - conf, err := config.GetConf(testConfigFile) + s, err := mgo.DialWithTimeout(ts.conf.URL, ts.conf.TimeoutDuration()) if err != nil { - return err + return nil } - if conf.Storage == nil || conf.Storage.Mongo == nil { - return fmt.Errorf("Invalid Mongo Storage Provided") - } - s, err := mgo.DialWithTimeout(conf.Storage.Mongo.URL, conf.Storage.Mongo.TimeoutDuration()) defer s.Close() - if err != nil { - return err - } - return s.DB("athens").C("modules").DropCollection() + return s.DB(ts.storage.d).C(ts.storage.c).DropCollection() } diff --git a/pkg/storage/storage_tests/module_storage/storage_benchmark_test.go b/pkg/storage/storage_tests/module_storage/storage_benchmark_test.go index 626b38a2..e493332b 100644 --- a/pkg/storage/storage_tests/module_storage/storage_benchmark_test.go +++ b/pkg/storage/storage_tests/module_storage/storage_benchmark_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/gobuffalo/suite" + "github.com/gomods/athens/pkg/config" "github.com/gomods/athens/pkg/errors" "github.com/gomods/athens/pkg/storage" "github.com/gomods/athens/pkg/storage/fs" @@ -124,13 +125,16 @@ func BenchmarkStorageExists(b *testing.B) { func getStores(b *testing.B) []storage.TestSuite { var stores []storage.TestSuite + conf, err := config.GetConf(testConfigFile) + require.NoError(b, err) + //TODO: create the instance without model or TestSuite model := suite.NewModel() fsStore, err := fs.NewTestSuite(model) require.NoError(b, err, "couldn't create filesystem store") stores = append(stores, fsStore) - mongoStore, err := mongo.NewTestSuite(model, testConfigFile) + mongoStore, err := mongo.NewTestSuite(model, conf.Storage.Mongo) require.NoError(b, err, "couldn't create mongo store") stores = append(stores, mongoStore) @@ -138,7 +142,7 @@ func getStores(b *testing.B) []storage.TestSuite { require.NoError(b, err) stores = append(stores, memStore) - minioStore, err := minio.NewTestSuite(model) + minioStore, err := minio.NewTestSuite(model, conf.Storage.Minio) require.NoError(b, err) stores = append(stores, minioStore) diff --git a/pkg/storage/storage_tests/module_storage/storage_test.go b/pkg/storage/storage_tests/module_storage/storage_test.go index 89cfc34b..e402da0f 100644 --- a/pkg/storage/storage_tests/module_storage/storage_test.go +++ b/pkg/storage/storage_tests/module_storage/storage_test.go @@ -10,12 +10,14 @@ package modulestorage import ( "bytes" "context" + "fmt" "io/ioutil" + "math/rand" "path/filepath" "testing" - "time" "github.com/gobuffalo/suite" + "github.com/gomods/athens/pkg/config" "github.com/gomods/athens/pkg/errors" "github.com/gomods/athens/pkg/storage" "github.com/gomods/athens/pkg/storage/fs" @@ -41,6 +43,9 @@ type TestSuites struct { func (d *TestSuites) SetupTest() { ra := d.Require() + conf, err := config.GetConf(testConfigFile) + ra.NoError(err) + // file system fsTests, err := fs.NewTestSuite(d.Model) ra.NoError(err) @@ -52,12 +57,12 @@ func (d *TestSuites) SetupTest() { d.storages = append(d.storages, memStore) // minio - minioStorage, err := minio.NewTestSuite(d.Model) + minioStorage, err := minio.NewTestSuite(d.Model, conf.Storage.Minio) ra.NoError(err) d.storages = append(d.storages, minioStorage) // mongo - mongoStore, err := mongo.NewTestSuite(d.Model, testConfigFile) + mongoStore, err := mongo.NewTestSuite(d.Model, conf.Storage.Mongo) ra.NoError(err) d.storages = append(d.storages, mongoStore) @@ -157,7 +162,7 @@ func (d *TestSuites) testGetSaveListRoundTrip(ts storage.TestSuite) { func (d *TestSuites) testDelete(ts storage.TestSuite) { r := d.Require() - version := "delete" + time.Now().String() + version := fmt.Sprintf("%s%d", "delete", rand.Int()) err := ts.Storage().Save(context.Background(), d.module, version, d.mod, bytes.NewReader(d.zip), d.info) r.NoError(err) diff --git a/scripts/create_default_config.sh b/scripts/create_default_config.sh index dca16bed..0510ac87 100755 --- a/scripts/create_default_config.sh +++ b/scripts/create_default_config.sh @@ -3,5 +3,5 @@ SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" if [ ! -e "${SCRIPTS_DIR}/../config.toml" ] ; then - cp "${SCRIPTS_DIR}/../config.example.toml" "${SCRIPTS_DIR}/../config.toml" + cp "${SCRIPTS_DIR}/../config.test.toml" "${SCRIPTS_DIR}/../config.toml" fi diff --git a/scripts/ps/create_default_config.ps1 b/scripts/ps/create_default_config.ps1 index 99873acb..f2ac8879 100644 --- a/scripts/ps/create_default_config.ps1 +++ b/scripts/ps/create_default_config.ps1 @@ -1,6 +1,6 @@ $repoDir = Join-Path $PSScriptRoot ".." | Join-Path -ChildPath ".." if (-not (Join-Path $repoDir config.toml | Test-Path)) { - $example = Join-Path $repoDir config.example.toml + $example = Join-Path $repoDir config.test.toml $target = Join-Path $repoDir config.toml Copy-Item $example $target } \ No newline at end of file