Removal of cdn rdbms driver and pop support from olympus (#551)

* Inital removal of cdn rdbms driver and pop support from olympus

* Update travis and other unit test scripts

* Remove postgres from Makefile

* Update comments and remove function that is not required

* Update .env file

* clean commented code

* Update docs to remove references to rdbms

* Remove pop and run dep ensure to make sure that the tests pass without pop

* Keeping GO_ENV=test for parametter logging

.travis.yml

@@ -7,7 +7,6 @@ go:
 env:
   global:
     - PATH=${PATH}:./bin
-    - POP_PATH=$PWD/cmd/olympus
     - GO_ENV=test
     - MINIO_ACCESS_KEY=minio
     - MINIO_SECRET_KEY=minio123
@@ -20,8 +19,6 @@ before_script:
   - make setup-dev-env
   - wget "https://dl.minio.io/server/minio/release/linux-amd64/minio"
   - chmod +x minio && nohup ./minio server . &
-  - buffalo db create -e $GO_ENV -d -c $POP_PATH/database.yml
-  - buffalo db migrate up -e $GO_ENV -d -c $POP_PATH/database.yml
 
 
 script:

Gopkg.toml

@@ -41,10 +41,6 @@
   name = "github.com/gobuffalo/packr"
   version = "1.10.6"
 
-[[constraint]]
-  name = "github.com/gobuffalo/pop"
-  version = "4.3.0"
-
 [[constraint]]
   name = "github.com/gobuffalo/suite"
   version = "2.1.0"

Makefile

@@ -51,7 +51,6 @@ bench:
 
 .PHONY: alldeps
 alldeps:
-	docker-compose -p athensdev up -d postgres
 	docker-compose -p athensdev up -d mongo
 	docker-compose -p athensdev up -d redis
 	docker-compose -p athensdev up -d minio

cmd/olympus/database.yml

@@ -1,24 +0,0 @@
-development:
-  dialect: "postgres"
-  database: athens
-  host: 127.0.0.1
-  port: 5432
-  user: postgres
-  password: postgres
-
-test:
-  dialect: "postgres"
-  database: athens_test
-  user: postgres
-  password: postgres
-  port: 5432
-  host: 127.0.0.1
-  pool: 5
-
-production:
-  dialect: "postgres"
-  database: olympusdb
-  host: {{ env "DB_HOST" }}
-  port: {{ env "DB_PORT" }}
-  user: {{ env "DB_USER" }}
-  password: {{ env "DB_PASS" }}

cmd/olympus/migrations/schema.sql

@@ -1,57 +0,0 @@
---
--- PostgreSQL database dump
---
-
--- Dumped from database version 9.6.9
--- Dumped by pg_dump version 10.4
-
-SET statement_timeout = 0;
-SET lock_timeout = 0;
-SET idle_in_transaction_session_timeout = 0;
-SET client_encoding = 'UTF8';
-SET standard_conforming_strings = on;
-SELECT pg_catalog.set_config('search_path', '', false);
-SET check_function_bodies = false;
-SET client_min_messages = warning;
-SET row_security = off;
-
---
--- Name: plpgsql; Type: EXTENSION; Schema: -; Owner: 
---
-
-CREATE EXTENSION IF NOT EXISTS plpgsql WITH SCHEMA pg_catalog;
-
-
---
--- Name: EXTENSION plpgsql; Type: COMMENT; Schema: -; Owner: 
---
-
-COMMENT ON EXTENSION plpgsql IS 'PL/pgSQL procedural language';
-
-
-SET default_tablespace = '';
-
-SET default_with_oids = false;
-
---
--- Name: schema_migration; Type: TABLE; Schema: public; Owner: postgres
---
-
-CREATE TABLE public.schema_migration (
-    version character varying(255) NOT NULL
-);
-
-
-ALTER TABLE public.schema_migration OWNER TO postgres;
-
---
--- Name: schema_migration_version_idx; Type: INDEX; Schema: public; Owner: postgres
---
-
-CREATE UNIQUE INDEX schema_migration_version_idx ON public.schema_migration USING btree (version);
-
-
---
--- PostgreSQL database dump complete
---
-

cmd/olympus/models/models.go

@@ -1,5 +0,0 @@
-package models
-
-func init() {
-
-}

cmd/olympus/models/models_test.go

@@ -1,9 +0,0 @@
-package models_test
-
-import (
-	"testing"
-)
-
-func Test_ModelSuite(t *testing.T) {
-
-}

cmd/proxy/.env

@@ -1,4 +1,3 @@
 ATHENS_MONGO_CONNECTION_STRING=mongodb://127.0.0.1:27017
 ATHENS_STORAGE_TYPE=mongo
-GO_ENV=test_postgres
+GO_ENV=test
-POP_PATH=$PWD/cmd/proxy

cmd/proxy/migrations/schema.sql

@@ -1,40 +0,0 @@
--- MySQL dump 10.13  Distrib 5.7.21, for osx10.13 (x86_64)
---
--- Host: 127.0.0.1    Database: athens
--- ------------------------------------------------------
--- Server version	5.7.21
-
-/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
-/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
-/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
-/*!40101 SET NAMES utf8 */;
-/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
-/*!40103 SET TIME_ZONE='+00:00' */;
-/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
-/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
-/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
-/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
-
---
--- Table structure for table `schema_migration`
---
-
-DROP TABLE IF EXISTS `schema_migration`;
-/*!40101 SET @saved_cs_client     = @@character_set_client */;
-/*!40101 SET character_set_client = utf8 */;
-CREATE TABLE `schema_migration` (
-  `version` varchar(255) NOT NULL,
-  UNIQUE KEY `version_idx` (`version`)
-) ENGINE=InnoDB DEFAULT CHARSET=utf8;
-/*!40101 SET character_set_client = @saved_cs_client */;
-/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;
-
-/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;
-/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
-/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */;
-/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */;
-/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */;
-/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;
-/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
-
--- Dump completed on 2018-06-06 12:32:35

docker-compose.yml

@@ -9,13 +9,6 @@ services:
     image: mongo:3.7.9-jessie
     ports:
       - 27017:27017
-  postgres:
-    image: postgres:9.6.9-alpine
-    ports:
-      - "5432:5432"
-    environment:
-      POSTGRES_USER: postgres
-      POSTGRES_PASSWORD: postgres
   minio:
     image: minio/minio:latest
     command: server /data

docs/content/design/proxy.md

@@ -24,7 +24,6 @@ First and foremost, a proxy exposes the same vgo download protocol as the regist
 
 - In-memory
 - Disk
-- RDBMS
 - Cloud blob storage
 
 Users who want to target a proxy configure their `vgo` CLI to point to the proxy, and then execute commands as normal.

migrations/20180326213718_create_modules.down.fizz

@@ -1 +0,0 @@
-drop_table("modules")

migrations/20180326213718_create_modules.up.fizz

@@ -1,8 +0,0 @@
-create_table("modules", func(t) {
-	t.Column("id", "uuid", {"primary": true})
-	t.Column("module", "text", {})
-	t.Column("version", "text", {})
-	t.Column("mod", "blob", {})
-	t.Column("zip", "blob", {})
-  t.Column("info", "blob", {})
-})

migrations/20180604072327_create_cdn_metadata_entries.down.fizz

@@ -1 +0,0 @@
-drop_table("cdn_metadata_entries")

migrations/20180604072327_create_cdn_metadata_entries.up.fizz

@@ -1,5 +0,0 @@
-create_table("cdn_metadata_entries", func(t) {
-	t.Column("id", "uuid", {"primary": true})
-	t.Column("module", "text", {})
-	t.Column("redirect_url", "text", {})
-})

migrations/schema.sql

@@ -1,106 +0,0 @@
---
--- PostgreSQL database dump
---
-
--- Dumped from database version 9.6.9
--- Dumped by pg_dump version 10.4
-
-SET statement_timeout = 0;
-SET lock_timeout = 0;
-SET idle_in_transaction_session_timeout = 0;
-SET client_encoding = 'UTF8';
-SET standard_conforming_strings = on;
-SELECT pg_catalog.set_config('search_path', '', false);
-SET check_function_bodies = false;
-SET client_min_messages = warning;
-SET row_security = off;
-
---
--- Name: plpgsql; Type: EXTENSION; Schema: -; Owner: 
---
-
-CREATE EXTENSION IF NOT EXISTS plpgsql WITH SCHEMA pg_catalog;
-
-
---
--- Name: EXTENSION plpgsql; Type: COMMENT; Schema: -; Owner: 
---
-
-COMMENT ON EXTENSION plpgsql IS 'PL/pgSQL procedural language';
-
-
-SET default_tablespace = '';
-
-SET default_with_oids = false;
-
---
--- Name: cdn_metadata_entries; Type: TABLE; Schema: public; Owner: postgres
---
-
-CREATE TABLE public.cdn_metadata_entries (
-    id uuid NOT NULL,
-    module text NOT NULL,
-    redirect_url text NOT NULL,
-    created_at timestamp without time zone NOT NULL,
-    updated_at timestamp without time zone NOT NULL
-);
-
-
-ALTER TABLE public.cdn_metadata_entries OWNER TO postgres;
-
---
--- Name: modules; Type: TABLE; Schema: public; Owner: postgres
---
-
-CREATE TABLE public.modules (
-    id uuid NOT NULL,
-    module text NOT NULL,
-    version text NOT NULL,
-    mod bytea NOT NULL,
-    zip bytea NOT NULL,
-    info bytea NOT NULL,
-    created_at timestamp without time zone NOT NULL,
-    updated_at timestamp without time zone NOT NULL
-);
-
-
-ALTER TABLE public.modules OWNER TO postgres;
-
---
--- Name: schema_migration; Type: TABLE; Schema: public; Owner: postgres
---
-
-CREATE TABLE public.schema_migration (
-    version character varying(255) NOT NULL
-);
-
-
-ALTER TABLE public.schema_migration OWNER TO postgres;
-
---
--- Name: cdn_metadata_entries cdn_metadata_entries_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres
---
-
-ALTER TABLE ONLY public.cdn_metadata_entries
-    ADD CONSTRAINT cdn_metadata_entries_pkey PRIMARY KEY (id);
-
-
---
--- Name: modules modules_pkey; Type: CONSTRAINT; Schema: public; Owner: postgres
---
-
-ALTER TABLE ONLY public.modules
-    ADD CONSTRAINT modules_pkey PRIMARY KEY (id);
-
-
---
--- Name: schema_migration_version_idx; Type: INDEX; Schema: public; Owner: postgres
---
-
-CREATE UNIQUE INDEX schema_migration_version_idx ON public.schema_migration USING btree (version);
-
-
---
--- PostgreSQL database dump complete
---
-

pkg/cdn/metadata/cdn_metadata_entry.go

@@ -8,11 +8,11 @@ import (
 
 // CDNMetadataEntry stores the module name and cdn URL.
 type CDNMetadataEntry struct {
-	ID          uuid.UUID `json:"id" db:"id" bson:"id"`
+	ID          uuid.UUID `json:"id" bson:"id"`
-	CreatedAt   time.Time `json:"created_at" db:"created_at" bson:"created_at"`
+	CreatedAt   time.Time `json:"created_at" bson:"created_at"`
-	UpdatedAt   time.Time `json:"updated_at" db:"updated_at" bson:"updated_at"`
+	UpdatedAt   time.Time `json:"updated_at" bson:"updated_at"`
-	Module      string    `json:"module" db:"module" bson:"module"`
+	Module      string    `json:"module" bson:"module"`
-	RedirectURL string    `json:"redirect_url" db:"redirect_url" bson:"redirect_url"`
+	RedirectURL string    `json:"redirect_url" bson:"redirect_url"`
 }
 
 // TableName changes the default name which would be c_d_n_metadata_entry

pkg/cdn/metadata/rdbms/all_test.go

@@ -1,27 +0,0 @@
-package rdbms
-
-import (
-	"testing"
-
-	"github.com/gobuffalo/suite"
-	"github.com/gomods/athens/pkg/cdn/metadata"
-)
-
-const (
-	module      = "testmodule"
-	redirectURL = "https://mycdn.com/gomods.io/my/testmodule"
-)
-
-type RDBMSTestSuite struct {
-	*suite.Model
-	storage metadata.StorageConnector
-}
-
-func (rd *RDBMSTestSuite) SetupTest() {
-	rd.storage = &MetadataStore{conn: rd.DB}
-	rd.Model.SetupTest()
-}
-
-func Test_ActionSuite(t *testing.T) {
-	suite.Run(t, &RDBMSTestSuite{Model: suite.NewModel()})
-}

pkg/cdn/metadata/rdbms/getter.go

@@ -1,11 +0,0 @@
-package rdbms
-
-import "github.com/gomods/athens/pkg/cdn/metadata"
-
-// Get retrieves the cdn base URL for a module
-func (s *MetadataStore) Get(module string) (string, error) {
-	result := metadata.CDNMetadataEntry{}
-	query := s.conn.Where("module = ?", module)
-	err := query.First(&result)
-	return result.RedirectURL, err
-}

pkg/cdn/metadata/rdbms/rdbms.go

@@ -1,31 +0,0 @@
-package rdbms
-
-import (
-	"github.com/gobuffalo/pop"
-)
-
-// MetadataStore represents a rdbms(postgres, mysql, sqlite, cockroachdb) backed metadata store.
-type MetadataStore struct {
-	conn           *pop.Connection
-	connectionName string // settings name from database.yml
-}
-
-// NewStorage  returns an unconnected RDBMS Metadata Storage
-// that satisfies the Getter and Setter interfaces. You must call
-// Connect() on the returned store before using it.
-// connectionName
-func NewStorage(connectionName string) *MetadataStore {
-	return &MetadataStore{
-		connectionName: connectionName,
-	}
-}
-
-// Connect creates connection to rdmbs backend.
-func (r *MetadataStore) Connect() error {
-	c, err := pop.Connect(r.connectionName)
-	if err != nil {
-		return err
-	}
-	r.conn = c
-	return nil
-}

pkg/cdn/metadata/rdbms/rdbms_test.go

@@ -1,25 +0,0 @@
-package rdbms
-
-import (
-	"github.com/gobuffalo/envy"
-)
-
-func (rd *RDBMSTestSuite) TestGetSaveRoundTrip() {
-	r := rd.Require()
-	err := rd.storage.Save(module, redirectURL)
-	r.NoError(err)
-
-	gotten, err := rd.storage.Get(module)
-	r.NoError(err)
-	r.Equal(gotten, redirectURL)
-}
-
-func (rd *RDBMSTestSuite) TestNewRDBMSStorage() {
-	r := rd.Require()
-	e := envy.Get("GO_ENV", "test")
-	getterSaver := NewStorage(e)
-	getterSaver.Connect()
-
-	r.NotNil(getterSaver.conn)
-	r.Equal(getterSaver.connectionName, e)
-}

pkg/cdn/metadata/rdbms/saver.go

@@ -1,9 +0,0 @@
-package rdbms
-
-import "github.com/gomods/athens/pkg/cdn/metadata"
-
-// Save saves the module and it's cdn base URL.
-func (s *MetadataStore) Save(module, redirectURL string) error {
-	r := metadata.CDNMetadataEntry{Module: module, RedirectURL: redirectURL}
-	return s.conn.Create(&r)
-}

pkg/config/env/storage.go

@@ -16,24 +16,14 @@ func DiskRoot() (string, error) {
 	return env, nil
 }
 
-// RdbmsName returns RDBMS storage name defined by ATHENS_RDBMS_STORAGE_NAME
-func RdbmsName() (string, error) {
-	env, err := envy.MustGet("ATHENS_RDBMS_STORAGE_NAME")
-	if err != nil {
-		return "", fmt.Errorf("missing RDBMS storage name: %s", err)
-	}
-
-	return env, nil
-}
-
 // StorageType returns storage type used by Athens with error if env is not set.
-// Possible values are memory, disk, mongo, postgres, sqlite, cockroach, mysql
+// Possible values are memory, disk, mongo
 func StorageType() (string, error) {
 	return envy.MustGet("ATHENS_STORAGE_TYPE")
 }
 
 // StorageTypeWithDefault returns storage type used by Athens with default value if env is not set.
-// Possible values are memory, disk, mongo, postgres, sqlite, cockroach, mysql
+// Possible values are memory, disk, mongo
 func StorageTypeWithDefault(value string) string {
 	return envy.Get("ATHENS_STORAGE_TYPE", value)
 }

pkg/storage/storage_tests/module_storage/storage_test.go

@@ -21,7 +21,6 @@ import (
 	"github.com/gomods/athens/pkg/storage/mem"
 	"github.com/gomods/athens/pkg/storage/minio"
 	"github.com/gomods/athens/pkg/storage/mongo"
-	// "github.com/gomods/athens/pkg/storage/rdbms"
 )
 
 type TestSuites struct {
@@ -57,11 +56,6 @@ func (d *TestSuites) SetupTest() {
 	ra.NoError(err)
 	d.storages = append(d.storages, mongoStore)
 
-	// rdbms
-	// rdbmsStore, err := rdbms.NewTestSuite(d.Model)
-	// d.Model.SetupTest()
-	// d.storages = append(d.storages, rdbmsStore)
-
 	d.module = "testmodule"
 	d.version = "v1.0.0"
 	d.mod = []byte("123")

scripts/test_unit.sh

@@ -2,7 +2,7 @@
 
 # test_unit.sh
 
-if [ -z ${ATHENS_MONGO_STORAGE_URL} ]; then 
+if [ -z ${ATHENS_MONGO_STORAGE_URL} ]; then
     export ATHENS_MONGO_STORAGE_URL="mongodb://127.0.0.1:27017"
 fi
 
@@ -10,10 +10,6 @@ if [ -z ${GO_ENV} ]; then
     export GO_ENV="test"
 fi
 
-if [ -z ${POP_PATH} ]; then
-    export POP_PATH="${PWD}/cmd/olympus"
-fi
-
 # Run the unit tests with the race detector and code coverage enabled
 set -xeuo pipefail
 go test -race -coverprofile cover.out -covermode atomic ./...