From 58d42a1b571d60c40313c91da3f4fd468d024dd0 Mon Sep 17 00:00:00 2001 From: marpio Date: Sat, 7 Jul 2018 17:46:43 +0200 Subject: [PATCH] Stream zips (#200) * storage.saver interface - use io.reader instead of []byte * stream zip - backendconnector * stream zip - azurecdn * stream zip - fs * storage - switch to io.Reader * update callers of Save * rm empty line * makezip stream * refactor mergedb * add multierror dep * mergedb multierror * scope err * update storages * return pipereader * fix test * use multierror for all strorage drivers * review feedback * azure cdn storage - use UploadStreamToBlockBlob * use numUpload * use timeout ctx --- Gopkg.lock | 14 +- Gopkg.toml | 4 + cmd/cli/upload.go | 10 +- cmd/olympus/actions/merge_db.go | 136 ++++--- cmd/olympus/actions/package_downloader.go | 8 +- cmd/proxy/actions/cache_miss_fetcher.go | 10 +- cmd/proxy/actions/fetch.go | 5 +- cmd/proxy/actions/upload.go | 3 +- pkg/module/module.go | 28 +- pkg/repo/github/github.go | 10 +- pkg/storage/azurecdn/storage.go | 107 ++++-- pkg/storage/backendconnector.go | 7 +- pkg/storage/fs/fs_test.go | 3 +- pkg/storage/fs/lister_test.go | 3 +- pkg/storage/fs/saver.go | 11 +- pkg/storage/gcp/save_test.go | 3 +- pkg/storage/gcp/saver.go | 26 +- pkg/storage/mem/mem_test.go | 3 +- pkg/storage/minio/lister_test.go | 3 +- pkg/storage/minio/minio_test.go | 3 +- pkg/storage/minio/saver.go | 5 +- pkg/storage/mongo/lister_test.go | 3 +- pkg/storage/mongo/mongo_test.go | 3 +- pkg/storage/mongo/saver.go | 10 +- pkg/storage/olympus/saver.go | 3 +- pkg/storage/rdbms/lister_test.go | 7 +- pkg/storage/rdbms/rdbms_test.go | 3 +- pkg/storage/rdbms/saver.go | 10 +- pkg/storage/s3/save_test.go | 3 +- pkg/storage/s3/storage.go | 23 +- pkg/storage/saver.go | 7 +- vendor/github.com/hashicorp/errwrap/LICENSE | 354 ++++++++++++++++++ vendor/github.com/hashicorp/errwrap/README.md | 89 +++++ .../github.com/hashicorp/errwrap/errwrap.go | 169 +++++++++ .../hashicorp/go-multierror/.travis.yml | 12 + .../hashicorp/go-multierror/LICENSE | 353 +++++++++++++++++ .../hashicorp/go-multierror/Makefile | 31 ++ .../hashicorp/go-multierror/README.md | 97 +++++ .../hashicorp/go-multierror/append.go | 41 ++ .../hashicorp/go-multierror/flatten.go | 26 ++ .../hashicorp/go-multierror/format.go | 27 ++ .../hashicorp/go-multierror/multierror.go | 51 +++ .../hashicorp/go-multierror/prefix.go | 37 ++ 43 files changed, 1580 insertions(+), 181 deletions(-) create mode 100644 vendor/github.com/hashicorp/errwrap/LICENSE create mode 100644 vendor/github.com/hashicorp/errwrap/README.md create mode 100644 vendor/github.com/hashicorp/errwrap/errwrap.go create mode 100644 vendor/github.com/hashicorp/go-multierror/.travis.yml create mode 100644 vendor/github.com/hashicorp/go-multierror/LICENSE create mode 100644 vendor/github.com/hashicorp/go-multierror/Makefile create mode 100644 vendor/github.com/hashicorp/go-multierror/README.md create mode 100644 vendor/github.com/hashicorp/go-multierror/append.go create mode 100644 vendor/github.com/hashicorp/go-multierror/flatten.go create mode 100644 vendor/github.com/hashicorp/go-multierror/format.go create mode 100644 vendor/github.com/hashicorp/go-multierror/multierror.go create mode 100644 vendor/github.com/hashicorp/go-multierror/prefix.go diff --git a/Gopkg.lock b/Gopkg.lock index 10a8db52..efc6b43e 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -329,6 +329,18 @@ revision = "ea4d1f681babbce9545c9c5f3d5194a789c89f5b" version = "v1.2.0" +[[projects]] + branch = "master" + name = "github.com/hashicorp/errwrap" + packages = ["."] + revision = "7554cd9344cec97297fa6649b055a8c98c2a1e55" + +[[projects]] + branch = "master" + name = "github.com/hashicorp/go-multierror" + packages = ["."] + revision = "b7773ae218740a7be65057fc60b366a49b538a44" + [[projects]] name = "github.com/inconshreveable/mousetrap" packages = ["."] @@ -932,6 +944,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "aa4ebb21d2f66618b1b1405408c30435de00ce8d88e2a07206ec56c2edd0a716" + inputs-digest = "f97f7787279ab52f2bb4d057552e115109606229dd6b424bd1b54aada52d94ba" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 6ae1cb26..49f347ad 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -116,3 +116,7 @@ [[constraint]] name = "google.golang.org/appengine" version = "1.0.0" + +[[constraint]] + branch = "master" + name = "github.com/hashicorp/go-multierror" diff --git a/cmd/cli/upload.go b/cmd/cli/upload.go index e914f6a1..87c2bfbb 100644 --- a/cmd/cli/upload.go +++ b/cmd/cli/upload.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "fmt" + "io/ioutil" "net/http" "net/url" "path" @@ -57,10 +58,7 @@ func upload(c *uploadCmd) func(*cobra.Command, []string) error { return fmt.Errorf("couldn't parse go.mod file (%s)", err) } - zipBytes, err := module.MakeZip(fs, fullDirectory, c.moduleName, c.version) - if err != nil { - return fmt.Errorf("couldn't make zip (%s)", err) - } + zipReader := module.MakeZip(fs, fullDirectory, c.moduleName, c.version) infoFilePath := filepath.Join(fullDirectory, c.version+".info") infoBytes, err := afero.ReadFile(fs, infoFilePath) @@ -74,6 +72,10 @@ func upload(c *uploadCmd) func(*cobra.Command, []string) error { } u.Path = path.Join(u.Path, c.moduleName, c.version) + zipBytes, err := ioutil.ReadAll(zipReader) + if err != nil { + return err + } postBody := &payloads.Upload{ Module: modBytes, Zip: zipBytes, diff --git a/cmd/olympus/actions/merge_db.go b/cmd/olympus/actions/merge_db.go index 873f54dc..80b063ff 100644 --- a/cmd/olympus/actions/merge_db.go +++ b/cmd/olympus/actions/merge_db.go @@ -2,13 +2,13 @@ package actions import ( "context" - "io/ioutil" "log" "time" "github.com/gomods/athens/pkg/cdn" "github.com/gomods/athens/pkg/eventlog" "github.com/gomods/athens/pkg/storage" + multierror "github.com/hashicorp/go-multierror" ) // mergeDB merges diff into the module database. @@ -23,74 +23,86 @@ import ( // // Both could be fixed by putting each 'for' loop into a (global) critical section func mergeDB(ctx context.Context, originURL string, diff dbDiff, eLog eventlog.Eventlog, storage storage.Backend) error { + var errors error for _, added := range diff.Added { - if _, err := eLog.ReadSingle(added.Module, added.Version); err != nil { - // the module/version already exists, is deprecated, or is - // tombstoned, so nothing to do - continue - } - - // download code from the origin - data, err := cdn.Download(originURL, added.Module, added.Version) - if err != nil { - log.Printf("error downloading new module %s/%s from %s (%s)", added.Module, added.Version, originURL, err) - continue - } - - // save module data to the CDN - zipBytes, err := ioutil.ReadAll(data.Zip) - if err != nil { - continue - } - if err := data.Zip.Close(); err != nil { - continue - } - - if err := storage.Save(ctx, added.Module, added.Version, data.Mod, zipBytes, data.Info); err != nil { - log.Printf("error saving new module %s/%s to CDN (%s)", added.Module, added.Version, err) - continue - } - - // save module metadata to the key/value store - if _, err := eLog.Append(eventlog.Event{Module: added.Module, Version: added.Version, Time: time.Now(), Op: eventlog.OpAdd}); err != nil { - log.Printf("error saving metadata for new module %s/%s (%s)", added.Module, added.Version, err) - continue + if err := add(ctx, added, originURL, eLog, storage); err != nil { + errors = multierror.Append(errors, err) } } for _, deprecated := range diff.Deprecated { - fromDB, err := eLog.ReadSingle(deprecated.Module, deprecated.Version) - if err != nil { - log.Printf("error getting deprecated module %s/%s (%s)", deprecated.Module, deprecated.Version, err) - continue - } - if fromDB.Op == eventlog.OpDel { - continue // can't deprecate something that's already deleted - } - // delete from the CDN - if err := storage.Delete(deprecated.Module, deprecated.Version); err != nil { - log.Printf("error deleting deprecated module %s/%s from CDN (%s)", deprecated.Module, deprecated.Version, err) - continue - } - - // add the tombstone to module metadata - if _, err := eLog.Append(eventlog.Event{Module: deprecated.Module, Version: deprecated.Version, Time: time.Now(), Op: eventlog.OpDel}); err != nil { - log.Printf("error saving metadata for deprecated module %s/%s from CDN (%s)", deprecated.Module, deprecated.Version, err) - continue + if err := deprecate(ctx, deprecated, originURL, eLog, storage); err != nil { + errors = multierror.Append(errors, err) } } for _, deleted := range diff.Deleted { - // delete in the CDN - if err := storage.Delete(deleted.Module, deleted.Version); err != nil { - log.Printf("error deleting deleted module %s/%s from CDN (%s)", deleted.Module, deleted.Version, err) - continue - } - // add tombstone to module metadata - if _, err := eLog.Append(eventlog.Event{Module: deleted.Module, Version: deleted.Version, Time: time.Now(), Op: eventlog.OpDel}); err != nil { - log.Printf("error inserting tombstone for deleted module %s/%s (%s)", deleted.Module, deleted.Version, err) - return err + if err := delete(deleted, eLog, storage); err != nil { + errors = multierror.Append(errors, err) } } - - return nil - + return errors +} + +func add(ctx context.Context, event eventlog.Event, originURL string, eLog eventlog.Eventlog, storage storage.Backend) error { + if _, err := eLog.ReadSingle(event.Module, event.Version); err != nil { + // the module/version already exists, is deprecated, or is + // tombstoned, so nothing to do + return err + } + + // download code from the origin + data, err := cdn.Download(originURL, event.Module, event.Version) + if err != nil { + log.Printf("error downloading new module %s/%s from %s (%s)", event.Module, event.Version, originURL, err) + return err + } + defer data.Zip.Close() + // save module data to the CDN + if err := storage.Save(ctx, event.Module, event.Version, data.Mod, data.Zip, data.Info); err != nil { + log.Printf("error saving new module %s/%s to CDN (%s)", event.Module, event.Version, err) + return err + } + + // save module metadata to the key/value store + if _, err := eLog.Append(eventlog.Event{Module: event.Module, Version: event.Version, Time: time.Now(), Op: eventlog.OpAdd}); err != nil { + log.Printf("error saving metadata for new module %s/%s (%s)", event.Module, event.Version, err) + return err + } + return nil +} + +func deprecate(ctx context.Context, event eventlog.Event, originURL string, eLog eventlog.Eventlog, storage storage.Backend) error { + fromDB, err := eLog.ReadSingle(event.Module, event.Version) + if err != nil { + log.Printf("error getting event module %s/%s (%s)", event.Module, event.Version, err) + return err + } + if fromDB.Op == eventlog.OpDel { + return err // can't deprecate something that's already deleted + } + // delete from the CDN + if err := storage.Delete(event.Module, event.Version); err != nil { + log.Printf("error deleting event module %s/%s from CDN (%s)", event.Module, event.Version, err) + return err + } + + // add the tombstone to module metadata + if _, err := eLog.Append(eventlog.Event{Module: event.Module, Version: event.Version, Time: time.Now(), Op: eventlog.OpDel}); err != nil { + log.Printf("error saving metadata for deprecated module %s/%s from CDN (%s)", event.Module, event.Version, err) + return err + } + return nil +} + +func delete(event eventlog.Event, eLog eventlog.Eventlog, storage storage.Backend) error { + // delete in the CDN + if err := storage.Delete(event.Module, event.Version); err != nil { + log.Printf("error deleting event module %s/%s from CDN (%s)", event.Module, event.Version, err) + return err + } + // add tombstone to module metadata + if _, err := eLog.Append(eventlog.Event{Module: event.Module, Version: event.Version, Time: time.Now(), Op: eventlog.OpDel}); err != nil { + log.Printf("error inserting tombstone for deleted module %s/%s (%s)", event.Module, event.Version, err) + return err + } + return nil } diff --git a/cmd/olympus/actions/package_downloader.go b/cmd/olympus/actions/package_downloader.go index 2ba5e062..b1a265c1 100644 --- a/cmd/olympus/actions/package_downloader.go +++ b/cmd/olympus/actions/package_downloader.go @@ -23,7 +23,8 @@ func GetPackageDownloaderJob(s storage.Backend, e eventlog.Eventlog, w worker.Wo } // download package - f, err := repo.NewGenericFetcher(afero.NewOsFs(), module, version) + fs := afero.NewOsFs() + f, err := repo.NewGenericFetcher(fs, module, version) if err != nil { return err } @@ -40,10 +41,11 @@ func GetPackageDownloaderJob(s storage.Backend, e eventlog.Eventlog, w worker.Wo } zipPath := filepath.Join(dirName, version+".zip") - zipBytes, err := ioutil.ReadFile(zipPath) + zipFile, err := fs.Open(zipPath) if err != nil { return err } + defer zipFile.Close() infoPath := filepath.Join(dirName, version+".info") infoBytes, err := ioutil.ReadFile(infoPath) @@ -52,7 +54,7 @@ func GetPackageDownloaderJob(s storage.Backend, e eventlog.Eventlog, w worker.Wo } // save it - if err := s.Save(context.Background(), module, version, modBytes, zipBytes, infoBytes); err != nil { + if err := s.Save(context.Background(), module, version, modBytes, zipFile, infoBytes); err != nil { return err } diff --git a/cmd/proxy/actions/cache_miss_fetcher.go b/cmd/proxy/actions/cache_miss_fetcher.go index 73573933..01cbd4a7 100644 --- a/cmd/proxy/actions/cache_miss_fetcher.go +++ b/cmd/proxy/actions/cache_miss_fetcher.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "io/ioutil" "github.com/gobuffalo/buffalo/worker" "github.com/gomods/athens/pkg/config/env" @@ -37,14 +36,9 @@ func GetProcessCacheMissJob(s storage.Backend, w worker.Worker) worker.Handler { process(module, version, args, w) return err } + defer v.Zip.Close() - zip, err := ioutil.ReadAll(v.Zip) - if err != nil { - process(module, version, args, w) - return err - } - - if err = s.Save(context.Background(), module, version, v.Mod, zip, v.Info); err != nil { + if err := s.Save(context.Background(), module, version, v.Mod, v.Zip, v.Info); err != nil { process(module, version, args, w) } diff --git a/cmd/proxy/actions/fetch.go b/cmd/proxy/actions/fetch.go index 2d7871e6..8470ae9a 100644 --- a/cmd/proxy/actions/fetch.go +++ b/cmd/proxy/actions/fetch.go @@ -50,10 +50,7 @@ func fetchHandler(store storage.Saver) func(c buffalo.Context) error { return fmt.Errorf("couldn't parse go.mod file (%s)", err) } - zipBytes, err := module.MakeZip(fs, path, moduleName, version) - if err != nil { - return fmt.Errorf("couldn't make zip (%s)", err) - } + zipBytes := module.MakeZip(fs, path, moduleName, version) infoFilePath := filepath.Join(path, version+".info") infoBytes, err := afero.ReadFile(fs, infoFilePath) diff --git a/cmd/proxy/actions/upload.go b/cmd/proxy/actions/upload.go index b57577bd..b0be9fa3 100644 --- a/cmd/proxy/actions/upload.go +++ b/cmd/proxy/actions/upload.go @@ -1,6 +1,7 @@ package actions import ( + "bytes" "net/http" "github.com/bketelsen/buffet" @@ -24,7 +25,7 @@ func uploadHandler(store storage.Saver) func(c buffalo.Context) error { if c.Bind(payload); err != nil { return errors.WithStack(err) } - saveErr := store.Save(c, mod, version, payload.Module, payload.Zip, payload.Info) + saveErr := store.Save(c, mod, version, payload.Module, bytes.NewReader(payload.Zip), payload.Info) if storage.IsVersionAlreadyExistsErr(saveErr) { return c.Error(http.StatusConflict, saveErr) } else if err != nil { diff --git a/pkg/module/module.go b/pkg/module/module.go index a3e17671..c013c8a0 100644 --- a/pkg/module/module.go +++ b/pkg/module/module.go @@ -2,8 +2,8 @@ package module import ( "archive/zip" - "bytes" "fmt" + "io" "os" "path/filepath" "strings" @@ -23,12 +23,23 @@ type file struct { // MakeZip takes dir and module info and generates vgo valid zip // the dir must end with a "/" -func MakeZip(fs afero.Fs, dir, module, version string) ([]byte, error) { +func MakeZip(fs afero.Fs, dir, module, version string) *io.PipeReader { ignoreParser := getIgnoreParser(fs, dir) - buf := new(bytes.Buffer) - w := zip.NewWriter(buf) + pr, pw := io.Pipe() - walkFunc := func(path string, info os.FileInfo, err error) error { + go func() { + zw := zip.NewWriter(pw) + defer zw.Close() + + walkFn := walkFunc(fs, zw, dir, module, version, ignoreParser) + err := afero.Walk(fs, dir, walkFn) + pw.CloseWithError(err) + }() + return pr +} + +func walkFunc(fs afero.Fs, zw *zip.Writer, dir, module, version string, ignoreParser ignore.IgnoreParser) filepath.WalkFunc { + return func(path string, info os.FileInfo, err error) error { if err != nil || info == nil || info.IsDir() { return err } @@ -44,7 +55,7 @@ func MakeZip(fs afero.Fs, dir, module, version string) ([]byte, error) { return err } - f, err := w.Create(fileName) + f, err := zw.Create(fileName) if err != nil { return err } @@ -52,11 +63,6 @@ func MakeZip(fs afero.Fs, dir, module, version string) ([]byte, error) { _, err = f.Write(fileContent) return err } - - err := afero.Walk(fs, dir, walkFunc) - w.Close() - - return buf.Bytes(), err } func getIgnoreParser(fs afero.Fs, dir string) ignore.IgnoreParser { diff --git a/pkg/repo/github/github.go b/pkg/repo/github/github.go index a1dc0eea..d79b3962 100644 --- a/pkg/repo/github/github.go +++ b/pkg/repo/github/github.go @@ -116,13 +116,15 @@ func (g *gitFetcher) Clear() error { } func (g *gitFetcher) generateZip(moduleName string) error { - zipContent, err := module.MakeZip(g.fs, g.dirName, moduleName, g.tag) + zipReader := module.MakeZip(g.fs, g.dirName, moduleName, g.tag) + + zipPath := filepath.Join(g.dirName, g.tag+".zip") + f, err := g.fs.OpenFile(zipPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.ModePerm) if err != nil { return err } - - zipPath := filepath.Join(g.dirName, g.tag+".zip") - return afero.WriteFile(g.fs, zipPath, zipContent, os.ModePerm) + _, err = io.Copy(f, zipReader) + return err } func (g *gitFetcher) generateInfo() error { diff --git a/pkg/storage/azurecdn/storage.go b/pkg/storage/azurecdn/storage.go index fc74d334..fac9f21b 100644 --- a/pkg/storage/azurecdn/storage.go +++ b/pkg/storage/azurecdn/storage.go @@ -3,13 +3,14 @@ package azurecdn import ( "bytes" "context" - "errors" "fmt" "io" "net/url" "github.com/Azure/azure-storage-blob-go/2017-07-29/azblob" "github.com/gomods/athens/pkg/config" + "github.com/gomods/athens/pkg/config/env" + multierror "github.com/hashicorp/go-multierror" ) // Storage implements (github.com/gomods/athens/pkg/storage).Saver and @@ -40,7 +41,10 @@ func (s Storage) BaseURL() *url.URL { } // Save implements the (github.com/gomods/athens/pkg/storage).Saver interface. -func (s *Storage) Save(ctx context.Context, module, version string, mod, zip, info []byte) error { +func (s *Storage) Save(ctx context.Context, module, version string, mod []byte, zip io.Reader, info []byte) error { + tctx, cancel := context.WithTimeout(ctx, env.Timeout()) + defer cancel() + pipe := azblob.NewPipeline(s.cred, azblob.PipelineOptions{}) serviceURL := azblob.NewServiceURL(*s.accountURL, pipe) // rules on container names: @@ -49,53 +53,88 @@ func (s *Storage) Save(ctx context.Context, module, version string, mod, zip, in // This container must exist containerURL := serviceURL.NewContainerURL("gomodules") - infoBlobURL := containerURL.NewBlockBlobURL(config.PackageVersionedName(module, version, "info")) - modBlobURL := containerURL.NewBlockBlobURL(config.PackageVersionedName(module, version, "mod")) - zipBlobURL := containerURL.NewBlockBlobURL(config.PackageVersionedName(module, version, "zip")) - + emptyMeta := map[string]string{} + emptyBlobAccessCond := azblob.BlobAccessConditions{} httpHeaders := func(contentType string) azblob.BlobHTTPHeaders { return azblob.BlobHTTPHeaders{ ContentType: contentType, } } - emptyMeta := map[string]string{} - emptyBlobAccessCond := azblob.BlobAccessConditions{} + uploadOpts := func(contentType string) uploadOptions { + return uploadOptions{ + accessConditions: emptyBlobAccessCond, + metadata: emptyMeta, + blobHTTPHeaders: httpHeaders(contentType), + } + } const numUpload = 3 - uploadErrs := make(chan error, numUpload) + errChan := make(chan error, numUpload) - upload := func(url azblob.BlockBlobURL, content io.ReadSeeker, contentType string) { - _, err := url.Upload(ctx, content, httpHeaders(contentType), emptyMeta, emptyBlobAccessCond) - uploadErrs <- err - } + go uploadBlob(tctx, errChan, containerURL, module, version, "info", bytes.NewReader(info), uploadOpts("application/json")) + go uploadBlob(tctx, errChan, containerURL, module, version, "mod", bytes.NewReader(mod), uploadOpts("text/plain")) + go uploadStream(tctx, errChan, containerURL, module, version, "zip", zip, uploadOpts("application/octet-stream")) - go upload(infoBlobURL, bytes.NewReader(info), "application/json") - go upload(modBlobURL, bytes.NewReader(mod), "text/plain") - go upload(zipBlobURL, bytes.NewReader(zip), "application/octet-stream") - - encountered := make([]error, 0, numUpload) + var errors error for i := 0; i < numUpload; i++ { - select { - case err := <-uploadErrs: - if err != nil { - encountered = append(encountered, err) - } - case <-ctx.Done(): - return ctx.Err() + err := <-errChan + if err != nil { + errors = multierror.Append(errors, err) } } - - if len(encountered) > 0 { - message := bytes.NewBufferString("encountered multiple errors during save:\n") - for _, err := range encountered { - fmt.Fprintln(message, err.Error()) - } - return errors.New(message.String()) - } + close(errChan) // TODO: take out lease on the /list file and add the version to it // // Do that only after module source+metadata is uploaded - return nil + return errors +} + +func blockBlobURL(containerURL azblob.ContainerURL, module, version, ext string) azblob.BlockBlobURL { + return containerURL.NewBlockBlobURL(config.PackageVersionedName(module, version, ext)) +} + +func uploadBlob(ctx context.Context, errChan chan<- error, containerURL azblob.ContainerURL, module, version, ext string, content io.ReadSeeker, opts uploadOptions) { + url := blockBlobURL(containerURL, module, version, ext) + save := func() error { + _, err := url.Upload(ctx, content, opts.blobHTTPHeaders, opts.metadata, opts.accessConditions) + return err + } + + select { + case errChan <- save(): + case <-ctx.Done(): + errChan <- fmt.Errorf("uploading %s/%s.%s timed out", module, version, ext) + } +} + +func uploadStream(ctx context.Context, errChan chan<- error, containerURL azblob.ContainerURL, module, version, ext string, stream io.Reader, opts uploadOptions) { + bufferSize := 1 * 1024 * 1024 // Size of the rotating buffers that are used when uploading + maxBuffers := 3 // Number of rotating buffers that are used when uploading + + uploadStreamOpts := azblob.UploadStreamToBlockBlobOptions{ + BufferSize: bufferSize, + MaxBuffers: maxBuffers, + BlobHTTPHeaders: opts.blobHTTPHeaders, + Metadata: opts.metadata, + AccessConditions: opts.accessConditions, + } + url := blockBlobURL(containerURL, module, version, ext) + save := func() error { + _, err := azblob.UploadStreamToBlockBlob(ctx, stream, url, uploadStreamOpts) + return err + } + + select { + case errChan <- save(): + case <-ctx.Done(): + errChan <- fmt.Errorf("uploading %s/%s.%s timed out", module, version, ext) + } +} + +type uploadOptions struct { + blobHTTPHeaders azblob.BlobHTTPHeaders + metadata map[string]string + accessConditions azblob.BlobAccessConditions } diff --git a/pkg/storage/backendconnector.go b/pkg/storage/backendconnector.go index 5afa557a..45846d41 100644 --- a/pkg/storage/backendconnector.go +++ b/pkg/storage/backendconnector.go @@ -1,6 +1,9 @@ package storage -import "context" +import ( + "context" + "io" +) // BackendConnector is a regular storage backend with Connect functionality type BackendConnector interface { @@ -31,7 +34,7 @@ func (n noOpConnectedBackend) Get(module, vsn string) (*Version, error) { func (n noOpConnectedBackend) List(module string) ([]string, error) { return n.backend.List(module) } -func (n noOpConnectedBackend) Save(ctx context.Context, module, version string, mod, zip, info []byte) error { +func (n noOpConnectedBackend) Save(ctx context.Context, module, version string, mod []byte, zip io.Reader, info []byte) error { return n.backend.Save(ctx, module, version, mod, zip, info) } func (n noOpConnectedBackend) Delete(module, version string) error { diff --git a/pkg/storage/fs/fs_test.go b/pkg/storage/fs/fs_test.go index a5c947a1..00640c2a 100644 --- a/pkg/storage/fs/fs_test.go +++ b/pkg/storage/fs/fs_test.go @@ -1,6 +1,7 @@ package fs import ( + "bytes" "context" "io/ioutil" "path/filepath" @@ -17,7 +18,7 @@ func (d *FsTests) TestLocationFuncs() { func (d *FsTests) TestGetSaveListRoundTrip() { r := d.Require() - r.NoError(d.storage.Save(context.Background(), module, version, mod, zip, info)) + r.NoError(d.storage.Save(context.Background(), module, version, mod, bytes.NewReader(zip), info)) listedVersions, err := d.storage.List(module) r.NoError(err) r.Equal(1, len(listedVersions)) diff --git a/pkg/storage/fs/lister_test.go b/pkg/storage/fs/lister_test.go index 2c359e16..6b026ed0 100644 --- a/pkg/storage/fs/lister_test.go +++ b/pkg/storage/fs/lister_test.go @@ -1,6 +1,7 @@ package fs import ( + "bytes" "context" ) @@ -8,7 +9,7 @@ func (d *FsTests) TestList() { r := d.Require() versions := []string{"v1.0.0", "v1.1.0", "v1.2.0"} for _, version := range versions { - r.NoError(d.storage.Save(context.Background(), module, version, mod, zip, info)) + r.NoError(d.storage.Save(context.Background(), module, version, mod, bytes.NewReader(zip), info)) } retVersions, err := d.storage.List(module) r.NoError(err) diff --git a/pkg/storage/fs/saver.go b/pkg/storage/fs/saver.go index f6da11ec..7e64ce03 100644 --- a/pkg/storage/fs/saver.go +++ b/pkg/storage/fs/saver.go @@ -2,13 +2,14 @@ package fs import ( "context" + "io" "os" "path/filepath" "github.com/spf13/afero" ) -func (s *storageImpl) Save(_ context.Context, module, vsn string, mod, zip, info []byte) error { +func (s *storageImpl) Save(_ context.Context, module, vsn string, mod []byte, zip io.Reader, info []byte) error { dir := s.versionLocation(module, vsn) // TODO: 777 is not the best filemode, use something better @@ -23,7 +24,13 @@ func (s *storageImpl) Save(_ context.Context, module, vsn string, mod, zip, info } // write the zipfile - if err := afero.WriteFile(s.filesystem, filepath.Join(dir, "source.zip"), zip, os.ModePerm); err != nil { + f, err := s.filesystem.OpenFile(filepath.Join(dir, "source.zip"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.ModePerm) + if err != nil { + return err + } + defer f.Close() + _, err = io.Copy(f, zip) + if err != nil { return err } diff --git a/pkg/storage/gcp/save_test.go b/pkg/storage/gcp/save_test.go index 617c4ba0..6b441d08 100644 --- a/pkg/storage/gcp/save_test.go +++ b/pkg/storage/gcp/save_test.go @@ -1,6 +1,7 @@ package gcp import ( + "bytes" "context" "cloud.google.com/go/storage" @@ -19,7 +20,7 @@ func (g *GcpTests) TestSave() { r := g.Require() store, err := New(g.context, g.options) r.NoError(err) - err = store.Save(g.context, g.module, g.version, mod, info, zip) + err = store.Save(g.context, g.module, g.version, mod, bytes.NewReader(zip), info) r.NoError(err) err = exists(g.context, g.options, g.bucket, g.module, g.version) diff --git a/pkg/storage/gcp/saver.go b/pkg/storage/gcp/saver.go index b7566f9a..debabbd0 100644 --- a/pkg/storage/gcp/saver.go +++ b/pkg/storage/gcp/saver.go @@ -1,15 +1,17 @@ package gcp import ( + "bytes" "context" "fmt" + "io" "log" - "strings" "time" "cloud.google.com/go/storage" "github.com/gomods/athens/pkg/config" "github.com/gomods/athens/pkg/config/env" + multierror "github.com/hashicorp/go-multierror" "google.golang.org/api/option" ) @@ -49,36 +51,32 @@ func New(ctx context.Context, cred option.ClientOption) (*Storage, error) { // // Uploaded files are publicly accessable in the storage bucket as per // an ACL rule. -func (s *Storage) Save(ctx context.Context, module, version string, mod, zip, info []byte) error { +func (s *Storage) Save(ctx context.Context, module, version string, mod []byte, zip io.Reader, info []byte) error { errs := make(chan error, 3) // create a context that will time out after 300 seconds / 5 minutes ctxWT, cancelCTX := context.WithTimeout(ctx, 300*time.Second) defer cancelCTX() // dispatch go routine for each file to upload - go upload(ctxWT, errs, s.bucket, module, version, "mod", mod) + go upload(ctxWT, errs, s.bucket, module, version, "mod", bytes.NewReader(mod)) go upload(ctxWT, errs, s.bucket, module, version, "zip", zip) - go upload(ctxWT, errs, s.bucket, module, version, "info", info) + go upload(ctxWT, errs, s.bucket, module, version, "info", bytes.NewReader(info)) - errsOut := make([]string, 0, 3) + var errors error // wait for each routine above to send a value for count := 0; count < 3; count++ { err := <-errs if err != nil { - errsOut = append(errsOut, err.Error()) + errors = multierror.Append(errors, err) } } close(errs) - // return concatenated error string if there is anything to report - if len(errsOut) > 0 { - return fmt.Errorf("one or more errors occured saving %s/@v/%s: %s", module, version, strings.Join(errsOut, ", ")) - } - return nil + return errors } // upload waits for either writeToBucket to complete or the context expires -func upload(ctx context.Context, errs chan<- error, bkt *storage.BucketHandle, module, version, ext string, file []byte) { +func upload(ctx context.Context, errs chan<- error, bkt *storage.BucketHandle, module, version, ext string, file io.Reader) { select { case errs <- writeToBucket(ctx, bkt, config.PackageVersionedName(module, version, ext), file): return @@ -88,7 +86,7 @@ func upload(ctx context.Context, errs chan<- error, bkt *storage.BucketHandle, m } // writeToBucket performs the actual write to a gcp storage bucket -func writeToBucket(ctx context.Context, bkt *storage.BucketHandle, filename string, file []byte) error { +func writeToBucket(ctx context.Context, bkt *storage.BucketHandle, filename string, file io.Reader) error { wc := bkt.Object(filename).NewWriter(ctx) defer func(w *storage.Writer) { if err := w.Close(); err != nil { @@ -98,7 +96,7 @@ func writeToBucket(ctx context.Context, bkt *storage.BucketHandle, filename stri wc.ContentType = "application/octet-stream" // TODO: have this configurable to allow for mixed public/private modules wc.ACL = []storage.ACLRule{{Entity: storage.AllUsers, Role: storage.RoleReader}} - if _, err := wc.Write(file); err != nil { + if _, err := io.Copy(wc, file); err != nil { return err } return nil diff --git a/pkg/storage/mem/mem_test.go b/pkg/storage/mem/mem_test.go index b05aa7dc..a8fd765a 100644 --- a/pkg/storage/mem/mem_test.go +++ b/pkg/storage/mem/mem_test.go @@ -1,6 +1,7 @@ package mem import ( + "bytes" "context" "io/ioutil" "testing" @@ -41,7 +42,7 @@ func (d *MemTests) TestGetSaveListRoundTrip() { d.Require().NoError(err) // save and list modules - r.NoError(storage.Save(context.Background(), module, version, mod, zip, info)) + r.NoError(storage.Save(context.Background(), module, version, mod, bytes.NewReader(zip), info)) listedVersions, err := storage.List(module) r.NoError(err) r.Equal(1, len(listedVersions)) diff --git a/pkg/storage/minio/lister_test.go b/pkg/storage/minio/lister_test.go index 03d2c0b5..d913b2c7 100644 --- a/pkg/storage/minio/lister_test.go +++ b/pkg/storage/minio/lister_test.go @@ -1,6 +1,7 @@ package minio import ( + "bytes" "context" ) @@ -8,7 +9,7 @@ func (d *MinioTests) TestList() { r := d.Require() versions := []string{"v1.0.0", "v1.1.0", "v1.2.0"} for _, version := range versions { - r.NoError(d.storage.Save(context.Background(), module, version, mod, zip, info)) + r.NoError(d.storage.Save(context.Background(), module, version, mod, bytes.NewReader(zip), info)) } retVersions, err := d.storage.List(module) r.NoError(err) diff --git a/pkg/storage/minio/minio_test.go b/pkg/storage/minio/minio_test.go index 7baebab5..32731b21 100644 --- a/pkg/storage/minio/minio_test.go +++ b/pkg/storage/minio/minio_test.go @@ -1,13 +1,14 @@ package minio import ( + "bytes" "context" "io/ioutil" ) func (d *MinioTests) TestGetSaveListRoundTrip() { r := d.Require() - r.NoError(d.storage.Save(context.Background(), module, version, mod, zip, info)) + r.NoError(d.storage.Save(context.Background(), module, version, mod, bytes.NewReader(zip), info)) listedVersions, err := d.storage.List(module) r.NoError(err) r.Equal(1, len(listedVersions)) diff --git a/pkg/storage/minio/saver.go b/pkg/storage/minio/saver.go index e2223dd3..a071d0c8 100644 --- a/pkg/storage/minio/saver.go +++ b/pkg/storage/minio/saver.go @@ -3,11 +3,12 @@ package minio import ( "bytes" "context" + "io" minio "github.com/minio/minio-go" ) -func (s *storageImpl) Save(_ context.Context, module, vsn string, mod, zip, info []byte) error { +func (s *storageImpl) Save(_ context.Context, module, vsn string, mod []byte, zip io.Reader, info []byte) error { dir := s.versionLocation(module, vsn) modFileName := dir + "/" + "go.mod" zipFileName := dir + "/" + "source.zip" @@ -16,7 +17,7 @@ func (s *storageImpl) Save(_ context.Context, module, vsn string, mod, zip, info if err != nil { return err } - _, err = s.minioClient.PutObject(s.bucketName, zipFileName, bytes.NewReader(zip), int64(len(zip)), minio.PutObjectOptions{}) + _, err = s.minioClient.PutObject(s.bucketName, zipFileName, zip, -1, minio.PutObjectOptions{}) if err != nil { return err } diff --git a/pkg/storage/mongo/lister_test.go b/pkg/storage/mongo/lister_test.go index bdae8a0e..da24b737 100644 --- a/pkg/storage/mongo/lister_test.go +++ b/pkg/storage/mongo/lister_test.go @@ -1,6 +1,7 @@ package mongo import ( + "bytes" "context" ) @@ -8,7 +9,7 @@ func (m *MongoTests) TestList() { r := m.Require() versions := []string{"v1.0.0", "v1.1.0", "v1.2.0"} for _, version := range versions { - m.storage.Save(context.Background(), module, version, mod, zip, info) + m.storage.Save(context.Background(), module, version, mod, bytes.NewReader(zip), info) } retVersions, err := m.storage.List(module) r.NoError(err) diff --git a/pkg/storage/mongo/mongo_test.go b/pkg/storage/mongo/mongo_test.go index 88f14346..62538148 100644 --- a/pkg/storage/mongo/mongo_test.go +++ b/pkg/storage/mongo/mongo_test.go @@ -1,13 +1,14 @@ package mongo import ( + "bytes" "context" "io/ioutil" ) func (m *MongoTests) TestGetSaveListRoundTrip() { r := m.Require() - m.storage.Save(context.Background(), module, version, mod, zip, info) + m.storage.Save(context.Background(), module, version, mod, bytes.NewReader(zip), info) listedVersions, err := m.storage.List(module) r.NoError(err) r.Equal(1, len(listedVersions)) diff --git a/pkg/storage/mongo/saver.go b/pkg/storage/mongo/saver.go index a57fb73e..98e74af7 100644 --- a/pkg/storage/mongo/saver.go +++ b/pkg/storage/mongo/saver.go @@ -2,17 +2,23 @@ package mongo import ( "context" + "io" + "io/ioutil" "github.com/gomods/athens/pkg/storage" ) // Save stores a module in mongo storage. -func (s *ModuleStore) Save(_ context.Context, module, version string, mod, zip, info []byte) error { +func (s *ModuleStore) Save(_ context.Context, module, version string, mod []byte, zip io.Reader, info []byte) error { + zipBytes, err := ioutil.ReadAll(zip) + if err != nil { + return err + } m := &storage.Module{ Module: module, Version: version, Mod: mod, - Zip: zip, + Zip: zipBytes, Info: info, } diff --git a/pkg/storage/olympus/saver.go b/pkg/storage/olympus/saver.go index 4239bc7e..b45fac3a 100644 --- a/pkg/storage/olympus/saver.go +++ b/pkg/storage/olympus/saver.go @@ -2,11 +2,12 @@ package olympus import ( "context" + "io" ) // Save stores a module in olympus. // This actually does not store anything just reports cache miss -func (s *ModuleStore) Save(_ context.Context, module, version string, _, _, _ []byte) error { +func (s *ModuleStore) Save(_ context.Context, module, version string, _ []byte, _ io.ReadSeeker, _ []byte) error { // dummy implementation so Olympus Store can be used everywhere as Backend iface return nil } diff --git a/pkg/storage/rdbms/lister_test.go b/pkg/storage/rdbms/lister_test.go index e41763de..d1434f68 100644 --- a/pkg/storage/rdbms/lister_test.go +++ b/pkg/storage/rdbms/lister_test.go @@ -1,12 +1,15 @@ package rdbms -import "context" +import ( + "bytes" + "context" +) func (rd *RDBMSTestSuite) TestList() { r := rd.Require() versions := []string{"v1.0.0", "v1.1.0", "v1.2.0"} for _, version := range versions { - rd.storage.Save(context.Background(), module, version, mod, zip, info) + rd.storage.Save(context.Background(), module, version, mod, bytes.NewReader(zip), info) } retVersions, err := rd.storage.List(module) r.NoError(err) diff --git a/pkg/storage/rdbms/rdbms_test.go b/pkg/storage/rdbms/rdbms_test.go index 90f42522..d435d513 100644 --- a/pkg/storage/rdbms/rdbms_test.go +++ b/pkg/storage/rdbms/rdbms_test.go @@ -1,13 +1,14 @@ package rdbms import ( + "bytes" "context" "io/ioutil" ) func (rd *RDBMSTestSuite) TestGetSaveListRoundTrip() { r := rd.Require() - err := rd.storage.Save(context.Background(), module, version, mod, zip, info) + err := rd.storage.Save(context.Background(), module, version, mod, bytes.NewReader(zip), info) r.NoError(err) listedVersions, err := rd.storage.List(module) r.NoError(err) diff --git a/pkg/storage/rdbms/saver.go b/pkg/storage/rdbms/saver.go index e9f79c2e..395e0c8b 100644 --- a/pkg/storage/rdbms/saver.go +++ b/pkg/storage/rdbms/saver.go @@ -2,17 +2,23 @@ package rdbms import ( "context" + "io" + "io/ioutil" "github.com/gomods/athens/pkg/storage/rdbms/models" ) // Save stores a module in rdbms storage. -func (r *ModuleStore) Save(_ context.Context, module, version string, mod, zip, info []byte) error { +func (r *ModuleStore) Save(_ context.Context, module, version string, mod []byte, zip io.Reader, info []byte) error { + zipBytes, err := ioutil.ReadAll(zip) + if err != nil { + return err + } m := &models.Module{ Module: module, Version: version, Mod: mod, - Zip: zip, + Zip: zipBytes, Info: info, } diff --git a/pkg/storage/s3/save_test.go b/pkg/storage/s3/save_test.go index df875a3e..75a758d0 100644 --- a/pkg/storage/s3/save_test.go +++ b/pkg/storage/s3/save_test.go @@ -1,6 +1,7 @@ package s3 import ( + "bytes" "context" "fmt" @@ -24,7 +25,7 @@ func (d *S3Tests) TestSave() { vinfo := append(info, []byte(version)...) vzip := append(zip, []byte(version)...) - r.NoError(d.storage.Save(context.Background(), module, version, vmod, vzip, vinfo)) + r.NoError(d.storage.Save(context.Background(), module, version, vmod, bytes.NewReader(vzip), vinfo)) expectedValues[config.PackageVersionedName(module, version, "info")] = vinfo expectedValues[config.PackageVersionedName(module, version, "mod")] = vmod expectedValues[config.PackageVersionedName(module, version, "zip")] = vzip diff --git a/pkg/storage/s3/storage.go b/pkg/storage/s3/storage.go index e5559d63..31aa74d8 100644 --- a/pkg/storage/s3/storage.go +++ b/pkg/storage/s3/storage.go @@ -4,14 +4,16 @@ import ( "bytes" "context" "fmt" + "io" + "io/ioutil" "net/url" - "strings" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3iface" "github.com/gomods/athens/pkg/config" "github.com/gomods/athens/pkg/config/env" + multierror "github.com/hashicorp/go-multierror" ) // Storage implements (github.com/gomods/athens/pkg/storage).Saver and @@ -75,30 +77,31 @@ func (s Storage) BaseURL() *url.URL { } // Save implements the (github.com/gomods/athens/pkg/storage).Saver interface. -func (s *Storage) Save(ctx context.Context, module, version string, mod, zip, info []byte) error { +func (s *Storage) Save(ctx context.Context, module, version string, mod []byte, zip io.Reader, info []byte) error { errChan := make(chan error, 3) tctx, cancel := context.WithTimeout(ctx, env.Timeout()) defer cancel() + zipBytes, err := ioutil.ReadAll(zip) + if err != nil { + return err + } + go s.upload(tctx, errChan, module, version, "mod", mod) - go s.upload(tctx, errChan, module, version, "zip", zip) + go s.upload(tctx, errChan, module, version, "zip", zipBytes) go s.upload(tctx, errChan, module, version, "info", info) - errs := make([]string, 0, 3) + var errors error for i := 0; i < 3; i++ { err := <-errChan if err != nil { - errs = append(errs, err.Error()) + errors = multierror.Append(errors, err) } } close(errChan) - if len(errs) > 0 { - return fmt.Errorf("One or more errors occured saving %s %s: %s", module, version, strings.Join(errs, ", ")) - } - - return nil + return errors } func (s *Storage) upload(ctx context.Context, errChan chan<- error, module, version, name string, content []byte) { diff --git a/pkg/storage/saver.go b/pkg/storage/saver.go index a720319f..47777c99 100644 --- a/pkg/storage/saver.go +++ b/pkg/storage/saver.go @@ -1,8 +1,11 @@ package storage -import "context" +import ( + "context" + "io" +) // Saver saves module metadata and its source to underlying storage type Saver interface { - Save(ctx context.Context, module, version string, mod, zip, info []byte) error + Save(ctx context.Context, module, version string, mod []byte, zip io.Reader, info []byte) error } diff --git a/vendor/github.com/hashicorp/errwrap/LICENSE b/vendor/github.com/hashicorp/errwrap/LICENSE new file mode 100644 index 00000000..c33dcc7c --- /dev/null +++ b/vendor/github.com/hashicorp/errwrap/LICENSE @@ -0,0 +1,354 @@ +Mozilla Public License, version 2.0 + +1. Definitions + +1.1. “Contributor” + + means each individual or legal entity that creates, contributes to the + creation of, or owns Covered Software. + +1.2. “Contributor Version” + + means the combination of the Contributions of others (if any) used by a + Contributor and that particular Contributor’s Contribution. + +1.3. “Contribution” + + means Covered Software of a particular Contributor. + +1.4. “Covered Software” + + means Source Code Form to which the initial Contributor has attached the + notice in Exhibit A, the Executable Form of such Source Code Form, and + Modifications of such Source Code Form, in each case including portions + thereof. + +1.5. “Incompatible With Secondary Licenses” + means + + a. that the initial Contributor has attached the notice described in + Exhibit B to the Covered Software; or + + b. that the Covered Software was made available under the terms of version + 1.1 or earlier of the License, but not also under the terms of a + Secondary License. + +1.6. “Executable Form” + + means any form of the work other than Source Code Form. + +1.7. “Larger Work” + + means a work that combines Covered Software with other material, in a separate + file or files, that is not Covered Software. + +1.8. “License” + + means this document. + +1.9. “Licensable” + + means having the right to grant, to the maximum extent possible, whether at the + time of the initial grant or subsequently, any and all of the rights conveyed by + this License. + +1.10. “Modifications” + + means any of the following: + + a. any file in Source Code Form that results from an addition to, deletion + from, or modification of the contents of Covered Software; or + + b. any new file in Source Code Form that contains any Covered Software. + +1.11. “Patent Claims” of a Contributor + + means any patent claim(s), including without limitation, method, process, + and apparatus claims, in any patent Licensable by such Contributor that + would be infringed, but for the grant of the License, by the making, + using, selling, offering for sale, having made, import, or transfer of + either its Contributions or its Contributor Version. + +1.12. “Secondary License” + + means either the GNU General Public License, Version 2.0, the GNU Lesser + General Public License, Version 2.1, the GNU Affero General Public + License, Version 3.0, or any later versions of those licenses. + +1.13. “Source Code Form” + + means the form of the work preferred for making modifications. + +1.14. “You” (or “Your”) + + means an individual or a legal entity exercising rights under this + License. For legal entities, “You” includes any entity that controls, is + controlled by, or is under common control with You. For purposes of this + definition, “control” means (a) the power, direct or indirect, to cause + the direction or management of such entity, whether by contract or + otherwise, or (b) ownership of more than fifty percent (50%) of the + outstanding shares or beneficial ownership of such entity. + + +2. License Grants and Conditions + +2.1. Grants + + Each Contributor hereby grants You a world-wide, royalty-free, + non-exclusive license: + + a. under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or as + part of a Larger Work; and + + b. under Patent Claims of such Contributor to make, use, sell, offer for + sale, have made, import, and otherwise transfer either its Contributions + or its Contributor Version. + +2.2. Effective Date + + The licenses granted in Section 2.1 with respect to any Contribution become + effective for each Contribution on the date the Contributor first distributes + such Contribution. + +2.3. Limitations on Grant Scope + + The licenses granted in this Section 2 are the only rights granted under this + License. No additional rights or licenses will be implied from the distribution + or licensing of Covered Software under this License. Notwithstanding Section + 2.1(b) above, no patent license is granted by a Contributor: + + a. for any code that a Contributor has removed from Covered Software; or + + b. for infringements caused by: (i) Your and any other third party’s + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + + c. under Patent Claims infringed by Covered Software in the absence of its + Contributions. + + This License does not grant any rights in the trademarks, service marks, or + logos of any Contributor (except as may be necessary to comply with the + notice requirements in Section 3.4). + +2.4. Subsequent Licenses + + No Contributor makes additional grants as a result of Your choice to + distribute the Covered Software under a subsequent version of this License + (see Section 10.2) or under the terms of a Secondary License (if permitted + under the terms of Section 3.3). + +2.5. Representation + + Each Contributor represents that the Contributor believes its Contributions + are its original creation(s) or it has sufficient rights to grant the + rights to its Contributions conveyed by this License. + +2.6. Fair Use + + This License is not intended to limit any rights You have under applicable + copyright doctrines of fair use, fair dealing, or other equivalents. + +2.7. Conditions + + Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in + Section 2.1. + + +3. Responsibilities + +3.1. Distribution of Source Form + + All distribution of Covered Software in Source Code Form, including any + Modifications that You create or to which You contribute, must be under the + terms of this License. You must inform recipients that the Source Code Form + of the Covered Software is governed by the terms of this License, and how + they can obtain a copy of this License. You may not attempt to alter or + restrict the recipients’ rights in the Source Code Form. + +3.2. Distribution of Executable Form + + If You distribute Covered Software in Executable Form then: + + a. such Covered Software must also be made available in Source Code Form, + as described in Section 3.1, and You must inform recipients of the + Executable Form how they can obtain a copy of such Source Code Form by + reasonable means in a timely manner, at a charge no more than the cost + of distribution to the recipient; and + + b. You may distribute such Executable Form under the terms of this License, + or sublicense it under different terms, provided that the license for + the Executable Form does not attempt to limit or alter the recipients’ + rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + + You may create and distribute a Larger Work under terms of Your choice, + provided that You also comply with the requirements of this License for the + Covered Software. If the Larger Work is a combination of Covered Software + with a work governed by one or more Secondary Licenses, and the Covered + Software is not Incompatible With Secondary Licenses, this License permits + You to additionally distribute such Covered Software under the terms of + such Secondary License(s), so that the recipient of the Larger Work may, at + their option, further distribute the Covered Software under the terms of + either this License or such Secondary License(s). + +3.4. Notices + + You may not remove or alter the substance of any license notices (including + copyright notices, patent notices, disclaimers of warranty, or limitations + of liability) contained within the Source Code Form of the Covered + Software, except that You may alter any license notices to the extent + required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + + You may choose to offer, and to charge a fee for, warranty, support, + indemnity or liability obligations to one or more recipients of Covered + Software. However, You may do so only on Your own behalf, and not on behalf + of any Contributor. You must make it absolutely clear that any such + warranty, support, indemnity, or liability obligation is offered by You + alone, and You hereby agree to indemnify every Contributor for any + liability incurred by such Contributor as a result of warranty, support, + indemnity or liability terms You offer. You may include additional + disclaimers of warranty and limitations of liability specific to any + jurisdiction. + +4. Inability to Comply Due to Statute or Regulation + + If it is impossible for You to comply with any of the terms of this License + with respect to some or all of the Covered Software due to statute, judicial + order, or regulation then You must: (a) comply with the terms of this License + to the maximum extent possible; and (b) describe the limitations and the code + they affect. Such description must be placed in a text file included with all + distributions of the Covered Software under this License. Except to the + extent prohibited by statute or regulation, such description must be + sufficiently detailed for a recipient of ordinary skill to be able to + understand it. + +5. Termination + +5.1. The rights granted under this License will terminate automatically if You + fail to comply with any of its terms. However, if You become compliant, + then the rights granted under this License from a particular Contributor + are reinstated (a) provisionally, unless and until such Contributor + explicitly and finally terminates Your grants, and (b) on an ongoing basis, + if such Contributor fails to notify You of the non-compliance by some + reasonable means prior to 60 days after You have come back into compliance. + Moreover, Your grants from a particular Contributor are reinstated on an + ongoing basis if such Contributor notifies You of the non-compliance by + some reasonable means, this is the first time You have received notice of + non-compliance with this License from such Contributor, and You become + compliant prior to 30 days after Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent + infringement claim (excluding declaratory judgment actions, counter-claims, + and cross-claims) alleging that a Contributor Version directly or + indirectly infringes any patent, then the rights granted to You by any and + all Contributors for the Covered Software under Section 2.1 of this License + shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user + license agreements (excluding distributors and resellers) which have been + validly granted by You or Your distributors under this License prior to + termination shall survive termination. + +6. Disclaimer of Warranty + + Covered Software is provided under this License on an “as is” basis, without + warranty of any kind, either expressed, implied, or statutory, including, + without limitation, warranties that the Covered Software is free of defects, + merchantable, fit for a particular purpose or non-infringing. The entire + risk as to the quality and performance of the Covered Software is with You. + Should any Covered Software prove defective in any respect, You (not any + Contributor) assume the cost of any necessary servicing, repair, or + correction. This disclaimer of warranty constitutes an essential part of this + License. No use of any Covered Software is authorized under this License + except under this disclaimer. + +7. Limitation of Liability + + Under no circumstances and under no legal theory, whether tort (including + negligence), contract, or otherwise, shall any Contributor, or anyone who + distributes Covered Software as permitted above, be liable to You for any + direct, indirect, special, incidental, or consequential damages of any + character including, without limitation, damages for lost profits, loss of + goodwill, work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses, even if such party shall have been + informed of the possibility of such damages. This limitation of liability + shall not apply to liability for death or personal injury resulting from such + party’s negligence to the extent applicable law prohibits such limitation. + Some jurisdictions do not allow the exclusion or limitation of incidental or + consequential damages, so this exclusion and limitation may not apply to You. + +8. Litigation + + Any litigation relating to this License may be brought only in the courts of + a jurisdiction where the defendant maintains its principal place of business + and such litigation shall be governed by laws of that jurisdiction, without + reference to its conflict-of-law provisions. Nothing in this Section shall + prevent a party’s ability to bring cross-claims or counter-claims. + +9. Miscellaneous + + This License represents the complete agreement concerning the subject matter + hereof. If any provision of this License is held to be unenforceable, such + provision shall be reformed only to the extent necessary to make it + enforceable. Any law or regulation which provides that the language of a + contract shall be construed against the drafter shall not be used to construe + this License against a Contributor. + + +10. Versions of the License + +10.1. New Versions + + Mozilla Foundation is the license steward. Except as provided in Section + 10.3, no one other than the license steward has the right to modify or + publish new versions of this License. Each version will be given a + distinguishing version number. + +10.2. Effect of New Versions + + You may distribute the Covered Software under the terms of the version of + the License under which You originally received the Covered Software, or + under the terms of any subsequent version published by the license + steward. + +10.3. Modified Versions + + If you create software not governed by this License, and you want to + create a new license for such software, you may create and use a modified + version of this License if you rename the license and remove any + references to the name of the license steward (except to note that such + modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses + If You choose to distribute Source Code Form that is Incompatible With + Secondary Licenses under the terms of this version of the License, the + notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice + + This Source Code Form is subject to the + terms of the Mozilla Public License, v. + 2.0. If a copy of the MPL was not + distributed with this file, You can + obtain one at + http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular file, then +You may include the notice in a location (such as a LICENSE file in a relevant +directory) where a recipient would be likely to look for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - “Incompatible With Secondary Licenses” Notice + + This Source Code Form is “Incompatible + With Secondary Licenses”, as defined by + the Mozilla Public License, v. 2.0. + diff --git a/vendor/github.com/hashicorp/errwrap/README.md b/vendor/github.com/hashicorp/errwrap/README.md new file mode 100644 index 00000000..1c95f597 --- /dev/null +++ b/vendor/github.com/hashicorp/errwrap/README.md @@ -0,0 +1,89 @@ +# errwrap + +`errwrap` is a package for Go that formalizes the pattern of wrapping errors +and checking if an error contains another error. + +There is a common pattern in Go of taking a returned `error` value and +then wrapping it (such as with `fmt.Errorf`) before returning it. The problem +with this pattern is that you completely lose the original `error` structure. + +Arguably the _correct_ approach is that you should make a custom structure +implementing the `error` interface, and have the original error as a field +on that structure, such [as this example](http://golang.org/pkg/os/#PathError). +This is a good approach, but you have to know the entire chain of possible +rewrapping that happens, when you might just care about one. + +`errwrap` formalizes this pattern (it doesn't matter what approach you use +above) by giving a single interface for wrapping errors, checking if a specific +error is wrapped, and extracting that error. + +## Installation and Docs + +Install using `go get github.com/hashicorp/errwrap`. + +Full documentation is available at +http://godoc.org/github.com/hashicorp/errwrap + +## Usage + +#### Basic Usage + +Below is a very basic example of its usage: + +```go +// A function that always returns an error, but wraps it, like a real +// function might. +func tryOpen() error { + _, err := os.Open("/i/dont/exist") + if err != nil { + return errwrap.Wrapf("Doesn't exist: {{err}}", err) + } + + return nil +} + +func main() { + err := tryOpen() + + // We can use the Contains helpers to check if an error contains + // another error. It is safe to do this with a nil error, or with + // an error that doesn't even use the errwrap package. + if errwrap.Contains(err, ErrNotExist) { + // Do something + } + if errwrap.ContainsType(err, new(os.PathError)) { + // Do something + } + + // Or we can use the associated `Get` functions to just extract + // a specific error. This would return nil if that specific error doesn't + // exist. + perr := errwrap.GetType(err, new(os.PathError)) +} +``` + +#### Custom Types + +If you're already making custom types that properly wrap errors, then +you can get all the functionality of `errwraps.Contains` and such by +implementing the `Wrapper` interface with just one function. Example: + +```go +type AppError { + Code ErrorCode + Err error +} + +func (e *AppError) WrappedErrors() []error { + return []error{e.Err} +} +``` + +Now this works: + +```go +err := &AppError{Err: fmt.Errorf("an error")} +if errwrap.ContainsType(err, fmt.Errorf("")) { + // This will work! +} +``` diff --git a/vendor/github.com/hashicorp/errwrap/errwrap.go b/vendor/github.com/hashicorp/errwrap/errwrap.go new file mode 100644 index 00000000..a733bef1 --- /dev/null +++ b/vendor/github.com/hashicorp/errwrap/errwrap.go @@ -0,0 +1,169 @@ +// Package errwrap implements methods to formalize error wrapping in Go. +// +// All of the top-level functions that take an `error` are built to be able +// to take any error, not just wrapped errors. This allows you to use errwrap +// without having to type-check and type-cast everywhere. +package errwrap + +import ( + "errors" + "reflect" + "strings" +) + +// WalkFunc is the callback called for Walk. +type WalkFunc func(error) + +// Wrapper is an interface that can be implemented by custom types to +// have all the Contains, Get, etc. functions in errwrap work. +// +// When Walk reaches a Wrapper, it will call the callback for every +// wrapped error in addition to the wrapper itself. Since all the top-level +// functions in errwrap use Walk, this means that all those functions work +// with your custom type. +type Wrapper interface { + WrappedErrors() []error +} + +// Wrap defines that outer wraps inner, returning an error type that +// can be cleanly used with the other methods in this package, such as +// Contains, GetAll, etc. +// +// This function won't modify the error message at all (the outer message +// will be used). +func Wrap(outer, inner error) error { + return &wrappedError{ + Outer: outer, + Inner: inner, + } +} + +// Wrapf wraps an error with a formatting message. This is similar to using +// `fmt.Errorf` to wrap an error. If you're using `fmt.Errorf` to wrap +// errors, you should replace it with this. +// +// format is the format of the error message. The string '{{err}}' will +// be replaced with the original error message. +func Wrapf(format string, err error) error { + outerMsg := "" + if err != nil { + outerMsg = err.Error() + } + + outer := errors.New(strings.Replace( + format, "{{err}}", outerMsg, -1)) + + return Wrap(outer, err) +} + +// Contains checks if the given error contains an error with the +// message msg. If err is not a wrapped error, this will always return +// false unless the error itself happens to match this msg. +func Contains(err error, msg string) bool { + return len(GetAll(err, msg)) > 0 +} + +// ContainsType checks if the given error contains an error with +// the same concrete type as v. If err is not a wrapped error, this will +// check the err itself. +func ContainsType(err error, v interface{}) bool { + return len(GetAllType(err, v)) > 0 +} + +// Get is the same as GetAll but returns the deepest matching error. +func Get(err error, msg string) error { + es := GetAll(err, msg) + if len(es) > 0 { + return es[len(es)-1] + } + + return nil +} + +// GetType is the same as GetAllType but returns the deepest matching error. +func GetType(err error, v interface{}) error { + es := GetAllType(err, v) + if len(es) > 0 { + return es[len(es)-1] + } + + return nil +} + +// GetAll gets all the errors that might be wrapped in err with the +// given message. The order of the errors is such that the outermost +// matching error (the most recent wrap) is index zero, and so on. +func GetAll(err error, msg string) []error { + var result []error + + Walk(err, func(err error) { + if err.Error() == msg { + result = append(result, err) + } + }) + + return result +} + +// GetAllType gets all the errors that are the same type as v. +// +// The order of the return value is the same as described in GetAll. +func GetAllType(err error, v interface{}) []error { + var result []error + + var search string + if v != nil { + search = reflect.TypeOf(v).String() + } + Walk(err, func(err error) { + var needle string + if err != nil { + needle = reflect.TypeOf(err).String() + } + + if needle == search { + result = append(result, err) + } + }) + + return result +} + +// Walk walks all the wrapped errors in err and calls the callback. If +// err isn't a wrapped error, this will be called once for err. If err +// is a wrapped error, the callback will be called for both the wrapper +// that implements error as well as the wrapped error itself. +func Walk(err error, cb WalkFunc) { + if err == nil { + return + } + + switch e := err.(type) { + case *wrappedError: + cb(e.Outer) + Walk(e.Inner, cb) + case Wrapper: + cb(err) + + for _, err := range e.WrappedErrors() { + Walk(err, cb) + } + default: + cb(err) + } +} + +// wrappedError is an implementation of error that has both the +// outer and inner errors. +type wrappedError struct { + Outer error + Inner error +} + +func (w *wrappedError) Error() string { + return w.Outer.Error() +} + +func (w *wrappedError) WrappedErrors() []error { + return []error{w.Outer, w.Inner} +} diff --git a/vendor/github.com/hashicorp/go-multierror/.travis.yml b/vendor/github.com/hashicorp/go-multierror/.travis.yml new file mode 100644 index 00000000..304a8359 --- /dev/null +++ b/vendor/github.com/hashicorp/go-multierror/.travis.yml @@ -0,0 +1,12 @@ +sudo: false + +language: go + +go: + - 1.x + +branches: + only: + - master + +script: make test testrace diff --git a/vendor/github.com/hashicorp/go-multierror/LICENSE b/vendor/github.com/hashicorp/go-multierror/LICENSE new file mode 100644 index 00000000..82b4de97 --- /dev/null +++ b/vendor/github.com/hashicorp/go-multierror/LICENSE @@ -0,0 +1,353 @@ +Mozilla Public License, version 2.0 + +1. Definitions + +1.1. “Contributor” + + means each individual or legal entity that creates, contributes to the + creation of, or owns Covered Software. + +1.2. “Contributor Version” + + means the combination of the Contributions of others (if any) used by a + Contributor and that particular Contributor’s Contribution. + +1.3. “Contribution” + + means Covered Software of a particular Contributor. + +1.4. “Covered Software” + + means Source Code Form to which the initial Contributor has attached the + notice in Exhibit A, the Executable Form of such Source Code Form, and + Modifications of such Source Code Form, in each case including portions + thereof. + +1.5. “Incompatible With Secondary Licenses” + means + + a. that the initial Contributor has attached the notice described in + Exhibit B to the Covered Software; or + + b. that the Covered Software was made available under the terms of version + 1.1 or earlier of the License, but not also under the terms of a + Secondary License. + +1.6. “Executable Form” + + means any form of the work other than Source Code Form. + +1.7. “Larger Work” + + means a work that combines Covered Software with other material, in a separate + file or files, that is not Covered Software. + +1.8. “License” + + means this document. + +1.9. “Licensable” + + means having the right to grant, to the maximum extent possible, whether at the + time of the initial grant or subsequently, any and all of the rights conveyed by + this License. + +1.10. “Modifications” + + means any of the following: + + a. any file in Source Code Form that results from an addition to, deletion + from, or modification of the contents of Covered Software; or + + b. any new file in Source Code Form that contains any Covered Software. + +1.11. “Patent Claims” of a Contributor + + means any patent claim(s), including without limitation, method, process, + and apparatus claims, in any patent Licensable by such Contributor that + would be infringed, but for the grant of the License, by the making, + using, selling, offering for sale, having made, import, or transfer of + either its Contributions or its Contributor Version. + +1.12. “Secondary License” + + means either the GNU General Public License, Version 2.0, the GNU Lesser + General Public License, Version 2.1, the GNU Affero General Public + License, Version 3.0, or any later versions of those licenses. + +1.13. “Source Code Form” + + means the form of the work preferred for making modifications. + +1.14. “You” (or “Your”) + + means an individual or a legal entity exercising rights under this + License. For legal entities, “You” includes any entity that controls, is + controlled by, or is under common control with You. For purposes of this + definition, “control” means (a) the power, direct or indirect, to cause + the direction or management of such entity, whether by contract or + otherwise, or (b) ownership of more than fifty percent (50%) of the + outstanding shares or beneficial ownership of such entity. + + +2. License Grants and Conditions + +2.1. Grants + + Each Contributor hereby grants You a world-wide, royalty-free, + non-exclusive license: + + a. under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or as + part of a Larger Work; and + + b. under Patent Claims of such Contributor to make, use, sell, offer for + sale, have made, import, and otherwise transfer either its Contributions + or its Contributor Version. + +2.2. Effective Date + + The licenses granted in Section 2.1 with respect to any Contribution become + effective for each Contribution on the date the Contributor first distributes + such Contribution. + +2.3. Limitations on Grant Scope + + The licenses granted in this Section 2 are the only rights granted under this + License. No additional rights or licenses will be implied from the distribution + or licensing of Covered Software under this License. Notwithstanding Section + 2.1(b) above, no patent license is granted by a Contributor: + + a. for any code that a Contributor has removed from Covered Software; or + + b. for infringements caused by: (i) Your and any other third party’s + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + + c. under Patent Claims infringed by Covered Software in the absence of its + Contributions. + + This License does not grant any rights in the trademarks, service marks, or + logos of any Contributor (except as may be necessary to comply with the + notice requirements in Section 3.4). + +2.4. Subsequent Licenses + + No Contributor makes additional grants as a result of Your choice to + distribute the Covered Software under a subsequent version of this License + (see Section 10.2) or under the terms of a Secondary License (if permitted + under the terms of Section 3.3). + +2.5. Representation + + Each Contributor represents that the Contributor believes its Contributions + are its original creation(s) or it has sufficient rights to grant the + rights to its Contributions conveyed by this License. + +2.6. Fair Use + + This License is not intended to limit any rights You have under applicable + copyright doctrines of fair use, fair dealing, or other equivalents. + +2.7. Conditions + + Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in + Section 2.1. + + +3. Responsibilities + +3.1. Distribution of Source Form + + All distribution of Covered Software in Source Code Form, including any + Modifications that You create or to which You contribute, must be under the + terms of this License. You must inform recipients that the Source Code Form + of the Covered Software is governed by the terms of this License, and how + they can obtain a copy of this License. You may not attempt to alter or + restrict the recipients’ rights in the Source Code Form. + +3.2. Distribution of Executable Form + + If You distribute Covered Software in Executable Form then: + + a. such Covered Software must also be made available in Source Code Form, + as described in Section 3.1, and You must inform recipients of the + Executable Form how they can obtain a copy of such Source Code Form by + reasonable means in a timely manner, at a charge no more than the cost + of distribution to the recipient; and + + b. You may distribute such Executable Form under the terms of this License, + or sublicense it under different terms, provided that the license for + the Executable Form does not attempt to limit or alter the recipients’ + rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + + You may create and distribute a Larger Work under terms of Your choice, + provided that You also comply with the requirements of this License for the + Covered Software. If the Larger Work is a combination of Covered Software + with a work governed by one or more Secondary Licenses, and the Covered + Software is not Incompatible With Secondary Licenses, this License permits + You to additionally distribute such Covered Software under the terms of + such Secondary License(s), so that the recipient of the Larger Work may, at + their option, further distribute the Covered Software under the terms of + either this License or such Secondary License(s). + +3.4. Notices + + You may not remove or alter the substance of any license notices (including + copyright notices, patent notices, disclaimers of warranty, or limitations + of liability) contained within the Source Code Form of the Covered + Software, except that You may alter any license notices to the extent + required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + + You may choose to offer, and to charge a fee for, warranty, support, + indemnity or liability obligations to one or more recipients of Covered + Software. However, You may do so only on Your own behalf, and not on behalf + of any Contributor. You must make it absolutely clear that any such + warranty, support, indemnity, or liability obligation is offered by You + alone, and You hereby agree to indemnify every Contributor for any + liability incurred by such Contributor as a result of warranty, support, + indemnity or liability terms You offer. You may include additional + disclaimers of warranty and limitations of liability specific to any + jurisdiction. + +4. Inability to Comply Due to Statute or Regulation + + If it is impossible for You to comply with any of the terms of this License + with respect to some or all of the Covered Software due to statute, judicial + order, or regulation then You must: (a) comply with the terms of this License + to the maximum extent possible; and (b) describe the limitations and the code + they affect. Such description must be placed in a text file included with all + distributions of the Covered Software under this License. Except to the + extent prohibited by statute or regulation, such description must be + sufficiently detailed for a recipient of ordinary skill to be able to + understand it. + +5. Termination + +5.1. The rights granted under this License will terminate automatically if You + fail to comply with any of its terms. However, if You become compliant, + then the rights granted under this License from a particular Contributor + are reinstated (a) provisionally, unless and until such Contributor + explicitly and finally terminates Your grants, and (b) on an ongoing basis, + if such Contributor fails to notify You of the non-compliance by some + reasonable means prior to 60 days after You have come back into compliance. + Moreover, Your grants from a particular Contributor are reinstated on an + ongoing basis if such Contributor notifies You of the non-compliance by + some reasonable means, this is the first time You have received notice of + non-compliance with this License from such Contributor, and You become + compliant prior to 30 days after Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent + infringement claim (excluding declaratory judgment actions, counter-claims, + and cross-claims) alleging that a Contributor Version directly or + indirectly infringes any patent, then the rights granted to You by any and + all Contributors for the Covered Software under Section 2.1 of this License + shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user + license agreements (excluding distributors and resellers) which have been + validly granted by You or Your distributors under this License prior to + termination shall survive termination. + +6. Disclaimer of Warranty + + Covered Software is provided under this License on an “as is” basis, without + warranty of any kind, either expressed, implied, or statutory, including, + without limitation, warranties that the Covered Software is free of defects, + merchantable, fit for a particular purpose or non-infringing. The entire + risk as to the quality and performance of the Covered Software is with You. + Should any Covered Software prove defective in any respect, You (not any + Contributor) assume the cost of any necessary servicing, repair, or + correction. This disclaimer of warranty constitutes an essential part of this + License. No use of any Covered Software is authorized under this License + except under this disclaimer. + +7. Limitation of Liability + + Under no circumstances and under no legal theory, whether tort (including + negligence), contract, or otherwise, shall any Contributor, or anyone who + distributes Covered Software as permitted above, be liable to You for any + direct, indirect, special, incidental, or consequential damages of any + character including, without limitation, damages for lost profits, loss of + goodwill, work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses, even if such party shall have been + informed of the possibility of such damages. This limitation of liability + shall not apply to liability for death or personal injury resulting from such + party’s negligence to the extent applicable law prohibits such limitation. + Some jurisdictions do not allow the exclusion or limitation of incidental or + consequential damages, so this exclusion and limitation may not apply to You. + +8. Litigation + + Any litigation relating to this License may be brought only in the courts of + a jurisdiction where the defendant maintains its principal place of business + and such litigation shall be governed by laws of that jurisdiction, without + reference to its conflict-of-law provisions. Nothing in this Section shall + prevent a party’s ability to bring cross-claims or counter-claims. + +9. Miscellaneous + + This License represents the complete agreement concerning the subject matter + hereof. If any provision of this License is held to be unenforceable, such + provision shall be reformed only to the extent necessary to make it + enforceable. Any law or regulation which provides that the language of a + contract shall be construed against the drafter shall not be used to construe + this License against a Contributor. + + +10. Versions of the License + +10.1. New Versions + + Mozilla Foundation is the license steward. Except as provided in Section + 10.3, no one other than the license steward has the right to modify or + publish new versions of this License. Each version will be given a + distinguishing version number. + +10.2. Effect of New Versions + + You may distribute the Covered Software under the terms of the version of + the License under which You originally received the Covered Software, or + under the terms of any subsequent version published by the license + steward. + +10.3. Modified Versions + + If you create software not governed by this License, and you want to + create a new license for such software, you may create and use a modified + version of this License if you rename the license and remove any + references to the name of the license steward (except to note that such + modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses + If You choose to distribute Source Code Form that is Incompatible With + Secondary Licenses under the terms of this version of the License, the + notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice + + This Source Code Form is subject to the + terms of the Mozilla Public License, v. + 2.0. If a copy of the MPL was not + distributed with this file, You can + obtain one at + http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular file, then +You may include the notice in a location (such as a LICENSE file in a relevant +directory) where a recipient would be likely to look for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - “Incompatible With Secondary Licenses” Notice + + This Source Code Form is “Incompatible + With Secondary Licenses”, as defined by + the Mozilla Public License, v. 2.0. diff --git a/vendor/github.com/hashicorp/go-multierror/Makefile b/vendor/github.com/hashicorp/go-multierror/Makefile new file mode 100644 index 00000000..b97cd6ed --- /dev/null +++ b/vendor/github.com/hashicorp/go-multierror/Makefile @@ -0,0 +1,31 @@ +TEST?=./... + +default: test + +# test runs the test suite and vets the code. +test: generate + @echo "==> Running tests..." + @go list $(TEST) \ + | grep -v "/vendor/" \ + | xargs -n1 go test -timeout=60s -parallel=10 ${TESTARGS} + +# testrace runs the race checker +testrace: generate + @echo "==> Running tests (race)..." + @go list $(TEST) \ + | grep -v "/vendor/" \ + | xargs -n1 go test -timeout=60s -race ${TESTARGS} + +# updatedeps installs all the dependencies needed to run and build. +updatedeps: + @sh -c "'${CURDIR}/scripts/deps.sh' '${NAME}'" + +# generate runs `go generate` to build the dynamically generated source files. +generate: + @echo "==> Generating..." + @find . -type f -name '.DS_Store' -delete + @go list ./... \ + | grep -v "/vendor/" \ + | xargs -n1 go generate + +.PHONY: default test testrace updatedeps generate diff --git a/vendor/github.com/hashicorp/go-multierror/README.md b/vendor/github.com/hashicorp/go-multierror/README.md new file mode 100644 index 00000000..ead5830f --- /dev/null +++ b/vendor/github.com/hashicorp/go-multierror/README.md @@ -0,0 +1,97 @@ +# go-multierror + +[![Build Status](http://img.shields.io/travis/hashicorp/go-multierror.svg?style=flat-square)][travis] +[![Go Documentation](http://img.shields.io/badge/go-documentation-blue.svg?style=flat-square)][godocs] + +[travis]: https://travis-ci.org/hashicorp/go-multierror +[godocs]: https://godoc.org/github.com/hashicorp/go-multierror + +`go-multierror` is a package for Go that provides a mechanism for +representing a list of `error` values as a single `error`. + +This allows a function in Go to return an `error` that might actually +be a list of errors. If the caller knows this, they can unwrap the +list and access the errors. If the caller doesn't know, the error +formats to a nice human-readable format. + +`go-multierror` implements the +[errwrap](https://github.com/hashicorp/errwrap) interface so that it can +be used with that library, as well. + +## Installation and Docs + +Install using `go get github.com/hashicorp/go-multierror`. + +Full documentation is available at +http://godoc.org/github.com/hashicorp/go-multierror + +## Usage + +go-multierror is easy to use and purposely built to be unobtrusive in +existing Go applications/libraries that may not be aware of it. + +**Building a list of errors** + +The `Append` function is used to create a list of errors. This function +behaves a lot like the Go built-in `append` function: it doesn't matter +if the first argument is nil, a `multierror.Error`, or any other `error`, +the function behaves as you would expect. + +```go +var result error + +if err := step1(); err != nil { + result = multierror.Append(result, err) +} +if err := step2(); err != nil { + result = multierror.Append(result, err) +} + +return result +``` + +**Customizing the formatting of the errors** + +By specifying a custom `ErrorFormat`, you can customize the format +of the `Error() string` function: + +```go +var result *multierror.Error + +// ... accumulate errors here, maybe using Append + +if result != nil { + result.ErrorFormat = func([]error) string { + return "errors!" + } +} +``` + +**Accessing the list of errors** + +`multierror.Error` implements `error` so if the caller doesn't know about +multierror, it will work just fine. But if you're aware a multierror might +be returned, you can use type switches to access the list of errors: + +```go +if err := something(); err != nil { + if merr, ok := err.(*multierror.Error); ok { + // Use merr.Errors + } +} +``` + +**Returning a multierror only if there are errors** + +If you build a `multierror.Error`, you can use the `ErrorOrNil` function +to return an `error` implementation only if there are errors to return: + +```go +var result *multierror.Error + +// ... accumulate errors here + +// Return the `error` only if errors were added to the multierror, otherwise +// return nil since there are no errors. +return result.ErrorOrNil() +``` diff --git a/vendor/github.com/hashicorp/go-multierror/append.go b/vendor/github.com/hashicorp/go-multierror/append.go new file mode 100644 index 00000000..775b6e75 --- /dev/null +++ b/vendor/github.com/hashicorp/go-multierror/append.go @@ -0,0 +1,41 @@ +package multierror + +// Append is a helper function that will append more errors +// onto an Error in order to create a larger multi-error. +// +// If err is not a multierror.Error, then it will be turned into +// one. If any of the errs are multierr.Error, they will be flattened +// one level into err. +func Append(err error, errs ...error) *Error { + switch err := err.(type) { + case *Error: + // Typed nils can reach here, so initialize if we are nil + if err == nil { + err = new(Error) + } + + // Go through each error and flatten + for _, e := range errs { + switch e := e.(type) { + case *Error: + if e != nil { + err.Errors = append(err.Errors, e.Errors...) + } + default: + if e != nil { + err.Errors = append(err.Errors, e) + } + } + } + + return err + default: + newErrs := make([]error, 0, len(errs)+1) + if err != nil { + newErrs = append(newErrs, err) + } + newErrs = append(newErrs, errs...) + + return Append(&Error{}, newErrs...) + } +} diff --git a/vendor/github.com/hashicorp/go-multierror/flatten.go b/vendor/github.com/hashicorp/go-multierror/flatten.go new file mode 100644 index 00000000..aab8e9ab --- /dev/null +++ b/vendor/github.com/hashicorp/go-multierror/flatten.go @@ -0,0 +1,26 @@ +package multierror + +// Flatten flattens the given error, merging any *Errors together into +// a single *Error. +func Flatten(err error) error { + // If it isn't an *Error, just return the error as-is + if _, ok := err.(*Error); !ok { + return err + } + + // Otherwise, make the result and flatten away! + flatErr := new(Error) + flatten(err, flatErr) + return flatErr +} + +func flatten(err error, flatErr *Error) { + switch err := err.(type) { + case *Error: + for _, e := range err.Errors { + flatten(e, flatErr) + } + default: + flatErr.Errors = append(flatErr.Errors, err) + } +} diff --git a/vendor/github.com/hashicorp/go-multierror/format.go b/vendor/github.com/hashicorp/go-multierror/format.go new file mode 100644 index 00000000..6c7a3cc9 --- /dev/null +++ b/vendor/github.com/hashicorp/go-multierror/format.go @@ -0,0 +1,27 @@ +package multierror + +import ( + "fmt" + "strings" +) + +// ErrorFormatFunc is a function callback that is called by Error to +// turn the list of errors into a string. +type ErrorFormatFunc func([]error) string + +// ListFormatFunc is a basic formatter that outputs the number of errors +// that occurred along with a bullet point list of the errors. +func ListFormatFunc(es []error) string { + if len(es) == 1 { + return fmt.Sprintf("1 error occurred:\n\n* %s", es[0]) + } + + points := make([]string, len(es)) + for i, err := range es { + points[i] = fmt.Sprintf("* %s", err) + } + + return fmt.Sprintf( + "%d errors occurred:\n\n%s", + len(es), strings.Join(points, "\n")) +} diff --git a/vendor/github.com/hashicorp/go-multierror/multierror.go b/vendor/github.com/hashicorp/go-multierror/multierror.go new file mode 100644 index 00000000..89b1422d --- /dev/null +++ b/vendor/github.com/hashicorp/go-multierror/multierror.go @@ -0,0 +1,51 @@ +package multierror + +import ( + "fmt" +) + +// Error is an error type to track multiple errors. This is used to +// accumulate errors in cases and return them as a single "error". +type Error struct { + Errors []error + ErrorFormat ErrorFormatFunc +} + +func (e *Error) Error() string { + fn := e.ErrorFormat + if fn == nil { + fn = ListFormatFunc + } + + return fn(e.Errors) +} + +// ErrorOrNil returns an error interface if this Error represents +// a list of errors, or returns nil if the list of errors is empty. This +// function is useful at the end of accumulation to make sure that the value +// returned represents the existence of errors. +func (e *Error) ErrorOrNil() error { + if e == nil { + return nil + } + if len(e.Errors) == 0 { + return nil + } + + return e +} + +func (e *Error) GoString() string { + return fmt.Sprintf("*%#v", *e) +} + +// WrappedErrors returns the list of errors that this Error is wrapping. +// It is an implementation of the errwrap.Wrapper interface so that +// multierror.Error can be used with that library. +// +// This method is not safe to be called concurrently and is no different +// than accessing the Errors field directly. It is implemented only to +// satisfy the errwrap.Wrapper interface. +func (e *Error) WrappedErrors() []error { + return e.Errors +} diff --git a/vendor/github.com/hashicorp/go-multierror/prefix.go b/vendor/github.com/hashicorp/go-multierror/prefix.go new file mode 100644 index 00000000..5c477abe --- /dev/null +++ b/vendor/github.com/hashicorp/go-multierror/prefix.go @@ -0,0 +1,37 @@ +package multierror + +import ( + "fmt" + + "github.com/hashicorp/errwrap" +) + +// Prefix is a helper function that will prefix some text +// to the given error. If the error is a multierror.Error, then +// it will be prefixed to each wrapped error. +// +// This is useful to use when appending multiple multierrors +// together in order to give better scoping. +func Prefix(err error, prefix string) error { + if err == nil { + return nil + } + + format := fmt.Sprintf("%s {{err}}", prefix) + switch err := err.(type) { + case *Error: + // Typed nils can reach here, so initialize if we are nil + if err == nil { + err = new(Error) + } + + // Wrap each of the errors + for i, e := range err.Errors { + err.Errors[i] = errwrap.Wrapf(format, e) + } + + return err + default: + return errwrap.Wrapf(format, err) + } +}