primal/athens
1
0
Fork 0
mirror of https://github.com/gomods/athens synced 2026-02-03 12:10:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add Content-Length to .zip requests (#1681)

Browse Source 
* Add Content-Length to .zip requests

* pr comments

Co-authored-by: Aaron Schlesinger <70865+arschles@users.noreply.github.com>
This commit is contained in:
Marwan Sulaiman
2020-11-24 10:00:47 -05:00
committed by GitHub
parent ebae08082e
commit 6ef4a793c3
18 changed files with 125 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/download/addons/with_pool.go
+2 -3
View File
@@ -2,7 +2,6 @@ package addons
import (
"context"
"io"
"github.com/gomods/athens/pkg/download"
"github.com/gomods/athens/pkg/errors"
@@ -112,9 +111,9 @@ func (p *withpool) GoMod(ctx context.Context, mod, ver string) ([]byte, error) {
return goMod, nil
}
func (p *withpool) Zip(ctx context.Context, mod, ver string) (io.ReadCloser, error) {
func (p *withpool) Zip(ctx context.Context, mod, ver string) (storage.SizeReadCloser, error) {
const op errors.Op = "pool.Zip"
var zip io.ReadCloser
var zip storage.SizeReadCloser
var err error
done := make(chan struct{}, 1)
p.jobCh <- func() {
pkg/download/addons/with_pool_test.go
+2 -3
View File
@@ -4,7 +4,6 @@ import (
"bytes"
"context"
"fmt"
"io"
"reflect"
"sync"
"testing"
@@ -98,7 +97,7 @@ type mockDP struct {
info []byte
latest *storage.RevInfo
gomod []byte
zip io.ReadCloser
zip storage.SizeReadCloser
inputMod string
inputVer string
catalog []paths.AllPathParams
@@ -143,7 +142,7 @@ func (m *mockDP) GoMod(ctx context.Context, mod, ver string) ([]byte, error) {
}
// Zip implements GET /{module}/@v/{version}.zip
func (m *mockDP) Zip(ctx context.Context, mod, ver string) (io.ReadCloser, error) {
func (m *mockDP) Zip(ctx context.Context, mod, ver string) (storage.SizeReadCloser, error) {
if m.inputMod != mod {
return nil, fmt.Errorf("expected mod input %v but got %v", m.inputMod, mod)
}
pkg/download/handler.go
+1 -1
View File
@@ -53,7 +53,7 @@ func RegisterHandlers(r *mux.Router, opts *HandlerOpts) {
r.Handle(PathVersionInfo, LogEntryHandler(InfoHandler, opts)).Methods(http.MethodGet)
r.Handle(PathVersionModule, LogEntryHandler(ModuleHandler, opts)).Methods(http.MethodGet)
r.Handle(PathVersionZip, LogEntryHandler(ZipHandler, opts)).Methods(http.MethodGet)
r.Handle(PathVersionZip, LogEntryHandler(ZipHandler, opts)).Methods(http.MethodGet, http.MethodHead)
}
func getRedirectURL(base, downloadPath string) (string, error) {
pkg/download/handler_test.go
+2 -2
View File
@@ -2,7 +2,6 @@ package download
import (
"context"
"io"
"net/http"
"net/http/httptest"
"testing"
@@ -10,6 +9,7 @@ import (
"github.com/gomods/athens/pkg/download/mode"
"github.com/gomods/athens/pkg/errors"
"github.com/gomods/athens/pkg/log"
"github.com/gomods/athens/pkg/storage"
"github.com/gorilla/mux"
)
@@ -58,7 +58,7 @@ func (mp *mockProtocol) GoMod(ctx context.Context, mod, ver string) ([]byte, err
return nil, errors.E(op, "not found", errors.KindRedirect)
}
func (mp *mockProtocol) Zip(ctx context.Context, mod, ver string) (io.ReadCloser, error) {
func (mp *mockProtocol) Zip(ctx context.Context, mod, ver string) (storage.SizeReadCloser, error) {
const op errors.Op = "mockProtocol.Zip"
return nil, errors.E(op, "not found", errors.KindRedirect)
}
pkg/download/protocol.go
+2 -3
View File
@@ -2,7 +2,6 @@ package download
import (
"context"
"io"
"regexp"
"strings"
"sync"
@@ -31,7 +30,7 @@ type Protocol interface {
GoMod(ctx context.Context, mod, ver string) ([]byte, error)
// Zip implements GET /{module}/@v/{version}.zip
Zip(ctx context.Context, mod, ver string) (io.ReadCloser, error)
Zip(ctx context.Context, mod, ver string) (storage.SizeReadCloser, error)
}
// Wrapper helps extend the main protocol's functionality with addons.
@@ -188,7 +187,7 @@ func (p *protocol) GoMod(ctx context.Context, mod, ver string) ([]byte, error) {
return goMod, nil
}
func (p *protocol) Zip(ctx context.Context, mod, ver string) (io.ReadCloser, error) {
func (p *protocol) Zip(ctx context.Context, mod, ver string) (storage.SizeReadCloser, error) {
const op errors.Op = "protocol.Zip"
ctx, span := observ.StartSpan(ctx, op.String())
defer span.End()
pkg/download/version_zip.go
+5
View File
@@ -3,6 +3,7 @@ package download
import (
"io"
"net/http"
"strconv"
"github.com/gomods/athens/pkg/download/mode"
"github.com/gomods/athens/pkg/errors"
@@ -43,6 +44,10 @@ func ZipHandler(dp Protocol, lggr log.Entry, df *mode.DownloadFile) http.Handler
defer zip.Close()
w.Header().Set("Content-Type", "application/zip")
w.Header().Set("Content-Length", strconv.FormatInt(zip.Size(), 10))
if r.Method == http.MethodHead {
return
}
_, err = io.Copy(w, zip)
if err != nil {
lggr.SystemErr(errors.E(op, errors.M(mod), errors.V(ver), err))
pkg/storage/azureblob/azureblob.go
+6 -3
View File
@@ -12,6 +12,7 @@ import (
"github.com/gomods/athens/pkg/config"
"github.com/gomods/athens/pkg/errors"
"github.com/gomods/athens/pkg/observ"
"github.com/gomods/athens/pkg/storage"
)
type client interface {
@@ -87,15 +88,17 @@ func (c *azureBlobStoreClient) BlobExists(ctx context.Context, path string) (boo
}
// ReadBlob returns an io.ReadCloser for the contents of a blob
func (c *azureBlobStoreClient) ReadBlob(ctx context.Context, path string) (io.ReadCloser, error) {
// ReadBlob returns a storage.SizeReadCloser for the contents of a blob
func (c *azureBlobStoreClient) ReadBlob(ctx context.Context, path string) (storage.SizeReadCloser, error) {
const op errors.Op = "azureblob.ReadBlob"
blobURL := c.containerURL.NewBlockBlobURL(path)
downloadResponse, err := blobURL.Download(ctx, 0, 0, azblob.BlobAccessConditions{}, false)
if err != nil {
return nil, errors.E(op, err)
}
return downloadResponse.Body(azblob.RetryReaderOptions{}), nil
rc := downloadResponse.Body(azblob.RetryReaderOptions{})
size := downloadResponse.ContentLength()
return storage.NewSizer(rc, size), nil
}
// ListBlobs will list all blobs which has the given prefix
pkg/storage/azureblob/getter.go
+2 -4
View File
@@ -3,12 +3,12 @@ package azureblob
import (
"context"
"fmt"
"io"
"io/ioutil"
"github.com/gomods/athens/pkg/config"
"github.com/gomods/athens/pkg/errors"
"github.com/gomods/athens/pkg/observ"
"github.com/gomods/athens/pkg/storage"
)
// Info implements the (./pkg/storage).Getter interface
@@ -76,7 +76,7 @@ func (s *Storage) GoMod(ctx context.Context, module string, version string) ([]b
}
// Zip implements the (./pkg/storage).Getter interface
func (s *Storage) Zip(ctx context.Context, module string, version string) (io.ReadCloser, error) {
func (s *Storage) Zip(ctx context.Context, module string, version string) (storage.SizeReadCloser, error) {
const op errors.Op = "azureblob.Zip"
ctx, span := observ.StartSpan(ctx, op.String())
defer span.End()
@@ -87,11 +87,9 @@ func (s *Storage) Zip(ctx context.Context, module string, version string) (io.Re
if !exists {
return nil, errors.E(op, errors.M(module), errors.V(version), errors.KindNotFound)
}
zipReader, err := s.client.ReadBlob(ctx, config.PackageVersionedName(module, version, "zip"))
if err != nil {
return nil, errors.E(op, err, errors.M(module), errors.V(version))
}
return zipReader, nil
}
pkg/storage/compliance/tests.go
+1
View File
@@ -153,6 +153,7 @@ func testGet(t *testing.T, b storage.Backend) {
givenZipBts, err := ioutil.ReadAll(zip)
require.NoError(t, err)
require.Equal(t, zipBts, givenZipBts)
require.Equal(t, int64(len(zipBts)), zip.Size())
}
func testExists(t *testing.T, b storage.Backend) {
pkg/storage/external/client.go
Vendored
+22 -14
View File
@@ -8,6 +8,7 @@ import (
"io/ioutil"
"mime/multipart"
"net/http"
"strconv"
"strings"
"github.com/gomods/athens/pkg/errors"
@@ -31,7 +32,7 @@ func NewClient(url string, c *http.Client) storage.Backend {
func (s *service) List(ctx context.Context, mod string) ([]string, error) {
const op errors.Op = "external.List"
body, err := s.getRequest(ctx, mod, "list", "")
body, _, err := s.getRequest(ctx, mod, "list", "")
if err != nil {
return nil, errors.E(op, err)
}
@@ -48,7 +49,7 @@ func (s *service) List(ctx context.Context, mod string) ([]string, error) {
func (s *service) Info(ctx context.Context, mod, ver string) ([]byte, error) {
const op errors.Op = "external.Info"
body, err := s.getRequest(ctx, mod, ver, "info")
body, _, err := s.getRequest(ctx, mod, ver, "info")
if err != nil {
return nil, errors.E(op, err)
}
@@ -61,7 +62,7 @@ func (s *service) Info(ctx context.Context, mod, ver string) ([]byte, error) {
func (s *service) GoMod(ctx context.Context, mod, ver string) ([]byte, error) {
const op errors.Op = "external.GoMod"
body, err := s.getRequest(ctx, mod, ver, "mod")
body, _, err := s.getRequest(ctx, mod, ver, "mod")
if err != nil {
return nil, errors.E(op, err)
}
@@ -72,13 +73,13 @@ func (s *service) GoMod(ctx context.Context, mod, ver string) ([]byte, error) {
return modFile, nil
}
func (s *service) Zip(ctx context.Context, mod, ver string) (io.ReadCloser, error) {
func (s *service) Zip(ctx context.Context, mod, ver string) (storage.SizeReadCloser, error) {
const op errors.Op = "external.Zip"
body, err := s.getRequest(ctx, mod, ver, "zip")
body, size, err := s.getRequest(ctx, mod, ver, "zip")
if err != nil {
return nil, errors.E(op, err)
}
return body, nil
return storage.NewSizer(body, size), nil
}
func (s *service) Save(ctx context.Context, mod, ver string, modFile []byte, zip io.Reader, info []byte) error {
@@ -114,7 +115,7 @@ func (s *service) Save(ctx context.Context, mod, ver string, modFile []byte, zip
func (s *service) Delete(ctx context.Context, mod, ver string) error {
const op errors.Op = "external.Delete"
body, err := s.doRequest(ctx, "DELETE", mod, ver, "delete")
body, _, err := s.doRequest(ctx, "DELETE", mod, ver, "delete")
if err != nil {
return errors.E(op, err)
}
@@ -151,16 +152,16 @@ func upload(mw *multipart.Writer, mod, info []byte, zip io.Reader) error {
return nil
}
func (s *service) getRequest(ctx context.Context, mod, ver, ext string) (io.ReadCloser, error) {
func (s *service) getRequest(ctx context.Context, mod, ver, ext string) (io.ReadCloser, int64, error) {
return s.doRequest(ctx, "GET", mod, ver, ext)
}
func (s *service) doRequest(ctx context.Context, method, mod, ver, ext string) (io.ReadCloser, error) {
func (s *service) doRequest(ctx context.Context, method, mod, ver, ext string) (io.ReadCloser, int64, error) {
const op errors.Op = "external.doRequest"
var err error
mod, err = module.EscapePath(mod)
if err != nil {
return nil, errors.E(op, err)
return nil, 0, errors.E(op, err)
}
url := s.url + "/" + mod + "/@v/" + ver
if ext != "" {
@@ -168,16 +169,23 @@ func (s *service) doRequest(ctx context.Context, method, mod, ver, ext string) (
}
req, err := http.NewRequestWithContext(ctx, method, url, nil)
if err != nil {
return nil, errors.E(op, err)
return nil, 0, errors.E(op, err)
}
resp, err := s.c.Do(req)
if err != nil {
return nil, errors.E(op, err)
return nil, 0, errors.E(op, err)
}
if resp.StatusCode != 200 {
body, _ := ioutil.ReadAll(resp.Body)
resp.Body.Close()
return nil, errors.E(op, fmt.Errorf("none 200 status code: %v - body: %s", resp.StatusCode, body), resp.StatusCode)
return nil, 0, errors.E(op, fmt.Errorf("none 200 status code: %v - body: %s", resp.StatusCode, body), resp.StatusCode)
}
return resp.Body, nil
var size int64
if cl := resp.Header.Get("Content-Length"); cl != "" {
size, err = strconv.ParseInt(cl, 10, 64)
if err != nil {
return nil, 0, errors.E(op, fmt.Errorf("could not parse content-length(%q): %w", cl, err))
}
}
return resp.Body, size, nil
}
pkg/storage/external/server.go
Vendored
+2
View File
@@ -5,6 +5,7 @@ import (
"io"
"io/ioutil"
"net/http"
"strconv"
"strings"
"github.com/gomods/athens/pkg/download"
@@ -67,6 +68,7 @@ func NewServer(strg storage.Backend) http.Handler {
return
}
defer zip.Close()
w.Header().Set("Content-Length", strconv.FormatInt(zip.Size(), 10))
io.Copy(w, zip)
}).Methods(http.MethodGet)
r.HandleFunc("/{module:.+}/@v/{version}.save", func(w http.ResponseWriter, r *http.Request) {
pkg/storage/fs/getter.go
+19 -4
View File
@@ -2,12 +2,12 @@ package fs
import (
"context"
"io"
"os"
"path/filepath"
"github.com/gomods/athens/pkg/errors"
"github.com/gomods/athens/pkg/observ"
"github.com/gomods/athens/pkg/storage"
"github.com/spf13/afero"
)
@@ -37,7 +37,7 @@ func (v *storageImpl) GoMod(ctx context.Context, module, version string) ([]byte
return mod, nil
}
func (v *storageImpl) Zip(ctx context.Context, module, version string) (io.ReadCloser, error) {
func (v *storageImpl) Zip(ctx context.Context, module, version string) (storage.SizeReadCloser, error) {
const op errors.Op = "fs.Zip"
ctx, span := observ.StartSpan(ctx, op.String())
defer span.End()
@@ -47,6 +47,21 @@ func (v *storageImpl) Zip(ctx context.Context, module, version string) (io.ReadC
if err != nil {
return nil, errors.E(op, errors.M(module), errors.V(version), errors.KindNotFound)
}
return src, nil
fi, err := src.Stat()
if err != nil {
return nil, errors.E(op, err)
}
return storage.NewSizer(src, fi.Size()), nil
}
func (v *storageImpl) ZipSize(ctx context.Context, module, version string) (int64, error) {
const op errors.Op = "fs.ZipFileSize"
ctx, span := observ.StartSpan(ctx, op.String())
defer span.End()
versionedPath := v.versionLocation(module, version)
fi, err := v.filesystem.Stat(filepath.Join(versionedPath))
if err != nil {
return 0, errors.E(op, err, errors.M(module), errors.V(version), errors.KindNotFound)
}
return fi.Size(), nil
}
pkg/storage/gcp/getter.go
+3 -4
View File
@@ -3,13 +3,13 @@ package gcp
import (
"context"
"fmt"
"io"
"io/ioutil"
"cloud.google.com/go/storage"
"github.com/gomods/athens/pkg/config"
"github.com/gomods/athens/pkg/errors"
"github.com/gomods/athens/pkg/observ"
pkgstorage "github.com/gomods/athens/pkg/storage"
)
// Info implements Getter
@@ -48,7 +48,7 @@ func (s *Storage) GoMod(ctx context.Context, module, version string) ([]byte, er
}
// Zip implements Getter
func (s *Storage) Zip(ctx context.Context, module, version string) (io.ReadCloser, error) {
func (s *Storage) Zip(ctx context.Context, module, version string) (pkgstorage.SizeReadCloser, error) {
const op errors.Op = "gcp.Zip"
ctx, span := observ.StartSpan(ctx, op.String())
defer span.End()
@@ -56,8 +56,7 @@ func (s *Storage) Zip(ctx context.Context, module, version string) (io.ReadClose
if err != nil {
return nil, errors.E(op, err, getErrorKind(err), errors.M(module), errors.V(version))
}
return zipReader, nil
return pkgstorage.NewSizer(zipReader, zipReader.Size()), nil
}
func getErrorKind(err error) int {
pkg/storage/getter.go
+24 -1
View File
@@ -9,5 +9,28 @@ import (
type Getter interface {
Info(ctx context.Context, module, vsn string) ([]byte, error)
GoMod(ctx context.Context, module, vsn string) ([]byte, error)
Zip(ctx context.Context, module, vsn string) (io.ReadCloser, error)
Zip(ctx context.Context, module, vsn string) (SizeReadCloser, error)
}
// SizeReadCloser extends io.ReadCloser
// with a Size() method that tells you the
// length of the io.ReadCloser if read in full
type SizeReadCloser interface {
io.ReadCloser
Size() int64
}
// NewSizer is a helper wrapper to return an implementation
// of ReadCloserSizer
func NewSizer(rc io.ReadCloser, size int64) SizeReadCloser {
return &sizeReadCloser{rc, size}
}
type sizeReadCloser struct {
io.ReadCloser
size int64
}
func (zf *sizeReadCloser) Size() int64 {
return zf.size
}
pkg/storage/minio/getter.go
+7 -4
View File
@@ -3,12 +3,12 @@ package minio
import (
"context"
"fmt"
"io"
"io/ioutil"
"net/http"
"github.com/gomods/athens/pkg/errors"
"github.com/gomods/athens/pkg/observ"
"github.com/gomods/athens/pkg/storage"
minio "github.com/minio/minio-go/v6"
)
@@ -45,7 +45,7 @@ func (v *storageImpl) GoMod(ctx context.Context, module, vsn string) ([]byte, er
return mod, nil
}
func (v *storageImpl) Zip(ctx context.Context, module, vsn string) (io.ReadCloser, error) {
func (v *storageImpl) Zip(ctx context.Context, module, vsn string) (storage.SizeReadCloser, error) {
const op errors.Op = "minio.Zip"
ctx, span := observ.StartSpan(ctx, op.String())
defer span.End()
@@ -60,8 +60,11 @@ func (v *storageImpl) Zip(ctx context.Context, module, vsn string) (io.ReadClose
if err != nil {
return nil, errors.E(op, err)
}
return zipReader, nil
oi, err := zipReader.Stat()
if err != nil {
return nil, errors.E(op, err)
}
return storage.NewSizer(zipReader, oi.Size), nil
}
func transformNotFoundErr(op errors.Op, module, version string, err error) error {
pkg/storage/module.go
+2 -1
View File
@@ -6,10 +6,11 @@ import (
// Module represents a vgo module saved in a storage backend.
type Module struct {
// TODO(marwan-at-work): ID is a mongo-specific field, it should not be
// in the generic storage.Module struct.
ID primitive.ObjectID `bson:"_id,omitempty"`
Module string `bson:"module"`
Version string `bson:"version"`
Mod []byte `bson:"mod"`
Zip []byte `bson:"zip"`
Info []byte `bson:"info"`
}
pkg/storage/mongo/getter.go
+15 -4
View File
@@ -2,7 +2,6 @@ package mongo
import (
"context"
"io"
"github.com/gomods/athens/pkg/errors"
"github.com/gomods/athens/pkg/observ"
@@ -11,6 +10,7 @@ import (
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/gridfs"
"go.mongodb.org/mongo-driver/mongo/options"
"go.mongodb.org/mongo-driver/x/bsonx"
)
// Info implements storage.Getter
@@ -44,7 +44,7 @@ func (s *ModuleStore) GoMod(ctx context.Context, module, vsn string) ([]byte, er
}
// Zip implements storage.Getter
func (s *ModuleStore) Zip(ctx context.Context, module, vsn string) (io.ReadCloser, error) {
func (s *ModuleStore) Zip(ctx context.Context, module, vsn string) (storage.SizeReadCloser, error) {
const op errors.Op = "mongo.Zip"
ctx, span := observ.StartSpan(ctx, op.String())
defer span.End()
@@ -64,8 +64,19 @@ func (s *ModuleStore) Zip(ctx context.Context, module, vsn string) (io.ReadClose
}
return nil, errors.E(op, err, kind, errors.M(module), errors.V(vsn))
}
return dStream, nil
res := s.client.Database(s.db).Collection("fs.files").FindOne(ctx, bson.M{
"filename": zipName,
})
if res.Err() != nil {
return nil, errors.E(op, res.Err())
}
var m bsonx.Doc
err = res.Decode(&m)
if err != nil {
return nil, errors.E(op, err)
}
size, _ := m.Lookup("length").Int64OK()
return storage.NewSizer(dStream, size), nil
}
// Query connects to and queries storage module
pkg/storage/s3/getter.go
+8 -5
View File
@@ -3,7 +3,6 @@ package s3
import (
"context"
"fmt"
"io"
"io/ioutil"
"github.com/aws/aws-sdk-go/aws"
@@ -11,6 +10,7 @@ import (
"github.com/gomods/athens/pkg/config"
"github.com/gomods/athens/pkg/errors"
"github.com/gomods/athens/pkg/observ"
"github.com/gomods/athens/pkg/storage"
)
// Info implements the (./pkg/storage).Getter interface
@@ -67,7 +67,7 @@ func (s *Storage) GoMod(ctx context.Context, module, version string) ([]byte, er
}
// Zip implements the (./pkg/storage).Getter interface
func (s *Storage) Zip(ctx context.Context, module, version string) (io.ReadCloser, error) {
func (s *Storage) Zip(ctx context.Context, module, version string) (storage.SizeReadCloser, error) {
const op errors.Op = "s3.Zip"
ctx, span := observ.StartSpan(ctx, op.String())
defer span.End()
@@ -87,7 +87,7 @@ func (s *Storage) Zip(ctx context.Context, module, version string) (io.ReadClose
return zipReader, nil
}
func (s *Storage) open(ctx context.Context, path string) (io.ReadCloser, error) {
func (s *Storage) open(ctx context.Context, path string) (storage.SizeReadCloser, error) {
const op errors.Op = "s3.open"
ctx, span := observ.StartSpan(ctx, op.String())
defer span.End()
@@ -100,6 +100,9 @@ func (s *Storage) open(ctx context.Context, path string) (io.ReadCloser, error)
if err != nil {
return nil, errors.E(op, err)
}
return goo.Body, nil
var size int64
if goo.ContentLength != nil {
size = *goo.ContentLength
}
return storage.NewSizer(goo.Body, size), nil
}