package actions import ( "fmt" "net/http" "net/url" "path" "strings" "github.com/gomods/athens/pkg/config" "github.com/gomods/athens/pkg/download" "github.com/gomods/athens/pkg/download/addons" "github.com/gomods/athens/pkg/download/mode" "github.com/gomods/athens/pkg/log" "github.com/gomods/athens/pkg/module" "github.com/gomods/athens/pkg/stash" "github.com/gomods/athens/pkg/storage" "github.com/gorilla/mux" "github.com/spf13/afero" ) func addProxyRoutes( r *mux.Router, s storage.Backend, l *log.Logger, c *config.Config, ) error { r.HandleFunc("/", proxyHomeHandler) r.HandleFunc("/healthz", healthHandler) r.HandleFunc("/readyz", getReadinessHandler(s)) r.HandleFunc("/version", versionHandler) r.HandleFunc("/catalog", catalogHandler(s)) for _, sumdb := range c.SumDBs { sumdbURL, err := url.Parse(sumdb) if err != nil { return err } if sumdbURL.Scheme != "https" { return fmt.Errorf("sumdb: %v must have an https scheme", sumdb) } supportPath := path.Join("/sumdb", sumdbURL.Host, "/supported") r.HandleFunc(supportPath, func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(200) }) sumHandler := sumdbPoxy(sumdbURL, c.NoSumPatterns) pathPrefix := "/sumdb/" + sumdbURL.Host r.PathPrefix(pathPrefix + "/").Handler( http.StripPrefix(pathPrefix, sumHandler), ) } // Download Protocol // the download.Protocol and the stash.Stasher interfaces are composable // in a middleware fashion. Therefore you can separate concerns // by the functionality: a download.Protocol that just takes care // of "go getting" things, and another Protocol that just takes care // of "pooling" requests etc. // In our case, we'd like to compose both interfaces in a particular // order to ensure logical ordering of execution. // Here's the order of an incoming request to the download.Protocol: // 1. The downloadpool gets hit first, and manages concurrent requests // 2. The downloadpool passes the request to its parent Protocol: stasher // 3. The stasher Protocol checks storage first, and if storage is empty // it makes a Stash request to the stash.Stasher interface. // Once the stasher picks up an order, here's how the requests go in order: // 1. The singleflight picks up the first request and latches duplicate ones. // 2. The singleflight passes the stash to its parent: stashpool. // 3. The stashpool manages limiting concurrent requests and passes them to stash. // 4. The plain stash.New just takes a request from upstream and saves it into storage. fs := afero.NewOsFs() mf, err := module.NewGoGetFetcher(c.GoBinary, fs) if err != nil { return err } lister := download.NewVCSLister(c.GoBinary, fs) withSingleFlight, err := getSingleFlight(c, s) if err != nil { return err } st := stash.New(mf, s, stash.WithPool(c.GoGetWorkers), withSingleFlight) df, err := mode.NewFile(c.DownloadMode, c.DownloadURL) if err != nil { return err } dpOpts := &download.Opts{ Storage: s, Stasher: st, Lister: lister, DownloadFile: df, } dp := download.New(dpOpts, addons.WithPool(c.ProtocolWorkers)) handlerOpts := &download.HandlerOpts{Protocol: dp, Logger: l, DownloadFile: df} download.RegisterHandlers(r, handlerOpts) return nil } func getSingleFlight(c *config.Config, checker storage.Checker) (stash.Wrapper, error) { switch c.SingleFlightType { case "", "memory": return stash.WithSingleflight, nil case "etcd": if c.SingleFlight == nil || c.SingleFlight.Etcd == nil { return nil, fmt.Errorf("Etcd config must be present") } endpoints := strings.Split(c.SingleFlight.Etcd.Endpoints, ",") return stash.WithEtcd(endpoints, checker) case "redis": if c.SingleFlight == nil || c.SingleFlight.Redis == nil { return nil, fmt.Errorf("Redis config must be present") } return stash.WithRedisLock(c.SingleFlight.Redis.Endpoint, checker) case "gcp": if c.StorageType != "gcp" { return nil, fmt.Errorf("gcp SingleFlight only works with a gcp storage type and not: %v", c.StorageType) } return stash.WithGCSLock, nil case "azureblob": if c.StorageType != "azureblob" { return nil, fmt.Errorf("azureblob SingleFlight only works with a azureblob storage type and not: %v", c.StorageType) } return stash.WithAzureBlobLock(c.Storage.AzureBlob, c.TimeoutDuration(), checker) default: return nil, fmt.Errorf("unrecognized single flight type: %v", c.SingleFlightType) } }