Make shutdown timeout configurable (#1806)

Currently, when shutting down the server (via SIGINT or SIGTERM), the
shutdown closes any open connections after only 10 seconds (via a
context.WithTimeout). This does not provie a lot of time for longer
operations, such as listing versions, or downloading a larger module zip
file.

When running in Kubernetes, and scaling instances or changing config,
this causes a lot of dropped connections and gateway errors. 10 seconds
is arguably much too short, and should be configurable.

This commit increases that default to 60 seconds, and adds a config
variable to allow users to specify their desired timeout.

cmd/proxy/main.go

@@ -55,7 +55,7 @@ func main() {
 		<-sigint
 
 		// We received an interrupt signal, shut down.
-		ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
+		ctx, cancel := context.WithTimeout(context.Background(), time.Second*time.Duration(conf.ShutdownTimeout))
 		defer cancel()
 		if err := srv.Shutdown(ctx); err != nil {
 			log.Fatal(err)

config.dev.toml

@@ -305,6 +305,12 @@ SingleFlightType = "memory"
 # Env override: ATHENS_INDEX_TYPE
 IndexType = "none"
 
+# ShutdownTimeout sets the timeout (in seconds) for open connections when shutting down
+# (via SIGINT or SIGTERM). Connections still open after the timeout will be dropped.
+# Defaults to 60
+# Env override: ATHENS_SHUTDOWN_TIMEOUT
+ShutdownTimeout = 60
+
 [SingleFlight]
     [SingleFlight.Etcd]
         # Endpoints are comma separated URLs that determine all distributed etcd servers.

pkg/config/config.go

@@ -56,6 +56,7 @@ type Config struct {
 	SingleFlightType string    `envconfig:"ATHENS_SINGLE_FLIGHT_TYPE"`
 	RobotsFile       string    `envconfig:"ATHENS_ROBOTS_FILE"`
 	IndexType        string    `envconfig:"ATHENS_INDEX_TYPE"`
+	ShutdownTimeout  int       `validate:"min=0" envconfig:"ATHENS_SHUTDOWN_TIMEOUT"`
 	SingleFlight     *SingleFlight
 	Storage          *Storage
 	Index            *Index
@@ -165,6 +166,7 @@ func defaultConfig() *Config {
 		NetworkMode:      "strict",
 		RobotsFile:       "robots.txt",
 		IndexType:        "none",
+		ShutdownTimeout:  60,
 		SingleFlight: &SingleFlight{
 			Etcd:  &Etcd{"localhost:2379,localhost:22379,localhost:32379"},
 			Redis: &Redis{"127.0.0.1:6379", "", DefaultRedisLockConfig()},

pkg/config/config_test.go

@@ -314,6 +314,7 @@ func TestParseExampleConfig(t *testing.T) {
 		DownloadMode:     "sync",
 		RobotsFile:       "robots.txt",
 		IndexType:        "none",
+		ShutdownTimeout:  60,
 		Index:            &Index{},
 	}