From 939e6955265f0a1290950974e9d672e35b65db79 Mon Sep 17 00:00:00 2001 From: Ted Wexler Date: Tue, 17 Mar 2020 16:04:37 -0400 Subject: [PATCH] Adds redis sentinel support (#1554) * Adds redis sentinel support Fixes #1553 * Fix redis-sentinel test hostnames * Fix redis master name again * Fix redis sentinel port in tests * Upgrade the redis client * Rmoeve accidental config change * Fix default config * Addresses review comments * Add documentation on single flight mechanisms * Fix spelling issues * Fix formatting Co-authored-by: Aaron Schlesinger <70865+arschles@users.noreply.github.com> --- .drone.yml | 13 +++- cmd/proxy/actions/app_proxy.go | 10 +++ config.dev.toml | 17 ++++- docker-compose.yml | 9 +++ docs/content/configuration/storage.md | 92 +++++++++++++++++++++++++++ go.mod | 2 + go.sum | 21 ++++++ pkg/config/config.go | 5 ++ pkg/config/singleflight.go | 13 +++- pkg/stash/with_redis.go | 14 ++-- pkg/stash/with_redis_sentinel.go | 30 +++++++++ pkg/stash/with_redis_sentinel_test.go | 48 ++++++++++++++ 12 files changed, 262 insertions(+), 12 deletions(-) create mode 100644 pkg/stash/with_redis_sentinel.go create mode 100644 pkg/stash/with_redis_sentinel_test.go diff --git a/.drone.yml b/.drone.yml index 0b09eaa4..f977eea1 100644 --- a/.drone.yml +++ b/.drone.yml @@ -46,6 +46,9 @@ steps: ATHENS_MONGO_STORAGE_URL: mongodb://mongo:27017 ATHENS_MINIO_ENDPOINT: minio:9000 REDIS_TEST_ENDPOINT: redis:6379 + REDIS_SENTINEL_TEST_ENDPOINT: redis-sentinel:26379 + REDIS_SENTINEL_TEST_MASTER_NAME: redis-1 + REDIS_SENTINEL_TEST_PASSWORD: sekret PROTECTED_REDIS_TEST_ENDPOINT: protectedredis:6380 ATHENS_PROTECTED_REDIS_PASSWORD: AthensPass1 GCS_SERVICE_ACCOUNT: @@ -156,13 +159,21 @@ services: image: redis ports: - 6379 +- name: redis-sentinel + image: bitnami/redis-sentinel + environment: + REDIS_MASTER_HOST: redis + REDIS_MASTER_SET: redis-1 + REDIS_SENTINEL_PASSWORD: sekret + REDIS_SENTINEL_QUORUM: "1" + ports: + - 26379 - name: protectedredis image: redis ports: - 6380 commands: - "redis-server ./test/redis.conf" - - name: athens-proxy image: gomods/athens:canary pull: always diff --git a/cmd/proxy/actions/app_proxy.go b/cmd/proxy/actions/app_proxy.go index 454695dd..ac036173 100644 --- a/cmd/proxy/actions/app_proxy.go +++ b/cmd/proxy/actions/app_proxy.go @@ -133,6 +133,16 @@ func getSingleFlight(c *config.Config, checker storage.Checker) (stash.Wrapper, return nil, fmt.Errorf("Redis config must be present") } return stash.WithRedisLock(c.SingleFlight.Redis.Endpoint, c.SingleFlight.Redis.Password, checker) + case "redis-sentinel": + if c.SingleFlight == nil || c.SingleFlight.RedisSentinel == nil { + return nil, fmt.Errorf("Redis config must be present") + } + return stash.WithRedisSentinelLock( + c.SingleFlight.RedisSentinel.Endpoints, + c.SingleFlight.RedisSentinel.MasterName, + c.SingleFlight.RedisSentinel.SentinelPassword, + checker, + ) case "gcp": if c.StorageType != "gcp" { return nil, fmt.Errorf("gcp SingleFlight only works with a gcp storage type and not: %v", c.StorageType) diff --git a/config.dev.toml b/config.dev.toml index 8eb7d610..decda6c8 100755 --- a/config.dev.toml +++ b/config.dev.toml @@ -263,7 +263,7 @@ DownloadURL = "" # and the second request will wait for the first one to finish so that # it doesn't override the storage. -# Options are ["memory", "etcd", "redis", "gcp", "azureblob"] +# Options are ["memory", "etcd", "redis", "redis-sentinel", "gcp", "azureblob"] # The default option is "memory" which means that only one instance of Athens # should be used. @@ -275,6 +275,10 @@ DownloadURL = "" # and therefore it will use its strong-consistency features to ensure # that only one module is ever written even when concurrent saves happen # at the same time. +# The "redis" single flight will use a single redis instance as a locking mechanism +# for updating the underlying storage +# The "redis-sentinel" single flight works similarly to "redis" but obtains a redis connection +# via a redis-sentinel # Env override: ATHENS_SINGLE_FLIGHT_TYPE SingleFlightType = "memory" @@ -290,6 +294,17 @@ SingleFlightType = "memory" # TODO(marwan): enable multiple endpoints for redis clusters. # Env override: ATHENS_REDIS_ENDPOINT Endpoint = "127.0.0.1:6379" + [SingleFlight.RedisSentinel] + # Endpoints is the redis sentinel endpoints to discover a redis + # master for a SingleFlight lock. + # Env override: ATHENS_REDIS_SENTINEL_ENDPOINTS + Endpoints = ["127.0.0.1:26379"] + # MasterName is the redis sentinel master name to use to discover + # the master for a SingleFlight lock + MasterName = "redis-1" + # SentinelPassword is an optional password for authenticating with + # redis sentinel + SentinelPassword = "sekret" # Password is the password for a redis SingleFlight lock. # Env override: ATHENS_REDIS_PASSWORD diff --git a/docker-compose.yml b/docker-compose.yml index 4a55d408..15c29aaf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -81,6 +81,15 @@ services: image: redis ports: - 6379:6379 + redis-sentinel: + image: bitnami/redis-sentinel + environment: + - REDIS_MASTER_HOST=redis + - REDIS_MASTER_SET=redis-1 + - REDIS_SENTINEL_PASSWORD=sekret + - REDIS_SENTINEL_QUORUM=1 + ports: + - 26379:26379 protectedredis: image: redis ports: diff --git a/docs/content/configuration/storage.md b/docs/content/configuration/storage.md index c036989d..4525d536 100644 --- a/docs/content/configuration/storage.md +++ b/docs/content/configuration/storage.md @@ -335,3 +335,95 @@ It assumes that you already have the following: # Name of container in the blob storage # Env override: ATHENS_AZURE_CONTAINER_NAME ContainerName = "MY_AZURE_BLOB_CONTAINER_NAME" + +## Running multiple Athens pointed at the same storage + +Athens has the ability to run concurrently pointed at the same storage medium, using +a distributed locking mechanism called "single flight". + +By default, Athens is configured to use the `memory` single flight, which +stores locks in local memory. This works when running a single Athens instance, given +the process has access to it's own memory. However, when running multiple Athens instances +pointed at the same storage, a distributed locking mechansism is required. + +Athens supports several distributed locking mechanisms: + +- `etcd` +- `redis` +- `redis-sentinel` +- `gcp` (available when using the `gcp` storage type) +- `azureblob` (available when using the `azureblob` storage type) + +Setting the `SingleFlightType` (or `ATHENS_SINGLE_FLIGHT TYPE` in the environment) configuration +value will enable usage of one of the above mechanisms. The `azureblob` and `gcp` types require +no extra configuration. + +### Using etcd as the single flight mechanism + +Using the `etcd` mechanism is very simple, just a comma separated list of etcd endpoints. +The recommend configuration is 3 endpoints, however, more can be used. + + SingleFlightType = "etcd" + + [SingleFlight] + [SingleFlight.Etcd] + # Env override: ATHENS_ETCD_ENDPOINTS + Endpoints = "localhost:2379,localhost:22379,localhost:32379" + +### Using redis as the single flight mechanism + +Athens supports two mechanisms of communicating with redis: direct connection, and +connecting via redis sentinels. + +#### Direct connection to redis + +Using a direct connection to redis is simple, and only requires a single `redis-server`. +You can also optionally specify a password to connect to the redis server with + + SingleFlightType = "redis" + + [SingleFlight] + [SingleFlight.Redis] + # Endpoint is the redis endpoint for the single flight mechanism + # Env override: ATHENS_REDIS_ENDPOINT + Endpoint = "127.0.0.1:6379" + + # Password is the password for the redis instance + # Env override: ATHENS_REDIS_PASSWORD + Password = "" + +#### Connecting to redis via redis sentinel + +**NOTE**: redis-sentinel requires a working knowledge of redis and is not recommended for +everyone. + +redis sentinel is a high-availability set up for redis, it provides automated monitoring, replication, +failover and configuration of multiple redis servers in a leader-follower setup. It is more +complex than running a single redis server and requires multiple disperate instances of redis +running distributed across nodes. + +For more details on redis-sentinel, check out the [documentation](https://redis.io/topics/sentinel) + +As redis-sentinel is a more complex set up of redis, it requires more configuration than standard redis. + +Required configuration: + +- `Endpoints` is a list of redis-sentinel endpoints to connect to, typically 3, but more can be used +- `MasterName` is the named master instance, as configured in the `redis-sentinel` [configuration](https://redis.io/topics/sentinel#configuring-sentinel) + +Optionally, like `redis`, you can also specify a password to connect to the `redis-sentinel` endpoints with + + SingleFlightType = "redis-sentinel" + + [SingleFlight] + [SingleFlight.RedisSentinel] + # Endpoints is the redis sentinel endpoints to discover a redis + # master for a SingleFlight lock. + # Env override: ATHENS_REDIS_SENTINEL_ENDPOINTS + Endpoints = ["127.0.0.1:26379"] + # MasterName is the redis sentinel master name to use to discover + # the master for a SingleFlight lock + MasterName = "redis-1" + # SentinelPassword is an optional password for authenticating with + # redis sentinel + SentinelPassword = "sekret" diff --git a/go.mod b/go.mod index 5d00f562..eab26e55 100644 --- a/go.mod +++ b/go.mod @@ -14,11 +14,13 @@ require ( github.com/DataDog/opencensus-go-exporter-datadog v0.0.0-20180917103902-e6c7f767dc57 github.com/aws/aws-sdk-go v1.15.24 github.com/bsm/redis-lock v8.0.0+incompatible + github.com/bsm/redislock v0.4.2 github.com/codegangsta/negroni v1.0.0 // indirect github.com/fatih/color v1.7.0 github.com/go-playground/locales v0.12.1 // indirect github.com/go-playground/universal-translator v0.16.0 // indirect github.com/go-redis/redis v6.15.2+incompatible + github.com/go-redis/redis/v7 v7.2.0 github.com/gobuffalo/envy v1.6.7 github.com/gobuffalo/httptest v1.0.4 github.com/golang/snappy v0.0.1 // indirect diff --git a/go.sum b/go.sum index 5685aacf..6c502944 100644 --- a/go.sum +++ b/go.sum @@ -54,6 +54,8 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB github.com/bsm/go-vlq v0.0.0-20150828105119-ec6e8d4f5f4e/go.mod h1:N+BjUcTjSxc2mtRGSCPsat1kze3CUtvJN3/jTXlp29k= github.com/bsm/redis-lock v8.0.0+incompatible h1:QgB0J2pNG8hUfndTIvpPh38F5XsUTTvO7x8Sls++9Mk= github.com/bsm/redis-lock v8.0.0+incompatible/go.mod h1:8dGkQ5GimBCahwF2R67tqGCJbyDZSp0gzO7wq3pDrik= +github.com/bsm/redislock v0.4.2 h1:+7WydoauDwf5Qw0ajaI/g3t26dQ/ovGU0Dv59sVvQzc= +github.com/bsm/redislock v0.4.2/go.mod h1:zeuSDdDFtEDtbAgKsw7NDucfSVR0zLWBv8tMpro/6UM= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/codegangsta/negroni v1.0.0 h1:+aYywywx4bnKXWvoWtRfJ91vC59NbEhEY03sZjQhbVY= github.com/codegangsta/negroni v1.0.0/go.mod h1:v0y3T5G7Y1UlFfyxFn/QLRU4a2EuNau2iZY63YTKWo0= @@ -90,6 +92,10 @@ github.com/go-playground/universal-translator v0.16.0 h1:X++omBR/4cE2MNg91AoC3rm github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEKwgtJRd2xk99HeFyHw3yid4rvQIY= github.com/go-redis/redis v6.15.2+incompatible h1:9SpNVG76gr6InJGxoZ6IuuxaCOQwDAhzyXg+Bs+0Sb4= github.com/go-redis/redis v6.15.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= +github.com/go-redis/redis v6.15.7+incompatible h1:3skhDh95XQMpnqeqNftPkQD9jL9e5e36z/1SUm6dy1U= +github.com/go-redis/redis/v7 v7.0.0-beta.4/go.mod h1:xhhSbUMTsleRPur+Vgx9sUHtyN33bdjxY+9/0n9Ig8s= +github.com/go-redis/redis/v7 v7.2.0 h1:CrCexy/jYWZjW0AyVoHlcJUeZN19VWlbepTh1Vq6dJs= +github.com/go-redis/redis/v7 v7.2.0/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-test/deep v1.0.1 h1:UQhStjbkDClarlmv0am7OXXO4/GaPdCGiUiMTvi28sg= @@ -111,6 +117,8 @@ github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -209,9 +217,15 @@ github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:v github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo= +github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= @@ -333,6 +347,8 @@ golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190502183928-7f726cade0ab/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092 h1:4QSRKanuywn15aTZvI/mIDEgPQpswuFndXpOj3rKEco= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478 h1:l5EDrHhldLYb3ZRHDUhXF7Om7MvYXnkV9/iQNo1lX6g= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 h1:Wo7BWFiOk0QRFMLYMqJGFMd9CgUAcGx7V+qEg/h5IBI= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -353,6 +369,8 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502175342-a43fa875dd82 h1:vsphBvatvfbhlb4PO1BYSr9dzugGxJ/SQHoNufZJq1w= golang.org/x/sys v0.0.0-20190502175342-a43fa875dd82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY= +golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -388,6 +406,7 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= @@ -403,6 +422,8 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= diff --git a/pkg/config/config.go b/pkg/config/config.go index 2599a0b8..3c82b455 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -164,6 +164,11 @@ func defaultConfig() *Config { SingleFlight: &SingleFlight{ Etcd: &Etcd{"localhost:2379,localhost:22379,localhost:32379"}, Redis: &Redis{"127.0.0.1:6379", ""}, + RedisSentinel: &RedisSentinel{ + Endpoints: []string{"127.0.0.1:26379"}, + MasterName: "redis-1", + SentinelPassword: "sekret", + }, }, } } diff --git a/pkg/config/singleflight.go b/pkg/config/singleflight.go index 52e76a21..f510b789 100644 --- a/pkg/config/singleflight.go +++ b/pkg/config/singleflight.go @@ -4,8 +4,9 @@ package config // backend configurations for a distributed // lock or single flight mechanism. type SingleFlight struct { - Etcd *Etcd - Redis *Redis + Etcd *Etcd + Redis *Redis + RedisSentinel *RedisSentinel } // Etcd holds client side configuration @@ -21,3 +22,11 @@ type Redis struct { Endpoint string `envconfig:"ATHENS_REDIS_ENDPOINT"` Password string `envconfig:"ATHENS_REDIS_PASSWORD"` } + +// RedisSentinel is the configuration for using redis with sentinel +// for SingleFlight +type RedisSentinel struct { + Endpoints []string `envconfig:"ATHENS_REDIS_SENTINEL_ENDPOINTS"` + MasterName string `envconfig:"ATHENS_REDIS_SENTINEL_MASTER_NAME"` + SentinelPassword string `envconfig:"ATHENS_REDIS_SENTINEL_PASSWORD"` +} diff --git a/pkg/stash/with_redis.go b/pkg/stash/with_redis.go index bf89f8c2..3565ed28 100644 --- a/pkg/stash/with_redis.go +++ b/pkg/stash/with_redis.go @@ -4,8 +4,8 @@ import ( "context" "time" - lock "github.com/bsm/redis-lock" - "github.com/go-redis/redis" + lock "github.com/bsm/redislock" + "github.com/go-redis/redis/v7" "github.com/gomods/athens/pkg/config" "github.com/gomods/athens/pkg/errors" "github.com/gomods/athens/pkg/observ" @@ -44,17 +44,15 @@ func (s *redisLock) Stash(ctx context.Context, mod, ver string) (newVer string, mv := config.FmtModVer(mod, ver) // Obtain a new lock with default settings - lock, err := lock.Obtain(s.client, mv, &lock.Options{ - LockTimeout: time.Minute * 5, - RetryCount: 60 * 5, - RetryDelay: time.Second, + lock, err := lock.Obtain(s.client, mv, time.Minute*5, &lock.Options{ + RetryStrategy: lock.LimitRetry(lock.LinearBackoff(time.Second), 60*5), }) if err != nil { return ver, errors.E(op, err) } defer func() { - const op errors.Op = "redis.Unlock" - lockErr := lock.Unlock() + const op errors.Op = "redis.Release" + lockErr := lock.Release() if err == nil && lockErr != nil { err = errors.E(op, lockErr) } diff --git a/pkg/stash/with_redis_sentinel.go b/pkg/stash/with_redis_sentinel.go new file mode 100644 index 00000000..aa92ffee --- /dev/null +++ b/pkg/stash/with_redis_sentinel.go @@ -0,0 +1,30 @@ +package stash + +import ( + "github.com/go-redis/redis/v7" + "github.com/gomods/athens/pkg/errors" + "github.com/gomods/athens/pkg/storage" +) + +// WithRedisSentinelLock returns a distributed singleflight +// with a redis cluster that utilizes sentinel for quorum and failover +func WithRedisSentinelLock(endpoints []string, master, password string, checker storage.Checker) (Wrapper, error) { + const op errors.Op = "stash.WithRedisSentinelLock" + // The redis client constructor does not return an error when no endpoints + // are provided, so we check for ourselves. + if len(endpoints) == 0 { + return nil, errors.E(op, "no endpoints specified") + } + client := redis.NewFailoverClient(&redis.FailoverOptions{ + MasterName: master, + SentinelAddrs: endpoints, + SentinelPassword: password, + }) + _, err := client.Ping().Result() + if err != nil { + return nil, errors.E(op, err) + } + return func(s Stasher) Stasher { + return &redisLock{client, s, checker} + }, nil +} diff --git a/pkg/stash/with_redis_sentinel_test.go b/pkg/stash/with_redis_sentinel_test.go new file mode 100644 index 00000000..5b3c0526 --- /dev/null +++ b/pkg/stash/with_redis_sentinel_test.go @@ -0,0 +1,48 @@ +package stash + +import ( + "context" + "os" + "testing" + "time" + + "github.com/gomods/athens/pkg/storage/mem" + "golang.org/x/sync/errgroup" +) + +// WithRedisLock will ensure that 5 concurrent requests will all get the first request's +// response. We can ensure that because only the first response does not return an error +// and therefore all 5 responses should have no error. +func TestWithRedisSentinelLock(t *testing.T) { + endpoint := os.Getenv("REDIS_SENTINEL_TEST_ENDPOINT") + masterName := os.Getenv("REDIS_SENTINEL_TEST_MASTER_NAME") + password := os.Getenv("REDIS_SENTINEL_TEST_PASSWORD") + if len(endpoint) == 0 || len(masterName) == 0 || len(password) == 0 { + t.SkipNow() + } + strg, err := mem.NewStorage() + if err != nil { + t.Fatal(err) + } + ms := &mockRedisStasher{strg: strg} + wrapper, err := WithRedisSentinelLock([]string{endpoint}, masterName, password, strg) + if err != nil { + t.Fatal(err) + } + s := wrapper(ms) + + var eg errgroup.Group + for i := 0; i < 5; i++ { + eg.Go(func() error { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + _, err := s.Stash(ctx, "mod", "ver") + return err + }) + } + + err = eg.Wait() + if err != nil { + t.Fatal(err) + } +}