diff --git a/cmd/proxy/actions/app_proxy.go b/cmd/proxy/actions/app_proxy.go index 5e39fe1b..fc562713 100644 --- a/cmd/proxy/actions/app_proxy.go +++ b/cmd/proxy/actions/app_proxy.go @@ -140,7 +140,7 @@ func getSingleFlight(c *config.Config, checker storage.Checker) (stash.Wrapper, if c.SingleFlight == nil || c.SingleFlight.Redis == nil { return nil, fmt.Errorf("Redis config must be present") } - return stash.WithRedisLock(c.SingleFlight.Redis.Endpoint, c.SingleFlight.Redis.Password, checker) + return stash.WithRedisLock(c.SingleFlight.Redis.Endpoint, c.SingleFlight.Redis.Password, checker, c.SingleFlight.Redis.LockConfig) case "redis-sentinel": if c.SingleFlight == nil || c.SingleFlight.RedisSentinel == nil { return nil, fmt.Errorf("Redis config must be present") @@ -150,6 +150,7 @@ func getSingleFlight(c *config.Config, checker storage.Checker) (stash.Wrapper, c.SingleFlight.RedisSentinel.MasterName, c.SingleFlight.RedisSentinel.SentinelPassword, checker, + c.SingleFlight.RedisSentinel.LockConfig, ) case "gcp": if c.StorageType != "gcp" { diff --git a/config.dev.toml b/config.dev.toml index 093d1a3e..e0089c8d 100755 --- a/config.dev.toml +++ b/config.dev.toml @@ -320,6 +320,18 @@ IndexType = "none" # Password is the password for a redis SingleFlight lock. # Env override: ATHENS_REDIS_PASSWORD Password = "" + + [SingleFlight.Redis.LockConfig] + # TTL for the lock in seconds. Defaults to 900 seconds (15 minutes). + # Env override: ATHENS_REDIS_LOCK_TTL + TTL = 900 + # Timeout for acquiring the lock in seconds. Defaults to 15 seconds. + # Env override: ATHENS_REDIS_LOCK_TIMEOUT + Timeout = 15 + # Max retries while acquiring the lock. Defaults to 10. + # Env override: ATHENS_REDIS_LOCK_MAX_RETRIES + MaxRetries = 10 + [SingleFlight.RedisSentinel] # Endpoints is the redis sentinel endpoints to discover a redis # master for a SingleFlight lock. @@ -327,11 +339,23 @@ IndexType = "none" Endpoints = ["127.0.0.1:26379"] # MasterName is the redis sentinel master name to use to discover # the master for a SingleFlight lock + # Env override: ATHENS_REDIS_SENTINEL_MASTER_NAME MasterName = "redis-1" # SentinelPassword is an optional password for authenticating with # redis sentinel + # Env override: ATHENS_REDIS_SENTINEL_PASSWORD SentinelPassword = "sekret" + [SingleFlight.RedisSentinel.LockConfig] + # TTL for the lock in seconds. Defaults to 900 seconds (15 minutes). + # Env override: ATHENS_REDIS_LOCK_TTL + TTL = 900 + # Timeout for acquiring the lock in seconds. Defaults to 15 seconds. + # Env override: ATHENS_REDIS_LOCK_TIMEOUT + Timeout = 15 + # Max retries while acquiring the lock. Defaults to 10. + # Env override: ATHENS_REDIS_LOCK_MAX_RETRIES + MaxRetries = 10 [Storage] # Only storage backends that are specified in Proxy.StorageType are required here [Storage.CDN] diff --git a/docs/content/configuration/storage.md b/docs/content/configuration/storage.md index 22716600..c9d3c32e 100644 --- a/docs/content/configuration/storage.md +++ b/docs/content/configuration/storage.md @@ -437,6 +437,24 @@ You can also optionally specify a password to connect to the redis server with # Env override: ATHENS_REDIS_PASSWORD Password = "" +##### Customizing lock configurations: +If you would like to customize the distributed lock options then you can optionally override the default lock config to better suit your use-case: + + [SingleFlight.Redis] + ... + [SingleFlight.Redis.LockConfig] + # TTL for the lock in seconds. Defaults to 900 seconds (15 minutes). + # Env override: ATHENS_REDIS_LOCK_TTL + TTL = 900 + # Timeout for acquiring the lock in seconds. Defaults to 15 seconds. + # Env override: ATHENS_REDIS_LOCK_TIMEOUT + Timeout = 15 + # Max retries while acquiring the lock. Defaults to 10. + # Env override: ATHENS_REDIS_LOCK_MAX_RETRIES + MaxRetries = 10 + +Customizations may be required in some cases for eg, you can set a higher TTL if it usually takes longer than 5 mins to fetch the modules in your case. + #### Connecting to redis via redis sentinel **NOTE**: redis-sentinel requires a working knowledge of redis and is not recommended for @@ -472,3 +490,5 @@ Optionally, like `redis`, you can also specify a password to connect to the `red # SentinelPassword is an optional password for authenticating with # redis sentinel SentinelPassword = "sekret" + +Distributed lock options can be customised for redis sentinal as well, in a similar manner as described above for redis. \ No newline at end of file diff --git a/go.mod b/go.mod index 82815761..5a636d4a 100644 --- a/go.mod +++ b/go.mod @@ -11,13 +11,13 @@ require ( github.com/BurntSushi/toml v1.0.0 github.com/DataDog/opencensus-go-exporter-datadog v0.0.0-20180917103902-e6c7f767dc57 github.com/aws/aws-sdk-go v1.32.7 - github.com/bsm/redislock v0.4.2 + github.com/bsm/redislock v0.7.2 github.com/fatih/color v1.13.0 - github.com/go-redis/redis/v7 v7.4.1 + github.com/go-redis/redis/v8 v8.11.4 github.com/go-sql-driver/mysql v1.6.0 github.com/gobuffalo/envy v1.7.0 github.com/gobuffalo/httptest v1.0.4 - github.com/google/go-cmp v0.5.7 + github.com/google/go-cmp v0.5.8 github.com/google/uuid v1.1.2 github.com/gorilla/mux v1.6.2 github.com/hashicorp/go-multierror v1.0.0 @@ -53,11 +53,12 @@ require ( github.com/ajg/form v0.0.0-20160822230020-523a5da1a92f // indirect github.com/apparentlymart/go-textseg v1.0.0 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.1.1 // indirect + github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/codegangsta/negroni v1.0.0 // indirect github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd/v22 v22.3.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/go-playground/locales v0.12.1 // indirect github.com/go-playground/universal-translator v0.16.0 // indirect github.com/go-stack/stack v1.8.0 // indirect @@ -106,8 +107,8 @@ require ( go.uber.org/multierr v1.7.0 // indirect go.uber.org/zap v1.21.0 // indirect golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa // indirect - golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect - golang.org/x/sys v0.0.0-20220209214540-3681064d5158 // indirect + golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 // indirect + golang.org/x/sys v0.0.0-20220422013727-9388b58f7150 // indirect golang.org/x/text v0.3.7 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/appengine v1.6.7 // indirect @@ -117,5 +118,5 @@ require ( gopkg.in/DataDog/dd-trace-go.v1 v1.10.0 // indirect gopkg.in/go-playground/assert.v1 v1.2.1 // indirect gopkg.in/ini.v1 v1.42.0 // indirect - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index d8ee6d53..2a08af22 100644 --- a/go.sum +++ b/go.sum @@ -116,12 +116,13 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bsm/go-vlq v0.0.0-20150828105119-ec6e8d4f5f4e/go.mod h1:N+BjUcTjSxc2mtRGSCPsat1kze3CUtvJN3/jTXlp29k= -github.com/bsm/redislock v0.4.2 h1:+7WydoauDwf5Qw0ajaI/g3t26dQ/ovGU0Dv59sVvQzc= -github.com/bsm/redislock v0.4.2/go.mod h1:zeuSDdDFtEDtbAgKsw7NDucfSVR0zLWBv8tMpro/6UM= +github.com/bsm/redislock v0.7.2 h1:jggqOio8JyX9FJBKIfjF3fTxAu/v7zC5mAID9LveqG4= +github.com/bsm/redislock v0.7.2/go.mod h1:kS2g0Yvlymc9Dz8V3iVYAtLAaSVruYbAFdYBDrmC5WU= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -145,6 +146,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -158,6 +161,8 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -172,14 +177,14 @@ github.com/go-playground/locales v0.12.1 h1:2FITxuFt/xuCNP1Acdhv62OzaCiviiE4kotf github.com/go-playground/locales v0.12.1/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3ygWanZIBtBW0W2TM= github.com/go-playground/universal-translator v0.16.0 h1:X++omBR/4cE2MNg91AoC3rmGrCjJ8eAeUP/K/EKx4DM= github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEKwgtJRd2xk99HeFyHw3yid4rvQIY= -github.com/go-redis/redis/v7 v7.0.0-beta.4/go.mod h1:xhhSbUMTsleRPur+Vgx9sUHtyN33bdjxY+9/0n9Ig8s= -github.com/go-redis/redis/v7 v7.4.1 h1:PASvf36gyUpr2zdOUS/9Zqc80GbM+9BDyiJSJDDOrTI= -github.com/go-redis/redis/v7 v7.4.1/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg= +github.com/go-redis/redis/v8 v8.11.4 h1:kHoYkfZP6+pe04aFTnhDH6GDROa5yJdHJVNxV3F46Tg= +github.com/go-redis/redis/v8 v8.11.4/go.mod h1:2Z2wHZXdQpCDXEGzqMockDpNyYvi2l4Pxt6RJr792+w= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-test/deep v1.0.1 h1:UQhStjbkDClarlmv0am7OXXO4/GaPdCGiUiMTvi28sg= github.com/go-test/deep v1.0.1/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= @@ -263,8 +268,9 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= @@ -319,7 +325,6 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl2 v0.0.0-20190503213020-640445e16309 h1:VBvyXC+b6Ix/MXMGIrOHjq+Ew1IRP52EzoQXf1KpwZo= github.com/hashicorp/hcl2 v0.0.0-20190503213020-640445e16309/go.mod h1:4oI94iqF3GB10QScn46WqbG0kgTUpha97SAzzg2+2ec= -github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -399,15 +404,19 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo= -github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= -github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.16.0 h1:6gjqkI8iiRHMvdccRJM8rVKjCWk6ZIm6FTm3ddIe4/c= +github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= @@ -596,7 +605,6 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -607,6 +615,7 @@ golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= @@ -620,9 +629,10 @@ golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 h1:HVyaeDAYux4pnY+D/SiwmLOR36ewZ4iGQIIrtnuCjFA= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -670,8 +680,10 @@ golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -696,6 +708,7 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -722,8 +735,8 @@ golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220209214540-3681064d5158 h1:rm+CHSpPEEW2IsXUib1ThaHIjuBVZjxNgSKmBLFfD4c= -golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220422013727-9388b58f7150 h1:xHms4gcpe1YE7A3yIllJXP16CMAGuqwO2lX1mTyyRRc= +golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -788,6 +801,7 @@ golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82u golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -967,7 +981,6 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= @@ -987,8 +1000,9 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/pkg/config/config.go b/pkg/config/config.go index 1bd9fa08..74dc1a83 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -167,11 +167,12 @@ func defaultConfig() *Config { IndexType: "none", SingleFlight: &SingleFlight{ Etcd: &Etcd{"localhost:2379,localhost:22379,localhost:32379"}, - Redis: &Redis{"127.0.0.1:6379", ""}, + Redis: &Redis{"127.0.0.1:6379", "", DefaultRedisLockConfig()}, RedisSentinel: &RedisSentinel{ Endpoints: []string{"127.0.0.1:26379"}, MasterName: "redis-1", SentinelPassword: "sekret", + LockConfig: DefaultRedisLockConfig(), }, }, Index: &Index{ diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 6e082b49..f45084be 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -23,9 +23,9 @@ func testConfigFile(t *testing.T) (testConfigFile string) { return testConfigFile } -func compareConfigs(parsedConf *Config, expConf *Config, t *testing.T) { +func compareConfigs(parsedConf *Config, expConf *Config, t *testing.T, ignoreTypes ...interface{}) { t.Helper() - opts := cmpopts.IgnoreTypes(Storage{}, SingleFlight{}, Index{}) + opts := cmpopts.IgnoreTypes(append([]interface{}{Index{}}, ignoreTypes...)...) eq := cmp.Equal(parsedConf, expConf, opts) if !eq { diff := cmp.Diff(parsedConf, expConf, opts) @@ -108,7 +108,7 @@ func TestEnvOverrides(t *testing.T) { if err != nil { t.Fatalf("Env override failed: %v", err) } - compareConfigs(conf, expConf, t) + compareConfigs(conf, expConf, t, Storage{}, SingleFlight{}) } func TestEnvOverridesPreservingPort(t *testing.T) { @@ -261,6 +261,27 @@ func TestParseExampleConfig(t *testing.T) { Token: "", Bucket: "MY_S3_BUCKET_NAME", }, + AzureBlob: &AzureBlobConfig{ + AccountName: "MY_AZURE_BLOB_ACCOUNT_NAME", + AccountKey: "MY_AZURE_BLOB_ACCOUNT_KEY", + ContainerName: "MY_AZURE_BLOB_CONTAINER_NAME", + }, + External: &External{URL: ""}, + } + + expSingleFlight := &SingleFlight{ + Redis: &Redis{ + Endpoint: "127.0.0.1:6379", + Password: "", + LockConfig: DefaultRedisLockConfig(), + }, + RedisSentinel: &RedisSentinel{ + Endpoints: []string{"127.0.0.1:26379"}, + MasterName: "redis-1", + SentinelPassword: "sekret", + LockConfig: DefaultRedisLockConfig(), + }, + Etcd: &Etcd{Endpoints: "localhost:2379,localhost:22379,localhost:32379"}, } expConf := &Config{ @@ -287,7 +308,7 @@ func TestParseExampleConfig(t *testing.T) { StatsExporter: "prometheus", SingleFlightType: "memory", GoBinaryEnvVars: []string{"GOPROXY=direct"}, - SingleFlight: &SingleFlight{}, + SingleFlight: expSingleFlight, SumDBs: []string{"https://sum.golang.org"}, NoSumPatterns: []string{}, DownloadMode: "sync", @@ -370,6 +391,33 @@ func getEnvMap(config *Config) map[string]string { envVars["ATHENS_S3_BUCKET_NAME"] = storage.S3.Bucket } } + + singleFlight := config.SingleFlight + if singleFlight != nil { + if singleFlight.Redis != nil { + envVars["ATHENS_SINGLE_FLIGHT_TYPE"] = "redis" + envVars["ATHENS_REDIS_ENDPOINT"] = singleFlight.Redis.Endpoint + envVars["ATHENS_REDIS_PASSWORD"] = singleFlight.Redis.Endpoint + if singleFlight.Redis.LockConfig != nil { + envVars["ATHENS_REDIS_LOCK_TTL"] = strconv.Itoa(singleFlight.Redis.LockConfig.TTL) + envVars["ATHENS_REDIS_LOCK_TIMEOUT"] = strconv.Itoa(singleFlight.Redis.LockConfig.Timeout) + envVars["ATHENS_REDIS_LOCK_MAX_RETRIES"] = strconv.Itoa(singleFlight.Redis.LockConfig.MaxRetries) + } + } else if singleFlight.RedisSentinel != nil { + envVars["ATHENS_SINGLE_FLIGHT_TYPE"] = "redis-sentinel" + envVars["ATHENS_REDIS_SENTINEL_ENDPOINTS"] = strings.Join(singleFlight.RedisSentinel.Endpoints, ",") + envVars["ATHENS_REDIS_SENTINEL_MASTER_NAME"] = singleFlight.RedisSentinel.MasterName + envVars["ATHENS_REDIS_SENTINEL_PASSWORD"] = singleFlight.RedisSentinel.SentinelPassword + if singleFlight.RedisSentinel.LockConfig != nil { + envVars["ATHENS_REDIS_LOCK_TTL"] = strconv.Itoa(singleFlight.RedisSentinel.LockConfig.TTL) + envVars["ATHENS_REDIS_LOCK_TIMEOUT"] = strconv.Itoa(singleFlight.RedisSentinel.LockConfig.Timeout) + envVars["ATHENS_REDIS_LOCK_MAX_RETRIES"] = strconv.Itoa(singleFlight.RedisSentinel.LockConfig.MaxRetries) + } + } else if singleFlight.Etcd != nil { + envVars["ATHENS_SINGLE_FLIGHT_TYPE"] = "etcd" + envVars["ATHENS_ETCD_ENDPOINTS"] = singleFlight.Etcd.Endpoints + } + } return envVars } diff --git a/pkg/config/singleflight.go b/pkg/config/singleflight.go index f510b789..49cc2568 100644 --- a/pkg/config/singleflight.go +++ b/pkg/config/singleflight.go @@ -19,8 +19,9 @@ type Etcd struct { // Redis holds the client side configuration // to connect to redis as a SingleFlight implementation. type Redis struct { - Endpoint string `envconfig:"ATHENS_REDIS_ENDPOINT"` - Password string `envconfig:"ATHENS_REDIS_PASSWORD"` + Endpoint string `envconfig:"ATHENS_REDIS_ENDPOINT"` + Password string `envconfig:"ATHENS_REDIS_PASSWORD"` + LockConfig *RedisLockConfig } // RedisSentinel is the configuration for using redis with sentinel @@ -29,4 +30,19 @@ type RedisSentinel struct { Endpoints []string `envconfig:"ATHENS_REDIS_SENTINEL_ENDPOINTS"` MasterName string `envconfig:"ATHENS_REDIS_SENTINEL_MASTER_NAME"` SentinelPassword string `envconfig:"ATHENS_REDIS_SENTINEL_PASSWORD"` + LockConfig *RedisLockConfig +} + +type RedisLockConfig struct { + Timeout int `envconfig:"ATHENS_REDIS_LOCK_TIMEOUT"` + TTL int `envconfig:"ATHENS_REDIS_LOCK_TTL"` + MaxRetries int `envconfig:"ATHENS_REDIS_LOCK_MAX_RETRIES"` +} + +func DefaultRedisLockConfig() *RedisLockConfig { + return &RedisLockConfig{ + TTL: 900, + Timeout: 15, + MaxRetries: 10, + } } diff --git a/pkg/stash/with_redis.go b/pkg/stash/with_redis.go index 3565ed28..7332c7ed 100644 --- a/pkg/stash/with_redis.go +++ b/pkg/stash/with_redis.go @@ -2,10 +2,11 @@ package stash import ( "context" + goerrors "errors" "time" - lock "github.com/bsm/redislock" - "github.com/go-redis/redis/v7" + "github.com/bsm/redislock" + "github.com/go-redis/redis/v8" "github.com/gomods/athens/pkg/config" "github.com/gomods/athens/pkg/errors" "github.com/gomods/athens/pkg/observ" @@ -13,28 +14,51 @@ import ( ) // WithRedisLock returns a distributed singleflight -// using an redis cluster. If it cannot connect, it will return an error. -func WithRedisLock(endpoint string, password string, checker storage.Checker) (Wrapper, error) { +// using a redis cluster. If it cannot connect, it will return an error. +func WithRedisLock(endpoint string, password string, checker storage.Checker, lockConfig *config.RedisLockConfig) (Wrapper, error) { const op errors.Op = "stash.WithRedisLock" client := redis.NewClient(&redis.Options{ Network: "tcp", Addr: endpoint, Password: password, }) - _, err := client.Ping().Result() + _, err := client.Ping(context.Background()).Result() + if err != nil { + return nil, errors.E(op, err) + } + + lockOptions, err := lockOptionsFromConfig(lockConfig) if err != nil { return nil, errors.E(op, err) } return func(s Stasher) Stasher { - return &redisLock{client, s, checker} + return &redisLock{client, s, checker, lockOptions} }, nil } +func lockOptionsFromConfig(lockConfig *config.RedisLockConfig) (redisLockOptions, error) { + if lockConfig.TTL <= 0 || lockConfig.Timeout <= 0 || lockConfig.MaxRetries <= 0 { + return redisLockOptions{}, goerrors.New("invalid lock options") + } + return redisLockOptions{ + ttl: time.Duration(lockConfig.TTL) * time.Second, + timeout: time.Duration(lockConfig.Timeout) * time.Second, + maxRetries: lockConfig.MaxRetries, + }, nil +} + +type redisLockOptions struct { + ttl time.Duration + timeout time.Duration + maxRetries int +} + type redisLock struct { client *redis.Client stasher Stasher checker storage.Checker + options redisLockOptions } func (s *redisLock) Stash(ctx context.Context, mod, ver string) (newVer string, err error) { @@ -42,17 +66,19 @@ func (s *redisLock) Stash(ctx context.Context, mod, ver string) (newVer string, ctx, span := observ.StartSpan(ctx, op.String()) defer span.End() mv := config.FmtModVer(mod, ver) + lockCtx, cancel := context.WithTimeout(ctx, s.options.timeout) + defer cancel() - // Obtain a new lock with default settings - lock, err := lock.Obtain(s.client, mv, time.Minute*5, &lock.Options{ - RetryStrategy: lock.LimitRetry(lock.LinearBackoff(time.Second), 60*5), + // Obtain a new lock using lock options + lock, err := redislock.Obtain(lockCtx, s.client, mv, s.options.ttl, &redislock.Options{ + RetryStrategy: redislock.LimitRetry(redislock.LinearBackoff(time.Second), s.options.maxRetries), }) if err != nil { return ver, errors.E(op, err) } defer func() { const op errors.Op = "redis.Release" - lockErr := lock.Release() + lockErr := lock.Release(ctx) if err == nil && lockErr != nil { err = errors.E(op, lockErr) } diff --git a/pkg/stash/with_redis_sentinel.go b/pkg/stash/with_redis_sentinel.go index aa92ffee..5922a64d 100644 --- a/pkg/stash/with_redis_sentinel.go +++ b/pkg/stash/with_redis_sentinel.go @@ -1,14 +1,16 @@ package stash import ( - "github.com/go-redis/redis/v7" + "context" + "github.com/go-redis/redis/v8" + "github.com/gomods/athens/pkg/config" "github.com/gomods/athens/pkg/errors" "github.com/gomods/athens/pkg/storage" ) // WithRedisSentinelLock returns a distributed singleflight // with a redis cluster that utilizes sentinel for quorum and failover -func WithRedisSentinelLock(endpoints []string, master, password string, checker storage.Checker) (Wrapper, error) { +func WithRedisSentinelLock(endpoints []string, master, password string, checker storage.Checker, lockConfig *config.RedisLockConfig) (Wrapper, error) { const op errors.Op = "stash.WithRedisSentinelLock" // The redis client constructor does not return an error when no endpoints // are provided, so we check for ourselves. @@ -20,11 +22,17 @@ func WithRedisSentinelLock(endpoints []string, master, password string, checker SentinelAddrs: endpoints, SentinelPassword: password, }) - _, err := client.Ping().Result() + _, err := client.Ping(context.Background()).Result() if err != nil { return nil, errors.E(op, err) } + + lockOptions, err := lockOptionsFromConfig(lockConfig) + if err != nil { + return nil, errors.E(op, err) + } + return func(s Stasher) Stasher { - return &redisLock{client, s, checker} + return &redisLock{client, s, checker, lockOptions} }, nil } diff --git a/pkg/stash/with_redis_sentinel_test.go b/pkg/stash/with_redis_sentinel_test.go index 05761add..9f347f26 100644 --- a/pkg/stash/with_redis_sentinel_test.go +++ b/pkg/stash/with_redis_sentinel_test.go @@ -6,6 +6,7 @@ import ( "testing" "time" + "github.com/gomods/athens/pkg/config" "github.com/gomods/athens/pkg/storage" "github.com/gomods/athens/pkg/storage/mem" "golang.org/x/sync/errgroup" @@ -27,7 +28,7 @@ func TestWithRedisSentinelLock(t *testing.T) { } ms := &mockRedisStasher{strg: strg} - wrapper, err := WithRedisSentinelLock([]string{endpoint}, masterName, password, storage.WithChecker(strg)) + wrapper, err := WithRedisSentinelLock([]string{endpoint}, masterName, password, storage.WithChecker(strg), config.DefaultRedisLockConfig()) if err != nil { t.Fatal(err) } diff --git a/pkg/stash/with_redis_test.go b/pkg/stash/with_redis_test.go index f9d0e2da..9f1dac9b 100644 --- a/pkg/stash/with_redis_test.go +++ b/pkg/stash/with_redis_test.go @@ -9,6 +9,7 @@ import ( "testing" "time" + "github.com/gomods/athens/pkg/config" "github.com/gomods/athens/pkg/storage" "github.com/gomods/athens/pkg/storage/mem" "golang.org/x/sync/errgroup" @@ -28,7 +29,7 @@ func TestWithRedisLock(t *testing.T) { t.Fatal(err) } ms := &mockRedisStasher{strg: strg} - wrapper, err := WithRedisLock(endpoint, password, storage.WithChecker(strg)) + wrapper, err := WithRedisLock(endpoint, password, storage.WithChecker(strg), config.DefaultRedisLockConfig()) if err != nil { t.Fatal(err) } @@ -63,7 +64,7 @@ func TestWithRedisLockWithPassword(t *testing.T) { t.Fatal(err) } ms := &mockRedisStasher{strg: strg} - wrapper, err := WithRedisLock(endpoint, password, storage.WithChecker(strg)) + wrapper, err := WithRedisLock(endpoint, password, storage.WithChecker(strg), config.DefaultRedisLockConfig()) if err != nil { t.Fatal(err) } @@ -97,7 +98,7 @@ func TestWithRedisLockWithWrongPassword(t *testing.T) { if err != nil { t.Fatal(err) } - _, err = WithRedisLock(endpoint, password, storage.WithChecker(strg)) + _, err = WithRedisLock(endpoint, password, storage.WithChecker(strg), config.DefaultRedisLockConfig()) if err == nil { t.Fatal("Expected Connection Error") }