Add Operation with recover

acme/challengeProvider.go

@@ -10,6 +10,7 @@ import (
 	"github.com/cenk/backoff"
 	"github.com/containous/traefik/cluster"
 	"github.com/containous/traefik/log"
+	"github.com/containous/traefik/safe"
 	"github.com/xenolf/lego/acme"
 )
 
@@ -49,7 +50,7 @@ func (c *challengeProvider) getCertificate(domain string) (cert *tls.Certificate
 	}
 	ebo := backoff.NewExponentialBackOff()
 	ebo.MaxElapsedTime = 60 * time.Second
-	err := backoff.RetryNotify(operation, ebo, notify)
+	err := backoff.RetryNotify(safe.OperationWithRecover(operation), ebo, notify)
 	if err != nil {
 		log.Errorf("Error getting cert: %v", err)
 		return nil, false

cluster/datastore.go

@@ -11,6 +11,7 @@ import (
 	"github.com/containous/staert"
 	"github.com/containous/traefik/job"
 	"github.com/containous/traefik/log"
+	"github.com/containous/traefik/safe"
 	"github.com/docker/libkv/store"
 	"github.com/satori/go.uuid"
 )
@@ -109,7 +110,7 @@ func (d *Datastore) watchChanges() error {
 		notify := func(err error, time time.Duration) {
 			log.Errorf("Error in watch datastore: %+v, retrying in %s", err, time)
 		}
-		err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify)
+		err := backoff.RetryNotify(safe.OperationWithRecover(operation), job.NewBackOff(backoff.NewExponentialBackOff()), notify)
 		if err != nil {
 			log.Errorf("Error in watch datastore: %v", err)
 		}

cluster/leadership.go

@@ -47,7 +47,7 @@ func (l *Leadership) Participate(pool *safe.Pool) {
 		notify := func(err error, time time.Duration) {
 			log.Errorf("Leadership election error %+v, retrying in %s", err, time)
 		}
-		err := backoff.RetryNotify(operation, backOff, notify)
+		err := backoff.RetryNotify(safe.OperationWithRecover(operation), backOff, notify)
 		if err != nil {
 			log.Errorf("Cannot elect leadership %+v", err)
 		}

provider/consul_catalog.go

@@ -334,7 +334,7 @@ func (provider *ConsulCatalog) Provide(configurationChan chan<- types.ConfigMess
 		operation := func() error {
 			return provider.watch(configurationChan, stop)
 		}
-		err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify)
+		err := backoff.RetryNotify(safe.OperationWithRecover(operation), job.NewBackOff(backoff.NewExponentialBackOff()), notify)
 		if err != nil {
 			log.Errorf("Cannot connect to consul server %+v", err)
 		}

provider/docker.go

@@ -229,7 +229,7 @@ func (provider *Docker) Provide(configurationChan chan<- types.ConfigMessage, po
 		notify := func(err error, time time.Duration) {
 			log.Errorf("Docker connection error %+v, retrying in %s", err, time)
 		}
-		err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify)
+		err := backoff.RetryNotify(safe.OperationWithRecover(operation), job.NewBackOff(backoff.NewExponentialBackOff()), notify)
 		if err != nil {
 			log.Errorf("Cannot connect to docker server %+v", err)
 		}

provider/kubernetes.go

@@ -91,7 +91,7 @@ func (provider *Kubernetes) Provide(configurationChan chan<- types.ConfigMessage
 		notify := func(err error, time time.Duration) {
 			log.Errorf("Kubernetes connection error %+v, retrying in %s", err, time)
 		}
-		err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify)
+		err := backoff.RetryNotify(safe.OperationWithRecover(operation), job.NewBackOff(backoff.NewExponentialBackOff()), notify)
 		if err != nil {
 			log.Errorf("Cannot connect to Kubernetes server %+v", err)
 		}

provider/kv.go

@@ -76,7 +76,7 @@ func (provider *Kv) watchKv(configurationChan chan<- types.ConfigMessage, prefix
 	notify := func(err error, time time.Duration) {
 		log.Errorf("KV connection error: %+v, retrying in %s", err, time)
 	}
-	err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify)
+	err := backoff.RetryNotify(safe.OperationWithRecover(operation), job.NewBackOff(backoff.NewExponentialBackOff()), notify)
 	if err != nil {
 		return fmt.Errorf("Cannot connect to KV server: %v", err)
 	}
@@ -107,7 +107,7 @@ func (provider *Kv) provide(configurationChan chan<- types.ConfigMessage, pool *
 	notify := func(err error, time time.Duration) {
 		log.Errorf("KV connection error: %+v, retrying in %s", err, time)
 	}
-	err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify)
+	err := backoff.RetryNotify(safe.OperationWithRecover(operation), job.NewBackOff(backoff.NewExponentialBackOff()), notify)
 	if err != nil {
 		return fmt.Errorf("Cannot connect to KV server: %v", err)
 	}

provider/marathon.go

@@ -120,7 +120,7 @@ func (provider *Marathon) Provide(configurationChan chan<- types.ConfigMessage,
 	notify := func(err error, time time.Duration) {
 		log.Errorf("Marathon connection error %+v, retrying in %s", err, time)
 	}
-	err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify)
+	err := backoff.RetryNotify(safe.OperationWithRecover(operation), job.NewBackOff(backoff.NewExponentialBackOff()), notify)
 	if err != nil {
 		log.Errorf("Cannot connect to Marathon server %+v", err)
 	}

provider/mesos.go

@@ -113,7 +113,7 @@ func (provider *Mesos) Provide(configurationChan chan<- types.ConfigMessage, poo
 	notify := func(err error, time time.Duration) {
 		log.Errorf("mesos connection error %+v, retrying in %s", err, time)
 	}
-	err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify)
+	err := backoff.RetryNotify(safe.OperationWithRecover(operation), job.NewBackOff(backoff.NewExponentialBackOff()), notify)
 	if err != nil {
 		log.Errorf("Cannot connect to mesos server %+v", err)
 	}

safe/routine.go

@@ -1,11 +1,12 @@
 package safe
 
 import (
+	"fmt"
+	"github.com/cenk/backoff"
+	"github.com/containous/traefik/log"
 	"context"
 	"runtime/debug"
 	"sync"
-
-	"github.com/containous/traefik/log"
 )
 
 type routine struct {
@@ -145,3 +146,17 @@ func defaultRecoverGoroutine(err interface{}) {
 	log.Errorf("Error in Go routine: %s", err)
 	debug.PrintStack()
 }
+
+// OperationWithRecover wrap a backoff operation in a Recover
+func OperationWithRecover(operation backoff.Operation) backoff.Operation {
+	return func() (err error) {
+		defer func() {
+			if res := recover(); res != nil {
+				defaultRecoverGoroutine(err)
+				err = fmt.Errorf("Panic in operation: %s", err)
+			}
+		}()
+		err = operation()
+		return nil
+	}
+}

safe/routine_test.go

@@ -0,0 +1,16 @@
+package safe
+
+import (
+	"github.com/cenk/backoff"
+	"testing"
+)
+
+func TestOperationWithRecover(t *testing.T) {
+	operation := func() error {
+		panic("BOOM")
+	}
+	err := backoff.Retry(OperationWithRecover(operation), &backoff.StopBackOff{})
+	if err == nil {
+		t.Fatalf("Error in OperationWithRecover: %s", err)
+	}
+}