diff --git a/acme/challengeProvider.go b/acme/challengeProvider.go index 2d5a8daae..efde44d67 100644 --- a/acme/challengeProvider.go +++ b/acme/challengeProvider.go @@ -9,6 +9,7 @@ import ( "github.com/cenk/backoff" "github.com/containous/traefik/cluster" "github.com/containous/traefik/log" + "github.com/containous/traefik/safe" "github.com/xenolf/lego/acme" "time" ) @@ -49,7 +50,7 @@ func (c *challengeProvider) getCertificate(domain string) (cert *tls.Certificate } ebo := backoff.NewExponentialBackOff() ebo.MaxElapsedTime = 60 * time.Second - err := backoff.RetryNotify(operation, ebo, notify) + err := backoff.RetryNotify(safe.OperationWithRecover(operation), ebo, notify) if err != nil { log.Errorf("Error getting cert: %v", err) return nil, false diff --git a/cluster/datastore.go b/cluster/datastore.go index 5cda3005e..042b1cd44 100644 --- a/cluster/datastore.go +++ b/cluster/datastore.go @@ -7,6 +7,7 @@ import ( "github.com/containous/staert" "github.com/containous/traefik/job" "github.com/containous/traefik/log" + "github.com/containous/traefik/safe" "github.com/docker/libkv/store" "github.com/satori/go.uuid" "golang.org/x/net/context" @@ -108,7 +109,7 @@ func (d *Datastore) watchChanges() error { notify := func(err error, time time.Duration) { log.Errorf("Error in watch datastore: %+v, retrying in %s", err, time) } - err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify) + err := backoff.RetryNotify(safe.OperationWithRecover(operation), job.NewBackOff(backoff.NewExponentialBackOff()), notify) if err != nil { log.Errorf("Error in watch datastore: %v", err) } diff --git a/cluster/leadership.go b/cluster/leadership.go index 05d81d1a0..d216105c3 100644 --- a/cluster/leadership.go +++ b/cluster/leadership.go @@ -46,7 +46,7 @@ func (l *Leadership) Participate(pool *safe.Pool) { notify := func(err error, time time.Duration) { log.Errorf("Leadership election error %+v, retrying in %s", err, time) } - err := backoff.RetryNotify(operation, backOff, notify) + err := backoff.RetryNotify(safe.OperationWithRecover(operation), backOff, notify) if err != nil { log.Errorf("Cannot elect leadership %+v", err) } diff --git a/provider/consul_catalog.go b/provider/consul_catalog.go index b83da3b1d..981ecda74 100644 --- a/provider/consul_catalog.go +++ b/provider/consul_catalog.go @@ -334,7 +334,7 @@ func (provider *ConsulCatalog) Provide(configurationChan chan<- types.ConfigMess operation := func() error { return provider.watch(configurationChan, stop) } - err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify) + err := backoff.RetryNotify(safe.OperationWithRecover(operation), job.NewBackOff(backoff.NewExponentialBackOff()), notify) if err != nil { log.Errorf("Cannot connect to consul server %+v", err) } diff --git a/provider/docker.go b/provider/docker.go index 4f4a7742a..27c117a81 100644 --- a/provider/docker.go +++ b/provider/docker.go @@ -230,7 +230,7 @@ func (provider *Docker) Provide(configurationChan chan<- types.ConfigMessage, po notify := func(err error, time time.Duration) { log.Errorf("Docker connection error %+v, retrying in %s", err, time) } - err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify) + err := backoff.RetryNotify(safe.OperationWithRecover(operation), job.NewBackOff(backoff.NewExponentialBackOff()), notify) if err != nil { log.Errorf("Cannot connect to docker server %+v", err) } diff --git a/provider/kubernetes.go b/provider/kubernetes.go index 546dbe564..792204563 100644 --- a/provider/kubernetes.go +++ b/provider/kubernetes.go @@ -150,7 +150,7 @@ func (provider *Kubernetes) Provide(configurationChan chan<- types.ConfigMessage notify := func(err error, time time.Duration) { log.Errorf("Kubernetes connection error %+v, retrying in %s", err, time) } - err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify) + err := backoff.RetryNotify(safe.OperationWithRecover(operation), job.NewBackOff(backoff.NewExponentialBackOff()), notify) if err != nil { log.Errorf("Cannot connect to Kubernetes server %+v", err) } diff --git a/provider/kv.go b/provider/kv.go index 0e29ec2be..212590bbd 100644 --- a/provider/kv.go +++ b/provider/kv.go @@ -76,7 +76,7 @@ func (provider *Kv) watchKv(configurationChan chan<- types.ConfigMessage, prefix notify := func(err error, time time.Duration) { log.Errorf("KV connection error: %+v, retrying in %s", err, time) } - err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify) + err := backoff.RetryNotify(safe.OperationWithRecover(operation), job.NewBackOff(backoff.NewExponentialBackOff()), notify) if err != nil { return fmt.Errorf("Cannot connect to KV server: %v", err) } @@ -107,7 +107,7 @@ func (provider *Kv) provide(configurationChan chan<- types.ConfigMessage, pool * notify := func(err error, time time.Duration) { log.Errorf("KV connection error: %+v, retrying in %s", err, time) } - err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify) + err := backoff.RetryNotify(safe.OperationWithRecover(operation), job.NewBackOff(backoff.NewExponentialBackOff()), notify) if err != nil { return fmt.Errorf("Cannot connect to KV server: %v", err) } diff --git a/provider/marathon.go b/provider/marathon.go index 9b98f0d72..cc4f5c156 100644 --- a/provider/marathon.go +++ b/provider/marathon.go @@ -113,7 +113,7 @@ func (provider *Marathon) Provide(configurationChan chan<- types.ConfigMessage, notify := func(err error, time time.Duration) { log.Errorf("Marathon connection error %+v, retrying in %s", err, time) } - err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify) + err := backoff.RetryNotify(safe.OperationWithRecover(operation), job.NewBackOff(backoff.NewExponentialBackOff()), notify) if err != nil { log.Errorf("Cannot connect to Marathon server %+v", err) } diff --git a/provider/mesos.go b/provider/mesos.go index 1822a3e60..c5f015b0f 100644 --- a/provider/mesos.go +++ b/provider/mesos.go @@ -113,7 +113,7 @@ func (provider *Mesos) Provide(configurationChan chan<- types.ConfigMessage, poo notify := func(err error, time time.Duration) { log.Errorf("mesos connection error %+v, retrying in %s", err, time) } - err := backoff.RetryNotify(operation, job.NewBackOff(backoff.NewExponentialBackOff()), notify) + err := backoff.RetryNotify(safe.OperationWithRecover(operation), job.NewBackOff(backoff.NewExponentialBackOff()), notify) if err != nil { log.Errorf("Cannot connect to mesos server %+v", err) } diff --git a/safe/routine.go b/safe/routine.go index a7f171968..292d52bcc 100644 --- a/safe/routine.go +++ b/safe/routine.go @@ -1,6 +1,8 @@ package safe import ( + "fmt" + "github.com/cenk/backoff" "github.com/containous/traefik/log" "golang.org/x/net/context" "runtime/debug" @@ -134,3 +136,17 @@ func defaultRecoverGoroutine(err interface{}) { log.Errorf("Error in Go routine: %s", err) debug.PrintStack() } + +// OperationWithRecover wrap a backoff operation in a Recover +func OperationWithRecover(operation backoff.Operation) backoff.Operation { + return func() (err error) { + defer func() { + if res := recover(); res != nil { + defaultRecoverGoroutine(err) + err = fmt.Errorf("Panic in operation: %s", err) + } + }() + err = operation() + return nil + } +} diff --git a/safe/routine_test.go b/safe/routine_test.go new file mode 100644 index 000000000..07fa89193 --- /dev/null +++ b/safe/routine_test.go @@ -0,0 +1,16 @@ +package safe + +import ( + "github.com/cenk/backoff" + "testing" +) + +func TestOperationWithRecover(t *testing.T) { + operation := func() error { + panic("BOOM") + } + err := backoff.Retry(OperationWithRecover(operation), &backoff.StopBackOff{}) + if err == nil { + t.Fatalf("Error in OperationWithRecover: %s", err) + } +}