From 8d3011fb9c495f5ecaa2c650dfe78f90a54e848f Mon Sep 17 00:00:00 2001 From: James Shubin Date: Mon, 29 Aug 2016 03:57:08 -0400 Subject: [PATCH] etcd: Add a timeout for etcd server to start correctly This also updates etcd to a newer version with a fix that allows this detection and timeout operation to be possible. --- etcd.go | 15 +++++++++++++-- vendor/github.com/coreos/etcd | 2 +- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/etcd.go b/etcd.go index c8fce194..76ec7dc6 100644 --- a/etcd.go +++ b/etcd.go @@ -72,6 +72,7 @@ import ( const ( NS = "_mgmt" // root namespace for mgmt operations seedSentinel = "_seed" // you must not name your hostname this + maxStartServerTimeout = 60 // max number of seconds to wait for server to start maxStartServerRetries = 3 // number of times to retry starting the etcd server maxClientConnectRetries = 5 // number of times to retry consecutive connect failures selfRemoveTimeout = 3 // give unnominated members a chance to self exit @@ -1629,11 +1630,21 @@ func (obj *EmbdEtcd) StartServer(newCluster bool, peerURLsMap etcdtypes.URLsMap) //cfg.ForceNewCluster = newCluster // TODO ? log.Printf("Etcd: StartServer: Starting server...") - obj.server, err = embed.StartEtcd(cfg) // we hang here if things are bad - log.Printf("Etcd: StartServer: Done starting server!") // it didn't hang! + obj.server, err = embed.StartEtcd(cfg) if err != nil { return err } + select { + case <-obj.server.Server.ReadyNotify(): // we hang here if things are bad + log.Printf("Etcd: StartServer: Done starting server!") // it didn't hang! + case <-time.After(time.Duration(maxStartServerTimeout) * time.Second): + e := fmt.Errorf("Etcd: StartServer: Timeout of %d seconds reached!", maxStartServerTimeout) + log.Printf(e.Error()) + obj.server.Server.Stop() // trigger a shutdown + obj.serverwg.Add(1) // add for the DestroyServer() + obj.DestroyServer() + return e + } //log.Fatal(<-obj.server.Err()) XXX log.Printf("Etcd: StartServer: Server running...") obj.memberId = uint64(obj.server.Server.ID()) // store member id for internal use diff --git a/vendor/github.com/coreos/etcd b/vendor/github.com/coreos/etcd index 6079be7d..9e9bbb82 160000 --- a/vendor/github.com/coreos/etcd +++ b/vendor/github.com/coreos/etcd @@ -1 +1 @@ -Subproject commit 6079be7dae57eda252a3fc9342dfcee687f481b3 +Subproject commit 9e9bbb829ef25181e5178138af6349f7971cba59