Refactor leader-elect related code (#1)

youtube · BlueBlue-Lee · Nov 30, 2017 · Nov 30, 2017 · Dec 1, 2017 · Dec 4, 2017
commit 0d09057511c10e59eb2cd1a57af05c205d072760
diff --git a/go/cmd/doorman/doorman_server.go b/go/cmd/doorman/doorman_server.go
@@ -65,7 +65,7 @@ var (
  keyFile = flag.String("key_file", "", "The TLS key file")
 
  etcdEndpoints = flag.String("etcd_endpoints", "", "comma separated list of etcd endpoints")
- masterDelay = flag.Duration("master_delay", 10*time.Second, "delay in master elections")
+ masterLease = flag.Duration("master_lease", 10*time.Second, "lease in master elections")
  masterElectionLock = flag.String("master_election_lock", "", "etcd path for the master election or empty for no master election")
 )
 
@@ -154,7 +154,7 @@ func main() {
  log.Exit("-etcd_endpoints cannot be empty if -master_election_lock is provided")
  }
 
- masterElection = election.Etcd(etcdEndpointsSlice, *masterElectionLock, *masterDelay)
+ masterElection = election.NewEtcdElection(etcdEndpointsSlice, *masterElectionLock, *masterLease)
  } else {
  masterElection = election.Trivial()
  }

diff --git a/go/server/election/election.go b/go/server/election/election.go
@@ -17,11 +17,6 @@
 package election
 
 import (
- "fmt"
- "time"
-
- "github.com/coreos/etcd/client"
- log "github.com/golang/glog"
  "golang.org/x/net/context"
 )
 
@@ -38,135 +33,3 @@ type Election interface {
  // master is currently unknown.
  Current() chan string
 }
-
-type trivial struct {
- isMaster chan bool
- current chan string
-}
-
-// Trivial returns a trivial election: the participant immediately
-// wins. Use this if you need the election interface, but don't really
-// care about the master election (eg. you'll never have more than one
-// candidate).
-func Trivial() Election {
- return &trivial{
- isMaster: make(chan bool, 1),
- current: make(chan string, 1),
- }
-}
-
-func (e *trivial) String() string {
- return "no election, acting as the master"
-}
-
-func (e *trivial) Current() chan string {
- return e.current
-}
-
-func (e *trivial) IsMaster() chan bool {
- return e.isMaster
-}
-
-func (e *trivial) Run(_ context.Context, id string) error {
- e.isMaster <- true
- e.current <- id
- return nil
-}
-
-type etcd struct {
- endpoints []string
- delay time.Duration
- isMaster chan bool
- lock string
- current chan string
-}
-
-// Etcd returns an etcd based master election (endpoints are used to
-// connect to the etcd cluster). The participants synchronize on lock,
-// and the master has delay time to renew its lease. Higher values of
-// delay may lead to more stable mastership at the cost of potentially
-// longer periods without any master.
-func Etcd(endpoints []string, lock string, delay time.Duration) Election {
- return &etcd{
- endpoints: endpoints,
- isMaster: make(chan bool),
- current: make(chan string),
- delay: delay,
- lock: lock,
- }
-}
-
-func (e *etcd) String() string {
- return fmt.Sprintf("etcd lock: %v (endpoints: %v)", e.lock, e.endpoints)
-}
-
-func (e *etcd) Current() chan string {
- return e.current
-}
-
-func (e *etcd) IsMaster() chan bool {
- return e.isMaster
-}
-
-func (e *etcd) Run(ctx context.Context, id string) error {
- c, err := client.New(client.Config{Endpoints: e.endpoints})
- if err != nil {
- return err
- }
- log.V(2).Infof("connected to etcd at %v", e.endpoints)
- kapi := client.NewKeysAPI(c)
-
- go func() {
- w := kapi.Watcher(e.lock, nil)
- log.V(2).Infof("watching %v for master updates", e.lock)
- var last string
- for {
- r, err := w.Next(ctx)
- if err != nil {
- if !client.IsKeyNotFound(err) {
- log.Errorf("Failed receiving new master: %v", err)
- }
- e.current <- ""
- time.Sleep(e.delay)
- continue
- }
- log.V(2).Infof("received master update: %v", r)
- if last != r.Node.Value {
- last = r.Node.Value
- e.current <- r.Node.Value
- }
- }
- }()
-
- go func() {
- for {
- log.V(2).Infof("trying to become master at %v", e.lock)
- if _, err := kapi.Set(ctx, e.lock, id, &client.SetOptions{
- TTL: e.delay,
- PrevExist: client.PrevNoExist,
- }); err != nil {
- log.V(2).Infof("failed becoming the master, retrying in %v: %v", e.delay, err)
- time.Sleep(e.delay)
- continue
- }
- e.isMaster <- true
- log.V(2).Info("Became master at %v as %v.", e.lock, id)
- for {
- time.Sleep(e.delay / 3)
- log.V(2).Infof("Renewing mastership lease at %v as %v", e.lock, id)
- _, err := kapi.Set(ctx, e.lock, id, &client.SetOptions{
- TTL: e.delay,
- PrevExist: client.PrevExist,
- PrevValue: id,
- })
-
- if err != nil {
- log.V(2).Info("lost mastership")
- e.isMaster <- false
- break
- }
- }
- }
- }()
- return nil
-}
diff --git a/go/server/election/etcd.go b/go/server/election/etcd.go
@@ -0,0 +1,147 @@
+// Copyright 2016 Google, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package election
+
+import (
+ "fmt"
+ "github.com/coreos/etcd/client"
+ log "github.com/golang/glog"
+ "golang.org/x/net/context"
+ "time"
+)
+
+type etcdElection struct {
+ endpoints []string
+ client client.KeysAPI
+
+ lease time.Duration
+ lock string
+ isMaster chan bool
+ current chan string
+}
+
+var _ Election = &etcdElection{}
+
+// NewEtcdElection returns an etcd based master election (endpoints are used to
+// connect to the etcd cluster). The participants synchronize on lock,
+// and the master has delay time to renew its lease. Higher values of
+// delay may lead to more stable mastership at the cost of potentially
+// longer periods without any master.
+func NewEtcdElection(endpoints []string, lock string, lease time.Duration) Election {
+ return &etcdElection{
+ endpoints: endpoints,
+ isMaster: make(chan bool),
+ current: make(chan string),
+ lease: lease,
+ lock: lock,
+ }
+}
+
+func (e *etcdElection) String() string {
+ return fmt.Sprintf("etcd lock: %v (endpoints: %v)", e.lock, e.endpoints)
+}
+
+func (e *etcdElection) Current() chan string {
+ return e.current
+}
+
+func (e *etcdElection) IsMaster() chan bool {
+ return e.isMaster
+}
+
+func (e *etcdElection) Run(ctx context.Context, id string) error {
+ err := e.init()
+ if err != nil {
+ return err
+ }
+
+ go e.refreshMastership(ctx)
+ go e.campaignAndRenew(ctx, id)
+ return nil
+}
+
+//
+func (e *etcdElection) init() error {
+ c, err := client.New(client.Config{Endpoints: e.endpoints})
+ if err != nil {
+ return err
+ }
+ log.V(2).Infof("connected to etcd at %v", e.endpoints)
+ e.client = client.NewKeysAPI(c)
+ return nil
+}
+
+//
+func (e *etcdElection) refreshMastership(ctx context.Context) {
+ w := e.client.Watcher(e.lock, nil)
+ log.V(2).Infof("watching %v for master updates", e.lock)
+ var last string
+ for {
+ r, err := w.Next(ctx)
+ if err != nil {
+ if !client.IsKeyNotFound(err) {
+ log.Errorf("Failed receiving new master: %v", err)
+ }
+ e.current <- ""
+ time.Sleep(e.lease)
+ continue
+ }
+ log.V(2).Infof("received master update: %v", r)
+ if last != r.Node.Value {
+ last = r.Node.Value
+ e.current <- r.Node.Value
+ }
+ }
+}
+
+//
+func (e *etcdElection) campaignAndRenew(ctx context.Context, id string) {
+ for {
+ log.V(2).Infof("trying to become master at %v", e.lock)
+ if _, err := e.client.Set(ctx, e.lock, id, &client.SetOptions{
+ TTL: e.lease,
+ PrevExist: client.PrevNoExist,
+ }); err != nil {
+ log.V(2).Infof("failed becoming the master, retrying in %v: %v", e.lease, err)
+ time.Sleep(e.lease)
+ continue
+ }
+ e.isMaster <- true
+ log.V(2).Info("Became master at %v as %v.", e.lock, id)
+ for {
+ time.Sleep(e.lease / 5)
+ log.V(2).Infof("Renewing mastership lease at %v as %v", e.lock, id)
+ _, err := e.client.Set(ctx, e.lock, id, &client.SetOptions{
+ TTL: e.lease,
+ PrevExist: client.PrevExist,
+ PrevValue: id,
+ })
+
+ for retry := 0; retry < 2; retry++ {
+ _, err = e.client.Set(ctx, e.lock, id, &client.SetOptions{
+ TTL: e.lease,
+ PrevExist: client.PrevExist,
+ PrevValue: id,
+ })
+ }
+
+ if err != nil {
+ log.V(2).Info("lost mastership")
+ e.isMaster <- false
+ break
+ }
+ }
+ }
+}
diff --git a/go/server/election/trivial.go b/go/server/election/trivial.go
@@ -0,0 +1,49 @@
+// Copyright 2016 Google, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package election
+
+type trivial struct {
+ isMaster chan bool
+ current chan string
+}
+
+// Trivial returns a trivial election: the participant immediately
+// wins. Use this if you need the election interface, but don't really
+// care about the master election (eg. you'll never have more than one
+// candidate).
+func Trivial() Election {
+ return &trivial{
+ isMaster: make(chan bool, 1),
+ current: make(chan string, 1),
+ }
+}
+
+func (e *trivial) String() string {
+ return "no election, acting as the master"
+}
+
+func (e *trivial) Current() chan string {
+ return e.current
+}
+
+func (e *trivial) IsMaster() chan bool {
+ return e.isMaster
+}
+
+func (e *trivial) Run(_ context.Context, id string) error {
+ e.isMaster <- true
+ e.current <- id
+ return nil
+}