From 8f9bb41155d3c3b7d9cb6dd9566c3b5f130c9c00 Mon Sep 17 00:00:00 2001 From: Dongsu Park Date: Wed, 7 Dec 2016 15:51:04 +0100 Subject: [PATCH] *: introduce an option UseLeaseTTL for lease manager's TTL * Introduce an option UseLeaseTTL. Default is false. * If UseLeaseTTL turned on, create lease manager with the TTL value. * When gRPC turned on, remove previous workaround of setting leaseTTL to a huge value. * Increase engine reconcile interval from 2 to 5 sec. Originally written by Hector Fernandez Taken from https://github.com/giantswarm/fleet/tree/patch_lease_ttl --- config/config.go | 1 + engine/engine.go | 7 ------- fleet.conf.sample | 2 +- fleetd/fleetd.go | 4 +++- pkg/lease/etcd.go | 32 +++++++++++++++++++++++++------- registry/rpc/registrymux_test.go | 2 +- server/server.go | 2 +- 7 files changed, 32 insertions(+), 18 deletions(-) diff --git a/config/config.go b/config/config.go index 1d7e4fdb9..08d4d2eb4 100644 --- a/config/config.go +++ b/config/config.go @@ -42,6 +42,7 @@ type Config struct { UnitsDirectory string SystemdUser bool AuthorizedKeysFile string + UseLeaseTTL bool } func (c *Config) Capabilities() machine.Capabilities { diff --git a/engine/engine.go b/engine/engine.go index 987bdf682..fd14b4538 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -67,13 +67,6 @@ func New(reg CompleteRegistry, lManager lease.Manager, rStream pkg.EventStream, func (e *Engine) Run(ival time.Duration, stop <-chan struct{}) { leaseTTL := ival * 5 - if e.machine.State().Capabilities.Has(machine.CapGRPC) { - // With grpc it doesn't make sense to set to 5secs the TTL of the etcd key. - // This has a special impact whenever we have high worload in the cluster, cause - // it'd provoke constant leader re-elections. - // TODO: IMHO, this should be configurable via a flag to disable the TTL. - leaseTTL = ival * 500000 - } machID := e.machine.State().ID reconcile := func() { diff --git a/fleet.conf.sample b/fleet.conf.sample index e22c4a214..533fab907 100644 --- a/fleet.conf.sample +++ b/fleet.conf.sample @@ -35,4 +35,4 @@ # agent_ttl="30s" # Interval at which the engine should reconcile the cluster schedule in etcd. -# engine_reconcile_interval=2 +# engine_reconcile_interval=5 diff --git a/fleetd/fleetd.go b/fleetd/fleetd.go index 1cfd06ba8..d8dfcd027 100644 --- a/fleetd/fleetd.go +++ b/fleetd/fleetd.go @@ -84,7 +84,7 @@ func Main() { cfgset.String("etcd_cafile", "", "SSL Certificate Authority file used to secure etcd communication") cfgset.String("etcd_key_prefix", registry.DefaultKeyPrefix, "Keyspace for fleet data in etcd") cfgset.Float64("etcd_request_timeout", 1.0, "Amount of time in seconds to allow a single etcd request before considering it failed.") - cfgset.Float64("engine_reconcile_interval", 2.0, "Interval at which the engine should reconcile the cluster schedule in etcd.") + cfgset.Float64("engine_reconcile_interval", 5.0, "Interval at which the engine should reconcile the cluster schedule in etcd.") cfgset.String("public_ip", "", "IP address that fleet machine should publish") cfgset.String("metadata", "", "List of key-value metadata to assign to the fleet machine") cfgset.String("agent_ttl", agent.DefaultTTL, "TTL in seconds of fleet machine state in etcd") @@ -96,6 +96,7 @@ func Main() { cfgset.Bool("disable_watches", false, "Disable the use of etcd watches. Increases scheduling latency") cfgset.Bool("verify_units", false, "DEPRECATED - This option is ignored") cfgset.String("authorized_keys_file", "", "DEPRECATED - This option is ignored") + cfgset.Bool("use_lease_ttl", false, "Enable the usage of TTL option when creating a lease key in etcd") globalconf.Register("", cfgset) cfg, err := getConfig(cfgset, *cfgPath) @@ -237,6 +238,7 @@ func getConfig(flagset *flag.FlagSet, userCfgFile string) (*config.Config, error SystemdUser: (*flagset.Lookup("systemd_user")).Value.(flag.Getter).Get().(bool), TokenLimit: (*flagset.Lookup("token_limit")).Value.(flag.Getter).Get().(int), AuthorizedKeysFile: (*flagset.Lookup("authorized_keys_file")).Value.(flag.Getter).Get().(string), + UseLeaseTTL: (*flagset.Lookup("use_lease_ttl")).Value.(flag.Getter).Get().(bool), } if cfg.VerifyUnits { diff --git a/pkg/lease/etcd.go b/pkg/lease/etcd.go index 83ad4693c..b0976b98d 100644 --- a/pkg/lease/etcd.go +++ b/pkg/lease/etcd.go @@ -20,6 +20,8 @@ import ( "time" etcd "github.com/coreos/etcd/client" + "github.com/coreos/fleet/log" + "golang.org/x/net/context" ) @@ -53,8 +55,14 @@ func (l *etcdLease) Renew(period time.Duration) error { val, err := serializeLeaseMetadata(l.meta.MachineID, l.meta.Version) opts := &etcd.SetOptions{ PrevIndex: l.idx, - TTL: period, } + + log.Debugf("Renew %v", l.mgr.UseLeaseTTL) + + if l.mgr.UseLeaseTTL { + opts.TTL = period + } + resp, err := l.mgr.kAPI.Set(l.mgr.ctx(), l.key, val, opts) if err != nil { return err @@ -97,12 +105,13 @@ func serializeLeaseMetadata(machID string, ver int) (string, error) { } type etcdLeaseManager struct { - kAPI etcd.KeysAPI - keyPrefix string + kAPI etcd.KeysAPI + keyPrefix string + UseLeaseTTL bool } -func NewEtcdLeaseManager(kAPI etcd.KeysAPI, keyPrefix string) *etcdLeaseManager { - return &etcdLeaseManager{kAPI: kAPI, keyPrefix: keyPrefix} +func NewEtcdLeaseManager(kAPI etcd.KeysAPI, keyPrefix string, useLeaseTTL bool) *etcdLeaseManager { + return &etcdLeaseManager{kAPI: kAPI, keyPrefix: keyPrefix, UseLeaseTTL: useLeaseTTL} } func (r *etcdLeaseManager) ctx() context.Context { @@ -136,7 +145,12 @@ func (r *etcdLeaseManager) StealLease(name, machID string, ver int, period time. key := r.leasePath(name) opts := &etcd.SetOptions{ PrevIndex: idx, - TTL: period, + } + + log.Debugf("StealLease %v", r.UseLeaseTTL) + + if r.UseLeaseTTL { + opts.TTL = period } resp, err := r.kAPI.Set(r.ctx(), key, val, opts) if err != nil { @@ -158,9 +172,13 @@ func (r *etcdLeaseManager) AcquireLease(name string, machID string, ver int, per key := r.leasePath(name) opts := &etcd.SetOptions{ - TTL: period, PrevExist: etcd.PrevNoExist, } + log.Debugf("AcquireLease %v", r.UseLeaseTTL) + + if r.UseLeaseTTL { + opts.TTL = period + } resp, err := r.kAPI.Set(r.ctx(), key, val, opts) if err != nil { diff --git a/registry/rpc/registrymux_test.go b/registry/rpc/registrymux_test.go index bdc4ffb7c..f8b6c1f90 100644 --- a/registry/rpc/registrymux_test.go +++ b/registry/rpc/registrymux_test.go @@ -111,7 +111,7 @@ func TestRegistryMuxUnitManagement(t *testing.T) { e := &testEtcdKeysAPI{} etcdReg := registry.NewEtcdRegistry(e, "/fleet/") - lManager := lease.NewEtcdLeaseManager(e, "/fleet/") + lManager := lease.NewEtcdLeaseManager(e, "/fleet/", false) reg := NewRegistryMux(etcdReg, mach, lManager) contents := ` diff --git a/server/server.go b/server/server.go index e121c9c7b..175415267 100644 --- a/server/server.go +++ b/server/server.go @@ -110,7 +110,7 @@ func New(cfg config.Config, listeners []net.Listener) (*Server, error) { reg engine.CompleteRegistry genericReg interface{} ) - lManager := lease.NewEtcdLeaseManager(kAPI, cfg.EtcdKeyPrefix) + lManager := lease.NewEtcdLeaseManager(kAPI, cfg.EtcdKeyPrefix, cfg.UseLeaseTTL) if !cfg.EnableGRPC { genericReg = registry.NewEtcdRegistry(kAPI, cfg.EtcdKeyPrefix)