diff --git a/.changelog/11720.txt b/.changelog/11720.txt new file mode 100644 index 0000000000..9c141b3c77 --- /dev/null +++ b/.changelog/11720.txt @@ -0,0 +1,11 @@ +```release-note:improvement +raft: Use bbolt instead of the legacy boltdb implementation +``` + +```release-note:improvement +raft: Emit boltdb related performance metrics +``` + +```release-note:improvement +raft: Added a configuration to disable boltdb freelist syncing +``` diff --git a/agent/agent.go b/agent/agent.go index bc6aea7503..ecf84fdea9 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -1263,6 +1263,7 @@ func newConsulConfig(runtimeCfg *config.RuntimeConfig, logger hclog.Logger) (*co } cfg.ConfigEntryBootstrap = runtimeCfg.ConfigEntryBootstrap + cfg.RaftBoltDBConfig = runtimeCfg.RaftBoltDBConfig // Duplicate our own serf config once to make sure that the duplication // function does not drift. diff --git a/agent/config/builder.go b/agent/config/builder.go index 012229ae34..55c9382126 100644 --- a/agent/config/builder.go +++ b/agent/config/builder.go @@ -1094,6 +1094,10 @@ func (b *builder) build() (rt RuntimeConfig, err error) { rt.UseStreamingBackend = boolValWithDefault(c.UseStreamingBackend, true) + if c.RaftBoltDBConfig != nil { + rt.RaftBoltDBConfig = *c.RaftBoltDBConfig + } + if rt.Cache.EntryFetchMaxBurst <= 0 { return RuntimeConfig{}, fmt.Errorf("cache.entry_fetch_max_burst must be strictly positive, was: %v", rt.Cache.EntryFetchMaxBurst) } diff --git a/agent/config/config.go b/agent/config/config.go index 61161b7b58..7d8ecadbb2 100644 --- a/agent/config/config.go +++ b/agent/config/config.go @@ -4,6 +4,8 @@ import ( "encoding/json" "fmt" + "github.com/hashicorp/consul/agent/consul" + "github.com/hashicorp/hcl" "github.com/mitchellh/mapstructure" @@ -256,6 +258,8 @@ type Config struct { RPC RPC `mapstructure:"rpc"` + RaftBoltDBConfig *consul.RaftBoltDBConfig `mapstructure:"raft_boltdb"` + // UseStreamingBackend instead of blocking queries for service health and // any other endpoints which support streaming. UseStreamingBackend *bool `mapstructure:"use_streaming_backend"` diff --git a/agent/config/runtime.go b/agent/config/runtime.go index 2e7d7cf972..e2393363f3 100644 --- a/agent/config/runtime.go +++ b/agent/config/runtime.go @@ -943,6 +943,8 @@ type RuntimeConfig struct { // hcl: raft_trailing_logs = int RaftTrailingLogs int + RaftBoltDBConfig consul.RaftBoltDBConfig + // ReconnectTimeoutLAN specifies the amount of time to wait to reconnect with // another agent before deciding it's permanently gone. This can be used to // control the time it takes to reap failed nodes from the cluster. diff --git a/agent/config/runtime_test.go b/agent/config/runtime_test.go index 390305902c..8abbcc4033 100644 --- a/agent/config/runtime_test.go +++ b/agent/config/runtime_test.go @@ -6015,6 +6015,7 @@ func TestLoad_FullConfig(t *testing.T) { "args": []interface{}{"dltjDJ2a", "flEa7C2d"}, }, }, + RaftBoltDBConfig: consul.RaftBoltDBConfig{NoFreelistSync: true}, } entFullRuntimeConfig(expected) diff --git a/agent/config/testdata/TestRuntimeConfig_Sanitize.golden b/agent/config/testdata/TestRuntimeConfig_Sanitize.golden index 2d1093d1eb..84c303c761 100644 --- a/agent/config/testdata/TestRuntimeConfig_Sanitize.golden +++ b/agent/config/testdata/TestRuntimeConfig_Sanitize.golden @@ -252,6 +252,9 @@ "RPCMaxConnsPerClient": 0, "RPCProtocol": 0, "RPCRateLimit": 0, + "RaftBoltDBConfig": { + "NoFreelistSync": false + }, "RaftProtocol": 3, "RaftSnapshotInterval": "0s", "RaftSnapshotThreshold": 0, diff --git a/agent/config/testdata/full-config.hcl b/agent/config/testdata/full-config.hcl index 939745c9b9..869f672528 100644 --- a/agent/config/testdata/full-config.hcl +++ b/agent/config/testdata/full-config.hcl @@ -328,6 +328,9 @@ raft_protocol = 3 raft_snapshot_threshold = 16384 raft_snapshot_interval = "30s" raft_trailing_logs = 83749 +raft_boltdb { + NoFreelistSync = true +} read_replica = true reconnect_timeout = "23739s" reconnect_timeout_wan = "26694s" diff --git a/agent/config/testdata/full-config.json b/agent/config/testdata/full-config.json index 4649c86bf7..017651d88f 100644 --- a/agent/config/testdata/full-config.json +++ b/agent/config/testdata/full-config.json @@ -326,6 +326,9 @@ "raft_snapshot_threshold": 16384, "raft_snapshot_interval": "30s", "raft_trailing_logs": 83749, + "raft_boltdb": { + "NoFreelistSync": true + }, "read_replica": true, "reconnect_timeout": "23739s", "reconnect_timeout_wan": "26694s", diff --git a/agent/consul/config.go b/agent/consul/config.go index 86c87f5d74..9c343494ac 100644 --- a/agent/consul/config.go +++ b/agent/consul/config.go @@ -391,6 +391,8 @@ type Config struct { RPCConfig RPCConfig + RaftBoltDBConfig RaftBoltDBConfig + // Embedded Consul Enterprise specific configuration *EnterpriseConfig } @@ -603,3 +605,7 @@ type ReloadableConfig struct { RaftSnapshotInterval time.Duration RaftTrailingLogs int } + +type RaftBoltDBConfig struct { + NoFreelistSync bool +} diff --git a/agent/consul/server.go b/agent/consul/server.go index b51fc7f7cb..554d5cbe58 100644 --- a/agent/consul/server.go +++ b/agent/consul/server.go @@ -18,6 +18,7 @@ import ( "time" "github.com/hashicorp/go-version" + "go.etcd.io/bbolt" "github.com/armon/go-metrics" connlimit "github.com/hashicorp/go-connlimit" @@ -25,7 +26,7 @@ import ( "github.com/hashicorp/go-memdb" "github.com/hashicorp/raft" autopilot "github.com/hashicorp/raft-autopilot" - raftboltdb "github.com/hashicorp/raft-boltdb" + raftboltdb "github.com/hashicorp/raft-boltdb/v2" "github.com/hashicorp/serf/serf" "golang.org/x/time/rate" "google.golang.org/grpc" @@ -729,13 +730,21 @@ func (s *Server) setupRaft() error { } // Create the backend raft store for logs and stable storage. - store, err := raftboltdb.NewBoltStore(filepath.Join(path, "raft.db")) + store, err := raftboltdb.New(raftboltdb.Options{ + BoltOptions: &bbolt.Options{ + NoFreelistSync: s.config.RaftBoltDBConfig.NoFreelistSync, + }, + Path: filepath.Join(path, "raft.db"), + }) if err != nil { return err } s.raftStore = store stable = store + // start publishing boltdb metrics + go store.RunMetrics(&lib.StopChannelContext{StopCh: s.shutdownCh}, 0) + // Wrap the store in a LogCache to improve performance. cacheStore, err := raft.NewLogCache(raftLogCacheSize, store) if err != nil { diff --git a/go.mod b/go.mod index 99c07da627..19ba3005fe 100644 --- a/go.mod +++ b/go.mod @@ -53,7 +53,8 @@ require ( github.com/hashicorp/net-rpc-msgpackrpc v0.0.0-20151116020338-a14192a58a69 github.com/hashicorp/raft v1.3.2 github.com/hashicorp/raft-autopilot v0.1.5 - github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea + github.com/hashicorp/raft-boltdb v0.0.0-20211202195631-7d34b9fb3f42 // indirect + github.com/hashicorp/raft-boltdb/v2 v2.2.0 github.com/hashicorp/serf v0.9.6 github.com/hashicorp/vault/api v1.0.5-0.20200717191844-f687267c8086 github.com/hashicorp/vault/sdk v0.1.14-0.20200519221838-e0cfd64bc267 @@ -79,6 +80,7 @@ require ( github.com/ryanuber/columnize v2.1.0+incompatible github.com/shirou/gopsutil/v3 v3.21.10 github.com/stretchr/testify v1.7.0 + go.etcd.io/bbolt v1.3.5 go.opencensus.io v0.22.0 // indirect go.uber.org/goleak v1.1.10 golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a diff --git a/go.sum b/go.sum index d99fe3b6b8..84979bb415 100644 --- a/go.sum +++ b/go.sum @@ -59,6 +59,7 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg= github.com/armon/go-metrics v0.3.0/go.mod h1:zXjbSimjXTd7vOpY8B0/2LpvNvDoXBuplAD+gJD3GYs= +github.com/armon/go-metrics v0.3.8/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= github.com/armon/go-metrics v0.3.10 h1:FR+drcQStOe+32sYyJYyZ7FIdgoGGBnwLl+flodp8Uo= github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= @@ -284,14 +285,19 @@ github.com/hashicorp/memberlist v0.3.0 h1:8+567mCcFDnS5ADl7lrpxPMWiFCElyUEeW0gtj github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/hashicorp/net-rpc-msgpackrpc v0.0.0-20151116020338-a14192a58a69 h1:lc3c72qGlIMDqQpQH82Y4vaglRMMFdJbziYWriR4UcE= github.com/hashicorp/net-rpc-msgpackrpc v0.0.0-20151116020338-a14192a58a69/go.mod h1:/z+jUGRBlwVpUZfjute9jWaF6/HuhjuFQuL1YXzVD1Q= +github.com/hashicorp/raft v1.1.0/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM= github.com/hashicorp/raft v1.1.1/go.mod h1:vPAJM8Asw6u8LxC3eJCUZmRP/E4QmUGE1R7g7k8sG/8= github.com/hashicorp/raft v1.2.0/go.mod h1:vPAJM8Asw6u8LxC3eJCUZmRP/E4QmUGE1R7g7k8sG/8= github.com/hashicorp/raft v1.3.2 h1:j2tqHqFnDdWCepLxzuo3b6WzS2krIweBrvEoqBbWMTo= github.com/hashicorp/raft v1.3.2/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM= github.com/hashicorp/raft-autopilot v0.1.5 h1:onEfMH5uHVdXQqtas36zXUHEZxLdsJVu/nXHLcLdL1I= github.com/hashicorp/raft-autopilot v0.1.5/go.mod h1:Af4jZBwaNOI+tXfIqIdbcAnh/UyyqIMj/pOISIfhArw= -github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea h1:xykPFhrBAS2J0VBzVa5e80b5ZtYuNQtgXjN40qBZlD4= github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea/go.mod h1:pNv7Wc3ycL6F5oOWn+tPGo2gWD4a5X+yp/ntwdKLjRk= +github.com/hashicorp/raft-boltdb v0.0.0-20210409134258-03c10cc3d4ea/go.mod h1:qRd6nFJYYS6Iqnc/8HcUmko2/2Gw8qTFEmxDLii6W5I= +github.com/hashicorp/raft-boltdb v0.0.0-20211202195631-7d34b9fb3f42 h1:Ye8SofeDHJzu9xvvaMmpMkqHELWW7rTcXwdUR0CWW48= +github.com/hashicorp/raft-boltdb v0.0.0-20211202195631-7d34b9fb3f42/go.mod h1:wcXL8otVu5cpJVLjcmq7pmfdRCdaP+xnvu7WQcKJAhs= +github.com/hashicorp/raft-boltdb/v2 v2.2.0 h1:/CVN9LSAcH50L3yp2TsPFIpeyHn1m3VF6kiutlDE3Nw= +github.com/hashicorp/raft-boltdb/v2 v2.2.0/go.mod h1:SgPUD5TP20z/bswEr210SnkUFvQP/YjKV95aaiTbeMQ= github.com/hashicorp/serf v0.9.6 h1:uuEX1kLR6aoda1TBttmJQKDLZE1Ob7KN0NPdE7EtCDc= github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= github.com/hashicorp/vault/api v1.0.5-0.20200717191844-f687267c8086 h1:OKsyxKi2sNmqm1Gv93adf2AID2FOBFdCbbZn9fGtIdg= @@ -520,6 +526,8 @@ github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1: github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= +go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0 h1:C9hSCOW830chIVkdja34wa6Ky+IzWllkUinR+BtRZd4= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= @@ -620,6 +628,7 @@ golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=