Snapshot restore tests (#16647)

* add snapshot restore test

* add logstore as test parameter

* Use the correct image version

* make sure we read the logs from a followers to test the follower snapshot install path.

* update to raf-wal v0.3.0

* add changelog.

* updating changelog for bug description and removed integration test.

* setting up test container builder to only set logStore for 1.15 and higher

---------

Co-authored-by: Paul Banks <pbanks@hashicorp.com>
Co-authored-by: John Murret <john.murret@hashicorp.com>

.changelog/16647.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+raft_logstore: Fixes a bug where restoring a snapshot when using the experimental WAL storage backend causes a panic.
+```

go.mod

@@ -19,7 +19,7 @@ require (
 	github.com/NYTimes/gziphandler v1.0.1
 	github.com/aliyun/alibaba-cloud-sdk-go v1.62.156
 	github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e
-	github.com/armon/go-metrics v0.3.10
+	github.com/armon/go-metrics v0.4.1
 	github.com/armon/go-radix v1.0.0
 	github.com/aws/aws-sdk-go v1.42.34
 	github.com/coredns/coredns v1.6.6
@@ -64,10 +64,10 @@ require (
 	github.com/hashicorp/hcp-sdk-go v0.23.1-0.20220921131124-49168300a7dc
 	github.com/hashicorp/hil v0.0.0-20200423225030-a18a1cd20038
 	github.com/hashicorp/memberlist v0.5.0
-	github.com/hashicorp/raft v1.3.11
+	github.com/hashicorp/raft v1.4.0
 	github.com/hashicorp/raft-autopilot v0.1.6
 	github.com/hashicorp/raft-boltdb/v2 v2.2.2
-	github.com/hashicorp/raft-wal v0.2.4
+	github.com/hashicorp/raft-wal v0.3.0
 	github.com/hashicorp/serf v0.10.1
 	github.com/hashicorp/vault-plugin-auth-alicloud v0.14.0
 	github.com/hashicorp/vault/api v1.8.3
@@ -92,7 +92,7 @@ require (
 	github.com/rboyer/safeio v0.2.1
 	github.com/ryanuber/columnize v2.1.2+incompatible
 	github.com/shirou/gopsutil/v3 v3.22.8
-	github.com/stretchr/testify v1.8.0
+	github.com/stretchr/testify v1.8.2
 	go.etcd.io/bbolt v1.3.6
 	go.uber.org/goleak v1.1.10
 	golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d
@@ -215,7 +215,7 @@ require (
 	github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect
 	github.com/softlayer/softlayer-go v0.0.0-20180806151055-260589d94c7d // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/stretchr/objx v0.4.0 // indirect
+	github.com/stretchr/objx v0.5.0 // indirect
 	github.com/tencentcloud/tencentcloud-sdk-go v1.0.162 // indirect
 	github.com/tklauser/go-sysconf v0.3.10 // indirect
 	github.com/tklauser/numcpus v0.4.0 // indirect

go.sum

@@ -144,8 +144,8 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
 github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg=
 github.com/armon/go-metrics v0.3.9/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc=
-github.com/armon/go-metrics v0.3.10 h1:FR+drcQStOe+32sYyJYyZ7FIdgoGGBnwLl+flodp8Uo=
+github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA=
-github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc=
+github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI=
 github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
@@ -610,8 +610,9 @@ github.com/hashicorp/net-rpc-msgpackrpc/v2 v2.0.0 h1:kBpVVl1sl3MaSrs97e0+pDQhSrq
 github.com/hashicorp/net-rpc-msgpackrpc/v2 v2.0.0/go.mod h1:6pdNz0vo0mF0GvhwDG56O3N18qBrAz/XRIcfINfTbwo=
 github.com/hashicorp/raft v1.1.0/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM=
 github.com/hashicorp/raft v1.2.0/go.mod h1:vPAJM8Asw6u8LxC3eJCUZmRP/E4QmUGE1R7g7k8sG/8=
-github.com/hashicorp/raft v1.3.11 h1:p3v6gf6l3S797NnK5av3HcczOC1T5CLoaRvg0g9ys4A=
 github.com/hashicorp/raft v1.3.11/go.mod h1:J8naEwc6XaaCfts7+28whSeRvCqTd6e20BlCU3LtEO4=
+github.com/hashicorp/raft v1.4.0 h1:tn28S/AWv0BtRQgwZv/1NELu8sCvI0FixqL8C8MYKeY=
+github.com/hashicorp/raft v1.4.0/go.mod h1:nz64BIjXphDLATfKGG5RzHtNUPioLeKFsXEm88yTVew=
 github.com/hashicorp/raft-autopilot v0.1.6 h1:C1q3RNF2FfXNZfHWbvVAu0QixaQK8K5pX4O5lh+9z4I=
 github.com/hashicorp/raft-autopilot v0.1.6/go.mod h1:Af4jZBwaNOI+tXfIqIdbcAnh/UyyqIMj/pOISIfhArw=
 github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea/go.mod h1:pNv7Wc3ycL6F5oOWn+tPGo2gWD4a5X+yp/ntwdKLjRk=
@@ -619,8 +620,8 @@ github.com/hashicorp/raft-boltdb v0.0.0-20210409134258-03c10cc3d4ea/go.mod h1:qR
 github.com/hashicorp/raft-boltdb v0.0.0-20220329195025-15018e9b97e0 h1:CO8dBMLH6dvE1jTn/30ZZw3iuPsNfajshWoJTnVc5cc=
 github.com/hashicorp/raft-boltdb/v2 v2.2.2 h1:rlkPtOllgIcKLxVT4nutqlTH2NRFn+tO1wwZk/4Dxqw=
 github.com/hashicorp/raft-boltdb/v2 v2.2.2/go.mod h1:N8YgaZgNJLpZC+h+by7vDu5rzsRgONThTEeUS3zWbfY=
-github.com/hashicorp/raft-wal v0.2.4 h1:Ke0ytMj8XyOVKQqFDmmgs/6hqkTJg0b/GO2a2XQBZ6A=
+github.com/hashicorp/raft-wal v0.3.0 h1:Mi6RPoRbsxIIYZryI+bSTXHD97Ua6rIYO51ibYV9bkY=
-github.com/hashicorp/raft-wal v0.2.4/go.mod h1:JQ/4RbnKFi5Q/4rA73CekaYtHCJhU7qM7AQ4X5Y6q4M=
+github.com/hashicorp/raft-wal v0.3.0/go.mod h1:A6vP5o8hGOs1LHfC1Okh9xPwWDcmb6Vvuz/QyqUXlOE=
 github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY=
 github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4=
 github.com/hashicorp/vault-plugin-auth-alicloud v0.14.0 h1:O6tNk0s/arubLUbLeCyaRs5xGo9VwmbQazISY/BfPK4=
@@ -703,8 +704,8 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
 github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
-github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@@ -928,8 +929,8 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L
 github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k=
 github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
-github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
 github.com/rs/zerolog v1.4.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
 github.com/russross/blackfriday v0.0.0-20170610170232-067529f716f4/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
@@ -984,8 +985,9 @@ github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DM
 github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
@@ -994,8 +996,9 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
-github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
+github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/tencentcloud/tencentcloud-sdk-go v1.0.162 h1:8fDzz4GuVg4skjY2B0nMN7h6uN61EDVkuLyI2+qGHhI=
 github.com/tencentcloud/tencentcloud-sdk-go v1.0.162/go.mod h1:asUz5BPXxgoPGaRgZaVm1iGcUAuHyYUo1nXqKa83cvI=
 github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=

test/integration/consul-container/libs/cluster/builder.go

@@ -20,6 +20,13 @@ const (
 	ConsulCACertKey = "consul-agent-ca-key.pem"
 )
 
+type LogStore string
+
+const (
+	LogStore_WAL    LogStore = "wal"
+	LogStore_BoltDB LogStore = "boltdb"
+)
+
 // BuildContext provides a reusable object meant to share common configuration settings
 // between agent configuration builders.
 type BuildContext struct {
@@ -41,6 +48,7 @@ type BuildContext struct {
 	tlsCertIndex int // keeps track of the certificates issued for naming purposes
 
 	aclEnabled bool
+	logStore   LogStore
 }
 
 func (c *BuildContext) DockerImage() string {
@@ -89,6 +97,9 @@ type BuildOptions struct {
 
 	// ACLEnabled configures acl in agent configuration
 	ACLEnabled bool
+
+	//StoreLog define which LogStore to use
+	LogStore LogStore
 }
 
 func NewBuildContext(t *testing.T, opts BuildOptions) *BuildContext {
@@ -103,6 +114,7 @@ func NewBuildContext(t *testing.T, opts BuildOptions) *BuildContext {
 		useAPIWithTLS:          opts.UseAPIWithTLS,
 		useGRPCWithTLS:         opts.UseGRPCWithTLS,
 		aclEnabled:             opts.ACLEnabled,
+		logStore:               opts.LogStore,
 	}
 
 	if ctx.consulImageName == "" {
@@ -202,6 +214,18 @@ func NewConfigBuilder(ctx *BuildContext) *Builder {
 		b.conf.Set("acl.enable_token_persistence", true)
 	}
 
+	ls := string(ctx.logStore)
+	if ls != "" && (ctx.consulVersion == "local" ||
+		semver.Compare("v"+ctx.consulVersion, "v1.15.0") >= 0) {
+		// Enable logstore backend for version after v1.15.0
+		if ls != string(LogStore_WAL) && ls != string(LogStore_BoltDB) {
+			ls = string(LogStore_BoltDB)
+		}
+		b.conf.Set("raft_logstore.backend", ls)
+	} else {
+		b.conf.Unset("raft_logstore.backend")
+	}
+
 	return b
 }
 

test/integration/consul-container/libs/cluster/container.go

@@ -154,11 +154,17 @@ func NewConsulContainer(ctx context.Context, config Config, cluster *Cluster, po
 		info AgentInfo
 	)
 	if httpPort > 0 {
-		uri, err := podContainer.PortEndpoint(ctx, "8500", "http")
+		for i := 0; i < 10; i++ {
+			uri, err := podContainer.PortEndpoint(ctx, "8500", "http")
+			if err != nil {
+				time.Sleep(500 * time.Millisecond)
+				continue
+			}
+			clientAddr = uri
+		}
 		if err != nil {
 			return nil, err
 		}
-		clientAddr = uri
 
 	} else if httpsPort > 0 {
 		uri, err := podContainer.PortEndpoint(ctx, "8501", "https")

test/integration/consul-container/libs/topology/peering_topology.go

@@ -198,6 +198,7 @@ func NewCluster(
 		AllowHTTPAnyway:        true,
 		ConsulVersion:          config.BuildOpts.ConsulVersion,
 		ACLEnabled:             config.BuildOpts.ACLEnabled,
+		LogStore:               config.BuildOpts.LogStore,
 	}
 	ctx := libcluster.NewBuildContext(t, opts)
 

test/integration/consul-container/test/snapshot/snapshot_restore_test.go

@@ -0,0 +1,97 @@
+package snapshot
+
+import (
+	"fmt"
+	"github.com/hashicorp/consul/api"
+	libcluster "github.com/hashicorp/consul/test/integration/consul-container/libs/cluster"
+	libtopology "github.com/hashicorp/consul/test/integration/consul-container/libs/topology"
+	"github.com/hashicorp/consul/test/integration/consul-container/libs/utils"
+	"github.com/stretchr/testify/require"
+	"io"
+	"testing"
+)
+
+func TestSnapshotRestore(t *testing.T) {
+
+	cases := []libcluster.LogStore{libcluster.LogStore_WAL, libcluster.LogStore_BoltDB}
+
+	for _, c := range cases {
+		t.Run(fmt.Sprintf("test log store: %s", c), func(t *testing.T) {
+			testSnapShotRestoreForLogStore(t, c)
+		})
+	}
+}
+
+func testSnapShotRestoreForLogStore(t *testing.T, logStore libcluster.LogStore) {
+
+	const (
+		numServers = 3
+	)
+
+	// Create initial cluster
+	cluster, _, _ := libtopology.NewCluster(t, &libtopology.ClusterConfig{
+		NumServers: numServers,
+		NumClients: 0,
+		BuildOpts: &libcluster.BuildOptions{
+			Datacenter:      "dc1",
+			ConsulImageName: utils.TargetImageName,
+			ConsulVersion:   utils.TargetVersion,
+			LogStore:        logStore,
+		},
+		ApplyDefaultProxySettings: true,
+	})
+
+	client := cluster.APIClient(0)
+	libcluster.WaitForLeader(t, cluster, client)
+	libcluster.WaitForMembers(t, client, 3)
+
+	for i := 0; i < 100; i++ {
+		_, err := client.KV().Put(&api.KVPair{Key: fmt.Sprintf("key-%d", i), Value: []byte(fmt.Sprintf("value-%d", i))}, nil)
+		require.NoError(t, err)
+	}
+
+	var snapshot io.ReadCloser
+	var err error
+	snapshot, _, err = client.Snapshot().Save(nil)
+	require.NoError(t, err)
+
+	err = cluster.Terminate()
+	require.NoError(t, err)
+	// Create a fresh cluster from scratch
+	cluster2, _, _ := libtopology.NewCluster(t, &libtopology.ClusterConfig{
+		NumServers: numServers,
+		NumClients: 0,
+		BuildOpts: &libcluster.BuildOptions{
+			Datacenter:      "dc1",
+			ConsulImageName: utils.TargetImageName,
+			ConsulVersion:   utils.TargetVersion,
+			LogStore:        logStore,
+		},
+		ApplyDefaultProxySettings: true,
+	})
+	client2 := cluster2.APIClient(0)
+
+	libcluster.WaitForLeader(t, cluster2, client2)
+	libcluster.WaitForMembers(t, client2, 3)
+
+	// Restore the saved snapshot
+	require.NoError(t, client2.Snapshot().Restore(nil, snapshot))
+
+	libcluster.WaitForLeader(t, cluster2, client2)
+
+	followers, err := cluster2.Followers()
+	require.NoError(t, err)
+	require.Len(t, followers, 2)
+
+	// use a follower api client and set `AllowStale` to true
+	// to test the follower snapshot install code path as well.
+	fc := followers[0].GetClient()
+
+	for i := 0; i < 100; i++ {
+		kv, _, err := fc.KV().Get(fmt.Sprintf("key-%d", i), &api.QueryOptions{AllowStale: true})
+		require.NoError(t, err)
+		require.Equal(t, kv.Key, fmt.Sprintf("key-%d", i))
+		require.Equal(t, kv.Value, []byte(fmt.Sprintf("value-%d", i)))
+	}
+
+}