diff --git a/api/lock_test.go b/api/lock_test.go index ceab5cdf9c..1996b29963 100644 --- a/api/lock_test.go +++ b/api/lock_test.go @@ -105,32 +105,40 @@ func TestLock_DeleteKey(t *testing.T) { c, s := makeClient(t) defer s.Stop() - lock, err := c.LockKey("test/lock") - if err != nil { - t.Fatalf("err: %v", err) - } + // This uncovered some issues around special-case handling of low index + // numbers where it would work with a low number but fail for higher + // ones, so we loop this a bit to sweep the index up out of that + // territory. + for i := 0; i < 10; i++ { + func() { + lock, err := c.LockKey("test/lock") + if err != nil { + t.Fatalf("err: %v", err) + } - // Should work - leaderCh, err := lock.Lock(nil) - if err != nil { - t.Fatalf("err: %v", err) - } - if leaderCh == nil { - t.Fatalf("not leader") - } - defer lock.Unlock() + // Should work + leaderCh, err := lock.Lock(nil) + if err != nil { + t.Fatalf("err: %v", err) + } + if leaderCh == nil { + t.Fatalf("not leader") + } + defer lock.Unlock() - go func() { - // Nuke the key, simulate an operator intervention - kv := c.KV() - kv.Delete("test/lock", nil) - }() + go func() { + // Nuke the key, simulate an operator intervention + kv := c.KV() + kv.Delete("test/lock", nil) + }() - // Should loose leadership - select { - case <-leaderCh: - case <-time.After(time.Second): - t.Fatalf("should not be leader") + // Should loose leadership + select { + case <-leaderCh: + case <-time.After(time.Second): + t.Fatalf("should not be leader") + } + }() } } diff --git a/consul/fsm_test.go b/consul/fsm_test.go index f4c9fd4adc..083d0fa89a 100644 --- a/consul/fsm_test.go +++ b/consul/fsm_test.go @@ -438,7 +438,7 @@ func TestFSM_SnapshotRestore(t *testing.T) { } // Verify key is set - d, err := fsm2.state.KVSGet("/test") + _, d, err := fsm2.state.KVSGet("/test") if err != nil { t.Fatalf("err: %v", err) } @@ -471,13 +471,17 @@ func TestFSM_SnapshotRestore(t *testing.T) { } // Verify tombstones are restored - idx, _, err = fsm2.state.KVSList("/remove") - if err != nil { - t.Fatalf("err: %s", err) - } - if idx != 12 { - t.Fatalf("bad index: %d", idx) - } + func() { + snap := fsm2.state.Snapshot() + defer snap.Close() + dump, err := snap.TombstoneDump() + if err != nil { + t.Fatalf("err: %s", err) + } + if len(dump) != 1 { + t.Fatalf("bad: %#v", dump) + } + }() } func TestFSM_KVSSet(t *testing.T) { @@ -505,7 +509,7 @@ func TestFSM_KVSSet(t *testing.T) { } // Verify key is set - d, err := fsm.state.KVSGet("/test/path") + _, d, err := fsm.state.KVSGet("/test/path") if err != nil { t.Fatalf("err: %v", err) } @@ -550,7 +554,7 @@ func TestFSM_KVSDelete(t *testing.T) { } // Verify key is not set - d, err := fsm.state.KVSGet("/test/path") + _, d, err := fsm.state.KVSGet("/test/path") if err != nil { t.Fatalf("err: %v", err) } @@ -596,7 +600,7 @@ func TestFSM_KVSDeleteTree(t *testing.T) { } // Verify key is not set - d, err := fsm.state.KVSGet("/test/path") + _, d, err := fsm.state.KVSGet("/test/path") if err != nil { t.Fatalf("err: %v", err) } @@ -630,7 +634,7 @@ func TestFSM_KVSDeleteCheckAndSet(t *testing.T) { } // Verify key is set - d, err := fsm.state.KVSGet("/test/path") + _, d, err := fsm.state.KVSGet("/test/path") if err != nil { t.Fatalf("err: %v", err) } @@ -651,7 +655,7 @@ func TestFSM_KVSDeleteCheckAndSet(t *testing.T) { } // Verify key is gone - d, err = fsm.state.KVSGet("/test/path") + _, d, err = fsm.state.KVSGet("/test/path") if err != nil { t.Fatalf("err: %v", err) } @@ -685,7 +689,7 @@ func TestFSM_KVSCheckAndSet(t *testing.T) { } // Verify key is set - d, err := fsm.state.KVSGet("/test/path") + _, d, err := fsm.state.KVSGet("/test/path") if err != nil { t.Fatalf("err: %v", err) } @@ -707,7 +711,7 @@ func TestFSM_KVSCheckAndSet(t *testing.T) { } // Verify key is updated - d, err = fsm.state.KVSGet("/test/path") + _, d, err = fsm.state.KVSGet("/test/path") if err != nil { t.Fatalf("err: %v", err) } @@ -824,7 +828,7 @@ func TestFSM_KVSLock(t *testing.T) { } // Verify key is locked - d, err := fsm.state.KVSGet("/test/path") + _, d, err := fsm.state.KVSGet("/test/path") if err != nil { t.Fatalf("err: %v", err) } @@ -886,7 +890,7 @@ func TestFSM_KVSUnlock(t *testing.T) { } // Verify key is unlocked - d, err := fsm.state.KVSGet("/test/path") + _, d, err := fsm.state.KVSGet("/test/path") if err != nil { t.Fatalf("err: %v", err) } @@ -1009,12 +1013,14 @@ func TestFSM_TombstoneReap(t *testing.T) { } // Verify the tombstones are gone - idx, _, err = fsm.state.KVSList("/remove") + snap := fsm.state.Snapshot() + defer snap.Close() + dump, err := snap.TombstoneDump() if err != nil { t.Fatalf("err: %s", err) } - if idx != 0 { - t.Fatalf("bad index: %d", idx) + if len(dump) != 0 { + t.Fatalf("bad: %#v", dump) } } diff --git a/consul/kvs_endpoint.go b/consul/kvs_endpoint.go index 04be2d97b3..570b7d83b2 100644 --- a/consul/kvs_endpoint.go +++ b/consul/kvs_endpoint.go @@ -95,7 +95,7 @@ func (k *KVS) Get(args *structs.KeyRequest, reply *structs.IndexedDirEntries) er &reply.QueryMeta, state.GetKVSWatch(args.Key), func() error { - ent, err := state.KVSGet(args.Key) + index, ent, err := state.KVSGet(args.Key) if err != nil { return err } @@ -105,7 +105,11 @@ func (k *KVS) Get(args *structs.KeyRequest, reply *structs.IndexedDirEntries) er if ent == nil { // Must provide non-zero index to prevent blocking // Index 1 is impossible anyways (due to Raft internals) - reply.Index = 1 + if index == 0 { + reply.Index = 1 + } else { + reply.Index = index + } reply.Entries = nil } else { reply.Index = ent.ModifyIndex diff --git a/consul/kvs_endpoint_test.go b/consul/kvs_endpoint_test.go index f2a7345a62..bdc43f1c9e 100644 --- a/consul/kvs_endpoint_test.go +++ b/consul/kvs_endpoint_test.go @@ -36,7 +36,7 @@ func TestKVS_Apply(t *testing.T) { // Verify state := s1.fsm.State() - d, err := state.KVSGet("test") + _, d, err := state.KVSGet("test") if err != nil { t.Fatalf("err: %v", err) } @@ -58,7 +58,7 @@ func TestKVS_Apply(t *testing.T) { } // Verify - d, err = state.KVSGet("test") + _, d, err = state.KVSGet("test") if err != nil { t.Fatalf("err: %v", err) } @@ -278,6 +278,18 @@ func TestKVSEndpoint_List(t *testing.T) { t.Fatalf("bad: %v", d) } } + + // Try listing a nonexistent prefix + getR.Key = "/nope" + if err := client.Call("KVS.List", &getR, &dirent); err != nil { + t.Fatalf("err: %v", err) + } + if dirent.Index == 0 { + t.Fatalf("Bad: %v", dirent) + } + if len(dirent.Entries) != 0 { + t.Fatalf("Bad: %v", dirent.Entries) + } } func TestKVSEndpoint_List_Blocking(t *testing.T) { @@ -514,6 +526,18 @@ func TestKVSEndpoint_ListKeys(t *testing.T) { if dirent.Keys[2] != "/test/sub/" { t.Fatalf("Bad: %v", dirent.Keys) } + + // Try listing a nonexistent prefix + getR.Prefix = "/nope" + if err := client.Call("KVS.ListKeys", &getR, &dirent); err != nil { + t.Fatalf("err: %v", err) + } + if dirent.Index == 0 { + t.Fatalf("Bad: %v", dirent) + } + if len(dirent.Keys) != 0 { + t.Fatalf("Bad: %v", dirent.Keys) + } } func TestKVSEndpoint_ListKeys_ACLDeny(t *testing.T) { diff --git a/consul/leader_test.go b/consul/leader_test.go index a1cb55d26f..524ef6ef8d 100644 --- a/consul/leader_test.go +++ b/consul/leader_test.go @@ -579,39 +579,36 @@ func TestLeader_ReapTombstones(t *testing.T) { t.Fatalf("err: %v", err) } - // Snag the pre-delete index that the tombstone should - // preserve. - state := s1.fsm.State() - keyIdx, _, err := state.KVSList("test") - if err != nil { - t.Fatalf("err: %v", err) - } - // Delete the KV entry (tombstoned). arg.Op = structs.KVSDelete if err := msgpackrpc.CallWithCodec(codec, "KVS.Apply", &arg, &out); err != nil { t.Fatalf("err: %v", err) } - // Make sure the index advances to reflect the delete, instead of sliding - // backwards. - idx, _, err := state.KVSList("test") - if err != nil { - t.Fatalf("err: %v", err) - } - if idx <= keyIdx { - t.Fatalf("tombstone not working: %d <= %d", idx, keyIdx) - } + // Make sure there's a tombstone. + state := s1.fsm.State() + func() { + snap := state.Snapshot() + defer snap.Close() + dump, err := snap.TombstoneDump() + if err != nil { + t.Fatalf("err: %s", err) + } + if len(dump) != 1 { + t.Fatalf("bad: %#v", dump) + } + }() // Check that the new leader has a pending GC expiration by - // watching for the index to slide back. + // watching for the tombstone to get removed. testutil.WaitForResult(func() (bool, error) { - idx, _, err := state.KVSList("test") + snap := state.Snapshot() + defer snap.Close() + dump, err := snap.TombstoneDump() if err != nil { - t.Fatalf("err: %v", err) + return false, err } - fmt.Printf("%d %d\n", idx, keyIdx) - return idx < keyIdx, err + return len(dump) == 0, nil }, func(err error) { t.Fatalf("err: %v", err) }) diff --git a/consul/state/state_store.go b/consul/state/state_store.go index bbf8bd0eef..42e7b06632 100644 --- a/consul/state/state_store.go +++ b/consul/state/state_store.go @@ -1319,32 +1319,35 @@ func (s *StateStore) kvsSetTxn(tx *memdb.Txn, idx uint64, entry *structs.DirEntr } // KVSGet is used to retrieve a key/value pair from the state store. -func (s *StateStore) KVSGet(key string) (*structs.DirEntry, error) { +func (s *StateStore) KVSGet(key string) (uint64, *structs.DirEntry, error) { tx := s.db.Txn(false) defer tx.Abort() + // Get the table index. + idx := maxIndexTxn(tx, "kvs") + + // Retrieve the key. entry, err := tx.First("kvs", "id", key) if err != nil { - return nil, fmt.Errorf("failed kvs lookup: %s", err) + return 0, nil, fmt.Errorf("failed kvs lookup: %s", err) } if entry != nil { - return entry.(*structs.DirEntry), nil + return idx, entry.(*structs.DirEntry), nil } - return nil, nil + return idx, nil, nil } -// TODO (slackpad) - We changed the behavior here to return 0 instead of the -// max index for the cases where they are no matching keys. Need to make sure -// this is sane. Seems ok from a watch perspective, as we integrate need to see -// if there are other impacts. - // KVSList is used to list out all keys under a given prefix. If the -// prefix is left empty, all keys in the KVS will be returned. The -// returned index is the max index of the returned kvs entries. +// prefix is left empty, all keys in the KVS will be returned. The returned +// is the max index of the returned kvs entries or applicable tombstones, or +// else it's the full table indexes for kvs and tombstones. func (s *StateStore) KVSList(prefix string) (uint64, structs.DirEntries, error) { tx := s.db.Txn(false) defer tx.Abort() + // Get the table indexes. + idx := maxIndexTxn(tx, "kvs", "tombstones") + // Query the prefix and list the available keys entries, err := tx.Get("kvs", "id_prefix", prefix) if err != nil { @@ -1370,7 +1373,13 @@ func (s *StateStore) KVSList(prefix string) (uint64, structs.DirEntries, error) if gindex > lindex { lindex = gindex } - return lindex, ents, nil + + // Use the sub index if it was set and there are entries, otherwise use + // the full table index from above. + if lindex != 0 { + idx = lindex + } + return idx, ents, nil } // KVSListKeys is used to query the KV store for keys matching the given prefix. @@ -1381,6 +1390,9 @@ func (s *StateStore) KVSListKeys(prefix, sep string) (uint64, []string, error) { tx := s.db.Txn(false) defer tx.Abort() + // Get the table indexes. + idx := maxIndexTxn(tx, "kvs", "tombstones") + // Fetch keys using the specified prefix entries, err := tx.Get("kvs", "id_prefix", prefix) if err != nil { @@ -1430,7 +1442,13 @@ func (s *StateStore) KVSListKeys(prefix, sep string) (uint64, []string, error) { if gindex > lindex { lindex = gindex } - return lindex, keys, nil + + // Use the sub index if it was set and there are entries, otherwise use + // the full table index from above. + if lindex != 0 { + idx = lindex + } + return idx, keys, nil } // KVSDelete is used to perform a shallow delete on a single key in the diff --git a/consul/state/state_store_test.go b/consul/state/state_store_test.go index 083d480bfd..f55e48f3f8 100644 --- a/consul/state/state_store_test.go +++ b/consul/state/state_store_test.go @@ -226,7 +226,7 @@ func TestStateStore_ReapTombstones(t *testing.T) { t.Fatalf("err: %s", err) } - // At this point the index will slide backwards. + // At this point the sub index will slide backwards. idx, _, err = s.KVSList("foo/") if err != nil { t.Fatalf("err: %s", err) @@ -234,6 +234,17 @@ func TestStateStore_ReapTombstones(t *testing.T) { if idx != 5 { t.Fatalf("bad index: %d", idx) } + + // Make sure the tombstones are actually gone. + snap := s.Snapshot() + defer snap.Close() + dump, err := snap.TombstoneDump() + if err != nil { + t.Fatalf("err: %s", err) + } + if len(dump) != 0 { + t.Fatalf("bad: %#v", dump) + } } func TestStateStore_GetWatches(t *testing.T) { @@ -1876,9 +1887,9 @@ func TestStateStore_KVSSet_KVSGet(t *testing.T) { s := testStateStore(t) // Get on an nonexistent key returns nil. - result, err := s.KVSGet("foo") - if result != nil || err != nil { - t.Fatalf("expected (nil, nil), got : (%#v, %#v)", result, err) + idx, result, err := s.KVSGet("foo") + if result != nil || err != nil || idx != 0 { + t.Fatalf("expected (0, nil, nil), got : (%#v, %#v, %#v)", idx, result, err) } // Write a new K/V entry to the store. @@ -1891,13 +1902,16 @@ func TestStateStore_KVSSet_KVSGet(t *testing.T) { } // Retrieve the K/V entry again. - result, err = s.KVSGet("foo") + idx, result, err = s.KVSGet("foo") if err != nil { t.Fatalf("err: %s", err) } if result == nil { t.Fatalf("expected k/v pair, got nothing") } + if idx != 1 { + t.Fatalf("bad index: %d", idx) + } // Check that the index was injected into the result. if result.CreateIndex != 1 || result.ModifyIndex != 1 { @@ -1909,11 +1923,6 @@ func TestStateStore_KVSSet_KVSGet(t *testing.T) { t.Fatalf("expected 'bar', got: '%s'", v) } - // Index was updated. - if idx := s.maxIndex("kvs"); idx != 1 { - t.Fatalf("bad index: %d", idx) - } - // Updating the entry works and changes the index. update := &structs.DirEntry{ Key: "foo", @@ -1924,7 +1933,7 @@ func TestStateStore_KVSSet_KVSGet(t *testing.T) { } // Fetch the kv pair and check. - result, err = s.KVSGet("foo") + idx, result, err = s.KVSGet("foo") if err != nil { t.Fatalf("err: %s", err) } @@ -1934,9 +1943,7 @@ func TestStateStore_KVSSet_KVSGet(t *testing.T) { if v := string(result.Value); v != "baz" { t.Fatalf("expected 'baz', got '%s'", v) } - - // Index was updated. - if idx := s.maxIndex("kvs"); idx != 2 { + if idx != 2 { t.Fatalf("bad index: %d", idx) } @@ -1951,7 +1958,7 @@ func TestStateStore_KVSSet_KVSGet(t *testing.T) { } // Fetch the kv pair and check. - result, err = s.KVSGet("foo") + idx, result, err = s.KVSGet("foo") if err != nil { t.Fatalf("err: %s", err) } @@ -1964,9 +1971,7 @@ func TestStateStore_KVSSet_KVSGet(t *testing.T) { if result.Session != "" { t.Fatalf("expected empty session, got '%s", result.Session) } - - // Index was updated. - if idx := s.maxIndex("kvs"); idx != 3 { + if idx != 3 { t.Fatalf("bad index: %d", idx) } @@ -1986,7 +1991,7 @@ func TestStateStore_KVSSet_KVSGet(t *testing.T) { } // Fetch the kv pair and check. - result, err = s.KVSGet("foo") + idx, result, err = s.KVSGet("foo") if err != nil { t.Fatalf("err: %s", err) } @@ -1999,9 +2004,7 @@ func TestStateStore_KVSSet_KVSGet(t *testing.T) { if result.Session != "session1" { t.Fatalf("expected session, got '%s", result.Session) } - - // Index was updated. - if idx := s.maxIndex("kvs"); idx != 6 { + if idx != 6 { t.Fatalf("bad index: %d", idx) } @@ -2016,7 +2019,7 @@ func TestStateStore_KVSSet_KVSGet(t *testing.T) { } // Fetch the kv pair and check. - result, err = s.KVSGet("foo") + idx, result, err = s.KVSGet("foo") if err != nil { t.Fatalf("err: %s", err) } @@ -2029,11 +2032,16 @@ func TestStateStore_KVSSet_KVSGet(t *testing.T) { if result.Session != "session1" { t.Fatalf("expected session, got '%s", result.Session) } - - // Index was updated. - if idx := s.maxIndex("kvs"); idx != 7 { + if idx != 7 { t.Fatalf("bad index: %d", idx) } + + // Fetch a key that doesn't exist and make sure we get the right + // response. + idx, result, err = s.KVSGet("nope") + if result != nil || err != nil || idx != 7 { + t.Fatalf("expected (7, nil, nil), got : (%#v, %#v, %#v)", idx, result, err) + } } func TestStateStore_KVSList(t *testing.T) { @@ -2057,8 +2065,6 @@ func TestStateStore_KVSList(t *testing.T) { if err != nil { t.Fatalf("err: %s", err) } - - // Check the index if idx != 5 { t.Fatalf("bad index: %d", idx) } @@ -2097,8 +2103,20 @@ func TestStateStore_KVSList(t *testing.T) { t.Fatalf("bad index: %d", idx) } - // Now reap the tombstones and make sure we get zero for the index - // if there are no matching keys. + // Set a different key to bump the index. + testSetKey(t, s, 7, "some/other/key", "") + + // Make sure we get the right index from the tombstone. + idx, _, err = s.KVSList("foo/bar/baz") + if err != nil { + t.Fatalf("err: %s", err) + } + if idx != 6 { + t.Fatalf("bad index: %d", idx) + } + + // Now reap the tombstones and make sure we get the latest index + // since there are no matching keys. if err := s.ReapTombstones(6); err != nil { t.Fatalf("err: %s", err) } @@ -2106,7 +2124,7 @@ func TestStateStore_KVSList(t *testing.T) { if err != nil { t.Fatalf("err: %s", err) } - if idx != 0 { + if idx != 7 { t.Fatalf("bad index: %d", idx) } } @@ -2170,8 +2188,20 @@ func TestStateStore_KVSListKeys(t *testing.T) { t.Fatalf("bad index: %d", idx) } - // Now reap the tombstones and make sure we get zero for the index - // if there are no matching keys. + // Set a different key to bump the index. + testSetKey(t, s, 9, "some/other/key", "") + + // Make sure the index still comes from the tombstone. + idx, _, err = s.KVSListKeys("foo/bar/baz", "") + if err != nil { + t.Fatalf("err: %s", err) + } + if idx != 8 { + t.Fatalf("bad index: %d", idx) + } + + // Now reap the tombstones and make sure we get the latest index + // since there are no matching keys. if err := s.ReapTombstones(8); err != nil { t.Fatalf("err: %s", err) } @@ -2179,7 +2209,7 @@ func TestStateStore_KVSListKeys(t *testing.T) { if err != nil { t.Fatalf("err: %s", err) } - if idx != 0 { + if idx != 9 { t.Fatalf("bad index: %d", idx) } } @@ -2270,16 +2300,16 @@ func TestStateStore_KVSDeleteCAS(t *testing.T) { // Check that the index is untouched and the entry // has not been deleted. - if idx := s.maxIndex("kvs"); idx != 3 { - t.Fatalf("bad index: %d", idx) - } - e, err := s.KVSGet("foo") + idx, e, err := s.KVSGet("foo") if err != nil { t.Fatalf("err: %s", err) } if e == nil { t.Fatalf("expected a kvs entry, got nil") } + if idx != 3 { + t.Fatalf("bad index: %d", idx) + } // Do another CAS delete, this time with the correct index // which should cause the delete to take place. @@ -2289,20 +2319,23 @@ func TestStateStore_KVSDeleteCAS(t *testing.T) { } // Entry was deleted and index was updated - if idx := s.maxIndex("kvs"); idx != 4 { - t.Fatalf("bad index: %d", idx) - } - e, err = s.KVSGet("bar") + idx, e, err = s.KVSGet("bar") if err != nil { t.Fatalf("err: %s", err) } if e != nil { t.Fatalf("entry should be deleted") } + if idx != 4 { + t.Fatalf("bad index: %d", idx) + } + + // Add another key to bump the index. + testSetKey(t, s, 5, "some/other/key", "baz") // Check that the tombstone was created and that prevents the index // from sliding backwards. - idx, _, err := s.KVSList("bar") + idx, _, err = s.KVSList("bar") if err != nil { t.Fatalf("err: %s", err) } @@ -2310,8 +2343,8 @@ func TestStateStore_KVSDeleteCAS(t *testing.T) { t.Fatalf("bad index: %d", idx) } - // Now reap the tombstone and watch the index revert to zero since - // there are no keys with this prefix. + // Now reap the tombstone and watch the index move up to the table + // index since there are no matching keys. if err := s.ReapTombstones(4); err != nil { t.Fatalf("err: %s", err) } @@ -2319,17 +2352,17 @@ func TestStateStore_KVSDeleteCAS(t *testing.T) { if err != nil { t.Fatalf("err: %s", err) } - if idx != 0 { + if idx != 5 { t.Fatalf("bad index: %d", idx) } // A delete on a nonexistent key should be idempotent and not return an // error - ok, err = s.KVSDeleteCAS(5, 2, "bar") + ok, err = s.KVSDeleteCAS(6, 2, "bar") if !ok || err != nil { t.Fatalf("expected (true, nil), got: (%v, %#v)", ok, err) } - if idx := s.maxIndex("kvs"); idx != 4 { + if idx := s.maxIndex("kvs"); idx != 5 { t.Fatalf("bad index: %d", idx) } } @@ -2380,16 +2413,14 @@ func TestStateStore_KVSSetCAS(t *testing.T) { } // Entry was inserted - entry, err = s.KVSGet("foo") + idx, entry, err := s.KVSGet("foo") if err != nil { t.Fatalf("err: %s", err) } if string(entry.Value) != "foo" || entry.CreateIndex != 2 || entry.ModifyIndex != 2 { t.Fatalf("bad entry: %#v", entry) } - - // Index was updated - if idx := s.maxIndex("kvs"); idx != 2 { + if idx != 2 { t.Fatalf("bad index: %d", idx) } @@ -2424,16 +2455,14 @@ func TestStateStore_KVSSetCAS(t *testing.T) { } // Entry was not updated in the store - entry, err = s.KVSGet("foo") + idx, entry, err = s.KVSGet("foo") if err != nil { t.Fatalf("err: %s", err) } if string(entry.Value) != "foo" || entry.CreateIndex != 2 || entry.ModifyIndex != 2 { t.Fatalf("bad entry: %#v", entry) } - - // Index was not modified - if idx := s.maxIndex("kvs"); idx != 2 { + if idx != 2 { t.Fatalf("bad index: %d", idx) } @@ -2453,16 +2482,14 @@ func TestStateStore_KVSSetCAS(t *testing.T) { } // Entry was updated - entry, err = s.KVSGet("foo") + idx, entry, err = s.KVSGet("foo") if err != nil { t.Fatalf("err: %s", err) } if string(entry.Value) != "bar" || entry.CreateIndex != 2 || entry.ModifyIndex != 3 { t.Fatalf("bad entry: %#v", entry) } - - // Index was updated - if idx := s.maxIndex("kvs"); idx != 3 { + if idx != 3 { t.Fatalf("bad index: %d", idx) } @@ -2482,7 +2509,7 @@ func TestStateStore_KVSSetCAS(t *testing.T) { } // Entry was updated, but the session should have been ignored. - entry, err = s.KVSGet("foo") + idx, entry, err = s.KVSGet("foo") if err != nil { t.Fatalf("err: %s", err) } @@ -2490,9 +2517,7 @@ func TestStateStore_KVSSetCAS(t *testing.T) { entry.Session != "" { t.Fatalf("bad entry: %#v", entry) } - - // Index was updated - if idx := s.maxIndex("kvs"); idx != 4 { + if idx != 4 { t.Fatalf("bad index: %d", idx) } @@ -2528,7 +2553,7 @@ func TestStateStore_KVSSetCAS(t *testing.T) { } // Entry was updated, and the lock status should have stayed the same. - entry, err = s.KVSGet("foo") + idx, entry, err = s.KVSGet("foo") if err != nil { t.Fatalf("err: %s", err) } @@ -2536,9 +2561,7 @@ func TestStateStore_KVSSetCAS(t *testing.T) { entry.Session != "session1" { t.Fatalf("bad entry: %#v", entry) } - - // Index was updated - if idx := s.maxIndex("kvs"); idx != 7 { + if idx != 7 { t.Fatalf("bad index: %d", idx) } } @@ -2614,7 +2637,6 @@ func TestStateStore_KVSDeleteTree(t *testing.T) { if idx != 4 { t.Fatalf("bad index: %d", idx) } - } func TestStateStore_KVSLockDelay(t *testing.T) { @@ -2656,7 +2678,7 @@ func TestStateStore_KVSLock(t *testing.T) { } // Make sure the indexes got set properly. - result, err := s.KVSGet("foo") + idx, result, err := s.KVSGet("foo") if err != nil { t.Fatalf("err: %s", err) } @@ -2664,7 +2686,7 @@ func TestStateStore_KVSLock(t *testing.T) { string(result.Value) != "foo" { t.Fatalf("bad entry: %#v", result) } - if idx := s.maxIndex("kvs"); idx != 4 { + if idx != 4 { t.Fatalf("bad index: %d", idx) } @@ -2677,7 +2699,7 @@ func TestStateStore_KVSLock(t *testing.T) { // Make sure the indexes got set properly, note that the lock index // won't go up since we didn't lock it again. - result, err = s.KVSGet("foo") + idx, result, err = s.KVSGet("foo") if err != nil { t.Fatalf("err: %s", err) } @@ -2685,7 +2707,7 @@ func TestStateStore_KVSLock(t *testing.T) { string(result.Value) != "bar" { t.Fatalf("bad entry: %#v", result) } - if idx := s.maxIndex("kvs"); idx != 5 { + if idx != 5 { t.Fatalf("bad index: %d", idx) } @@ -2700,7 +2722,7 @@ func TestStateStore_KVSLock(t *testing.T) { } // Make sure the indexes got set properly. - result, err = s.KVSGet("foo") + idx, result, err = s.KVSGet("foo") if err != nil { t.Fatalf("err: %s", err) } @@ -2708,7 +2730,7 @@ func TestStateStore_KVSLock(t *testing.T) { string(result.Value) != "zoo" { t.Fatalf("bad entry: %#v", result) } - if idx := s.maxIndex("kvs"); idx != 7 { + if idx != 7 { t.Fatalf("bad index: %d", idx) } @@ -2720,7 +2742,7 @@ func TestStateStore_KVSLock(t *testing.T) { } // Make sure the indexes got set properly. - result, err = s.KVSGet("bar") + idx, result, err = s.KVSGet("bar") if err != nil { t.Fatalf("err: %s", err) } @@ -2728,7 +2750,7 @@ func TestStateStore_KVSLock(t *testing.T) { string(result.Value) != "xxx" { t.Fatalf("bad entry: %#v", result) } - if idx := s.maxIndex("kvs"); idx != 9 { + if idx != 9 { t.Fatalf("bad index: %d", idx) } @@ -2745,7 +2767,7 @@ func TestStateStore_KVSLock(t *testing.T) { } // Make sure the indexes didn't update. - result, err = s.KVSGet("bar") + idx, result, err = s.KVSGet("bar") if err != nil { t.Fatalf("err: %s", err) } @@ -2753,7 +2775,7 @@ func TestStateStore_KVSLock(t *testing.T) { string(result.Value) != "xxx" { t.Fatalf("bad entry: %#v", result) } - if idx := s.maxIndex("kvs"); idx != 9 { + if idx != 9 { t.Fatalf("bad index: %d", idx) } } @@ -2788,7 +2810,7 @@ func TestStateStore_KVSUnlock(t *testing.T) { } // Make sure the indexes didn't update. - result, err := s.KVSGet("foo") + idx, result, err := s.KVSGet("foo") if err != nil { t.Fatalf("err: %s", err) } @@ -2796,7 +2818,7 @@ func TestStateStore_KVSUnlock(t *testing.T) { string(result.Value) != "bar" { t.Fatalf("bad entry: %#v", result) } - if idx := s.maxIndex("kvs"); idx != 4 { + if idx != 4 { t.Fatalf("bad index: %d", idx) } @@ -2816,7 +2838,7 @@ func TestStateStore_KVSUnlock(t *testing.T) { } // Make sure the indexes didn't update. - result, err = s.KVSGet("foo") + idx, result, err = s.KVSGet("foo") if err != nil { t.Fatalf("err: %s", err) } @@ -2824,7 +2846,7 @@ func TestStateStore_KVSUnlock(t *testing.T) { string(result.Value) != "bar" { t.Fatalf("bad entry: %#v", result) } - if idx := s.maxIndex("kvs"); idx != 6 { + if idx != 6 { t.Fatalf("bad index: %d", idx) } @@ -2835,7 +2857,7 @@ func TestStateStore_KVSUnlock(t *testing.T) { } // Make sure the indexes got set properly. - result, err = s.KVSGet("foo") + idx, result, err = s.KVSGet("foo") if err != nil { t.Fatalf("err: %s", err) } @@ -2843,7 +2865,7 @@ func TestStateStore_KVSUnlock(t *testing.T) { string(result.Value) != "zoo" { t.Fatalf("bad entry: %#v", result) } - if idx := s.maxIndex("kvs"); idx != 9 { + if idx != 9 { t.Fatalf("bad index: %d", idx) } @@ -2854,7 +2876,7 @@ func TestStateStore_KVSUnlock(t *testing.T) { } // Make sure the indexes didn't update. - result, err = s.KVSGet("foo") + idx, result, err = s.KVSGet("foo") if err != nil { t.Fatalf("err: %s", err) } @@ -2862,7 +2884,7 @@ func TestStateStore_KVSUnlock(t *testing.T) { string(result.Value) != "zoo" { t.Fatalf("bad entry: %#v", result) } - if idx := s.maxIndex("kvs"); idx != 9 { + if idx != 9 { t.Fatalf("bad index: %d", idx) } } @@ -3094,7 +3116,9 @@ func TestStateStore_Tombstone_Snapshot_Restore(t *testing.T) { // Insert a key and then delete it to create a tombstone. testSetKey(t, s, 1, "foo/bar", "bar") - if err := s.KVSDelete(2, "foo/bar"); err != nil { + testSetKey(t, s, 2, "foo/bar/baz", "bar") + testSetKey(t, s, 3, "foo/bar/zoo", "bar") + if err := s.KVSDelete(4, "foo/bar"); err != nil { t.Fatalf("err: %s", err) } @@ -3103,14 +3127,14 @@ func TestStateStore_Tombstone_Snapshot_Restore(t *testing.T) { defer snap.Close() // Alter the real state store. - if err := s.ReapTombstones(2); err != nil { + if err := s.ReapTombstones(4); err != nil { t.Fatalf("err: %s", err) } idx, _, err := s.KVSList("foo/bar") if err != nil { t.Fatalf("err: %s", err) } - if idx != 0 { + if idx != 3 { t.Fatalf("bad index: %d", idx) } @@ -3123,7 +3147,7 @@ func TestStateStore_Tombstone_Snapshot_Restore(t *testing.T) { t.Fatalf("bad %#v", dump) } stone := dump[0] - if stone.Key != "foo/bar" || stone.Index != 2 { + if stone.Key != "foo/bar" || stone.Index != 4 { t.Fatalf("bad: %#v", stone) } @@ -3142,21 +3166,34 @@ func TestStateStore_Tombstone_Snapshot_Restore(t *testing.T) { if err != nil { t.Fatalf("err: %s", err) } - if idx != 2 { + if idx != 4 { t.Fatalf("bad index: %d", idx) } - // Make sure it reaps correctly. - if err := s.ReapTombstones(2); err != nil { + // Make sure it reaps correctly. We should still get a 4 for + // the index here because it will be using the last index from + // the tombstone table. + if err := s.ReapTombstones(4); err != nil { t.Fatalf("err: %s", err) } idx, _, err = s.KVSList("foo/bar") if err != nil { t.Fatalf("err: %s", err) } - if idx != 0 { + if idx != 4 { t.Fatalf("bad index: %d", idx) } + + // But make sure the tombstone is actually gone. + snap := s.Snapshot() + defer snap.Close() + dump, err := snap.TombstoneDump() + if err != nil { + t.Fatalf("err: %s", err) + } + if len(dump) != 0 { + t.Fatalf("bad %#v", dump) + } }() } @@ -3835,7 +3872,7 @@ func TestStateStore_Session_Invalidate_Key_Unlock_Behavior(t *testing.T) { } // Key should be unlocked. - d2, err := s.KVSGet("/foo") + idx, d2, err := s.KVSGet("/foo") if err != nil { t.Fatalf("err: %s", err) } @@ -3848,6 +3885,9 @@ func TestStateStore_Session_Invalidate_Key_Unlock_Behavior(t *testing.T) { if d2.Session != "" { t.Fatalf("bad: %v", *d2) } + if idx != 6 { + t.Fatalf("bad index: %d", idx) + } // Key should have a lock delay. expires := s.KVSLockDelay("/foo") @@ -3912,12 +3952,15 @@ func TestStateStore_Session_Invalidate_Key_Delete_Behavior(t *testing.T) { } // Key should be deleted. - d2, err := s.KVSGet("/bar") + idx, d2, err := s.KVSGet("/bar") if err != nil { t.Fatalf("err: %s", err) } if d2 != nil { - t.Fatalf("unexpected undeleted key") + t.Fatalf("unexpected deleted key") + } + if idx != 6 { + t.Fatalf("bad index: %d", idx) } // Key should have a lock delay.