mirror of https://github.com/status-im/consul.git
1539 lines
44 KiB
Go
1539 lines
44 KiB
Go
package cache
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"sort"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/mock"
|
|
"github.com/stretchr/testify/require"
|
|
"golang.org/x/time/rate"
|
|
|
|
"github.com/hashicorp/consul/lib/ttlcache"
|
|
"github.com/hashicorp/consul/sdk/testutil"
|
|
)
|
|
|
|
// Test a basic Get with no indexes (and therefore no blocking queries).
|
|
func TestCacheGet_noIndex(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
require := require.New(t)
|
|
|
|
typ := TestType(t)
|
|
defer typ.AssertExpectations(t)
|
|
c := New(Options{})
|
|
c.RegisterType("t", typ)
|
|
|
|
// Configure the type
|
|
typ.Static(FetchResult{Value: 42}, nil).Times(1)
|
|
|
|
// Get, should fetch
|
|
req := TestRequest(t, RequestInfo{Key: "hello"})
|
|
result, meta, err := c.Get(context.Background(), "t", req)
|
|
require.NoError(err)
|
|
require.Equal(42, result)
|
|
require.False(meta.Hit)
|
|
|
|
// Get, should not fetch since we already have a satisfying value
|
|
result, meta, err = c.Get(context.Background(), "t", req)
|
|
require.NoError(err)
|
|
require.Equal(42, result)
|
|
require.True(meta.Hit)
|
|
|
|
// Sleep a tiny bit just to let maybe some background calls happen
|
|
// then verify that we still only got the one call
|
|
time.Sleep(20 * time.Millisecond)
|
|
typ.AssertExpectations(t)
|
|
}
|
|
|
|
// Test a basic Get with no index and a failed fetch.
|
|
func TestCacheGet_initError(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
require := require.New(t)
|
|
|
|
typ := TestType(t)
|
|
defer typ.AssertExpectations(t)
|
|
c := New(Options{})
|
|
c.RegisterType("t", typ)
|
|
|
|
// Configure the type
|
|
fetcherr := fmt.Errorf("error")
|
|
typ.Static(FetchResult{}, fetcherr).Times(2)
|
|
|
|
// Get, should fetch
|
|
req := TestRequest(t, RequestInfo{Key: "hello"})
|
|
result, meta, err := c.Get(context.Background(), "t", req)
|
|
require.Error(err)
|
|
require.Nil(result)
|
|
require.False(meta.Hit)
|
|
|
|
// Get, should fetch again since our last fetch was an error
|
|
result, meta, err = c.Get(context.Background(), "t", req)
|
|
require.Error(err)
|
|
require.Nil(result)
|
|
require.False(meta.Hit)
|
|
|
|
// Sleep a tiny bit just to let maybe some background calls happen
|
|
// then verify that we still only got the one call
|
|
time.Sleep(20 * time.Millisecond)
|
|
typ.AssertExpectations(t)
|
|
}
|
|
|
|
// Test a cached error is replaced by a successful result. See
|
|
// https://github.com/hashicorp/consul/issues/4480
|
|
func TestCacheGet_cachedErrorsDontStick(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("too slow for testing.Short")
|
|
}
|
|
|
|
t.Parallel()
|
|
|
|
require := require.New(t)
|
|
|
|
typ := TestType(t)
|
|
defer typ.AssertExpectations(t)
|
|
c := New(Options{})
|
|
c.RegisterType("t", typ)
|
|
|
|
// Configure the type
|
|
fetcherr := fmt.Errorf("initial error")
|
|
// First fetch errors, subsequent fetches are successful and then block
|
|
typ.Static(FetchResult{}, fetcherr).Times(1)
|
|
typ.Static(FetchResult{Value: 42, Index: 123}, nil).Times(1)
|
|
// We trigger this to return same value to simulate a timeout.
|
|
triggerCh := make(chan time.Time)
|
|
typ.Static(FetchResult{Value: 42, Index: 123}, nil).WaitUntil(triggerCh)
|
|
|
|
// Get, should fetch and get error
|
|
req := TestRequest(t, RequestInfo{Key: "hello"})
|
|
result, meta, err := c.Get(context.Background(), "t", req)
|
|
require.Error(err)
|
|
require.Nil(result)
|
|
require.False(meta.Hit)
|
|
|
|
// Get, should fetch again since our last fetch was an error, but get success
|
|
result, meta, err = c.Get(context.Background(), "t", req)
|
|
require.NoError(err)
|
|
require.Equal(42, result)
|
|
require.False(meta.Hit)
|
|
|
|
// Now get should block until timeout and then get the same response NOT the
|
|
// cached error.
|
|
getCh1 := TestCacheGetCh(t, c, "t", TestRequest(t, RequestInfo{
|
|
Key: "hello",
|
|
MinIndex: 123,
|
|
// We _don't_ set a timeout here since that doesn't trigger the bug - the
|
|
// bug occurs when the Fetch call times out and returns the same value when
|
|
// an error is set. If it returns a new value the blocking loop works too.
|
|
}))
|
|
time.AfterFunc(50*time.Millisecond, func() {
|
|
// "Timeout" the Fetch after a short time.
|
|
close(triggerCh)
|
|
})
|
|
select {
|
|
case result := <-getCh1:
|
|
t.Fatalf("result or error returned before an update happened. "+
|
|
"If this is nil look above for the error log: %v", result)
|
|
case <-time.After(100 * time.Millisecond):
|
|
// It _should_ keep blocking for a new value here
|
|
}
|
|
|
|
// Sleep a tiny bit just to let maybe some background calls happen
|
|
// then verify the calls.
|
|
time.Sleep(20 * time.Millisecond)
|
|
typ.AssertExpectations(t)
|
|
}
|
|
|
|
// Test a Get with a request that returns a blank cache key. This should
|
|
// force a backend request and skip the cache entirely.
|
|
func TestCacheGet_blankCacheKey(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
require := require.New(t)
|
|
|
|
typ := TestType(t)
|
|
defer typ.AssertExpectations(t)
|
|
c := New(Options{})
|
|
c.RegisterType("t", typ)
|
|
|
|
// Configure the type
|
|
typ.Static(FetchResult{Value: 42}, nil).Times(2)
|
|
|
|
// Get, should fetch
|
|
req := TestRequest(t, RequestInfo{Key: ""})
|
|
result, meta, err := c.Get(context.Background(), "t", req)
|
|
require.NoError(err)
|
|
require.Equal(42, result)
|
|
require.False(meta.Hit)
|
|
|
|
// Get, should not fetch since we already have a satisfying value
|
|
result, meta, err = c.Get(context.Background(), "t", req)
|
|
require.NoError(err)
|
|
require.Equal(42, result)
|
|
require.False(meta.Hit)
|
|
|
|
// Sleep a tiny bit just to let maybe some background calls happen
|
|
// then verify that we still only got the one call
|
|
time.Sleep(20 * time.Millisecond)
|
|
typ.AssertExpectations(t)
|
|
}
|
|
|
|
// Test that Get blocks on the initial value
|
|
func TestCacheGet_blockingInitSameKey(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
typ := TestType(t)
|
|
defer typ.AssertExpectations(t)
|
|
c := New(Options{})
|
|
c.RegisterType("t", typ)
|
|
|
|
// Configure the type
|
|
triggerCh := make(chan time.Time)
|
|
typ.Static(FetchResult{Value: 42}, nil).WaitUntil(triggerCh).Times(1)
|
|
|
|
// Perform multiple gets
|
|
getCh1 := TestCacheGetCh(t, c, "t", TestRequest(t, RequestInfo{Key: "hello"}))
|
|
getCh2 := TestCacheGetCh(t, c, "t", TestRequest(t, RequestInfo{Key: "hello"}))
|
|
|
|
// They should block
|
|
select {
|
|
case <-getCh1:
|
|
t.Fatal("should block (ch1)")
|
|
case <-getCh2:
|
|
t.Fatal("should block (ch2)")
|
|
case <-time.After(50 * time.Millisecond):
|
|
}
|
|
|
|
// Trigger it
|
|
close(triggerCh)
|
|
|
|
// Should return
|
|
TestCacheGetChResult(t, getCh1, 42)
|
|
TestCacheGetChResult(t, getCh2, 42)
|
|
}
|
|
|
|
// Test that Get with different cache keys both block on initial value
|
|
// but that the fetches were both properly called.
|
|
func TestCacheGet_blockingInitDiffKeys(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
require := require.New(t)
|
|
|
|
typ := TestType(t)
|
|
defer typ.AssertExpectations(t)
|
|
c := New(Options{})
|
|
c.RegisterType("t", typ)
|
|
|
|
// Keep track of the keys
|
|
var keysLock sync.Mutex
|
|
var keys []string
|
|
|
|
// Configure the type
|
|
triggerCh := make(chan time.Time)
|
|
typ.Static(FetchResult{Value: 42}, nil).
|
|
WaitUntil(triggerCh).
|
|
Times(2).
|
|
Run(func(args mock.Arguments) {
|
|
keysLock.Lock()
|
|
defer keysLock.Unlock()
|
|
keys = append(keys, args.Get(1).(Request).CacheInfo().Key)
|
|
})
|
|
|
|
// Perform multiple gets
|
|
getCh1 := TestCacheGetCh(t, c, "t", TestRequest(t, RequestInfo{Key: "hello"}))
|
|
getCh2 := TestCacheGetCh(t, c, "t", TestRequest(t, RequestInfo{Key: "goodbye"}))
|
|
|
|
// They should block
|
|
select {
|
|
case <-getCh1:
|
|
t.Fatal("should block (ch1)")
|
|
case <-getCh2:
|
|
t.Fatal("should block (ch2)")
|
|
case <-time.After(50 * time.Millisecond):
|
|
}
|
|
|
|
// Trigger it
|
|
close(triggerCh)
|
|
|
|
// Should return both!
|
|
TestCacheGetChResult(t, getCh1, 42)
|
|
TestCacheGetChResult(t, getCh2, 42)
|
|
|
|
// Verify proper keys
|
|
sort.Strings(keys)
|
|
require.Equal([]string{"goodbye", "hello"}, keys)
|
|
}
|
|
|
|
// Test a get with an index set will wait until an index that is higher
|
|
// is set in the cache.
|
|
func TestCacheGet_blockingIndex(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
typ := TestType(t)
|
|
defer typ.AssertExpectations(t)
|
|
c := New(Options{})
|
|
c.RegisterType("t", typ)
|
|
|
|
// Configure the type
|
|
triggerCh := make(chan time.Time)
|
|
typ.Static(FetchResult{Value: 1, Index: 4}, nil).Once()
|
|
typ.Static(FetchResult{Value: 12, Index: 5}, nil).Once()
|
|
typ.Static(FetchResult{Value: 42, Index: 6}, nil).WaitUntil(triggerCh)
|
|
|
|
// Fetch should block
|
|
resultCh := TestCacheGetCh(t, c, "t", TestRequest(t, RequestInfo{
|
|
Key: "hello", MinIndex: 5}))
|
|
|
|
// Should block
|
|
select {
|
|
case <-resultCh:
|
|
t.Fatal("should block")
|
|
case <-time.After(50 * time.Millisecond):
|
|
}
|
|
|
|
// Wait a bit
|
|
close(triggerCh)
|
|
|
|
// Should return
|
|
TestCacheGetChResult(t, resultCh, 42)
|
|
}
|
|
|
|
func TestCacheGet_cancellation(t *testing.T) {
|
|
typ := TestType(t)
|
|
defer typ.AssertExpectations(t)
|
|
c := New(Options{})
|
|
c.RegisterType("t", typ)
|
|
|
|
typ.Static(FetchResult{Value: 1, Index: 4}, nil).Times(0).WaitUntil(time.After(1 * time.Millisecond))
|
|
|
|
ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(50*time.Millisecond))
|
|
// this is just to keep the linter happy
|
|
defer cancel()
|
|
|
|
result, _, err := c.Get(ctx, "t", TestRequest(t, RequestInfo{
|
|
Key: "hello", MinIndex: 5}))
|
|
|
|
require.Nil(t, result)
|
|
require.Error(t, err)
|
|
testutil.RequireErrorContains(t, err, context.DeadlineExceeded.Error())
|
|
}
|
|
|
|
// Test a get with an index set will timeout if the fetch doesn't return
|
|
// anything.
|
|
func TestCacheGet_blockingIndexTimeout(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("too slow for testing.Short")
|
|
}
|
|
|
|
t.Parallel()
|
|
|
|
typ := TestType(t)
|
|
defer typ.AssertExpectations(t)
|
|
c := New(Options{})
|
|
c.RegisterType("t", typ)
|
|
|
|
// Configure the type
|
|
triggerCh := make(chan time.Time)
|
|
typ.Static(FetchResult{Value: 1, Index: 4}, nil).Once()
|
|
typ.Static(FetchResult{Value: 12, Index: 5}, nil).Once()
|
|
typ.Static(FetchResult{Value: 42, Index: 6}, nil).WaitUntil(triggerCh)
|
|
|
|
// Fetch should block
|
|
resultCh := TestCacheGetCh(t, c, "t", TestRequest(t, RequestInfo{
|
|
Key: "hello", MinIndex: 5, Timeout: 200 * time.Millisecond}))
|
|
|
|
// Should block
|
|
select {
|
|
case <-resultCh:
|
|
t.Fatal("should block")
|
|
case <-time.After(50 * time.Millisecond):
|
|
}
|
|
|
|
// Should return after more of the timeout
|
|
select {
|
|
case result := <-resultCh:
|
|
require.Equal(t, 12, result)
|
|
case <-time.After(300 * time.Millisecond):
|
|
t.Fatal("should've returned")
|
|
}
|
|
}
|
|
|
|
// Test a get with an index set with requests returning an error
|
|
// will return that error.
|
|
func TestCacheGet_blockingIndexError(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("too slow for testing.Short")
|
|
}
|
|
|
|
t.Parallel()
|
|
|
|
typ := TestType(t)
|
|
defer typ.AssertExpectations(t)
|
|
c := New(Options{})
|
|
c.RegisterType("t", typ)
|
|
|
|
// Configure the type
|
|
var retries uint32
|
|
fetchErr := fmt.Errorf("test fetch error")
|
|
typ.Static(FetchResult{Value: 1, Index: 4}, nil).Once()
|
|
typ.Static(FetchResult{Value: nil, Index: 5}, fetchErr).Run(func(args mock.Arguments) {
|
|
atomic.AddUint32(&retries, 1)
|
|
})
|
|
|
|
// First good fetch to populate catch
|
|
resultCh := TestCacheGetCh(t, c, "t", TestRequest(t, RequestInfo{Key: "hello"}))
|
|
TestCacheGetChResult(t, resultCh, 1)
|
|
|
|
// Fetch should not block and should return error
|
|
resultCh = TestCacheGetCh(t, c, "t", TestRequest(t, RequestInfo{
|
|
Key: "hello", MinIndex: 7, Timeout: 1 * time.Minute}))
|
|
TestCacheGetChResult(t, resultCh, nil)
|
|
|
|
// Wait a bit
|
|
time.Sleep(100 * time.Millisecond)
|
|
|
|
// Check the number
|
|
actual := atomic.LoadUint32(&retries)
|
|
require.True(t, actual < 10, fmt.Sprintf("actual: %d", actual))
|
|
}
|
|
|
|
// Test that if a Type returns an empty value on Fetch that the previous
|
|
// value is preserved.
|
|
func TestCacheGet_emptyFetchResult(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("too slow for testing.Short")
|
|
}
|
|
|
|
t.Parallel()
|
|
|
|
require := require.New(t)
|
|
|
|
typ := TestType(t)
|
|
defer typ.AssertExpectations(t)
|
|
c := New(Options{})
|
|
c.RegisterType("t", typ)
|
|
|
|
stateCh := make(chan int, 1)
|
|
|
|
// Configure the type
|
|
typ.Static(FetchResult{Value: 42, State: 31, Index: 1}, nil).Times(1)
|
|
// Return different State, it should NOT be ignored
|
|
typ.Static(FetchResult{Value: nil, State: 32}, nil).Run(func(args mock.Arguments) {
|
|
// We should get back the original state
|
|
opts := args.Get(0).(FetchOptions)
|
|
require.NotNil(opts.LastResult)
|
|
stateCh <- opts.LastResult.State.(int)
|
|
})
|
|
|
|
// Get, should fetch
|
|
req := TestRequest(t, RequestInfo{Key: "hello"})
|
|
result, meta, err := c.Get(context.Background(), "t", req)
|
|
require.NoError(err)
|
|
require.Equal(42, result)
|
|
require.False(meta.Hit)
|
|
|
|
// Get, should not fetch since we already have a satisfying value
|
|
req = TestRequest(t, RequestInfo{
|
|
Key: "hello", MinIndex: 1, Timeout: 100 * time.Millisecond})
|
|
result, meta, err = c.Get(context.Background(), "t", req)
|
|
require.NoError(err)
|
|
require.Equal(42, result)
|
|
require.False(meta.Hit)
|
|
|
|
// State delivered to second call should be the result from first call.
|
|
select {
|
|
case state := <-stateCh:
|
|
require.Equal(31, state)
|
|
case <-time.After(20 * time.Millisecond):
|
|
t.Fatal("timed out")
|
|
}
|
|
|
|
// Next request should get the SECOND returned state even though the fetch
|
|
// returns nil and so the previous result is used.
|
|
req = TestRequest(t, RequestInfo{
|
|
Key: "hello", MinIndex: 1, Timeout: 100 * time.Millisecond})
|
|
result, meta, err = c.Get(context.Background(), "t", req)
|
|
require.NoError(err)
|
|
require.Equal(42, result)
|
|
require.False(meta.Hit)
|
|
select {
|
|
case state := <-stateCh:
|
|
require.Equal(32, state)
|
|
case <-time.After(20 * time.Millisecond):
|
|
t.Fatal("timed out")
|
|
}
|
|
|
|
// Sleep a tiny bit just to let maybe some background calls happen
|
|
// then verify that we still only got the one call
|
|
time.Sleep(20 * time.Millisecond)
|
|
typ.AssertExpectations(t)
|
|
}
|
|
|
|
// Test that a type registered with a periodic refresh will perform
|
|
// that refresh after the timer is up.
|
|
func TestCacheGet_periodicRefresh(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("too slow for testing.Short")
|
|
}
|
|
|
|
t.Parallel()
|
|
|
|
typ := &MockType{}
|
|
typ.On("RegisterOptions").Return(RegisterOptions{
|
|
Refresh: true,
|
|
RefreshTimer: 100 * time.Millisecond,
|
|
QueryTimeout: 5 * time.Minute,
|
|
})
|
|
defer typ.AssertExpectations(t)
|
|
c := New(Options{})
|
|
c.RegisterType("t", typ)
|
|
|
|
// This is a bit weird, but we do this to ensure that the final
|
|
// call to the Fetch (if it happens, depends on timing) just blocks.
|
|
triggerCh := make(chan time.Time)
|
|
defer close(triggerCh)
|
|
|
|
// Configure the type
|
|
typ.Static(FetchResult{Value: 1, Index: 4}, nil).Once()
|
|
typ.Static(FetchResult{Value: 12, Index: 5}, nil).Once()
|
|
typ.Static(FetchResult{Value: 12, Index: 5}, nil).WaitUntil(triggerCh)
|
|
|
|
// Fetch should block
|
|
resultCh := TestCacheGetCh(t, c, "t", TestRequest(t, RequestInfo{Key: "hello"}))
|
|
TestCacheGetChResult(t, resultCh, 1)
|
|
|
|
// Fetch again almost immediately should return old result
|
|
time.Sleep(5 * time.Millisecond)
|
|
resultCh = TestCacheGetCh(t, c, "t", TestRequest(t, RequestInfo{Key: "hello"}))
|
|
TestCacheGetChResult(t, resultCh, 1)
|
|
|
|
// Wait for the timer
|
|
time.Sleep(200 * time.Millisecond)
|
|
resultCh = TestCacheGetCh(t, c, "t", TestRequest(t, RequestInfo{Key: "hello"}))
|
|
TestCacheGetChResult(t, resultCh, 12)
|
|
}
|
|
|
|
// Test that a type registered with a periodic refresh will perform
|
|
// that refresh after the timer is up.
|
|
func TestCacheGet_periodicRefreshMultiple(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("too slow for testing.Short")
|
|
}
|
|
|
|
t.Parallel()
|
|
|
|
typ := &MockType{}
|
|
typ.On("RegisterOptions").Return(RegisterOptions{
|
|
Refresh: true,
|
|
RefreshTimer: 0 * time.Millisecond,
|
|
QueryTimeout: 5 * time.Minute,
|
|
})
|
|
defer typ.AssertExpectations(t)
|
|
c := New(Options{})
|
|
c.RegisterType("t", typ)
|
|
|
|
// This is a bit weird, but we do this to ensure that the final
|
|
// call to the Fetch (if it happens, depends on timing) just blocks.
|
|
trigger := make([]chan time.Time, 3)
|
|
for i := range trigger {
|
|
trigger[i] = make(chan time.Time)
|
|
}
|
|
|
|
// Configure the type
|
|
typ.Static(FetchResult{Value: 1, Index: 4}, nil).Once()
|
|
typ.Static(FetchResult{Value: 12, Index: 5}, nil).Once().WaitUntil(trigger[0])
|
|
typ.Static(FetchResult{Value: 24, Index: 6}, nil).Once().WaitUntil(trigger[1])
|
|
typ.Static(FetchResult{Value: 42, Index: 7}, nil).WaitUntil(trigger[2])
|
|
|
|
// Fetch should block
|
|
resultCh := TestCacheGetCh(t, c, "t", TestRequest(t, RequestInfo{Key: "hello"}))
|
|
TestCacheGetChResult(t, resultCh, 1)
|
|
|
|
// Fetch again almost immediately should return old result
|
|
time.Sleep(5 * time.Millisecond)
|
|
resultCh = TestCacheGetCh(t, c, "t", TestRequest(t, RequestInfo{Key: "hello"}))
|
|
TestCacheGetChResult(t, resultCh, 1)
|
|
|
|
// Trigger the next, sleep a bit, and verify we get the next result
|
|
close(trigger[0])
|
|
time.Sleep(100 * time.Millisecond)
|
|
resultCh = TestCacheGetCh(t, c, "t", TestRequest(t, RequestInfo{Key: "hello"}))
|
|
TestCacheGetChResult(t, resultCh, 12)
|
|
|
|
// Trigger the next, sleep a bit, and verify we get the next result
|
|
close(trigger[1])
|
|
time.Sleep(100 * time.Millisecond)
|
|
resultCh = TestCacheGetCh(t, c, "t", TestRequest(t, RequestInfo{Key: "hello"}))
|
|
TestCacheGetChResult(t, resultCh, 24)
|
|
}
|
|
|
|
// Test that a refresh performs a backoff.
|
|
func TestCacheGet_periodicRefreshErrorBackoff(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("too slow for testing.Short")
|
|
}
|
|
|
|
t.Parallel()
|
|
|
|
typ := &MockType{}
|
|
typ.On("RegisterOptions").Return(RegisterOptions{
|
|
Refresh: true,
|
|
RefreshTimer: 0,
|
|
QueryTimeout: 5 * time.Minute,
|
|
})
|
|
defer typ.AssertExpectations(t)
|
|
c := New(Options{})
|
|
c.RegisterType("t", typ)
|
|
|
|
// Configure the type
|
|
var retries uint32
|
|
fetchErr := fmt.Errorf("test fetch error")
|
|
typ.Static(FetchResult{Value: 1, Index: 4}, nil).Once()
|
|
typ.Static(FetchResult{Value: nil, Index: 5}, fetchErr).Run(func(args mock.Arguments) {
|
|
atomic.AddUint32(&retries, 1)
|
|
})
|
|
|
|
// Fetch
|
|
resultCh := TestCacheGetCh(t, c, "t", TestRequest(t, RequestInfo{Key: "hello"}))
|
|
TestCacheGetChResult(t, resultCh, 1)
|
|
|
|
// Sleep a bit. The refresh will quietly fail in the background. What we
|
|
// want to verify is that it doesn't retry too much. "Too much" is hard
|
|
// to measure since its CPU dependent if this test is failing. But due
|
|
// to the short sleep below, we can calculate about what we'd expect if
|
|
// backoff IS working.
|
|
time.Sleep(500 * time.Millisecond)
|
|
|
|
// Fetch should work, we should get a 1 still. Errors are ignored.
|
|
resultCh = TestCacheGetCh(t, c, "t", TestRequest(t, RequestInfo{Key: "hello"}))
|
|
TestCacheGetChResult(t, resultCh, 1)
|
|
|
|
// Check the number
|
|
actual := atomic.LoadUint32(&retries)
|
|
require.True(t, actual < 10, fmt.Sprintf("actual: %d", actual))
|
|
}
|
|
|
|
// Test that a badly behaved RPC that returns 0 index will perform a backoff.
|
|
func TestCacheGet_periodicRefreshBadRPCZeroIndexErrorBackoff(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("too slow for testing.Short")
|
|
}
|
|
|
|
t.Parallel()
|
|
|
|
typ := &MockType{}
|
|
typ.On("RegisterOptions").Return(RegisterOptions{
|
|
Refresh: true,
|
|
RefreshTimer: 0,
|
|
QueryTimeout: 5 * time.Minute,
|
|
})
|
|
defer typ.AssertExpectations(t)
|
|
c := New(Options{})
|
|
c.RegisterType("t", typ)
|
|
|
|
// Configure the type
|
|
var retries uint32
|
|
typ.Static(FetchResult{Value: 0, Index: 0}, nil).Run(func(args mock.Arguments) {
|
|
atomic.AddUint32(&retries, 1)
|
|
})
|
|
|
|
// Fetch
|
|
resultCh := TestCacheGetCh(t, c, "t", TestRequest(t, RequestInfo{Key: "hello"}))
|
|
TestCacheGetChResult(t, resultCh, 0)
|
|
|
|
// Sleep a bit. The refresh will quietly fail in the background. What we
|
|
// want to verify is that it doesn't retry too much. "Too much" is hard
|
|
// to measure since its CPU dependent if this test is failing. But due
|
|
// to the short sleep below, we can calculate about what we'd expect if
|
|
// backoff IS working.
|
|
time.Sleep(500 * time.Millisecond)
|
|
|
|
// Fetch should work, we should get a 0 still. Errors are ignored.
|
|
resultCh = TestCacheGetCh(t, c, "t", TestRequest(t, RequestInfo{Key: "hello"}))
|
|
TestCacheGetChResult(t, resultCh, 0)
|
|
|
|
// Check the number
|
|
actual := atomic.LoadUint32(&retries)
|
|
require.True(t, actual < 10, fmt.Sprintf("%d retries, should be < 10", actual))
|
|
}
|
|
|
|
// Test that fetching with no index makes an initial request with no index, but
|
|
// then ensures all background refreshes have > 0. This ensures we don't end up
|
|
// with any index 0 loops from background refreshed while also returning
|
|
// immediately on the initial request if there is no data written to that table
|
|
// yet.
|
|
func TestCacheGet_noIndexSetsOne(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("too slow for testing.Short")
|
|
}
|
|
|
|
t.Parallel()
|
|
|
|
typ := &MockType{}
|
|
typ.On("RegisterOptions").Return(RegisterOptions{
|
|
SupportsBlocking: true,
|
|
Refresh: true,
|
|
RefreshTimer: 0,
|
|
QueryTimeout: 5 * time.Minute,
|
|
})
|
|
defer typ.AssertExpectations(t)
|
|
c := New(Options{})
|
|
c.RegisterType("t", typ)
|
|
|
|
// Simulate "well behaved" RPC with no data yet but returning 1
|
|
{
|
|
first := int32(1)
|
|
|
|
typ.Static(FetchResult{Value: 0, Index: 1}, nil).Run(func(args mock.Arguments) {
|
|
opts := args.Get(0).(FetchOptions)
|
|
isFirst := atomic.SwapInt32(&first, 0)
|
|
if isFirst == 1 {
|
|
assert.Equal(t, uint64(0), opts.MinIndex)
|
|
} else {
|
|
assert.True(t, opts.MinIndex > 0, "minIndex > 0")
|
|
}
|
|
})
|
|
|
|
// Fetch
|
|
resultCh := TestCacheGetCh(t, c, "t", TestRequest(t, RequestInfo{Key: "hello"}))
|
|
TestCacheGetChResult(t, resultCh, 0)
|
|
|
|
// Sleep a bit so background refresh happens
|
|
time.Sleep(100 * time.Millisecond)
|
|
}
|
|
|
|
// Same for "badly behaved" RPC that returns 0 index and no data
|
|
{
|
|
first := int32(1)
|
|
|
|
typ.Static(FetchResult{Value: 0, Index: 0}, nil).Run(func(args mock.Arguments) {
|
|
opts := args.Get(0).(FetchOptions)
|
|
isFirst := atomic.SwapInt32(&first, 0)
|
|
if isFirst == 1 {
|
|
assert.Equal(t, uint64(0), opts.MinIndex)
|
|
} else {
|
|
assert.True(t, opts.MinIndex > 0, "minIndex > 0")
|
|
}
|
|
})
|
|
|
|
// Fetch
|
|
resultCh := TestCacheGetCh(t, c, "t", TestRequest(t, RequestInfo{Key: "hello"}))
|
|
TestCacheGetChResult(t, resultCh, 0)
|
|
|
|
// Sleep a bit so background refresh happens
|
|
time.Sleep(100 * time.Millisecond)
|
|
}
|
|
}
|
|
|
|
// Test that the backend fetch sets the proper timeout.
|
|
func TestCacheGet_fetchTimeout(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
require := require.New(t)
|
|
|
|
typ := &MockType{}
|
|
timeout := 10 * time.Minute
|
|
typ.On("RegisterOptions").Return(RegisterOptions{
|
|
QueryTimeout: timeout,
|
|
SupportsBlocking: true,
|
|
})
|
|
defer typ.AssertExpectations(t)
|
|
c := New(Options{})
|
|
|
|
// Register the type with a timeout
|
|
c.RegisterType("t", typ)
|
|
|
|
// Configure the type
|
|
var actual time.Duration
|
|
typ.Static(FetchResult{Value: 42}, nil).Times(1).Run(func(args mock.Arguments) {
|
|
opts := args.Get(0).(FetchOptions)
|
|
actual = opts.Timeout
|
|
})
|
|
|
|
// Get, should fetch
|
|
req := TestRequest(t, RequestInfo{Key: "hello"})
|
|
result, meta, err := c.Get(context.Background(), "t", req)
|
|
require.NoError(err)
|
|
require.Equal(42, result)
|
|
require.False(meta.Hit)
|
|
|
|
// Test the timeout
|
|
require.Equal(timeout, actual)
|
|
}
|
|
|
|
// Test that entries expire
|
|
func TestCacheGet_expire(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("too slow for testing.Short")
|
|
}
|
|
|
|
t.Parallel()
|
|
|
|
require := require.New(t)
|
|
|
|
typ := &MockType{}
|
|
typ.On("RegisterOptions").Return(RegisterOptions{
|
|
LastGetTTL: 400 * time.Millisecond,
|
|
})
|
|
defer typ.AssertExpectations(t)
|
|
c := New(Options{})
|
|
|
|
// Register the type with a timeout
|
|
c.RegisterType("t", typ)
|
|
|
|
// Configure the type
|
|
typ.Static(FetchResult{Value: 42}, nil).Times(2)
|
|
|
|
// Get, should fetch
|
|
req := TestRequest(t, RequestInfo{Key: "hello"})
|
|
result, meta, err := c.Get(context.Background(), "t", req)
|
|
require.NoError(err)
|
|
require.Equal(42, result)
|
|
require.False(meta.Hit)
|
|
|
|
// Wait for a non-trivial amount of time to sanity check the age increases at
|
|
// least this amount. Note that this is not a fudge for some timing-dependent
|
|
// background work it's just ensuring a non-trivial time elapses between the
|
|
// request above and below serilaly in this thread so short time is OK.
|
|
time.Sleep(5 * time.Millisecond)
|
|
|
|
// Get, should not fetch, verified via the mock assertions above
|
|
req = TestRequest(t, RequestInfo{Key: "hello"})
|
|
result, meta, err = c.Get(context.Background(), "t", req)
|
|
require.NoError(err)
|
|
require.Equal(42, result)
|
|
require.True(meta.Hit)
|
|
require.True(meta.Age > 5*time.Millisecond)
|
|
|
|
// Sleep for the expiry
|
|
time.Sleep(500 * time.Millisecond)
|
|
|
|
// Get, should fetch
|
|
req = TestRequest(t, RequestInfo{Key: "hello"})
|
|
result, meta, err = c.Get(context.Background(), "t", req)
|
|
require.NoError(err)
|
|
require.Equal(42, result)
|
|
require.False(meta.Hit)
|
|
|
|
// Sleep a tiny bit just to let maybe some background calls happen
|
|
// then verify that we still only got the one call
|
|
time.Sleep(20 * time.Millisecond)
|
|
typ.AssertExpectations(t)
|
|
}
|
|
|
|
// Test that entries reset their TTL on Get
|
|
func TestCacheGet_expireResetGet(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("too slow for testing.Short")
|
|
}
|
|
|
|
t.Parallel()
|
|
|
|
require := require.New(t)
|
|
|
|
typ := &MockType{}
|
|
typ.On("RegisterOptions").Return(RegisterOptions{
|
|
LastGetTTL: 150 * time.Millisecond,
|
|
})
|
|
defer typ.AssertExpectations(t)
|
|
c := New(Options{})
|
|
|
|
// Register the type with a timeout
|
|
c.RegisterType("t", typ)
|
|
|
|
// Configure the type
|
|
typ.Static(FetchResult{Value: 42}, nil).Times(2)
|
|
|
|
// Get, should fetch
|
|
req := TestRequest(t, RequestInfo{Key: "hello"})
|
|
result, meta, err := c.Get(context.Background(), "t", req)
|
|
require.NoError(err)
|
|
require.Equal(42, result)
|
|
require.False(meta.Hit)
|
|
|
|
// Fetch multiple times, where the total time is well beyond
|
|
// the TTL. We should not trigger any fetches during this time.
|
|
for i := 0; i < 5; i++ {
|
|
// Sleep a bit
|
|
time.Sleep(50 * time.Millisecond)
|
|
|
|
// Get, should not fetch
|
|
req = TestRequest(t, RequestInfo{Key: "hello"})
|
|
result, meta, err = c.Get(context.Background(), "t", req)
|
|
require.NoError(err)
|
|
require.Equal(42, result)
|
|
require.True(meta.Hit)
|
|
}
|
|
|
|
time.Sleep(200 * time.Millisecond)
|
|
|
|
// Get, should fetch
|
|
req = TestRequest(t, RequestInfo{Key: "hello"})
|
|
result, meta, err = c.Get(context.Background(), "t", req)
|
|
require.NoError(err)
|
|
require.Equal(42, result)
|
|
require.False(meta.Hit)
|
|
|
|
// Sleep a tiny bit just to let maybe some background calls happen
|
|
// then verify that we still only got the one call
|
|
time.Sleep(20 * time.Millisecond)
|
|
typ.AssertExpectations(t)
|
|
}
|
|
|
|
// Test that entries with state that satisfies io.Closer get cleaned up
|
|
func TestCacheGet_expireClose(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("too slow for testing.Short")
|
|
}
|
|
|
|
t.Parallel()
|
|
|
|
require := require.New(t)
|
|
|
|
typ := &MockType{}
|
|
defer typ.AssertExpectations(t)
|
|
c := New(Options{})
|
|
defer c.Close()
|
|
typ.On("RegisterOptions").Return(RegisterOptions{
|
|
SupportsBlocking: true,
|
|
LastGetTTL: 100 * time.Millisecond,
|
|
})
|
|
|
|
// Register the type with a timeout
|
|
c.RegisterType("t", typ)
|
|
|
|
// Configure the type
|
|
state := &testCloser{}
|
|
typ.Static(FetchResult{Value: 42, State: state}, nil).Times(1)
|
|
|
|
ctx := context.Background()
|
|
req := TestRequest(t, RequestInfo{Key: "hello"})
|
|
result, meta, err := c.Get(ctx, "t", req)
|
|
require.NoError(err)
|
|
require.Equal(42, result)
|
|
require.False(meta.Hit)
|
|
require.False(state.isClosed())
|
|
|
|
// Sleep for the expiry
|
|
time.Sleep(200 * time.Millisecond)
|
|
|
|
// state.Close() should have been called
|
|
require.True(state.isClosed())
|
|
}
|
|
|
|
type testCloser struct {
|
|
closed uint32
|
|
}
|
|
|
|
func (t *testCloser) Close() error {
|
|
atomic.SwapUint32(&t.closed, 1)
|
|
return nil
|
|
}
|
|
|
|
func (t *testCloser) isClosed() bool {
|
|
return atomic.LoadUint32(&t.closed) == 1
|
|
}
|
|
|
|
// Test a Get with a request that returns the same cache key across
|
|
// two different "types" returns two separate results.
|
|
func TestCacheGet_duplicateKeyDifferentType(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
require := require.New(t)
|
|
|
|
typ := TestType(t)
|
|
defer typ.AssertExpectations(t)
|
|
typ2 := TestType(t)
|
|
defer typ2.AssertExpectations(t)
|
|
|
|
c := New(Options{})
|
|
c.RegisterType("t", typ)
|
|
c.RegisterType("t2", typ2)
|
|
|
|
// Configure the types
|
|
typ.Static(FetchResult{Value: 100}, nil)
|
|
typ2.Static(FetchResult{Value: 200}, nil)
|
|
|
|
// Get, should fetch
|
|
req := TestRequest(t, RequestInfo{Key: "foo"})
|
|
result, meta, err := c.Get(context.Background(), "t", req)
|
|
require.NoError(err)
|
|
require.Equal(100, result)
|
|
require.False(meta.Hit)
|
|
|
|
// Get from t2 with same key, should fetch
|
|
req = TestRequest(t, RequestInfo{Key: "foo"})
|
|
result, meta, err = c.Get(context.Background(), "t2", req)
|
|
require.NoError(err)
|
|
require.Equal(200, result)
|
|
require.False(meta.Hit)
|
|
|
|
// Get from t again with same key, should cache
|
|
req = TestRequest(t, RequestInfo{Key: "foo"})
|
|
result, meta, err = c.Get(context.Background(), "t", req)
|
|
require.NoError(err)
|
|
require.Equal(100, result)
|
|
require.True(meta.Hit)
|
|
|
|
// Sleep a tiny bit just to let maybe some background calls happen
|
|
// then verify that we still only got the one call
|
|
time.Sleep(20 * time.Millisecond)
|
|
typ.AssertExpectations(t)
|
|
typ2.AssertExpectations(t)
|
|
}
|
|
|
|
// Test that Get partitions the caches based on DC so two equivalent requests
|
|
// to different datacenters are automatically cached even if their keys are
|
|
// the same.
|
|
func TestCacheGet_partitionDC(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
c := New(Options{})
|
|
c.RegisterType("t", &testPartitionType{})
|
|
|
|
// Perform multiple gets
|
|
getCh1 := TestCacheGetCh(t, c, "t", TestRequest(t, RequestInfo{
|
|
Datacenter: "dc1", Key: "hello"}))
|
|
getCh2 := TestCacheGetCh(t, c, "t", TestRequest(t, RequestInfo{
|
|
Datacenter: "dc9", Key: "hello"}))
|
|
|
|
// Should return both!
|
|
TestCacheGetChResult(t, getCh1, "dc1")
|
|
TestCacheGetChResult(t, getCh2, "dc9")
|
|
}
|
|
|
|
// Test that Get partitions the caches based on token so two equivalent requests
|
|
// with different ACL tokens do not return the same result.
|
|
func TestCacheGet_partitionToken(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
c := New(Options{})
|
|
c.RegisterType("t", &testPartitionType{})
|
|
|
|
// Perform multiple gets
|
|
getCh1 := TestCacheGetCh(t, c, "t", TestRequest(t, RequestInfo{
|
|
Token: "", Key: "hello"}))
|
|
getCh2 := TestCacheGetCh(t, c, "t", TestRequest(t, RequestInfo{
|
|
Token: "foo", Key: "hello"}))
|
|
|
|
// Should return both!
|
|
TestCacheGetChResult(t, getCh1, "")
|
|
TestCacheGetChResult(t, getCh2, "foo")
|
|
}
|
|
|
|
// testPartitionType implements Type for testing that simply returns a value
|
|
// comprised of the request DC and ACL token, used for testing cache
|
|
// partitioning.
|
|
type testPartitionType struct{}
|
|
|
|
func (t *testPartitionType) Fetch(opts FetchOptions, r Request) (FetchResult, error) {
|
|
info := r.CacheInfo()
|
|
return FetchResult{
|
|
Value: fmt.Sprintf("%s%s", info.Datacenter, info.Token),
|
|
}, nil
|
|
}
|
|
|
|
func (t *testPartitionType) RegisterOptions() RegisterOptions {
|
|
return RegisterOptions{
|
|
SupportsBlocking: true,
|
|
}
|
|
}
|
|
|
|
// Test that background refreshing reports correct Age in failure and happy
|
|
// states.
|
|
func TestCacheGet_refreshAge(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("too slow for testing.Short")
|
|
}
|
|
|
|
if testing.Short() {
|
|
t.Skip("too slow for -short run")
|
|
}
|
|
t.Parallel()
|
|
|
|
require := require.New(t)
|
|
|
|
typ := &MockType{}
|
|
typ.On("RegisterOptions").Return(RegisterOptions{
|
|
Refresh: true,
|
|
RefreshTimer: 0,
|
|
QueryTimeout: 5 * time.Minute,
|
|
})
|
|
defer typ.AssertExpectations(t)
|
|
c := New(Options{})
|
|
c.RegisterType("t", typ)
|
|
|
|
// Configure the type
|
|
var index, shouldFail uint64
|
|
|
|
typ.On("Fetch", mock.Anything, mock.Anything).
|
|
Return(func(o FetchOptions, r Request) FetchResult {
|
|
idx := atomic.LoadUint64(&index)
|
|
if atomic.LoadUint64(&shouldFail) == 1 {
|
|
return FetchResult{Value: nil, Index: idx}
|
|
}
|
|
if o.MinIndex == idx {
|
|
// Simulate waiting for a new value
|
|
time.Sleep(5 * time.Millisecond)
|
|
}
|
|
return FetchResult{Value: int(idx * 2), Index: idx}
|
|
}, func(o FetchOptions, r Request) error {
|
|
if atomic.LoadUint64(&shouldFail) == 1 {
|
|
return errors.New("test error")
|
|
}
|
|
return nil
|
|
})
|
|
|
|
// Set initial index/value
|
|
atomic.StoreUint64(&index, 4)
|
|
|
|
// Fetch
|
|
resultCh := TestCacheGetCh(t, c, "t", TestRequest(t, RequestInfo{Key: "hello"}))
|
|
TestCacheGetChResult(t, resultCh, 8)
|
|
|
|
{
|
|
// Wait a few milliseconds after initial fetch to check age is not reporting
|
|
// actual age.
|
|
time.Sleep(2 * time.Millisecond)
|
|
|
|
// Fetch again, non-blocking
|
|
result, meta, err := c.Get(context.Background(), "t", TestRequest(t, RequestInfo{Key: "hello"}))
|
|
require.NoError(err)
|
|
require.Equal(8, result)
|
|
require.True(meta.Hit)
|
|
// Age should be zero since background refresh was "active"
|
|
require.Equal(time.Duration(0), meta.Age)
|
|
}
|
|
|
|
// Now fail the next background sync
|
|
atomic.StoreUint64(&shouldFail, 1)
|
|
|
|
// Wait until the current request times out and starts failing. The request
|
|
// should take a maximum of 5ms to return but give it some headroom to allow
|
|
// it to finish 5ms sleep, unblock and next background request to be attemoted
|
|
// and fail and state updated in noisy CI... We might want to retry if this is
|
|
// still flaky but see if a longer wait is sufficient for now.
|
|
time.Sleep(50 * time.Millisecond)
|
|
|
|
var lastAge time.Duration
|
|
{
|
|
result, meta, err := c.Get(context.Background(), "t", TestRequest(t, RequestInfo{Key: "hello"}))
|
|
require.NoError(err)
|
|
require.Equal(8, result)
|
|
require.True(meta.Hit)
|
|
// Age should be non-zero since background refresh was "active"
|
|
require.True(meta.Age > 0)
|
|
lastAge = meta.Age
|
|
}
|
|
// Wait a bit longer - age should increase by at least this much
|
|
time.Sleep(5 * time.Millisecond)
|
|
{
|
|
result, meta, err := c.Get(context.Background(), "t", TestRequest(t, RequestInfo{Key: "hello"}))
|
|
require.NoError(err)
|
|
require.Equal(8, result)
|
|
require.True(meta.Hit)
|
|
require.True(meta.Age > (lastAge + (1 * time.Millisecond)))
|
|
}
|
|
|
|
// Now unfail the background refresh
|
|
atomic.StoreUint64(&shouldFail, 0)
|
|
|
|
// And update the data so we can see when the background task is working again
|
|
// (won't be immediate due to backoff on the errors).
|
|
atomic.AddUint64(&index, 1)
|
|
|
|
t0 := time.Now()
|
|
|
|
timeout := true
|
|
// Allow up to 5 seconds since the error backoff is likely to have kicked in
|
|
// and causes this to take different amounts of time depending on how quickly
|
|
// the test thread got down here relative to the failures.
|
|
for attempts := 0; attempts < 50; attempts++ {
|
|
time.Sleep(100 * time.Millisecond)
|
|
result, meta, err := c.Get(context.Background(), "t", TestRequest(t, RequestInfo{Key: "hello"}))
|
|
// Should never error even if background is failing as we have cached value
|
|
require.NoError(err)
|
|
require.True(meta.Hit)
|
|
// Got the new value!
|
|
if result == 10 {
|
|
// Age should be zero since background refresh is "active" again
|
|
t.Logf("Succeeded after %d attempts", attempts)
|
|
require.Equal(time.Duration(0), meta.Age)
|
|
timeout = false
|
|
break
|
|
}
|
|
}
|
|
require.False(timeout, "failed to observe update after %s", time.Since(t0))
|
|
}
|
|
|
|
func TestCacheGet_nonRefreshAge(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("too slow for testing.Short")
|
|
}
|
|
|
|
t.Parallel()
|
|
|
|
require := require.New(t)
|
|
|
|
typ := &MockType{}
|
|
typ.On("RegisterOptions").Return(RegisterOptions{
|
|
Refresh: false,
|
|
LastGetTTL: 100 * time.Millisecond,
|
|
})
|
|
defer typ.AssertExpectations(t)
|
|
c := New(Options{})
|
|
c.RegisterType("t", typ)
|
|
|
|
// Configure the type
|
|
var index uint64
|
|
|
|
typ.On("Fetch", mock.Anything, mock.Anything).
|
|
Return(func(o FetchOptions, r Request) FetchResult {
|
|
idx := atomic.LoadUint64(&index)
|
|
return FetchResult{Value: int(idx * 2), Index: idx}
|
|
}, nil)
|
|
|
|
// Set initial index/value
|
|
atomic.StoreUint64(&index, 4)
|
|
|
|
// Fetch
|
|
resultCh := TestCacheGetCh(t, c, "t", TestRequest(t, RequestInfo{Key: "hello"}))
|
|
TestCacheGetChResult(t, resultCh, 8)
|
|
|
|
var lastAge time.Duration
|
|
{
|
|
// Wait a few milliseconds after initial fetch to check age IS reporting
|
|
// actual age.
|
|
time.Sleep(5 * time.Millisecond)
|
|
|
|
// Fetch again, non-blocking
|
|
result, meta, err := c.Get(context.Background(), "t", TestRequest(t, RequestInfo{Key: "hello"}))
|
|
require.NoError(err)
|
|
require.Equal(8, result)
|
|
require.True(meta.Hit)
|
|
require.True(meta.Age > (5 * time.Millisecond))
|
|
lastAge = meta.Age
|
|
}
|
|
|
|
// Wait for expiry
|
|
time.Sleep(200 * time.Millisecond)
|
|
|
|
{
|
|
result, meta, err := c.Get(context.Background(), "t", TestRequest(t, RequestInfo{Key: "hello"}))
|
|
require.NoError(err)
|
|
require.Equal(8, result)
|
|
require.False(meta.Hit)
|
|
// Age should smaller again
|
|
require.True(meta.Age < lastAge)
|
|
}
|
|
|
|
{
|
|
// Wait for a non-trivial amount of time to sanity check the age increases at
|
|
// least this amount. Note that this is not a fudge for some timing-dependent
|
|
// background work it's just ensuring a non-trivial time elapses between the
|
|
// request above and below serilaly in this thread so short time is OK.
|
|
time.Sleep(5 * time.Millisecond)
|
|
|
|
// Fetch again, non-blocking
|
|
result, meta, err := c.Get(context.Background(), "t", TestRequest(t, RequestInfo{Key: "hello"}))
|
|
require.NoError(err)
|
|
require.Equal(8, result)
|
|
require.True(meta.Hit)
|
|
require.True(meta.Age > (5 * time.Millisecond))
|
|
lastAge = meta.Age
|
|
}
|
|
|
|
// Now verify that setting MaxAge results in cache invalidation
|
|
{
|
|
result, meta, err := c.Get(context.Background(), "t", TestRequest(t, RequestInfo{
|
|
Key: "hello",
|
|
MaxAge: 1 * time.Millisecond,
|
|
}))
|
|
require.NoError(err)
|
|
require.Equal(8, result)
|
|
require.False(meta.Hit)
|
|
// Age should smaller again
|
|
require.True(meta.Age < lastAge)
|
|
}
|
|
}
|
|
|
|
func TestCacheGet_nonBlockingType(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
typ := TestTypeNonBlocking(t)
|
|
c := New(Options{})
|
|
c.RegisterType("t", typ)
|
|
|
|
// Configure the type
|
|
typ.Static(FetchResult{Value: 42, Index: 1}, nil).Once()
|
|
typ.Static(FetchResult{Value: 43, Index: 2}, nil).Twice().
|
|
Run(func(args mock.Arguments) {
|
|
opts := args.Get(0).(FetchOptions)
|
|
// MinIndex should never be set for a non-blocking type.
|
|
require.Equal(t, uint64(0), opts.MinIndex)
|
|
})
|
|
|
|
require := require.New(t)
|
|
|
|
// Get, should fetch
|
|
req := TestRequest(t, RequestInfo{Key: "hello"})
|
|
result, meta, err := c.Get(context.Background(), "t", req)
|
|
require.NoError(err)
|
|
require.Equal(42, result)
|
|
require.False(meta.Hit)
|
|
|
|
// Get, should not fetch since we have a cached value
|
|
req = TestRequest(t, RequestInfo{Key: "hello"})
|
|
result, meta, err = c.Get(context.Background(), "t", req)
|
|
require.NoError(err)
|
|
require.Equal(42, result)
|
|
require.True(meta.Hit)
|
|
|
|
// Get, should not attempt to fetch with blocking even if requested. The
|
|
// assertions below about the value being the same combined with the fact the
|
|
// mock will only return that value on first call suffice to show that
|
|
// blocking request is not being attempted.
|
|
req = TestRequest(t, RequestInfo{
|
|
Key: "hello",
|
|
MinIndex: 1,
|
|
Timeout: 10 * time.Minute,
|
|
})
|
|
result, meta, err = c.Get(context.Background(), "t", req)
|
|
require.NoError(err)
|
|
require.Equal(42, result)
|
|
require.True(meta.Hit)
|
|
|
|
time.Sleep(10 * time.Millisecond)
|
|
|
|
// Get with a max age should fetch again
|
|
req = TestRequest(t, RequestInfo{Key: "hello", MaxAge: 5 * time.Millisecond})
|
|
result, meta, err = c.Get(context.Background(), "t", req)
|
|
require.NoError(err)
|
|
require.Equal(43, result)
|
|
require.False(meta.Hit)
|
|
|
|
// Get with a must revalidate should fetch again even without a delay.
|
|
req = TestRequest(t, RequestInfo{Key: "hello", MustRevalidate: true})
|
|
result, meta, err = c.Get(context.Background(), "t", req)
|
|
require.NoError(err)
|
|
require.Equal(43, result)
|
|
require.False(meta.Hit)
|
|
|
|
// Sleep a tiny bit just to let maybe some background calls happen
|
|
// then verify that we still only got the one call
|
|
time.Sleep(20 * time.Millisecond)
|
|
typ.AssertExpectations(t)
|
|
}
|
|
|
|
// Test a get with an index set will wait until an index that is higher
|
|
// is set in the cache.
|
|
func TestCacheReload(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
typ1 := TestType(t)
|
|
defer typ1.AssertExpectations(t)
|
|
|
|
c := New(Options{EntryFetchRate: rate.Limit(1), EntryFetchMaxBurst: 1})
|
|
c.RegisterType("t1", typ1)
|
|
typ1.Mock.On("Fetch", mock.Anything, mock.Anything).Return(FetchResult{Value: 42, Index: 42}, nil).Maybe()
|
|
|
|
require.False(t, c.ReloadOptions(Options{EntryFetchRate: rate.Limit(1), EntryFetchMaxBurst: 1}), "Value should not be reloaded")
|
|
|
|
_, meta, err := c.Get(context.Background(), "t1", TestRequest(t, RequestInfo{Key: "hello1", MinIndex: uint64(1)}))
|
|
require.NoError(t, err)
|
|
require.Equal(t, meta.Index, uint64(42))
|
|
|
|
testEntry := func(t *testing.T, doTest func(t *testing.T, entry cacheEntry)) {
|
|
c.entriesLock.Lock()
|
|
tEntry, ok := c.types["t1"]
|
|
require.True(t, ok)
|
|
keyName := makeEntryKey("t1", "", "", "hello1")
|
|
ok, entryValid, entry := c.getEntryLocked(tEntry, keyName, RequestInfo{})
|
|
require.True(t, ok)
|
|
require.True(t, entryValid)
|
|
doTest(t, entry)
|
|
c.entriesLock.Unlock()
|
|
|
|
}
|
|
testEntry(t, func(t *testing.T, entry cacheEntry) {
|
|
require.Equal(t, entry.FetchRateLimiter.Limit(), rate.Limit(1))
|
|
require.Equal(t, entry.FetchRateLimiter.Burst(), 1)
|
|
})
|
|
|
|
// Modify only rateLimit
|
|
require.True(t, c.ReloadOptions(Options{EntryFetchRate: rate.Limit(100), EntryFetchMaxBurst: 1}))
|
|
testEntry(t, func(t *testing.T, entry cacheEntry) {
|
|
require.Equal(t, entry.FetchRateLimiter.Limit(), rate.Limit(100))
|
|
require.Equal(t, entry.FetchRateLimiter.Burst(), 1)
|
|
})
|
|
|
|
// Modify only Burst
|
|
require.True(t, c.ReloadOptions(Options{EntryFetchRate: rate.Limit(100), EntryFetchMaxBurst: 5}))
|
|
testEntry(t, func(t *testing.T, entry cacheEntry) {
|
|
require.Equal(t, entry.FetchRateLimiter.Limit(), rate.Limit(100))
|
|
require.Equal(t, entry.FetchRateLimiter.Burst(), 5)
|
|
})
|
|
|
|
// Modify only Burst and Limit at the same time
|
|
require.True(t, c.ReloadOptions(Options{EntryFetchRate: rate.Limit(1000), EntryFetchMaxBurst: 42}))
|
|
|
|
testEntry(t, func(t *testing.T, entry cacheEntry) {
|
|
require.Equal(t, entry.FetchRateLimiter.Limit(), rate.Limit(1000))
|
|
require.Equal(t, entry.FetchRateLimiter.Burst(), 42)
|
|
})
|
|
}
|
|
|
|
// TestCacheThrottle checks the assumptions for the cache throttling. It sets
|
|
// up a cache with Options{EntryFetchRate: 10.0, EntryFetchMaxBurst: 1}, which
|
|
// allows for 10req/s, or one request every 100ms.
|
|
// It configures two different cache types with each 3 updates. Each type has
|
|
// its own rate limiter which starts initially full, and we expect the
|
|
// following requests when creating blocking queries against both waiting for
|
|
// the third update:
|
|
// at ~0ms: typ1 and typ2 receive first value and are blocked until
|
|
// ~100ms: typ1 and typ2 receive second value and are blocked until
|
|
// ~200ms: typ1 and typ2 receive third value which we check
|
|
//
|
|
// This test will verify waiting with a blocking query for the last update will
|
|
// block for 190ms and only afterwards have the expected result.
|
|
// It demonstrates the ratelimiting waits for the expected amount of time and
|
|
// also that each type has its own ratelimiter, because results for both types
|
|
// are arriving at similar times, which wouldn't be the case if they use a
|
|
// shared limiter.
|
|
func TestCacheThrottle(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("too slow for testing.Short")
|
|
}
|
|
|
|
t.Parallel()
|
|
|
|
typ1 := TestType(t)
|
|
typ2 := TestType(t)
|
|
defer typ1.AssertExpectations(t)
|
|
defer typ2.AssertExpectations(t)
|
|
|
|
c := New(Options{EntryFetchRate: 10.0, EntryFetchMaxBurst: 1})
|
|
c.RegisterType("t1", typ1)
|
|
c.RegisterType("t2", typ2)
|
|
|
|
// Configure the type
|
|
typ1.Static(FetchResult{Value: 1, Index: 4}, nil).Once()
|
|
typ1.Static(FetchResult{Value: 12, Index: 5}, nil).Once()
|
|
typ1.Static(FetchResult{Value: 42, Index: 6}, nil).Once()
|
|
|
|
typ2.Static(FetchResult{Value: 1, Index: 4}, nil).Once()
|
|
typ2.Static(FetchResult{Value: 12, Index: 5}, nil).Once()
|
|
typ2.Static(FetchResult{Value: 43, Index: 6}, nil).Once()
|
|
|
|
result1Ch := TestCacheGetCh(t, c, "t1", TestRequest(t, RequestInfo{
|
|
Key: "hello1", MinIndex: 5}))
|
|
|
|
result2Ch := TestCacheGetCh(t, c, "t2", TestRequest(t, RequestInfo{
|
|
Key: "hello2", MinIndex: 5}))
|
|
|
|
select {
|
|
case <-result1Ch:
|
|
t.Fatal("result1Ch should block")
|
|
case <-result2Ch:
|
|
t.Fatal("result2Ch should block")
|
|
case <-time.After(190 * time.Millisecond):
|
|
}
|
|
|
|
after := time.After(30 * time.Millisecond)
|
|
var res1, res2 bool
|
|
OUT:
|
|
for {
|
|
select {
|
|
case result := <-result1Ch:
|
|
require.Equal(t, 42, result)
|
|
|
|
res1 = true
|
|
case result := <-result2Ch:
|
|
require.Equal(t, 43, result)
|
|
res2 = true
|
|
case <-after:
|
|
t.Fatal("shouldn't block that long")
|
|
}
|
|
if res1 && res2 {
|
|
break OUT
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestCache_ExpiryLoop_ExitsWhenStopped(t *testing.T) {
|
|
c := &Cache{
|
|
stopCh: make(chan struct{}),
|
|
entries: make(map[string]cacheEntry),
|
|
entriesExpiryHeap: ttlcache.NewExpiryHeap(),
|
|
}
|
|
chStart := make(chan struct{})
|
|
chDone := make(chan struct{})
|
|
go func() {
|
|
close(chStart)
|
|
c.runExpiryLoop()
|
|
close(chDone)
|
|
}()
|
|
|
|
<-chStart
|
|
close(c.stopCh)
|
|
|
|
select {
|
|
case <-chDone:
|
|
case <-time.After(50 * time.Millisecond):
|
|
t.Fatalf("expected loop to exit when stopped")
|
|
}
|
|
}
|
|
|
|
func TestCache_Prepopulate(t *testing.T) {
|
|
typ := &fakeType{index: 5}
|
|
c := New(Options{})
|
|
c.RegisterType("t", typ)
|
|
|
|
c.Prepopulate("t", FetchResult{Value: 17, Index: 1}, "dc1", "token", "v1")
|
|
|
|
ctx := context.Background()
|
|
req := fakeRequest{
|
|
info: RequestInfo{
|
|
Key: "v1",
|
|
Token: "token",
|
|
Datacenter: "dc1",
|
|
MinIndex: 1,
|
|
},
|
|
}
|
|
result, _, err := c.Get(ctx, "t", req)
|
|
require.NoError(t, err)
|
|
require.Equal(t, 17, result)
|
|
}
|
|
|
|
type fakeType struct {
|
|
index uint64
|
|
}
|
|
|
|
func (f fakeType) Fetch(_ FetchOptions, _ Request) (FetchResult, error) {
|
|
idx := atomic.LoadUint64(&f.index)
|
|
return FetchResult{Value: int(idx * 2), Index: idx}, nil
|
|
}
|
|
|
|
func (f fakeType) RegisterOptions() RegisterOptions {
|
|
return RegisterOptions{Refresh: true}
|
|
}
|
|
|
|
var _ Type = (*fakeType)(nil)
|
|
|
|
type fakeRequest struct {
|
|
info RequestInfo
|
|
}
|
|
|
|
func (f fakeRequest) CacheInfo() RequestInfo {
|
|
return f.info
|
|
}
|
|
|
|
var _ Request = (*fakeRequest)(nil)
|