go-watchdog/watchdog_test.go

181 lines
4.2 KiB
Go

package watchdog
import (
"fmt"
"log"
"os"
"runtime"
"runtime/debug"
"strconv"
"testing"
"time"
"github.com/elastic/gosigar"
"github.com/raulk/clock"
"github.com/stretchr/testify/require"
)
const (
// EnvTestIsolated is a marker property for the runner to confirm that this
// test is running in isolation (i.e. a dedicated process).
EnvTestIsolated = "TEST_ISOLATED"
// EnvTestDockerMemLimit is the memory limit applied in a docker container.
EnvTestDockerMemLimit = "TEST_DOCKER_MEMLIMIT"
)
// DockerMemLimit is initialized in the init() function from the
// EnvTestDockerMemLimit env variable.
var DockerMemLimit int // bytes
func init() {
Logger = &stdlog{log: log.New(os.Stdout, "[watchdog test] ", log.LstdFlags|log.Lmsgprefix), debug: true}
if l := os.Getenv(EnvTestDockerMemLimit); l != "" {
l, err := strconv.Atoi(l)
if err != nil {
panic(err)
}
DockerMemLimit = l
}
}
func skipIfNotIsolated(t *testing.T) {
if os.Getenv(EnvTestIsolated) != "1" {
t.Skipf("skipping test in non-isolated mode")
}
}
var (
limit uint64 = 64 << 20 // 64MiB.
)
func TestControl_Isolated(t *testing.T) {
skipIfNotIsolated(t)
debug.SetGCPercent(100)
rounds := 100
if DockerMemLimit != 0 {
rounds /= int(float64(DockerMemLimit)*0.8) / 1024 / 1024
}
// retain 1MiB every iteration.
var retained [][]byte
for i := 0; i < rounds; i++ {
b := make([]byte, 1*1024*1024)
for i := range b {
b[i] = byte(i)
}
retained = append(retained, b)
}
for _, b := range retained {
for i := range b {
b[i] = byte(i)
}
}
var ms runtime.MemStats
runtime.ReadMemStats(&ms)
require.NotZero(t, ms.NumGC) // GCs have taken place, but...
require.Zero(t, ms.NumForcedGC) // ... no forced GCs beyond our initial one.
}
func TestHeapDriven_Isolated(t *testing.T) {
skipIfNotIsolated(t)
// we can't mock ReadMemStats, because we're relying on the go runtime to
// enforce the GC run, and the go runtime won't use our mock. Therefore, we
// need to do the actual thing.
debug.SetGCPercent(100)
clk := clock.NewMock()
Clock = clk
observations := make([]*runtime.MemStats, 0, 100)
NotifyGC = func() {
var ms runtime.MemStats
runtime.ReadMemStats(&ms)
observations = append(observations, &ms)
}
// limit is 64MiB.
err, stopFn := HeapDriven(limit, NewAdaptivePolicy(0.5))
require.NoError(t, err)
defer stopFn()
time.Sleep(500 * time.Millisecond) // give time for the watchdog to init.
// retain 1MiB every iteration, up to 100MiB (beyond heap limit!).
var retained [][]byte
for i := 0; i < 100; i++ {
retained = append(retained, make([]byte, 1*1024*1024))
}
for _, o := range observations {
fmt.Println("heap alloc:", o.HeapAlloc, "next gc:", o.NextGC, "gc count:", o.NumGC, "forced gc:", o.NumForcedGC)
}
var ms runtime.MemStats
runtime.ReadMemStats(&ms)
require.GreaterOrEqual(t, ms.NumGC, uint32(9)) // over 9 GCs should've taken place.
}
func TestSystemDriven_Isolated(t *testing.T) {
skipIfNotIsolated(t)
debug.SetGCPercent(100)
clk := clock.NewMock()
Clock = clk
// mock the system reporting.
var actualUsed uint64
sysmemFn = func(g *gosigar.Mem) error {
g.ActualUsed = actualUsed
return nil
}
// limit is 64MiB.
err, stopFn := SystemDriven(limit, 5*time.Second, NewAdaptivePolicy(0.5))
require.NoError(t, err)
defer stopFn()
time.Sleep(200 * time.Millisecond) // give time for the watchdog to init.
notifyCh := make(chan struct{}, 1)
NotifyGC = func() {
notifyCh <- struct{}{}
}
// first tick; used = 0.
clk.Add(5 * time.Second)
time.Sleep(200 * time.Millisecond)
require.Len(t, notifyCh, 0) // no GC has taken place.
// second tick; used = just over 50%; will trigger GC.
actualUsed = (limit / 2) + 1
clk.Add(5 * time.Second)
time.Sleep(200 * time.Millisecond)
require.Len(t, notifyCh, 1)
<-notifyCh
// third tick; just below 75%; no GC.
actualUsed = uint64(float64(limit)*0.75) - 1
clk.Add(5 * time.Second)
time.Sleep(200 * time.Millisecond)
require.Len(t, notifyCh, 0)
// fourth tick; 75% exactly; will trigger GC.
actualUsed = uint64(float64(limit)*0.75) + 1
clk.Add(5 * time.Second)
time.Sleep(200 * time.Millisecond)
require.Len(t, notifyCh, 1)
<-notifyCh
var ms runtime.MemStats
runtime.ReadMemStats(&ms)
require.GreaterOrEqual(t, ms.NumForcedGC, uint32(2))
}