package watchdog import ( "fmt" "log" "os" "runtime" "runtime/debug" "strconv" "testing" "time" "github.com/elastic/gosigar" "github.com/raulk/clock" "github.com/stretchr/testify/require" ) const ( // EnvTestIsolated is a marker property for the runner to confirm that this // test is running in isolation (i.e. a dedicated process). EnvTestIsolated = "TEST_ISOLATED" // EnvTestDockerMemLimit is the memory limit applied in a docker container. EnvTestDockerMemLimit = "TEST_DOCKER_MEMLIMIT" ) // DockerMemLimit is initialized in the init() function from the // EnvTestDockerMemLimit env variable. var DockerMemLimit int // bytes func init() { Logger = &stdlog{log: log.New(os.Stdout, "[watchdog test] ", log.LstdFlags|log.Lmsgprefix), debug: true} if l := os.Getenv(EnvTestDockerMemLimit); l != "" { l, err := strconv.Atoi(l) if err != nil { panic(err) } DockerMemLimit = l } } func skipIfNotIsolated(t *testing.T) { if os.Getenv(EnvTestIsolated) != "1" { t.Skipf("skipping test in non-isolated mode") } } var ( limit uint64 = 64 << 20 // 64MiB. ) func TestControl_Isolated(t *testing.T) { skipIfNotIsolated(t) debug.SetGCPercent(100) rounds := 100 if DockerMemLimit != 0 { rounds /= int(float64(DockerMemLimit)*0.8) / 1024 / 1024 } // retain 1MiB every iteration. var retained [][]byte for i := 0; i < rounds; i++ { b := make([]byte, 1*1024*1024) for i := range b { b[i] = byte(i) } retained = append(retained, b) } for _, b := range retained { for i := range b { b[i] = byte(i) } } var ms runtime.MemStats runtime.ReadMemStats(&ms) require.NotZero(t, ms.NumGC) // GCs have taken place, but... require.Zero(t, ms.NumForcedGC) // ... no forced GCs beyond our initial one. } func TestHeapDriven_Isolated(t *testing.T) { skipIfNotIsolated(t) // we can't mock ReadMemStats, because we're relying on the go runtime to // enforce the GC run, and the go runtime won't use our mock. Therefore, we // need to do the actual thing. debug.SetGCPercent(100) clk := clock.NewMock() Clock = clk observations := make([]*runtime.MemStats, 0, 100) NotifyGC = func() { var ms runtime.MemStats runtime.ReadMemStats(&ms) observations = append(observations, &ms) } // limit is 64MiB. err, stopFn := HeapDriven(limit, NewAdaptivePolicy(0.5)) require.NoError(t, err) defer stopFn() time.Sleep(500 * time.Millisecond) // give time for the watchdog to init. // retain 1MiB every iteration, up to 100MiB (beyond heap limit!). var retained [][]byte for i := 0; i < 100; i++ { retained = append(retained, make([]byte, 1*1024*1024)) } for _, o := range observations { fmt.Println("heap alloc:", o.HeapAlloc, "next gc:", o.NextGC, "gc count:", o.NumGC, "forced gc:", o.NumForcedGC) } var ms runtime.MemStats runtime.ReadMemStats(&ms) require.GreaterOrEqual(t, ms.NumGC, uint32(9)) // over 9 GCs should've taken place. } func TestSystemDriven_Isolated(t *testing.T) { skipIfNotIsolated(t) debug.SetGCPercent(100) clk := clock.NewMock() Clock = clk // mock the system reporting. var actualUsed uint64 sysmemFn = func(g *gosigar.Mem) error { g.ActualUsed = actualUsed return nil } // limit is 64MiB. err, stopFn := SystemDriven(limit, 5*time.Second, NewAdaptivePolicy(0.5)) require.NoError(t, err) defer stopFn() time.Sleep(200 * time.Millisecond) // give time for the watchdog to init. notifyCh := make(chan struct{}, 1) NotifyGC = func() { notifyCh <- struct{}{} } // first tick; used = 0. clk.Add(5 * time.Second) time.Sleep(200 * time.Millisecond) require.Len(t, notifyCh, 0) // no GC has taken place. // second tick; used = just over 50%; will trigger GC. actualUsed = (limit / 2) + 1 clk.Add(5 * time.Second) time.Sleep(200 * time.Millisecond) require.Len(t, notifyCh, 1) <-notifyCh // third tick; just below 75%; no GC. actualUsed = uint64(float64(limit)*0.75) - 1 clk.Add(5 * time.Second) time.Sleep(200 * time.Millisecond) require.Len(t, notifyCh, 0) // fourth tick; 75% exactly; will trigger GC. actualUsed = uint64(float64(limit)*0.75) + 1 clk.Add(5 * time.Second) time.Sleep(200 * time.Millisecond) require.Len(t, notifyCh, 1) <-notifyCh var ms runtime.MemStats runtime.ReadMemStats(&ms) require.GreaterOrEqual(t, ms.NumForcedGC, uint32(2)) }