major rewrite of go-watchdog.
This commit introduces a major rewrite of go-watchdog. * HeapDriven and SystemDriven are now distinct run modes. * WIP ProcessDriven that uses cgroups. * Policies are now stateless, pure and greatly simplified. * Policies now return the next utilization at which GC should run. The watchdog enforces that value differently depending on the run mode. * The heap-driven run mode adjusts GOGC dynamically. This places the responsibility on the Go runtime to honour the trigger point, and results in more robust logic that is not vulnerable to very quick bursts within sampling periods. * The heap-driven run mode is no longer polling (interval-driven). Instead, it relies entirely on GC signals. * The Silence and Emergency features of the watermark policy have been removed. If utilization is above the last watermark, the policy will request immediate GC. * Races removed.
This commit is contained in:
Raúl Kripalani
parent
ffbfd5e37a
commit
4558d98653
20
README.md
20
README.md
|
|
@ -4,9 +4,17 @@
|
||||||
|
|
||||||
[](https://godoc.org/github.com/raulk/go-watchdog)
|
[](https://godoc.org/github.com/raulk/go-watchdog)
|
||||||
|
|
||||||
go-watchdog runs a singleton memory watchdog. It takes system and heap memory
|
go-watchdog runs a singleton memory watchdog in the process, which watches
|
||||||
readings periodically, and feeds them to a user-defined policy to determine
|
memory utilization and forces Go GC in accordance with a user-defined policy.
|
||||||
whether GC needs to run immediately.
|
|
||||||
|
There are two kinds of watchdog so far:
|
||||||
|
|
||||||
|
* **heap-driven:** applies a limit to the heap, and obtains current usage through
|
||||||
|
`runtime.ReadMemStats()`.
|
||||||
|
* **system-driven:** applies a limit to the total system memory used, and obtains
|
||||||
|
current usage through [`elastic/go-sigar`](https://github.com/elastic/gosigar).
|
||||||
|
|
||||||
|
A third process-driven watchdog that uses cgroups is underway.
|
||||||
|
|
||||||
This library ships with two policies out of the box:
|
This library ships with two policies out of the box:
|
||||||
|
|
||||||
|
|
@ -18,12 +26,6 @@ This library ships with two policies out of the box:
|
||||||
You can easily build a custom policy tailored to the allocation patterns of your
|
You can easily build a custom policy tailored to the allocation patterns of your
|
||||||
program.
|
program.
|
||||||
|
|
||||||
It is recommended that you set both (a) a memory limit and (b) a scope of
|
|
||||||
application of that limit (system or heap) when you start the watchdog.
|
|
||||||
Otherwise, go-watchdog will use the system scope, and will default to the
|
|
||||||
total system memory as the limit. [elastic/go-sigar](https://github.com/elastic/gosigar)
|
|
||||||
is used to make the discovery.
|
|
||||||
|
|
||||||
## Why is this even needed?
|
## Why is this even needed?
|
||||||
|
|
||||||
The garbage collector that ships with the go runtime is pretty good in some
|
The garbage collector that ships with the go runtime is pretty good in some
|
||||||
|
|
|
||||||
74
adaptive.go
74
adaptive.go
|
|
@ -1,54 +1,32 @@
|
||||||
package watchdog
|
package watchdog
|
||||||
|
|
||||||
// AdaptivePolicy is a policy that forces GC when the usage surpasses a
|
// NewAdaptivePolicy creates a policy that forces GC when the usage surpasses a
|
||||||
// user-configured percentage (Factor) of the available memory that remained
|
// user-configured percentage (factor) of the available memory.
|
||||||
// after the last GC run.
|
|
||||||
//
|
//
|
||||||
// TODO tests
|
// This policy recalculates the next target as usage+(limit-usage)*factor, and
|
||||||
type AdaptivePolicy struct {
|
// forces immediate GC when used >= limit.
|
||||||
// Factor determines how much this policy will let the heap expand
|
func NewAdaptivePolicy(factor float64) PolicyCtor {
|
||||||
// before it triggers.
|
return func(limit uint64) (Policy, error) {
|
||||||
//
|
return &adaptivePolicy{
|
||||||
// On every GC run, this policy recalculates the next target as
|
factor: factor,
|
||||||
// (limit-currentHeap)*Factor (i.e. available*Factor).
|
limit: limit,
|
||||||
//
|
}, nil
|
||||||
// If the GC target calculated by the runtime is lower than the one
|
}
|
||||||
// calculated by this policy, this policy will set the new target, but the
|
|
||||||
// effect will be nil, since the the go runtime will run GC sooner than us
|
|
||||||
// anyway.
|
|
||||||
Factor float64
|
|
||||||
|
|
||||||
active bool
|
|
||||||
target uint64
|
|
||||||
initialized bool
|
|
||||||
}
|
}
|
||||||
|
|
||||||
var _ Policy = (*AdaptivePolicy)(nil)
|
type adaptivePolicy struct {
|
||||||
|
factor float64
|
||||||
func (a *AdaptivePolicy) Evaluate(input PolicyInput) (trigger bool) {
|
limit uint64
|
||||||
if !a.initialized {
|
}
|
||||||
// when initializing, set the target to the limit; it will be reset
|
|
||||||
// when the first GC happens.
|
var _ Policy = (*adaptivePolicy)(nil)
|
||||||
a.target = input.Limit
|
|
||||||
a.initialized = true
|
func (p *adaptivePolicy) Evaluate(_ UtilizationType, used uint64) (next uint64, immediate bool) {
|
||||||
}
|
if used >= p.limit {
|
||||||
|
return used, true
|
||||||
// determine the value to compare utilisation against.
|
}
|
||||||
var actual uint64
|
|
||||||
switch input.Scope {
|
available := float64(p.limit) - float64(used)
|
||||||
case ScopeSystem:
|
next = used + uint64(available*p.factor)
|
||||||
actual = input.SysStats.ActualUsed
|
return next, false
|
||||||
case ScopeHeap:
|
|
||||||
actual = input.MemStats.HeapAlloc
|
|
||||||
}
|
|
||||||
|
|
||||||
if input.GCTrigger {
|
|
||||||
available := float64(input.Limit) - float64(actual)
|
|
||||||
calc := uint64(available * a.Factor)
|
|
||||||
a.target = calc
|
|
||||||
}
|
|
||||||
if actual >= a.target {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,31 @@
|
||||||
|
package watchdog
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/raulk/clock"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestAdaptivePolicy(t *testing.T) {
|
||||||
|
clk := clock.NewMock()
|
||||||
|
Clock = clk
|
||||||
|
|
||||||
|
p, err := NewAdaptivePolicy(0.5)(limit)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
// at zero; next = 50%.
|
||||||
|
next, immediate := p.Evaluate(UtilizationSystem, 0)
|
||||||
|
require.False(t, immediate)
|
||||||
|
require.EqualValues(t, limit/2, next)
|
||||||
|
|
||||||
|
// at half; next = 75%.
|
||||||
|
next, immediate = p.Evaluate(UtilizationSystem, limit/2)
|
||||||
|
require.False(t, immediate)
|
||||||
|
require.EqualValues(t, 3*(limit/4), next)
|
||||||
|
|
||||||
|
// at limit; immediate = true.
|
||||||
|
next, immediate = p.Evaluate(UtilizationSystem, limit)
|
||||||
|
require.True(t, immediate)
|
||||||
|
require.EqualValues(t, limit, next)
|
||||||
|
}
|
||||||
3
go.mod
3
go.mod
|
|
@ -3,11 +3,10 @@ module github.com/raulk/go-watchdog
|
||||||
go 1.15
|
go 1.15
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
github.com/containerd/cgroups v0.0.0-20201119153540-4cbc285b3327
|
||||||
github.com/elastic/gosigar v0.12.0
|
github.com/elastic/gosigar v0.12.0
|
||||||
github.com/kr/pretty v0.1.0 // indirect
|
github.com/kr/pretty v0.1.0 // indirect
|
||||||
github.com/raulk/clock v1.1.0
|
github.com/raulk/clock v1.1.0
|
||||||
github.com/stretchr/testify v1.4.0
|
github.com/stretchr/testify v1.4.0
|
||||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d // indirect
|
|
||||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
|
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
|
||||||
)
|
)
|
||||||
|
|
|
||||||
31
go.sum
31
go.sum
|
|
@ -1,27 +1,54 @@
|
||||||
|
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||||
|
github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs=
|
||||||
|
github.com/containerd/cgroups v0.0.0-20201119153540-4cbc285b3327 h1:7grrpcfCtbZLsjtB0DgMuzs1umsJmpzaHMZ6cO6iAWw=
|
||||||
|
github.com/containerd/cgroups v0.0.0-20201119153540-4cbc285b3327/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE=
|
||||||
|
github.com/coreos/go-systemd/v22 v22.1.0 h1:kq/SbG2BCKLkDKkjQf5OWwKWUKj1lgs3lFI4PxnR5lg=
|
||||||
|
github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk=
|
||||||
|
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
|
||||||
|
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
|
||||||
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
|
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
|
||||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
|
github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
|
||||||
|
github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
|
||||||
github.com/elastic/gosigar v0.12.0 h1:AsdhYCJlTudhfOYQyFNgx+fIVTfrDO0V1ST0vHgiapU=
|
github.com/elastic/gosigar v0.12.0 h1:AsdhYCJlTudhfOYQyFNgx+fIVTfrDO0V1ST0vHgiapU=
|
||||||
github.com/elastic/gosigar v0.12.0/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs=
|
github.com/elastic/gosigar v0.12.0/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs=
|
||||||
|
github.com/godbus/dbus/v5 v5.0.3 h1:ZqHaoEF7TBzh4jzPmqVhE/5A1z9of6orkAe5uHoAeME=
|
||||||
|
github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
|
||||||
|
github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
|
||||||
|
github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
|
||||||
|
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||||
|
github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
|
||||||
|
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||||
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
|
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
|
||||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||||
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
|
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
|
||||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||||
|
github.com/opencontainers/runtime-spec v1.0.2 h1:UfAcuLBJB9Coz72x1hgl8O5RVzTdNiaglX6v2DM6FI0=
|
||||||
|
github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
|
||||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
github.com/raulk/clock v1.1.0 h1:dpb29+UKMbLqiU/jqIJptgLR1nn23HLgMY0sTCDza5Y=
|
github.com/raulk/clock v1.1.0 h1:dpb29+UKMbLqiU/jqIJptgLR1nn23HLgMY0sTCDza5Y=
|
||||||
github.com/raulk/clock v1.1.0/go.mod h1:3MpVxdZ/ODBQDxbN+kzshf5OSZwPjtMDx6BBXBmOeY0=
|
github.com/raulk/clock v1.1.0/go.mod h1:3MpVxdZ/ODBQDxbN+kzshf5OSZwPjtMDx6BBXBmOeY0=
|
||||||
|
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||||
|
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
|
||||||
|
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
|
||||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||||
|
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||||
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
|
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
|
||||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||||
|
github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
|
||||||
golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0 h1:8H8QZJ30plJyIVj60H3lr8TZGIq2Fh3Cyrs/ZNg1foU=
|
golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0 h1:8H8QZJ30plJyIVj60H3lr8TZGIq2Fh3Cyrs/ZNg1foU=
|
||||||
golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI=
|
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9 h1:1/DFK4b7JH8DmkqhUk48onnSfrPzImPoVxuomtbT2nk=
|
||||||
|
golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||||
|
golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||||
|
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
|
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
|
||||||
|
|
|
||||||
8
log.go
8
log.go
|
|
@ -2,17 +2,17 @@ package watchdog
|
||||||
|
|
||||||
import "log"
|
import "log"
|
||||||
|
|
||||||
// Logger is an interface to be implemented by custom loggers.
|
// logger is an interface to be implemented by custom loggers.
|
||||||
type Logger interface {
|
type logger interface {
|
||||||
Debugf(template string, args ...interface{})
|
Debugf(template string, args ...interface{})
|
||||||
Infof(template string, args ...interface{})
|
Infof(template string, args ...interface{})
|
||||||
Warnf(template string, args ...interface{})
|
Warnf(template string, args ...interface{})
|
||||||
Errorf(template string, args ...interface{})
|
Errorf(template string, args ...interface{})
|
||||||
}
|
}
|
||||||
|
|
||||||
var _ Logger = (*stdlog)(nil)
|
var _ logger = (*stdlog)(nil)
|
||||||
|
|
||||||
// stdlog is a Logger that proxies to a standard log.Logger.
|
// stdlog is a logger that proxies to a standard log.logger.
|
||||||
type stdlog struct {
|
type stdlog struct {
|
||||||
log *log.Logger
|
log *log.Logger
|
||||||
debug bool
|
debug bool
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,24 @@
|
||||||
|
package watchdog
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/containerd/cgroups"
|
||||||
|
)
|
||||||
|
|
||||||
|
func ProcessMemoryLimit() uint64 {
|
||||||
|
var (
|
||||||
|
pid = os.Getpid()
|
||||||
|
memSubsystem = cgroups.SingleSubsystem(cgroups.V1, cgroups.Memory)
|
||||||
|
)
|
||||||
|
cgroup, err := cgroups.Load(memSubsystem, cgroups.PidPath(pid))
|
||||||
|
if err != nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
metrics, err := cgroup.Stat()
|
||||||
|
if err != nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
if metrics.Memory == nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return metrics.Memory.HierarchicalMemoryLimit
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,7 @@
|
||||||
|
// +build !linux
|
||||||
|
|
||||||
|
package watchdog
|
||||||
|
|
||||||
|
func ProcessMemoryLimit() uint64 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
505
watchdog.go
505
watchdog.go
|
|
@ -4,6 +4,7 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"log"
|
"log"
|
||||||
"runtime"
|
"runtime"
|
||||||
|
"runtime/debug"
|
||||||
"sync"
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"time"
|
"time"
|
||||||
|
|
@ -12,173 +13,22 @@ import (
|
||||||
"github.com/raulk/clock"
|
"github.com/raulk/clock"
|
||||||
)
|
)
|
||||||
|
|
||||||
// DecimalPrecision is the rounding precision that float calculations will use.
|
// The watchdog is designed to be used as a singleton; global vars are OK for
|
||||||
// By default, 4 decimal places.
|
// that reason.
|
||||||
var DecimalPrecision = 1e4
|
|
||||||
|
|
||||||
// Clock can be used to inject a mock clock for testing.
|
|
||||||
var Clock = clock.New()
|
|
||||||
|
|
||||||
var (
|
var (
|
||||||
// ErrAlreadyStarted is returned when the user tries to start the watchdog more than once.
|
// Logger is the logger to use. If nil, it will default to a logger that
|
||||||
ErrAlreadyStarted = fmt.Errorf("singleton memory watchdog was already started")
|
// proxies to a standard logger using the "[watchdog]" prefix.
|
||||||
)
|
Logger logger = &stdlog{log: log.New(log.Writer(), "[watchdog] ", log.LstdFlags|log.Lmsgprefix)}
|
||||||
|
|
||||||
const (
|
// Clock can be used to inject a mock clock for testing.
|
||||||
// stateUnstarted represents an unstarted state.
|
Clock = clock.New()
|
||||||
stateUnstarted int64 = iota
|
|
||||||
// stateStarted represents a started state.
|
|
||||||
stateStarted
|
|
||||||
)
|
|
||||||
|
|
||||||
// watchdog is a global singleton watchdog.
|
|
||||||
var watchdog struct {
|
|
||||||
state int64
|
|
||||||
config MemConfig
|
|
||||||
|
|
||||||
closing chan struct{}
|
|
||||||
wg sync.WaitGroup
|
|
||||||
}
|
|
||||||
|
|
||||||
// ScopeType defines the scope of the utilisation that we'll apply the limit to.
|
|
||||||
type ScopeType int
|
|
||||||
|
|
||||||
const (
|
|
||||||
// ScopeSystem specifies that the policy compares against actual used
|
|
||||||
// system memory.
|
|
||||||
ScopeSystem ScopeType = iota
|
|
||||||
// ScopeHeap specifies that the policy compares against heap used.
|
|
||||||
ScopeHeap
|
|
||||||
)
|
|
||||||
|
|
||||||
// PolicyInput is the object that's passed to a policy when evaluating it.
|
|
||||||
type PolicyInput struct {
|
|
||||||
Scope ScopeType
|
|
||||||
Limit uint64
|
|
||||||
MemStats *runtime.MemStats
|
|
||||||
SysStats *gosigar.Mem
|
|
||||||
GCTrigger bool // is this a GC trigger?
|
|
||||||
Logger Logger
|
|
||||||
}
|
|
||||||
|
|
||||||
// Policy encapsulates the logic that the watchdog will run on every tick.
|
|
||||||
type Policy interface {
|
|
||||||
// Evaluate determines whether the policy should fire. It receives the
|
|
||||||
// limit (either guessed or manually set), go runtime memory stats, and
|
|
||||||
// system memory stats, amongst other things. It returns whether the policy
|
|
||||||
// has fired or not.
|
|
||||||
Evaluate(input PolicyInput) (trigger bool)
|
|
||||||
}
|
|
||||||
|
|
||||||
type MemConfig struct {
|
|
||||||
// Scope is the scope at which the limit will be applied.
|
|
||||||
Scope ScopeType
|
|
||||||
|
|
||||||
// Limit is the memory available to this process. If zero, we will fall
|
|
||||||
// back to querying the system total memory via SIGAR.
|
|
||||||
Limit uint64
|
|
||||||
|
|
||||||
// Resolution is the interval at which the watchdog will retrieve memory
|
|
||||||
// stats and invoke the Policy.
|
|
||||||
Resolution time.Duration
|
|
||||||
|
|
||||||
// Policy sets the firing policy of this watchdog.
|
|
||||||
Policy Policy
|
|
||||||
|
|
||||||
// NotifyFired, if non-nil, will be called when the policy has fired,
|
// NotifyFired, if non-nil, will be called when the policy has fired,
|
||||||
// prior to calling GC, even if GC is disabled.
|
// prior to calling GC, even if GC is disabled.
|
||||||
NotifyFired func()
|
NotifyFired func() = func() {}
|
||||||
|
)
|
||||||
// NotifyOnly, if true, will cause the watchdog to only notify via the
|
|
||||||
// callbacks, without triggering actual GC.
|
|
||||||
NotifyOnly bool
|
|
||||||
|
|
||||||
// Logger is the logger to use. If nil, it will default to a logger that
|
|
||||||
// proxies to a standard logger using the "[watchdog]" prefix.
|
|
||||||
Logger Logger
|
|
||||||
}
|
|
||||||
|
|
||||||
// Memory starts the singleton memory watchdog with the provided configuration.
|
|
||||||
func Memory(config MemConfig) (err error, stop func()) {
|
|
||||||
if !atomic.CompareAndSwapInt64(&watchdog.state, stateUnstarted, stateStarted) {
|
|
||||||
return ErrAlreadyStarted, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if config.Logger == nil {
|
|
||||||
config.Logger = &stdlog{log: log.New(log.Writer(), "[watchdog] ", log.LstdFlags|log.Lmsgprefix)}
|
|
||||||
}
|
|
||||||
|
|
||||||
// if the user didn't provide a limit, get the total memory.
|
|
||||||
if config.Limit == 0 {
|
|
||||||
var mem gosigar.Mem
|
|
||||||
if err := mem.Get(); err != nil {
|
|
||||||
return fmt.Errorf("failed to get system memory limit via SIGAR: %w", err), nil
|
|
||||||
}
|
|
||||||
config.Limit = mem.Total
|
|
||||||
}
|
|
||||||
|
|
||||||
watchdog.config = config
|
|
||||||
watchdog.closing = make(chan struct{})
|
|
||||||
|
|
||||||
watchdog.wg.Add(1)
|
|
||||||
go watch()
|
|
||||||
|
|
||||||
return nil, stopMemory
|
|
||||||
}
|
|
||||||
|
|
||||||
func watch() {
|
|
||||||
var (
|
|
||||||
lk sync.Mutex // guards gcTriggered channel, which is drained and flipped to nil on closure.
|
|
||||||
gcTriggered = make(chan struct{}, 16)
|
|
||||||
|
|
||||||
memstats runtime.MemStats
|
|
||||||
sysmem gosigar.Mem
|
|
||||||
config = watchdog.config
|
|
||||||
)
|
|
||||||
|
|
||||||
// this non-zero sized struct is used as a sentinel to detect when a GC
|
|
||||||
// run has finished, by setting and resetting a finalizer on it.
|
|
||||||
type sentinel struct{ a *int }
|
|
||||||
var sentinelObj sentinel
|
|
||||||
var finalizer func(o *sentinel)
|
|
||||||
finalizer = func(o *sentinel) {
|
|
||||||
lk.Lock()
|
|
||||||
defer lk.Unlock()
|
|
||||||
select {
|
|
||||||
case gcTriggered <- struct{}{}:
|
|
||||||
default:
|
|
||||||
config.Logger.Warnf("failed to queue gc trigger; channel backlogged")
|
|
||||||
}
|
|
||||||
runtime.SetFinalizer(o, finalizer)
|
|
||||||
}
|
|
||||||
finalizer(&sentinelObj)
|
|
||||||
|
|
||||||
defer watchdog.wg.Done()
|
|
||||||
for {
|
|
||||||
var eventIsGc bool
|
|
||||||
select {
|
|
||||||
case <-Clock.After(config.Resolution):
|
|
||||||
// exit select.
|
|
||||||
|
|
||||||
case <-gcTriggered:
|
|
||||||
eventIsGc = true
|
|
||||||
// exit select.
|
|
||||||
|
|
||||||
case <-watchdog.closing:
|
|
||||||
runtime.SetFinalizer(&sentinelObj, nil) // clear the sentinel finalizer.
|
|
||||||
|
|
||||||
lk.Lock()
|
|
||||||
ch := gcTriggered
|
|
||||||
gcTriggered = nil
|
|
||||||
lk.Unlock()
|
|
||||||
|
|
||||||
// close and drain
|
|
||||||
close(ch)
|
|
||||||
for range ch {
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
|
var (
|
||||||
// ReadMemStats stops the world. But as of go1.9, it should only
|
// ReadMemStats stops the world. But as of go1.9, it should only
|
||||||
// take ~25µs to complete.
|
// take ~25µs to complete.
|
||||||
//
|
//
|
||||||
|
|
@ -215,50 +65,327 @@ func watch() {
|
||||||
//
|
//
|
||||||
// See: https://github.com/golang/go/issues/19812
|
// See: https://github.com/golang/go/issues/19812
|
||||||
// See: https://github.com/prometheus/client_golang/issues/403
|
// See: https://github.com/prometheus/client_golang/issues/403
|
||||||
|
memstatsFn = runtime.ReadMemStats
|
||||||
|
sysmemFn = (*gosigar.Mem).Get
|
||||||
|
)
|
||||||
|
|
||||||
if eventIsGc {
|
var (
|
||||||
config.Logger.Infof("watchdog after GC")
|
// ErrAlreadyStarted is returned when the user tries to start the watchdog more than once.
|
||||||
|
ErrAlreadyStarted = fmt.Errorf("singleton memory watchdog was already started")
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
// stateUnstarted represents an unstarted state.
|
||||||
|
stateUnstarted int32 = iota
|
||||||
|
// stateRunning represents an operational state.
|
||||||
|
stateRunning
|
||||||
|
// stateClosing represents a temporary closing state.
|
||||||
|
stateClosing
|
||||||
|
)
|
||||||
|
|
||||||
|
// _watchdog is a global singleton watchdog.
|
||||||
|
var _watchdog struct {
|
||||||
|
state int32 // guarded by atomic, one of state* constants.
|
||||||
|
|
||||||
|
scope UtilizationType
|
||||||
|
|
||||||
|
closing chan struct{}
|
||||||
|
wg sync.WaitGroup
|
||||||
|
}
|
||||||
|
|
||||||
|
// UtilizationType is the utilization metric in use.
|
||||||
|
type UtilizationType int
|
||||||
|
|
||||||
|
const (
|
||||||
|
// UtilizationSystem specifies that the policy compares against actual used
|
||||||
|
// system memory.
|
||||||
|
UtilizationSystem UtilizationType = iota
|
||||||
|
// UtilizationHeap specifies that the policy compares against heap used.
|
||||||
|
UtilizationHeap
|
||||||
|
)
|
||||||
|
|
||||||
|
// PolicyCtor is a policy constructor.
|
||||||
|
type PolicyCtor func(limit uint64) (Policy, error)
|
||||||
|
|
||||||
|
// Policy is polled by the watchdog to determine the next utilisation at which
|
||||||
|
// a GC should be forced.
|
||||||
|
type Policy interface {
|
||||||
|
// Evaluate determines when the next GC should take place. It receives the
|
||||||
|
// current usage, and it returns the next usage at which to trigger GC.
|
||||||
|
//
|
||||||
|
// The policy can request immediate GC, in which case next should match the
|
||||||
|
// used memory.
|
||||||
|
Evaluate(scope UtilizationType, used uint64) (next uint64, immediate bool)
|
||||||
|
}
|
||||||
|
|
||||||
|
// HeapDriven starts a singleton heap-driven watchdog.
|
||||||
|
//
|
||||||
|
// The heap-driven watchdog adjusts GOGC dynamically after every GC, to honour
|
||||||
|
// the policy. When an immediate GC is requested, runtime.GC() is called, and
|
||||||
|
// the policy is re-evaluated at the end of GC.
|
||||||
|
//
|
||||||
|
// It is entirely possible for the policy to keep requesting immediate GC
|
||||||
|
// repeateadly. This usually signals an emergency situation, and won't prevent
|
||||||
|
// the program from making progress, since the Go's garbage collection is not
|
||||||
|
// stop-the-world (for the major part).
|
||||||
|
//
|
||||||
|
// A limit value of 0 will error.
|
||||||
|
func HeapDriven(limit uint64, policyCtor PolicyCtor) (err error, stopFn func()) {
|
||||||
|
if !atomic.CompareAndSwapInt32(&_watchdog.state, stateUnstarted, stateRunning) {
|
||||||
|
return ErrAlreadyStarted, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
runtime.ReadMemStats(&memstats)
|
if limit == 0 {
|
||||||
|
return fmt.Errorf("cannot use zero limit for heap-driven watchdog"), nil
|
||||||
if err := sysmem.Get(); err != nil {
|
|
||||||
config.Logger.Warnf("failed to obtain system memory stats; err: %s", err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
trigger := config.Policy.Evaluate(PolicyInput{
|
policy, err := policyCtor(limit)
|
||||||
Scope: config.Scope,
|
if err != nil {
|
||||||
Limit: config.Limit,
|
return fmt.Errorf("failed to construct policy with limit %d: %w", limit, err), nil
|
||||||
MemStats: &memstats,
|
}
|
||||||
SysStats: &sysmem,
|
|
||||||
GCTrigger: eventIsGc,
|
|
||||||
Logger: config.Logger,
|
|
||||||
})
|
|
||||||
|
|
||||||
if !trigger {
|
_watchdog.scope = UtilizationHeap
|
||||||
|
_watchdog.closing = make(chan struct{})
|
||||||
|
|
||||||
|
var gcTriggered = make(chan struct{}, 16)
|
||||||
|
setupGCSentinel(gcTriggered)
|
||||||
|
|
||||||
|
_watchdog.wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer _watchdog.wg.Done()
|
||||||
|
|
||||||
|
// get the initial effective GOGC; guess it's 100 (default), and restore
|
||||||
|
// it to whatever it actually was. This works because SetGCPercent
|
||||||
|
// returns the previous value.
|
||||||
|
originalGOGC := debug.SetGCPercent(debug.SetGCPercent(100))
|
||||||
|
currGOGC := originalGOGC
|
||||||
|
|
||||||
|
var memstats runtime.MemStats
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-gcTriggered:
|
||||||
|
NotifyFired()
|
||||||
|
|
||||||
|
case <-_watchdog.closing:
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// recompute the next trigger.
|
||||||
|
memstatsFn(&memstats)
|
||||||
|
|
||||||
|
// heapMarked is the amount of heap that was marked as live by GC.
|
||||||
|
// it is inferred from our current GOGC and the new target picked.
|
||||||
|
heapMarked := uint64(float64(memstats.NextGC) / (1 + float64(currGOGC)/100))
|
||||||
|
if heapMarked == 0 {
|
||||||
|
// this shouldn't happen, but just in case; avoiding a div by 0.
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
config.Logger.Infof("watchdog policy fired")
|
// evaluate the policy.
|
||||||
|
next, immediate := policy.Evaluate(UtilizationHeap, memstats.HeapAlloc)
|
||||||
|
|
||||||
if f := config.NotifyFired; f != nil {
|
if immediate {
|
||||||
f()
|
// trigger a forced GC; because we're not making the finalizer
|
||||||
|
// skip sending to the trigger channel, we will get fired again.
|
||||||
|
// at this stage, the program is under significant pressure, and
|
||||||
|
// given that Go GC is not STW for the largest part, the worse
|
||||||
|
// thing that could happen from infinitely GC'ing is that the
|
||||||
|
// program will run in a degrated state for longer, possibly
|
||||||
|
// long enough for an operator to intervene.
|
||||||
|
Logger.Warnf("heap-driven watchdog requested immediate GC; " +
|
||||||
|
"system is probably under significant pressure; " +
|
||||||
|
"performance compromised")
|
||||||
|
forceGC(&memstats)
|
||||||
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
if !config.NotifyOnly {
|
// calculate how much to set GOGC to honour the next trigger point.
|
||||||
config.Logger.Infof("watchdog is triggering GC")
|
currGOGC = int(((float64(next) / float64(heapMarked)) - float64(1)) * 100)
|
||||||
start := time.Now()
|
if currGOGC >= originalGOGC {
|
||||||
runtime.GC()
|
Logger.Infof("heap watchdog: requested GOGC percent higher than default; capping at default; requested: %d; default: %d", currGOGC, originalGOGC)
|
||||||
runtime.ReadMemStats(&memstats)
|
currGOGC = originalGOGC
|
||||||
config.Logger.Infof("watchdog-triggered GC finished; took: %s; current heap allocated: %d bytes", time.Since(start), memstats.HeapAlloc)
|
} else {
|
||||||
|
if currGOGC < 1 {
|
||||||
|
currGOGC = 1
|
||||||
}
|
}
|
||||||
|
Logger.Infof("heap watchdog: setting GOGC percent: %d", currGOGC)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
debug.SetGCPercent(currGOGC)
|
||||||
|
|
||||||
|
memstatsFn(&memstats)
|
||||||
|
Logger.Infof("heap watchdog stats: heap_alloc: %d, heap_marked: %d, next_gc: %d, policy_next_gc: %d, gogc: %d",
|
||||||
|
memstats.HeapAlloc, heapMarked, memstats.NextGC, next, currGOGC)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
return nil, stop
|
||||||
}
|
}
|
||||||
|
|
||||||
func stopMemory() {
|
// SystemDriven starts a singleton system-driven watchdog.
|
||||||
if !atomic.CompareAndSwapInt64(&watchdog.state, stateStarted, stateUnstarted) {
|
//
|
||||||
|
// The system-driven watchdog keeps a threshold, above which GC will be forced.
|
||||||
|
// The watchdog polls the system utilization at the specified frequency. When
|
||||||
|
// the actual utilization exceeds the threshold, a GC is forced.
|
||||||
|
//
|
||||||
|
// This threshold is calculated by querying the policy every time that GC runs,
|
||||||
|
// either triggered by the runtime, or forced by us.
|
||||||
|
func SystemDriven(limit uint64, frequency time.Duration, policyCtor PolicyCtor) (err error, stopFn func()) {
|
||||||
|
if !atomic.CompareAndSwapInt32(&_watchdog.state, stateUnstarted, stateRunning) {
|
||||||
|
return ErrAlreadyStarted, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if limit == 0 {
|
||||||
|
limit, err = determineLimit(false)
|
||||||
|
if err != nil {
|
||||||
|
return err, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
policy, err := policyCtor(limit)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to construct policy with limit %d: %w", limit, err), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
_watchdog.scope = UtilizationSystem
|
||||||
|
_watchdog.closing = make(chan struct{})
|
||||||
|
|
||||||
|
var gcTriggered = make(chan struct{}, 16)
|
||||||
|
setupGCSentinel(gcTriggered)
|
||||||
|
|
||||||
|
_watchdog.wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer _watchdog.wg.Done()
|
||||||
|
|
||||||
|
var (
|
||||||
|
memstats runtime.MemStats
|
||||||
|
sysmem gosigar.Mem
|
||||||
|
threshold uint64
|
||||||
|
)
|
||||||
|
|
||||||
|
// initialize the threshold.
|
||||||
|
threshold, immediate := policy.Evaluate(UtilizationSystem, sysmem.ActualUsed)
|
||||||
|
if immediate {
|
||||||
|
Logger.Warnf("system-driven watchdog requested immediate GC upon startup; " +
|
||||||
|
"policy is probably misconfigured; " +
|
||||||
|
"performance compromised")
|
||||||
|
forceGC(&memstats)
|
||||||
|
}
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-Clock.After(frequency):
|
||||||
|
// get the current usage.
|
||||||
|
if err := sysmemFn(&sysmem); err != nil {
|
||||||
|
Logger.Warnf("failed to obtain system memory stats; err: %s", err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
actual := sysmem.ActualUsed
|
||||||
|
if actual < threshold {
|
||||||
|
// nothing to do.
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// trigger GC; this will emit a gcTriggered event which we'll
|
||||||
|
// consume next to readjust the threshold.
|
||||||
|
Logger.Warnf("system-driven watchdog triggering GC; %d/%d bytes (used/threshold)", actual, threshold)
|
||||||
|
forceGC(&memstats)
|
||||||
|
|
||||||
|
case <-gcTriggered:
|
||||||
|
NotifyFired()
|
||||||
|
|
||||||
|
// get the current usage.
|
||||||
|
if err := sysmemFn(&sysmem); err != nil {
|
||||||
|
Logger.Warnf("failed to obtain system memory stats; err: %s", err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// adjust the threshold.
|
||||||
|
threshold, immediate = policy.Evaluate(UtilizationSystem, sysmem.ActualUsed)
|
||||||
|
if immediate {
|
||||||
|
Logger.Warnf("system-driven watchdog triggering immediate GC; %d used bytes", sysmem.ActualUsed)
|
||||||
|
forceGC(&memstats)
|
||||||
|
}
|
||||||
|
|
||||||
|
case <-_watchdog.closing:
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
close(watchdog.closing)
|
}
|
||||||
watchdog.wg.Wait()
|
}()
|
||||||
|
|
||||||
|
return nil, stop
|
||||||
|
}
|
||||||
|
|
||||||
|
func determineLimit(restrictByProcess bool) (uint64, error) {
|
||||||
|
// TODO.
|
||||||
|
// if restrictByProcess {
|
||||||
|
// if pmem := ProcessMemoryLimit(); pmem > 0 {
|
||||||
|
// Logger.Infof("watchdog using process limit: %d bytes", pmem)
|
||||||
|
// return pmem, nil
|
||||||
|
// }
|
||||||
|
// Logger.Infof("watchdog was unable to determine process limit; falling back to total system memory")
|
||||||
|
// }
|
||||||
|
|
||||||
|
// populate initial utilisation and system stats.
|
||||||
|
var sysmem gosigar.Mem
|
||||||
|
if err := sysmemFn(&sysmem); err != nil {
|
||||||
|
return 0, fmt.Errorf("failed to get system memory stats: %w", err)
|
||||||
|
}
|
||||||
|
return sysmem.Total, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// forceGC forces a manual GC.
|
||||||
|
func forceGC(memstats *runtime.MemStats) {
|
||||||
|
Logger.Infof("watchdog is forcing GC")
|
||||||
|
|
||||||
|
// it's safe to assume that the finalizer will attempt to run before
|
||||||
|
// runtime.GC() returns because runtime.GC() waits for the sweep phase to
|
||||||
|
// finish before returning.
|
||||||
|
// finalizers are run in the sweep phase.
|
||||||
|
start := time.Now()
|
||||||
|
runtime.GC()
|
||||||
|
took := time.Since(start)
|
||||||
|
|
||||||
|
memstatsFn(memstats)
|
||||||
|
Logger.Infof("watchdog-triggered GC finished; took: %s; current heap allocated: %d bytes", took, memstats.HeapAlloc)
|
||||||
|
}
|
||||||
|
|
||||||
|
func setupGCSentinel(gcTriggered chan struct{}) {
|
||||||
|
logger := Logger
|
||||||
|
|
||||||
|
// this non-zero sized struct is used as a sentinel to detect when a GC
|
||||||
|
// run has finished, by setting and resetting a finalizer on it.
|
||||||
|
// it essentially creates a GC notification "flywheel"; every GC will
|
||||||
|
// trigger this finalizer, which will reset itself so it gets notified
|
||||||
|
// of the next GC, breaking the cycle when the watchdog is stopped.
|
||||||
|
type sentinel struct{ a *int }
|
||||||
|
var finalizer func(o *sentinel)
|
||||||
|
finalizer = func(o *sentinel) {
|
||||||
|
if atomic.LoadInt32(&_watchdog.state) != stateRunning {
|
||||||
|
// this GC triggered after the watchdog was stopped; ignore
|
||||||
|
// and do not reset the finalizer.
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// reset so it triggers on the next GC.
|
||||||
|
runtime.SetFinalizer(o, finalizer)
|
||||||
|
|
||||||
|
select {
|
||||||
|
case gcTriggered <- struct{}{}:
|
||||||
|
default:
|
||||||
|
logger.Warnf("failed to queue gc trigger; channel backlogged")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
runtime.SetFinalizer(&sentinel{}, finalizer) // start the flywheel.
|
||||||
|
}
|
||||||
|
|
||||||
|
func stop() {
|
||||||
|
if !atomic.CompareAndSwapInt32(&_watchdog.state, stateRunning, stateClosing) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
close(_watchdog.closing)
|
||||||
|
_watchdog.wg.Wait()
|
||||||
|
|
||||||
|
atomic.StoreInt32(&_watchdog.state, stateUnstarted)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
191
watchdog_test.go
191
watchdog_test.go
|
|
@ -1,92 +1,151 @@
|
||||||
package watchdog
|
package watchdog
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"fmt"
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
"runtime"
|
"runtime"
|
||||||
|
"runtime/debug"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/elastic/gosigar"
|
||||||
"github.com/raulk/clock"
|
"github.com/raulk/clock"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
type testPolicy struct {
|
// These integration tests are a hugely non-deterministic, but necessary to get
|
||||||
ch chan *PolicyInput
|
// good coverage and confidence. The Go runtime makes its own pacing decisions,
|
||||||
trigger bool
|
// and those may vary based on machine, OS, kernel memory management, other
|
||||||
|
// running programs, exogenous memory pressure, and Go runtime versions.
|
||||||
|
//
|
||||||
|
// The assertions we use here are lax, but should be sufficient to serve as a
|
||||||
|
// reasonable litmus test of whether the watchdog is doing what it's supposed
|
||||||
|
// to or not.
|
||||||
|
|
||||||
|
var (
|
||||||
|
limit uint64 = 64 << 20 // 64MiB.
|
||||||
|
)
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
Logger = &stdlog{log: log.New(os.Stdout, "[watchdog test] ", log.LstdFlags|log.Lmsgprefix), debug: true}
|
||||||
}
|
}
|
||||||
|
|
||||||
var _ Policy = (*testPolicy)(nil)
|
func TestControl(t *testing.T) {
|
||||||
|
debug.SetGCPercent(100)
|
||||||
|
|
||||||
func (t *testPolicy) Evaluate(input PolicyInput) (trigger bool) {
|
// retain 1MiB every iteration, up to 100MiB (beyond heap limit!).
|
||||||
t.ch <- &input
|
var retained [][]byte
|
||||||
return t.trigger
|
for i := 0; i < 100; i++ {
|
||||||
}
|
b := make([]byte, 1*1024*1024)
|
||||||
|
for i := range b {
|
||||||
|
b[i] = byte(i)
|
||||||
|
}
|
||||||
|
retained = append(retained, b)
|
||||||
|
}
|
||||||
|
|
||||||
func TestWatchdog(t *testing.T) {
|
for _, b := range retained {
|
||||||
clk := clock.NewMock()
|
for i := range b {
|
||||||
Clock = clk
|
b[i] = byte(i)
|
||||||
|
}
|
||||||
tp := &testPolicy{ch: make(chan *PolicyInput, 1)}
|
}
|
||||||
notifyCh := make(chan struct{}, 1)
|
|
||||||
|
|
||||||
err, stop := Memory(MemConfig{
|
|
||||||
Scope: ScopeHeap,
|
|
||||||
Limit: 100,
|
|
||||||
Resolution: 10 * time.Second,
|
|
||||||
Policy: tp,
|
|
||||||
NotifyFired: func() { notifyCh <- struct{}{} },
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
|
||||||
defer stop()
|
|
||||||
|
|
||||||
time.Sleep(500 * time.Millisecond) // wait til the watchdog goroutine starts.
|
|
||||||
|
|
||||||
clk.Add(10 * time.Second)
|
|
||||||
pi := <-tp.ch
|
|
||||||
require.EqualValues(t, 100, pi.Limit)
|
|
||||||
require.NotNil(t, pi.MemStats)
|
|
||||||
require.NotNil(t, pi.SysStats)
|
|
||||||
require.Equal(t, ScopeHeap, pi.Scope)
|
|
||||||
|
|
||||||
require.Len(t, notifyCh, 0)
|
|
||||||
|
|
||||||
var ms runtime.MemStats
|
var ms runtime.MemStats
|
||||||
runtime.ReadMemStats(&ms)
|
runtime.ReadMemStats(&ms)
|
||||||
require.EqualValues(t, 0, ms.NumGC)
|
require.LessOrEqual(t, ms.NumGC, uint32(8)) // a maximum of 8 GCs should've happened.
|
||||||
|
require.Zero(t, ms.NumForcedGC) // no forced GCs.
|
||||||
|
}
|
||||||
|
|
||||||
// now fire.
|
func TestHeapDriven(t *testing.T) {
|
||||||
tp.trigger = true
|
// we can't mock ReadMemStats, because we're relying on the go runtime to
|
||||||
|
// enforce the GC run, and the go runtime won't use our mock. Therefore, we
|
||||||
|
// need to do the actual thing.
|
||||||
|
debug.SetGCPercent(100)
|
||||||
|
|
||||||
clk.Add(10 * time.Second)
|
clk := clock.NewMock()
|
||||||
pi = <-tp.ch
|
Clock = clk
|
||||||
require.EqualValues(t, 100, pi.Limit)
|
|
||||||
require.NotNil(t, pi.MemStats)
|
|
||||||
require.NotNil(t, pi.SysStats)
|
|
||||||
require.Equal(t, ScopeHeap, pi.Scope)
|
|
||||||
|
|
||||||
time.Sleep(500 * time.Millisecond) // wait until the watchdog runs.
|
|
||||||
|
|
||||||
require.Len(t, notifyCh, 1)
|
|
||||||
|
|
||||||
|
observations := make([]*runtime.MemStats, 0, 100)
|
||||||
|
NotifyFired = func() {
|
||||||
|
var ms runtime.MemStats
|
||||||
runtime.ReadMemStats(&ms)
|
runtime.ReadMemStats(&ms)
|
||||||
require.EqualValues(t, 1, ms.NumGC)
|
observations = append(observations, &ms)
|
||||||
}
|
|
||||||
|
|
||||||
func TestDoubleClose(t *testing.T) {
|
|
||||||
defer func() {
|
|
||||||
if r := recover(); r != nil {
|
|
||||||
t.Fatal(r)
|
|
||||||
}
|
}
|
||||||
}()
|
|
||||||
|
|
||||||
tp := &testPolicy{ch: make(chan *PolicyInput, 1)}
|
// limit is 64MiB.
|
||||||
|
err, stopFn := HeapDriven(limit, NewAdaptivePolicy(0.5))
|
||||||
err, stop := Memory(MemConfig{
|
|
||||||
Scope: ScopeHeap,
|
|
||||||
Limit: 100,
|
|
||||||
Resolution: 10 * time.Second,
|
|
||||||
Policy: tp,
|
|
||||||
})
|
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
stop()
|
defer stopFn()
|
||||||
stop()
|
|
||||||
|
time.Sleep(500 * time.Millisecond) // give time for the watchdog to init.
|
||||||
|
|
||||||
|
// retain 1MiB every iteration, up to 100MiB (beyond heap limit!).
|
||||||
|
var retained [][]byte
|
||||||
|
for i := 0; i < 100; i++ {
|
||||||
|
retained = append(retained, make([]byte, 1*1024*1024))
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, o := range observations {
|
||||||
|
fmt.Println("heap alloc:", o.HeapAlloc, "next gc:", o.NextGC, "gc count:", o.NumGC, "forced gc:", o.NumForcedGC)
|
||||||
|
}
|
||||||
|
|
||||||
|
var ms runtime.MemStats
|
||||||
|
runtime.ReadMemStats(&ms)
|
||||||
|
require.GreaterOrEqual(t, ms.NumGC, uint32(12)) // over 12 GCs should've taken place.
|
||||||
|
require.GreaterOrEqual(t, ms.NumForcedGC, uint32(5)) // at least 5 forced GCs.
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSystemDriven(t *testing.T) {
|
||||||
|
debug.SetGCPercent(100)
|
||||||
|
|
||||||
|
clk := clock.NewMock()
|
||||||
|
Clock = clk
|
||||||
|
|
||||||
|
// mock the system reporting.
|
||||||
|
var actualUsed uint64
|
||||||
|
sysmemFn = func(g *gosigar.Mem) error {
|
||||||
|
g.ActualUsed = actualUsed
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// limit is 64MiB.
|
||||||
|
err, stopFn := SystemDriven(limit, 5*time.Second, NewAdaptivePolicy(0.5))
|
||||||
|
require.NoError(t, err)
|
||||||
|
defer stopFn()
|
||||||
|
|
||||||
|
time.Sleep(200 * time.Millisecond) // give time for the watchdog to init.
|
||||||
|
|
||||||
|
notifyCh := make(chan struct{}, 1)
|
||||||
|
NotifyFired = func() {
|
||||||
|
notifyCh <- struct{}{}
|
||||||
|
}
|
||||||
|
|
||||||
|
// first tick; used = 0.
|
||||||
|
clk.Add(5 * time.Second)
|
||||||
|
time.Sleep(200 * time.Millisecond)
|
||||||
|
require.Len(t, notifyCh, 0) // no GC has taken place.
|
||||||
|
|
||||||
|
// second tick; used = just over 50%; will trigger GC.
|
||||||
|
actualUsed = (limit / 2) + 1
|
||||||
|
clk.Add(5 * time.Second)
|
||||||
|
time.Sleep(200 * time.Millisecond)
|
||||||
|
require.Len(t, notifyCh, 1)
|
||||||
|
<-notifyCh
|
||||||
|
|
||||||
|
// third tick; just below 75%; no GC.
|
||||||
|
actualUsed = uint64(float64(limit)*0.75) - 1
|
||||||
|
clk.Add(5 * time.Second)
|
||||||
|
time.Sleep(200 * time.Millisecond)
|
||||||
|
require.Len(t, notifyCh, 0)
|
||||||
|
|
||||||
|
// fourth tick; 75% exactly; will trigger GC.
|
||||||
|
actualUsed = uint64(float64(limit)*0.75) + 1
|
||||||
|
clk.Add(5 * time.Second)
|
||||||
|
time.Sleep(200 * time.Millisecond)
|
||||||
|
require.Len(t, notifyCh, 1)
|
||||||
|
<-notifyCh
|
||||||
|
|
||||||
|
var ms runtime.MemStats
|
||||||
|
runtime.ReadMemStats(&ms)
|
||||||
|
require.GreaterOrEqual(t, ms.NumForcedGC, uint32(2))
|
||||||
}
|
}
|
||||||
|
|
|
||||||
163
watermarks.go
163
watermarks.go
|
|
@ -1,142 +1,41 @@
|
||||||
package watchdog
|
package watchdog
|
||||||
|
|
||||||
import (
|
// NewWatermarkPolicy creates a watchdog policy that schedules GC at concrete
|
||||||
"math"
|
// watermarks. When queried, it will determine the next trigger point based
|
||||||
"time"
|
// on the current utilisation. If the last watermark is surpassed,
|
||||||
)
|
// the policy will request immediate GC.
|
||||||
|
func NewWatermarkPolicy(watermarks ...float64) PolicyCtor {
|
||||||
|
return func(limit uint64) (Policy, error) {
|
||||||
|
p := new(watermarkPolicy)
|
||||||
|
p.limit = limit
|
||||||
|
p.thresholds = make([]uint64, 0, len(watermarks))
|
||||||
|
for _, m := range watermarks {
|
||||||
|
p.thresholds = append(p.thresholds, uint64(float64(limit)*m))
|
||||||
|
}
|
||||||
|
Logger.Infof("initialized watermark watchdog policy; watermarks: %v; thresholds: %v", p.watermarks, p.thresholds)
|
||||||
|
return p, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// WatermarkPolicy is a watchdog firing policy that triggers when watermarks are
|
type watermarkPolicy struct {
|
||||||
// surpassed in the increasing direction.
|
// watermarks are the percentual amounts of limit.
|
||||||
//
|
watermarks []float64
|
||||||
// For example, a policy configured with the watermarks 0.50, 0.75, 0.80, and
|
// thresholds are the absolute trigger points of this policy.
|
||||||
// 0.99 will trigger at most once, and only once, each time that a watermark
|
|
||||||
// is surpassed upwards.
|
|
||||||
//
|
|
||||||
// Even if utilisation pierces through several watermarks at once between two
|
|
||||||
// subsequent calls to Evaluate, the policy will fire only once.
|
|
||||||
//
|
|
||||||
// It is possible to suppress the watermark policy from firing too often by
|
|
||||||
// setting a Silence period. When non-zero, if a watermark is surpassed within
|
|
||||||
// the Silence window (using the last GC timestamp as basis), that event will
|
|
||||||
// not immediately trigger firing. Instead, the policy will wait until the
|
|
||||||
// silence period is over, and only then will it fire if utilisation is still
|
|
||||||
// beyond that watermark.
|
|
||||||
//
|
|
||||||
// At last, if an EmergencyWatermark is set, when utilisation is above that
|
|
||||||
// level, the Silence period will be ignored and the policy will fire
|
|
||||||
// persistenly, as long as utilisation stays above that watermark.
|
|
||||||
type WatermarkPolicy struct {
|
|
||||||
// Watermarks are the percentual amounts of limit. The policy will panic if
|
|
||||||
// Watermarks is zero length.
|
|
||||||
Watermarks []float64
|
|
||||||
|
|
||||||
// EmergencyWatermark is a watermark that, when surpassed, puts this
|
|
||||||
// watchdog in emergency mode. During emergency mode, the system is
|
|
||||||
// considered to be under significant memory pressure, and the Quiesce
|
|
||||||
// period is not honoured.
|
|
||||||
EmergencyWatermark float64
|
|
||||||
|
|
||||||
// Silence is the quiet period the watchdog will honour except when in
|
|
||||||
// emergency mode.
|
|
||||||
Silence time.Duration
|
|
||||||
|
|
||||||
// internal state.
|
|
||||||
thresholds []uint64
|
thresholds []uint64
|
||||||
currIdx int // idx of the current watermark.
|
limit uint64
|
||||||
lastIdx int
|
|
||||||
firedLast bool
|
|
||||||
silenceNs int64
|
|
||||||
initialized bool
|
|
||||||
}
|
}
|
||||||
|
|
||||||
var _ Policy = (*WatermarkPolicy)(nil)
|
var _ Policy = (*watermarkPolicy)(nil)
|
||||||
|
|
||||||
func (w *WatermarkPolicy) Evaluate(input PolicyInput) (trigger bool) {
|
func (w *watermarkPolicy) Evaluate(_ UtilizationType, used uint64) (next uint64, immediate bool) {
|
||||||
if !w.initialized {
|
Logger.Debugf("watermark policy: evaluating; utilization: %d/%d (used/limit)", used, w.limit)
|
||||||
w.thresholds = make([]uint64, 0, len(w.Watermarks))
|
var i int
|
||||||
for _, m := range w.Watermarks {
|
for ; i < len(w.thresholds); i++ {
|
||||||
w.thresholds = append(w.thresholds, uint64(float64(input.Limit)*m))
|
t := w.thresholds[i]
|
||||||
}
|
if used < t {
|
||||||
w.silenceNs = w.Silence.Nanoseconds()
|
return t, false
|
||||||
w.lastIdx = len(w.Watermarks) - 1
|
|
||||||
w.initialized = true
|
|
||||||
input.Logger.Infof("initialized watermark watchdog policy; watermarks: %v; emergency watermark: %f, thresholds: %v; silence period: %s",
|
|
||||||
w.Watermarks, w.EmergencyWatermark, w.thresholds, w.Silence)
|
|
||||||
}
|
|
||||||
|
|
||||||
// determine the value to compare utilisation against.
|
|
||||||
var actual uint64
|
|
||||||
switch input.Scope {
|
|
||||||
case ScopeSystem:
|
|
||||||
actual = input.SysStats.ActualUsed
|
|
||||||
case ScopeHeap:
|
|
||||||
actual = input.MemStats.HeapAlloc
|
|
||||||
}
|
|
||||||
|
|
||||||
input.Logger.Debugf("watermark policy: evaluating; curr_watermark: %f, utilization: %d/%d/%d (used/curr_threshold/limit)",
|
|
||||||
w.Watermarks[w.currIdx], actual, w.thresholds[w.currIdx], input.Limit)
|
|
||||||
|
|
||||||
// determine whether we're past the emergency watermark; if we are, we fire
|
|
||||||
// unconditionally. Disabled if 0.
|
|
||||||
if w.EmergencyWatermark > 0 {
|
|
||||||
currPercentage := math.Round((float64(actual)/float64(input.Limit))*DecimalPrecision) / DecimalPrecision // round float.
|
|
||||||
if pastEmergency := currPercentage >= w.EmergencyWatermark; pastEmergency {
|
|
||||||
emergencyThreshold := uint64(float64(input.Limit) / w.EmergencyWatermark)
|
|
||||||
input.Logger.Infof("watermark policy: emergency trigger; perc: %f/%f (%% used/%% emergency), utilization: %d/%d/%d (used/emergency/limit), used: %d, limit: %d, ",
|
|
||||||
currPercentage, w.EmergencyWatermark, actual, emergencyThreshold, input.Limit)
|
|
||||||
return true
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// we reached the maximum threshold, so fire immediately.
|
||||||
// short-circuit if within the silencing period.
|
return used, true
|
||||||
if silencing := w.silenceNs > 0; silencing {
|
|
||||||
now := Clock.Now().UnixNano()
|
|
||||||
if elapsed := now - int64(input.MemStats.LastGC); elapsed < w.silenceNs {
|
|
||||||
input.Logger.Debugf("watermark policy: silenced")
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// check if the the utilisation is below our current threshold; try
|
|
||||||
// to downscale before returning false.
|
|
||||||
if actual < w.thresholds[w.currIdx] {
|
|
||||||
for w.currIdx > 0 {
|
|
||||||
if actual >= w.thresholds[w.currIdx-1] {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
w.firedLast = false
|
|
||||||
w.currIdx--
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// we are above our current watermark, but if this is the last watermark and
|
|
||||||
// we've already fired, suppress the firing.
|
|
||||||
if w.currIdx == w.lastIdx && w.firedLast {
|
|
||||||
input.Logger.Debugf("watermark policy: last watermark already fired; skipping")
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// if our value is above the last threshold, record the last threshold as
|
|
||||||
// our current watermark, and also the fact that we've already fired for
|
|
||||||
// it.
|
|
||||||
if actual >= w.thresholds[w.lastIdx] {
|
|
||||||
w.currIdx = w.lastIdx
|
|
||||||
w.firedLast = true
|
|
||||||
input.Logger.Infof("watermark policy triggering: %f watermark surpassed", w.Watermarks[w.currIdx])
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
input.Logger.Infof("watermark policy triggering: %f watermark surpassed", w.Watermarks[w.currIdx])
|
|
||||||
|
|
||||||
// we are triggering; update the current watermark, upscaling it to the
|
|
||||||
// next threshold.
|
|
||||||
for w.currIdx < w.lastIdx {
|
|
||||||
w.currIdx++
|
|
||||||
if actual < w.thresholds[w.currIdx] {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return true
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,288 +1,61 @@
|
||||||
package watchdog
|
package watchdog
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"log"
|
|
||||||
"os"
|
|
||||||
"runtime"
|
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/elastic/gosigar"
|
|
||||||
"github.com/raulk/clock"
|
"github.com/raulk/clock"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
limit uint64 = 64 << 20 // 64MiB.
|
watermarks = []float64{0.50, 0.75, 0.80}
|
||||||
firstWatermark = 0.50
|
thresholds = func() []uint64 {
|
||||||
secondWatermark = 0.75
|
var ret []uint64
|
||||||
thirdWatermark = 0.80
|
for _, w := range watermarks {
|
||||||
emergencyWatermark = 0.90
|
ret = append(ret, uint64(float64(limit)*w))
|
||||||
|
}
|
||||||
logger = &stdlog{log: log.New(os.Stdout, "[watchdog test] ", log.LstdFlags|log.Lmsgprefix), debug: true}
|
return ret
|
||||||
|
}()
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestProgressiveWatermarksSystem(t *testing.T) {
|
func TestProgressiveWatermarks(t *testing.T) {
|
||||||
clk := clock.NewMock()
|
clk := clock.NewMock()
|
||||||
Clock = clk
|
Clock = clk
|
||||||
|
|
||||||
p := WatermarkPolicy{
|
p, err := NewWatermarkPolicy(watermarks...)(limit)
|
||||||
Watermarks: []float64{firstWatermark, secondWatermark, thirdWatermark},
|
require.NoError(t, err)
|
||||||
EmergencyWatermark: emergencyWatermark,
|
|
||||||
}
|
|
||||||
|
|
||||||
require.False(t, p.Evaluate(PolicyInput{
|
// at zero
|
||||||
Logger: logger,
|
next, immediate := p.Evaluate(UtilizationSystem, uint64(0))
|
||||||
Scope: ScopeSystem,
|
require.False(t, immediate)
|
||||||
Limit: limit,
|
require.EqualValues(t, thresholds[0], next)
|
||||||
SysStats: &gosigar.Mem{ActualUsed: uint64(float64(limit)*firstWatermark) - 1},
|
|
||||||
}))
|
|
||||||
|
|
||||||
// trigger the first watermark.
|
// before the watermark.
|
||||||
require.True(t, p.Evaluate(PolicyInput{
|
next, immediate = p.Evaluate(UtilizationSystem, uint64(float64(limit)*watermarks[0])-1)
|
||||||
Logger: logger,
|
require.False(t, immediate)
|
||||||
Scope: ScopeSystem,
|
require.EqualValues(t, thresholds[0], next)
|
||||||
Limit: limit,
|
|
||||||
MemStats: &runtime.MemStats{LastGC: uint64(time.Now().UnixNano())},
|
|
||||||
SysStats: &gosigar.Mem{ActualUsed: uint64(float64(limit) * firstWatermark)},
|
|
||||||
}))
|
|
||||||
|
|
||||||
// this won't fire because we're still on the same watermark.
|
// exactly at the watermark; gives us the next watermark, as the watchdodg would've
|
||||||
for i := 0; i < 100; i++ {
|
// taken care of triggering the first watermark.
|
||||||
require.False(t, p.Evaluate(PolicyInput{
|
next, immediate = p.Evaluate(UtilizationSystem, uint64(float64(limit)*watermarks[0]))
|
||||||
Logger: logger,
|
require.False(t, immediate)
|
||||||
Scope: ScopeSystem,
|
require.EqualValues(t, thresholds[1], next)
|
||||||
Limit: limit,
|
|
||||||
MemStats: &runtime.MemStats{LastGC: uint64(time.Now().UnixNano())},
|
|
||||||
SysStats: &gosigar.Mem{ActualUsed: uint64(float64(limit) * firstWatermark)},
|
|
||||||
}))
|
|
||||||
}
|
|
||||||
|
|
||||||
// now let's move to the second watermark.
|
// after the watermark gives us the next watermark.
|
||||||
require.True(t, p.Evaluate(PolicyInput{
|
next, immediate = p.Evaluate(UtilizationSystem, uint64(float64(limit)*watermarks[0])+1)
|
||||||
Logger: logger,
|
require.False(t, immediate)
|
||||||
Scope: ScopeSystem,
|
require.EqualValues(t, thresholds[1], next)
|
||||||
Limit: limit,
|
|
||||||
MemStats: &runtime.MemStats{LastGC: uint64(time.Now().UnixNano())},
|
|
||||||
SysStats: &gosigar.Mem{ActualUsed: uint64(float64(limit) * secondWatermark)},
|
|
||||||
}))
|
|
||||||
|
|
||||||
// this won't fire because we're still on the same watermark.
|
// last watermark; always triggers.
|
||||||
for i := 0; i < 100; i++ {
|
next, immediate = p.Evaluate(UtilizationSystem, uint64(float64(limit)*watermarks[2]))
|
||||||
require.False(t, p.Evaluate(PolicyInput{
|
require.True(t, immediate)
|
||||||
Logger: logger,
|
require.EqualValues(t, uint64(float64(limit)*watermarks[2]), next)
|
||||||
Scope: ScopeSystem,
|
|
||||||
Limit: limit,
|
|
||||||
MemStats: &runtime.MemStats{LastGC: uint64(0)},
|
|
||||||
SysStats: &gosigar.Mem{ActualUsed: uint64(float64(limit) * secondWatermark)},
|
|
||||||
}))
|
|
||||||
}
|
|
||||||
|
|
||||||
// now let's move to the third and last watermark.
|
next, immediate = p.Evaluate(UtilizationSystem, uint64(float64(limit)*watermarks[2]+1))
|
||||||
require.True(t, p.Evaluate(PolicyInput{
|
require.True(t, immediate)
|
||||||
Logger: logger,
|
require.EqualValues(t, uint64(float64(limit)*watermarks[2])+1, next)
|
||||||
Scope: ScopeSystem,
|
|
||||||
Limit: limit,
|
|
||||||
MemStats: &runtime.MemStats{LastGC: uint64(0)},
|
|
||||||
SysStats: &gosigar.Mem{ActualUsed: uint64(float64(limit) * thirdWatermark)},
|
|
||||||
}))
|
|
||||||
|
|
||||||
// this won't fire because we're still on the same watermark.
|
next, immediate = p.Evaluate(UtilizationSystem, limit)
|
||||||
for i := 0; i < 100; i++ {
|
require.True(t, immediate)
|
||||||
require.False(t, p.Evaluate(PolicyInput{
|
require.EqualValues(t, limit, next)
|
||||||
Logger: logger,
|
|
||||||
Scope: ScopeSystem,
|
|
||||||
Limit: limit,
|
|
||||||
MemStats: &runtime.MemStats{LastGC: uint64(0)},
|
|
||||||
SysStats: &gosigar.Mem{ActualUsed: uint64(float64(limit) * thirdWatermark)},
|
|
||||||
}))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestProgressiveWatermarksHeap(t *testing.T) {
|
|
||||||
clk := clock.NewMock()
|
|
||||||
Clock = clk
|
|
||||||
|
|
||||||
p := WatermarkPolicy{
|
|
||||||
Watermarks: []float64{firstWatermark, secondWatermark, thirdWatermark},
|
|
||||||
}
|
|
||||||
|
|
||||||
// now back up step by step, check that all of them fire.
|
|
||||||
for _, wm := range []float64{firstWatermark, secondWatermark, thirdWatermark} {
|
|
||||||
require.True(t, p.Evaluate(PolicyInput{
|
|
||||||
Logger: logger,
|
|
||||||
Scope: ScopeHeap,
|
|
||||||
Limit: limit,
|
|
||||||
MemStats: &runtime.MemStats{LastGC: uint64(0), HeapAlloc: uint64(float64(limit) * wm)},
|
|
||||||
SysStats: &gosigar.Mem{ActualUsed: 0},
|
|
||||||
}))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestDownscalingWatermarks_Reentrancy(t *testing.T) {
|
|
||||||
clk := clock.NewMock()
|
|
||||||
Clock = clk
|
|
||||||
|
|
||||||
p := WatermarkPolicy{
|
|
||||||
Watermarks: []float64{firstWatermark, secondWatermark, thirdWatermark},
|
|
||||||
}
|
|
||||||
|
|
||||||
// crank all the way to the top.
|
|
||||||
require.True(t, p.Evaluate(PolicyInput{
|
|
||||||
Logger: logger,
|
|
||||||
Scope: ScopeSystem,
|
|
||||||
Limit: limit,
|
|
||||||
MemStats: &runtime.MemStats{LastGC: uint64(0)},
|
|
||||||
SysStats: &gosigar.Mem{ActualUsed: limit},
|
|
||||||
}))
|
|
||||||
|
|
||||||
// now back down, checking that none of the boundaries fire.
|
|
||||||
for _, wm := range []float64{thirdWatermark, secondWatermark, firstWatermark} {
|
|
||||||
require.False(t, p.Evaluate(PolicyInput{
|
|
||||||
Logger: logger,
|
|
||||||
Scope: ScopeSystem,
|
|
||||||
Limit: limit,
|
|
||||||
MemStats: &runtime.MemStats{LastGC: uint64(0)},
|
|
||||||
SysStats: &gosigar.Mem{ActualUsed: uint64(float64(limit)*wm - 1)},
|
|
||||||
}))
|
|
||||||
}
|
|
||||||
|
|
||||||
// now back up step by step, check that all of them fire.
|
|
||||||
for _, wm := range []float64{firstWatermark, secondWatermark, thirdWatermark} {
|
|
||||||
require.True(t, p.Evaluate(PolicyInput{
|
|
||||||
Logger: logger,
|
|
||||||
Scope: ScopeSystem,
|
|
||||||
Limit: limit,
|
|
||||||
MemStats: &runtime.MemStats{LastGC: uint64(0)},
|
|
||||||
SysStats: &gosigar.Mem{ActualUsed: uint64(float64(limit)*wm + 1)},
|
|
||||||
}))
|
|
||||||
}
|
|
||||||
|
|
||||||
// check the top does not fire because it already fired.
|
|
||||||
for i := 0; i < 100; i++ {
|
|
||||||
require.False(t, p.Evaluate(PolicyInput{
|
|
||||||
Logger: logger,
|
|
||||||
Scope: ScopeSystem,
|
|
||||||
Limit: limit,
|
|
||||||
MemStats: &runtime.MemStats{LastGC: uint64(0)},
|
|
||||||
SysStats: &gosigar.Mem{ActualUsed: limit},
|
|
||||||
}))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestEmergencyWatermark(t *testing.T) {
|
|
||||||
clk := clock.NewMock()
|
|
||||||
Clock = clk
|
|
||||||
|
|
||||||
p := WatermarkPolicy{
|
|
||||||
Watermarks: []float64{firstWatermark},
|
|
||||||
EmergencyWatermark: emergencyWatermark,
|
|
||||||
Silence: 1 * time.Minute,
|
|
||||||
}
|
|
||||||
|
|
||||||
// every tick triggers, even within the silence period.
|
|
||||||
for i := 0; i < 100; i++ {
|
|
||||||
require.True(t, p.Evaluate(PolicyInput{
|
|
||||||
Logger: logger,
|
|
||||||
Scope: ScopeSystem,
|
|
||||||
Limit: limit,
|
|
||||||
MemStats: &runtime.MemStats{LastGC: uint64(clk.Now().UnixNano())},
|
|
||||||
SysStats: &gosigar.Mem{ActualUsed: uint64(float64(limit) * emergencyWatermark)},
|
|
||||||
}))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestJumpWatermark(t *testing.T) {
|
|
||||||
clk := clock.NewMock()
|
|
||||||
Clock = clk
|
|
||||||
|
|
||||||
p := WatermarkPolicy{
|
|
||||||
Watermarks: []float64{firstWatermark, secondWatermark, thirdWatermark},
|
|
||||||
}
|
|
||||||
|
|
||||||
// check that jumping to the top only fires once.
|
|
||||||
require.True(t, p.Evaluate(PolicyInput{
|
|
||||||
Logger: logger,
|
|
||||||
Scope: ScopeSystem,
|
|
||||||
Limit: limit,
|
|
||||||
MemStats: &runtime.MemStats{LastGC: uint64(0)},
|
|
||||||
SysStats: &gosigar.Mem{ActualUsed: limit},
|
|
||||||
}))
|
|
||||||
|
|
||||||
for i := 0; i < 100; i++ {
|
|
||||||
require.False(t, p.Evaluate(PolicyInput{
|
|
||||||
Logger: logger,
|
|
||||||
Scope: ScopeSystem,
|
|
||||||
Limit: limit,
|
|
||||||
MemStats: &runtime.MemStats{LastGC: uint64(0)},
|
|
||||||
SysStats: &gosigar.Mem{ActualUsed: limit},
|
|
||||||
}))
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSilencePeriod(t *testing.T) {
|
|
||||||
clk := clock.NewMock()
|
|
||||||
Clock = clk
|
|
||||||
|
|
||||||
var (
|
|
||||||
limit uint64 = 64 << 20 // 64MiB.
|
|
||||||
firstWatermark = 0.50
|
|
||||||
secondWatermark = 0.75
|
|
||||||
thirdWatermark = 0.80
|
|
||||||
)
|
|
||||||
|
|
||||||
p := WatermarkPolicy{
|
|
||||||
Watermarks: []float64{firstWatermark, secondWatermark, thirdWatermark},
|
|
||||||
Silence: 1 * time.Minute,
|
|
||||||
}
|
|
||||||
|
|
||||||
// going above the first threshold, but within silencing period, so nothing happens.
|
|
||||||
for i := 0; i < 100; i++ {
|
|
||||||
require.False(t, p.Evaluate(PolicyInput{
|
|
||||||
Logger: logger,
|
|
||||||
Scope: ScopeSystem,
|
|
||||||
Limit: limit,
|
|
||||||
MemStats: &runtime.MemStats{LastGC: uint64(clk.Now().UnixNano())},
|
|
||||||
SysStats: &gosigar.Mem{ActualUsed: uint64(float64(limit) * firstWatermark)},
|
|
||||||
}))
|
|
||||||
}
|
|
||||||
|
|
||||||
// now outside the silencing period, we do fire.
|
|
||||||
clk.Add(time.Minute)
|
|
||||||
|
|
||||||
require.True(t, p.Evaluate(PolicyInput{
|
|
||||||
Logger: logger,
|
|
||||||
Scope: ScopeSystem,
|
|
||||||
Limit: limit,
|
|
||||||
MemStats: &runtime.MemStats{LastGC: 0},
|
|
||||||
SysStats: &gosigar.Mem{ActualUsed: uint64(float64(limit) * firstWatermark)},
|
|
||||||
}))
|
|
||||||
|
|
||||||
// but not the second time.
|
|
||||||
require.False(t, p.Evaluate(PolicyInput{
|
|
||||||
Logger: logger,
|
|
||||||
Scope: ScopeSystem,
|
|
||||||
Limit: limit,
|
|
||||||
MemStats: &runtime.MemStats{LastGC: 0},
|
|
||||||
SysStats: &gosigar.Mem{ActualUsed: uint64(float64(limit) * firstWatermark)},
|
|
||||||
}))
|
|
||||||
|
|
||||||
// now let's go up inside the silencing period, nothing happens.
|
|
||||||
require.False(t, p.Evaluate(PolicyInput{
|
|
||||||
Logger: logger,
|
|
||||||
Scope: ScopeSystem,
|
|
||||||
Limit: limit,
|
|
||||||
MemStats: &runtime.MemStats{LastGC: uint64(clk.Now().UnixNano())},
|
|
||||||
SysStats: &gosigar.Mem{ActualUsed: uint64(float64(limit) * secondWatermark)},
|
|
||||||
}))
|
|
||||||
|
|
||||||
// same thing, outside the silencing period should trigger.
|
|
||||||
require.True(t, p.Evaluate(PolicyInput{
|
|
||||||
Logger: logger,
|
|
||||||
Scope: ScopeSystem,
|
|
||||||
Limit: limit,
|
|
||||||
MemStats: &runtime.MemStats{LastGC: uint64(0)},
|
|
||||||
SysStats: &gosigar.Mem{ActualUsed: uint64(float64(limit) * secondWatermark)},
|
|
||||||
}))
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue