diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 23f94d6..efa43c2 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -1,8 +1,6 @@ name: Endurance Tests on: - push: - branches: [ "stress_test" ] pull_request: branches: [ "stress_test" ] schedule: diff --git a/go.mod b/go.mod index 166611d..dcfaad8 100644 --- a/go.mod +++ b/go.mod @@ -38,10 +38,7 @@ require ( google.golang.org/protobuf v1.34.2 ) -require ( - github.com/sirupsen/logrus v1.2.0 - github.com/waku-org/go-waku v0.8.1-0.20241028194639-dd82c24e0057 -) +require github.com/waku-org/go-waku v0.8.1-0.20241028194639-dd82c24e0057 require ( github.com/beorn7/perks v1.0.1 // indirect @@ -59,7 +56,6 @@ require ( github.com/ipfs/go-log/v2 v2.5.1 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/klauspost/cpuid/v2 v2.2.8 // indirect - github.com/konsorten/go-windows-terminal-sequences v1.0.1 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/libp2p/go-msgio v0.3.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect @@ -83,7 +79,6 @@ require ( golang.org/x/net v0.28.0 // indirect golang.org/x/sync v0.8.0 // indirect golang.org/x/sys v0.24.0 // indirect - golang.org/x/term v0.23.0 // indirect golang.org/x/time v0.5.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.3.0 // indirect diff --git a/go.sum b/go.sum index 1d53bd8..dcf97e7 100644 --- a/go.sum +++ b/go.sum @@ -297,7 +297,6 @@ github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= -github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koron/go-ssdp v0.0.4 h1:1IDwrghSKYM7yLf7XCzbByg2sJ/JcNOZRXS2jczTwz0= github.com/koron/go-ssdp v0.0.4/go.mod h1:oDXq+E5IL5q0U8uSBcoAXzTzInwy5lEgC91HoKtbmZk= @@ -515,7 +514,6 @@ github.com/segmentio/kafka-go v0.2.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfP github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= @@ -734,8 +732,6 @@ golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= -golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= diff --git a/tools/memory_record.go b/tools/memory_record.go new file mode 100644 index 0000000..6ad63af --- /dev/null +++ b/tools/memory_record.go @@ -0,0 +1,38 @@ +package main + +import ( + "flag" + "fmt" + "os" + "runtime" + + "github.com/waku-org/waku-go-bindings/utils" +) + +var ( + testName string + iteration int + phase string +) + +func main() { + flag.StringVar(&testName, "testName", "FullTestSuite", "Name of the test ") + flag.IntVar(&iteration, "iteration", 0, "Iteration number") + flag.StringVar(&phase, "phase", "", "'start' or 'end')") + flag.Parse() + + var memStats runtime.MemStats + runtime.ReadMemStats(&memStats) + heapKB := memStats.HeapAlloc / 1024 + + rssKB, err := utils.GetRSSKB() + if err != nil { + fmt.Fprintln(os.Stderr, "Failed to get RSS:", err) + rssKB = 0 + } + + if err := utils.RecordMemoryMetricsCSV(testName, iteration, phase, heapKB, rssKB); err != nil { + fmt.Fprintln(os.Stderr, "Error recording metrics:", err) + os.Exit(1) + } +} diff --git a/waku/memory_utils.go b/utils/utils.go similarity index 55% rename from waku/memory_utils.go rename to utils/utils.go index 6187a6c..c1f03bd 100644 --- a/waku/memory_utils.go +++ b/utils/utils.go @@ -1,12 +1,12 @@ -package waku +package utils import ( "encoding/csv" - "flag" "fmt" + "io" "os" - "runtime" "strconv" + "strings" "sync" "time" ) @@ -18,29 +18,7 @@ var ( mu sync.Mutex ) -func main() { - flag.StringVar(&testName, "testName", "FullTestSuite", "Name of the test ") - flag.IntVar(&iteration, "iteration", 0, "Iteration number") - flag.StringVar(&phase, "phase", "", "'start' or 'end')") - flag.Parse() - - var memStats runtime.MemStats - runtime.ReadMemStats(&memStats) - heapKB := memStats.HeapAlloc / 1024 - - rssKB, err := getRSSKB() - if err != nil { - fmt.Fprintln(os.Stderr, "Failed to get RSS:", err) - rssKB = 0 - } - - if err := recordMemoryMetricsCSV(testName, iteration, phase, heapKB, rssKB); err != nil { - fmt.Fprintln(os.Stderr, "Error recording metrics:", err) - os.Exit(1) - } -} - -func recordMemoryMetricsCSV(testName string, iter int, phase string, heapKB, rssKB uint64) error { +func RecordMemoryMetricsCSV(testName string, iter int, phase string, heapKB, rssKB uint64) error { mu.Lock() defer mu.Unlock() @@ -75,3 +53,25 @@ func recordMemoryMetricsCSV(testName string, iter int, phase string, heapKB, rss return w.Write(row) } + +func GetRSSKB() (uint64, error) { + f, err := os.Open("/proc/self/statm") + if err != nil { + return 0, err + } + defer f.Close() + data, err := io.ReadAll(f) + if err != nil { + return 0, err + } + fields := strings.Fields(string(data)) + if len(fields) < 2 { + return 0, fmt.Errorf("unexpected /proc/self/statm format") + } + rssPages, err := strconv.ParseUint(fields[1], 10, 64) + if err != nil { + return 0, err + } + pageSize := os.Getpagesize() + return (rssPages * uint64(pageSize)) / 1024, nil +} diff --git a/waku/nwaku_test_utils.go b/waku/nwaku_test_utils.go index 1dd4754..7e319d7 100644 --- a/waku/nwaku_test_utils.go +++ b/waku/nwaku_test_utils.go @@ -9,7 +9,6 @@ import ( "net/http" "os" "strconv" - "strings" "time" "github.com/cenkalti/backoff/v3" @@ -256,25 +255,3 @@ func (n *WakuNode) GetStoredMessages(storeNode *WakuNode, storeRequest *common.S Debug("Store query successful, retrieved %d messages", len(*res.Messages)) return res, nil } - -func getRSSKB() (uint64, error) { - f, err := os.Open("/proc/self/statm") - if err != nil { - return 0, err - } - defer f.Close() - data, err := io.ReadAll(f) - if err != nil { - return 0, err - } - fields := strings.Fields(string(data)) - if len(fields) < 2 { - return 0, fmt.Errorf("unexpected /proc/self/statm format") - } - rssPages, err := strconv.ParseUint(fields[1], 10, 64) - if err != nil { - return 0, err - } - pageSize := os.Getpagesize() - return (rssPages * uint64(pageSize)) / 1024, nil -} diff --git a/waku/stress_test.go b/waku/stress_test.go index e652c26..f3ad305 100644 --- a/waku/stress_test.go +++ b/waku/stress_test.go @@ -1,4 +1,4 @@ - +//go:build !stress // +build !stress package waku @@ -12,6 +12,7 @@ import ( "time" "github.com/stretchr/testify/require" + "github.com/waku-org/waku-go-bindings/utils" "github.com/waku-org/waku-go-bindings/waku/common" "go.uber.org/zap" "go.uber.org/zap/zapcore" @@ -111,7 +112,7 @@ func Test2Nodes500IterationTearDown(t *testing.T) { initialMem := memStats.HeapAlloc Debug("[%s] Memory usage at test START: %d KB", t.Name(), initialMem/1024) - initialRSS, err := getRSSKB() + initialRSS, err := utils.GetRSSKB() require.NoError(t, err) Debug("[%s] OS-level RSS at test START: %d KB", t.Name(), initialRSS) @@ -148,7 +149,7 @@ func Test2Nodes500IterationTearDown(t *testing.T) { runtime.ReadMemStats(&memStats) Debug("Iteration %d, usage after teardown: %d KB", i, memStats.HeapAlloc/1024) require.LessOrEqual(t, memStats.HeapAlloc, initialMem*3, "Memory usage soared above threshold after iteration %d", i) - rssNow, err := getRSSKB() + rssNow, err := utils.GetRSSKB() require.NoError(t, err) Debug("Iteration %d, OS-level RSS after teardown: %d KB", i, rssNow) //require.LessOrEqual(t, rssNow, initialRSS*10, "OS-level RSS soared above threshold after iteration %d", i) @@ -162,7 +163,7 @@ func Test2Nodes500IterationTearDown(t *testing.T) { finalMem := memStats.HeapAlloc Debug("[%s] Memory usage at test END: %d KB", t.Name(), finalMem/1024) require.LessOrEqual(t, finalMem, initialMem*3, "Memory usage soared above threshold after %d cycles", totalIterations) - finalRSS, err := getRSSKB() + finalRSS, err := utils.GetRSSKB() require.NoError(t, err) Debug("[%s] OS-level RSS at test END: %d KB", t.Name(), finalRSS) require.LessOrEqual(t, finalRSS, initialRSS*3, "OS-level RSS soared above threshold after %d cycles", totalIterations)