From fd5d5df4f34955c091134056409aac291ec64c20 Mon Sep 17 00:00:00 2001 From: aya Date: Sun, 6 Apr 2025 13:36:14 +0200 Subject: [PATCH 01/14] Add modifications for failed tests --- waku/stress_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/waku/stress_test.go b/waku/stress_test.go index b06a059..c898a34 100644 --- a/waku/stress_test.go +++ b/waku/stress_test.go @@ -157,7 +157,7 @@ func TestStressHighThroughput10kPublish(t *testing.T) { Debug("Memory usage BEFORE sending => HeapAlloc: %d KB, RSS: %d KB", startHeapKB, startRSSKB) - totalMessages := 5000 + totalMessages := 1000 pubsubTopic := DefaultPubsubTopic for i := 0; i < totalMessages; i++ { From 8cd84f6d83a604a51dc3a2a235ffdc5286349242 Mon Sep 17 00:00:00 2001 From: aya Date: Sun, 6 Apr 2025 18:20:05 +0200 Subject: [PATCH 02/14] reduce iterations to 100 --- waku/stress_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/waku/stress_test.go b/waku/stress_test.go index c898a34..f5151c2 100644 --- a/waku/stress_test.go +++ b/waku/stress_test.go @@ -157,7 +157,7 @@ func TestStressHighThroughput10kPublish(t *testing.T) { Debug("Memory usage BEFORE sending => HeapAlloc: %d KB, RSS: %d KB", startHeapKB, startRSSKB) - totalMessages := 1000 + totalMessages := 100 pubsubTopic := DefaultPubsubTopic for i := 0; i < totalMessages; i++ { From 53158bf20a8b3fcc80a0ab887df251479b5b2c8f Mon Sep 17 00:00:00 2001 From: aya Date: Sun, 6 Apr 2025 18:24:52 +0200 Subject: [PATCH 03/14] edit iterations to correct test name --- waku/stress_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/waku/stress_test.go b/waku/stress_test.go index f5151c2..ec4d893 100644 --- a/waku/stress_test.go +++ b/waku/stress_test.go @@ -157,7 +157,7 @@ func TestStressHighThroughput10kPublish(t *testing.T) { Debug("Memory usage BEFORE sending => HeapAlloc: %d KB, RSS: %d KB", startHeapKB, startRSSKB) - totalMessages := 100 + totalMessages := 2000 pubsubTopic := DefaultPubsubTopic for i := 0; i < totalMessages; i++ { @@ -343,7 +343,7 @@ func TestStressLargePayloadEphemeralMessagesEndurance(t *testing.T) { require.NoError(t, err) Debug("Before endurance test: HeapAlloc = %d KB, RSS = %d KB", startHeapKB, startRSSKB) - maxIterations := 2000 + maxIterations := 1000 payloadSize := 100 * 1024 largePayload := make([]byte, payloadSize) for i := range largePayload { From 99cc621c8812079f6b788e929dbf291a39774f84 Mon Sep 17 00:00:00 2001 From: aya Date: Sun, 6 Apr 2025 20:12:20 +0200 Subject: [PATCH 04/14] adding more wait time --- waku/stress_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/waku/stress_test.go b/waku/stress_test.go index ec4d893..a1da267 100644 --- a/waku/stress_test.go +++ b/waku/stress_test.go @@ -157,7 +157,7 @@ func TestStressHighThroughput10kPublish(t *testing.T) { Debug("Memory usage BEFORE sending => HeapAlloc: %d KB, RSS: %d KB", startHeapKB, startRSSKB) - totalMessages := 2000 + totalMessages := 1500 pubsubTopic := DefaultPubsubTopic for i := 0; i < totalMessages; i++ { @@ -373,6 +373,7 @@ func TestStressLargePayloadEphemeralMessagesEndurance(t *testing.T) { endRSSKB, err := utils.GetRSSKB() require.NoError(t, err) Debug("After endurance test: HeapAlloc = %d KB, RSS = %d KB", endHeapKB, endRSSKB) + time.Sleep(10 * time.Second) } func TestStress2Nodes500IterationTearDown(t *testing.T) { From 722005d150d7949e1d6a9dfa0bcd55018b3516ed Mon Sep 17 00:00:00 2001 From: aya Date: Sun, 6 Apr 2025 21:57:48 +0200 Subject: [PATCH 05/14] make modifications for jobs --- .github/workflows/CI_endurance.yml | 4 ++-- waku/stress_test.go | 14 +++++++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/workflows/CI_endurance.yml b/.github/workflows/CI_endurance.yml index 314bc19..f411e68 100644 --- a/.github/workflows/CI_endurance.yml +++ b/.github/workflows/CI_endurance.yml @@ -37,7 +37,7 @@ jobs: - name: Run Endurance Test (Group 1) run: | - go test -p=1 -v ./waku -count=1 -timeout=360m -run '^(TestStressMemoryUsageForThreeNodes|TestStressStoreQuery5kMessagesWithPagination|TestStressHighThroughput10kPublish|TestStressConnectDisconnect500Iteration)$' | tee testlogs1.log + go test -p=1 -v ./waku -count=1 -timeout=360m -run '^(TestStressMemoryUsageForThreeNodes|TestStressStoreQuery5kMessagesWithPagination|TestStressConnectDisconnect500Iteration|TestStressHighThroughput10kPublish)$' | tee testlogs1.log - name: Upload Test Logs (Group 1) uses: actions/upload-artifact@v4 @@ -78,7 +78,7 @@ jobs: - name: Run Endurance Test (Group 2) run: | - go test -p=1 -v ./waku -count=1 -timeout=360m -run '^(TestStressRandomNodesInMesh|TestStressLargePayloadEphemeralMessagesEndurance|TestStress2Nodes500IterationTearDown|TestPeerExchangePXLoad)$' | tee testlogs2.log + go test -p=1 -v ./waku -count=1 -timeout=360m -run '^(TestStressRandomNodesInMesh|TestStress2Nodes500IterationTearDown|TestPeerExchangePXLoad)$' | tee testlogs2.log - name: Upload Test Logs (Group 2) uses: actions/upload-artifact@v4 diff --git a/waku/stress_test.go b/waku/stress_test.go index a1da267..e825214 100644 --- a/waku/stress_test.go +++ b/waku/stress_test.go @@ -92,7 +92,7 @@ func TestStressStoreQuery5kMessagesWithPagination(t *testing.T) { }() var memStats runtime.MemStats - iterations := 50 + iterations := 500 runtime.ReadMemStats(&memStats) initialHeapAlloc := memStats.HeapAlloc @@ -157,7 +157,7 @@ func TestStressHighThroughput10kPublish(t *testing.T) { Debug("Memory usage BEFORE sending => HeapAlloc: %d KB, RSS: %d KB", startHeapKB, startRSSKB) - totalMessages := 1500 + totalMessages := 1000 pubsubTopic := DefaultPubsubTopic for i := 0; i < totalMessages; i++ { @@ -320,17 +320,21 @@ func TestStressLargePayloadEphemeralMessagesEndurance(t *testing.T) { nodePubCfg.Relay = true publisher, err := StartWakuNode("publisher", &nodePubCfg) require.NoError(t, err) - defer publisher.StopAndDestroy() nodeRecvCfg := DefaultWakuConfig nodeRecvCfg.Relay = true receiver, err := StartWakuNode("receiver", &nodeRecvCfg) require.NoError(t, err) - defer receiver.StopAndDestroy() err = receiver.RelaySubscribe(DefaultPubsubTopic) require.NoError(t, err) + defer func() { + publisher.StopAndDestroy() + time.Sleep(30 * time.Second) + receiver.StopAndDestroy() + + }() err = publisher.ConnectPeer(receiver) require.NoError(t, err) @@ -373,7 +377,7 @@ func TestStressLargePayloadEphemeralMessagesEndurance(t *testing.T) { endRSSKB, err := utils.GetRSSKB() require.NoError(t, err) Debug("After endurance test: HeapAlloc = %d KB, RSS = %d KB", endHeapKB, endRSSKB) - time.Sleep(10 * time.Second) + } func TestStress2Nodes500IterationTearDown(t *testing.T) { From a0e47ac12a1f79b7304d098515fe4f1a016bd011 Mon Sep 17 00:00:00 2001 From: aya Date: Tue, 8 Apr 2025 20:44:29 +0200 Subject: [PATCH 06/14] Merge master to branch --- .github/workflows/CI_endurance.yml | 2 +- waku/stress_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/CI_endurance.yml b/.github/workflows/CI_endurance.yml index f411e68..6279500 100644 --- a/.github/workflows/CI_endurance.yml +++ b/.github/workflows/CI_endurance.yml @@ -78,7 +78,7 @@ jobs: - name: Run Endurance Test (Group 2) run: | - go test -p=1 -v ./waku -count=1 -timeout=360m -run '^(TestStressRandomNodesInMesh|TestStress2Nodes500IterationTearDown|TestPeerExchangePXLoad)$' | tee testlogs2.log + go test -p=1 -v ./waku -count=1 -timeout=360m -run '^(TestStressRandomNodesInMesh|TestStress2Nodes500IterationTearDown|TestPeerExchangePXLoad|TestStressHighThroughput10kPublish)$' | tee testlogs2.log - name: Upload Test Logs (Group 2) uses: actions/upload-artifact@v4 diff --git a/waku/stress_test.go b/waku/stress_test.go index e825214..1d01617 100644 --- a/waku/stress_test.go +++ b/waku/stress_test.go @@ -157,7 +157,7 @@ func TestStressHighThroughput10kPublish(t *testing.T) { Debug("Memory usage BEFORE sending => HeapAlloc: %d KB, RSS: %d KB", startHeapKB, startRSSKB) - totalMessages := 1000 + totalMessages := 1100 pubsubTopic := DefaultPubsubTopic for i := 0; i < totalMessages; i++ { @@ -347,7 +347,7 @@ func TestStressLargePayloadEphemeralMessagesEndurance(t *testing.T) { require.NoError(t, err) Debug("Before endurance test: HeapAlloc = %d KB, RSS = %d KB", startHeapKB, startRSSKB) - maxIterations := 1000 + maxIterations := 1100 payloadSize := 100 * 1024 largePayload := make([]byte, payloadSize) for i := range largePayload { From d8e7fb1d954e4d44798b56a98bdbeb478c28c2d3 Mon Sep 17 00:00:00 2001 From: aya Date: Wed, 9 Apr 2025 00:51:39 +0200 Subject: [PATCH 07/14] increase iterations limit to 3k --- waku/stress_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/waku/stress_test.go b/waku/stress_test.go index 1d01617..316a98b 100644 --- a/waku/stress_test.go +++ b/waku/stress_test.go @@ -157,7 +157,7 @@ func TestStressHighThroughput10kPublish(t *testing.T) { Debug("Memory usage BEFORE sending => HeapAlloc: %d KB, RSS: %d KB", startHeapKB, startRSSKB) - totalMessages := 1100 + totalMessages := 3000 pubsubTopic := DefaultPubsubTopic for i := 0; i < totalMessages; i++ { @@ -347,7 +347,7 @@ func TestStressLargePayloadEphemeralMessagesEndurance(t *testing.T) { require.NoError(t, err) Debug("Before endurance test: HeapAlloc = %d KB, RSS = %d KB", startHeapKB, startRSSKB) - maxIterations := 1100 + maxIterations := 3000 payloadSize := 100 * 1024 largePayload := make([]byte, payloadSize) for i := range largePayload { From b6121264e4dbcac080ba27a5b82f847fbbb640f6 Mon Sep 17 00:00:00 2001 From: aya Date: Wed, 14 May 2025 15:47:52 +0300 Subject: [PATCH 08/14] Increase limit to stress tests --- waku/stress_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/waku/stress_test.go b/waku/stress_test.go index 316a98b..ba7c3cc 100644 --- a/waku/stress_test.go +++ b/waku/stress_test.go @@ -92,7 +92,7 @@ func TestStressStoreQuery5kMessagesWithPagination(t *testing.T) { }() var memStats runtime.MemStats - iterations := 500 + iterations := 2000 runtime.ReadMemStats(&memStats) initialHeapAlloc := memStats.HeapAlloc @@ -157,7 +157,7 @@ func TestStressHighThroughput10kPublish(t *testing.T) { Debug("Memory usage BEFORE sending => HeapAlloc: %d KB, RSS: %d KB", startHeapKB, startRSSKB) - totalMessages := 3000 + totalMessages := 6000 pubsubTopic := DefaultPubsubTopic for i := 0; i < totalMessages; i++ { @@ -199,7 +199,7 @@ func TestStressConnectDisconnect500Iteration(t *testing.T) { node1.StopAndDestroy() }() - iterations := 200 + iterations := 1000 for i := 1; i <= iterations; i++ { err := node0.ConnectPeer(node1) require.NoError(t, err, "Iteration %d: node0 failed to connect to node1", i) @@ -347,7 +347,7 @@ func TestStressLargePayloadEphemeralMessagesEndurance(t *testing.T) { require.NoError(t, err) Debug("Before endurance test: HeapAlloc = %d KB, RSS = %d KB", startHeapKB, startRSSKB) - maxIterations := 3000 + maxIterations := 5000 payloadSize := 100 * 1024 largePayload := make([]byte, payloadSize) for i := range largePayload { @@ -391,7 +391,7 @@ func TestStress2Nodes500IterationTearDown(t *testing.T) { require.NoError(t, err) Debug("[%s] OS-level RSS at test START: %d KB", t.Name(), initialRSS) - totalIterations := 500 + totalIterations := 2000 for i := 1; i <= totalIterations; i++ { var nodes []*WakuNode for n := 1; n <= 2; n++ { From e328fa0c8649f1be17cc9f83c798bc8022ccd6ab Mon Sep 17 00:00:00 2001 From: aya Date: Wed, 14 May 2025 18:53:27 +0300 Subject: [PATCH 09/14] save results to csv file --- utils/utils.go | 1 + waku/nwaku_test_utils.go | 14 ++++ waku/stress_test.go | 141 +++++++-------------------------------- 3 files changed, 39 insertions(+), 117 deletions(-) diff --git a/utils/utils.go b/utils/utils.go index c1f03bd..0df2b6b 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -75,3 +75,4 @@ func GetRSSKB() (uint64, error) { pageSize := os.Getpagesize() return (rssPages * uint64(pageSize)) / 1024, nil } + diff --git a/waku/nwaku_test_utils.go b/waku/nwaku_test_utils.go index a2d64a0..4da8272 100644 --- a/waku/nwaku_test_utils.go +++ b/waku/nwaku_test_utils.go @@ -9,6 +9,7 @@ import ( "io" "net/http" "os" + "runtime" "strconv" "sync" "time" @@ -16,6 +17,7 @@ import ( "github.com/cenkalti/backoff/v3" "github.com/libp2p/go-libp2p/core/peer" "github.com/waku-org/go-waku/waku/v2/protocol/pb" + "github.com/waku-org/waku-go-bindings/utils" "github.com/waku-org/waku-go-bindings/waku/common" "google.golang.org/protobuf/proto" ) @@ -288,3 +290,15 @@ func recordMemoryMetricsPX(testName, phase string, heapAllocKB, rssKB uint64) er } return writer.Write(row) } + +func captureMemory(testName, phase string) { + var ms runtime.MemStats + runtime.ReadMemStats(&ms) + + heapKB := ms.HeapAlloc / 1024 + rssKB, _ := utils.GetRSSKB() + + Debug("[%s] Memory usage (%s): %d KB (RSS %d KB)", testName, phase, heapKB, rssKB) + + _ = recordMemoryMetricsPX(testName, phase, heapKB, rssKB) +} \ No newline at end of file diff --git a/waku/stress_test.go b/waku/stress_test.go index ba7c3cc..dc65b65 100644 --- a/waku/stress_test.go +++ b/waku/stress_test.go @@ -11,7 +11,6 @@ import ( "time" "github.com/stretchr/testify/require" - "github.com/waku-org/waku-go-bindings/utils" "github.com/waku-org/waku-go-bindings/waku/common" // "go.uber.org/zap/zapcore" @@ -21,9 +20,7 @@ import ( func TestStressMemoryUsageForThreeNodes(t *testing.T) { testName := t.Name() var err error - var memStats runtime.MemStats - runtime.ReadMemStats(&memStats) - Debug("[%s] Memory usage BEFORE creating nodes: %d KB", testName, memStats.HeapAlloc/1024) + captureMemory(t.Name(), "start") node1Cfg := DefaultWakuConfig node1Cfg.TcpPort, node1Cfg.Discv5UdpPort, err = GetFreePortIfNeeded(0, 0) require.NoError(t, err) @@ -41,8 +38,7 @@ func TestStressMemoryUsageForThreeNodes(t *testing.T) { node3, err := NewWakuNode(&node3Cfg, "node3") require.NoError(t, err) - runtime.ReadMemStats(&memStats) - Debug("[%s] Memory usage AFTER creating nodes: %d KB", testName, memStats.HeapAlloc/1024) + captureMemory(t.Name(), "before nodes start") err = node1.Start() require.NoError(t, err) @@ -51,8 +47,7 @@ func TestStressMemoryUsageForThreeNodes(t *testing.T) { err = node3.Start() require.NoError(t, err) - runtime.ReadMemStats(&memStats) - Debug("[%s] Memory usage AFTER starting nodes: %d KB", testName, memStats.HeapAlloc/1024) + captureMemory(t.Name(), "after nodes run") time.Sleep(2 * time.Second) @@ -64,8 +59,7 @@ func TestStressMemoryUsageForThreeNodes(t *testing.T) { time.Sleep(1 * time.Second) runtime.GC() - runtime.ReadMemStats(&memStats) - Debug("[%s] Memory usage AFTER destroying nodes: %d KB", testName, memStats.HeapAlloc/1024) + captureMemory(t.Name(), "at end") Debug("[%s] Test completed successfully", testName) } @@ -91,12 +85,10 @@ func TestStressStoreQuery5kMessagesWithPagination(t *testing.T) { node2.StopAndDestroy() }() - var memStats runtime.MemStats - iterations := 2000 + + iterations := 5000 - runtime.ReadMemStats(&memStats) - initialHeapAlloc := memStats.HeapAlloc - Debug("Initial memory usage check before publishing %d MB", initialHeapAlloc/1024/1024) + captureMemory(t.Name(), "at start") queryTimestamp := proto.Int64(time.Now().UnixNano()) @@ -107,9 +99,6 @@ func TestStressStoreQuery5kMessagesWithPagination(t *testing.T) { require.NoError(t, err, "Failed to publish message") if i%10 == 0 { - runtime.ReadMemStats(&memStats) - Debug("Memory usage at iteration %d: HeapAlloc=%v MB, NumGC=%v", - i, memStats.HeapAlloc/1024/1024, memStats.NumGC) storeQueryRequest := &common.StoreQueryRequest{ TimeStart: queryTimestamp, @@ -123,11 +112,7 @@ func TestStressStoreQuery5kMessagesWithPagination(t *testing.T) { } } - runtime.ReadMemStats(&memStats) - finalHeapAlloc := memStats.HeapAlloc - Debug("Memory before test: %v KB, Memory after test: %v KB", initialHeapAlloc/1024, finalHeapAlloc/1024) - - //require.LessOrEqual(t, finalHeapAlloc, initialHeapAlloc*2, "Memory usage has grown too much") + captureMemory(t.Name(), "at end") Debug("[%s] Test completed successfully", t.Name()) } @@ -149,15 +134,9 @@ func TestStressHighThroughput10kPublish(t *testing.T) { err = node1.ConnectPeer(node2) require.NoError(t, err, "Failed to connect node1 to node2") - var memStats runtime.MemStats - runtime.ReadMemStats(&memStats) - startHeapKB := memStats.HeapAlloc / 1024 - startRSSKB, err := utils.GetRSSKB() - require.NoError(t, err, "Failed to read initial RSS") + captureMemory(t.Name(), "at start") - Debug("Memory usage BEFORE sending => HeapAlloc: %d KB, RSS: %d KB", startHeapKB, startRSSKB) - - totalMessages := 6000 + totalMessages := 2000 pubsubTopic := DefaultPubsubTopic for i := 0; i < totalMessages; i++ { @@ -170,21 +149,11 @@ func TestStressHighThroughput10kPublish(t *testing.T) { Debug("###Iteration number#%d", i) } - runtime.ReadMemStats(&memStats) - endHeapKB := memStats.HeapAlloc / 1024 - endRSSKB, err := utils.GetRSSKB() - require.NoError(t, err, "Failed to read final RSS") - - Debug("Memory usage AFTER sending => HeapAlloc: %d KB, RSS: %d KB", endHeapKB, endRSSKB) + captureMemory(t.Name(), "at end") } -func TestStressConnectDisconnect500Iteration(t *testing.T) { - var memStats runtime.MemStats - runtime.ReadMemStats(&memStats) - startHeapKB := memStats.HeapAlloc / 1024 - startRSSKB, err := utils.GetRSSKB() - require.NoError(t, err) - Debug("Before test: HeapAlloc = %d KB, RSS = %d KB", startHeapKB, startRSSKB) +func TestStressConnectDisconnect1kIteration(t *testing.T) { + captureMemory(t.Name(), "at start") node0Cfg := DefaultWakuConfig node0Cfg.Relay = true @@ -219,11 +188,7 @@ func TestStressConnectDisconnect500Iteration(t *testing.T) { Debug("Iteration %d: node0 disconnected from node1", i) time.Sleep(2 * time.Second) } - runtime.ReadMemStats(&memStats) - endHeapKB := memStats.HeapAlloc / 1024 - endRSSKB, err := utils.GetRSSKB() - require.NoError(t, err) - Debug("After test: HeapAlloc = %d KB, RSS = %d KB", endHeapKB, endRSSKB) + captureMemory(t.Name(), "at end") } func TestStressRandomNodesInMesh(t *testing.T) { @@ -245,12 +210,7 @@ func TestStressRandomNodesInMesh(t *testing.T) { time.Sleep(1 * time.Second) require.NoError(t, err, "Failed to connect initial nodes with ConnectAllPeers") - var memStats runtime.MemStats - runtime.ReadMemStats(&memStats) - startHeapKB := memStats.HeapAlloc / 1024 - startRSSKB, err2 := utils.GetRSSKB() - require.NoError(t, err2, "Failed to read initial RSS") - Debug("Memory at start of test: HeapAlloc=%d KB, RSS=%d KB", startHeapKB, startRSSKB) + captureMemory(t.Name(), "at start") testDuration := 30 * time.Minute endTime := time.Now().Add(testDuration) @@ -308,11 +268,7 @@ func TestStressRandomNodesInMesh(t *testing.T) { n.StopAndDestroy() } - runtime.ReadMemStats(&memStats) - endHeapKB := memStats.HeapAlloc / 1024 - endRSSKB, err3 := utils.GetRSSKB() - require.NoError(t, err3, "Failed to read final RSS") - Debug("Memory at end of test: HeapAlloc=%d KB, RSS=%d KB", endHeapKB, endRSSKB) + captureMemory(t.Name(), "at end") } func TestStressLargePayloadEphemeralMessagesEndurance(t *testing.T) { @@ -340,12 +296,7 @@ func TestStressLargePayloadEphemeralMessagesEndurance(t *testing.T) { time.Sleep(2 * time.Second) - var memStats runtime.MemStats - runtime.ReadMemStats(&memStats) - startHeapKB := memStats.HeapAlloc / 1024 - startRSSKB, err := utils.GetRSSKB() - require.NoError(t, err) - Debug("Before endurance test: HeapAlloc = %d KB, RSS = %d KB", startHeapKB, startRSSKB) + captureMemory(t.Name(), "at start") maxIterations := 5000 payloadSize := 100 * 1024 @@ -372,25 +323,14 @@ func TestStressLargePayloadEphemeralMessagesEndurance(t *testing.T) { Debug("###Iteration number %d", i+1) } - runtime.ReadMemStats(&memStats) - endHeapKB := memStats.HeapAlloc / 1024 - endRSSKB, err := utils.GetRSSKB() - require.NoError(t, err) - Debug("After endurance test: HeapAlloc = %d KB, RSS = %d KB", endHeapKB, endRSSKB) + captureMemory(t.Name(), "at end") } -func TestStress2Nodes500IterationTearDown(t *testing.T) { - - var memStats runtime.MemStats - runtime.ReadMemStats(&memStats) - initialMem := memStats.HeapAlloc - Debug("[%s] Memory usage at test START: %d KB", t.Name(), initialMem/1024) - - initialRSS, err := utils.GetRSSKB() - require.NoError(t, err) - Debug("[%s] OS-level RSS at test START: %d KB", t.Name(), initialRSS) +func TestStress2Nodes2kIterationTearDown(t *testing.T) { + captureMemory(t.Name(), "at start") + var err error totalIterations := 2000 for i := 1; i <= totalIterations; i++ { var nodes []*WakuNode @@ -421,32 +361,16 @@ func TestStress2Nodes500IterationTearDown(t *testing.T) { runtime.GC() time.Sleep(250 * time.Millisecond) runtime.GC() - if i == 250 || i == 500 { - runtime.ReadMemStats(&memStats) - Debug("Iteration %d, usage after teardown: %d KB", i, memStats.HeapAlloc/1024) - require.LessOrEqual(t, memStats.HeapAlloc, initialMem*3, "Memory usage soared above threshold after iteration %d", i) - rssNow, err := utils.GetRSSKB() - require.NoError(t, err) - Debug("Iteration %d, OS-level RSS after teardown: %d KB", i, rssNow) - //require.LessOrEqual(t, rssNow, initialRSS*10, "OS-level RSS soared above threshold after iteration %d", i) - } Debug("Iteration numberrrrrr %d", i) } runtime.GC() time.Sleep(500 * time.Millisecond) runtime.GC() - runtime.ReadMemStats(&memStats) - finalMem := memStats.HeapAlloc - Debug("[%s] Memory usage at test END: %d KB", t.Name(), finalMem/1024) - // require.LessOrEqual(t, finalMem, initialMem*3, "Memory usage soared above threshold after %d cycles", totalIterations) - finalRSS, err := utils.GetRSSKB() - require.NoError(t, err) - Debug("[%s] OS-level RSS at test END: %d KB", t.Name(), finalRSS) + captureMemory(t.Name(), "at end") //require.LessOrEqual(t, finalRSS, initialRSS*3, "OS-level RSS soared above threshold after %d cycles", totalIterations) } func TestPeerExchangePXLoad(t *testing.T) { - testName := "PeerExchangePXLoad" pxServerCfg := DefaultWakuConfig pxServerCfg.PeerExchange = true pxServerCfg.Relay = true @@ -469,16 +393,7 @@ func TestPeerExchangePXLoad(t *testing.T) { time.Sleep(2 * time.Second) - var memStats runtime.MemStats - runtime.ReadMemStats(&memStats) - startHeapKB := memStats.HeapAlloc / 1024 - startRSSKB, err := utils.GetRSSKB() - require.NoError(t, err, "Failed to get initial RSS") - Debug("%s: Before test: HeapAlloc=%d KB, RSS=%d KB", testName, startHeapKB, startRSSKB) - - // Save the initial memory reading to CSV - err = recordMemoryMetricsPX(testName, "start", startHeapKB, startRSSKB) - require.NoError(t, err, "Failed to record start metrics") + captureMemory(t.Name(), "at start") testDuration := 30 * time.Minute endTime := time.Now().Add(testDuration) @@ -514,13 +429,5 @@ func TestPeerExchangePXLoad(t *testing.T) { time.Sleep(1 * time.Second) } - runtime.ReadMemStats(&memStats) - endHeapKB := memStats.HeapAlloc / 1024 - endRSSKB, err := utils.GetRSSKB() - require.NoError(t, err, "Failed to get final RSS") - Debug("Memory %s: After test: HeapAlloc=%d KB, RSS=%d KB", testName, endHeapKB, endRSSKB) - - // Save the final memory reading to CSV - err = recordMemoryMetricsPX(testName, "end", endHeapKB, endRSSKB) - require.NoError(t, err, "Failed to record end metrics") + captureMemory(t.Name(), "at end") } From 6564d7a0c095df0dfde58f058e04dd0736592d68 Mon Sep 17 00:00:00 2001 From: aya Date: Thu, 15 May 2025 13:49:15 +0300 Subject: [PATCH 10/14] upload results to csv file in CI job --- .github/workflows/CI_endurance.yml | 12 ++++++++++++ waku/stress_test.go | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/CI_endurance.yml b/.github/workflows/CI_endurance.yml index 6279500..6dc0f5d 100644 --- a/.github/workflows/CI_endurance.yml +++ b/.github/workflows/CI_endurance.yml @@ -45,6 +45,12 @@ jobs: name: endurance-logs-group1 path: testlogs1.log + - name: Upload Memory Metrics (Group 1) + uses: actions/upload-artifact@v4 + with: + name: memory-metrics-group1 + path: px_load_metrics.csv + endurance2: runs-on: ubuntu-latest @@ -85,3 +91,9 @@ jobs: with: name: endurance-logs-group2 path: testlogs2.log + + - name: Upload Memory Metrics (Group 2) + uses: actions/upload-artifact@v4 + with: + name: memory-metrics-group2 + path: px_load_metrics.csv \ No newline at end of file diff --git a/waku/stress_test.go b/waku/stress_test.go index dc65b65..b5600d9 100644 --- a/waku/stress_test.go +++ b/waku/stress_test.go @@ -136,7 +136,7 @@ func TestStressHighThroughput10kPublish(t *testing.T) { captureMemory(t.Name(), "at start") - totalMessages := 2000 + totalMessages := 2500 pubsubTopic := DefaultPubsubTopic for i := 0; i < totalMessages; i++ { From 25d5df945d65810854bcf9dcadc7a34494ff6563 Mon Sep 17 00:00:00 2001 From: aya Date: Thu, 15 May 2025 15:18:53 +0300 Subject: [PATCH 11/14] try using self hosted runner for the repeated yml file --- .github/workflows/Repeated_tests_endurancce.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/Repeated_tests_endurancce.yml b/.github/workflows/Repeated_tests_endurancce.yml index 0e51b4d..42e1aa2 100644 --- a/.github/workflows/Repeated_tests_endurancce.yml +++ b/.github/workflows/Repeated_tests_endurancce.yml @@ -5,7 +5,8 @@ on: jobs: repeated-tests: - runs-on: ubuntu-latest + runs-on: [self-hosted, ubuntu-22.04] + timeout-minutes: 900 steps: - name: Check out repository uses: actions/checkout@v3 @@ -41,7 +42,7 @@ jobs: - name: Repeated test runs run: | set +e - for i in {1..100}; do + for i in {1..80}; do echo "Iteration $i: measuring memory BEFORE the tests..." go run tools/memory_record.go --iteration $i --phase start echo "Running tests (iteration $i)..." From 2184bd7bc01517b6c4fe6c54e056cf71580361a2 Mon Sep 17 00:00:00 2001 From: aya Date: Thu, 15 May 2025 16:35:50 +0300 Subject: [PATCH 12/14] Add correct path for csv file --- .github/workflows/CI_endurance.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/CI_endurance.yml b/.github/workflows/CI_endurance.yml index 6dc0f5d..8f22e3f 100644 --- a/.github/workflows/CI_endurance.yml +++ b/.github/workflows/CI_endurance.yml @@ -49,7 +49,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: memory-metrics-group1 - path: px_load_metrics.csv + path: waku/px_load_metrics.csv endurance2: runs-on: ubuntu-latest @@ -84,7 +84,7 @@ jobs: - name: Run Endurance Test (Group 2) run: | - go test -p=1 -v ./waku -count=1 -timeout=360m -run '^(TestStressRandomNodesInMesh|TestStress2Nodes500IterationTearDown|TestPeerExchangePXLoad|TestStressHighThroughput10kPublish)$' | tee testlogs2.log + go test -p=1 -v ./waku -count=1 -timeout=360m -run '^(TestStressRandomNodesInMesh|TestStress2Nodes500IterationTearDown|TestPeerExchangePXLoad)$' | tee testlogs2.log - name: Upload Test Logs (Group 2) uses: actions/upload-artifact@v4 @@ -96,4 +96,4 @@ jobs: uses: actions/upload-artifact@v4 with: name: memory-metrics-group2 - path: px_load_metrics.csv \ No newline at end of file + path: waku/px_load_metrics.csv \ No newline at end of file From f17a654158384fb3e2093406d4feeef6d80fc399 Mon Sep 17 00:00:00 2001 From: aya Date: Thu, 15 May 2025 18:50:29 +0300 Subject: [PATCH 13/14] make changes in sore.go fro json wrong format and make CI job exit on errors --- .github/workflows/CI_endurance.yml | 5 +++++ waku/common/store.go | 6 +++--- waku/stress_test.go | 5 +++-- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/CI_endurance.yml b/.github/workflows/CI_endurance.yml index 8f22e3f..4cea715 100644 --- a/.github/workflows/CI_endurance.yml +++ b/.github/workflows/CI_endurance.yml @@ -36,7 +36,9 @@ jobs: run: sudo sh -c "ulimit -n 8192" - name: Run Endurance Test (Group 1) + shell: bash run: | + set -euo pipefail go test -p=1 -v ./waku -count=1 -timeout=360m -run '^(TestStressMemoryUsageForThreeNodes|TestStressStoreQuery5kMessagesWithPagination|TestStressConnectDisconnect500Iteration|TestStressHighThroughput10kPublish)$' | tee testlogs1.log - name: Upload Test Logs (Group 1) @@ -83,7 +85,10 @@ jobs: run: sudo sh -c "ulimit -n 8192" - name: Run Endurance Test (Group 2) + + shell: bash run: | + set -euo pipefail go test -p=1 -v ./waku -count=1 -timeout=360m -run '^(TestStressRandomNodesInMesh|TestStress2Nodes500IterationTearDown|TestPeerExchangePXLoad)$' | tee testlogs2.log - name: Upload Test Logs (Group 2) diff --git a/waku/common/store.go b/waku/common/store.go index 066ad4e..fd73c9c 100644 --- a/waku/common/store.go +++ b/waku/common/store.go @@ -1,15 +1,15 @@ package common type StoreQueryRequest struct { - RequestId string `json:"requestId"` - IncludeData bool `json:"includeData"` + RequestId string `json:"request_id"` + IncludeData bool `json:"include_data"` PubsubTopic string `json:"pubsubTopic,omitempty"` ContentTopics *[]string `json:"contentTopics,omitempty"` TimeStart *int64 `json:"timeStart,omitempty"` TimeEnd *int64 `json:"timeEnd,omitempty"` MessageHashes *[]MessageHash `json:"messageHashes,omitempty"` PaginationCursor *MessageHash `json:"paginationCursor,omitempty"` - PaginationForward bool `json:"paginationForward"` + PaginationForward bool `json:"pagination_forward"` PaginationLimit *uint64 `json:"paginationLimit,omitempty"` } diff --git a/waku/stress_test.go b/waku/stress_test.go index b5600d9..80b54ee 100644 --- a/waku/stress_test.go +++ b/waku/stress_test.go @@ -86,7 +86,7 @@ func TestStressStoreQuery5kMessagesWithPagination(t *testing.T) { }() - iterations := 5000 + iterations := 3000 captureMemory(t.Name(), "at start") @@ -104,6 +104,7 @@ func TestStressStoreQuery5kMessagesWithPagination(t *testing.T) { TimeStart: queryTimestamp, IncludeData: true, PaginationLimit: proto.Uint64(50), + PaginationForward: false, } storedmsgs, err := wakuNode.GetStoredMessages(node2, storeQueryRequest) @@ -136,7 +137,7 @@ func TestStressHighThroughput10kPublish(t *testing.T) { captureMemory(t.Name(), "at start") - totalMessages := 2500 + totalMessages := 5 pubsubTopic := DefaultPubsubTopic for i := 0; i < totalMessages; i++ { From 47c83091dc337abfb96d3612c6573ff900748256 Mon Sep 17 00:00:00 2001 From: aya Date: Thu, 15 May 2025 19:00:04 +0300 Subject: [PATCH 14/14] Update some ranges for data --- .github/workflows/CI_endurance.yml | 2 +- waku/stress_test.go | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/CI_endurance.yml b/.github/workflows/CI_endurance.yml index 4cea715..16a8fe6 100644 --- a/.github/workflows/CI_endurance.yml +++ b/.github/workflows/CI_endurance.yml @@ -89,7 +89,7 @@ jobs: shell: bash run: | set -euo pipefail - go test -p=1 -v ./waku -count=1 -timeout=360m -run '^(TestStressRandomNodesInMesh|TestStress2Nodes500IterationTearDown|TestPeerExchangePXLoad)$' | tee testlogs2.log + go test -p=1 -v ./waku -count=1 -timeout=360m -run '^(TestStressRandomNodesInMesh|TestStress2Nodes2kIterationTearDown|TestPeerExchangePXLoad)$' | tee testlogs2.log - name: Upload Test Logs (Group 2) uses: actions/upload-artifact@v4 diff --git a/waku/stress_test.go b/waku/stress_test.go index 80b54ee..45137ae 100644 --- a/waku/stress_test.go +++ b/waku/stress_test.go @@ -86,7 +86,7 @@ func TestStressStoreQuery5kMessagesWithPagination(t *testing.T) { }() - iterations := 3000 + iterations := 4000 captureMemory(t.Name(), "at start") @@ -137,7 +137,7 @@ func TestStressHighThroughput10kPublish(t *testing.T) { captureMemory(t.Name(), "at start") - totalMessages := 5 + totalMessages := 2000 pubsubTopic := DefaultPubsubTopic for i := 0; i < totalMessages; i++ { @@ -169,7 +169,7 @@ func TestStressConnectDisconnect1kIteration(t *testing.T) { node1.StopAndDestroy() }() - iterations := 1000 + iterations := 2000 for i := 1; i <= iterations; i++ { err := node0.ConnectPeer(node1) require.NoError(t, err, "Iteration %d: node0 failed to connect to node1", i) @@ -196,7 +196,7 @@ func TestStressRandomNodesInMesh(t *testing.T) { r := rand.New(rand.NewSource(time.Now().UnixNano())) minNodes := 5 - maxNodes := 10 + maxNodes := 20 nodes := make([]*WakuNode, 0, maxNodes) for i := 0; i < minNodes; i++ {